[verified] fix Stage helper lock under daemon userns remapping
Deploy Canina / deploy (push) Failing after 7m24s
Deploy Canina / deploy (push) Failing after 7m24s
This commit is contained in:
1 parent
2a32f31097
commit
0fa6abc53d
2 files changed
+61
-9
No files matched your search
@@ -46,6 +46,21 @@ cleanup_stage_lock() {
|
||||
wait "$release_pid" 2>/dev/null || :
|
||||
fi
|
||||
if [[ -n $lock_container_id ]]; then
|
||||
if [[ $status != 0 ]]; then
|
||||
# Never emit raw helper/daemon output (image entrypoints may contain
|
||||
# secrets). Classify only a bounded tail before deleting the owned ID.
|
||||
local helper_log diagnostic=unavailable
|
||||
helper_log=$(timeout --foreground --kill-after=2s 10 docker logs --tail 10 "$lock_container_id" 2>&1 | head -c 4096) || :
|
||||
case "$helper_log" in
|
||||
*'Permission denied'*) diagnostic=permission-denied ;;
|
||||
*'No such file or directory'*) diagnostic=missing-host-lock ;;
|
||||
*'Read-only file system'*) diagnostic=read-only-filesystem ;;
|
||||
*'flock: not found'*) diagnostic=flock-unavailable ;;
|
||||
*CANINA_STAGE_DEPLOY_LOCK_ACQUIRED*) diagnostic=previously-acquired ;;
|
||||
?*) diagnostic=unclassified-output ;;
|
||||
esac
|
||||
printf 'Stage lock helper diagnostic: %s\n' "$diagnostic" >&2
|
||||
fi
|
||||
timeout --kill-after=2s 15 docker rm -f "$lock_container_id" >/dev/null || {
|
||||
printf 'Stage lock helper cleanup failed; manual inspection required\n' >&2
|
||||
[[ $status != 0 ]] || status=1
|
||||
@@ -59,10 +74,12 @@ trap 'exit 143' TERM
|
||||
runner_image=$(timeout --foreground --kill-after=2s 10 docker inspect -f '{{.Config.Image}}' gitea_runner)
|
||||
[[ -n $runner_image ]] || exit 1
|
||||
lock_container_id=$(timeout --kill-after=2s 15 docker create --name canina-stage-deploy-lock \
|
||||
--pull=never --network none \
|
||||
--mount type=bind,src=/opt/services/canina,dst=/host-canina \
|
||||
--pull=never --network none --cap-drop=ALL --security-opt=no-new-privileges \
|
||||
--mount type=bind,src=/opt/services/canina,dst=/host-canina,readonly \
|
||||
--entrypoint /bin/sh "$runner_image" -c '
|
||||
exec 9>/host-canina/stage-backend-deploy.lock
|
||||
# Linux flock permits an exclusive lock on a read-only descriptor. Preserve
|
||||
# daemon userns remapping; require the existing host inode, never create it.
|
||||
exec 9</host-canina/stage-backend-deploy.lock
|
||||
flock -n 9 || exit 1
|
||||
printf "CANINA_STAGE_DEPLOY_LOCK_ACQUIRED\n"
|
||||
while :; do sleep 60; done
|
||||
@@ -72,7 +89,7 @@ timeout --foreground --kill-after=2s 10 docker start "$lock_container_id" >/dev/
|
||||
lock_acquired=false
|
||||
for lock_attempt in {1..10}; do
|
||||
[[ $(timeout --foreground --kill-after=2s 10 docker inspect -f '{{.State.Running}}' "$lock_container_id") == true ]] || break
|
||||
if [[ $(timeout --foreground --kill-after=2s 10 docker logs "$lock_container_id") == CANINA_STAGE_DEPLOY_LOCK_ACQUIRED ]]; then
|
||||
if [[ $(timeout --foreground --kill-after=2s 10 docker logs --tail 1 "$lock_container_id" 2>/dev/null) == CANINA_STAGE_DEPLOY_LOCK_ACQUIRED ]]; then
|
||||
lock_acquired=true
|
||||
break
|
||||
fi
|
||||
|
||||
@@ -27,15 +27,25 @@ if (mode == 'stubborn-operation' and args[-1] == 'gitea_runner'
|
||||
time.sleep(6) # Self-bounded even before the production fix.
|
||||
(root / 'stubborn-completed').touch()
|
||||
if args[0] == 'inspect':
|
||||
print('runner:local' if args[-1] == 'gitea_runner' else ('false' if (root / 'lost').exists() else 'true'))
|
||||
denied = False
|
||||
if mode == 'remapped-host' and (root / 'create.json').exists():
|
||||
create = json.loads((root / 'create.json').read_text())
|
||||
# uid0 maps to host100000: directory1001:755/file0:644.
|
||||
# DAC allows read/traversal, not truncation/creation. Model transport
|
||||
# access only; the separate real-shell test proves actual host flock.
|
||||
denied = '9>/host-canina/' in create[-1]
|
||||
print('runner:local' if args[-1] == 'gitea_runner' else ('false' if denied or (root / 'lost').exists() else 'true'))
|
||||
elif args[0] == 'create':
|
||||
if mode == 'contended': sys.exit(1)
|
||||
(root / 'held').touch()
|
||||
(root / 'create.json').write_text(json.dumps(args))
|
||||
print('abc123owned')
|
||||
elif args[0] == 'start':
|
||||
if mode == 'start-failure': sys.exit(1)
|
||||
elif args[0] == 'logs':
|
||||
if mode != 'host-contended': print('CANINA_STAGE_DEPLOY_LOCK_ACQUIRED')
|
||||
if mode == 'diagnostic-failure':
|
||||
print('sh: cannot open lock: Permission denied; SECRET_MUST_NOT_LEAK', file=sys.stderr)
|
||||
elif mode != 'host-contended': print('CANINA_STAGE_DEPLOY_LOCK_ACQUIRED')
|
||||
elif args[0] == 'rm':
|
||||
assert args[-1] == 'abc123owned', 'must only remove owned ID'
|
||||
(root / 'held').unlink()
|
||||
@@ -78,7 +88,7 @@ else: sys.exit(99)
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
create = next(c for c in calls if c[0] == 'create')
|
||||
self.assertIn('--pull=never', create)
|
||||
self.assertIn('type=bind,src=/opt/services/canina,dst=/host-canina', create)
|
||||
self.assertIn('type=bind,src=/opt/services/canina,dst=/host-canina,readonly', create)
|
||||
self.assertIn('runner:local', create)
|
||||
self.assertIn('flock', ' '.join(create))
|
||||
self.assertIn('/host-canina/stage-backend-deploy.lock', ' '.join(create))
|
||||
@@ -86,6 +96,17 @@ else: sys.exit(99)
|
||||
self.assertEqual(len([c for c in calls if c[0] == 'compose']), 2)
|
||||
self.assertFalse(held)
|
||||
|
||||
def test_remapped_uid_can_lock_existing_root_owned_readable_host_file(self):
|
||||
result, calls, held = self.exercise('remapped-host')
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
create = next(c for c in calls if c[0] == 'create')
|
||||
self.assertIn('type=bind,src=/opt/services/canina,dst=/host-canina,readonly', create)
|
||||
self.assertIn('--cap-drop=ALL', create)
|
||||
self.assertIn('--security-opt=no-new-privileges', create)
|
||||
self.assertFalse(any(c.startswith('--userns') or c == '--privileged' for c in create))
|
||||
self.assertNotIn('9>/host-canina/', create[-1])
|
||||
self.assertFalse(held)
|
||||
|
||||
def test_existing_named_helper_fails_closed_without_removal(self):
|
||||
result, calls, _ = self.exercise('contended')
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
@@ -95,10 +116,20 @@ else: sys.exit(99)
|
||||
result, calls, held = self.exercise('host-contended')
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertFalse(any(c[0] == 'compose' for c in calls))
|
||||
self.assertEqual(len([c for c in calls if c[0] == 'logs']), 10)
|
||||
self.assertEqual(len([c for c in calls if c[:3] == ['logs', '--tail', '1']]), 10)
|
||||
self.assertEqual(calls[-1], ['rm', '-f', 'abc123owned'])
|
||||
self.assertFalse(held)
|
||||
|
||||
def test_failure_diagnostics_are_sanitized_and_captured_before_owned_cleanup(self):
|
||||
result, calls, held = self.exercise('diagnostic-failure')
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertIn('Stage lock helper diagnostic: permission-denied', result.stderr)
|
||||
self.assertNotIn('SECRET_MUST_NOT_LEAK', result.stderr)
|
||||
diagnostic = next(i for i, c in enumerate(calls) if c[:3] == ['logs', '--tail', '10'])
|
||||
self.assertLess(diagnostic, len(calls) - 1)
|
||||
self.assertFalse(any(c[0] == 'compose' for c in calls))
|
||||
self.assertFalse(held)
|
||||
|
||||
def test_start_failure_cleans_owned_helper(self):
|
||||
result, calls, held = self.exercise('start-failure')
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
@@ -165,6 +196,8 @@ else: sys.exit(99)
|
||||
blocked = subprocess.run(['sh', '-c', helper], capture_output=True, text=True, timeout=2)
|
||||
self.assertNotEqual(blocked.returncode, 0)
|
||||
self.assertNotIn('ACQUIRED', blocked.stdout)
|
||||
lock_path.write_text('existing-host-inode\n')
|
||||
lock_path.chmod(0o444)
|
||||
with subprocess.Popen(['sh', '-c', helper], stdout=subprocess.PIPE, text=True) as proc:
|
||||
try:
|
||||
assert proc.stdout is not None
|
||||
@@ -172,7 +205,9 @@ else: sys.exit(99)
|
||||
selector.register(proc.stdout, selectors.EVENT_READ)
|
||||
self.assertTrue(selector.select(timeout=2), 'helper did not confirm flock')
|
||||
self.assertEqual(proc.stdout.readline().strip(), 'CANINA_STAGE_DEPLOY_LOCK_ACQUIRED')
|
||||
with lock_path.open('w') as host:
|
||||
self.assertEqual(lock_path.read_text(), 'existing-host-inode\n')
|
||||
self.assertEqual(lock_path.stat().st_mode & 0o777, 0o444)
|
||||
with lock_path.open('r') as host:
|
||||
with self.assertRaises(BlockingIOError):
|
||||
fcntl.flock(host, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
finally:
|
||||
|
||||
Reference in new issue
Block a user