fix(stage): fence runtime authority and bound release transports
Deploy Canina / deploy (push) Failing after 5s

This commit is contained in:
Canina Automation committed 2026-10-26 18:25:16 +00:00
1 parent 52a7aebb51
commit 2a32f31097
6 files changed
+906 -12

No files matched your search

+158 -12
View File
@@ -5,18 +5,111 @@ set +x
umask 077
[[ ${GITHUB_ACTIONS:-${GITEA_ACTIONS:-}} == true && ${DEPLOY_SHA:-} =~ ^[0-9a-f]{40}$ ]] || exit 2
STAGE_DIR=/opt/services/canina/stage
exec 9>/opt/services/canina/stage-backend-deploy.lock
flock -n 9 || { printf 'Stage deployment locked\n' >&2; exit 1; }
# Alpine runner has BusyBox timeout; supervision needs GNU --foreground.
if ! timeout --help 2>&1 | grep -q -- --foreground; then
apk add --no-cache coreutils
fi
timeout --help 2>&1 | grep -q -- --foreground || exit 2
# The runner and host share a daemon, not a filesystem inode. The atomic
# container name serializes runners; the bind-mounted flock also fences host jobs.
# Never reclaim an existing name automatically (including stopped/stale helpers).
lock_container_id=
release_pid=
check_stage_lock() {
[[ $(timeout --foreground --kill-after=2s 10 docker inspect -f '{{.State.Running}}' "$lock_container_id") == true ]] || {
printf 'Stage host lock lost; aborting release\n' >&2
return 1
}
}
supervise_release() {
# A separate process group lets lock loss/signals stop the active command too.
set -m
(set +m; "$@") &
release_pid=$!
while kill -0 "$release_pid" 2>/dev/null; do
check_stage_lock || return 1
sleep 1
done
local status=0
wait "$release_pid" || status=$?
release_pid=
check_stage_lock || return 1
return "$status"
}
cleanup_stage_lock() {
local status=$?
trap - EXIT
if [[ -n $release_pid ]]; then
kill -TERM -- "-$release_pid" 2>/dev/null || :
sleep 1
kill -KILL -- "-$release_pid" 2>/dev/null || :
wait "$release_pid" 2>/dev/null || :
fi
if [[ -n $lock_container_id ]]; then
timeout --kill-after=2s 15 docker rm -f "$lock_container_id" >/dev/null || {
printf 'Stage lock helper cleanup failed; manual inspection required\n' >&2
[[ $status != 0 ]] || status=1
}
fi
exit "$status"
}
trap cleanup_stage_lock EXIT
trap 'exit 130' INT
trap 'exit 143' TERM
runner_image=$(timeout --foreground --kill-after=2s 10 docker inspect -f '{{.Config.Image}}' gitea_runner)
[[ -n $runner_image ]] || exit 1
lock_container_id=$(timeout --kill-after=2s 15 docker create --name canina-stage-deploy-lock \
--pull=never --network none \
--mount type=bind,src=/opt/services/canina,dst=/host-canina \
--entrypoint /bin/sh "$runner_image" -c '
exec 9>/host-canina/stage-backend-deploy.lock
flock -n 9 || exit 1
printf "CANINA_STAGE_DEPLOY_LOCK_ACQUIRED\n"
while :; do sleep 60; done
') || { printf 'Stage deployment locked or helper creation failed\n' >&2; exit 1; }
[[ -n $lock_container_id ]] || exit 1
timeout --foreground --kill-after=2s 10 docker start "$lock_container_id" >/dev/null
lock_acquired=false
for lock_attempt in {1..10}; do
[[ $(timeout --foreground --kill-after=2s 10 docker inspect -f '{{.State.Running}}' "$lock_container_id") == true ]] || break
if [[ $(timeout --foreground --kill-after=2s 10 docker logs "$lock_container_id") == CANINA_STAGE_DEPLOY_LOCK_ACQUIRED ]]; then
lock_acquired=true
break
fi
[[ $lock_attempt == 10 ]] || sleep 1
done
[[ $lock_acquired == true ]] || { printf 'Stage host lock not confirmed\n' >&2; exit 1; }
release_stage() {
cd "$STAGE_DIR"
git fetch origin develop
timeout --foreground --kill-after=2s 60 git fetch origin develop
[[ $(git rev-parse origin/develop) == "$DEPLOY_SHA" ]] || exit 2
# Never reset/clean the operational checkout or overwrite its generated configs.
TARGET_DIR="/opt/services/canina/cache-releases/$DEPLOY_SHA"
[[ ! -e "$TARGET_DIR" ]] || { printf 'Release directory already exists\n' >&2; exit 2; }
mkdir -p "$TARGET_DIR"
# Every attempt has its own directory: retain incomplete and active releases.
RELEASE_ROOT=/opt/services/canina/cache-releases
mkdir -p "$RELEASE_ROOT"
TARGET_DIR=$(mktemp -d "$RELEASE_ROOT/$DEPLOY_SHA.attempt.XXXXXXXX")
# Archive failure leaves only this isolated attempt, never a reusable release.
git archive "$DEPLOY_SHA" | tar -x -C "$TARGET_DIR"
[[ $(git rev-parse HEAD) == "$DEPLOY_SHA" ]] || printf 'Operational HEAD intentionally preserved\n'
[[ $(git rev-parse "$DEPLOY_SHA^{commit}") == "$DEPLOY_SHA" ]] || exit 2
# Record the exact source identity before introducing operational configuration.
command -v python3 >/dev/null || apk add --no-cache python3
python3 - "$TARGET_DIR" "$DEPLOY_SHA" <<'PY'
import hashlib, json, sys
from pathlib import Path
root = Path(sys.argv[1]); sha = sys.argv[2]
files = {str(p.relative_to(root)): hashlib.sha256(p.read_bytes()).hexdigest()
for p in sorted(root.rglob('*')) if p.is_file() and not p.is_symlink()}
manifest = {'sha': sha, 'files': files}
(root / '.release-manifest.json').write_text(json.dumps(manifest, sort_keys=True))
loaded = json.loads((root / '.release-manifest.json').read_text())
assert loaded['sha'] == sha
assert all(hashlib.sha256((root / name).read_bytes()).hexdigest() == digest
for name, digest in loaded['files'].items())
(root / '.release-ready').write_text(sha + '\n')
PY
[[ $(cat "$TARGET_DIR/.release-ready") == "$DEPLOY_SHA" ]] || exit 2
# The ready marker certifies extraction only, not deployment/readiness.
[[ $(git rev-parse HEAD) == "$DEPLOY_SHA" ]] || printf 'Operational HEAD intentionally preserved\n'
cd "$TARGET_DIR"
python3 scripts/provision-revalidation-env.py --environment stage --repair-empty \
--source "$STAGE_DIR/.revalidation.env" --output "$TARGET_DIR/.revalidation.env"
@@ -45,11 +138,64 @@ services['frontend_stage']['environment']={'INTERNAL_API_URL':'http://backend_st
Path('release.override.json').write_text(json.dumps({'services':services}))
PY
compose=(docker compose -p stage --project-directory "$STAGE_DIR" -f "$STAGE_DIR/docker-compose.yml" -f "$TARGET_DIR/release.override.json")
"${compose[@]}" config --quiet
"${compose[@]}" --progress plain build backend_stage frontend_stage
"${compose[@]}" up -d --no-deps --force-recreate backend_stage frontend_stage
timeout --foreground --kill-after=2s 60 "${compose[@]}" config --quiet
python3 scripts/verify-stage-runtime.py --base-compose "$STAGE_DIR/docker-compose.yml" \
--override "$TARGET_DIR/release.override.json" --project-directory "$STAGE_DIR" \
--nginx "$TARGET_DIR/nginx.conf"
check_stage_lock
timeout --foreground --kill-after=2s 1800 "${compose[@]}" --progress plain build backend_stage frontend_stage
check_stage_lock
timeout --foreground --kill-after=2s 120 "${compose[@]}" up -d --no-deps --force-recreate backend_stage frontend_stage
for name in canino_backend_stage canino_frontend_stage; do
[[ $(docker inspect -f '{{index .Config.Labels "ir.canina.release.sha"}}' "$name") == "$DEPLOY_SHA" ]]
docker inspect -f '{{.Name}} image={{.Image}} release={{index .Config.Labels "ir.canina.release.sha"}}' "$name"
# Every daemon interaction is bounded, including a hung exec/inspect.
[[ $(timeout --foreground --kill-after=2s 10 docker inspect -f '{{index .Config.Labels "ir.canina.release.sha"}}' "$name") == "$DEPLOY_SHA" ]]
image=$(timeout --foreground --kill-after=2s 10 docker inspect -f '{{.Image}}' "$name")
[[ $(timeout --foreground --kill-after=2s 10 docker image inspect -f '{{index .Config.Labels "org.opencontainers.image.revision"}}' "$image") == "$DEPLOY_SHA" ]]
path=/
[[ $name != canino_backend_stage ]] || path=/api
ready=false
for attempt in {1..12}; do
[[ $(timeout --foreground --kill-after=2s 10 docker inspect -f '{{.State.Running}} {{.RestartCount}}' "$name") == 'true 0' ]] || { printf 'Container stopped or restarted: %s\n' "$name" >&2; exit 1; }
if timeout --foreground --kill-after=2s 10 docker exec "$name" node -e '
const http = require("http");
const req = http.get({host: "127.0.0.1", port: 3000, path: process.argv[1]}, res => {
res.resume(); process.exit(res.statusCode === 200 ? 0 : 1);
});
req.setTimeout(5000, () => req.destroy(new Error("readiness timeout")));
req.on("error", () => process.exit(1));
' "$path"; then
ready=true
break
fi
[[ $attempt == 12 ]] || sleep 2
done
[[ $ready == true ]] || { printf 'Readiness timed out: %s\n' "$name" >&2; exit 1; }
if [[ $name == canino_frontend_stage ]]; then
for host in stage.canina.ir stageadmin.canina.ir; do
ready=false
for attempt in {1..12}; do
if timeout --foreground --kill-after=2s 10 docker exec "$name" node -e '
const http = require("http");
const req = http.get({host: "127.0.0.1", port: 8080, path: "/", headers: {Host: process.argv[1]}}, res => {
res.resume(); process.exit(res.statusCode === 200 ? 0 : 1);
});
req.setTimeout(5000, () => req.destroy(new Error("nginx readiness timeout")));
req.on("error", () => process.exit(1));
' "$host"; then
ready=true
break
fi
[[ $attempt == 12 ]] || sleep 2
done
[[ $ready == true ]] || { printf 'Nginx readiness timed out: %s\n' "$host" >&2; exit 1; }
done
fi
# Fence replacement/restart during probing, not just the initial state.
[[ $(timeout --foreground --kill-after=2s 10 docker inspect -f '{{.State.Running}} {{.RestartCount}}' "$name") == 'true 0' ]]
[[ $(timeout --foreground --kill-after=2s 10 docker inspect -f '{{.Image}}' "$name") == "$image" ]]
[[ $(timeout --foreground --kill-after=2s 10 docker inspect -f '{{index .Config.Labels "ir.canina.release.sha"}}' "$name") == "$DEPLOY_SHA" ]]
printf '%s image=%s release=%s ready\n' "$name" "$image" "$DEPLOY_SHA"
done
printf 'Code-only Stage release complete: %s\n' "$DEPLOY_SHA"
}
supervise_release release_stage
+101
View File
@@ -0,0 +1,101 @@
"""Local shell harnesses: no Docker daemon or remote host is contacted."""
import json
import os
from pathlib import Path
import subprocess
import tempfile
import unittest
ROOT = Path(__file__).resolve().parents[2]
SCRIPT = ROOT / 'scripts/release-cache-stage.sh'
class ReleaseSafetyTests(unittest.TestCase):
def test_release_commands_have_deadlines_and_fresh_lock_fences(self):
source = SCRIPT.read_text()
self.assertIn('timeout --foreground --kill-after=2s 60 git fetch origin develop', source)
self.assertIn('timeout --foreground --kill-after=2s 60 "${compose[@]}" config --quiet', source)
self.assertIn('check_stage_lock\ntimeout --foreground --kill-after=2s 1800 "${compose[@]}" --progress plain build', source)
self.assertIn('check_stage_lock\ntimeout --foreground --kill-after=2s 120 "${compose[@]}" up', source)
def test_incomplete_sha_retry_preserves_previous_release(self):
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
stage = root / 'stage'
stage.mkdir()
def git(*args):
return subprocess.check_output(['git', '-C', str(stage), *args], text=True).strip()
git('init', '-q')
git('config', 'user.email', 'test@example.invalid')
git('config', 'user.name', 'Test')
(stage / 'source').write_text('exact source')
git('add', 'source')
git('commit', '-qm', 'fixture')
sha = git('rev-parse', 'HEAD')
git('update-ref', 'refs/remotes/origin/develop', sha)
releases = root / 'releases'
old = releases / sha
old.mkdir(parents=True)
(old / 'operational-config').write_text('preserve me')
source = SCRIPT.read_text().replace('/opt/services/canina/stage', str(stage)).replace('/opt/services/canina/cache-releases', str(releases))
source = source.replace('timeout --foreground --kill-after=2s 60 git fetch origin develop', ': # offline fixture')
# Lock behavior has its own daemon-double suite; isolate extraction.
start = source.index('# The runner and host share a daemon')
end = source.index('cd "$STAGE_DIR"')
source = source[:start] + source[end:]
source = source.split('cd "$TARGET_DIR"')[0] + '\nprintf "TARGET=%s\\n" "$TARGET_DIR"\n'
result = subprocess.run(['bash', '-c', source], env={**os.environ, 'GITHUB_ACTIONS': 'true', 'DEPLOY_SHA': sha}, capture_output=True, text=True)
self.assertEqual(result.returncode, 0, result.stderr)
target = Path(result.stdout.split('TARGET=')[1].strip())
self.assertNotEqual(target, old)
self.assertEqual((old / 'operational-config').read_text(), 'preserve me')
self.assertEqual((target / 'source').read_text(), 'exact source')
self.assertEqual((target / '.release-ready').read_text().strip(), sha)
manifest = json.loads((target / '.release-manifest.json').read_text())
self.assertEqual(manifest['sha'], sha)
def readiness(self, mode):
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
log = root / 'calls'
docker = root / 'docker'
docker.write_text('''#!/usr/bin/env python3
import os, sys, json
args = sys.argv[1:]
with open(os.environ['CALL_LOG'], 'a') as f: f.write(json.dumps(' '.join(args)) + '\\n')
mode = os.environ['MODE']
if args[0] == 'exec': sys.exit(1 if mode == 'http' else 0)
fmt = args[args.index('-f') + 1] if '-f' in args else ''
if 'org.opencontainers.image.revision' in fmt:
print('wrong' if mode == 'revision' else os.environ['DEPLOY_SHA'])
elif 'RestartCount' in fmt:
print('true 1' if mode == 'restart' else ('false 0' if mode == 'stopped' else 'true 0'))
elif '.Image' in fmt: print('sha256:fixture')
else: print(os.environ['DEPLOY_SHA'])
''')
docker.chmod(0o755)
(root / 'sleep').write_text('#!/bin/sh\nexit 0\n')
(root / 'sleep').chmod(0o755)
source = SCRIPT.read_text().split('for name in canino_backend_stage')[1].rsplit('\n}\nsupervise_release release_stage', 1)[0]
result = subprocess.run(['bash', '-c', 'set -euo pipefail\nfor name in canino_backend_stage' + source], env={**os.environ, 'PATH': str(root) + ':' + os.environ['PATH'], 'DEPLOY_SHA': 'a'*40, 'MODE': mode, 'CALL_LOG': str(log)}, capture_output=True, text=True, timeout=20)
return result, [json.loads(line) for line in log.read_text().splitlines()]
def test_readiness_is_bounded_and_checks_both_services(self):
result, calls = self.readiness('ok')
self.assertEqual(result.returncode, 0, result.stderr)
probes = [c for c in calls if c.startswith('exec ')]
self.assertEqual(len(probes), 4)
self.assertTrue(any('8080' in c and 'stage.canina.ir' in c for c in probes))
self.assertTrue(any('8080' in c and 'stageadmin.canina.ir' in c for c in probes))
self.assertTrue(any('canino_backend_stage' in c for c in probes))
self.assertTrue(any('canino_frontend_stage' in c for c in probes))
for mode in ('revision', 'restart', 'stopped', 'http'):
with self.subTest(mode=mode):
result, calls = self.readiness(mode)
self.assertNotEqual(result.returncode, 0)
self.assertLessEqual(len([c for c in calls if c.startswith('exec ')]), 24)
self.assertNotIn('Code-only Stage release complete:', result.stdout)
if __name__ == '__main__':
unittest.main()
@@ -7,6 +7,10 @@ import unittest
ROOT = Path(__file__).resolve().parents[2]
class CacheStageReleaseTests(unittest.TestCase):
def test_gnu_timeout_is_provisioned_before_use(self):
script = (ROOT / 'scripts/release-cache-stage.sh').read_text()
self.assertLess(script.index('apk add --no-cache coreutils'), script.index('runner_image=$(timeout --foreground --kill-after=2s'))
def test_develop_uses_isolated_code_only_release(self):
deploy = (ROOT/'scripts/deploy.sh').read_text()
self.assertIn('exec bash "$SCRIPT_DIR/release-cache-stage.sh"', deploy)
+184
View File
@@ -0,0 +1,184 @@
"""Exercise the real release lock shell with a strictly local Docker double."""
import json
import os
from pathlib import Path
import subprocess
import tempfile
import unittest
SCRIPT = Path(__file__).resolve().parents[1] / 'release-cache-stage.sh'
class StageDeployLockTests(unittest.TestCase):
def exercise(self, mode='ok', body='docker compose build; docker compose up', accelerated=False):
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
docker = root / 'docker'
docker.write_text('''#!/usr/bin/env python3
import json, os, sys, time, signal
from pathlib import Path
args = sys.argv[1:]
root = Path(os.environ['FAKE_ROOT'])
with (root / 'calls').open('a') as out: out.write(json.dumps(args) + '\\n')
mode = os.environ['MODE']
if (mode == 'stubborn-operation' and args[-1] == 'gitea_runner'
or mode == 'stubborn-cleanup' and args[0] == 'rm'):
signal.signal(signal.SIGTERM, signal.SIG_IGN)
time.sleep(6) # Self-bounded even before the production fix.
(root / 'stubborn-completed').touch()
if args[0] == 'inspect':
print('runner:local' if args[-1] == 'gitea_runner' else ('false' if (root / 'lost').exists() else 'true'))
elif args[0] == 'create':
if mode == 'contended': sys.exit(1)
(root / 'held').touch()
print('abc123owned')
elif args[0] == 'start':
if mode == 'start-failure': sys.exit(1)
elif args[0] == 'logs':
if mode != 'host-contended': print('CANINA_STAGE_DEPLOY_LOCK_ACQUIRED')
elif args[0] == 'rm':
assert args[-1] == 'abc123owned', 'must only remove owned ID'
(root / 'held').unlink()
elif args[0] == 'compose':
assert (root / 'held').exists(), 'deployment ran without lock'
if mode == 'loss-during-build' and 'build' in args:
(root / 'lost').touch()
def stopped(*_):
with (root / 'calls').open('a') as out: out.write(json.dumps(['child-stopped']) + '\\n')
sys.exit(143)
signal.signal(signal.SIGTERM, stopped)
time.sleep(5)
(root / 'child-completed').touch()
else: sys.exit(99)
''')
docker.chmod(0o755)
(root / 'sleep').write_text('#!/bin/sh\nexec /bin/sleep 0.1\n')
(root / 'sleep').chmod(0o755)
source = SCRIPT.read_text().split('release_stage() {')[0]
if accelerated:
# Keep GNU timeout real; shorten only test-local deadlines.
source = source.replace('--kill-after=2s', '--kill-after=0.1s')
source = source.replace('--foreground --kill-after=0.1s 10 ', '--foreground --kill-after=0.1s 2 ')
source = source.replace('timeout 15 ', 'timeout 2 ')
source = source.replace('--kill-after=0.1s 15 ', '--kill-after=0.1s 2 ')
# Only the isolated lock prefix runs; no operational paths are touched.
result = subprocess.run(['bash', '-c', source + '\nrelease_stage() {\n' + body + '\n}\nsupervise_release release_stage'],
env={**os.environ, 'PATH': str(root) + ':' + os.environ['PATH'],
'GITHUB_ACTIONS': 'true', 'DEPLOY_SHA': 'a' * 40,
'FAKE_ROOT': str(root), 'MODE': mode},
capture_output=True, text=True, timeout=20)
calls = [json.loads(line) for line in (root / 'calls').read_text().splitlines()] if (root / 'calls').exists() else []
if accelerated:
self.assertFalse((root / 'stubborn-completed').exists(),
'TERM-ignoring transport outlived its kill deadline')
return result, calls, (root / 'held').exists()
def test_success_holds_host_lock_through_build_and_deploy(self):
result, calls, held = self.exercise()
self.assertEqual(result.returncode, 0, result.stderr)
create = next(c for c in calls if c[0] == 'create')
self.assertIn('--pull=never', create)
self.assertIn('type=bind,src=/opt/services/canina,dst=/host-canina', create)
self.assertIn('runner:local', create)
self.assertIn('flock', ' '.join(create))
self.assertIn('/host-canina/stage-backend-deploy.lock', ' '.join(create))
self.assertEqual(calls[-1], ['rm', '-f', 'abc123owned'])
self.assertEqual(len([c for c in calls if c[0] == 'compose']), 2)
self.assertFalse(held)
def test_existing_named_helper_fails_closed_without_removal(self):
result, calls, _ = self.exercise('contended')
self.assertNotEqual(result.returncode, 0)
self.assertFalse(any(c[0] in ('rm', 'compose', 'start') for c in calls))
def test_unconfirmed_host_lock_is_bounded_and_removes_only_owned_id(self):
result, calls, held = self.exercise('host-contended')
self.assertNotEqual(result.returncode, 0)
self.assertFalse(any(c[0] == 'compose' for c in calls))
self.assertEqual(len([c for c in calls if c[0] == 'logs']), 10)
self.assertEqual(calls[-1], ['rm', '-f', 'abc123owned'])
self.assertFalse(held)
def test_start_failure_cleans_owned_helper(self):
result, calls, held = self.exercise('start-failure')
self.assertNotEqual(result.returncode, 0)
self.assertEqual(calls[-1], ['rm', '-f', 'abc123owned'])
self.assertFalse(held)
self.assertFalse(any(c[0] == 'compose' for c in calls))
def test_deploy_failure_preserves_status_and_releases_owned_helper(self):
result, calls, held = self.exercise(body='docker compose build; exit 42')
self.assertEqual(result.returncode, 42)
self.assertEqual(calls[-1], ['rm', '-f', 'abc123owned'])
self.assertFalse(held)
def test_termination_releases_owned_helper(self):
result, calls, held = self.exercise(body='kill -TERM $$')
self.assertEqual(result.returncode, 143)
self.assertEqual(calls[-1], ['rm', '-f', 'abc123owned'])
self.assertFalse(held)
def test_loss_after_acquisition_before_rollout_prevents_up(self):
result, calls, held = self.exercise(body='touch "$FAKE_ROOT/lost"; /bin/sleep 2; docker compose up')
self.assertNotEqual(result.returncode, 0, result.stderr)
self.assertFalse(any(c[0] == 'compose' for c in calls))
self.assertFalse(held)
def test_loss_during_build_interrupts_child_and_prevents_up(self):
result, calls, held = self.exercise('loss-during-build')
self.assertNotEqual(result.returncode, 0, result.stderr)
self.assertFalse(any(c[0] == 'compose' and 'up' in c for c in calls))
self.assertIn('Stage host lock lost', result.stderr)
self.assertIn(['child-stopped'], calls)
self.assertFalse(held)
def test_loss_interrupts_timeout_wrapped_docker_child(self):
result, calls, held = self.exercise('loss-during-build', body='timeout --foreground 20 docker compose build; docker compose up')
self.assertNotEqual(result.returncode, 0, result.stderr)
self.assertIn(['child-stopped'], calls)
self.assertFalse(any(c[0] == 'compose' and 'up' in c for c in calls))
self.assertFalse(held)
def test_term_ignoring_operational_transport_is_killed(self):
result, calls, held = self.exercise('stubborn-operation', accelerated=True)
self.assertNotEqual(result.returncode, 0)
self.assertFalse(any(c[0] in ('create', 'compose') for c in calls))
self.assertFalse(held)
def test_term_ignoring_cleanup_transport_is_killed_and_fails_closed(self):
result, calls, held = self.exercise('stubborn-cleanup', accelerated=True)
self.assertNotEqual(result.returncode, 0)
self.assertIn('manual inspection required', result.stderr)
self.assertTrue(held, 'failed cleanup must not claim removal')
self.assertEqual(calls[-1], ['rm', '-f', 'abc123owned'])
def test_helper_shell_contends_on_exact_host_inode(self):
import fcntl
import selectors
source = SCRIPT.read_text()
helper = source.split('--entrypoint /bin/sh "$runner_image" -c \'')[1].split("\n ')")[0]
with tempfile.TemporaryDirectory() as tmp:
helper = helper.replace('/host-canina', tmp).replace('sleep 60', 'sleep 0.1')
lock_path = Path(tmp) / 'stage-backend-deploy.lock'
with lock_path.open('w') as host:
fcntl.flock(host, fcntl.LOCK_EX | fcntl.LOCK_NB)
blocked = subprocess.run(['sh', '-c', helper], capture_output=True, text=True, timeout=2)
self.assertNotEqual(blocked.returncode, 0)
self.assertNotIn('ACQUIRED', blocked.stdout)
with subprocess.Popen(['sh', '-c', helper], stdout=subprocess.PIPE, text=True) as proc:
try:
assert proc.stdout is not None
with selectors.DefaultSelector() as selector:
selector.register(proc.stdout, selectors.EVENT_READ)
self.assertTrue(selector.select(timeout=2), 'helper did not confirm flock')
self.assertEqual(proc.stdout.readline().strip(), 'CANINA_STAGE_DEPLOY_LOCK_ACQUIRED')
with lock_path.open('w') as host:
with self.assertRaises(BlockingIOError):
fcntl.flock(host, fcntl.LOCK_EX | fcntl.LOCK_NB)
finally:
proc.terminate()
proc.wait(timeout=2)
if __name__ == '__main__':
unittest.main()
@@ -0,0 +1,295 @@
"""Read-only Stage config authority contract; fixtures contain no real secrets."""
import copy
import importlib.util
from pathlib import Path
import unittest
import contextlib
import io
import json
import subprocess
from unittest.mock import patch
SCRIPT = Path(__file__).resolve().parents[1] / 'verify-stage-runtime.py'
def fixture():
config = {'name': 'stage', 'services': {},
'volumes': {'uploads': {'name': 'stage_uploads'}},
'networks': {'public': {'name': 'traefik_public'}}}
live = {}
for kind in ('backend', 'frontend'):
name = kind + '_stage'
config['services'][name] = {
'environment': {'OPERATIONAL': 'fixture-value', 'REVALIDATION_SECRET': 'a' * 64},
'volumes': [{'type': 'volume', 'source': 'uploads', 'target': '/app/uploads'}],
'networks': {'public': {}},
'labels': {'traefik.enable': 'true'},
'build': {'args': {'NPM_REGISTRY': 'https://registry.parsaaghayi.ir/repository/npm/'}},
}
live[name] = {
'Config': {'Env': ['OPERATIONAL=fixture-value', 'PATH=/bin'],
'Labels': {'traefik.enable': 'true'}},
'Mounts': [{'Type': 'volume', 'Name': 'stage_uploads',
'Source': '/var/lib/docker/volumes/stage_uploads/_data',
'Destination': '/app/uploads', 'RW': True}],
'NetworkSettings': {'Networks': {'traefik_public': {}}},
}
config['services']['frontend_stage']['build']['args'].update(
NEXT_PUBLIC_API_URL='https://stageapi.canina.ir/api', VITE_API_URL='https://stageapi.canina.ir')
return config, copy.deepcopy(config), live
class AuthorityTests(unittest.TestCase):
def setUp(self):
self.assertTrue(SCRIPT.exists(), 'authority preflight is missing')
spec = importlib.util.spec_from_file_location('stage_runtime', SCRIPT)
self.module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(self.module)
def test_environment_authority_mismatch_is_key_only(self):
for scenario in ('base-value', 'planned-value', 'removed', 'extra-live', 'extra-planned', 'null'):
with self.subTest(scenario=scenario):
base, planned, live = fixture()
if scenario == 'base-value':
base['services']['backend_stage']['environment']['OPERATIONAL'] = 'DO-NOT-PRINT'
elif scenario == 'planned-value':
planned['services']['backend_stage']['environment']['OPERATIONAL'] = 'DO-NOT-PRINT'
elif scenario == 'removed':
del planned['services']['backend_stage']['environment']['OPERATIONAL']
elif scenario == 'extra-live':
live['backend_stage']['Config']['Env'].append('MISSING=DO-NOT-PRINT')
elif scenario == 'null':
planned['services']['backend_stage']['environment']['OPERATIONAL'] = None
else:
planned['services']['backend_stage']['environment']['NEW'] = 'DO-NOT-PRINT'
errors = self.module.verify(base, planned, live, b'n', b'n')
self.assertTrue(errors)
self.assertIn('environment', ' '.join(errors))
self.assertNotIn('DO-NOT-PRINT', ' '.join(errors))
def test_release_environment_exceptions_and_image_defaults(self):
base, planned, live = fixture()
for name in ('backend_stage', 'frontend_stage'):
planned['services'][name]['environment']['REVALIDATION_SECRET'] = 'b' * 64
live[name]['Config']['Env'].append('REVALIDATION_SECRET=' + 'a' * 64)
for key in ('FRONTEND_INTERNAL_URL', 'INTERNAL_API_URL'):
planned['services']['backend_stage']['environment'][key] = 'new-release'
live['backend_stage']['Config']['Env'].append(key + '=old')
for key in ('NODE_VERSION', 'YARN_VERSION', 'PRISMA_HIDE_UPDATE_MESSAGE'):
live['backend_stage']['Config']['Env'].append(key + '=image-default')
self.assertEqual([], self.module.verify(base, planned, live, b'n', b'n'))
def test_runtime_properties_cannot_drift(self):
for prop in ('mount-source', 'mount-type', 'mount-target', 'network', 'label', 'label-added', 'nginx'):
for side in ('base', 'planned', 'live'):
with self.subTest(prop=prop, side=side):
base, planned, live = fixture()
local = b'nginx'
if prop == 'nginx':
local = b'different'
elif side == 'live':
item = live['backend_stage']
if prop.startswith('mount'):
field = {'mount-source': 'Name', 'mount-type': 'Type', 'mount-target': 'Destination'}[prop]
item['Mounts'][0][field] = 'different'
elif prop == 'network':
item['NetworkSettings']['Networks'] = {'wrong': {}}
else:
item['Config']['Labels']['traefik.extra' if prop == 'label-added' else 'traefik.enable'] = 'different'
else:
item = (base if side == 'base' else planned)['services']['backend_stage']
if prop.startswith('mount'):
field = {'mount-source': 'source', 'mount-type': 'type', 'mount-target': 'target'}[prop]
item['volumes'][0][field] = 'different'
elif prop == 'network':
item['networks'] = {'wrong': {}}
else:
item['labels']['traefik.extra' if prop == 'label-added' else 'traefik.enable'] = 'different'
self.assertTrue(self.module.verify(base, planned, live, local, b'nginx'))
def test_bind_mount_normalization_and_non_operational_labels(self):
base, planned, live = fixture()
for config in (base, planned):
config['services']['backend_stage']['volumes'].append(
{'type': 'bind', 'source': '/opt/certs/../certs', 'target': '/app/certs'})
live['backend_stage']['Mounts'].append(
{'Type': 'bind', 'Source': '/opt/certs', 'Destination': '/app/certs', 'RW': True})
planned['services']['backend_stage']['labels']['ir.canina.release.sha'] = 'release'
self.assertEqual([], self.module.verify(base, planned, live, b'n', b'n'))
def test_mount_read_only_matches_live_rw_for_both_authorities(self):
for side in ('base', 'planned'):
for read_only in (False, True):
with self.subTest(side=side, read_only=read_only):
base, planned, live = fixture()
for config in (base, planned):
config['services']['backend_stage']['volumes'][0]['read_only'] = read_only
live['backend_stage']['Mounts'][0]['RW'] = not read_only
self.assertEqual([], self.module.verify(base, planned, live, b'n', b'n'))
selected = base if side == 'base' else planned
selected['services']['backend_stage']['volumes'][0]['read_only'] = not read_only
self.assertEqual(['backend_stage.mounts'],
self.module.verify(base, planned, live, b'n', b'n'))
base, planned, live = fixture()
del live['backend_stage']['Mounts'][0]['RW']
self.assertEqual(['backend_stage.mounts'],
self.module.verify(base, planned, live, b'n', b'n'))
def test_explicit_bind_propagation_is_checked_for_each_authority(self):
for side in ('base', 'planned'):
for actual in ('rshared', 'rprivate', None):
with self.subTest(side=side, actual=actual):
base, planned, live = fixture()
for config in (base, planned):
config['services']['backend_stage']['volumes'][0] = {
'type': 'bind', 'source': '/fixture', 'target': '/app/uploads'}
selected = base if side == 'base' else planned
selected['services']['backend_stage']['volumes'][0]['bind'] = {'propagation': 'rshared'}
mount = live['backend_stage']['Mounts'][0]
mount.update(Type='bind', Source='/fixture')
if actual is not None:
mount['Propagation'] = actual
self.assertEqual([] if actual == 'rshared' else ['backend_stage.mounts'],
self.module.verify(base, planned, live, b'n', b'n'))
def test_configured_volume_nocopy_requires_inspectable_matching_option(self):
for side in ('base', 'planned'):
for expected in (False, True):
for actual in (False, True, None):
with self.subTest(side=side, expected=expected, actual=actual):
base, planned, live = fixture()
selected = base if side == 'base' else planned
selected['services']['backend_stage']['volumes'][0]['volume'] = {'nocopy': expected}
if actual is not None:
live['backend_stage']['HostConfig'] = {'Mounts': [{
'Type': 'volume', 'Source': 'stage_uploads', 'Target': '/app/uploads',
'VolumeOptions': {'NoCopy': actual}}]}
self.assertEqual([] if actual is expected else ['backend_stage.mounts'],
self.module.verify(base, planned, live, b'n', b'n'))
def test_explicit_network_aliases_are_required_but_defaults_are_allowed(self):
for side in ('base', 'planned'):
for aliases in (['fixture-alias', 'backend_stage', 'container-id'],
['backend_stage'], None):
with self.subTest(side=side, aliases=aliases):
base, planned, live = fixture()
selected = base if side == 'base' else planned
selected['services']['backend_stage']['networks']['public'] = {
'aliases': ['fixture-alias']}
endpoint = live['backend_stage']['NetworkSettings']['Networks']['traefik_public']
endpoint['Aliases'] = aliases
self.assertEqual([] if aliases and 'fixture-alias' in aliases
else ['backend_stage.networks'],
self.module.verify(base, planned, live, b'n', b'n'))
base, planned, live = fixture()
live['backend_stage']['NetworkSettings']['Networks']['traefik_public']['Aliases'] = ['backend_stage']
self.assertEqual([], self.module.verify(base, planned, live, b'n', b'n'))
def test_static_network_addresses_match_live_for_each_authority(self):
for side in ('base', 'planned'):
for key, field, address in (('ipv4_address', 'IPAddress', '192.0.2.10'),
('ipv6_address', 'GlobalIPv6Address', '2001:db8::10')):
for actual in (address, 'different', None):
with self.subTest(side=side, key=key, actual=actual):
base, planned, live = fixture()
selected = base if side == 'base' else planned
selected['services']['backend_stage']['networks']['public'] = {key: address}
endpoint = live['backend_stage']['NetworkSettings']['Networks']['traefik_public']
if actual is not None:
endpoint[field] = actual
self.assertEqual([] if actual == address else ['backend_stage.networks'],
self.module.verify(base, planned, live, b'n', b'n'))
base, planned, live = fixture()
live['backend_stage']['NetworkSettings']['Networks']['traefik_public'].update(
IPAddress='192.0.2.20', GlobalIPv6Address='2001:db8::20')
self.assertEqual([], self.module.verify(base, planned, live, b'n', b'n'))
def test_release_exceptions_do_not_allow_removed_operational_keys(self):
base, planned, live = fixture()
del planned['services']['backend_stage']['environment']['REVALIDATION_SECRET']
base['services']['backend_stage']['environment']['REVALIDATION_SECRET'] = 'old'
live['backend_stage']['Config']['Env'].append('REVALIDATION_SECRET=old')
self.assertIn('backend_stage.environment.REVALIDATION_SECRET',
self.module.verify(base, planned, live, b'n', b'n'))
def test_resolved_secret_rejects_empty_environment_override_and_invalid_values(self):
# Compose environment overrides env_file, even with an empty value.
for name in ('backend_stage', 'frontend_stage'):
for value in ('', None, 'short', 'x' * 257, '!' * 64):
with self.subTest(name=name, kind=type(value).__name__):
base, planned, live = fixture()
planned['services'][name]['environment']['REVALIDATION_SECRET'] = value
errors = self.module.verify(base, planned, live, b'n', b'n')
self.assertIn(name + '.environment.REVALIDATION_SECRET', errors)
if value:
self.assertNotIn(value, ' '.join(errors))
base, planned, live = fixture()
del planned['services']['backend_stage']['environment']['REVALIDATION_SECRET']
self.assertIn('backend_stage.environment.REVALIDATION_SECRET',
self.module.verify(base, planned, live, b'n', b'n'))
def test_resolved_service_secrets_must_match_without_disclosing_values(self):
base, planned, live = fixture()
planned['services']['frontend_stage']['environment']['REVALIDATION_SECRET'] = 'b' * 64
errors = self.module.verify(base, planned, live, b'n', b'n')
self.assertTrue(errors, 'mismatched effective secrets were accepted')
self.assertNotIn('a' * 64, ' '.join(errors))
self.assertNotIn('b' * 64, ' '.join(errors))
def test_required_runner_build_arguments(self):
for name, key in (('backend_stage', 'NPM_REGISTRY'), ('frontend_stage', 'NPM_REGISTRY'),
('frontend_stage', 'NEXT_PUBLIC_API_URL'), ('frontend_stage', 'VITE_API_URL')):
for value in (None, '', 'https://wrong.invalid'):
with self.subTest(name=name, key=key, value=value):
base, planned, live = fixture()
args = planned['services'][name]['build']['args']
if value is None:
del args[key]
else:
args[key] = value
errors = self.module.verify(base, planned, live, b'n', b'n')
self.assertIn(name + '.build.args.' + key, errors)
def test_cli_captures_read_only_commands_and_fails_closed(self):
self.assertTrue(hasattr(self.module, 'main'), 'CLI is missing')
base, planned, live = fixture()
payloads = [json.dumps(base).encode(), json.dumps(planned).encode(),
json.dumps([live['backend_stage'], live['frontend_stage']]).encode(), b'nginx']
argv = ['--base-compose', '/stage/docker-compose.yml', '--override', '/release/override.json',
'--project-directory', '/stage', '--nginx', '/release/nginx.conf']
for failure in (None, subprocess.CalledProcessError(1, ['docker'], output=b'SECRET', stderr=b'SECRET'),
b'not-json-SECRET'):
with self.subTest(failure=type(failure).__name__):
output = io.StringIO()
responses = ([subprocess.CompletedProcess([], 0, p, b'SECRET') for p in payloads]
if failure is None else [failure if isinstance(failure, Exception)
else subprocess.CompletedProcess([], 0, failure, b'SECRET')])
with patch('subprocess.run', side_effect=responses) as run, \
patch.object(Path, 'read_bytes', return_value=b'nginx'), \
contextlib.redirect_stdout(output), contextlib.redirect_stderr(output):
status = self.module.main(argv)
self.assertEqual(0 if failure is None else 1, status)
self.assertNotIn('SECRET', output.getvalue())
if failure is None:
commands = [call.args[0] for call in run.call_args_list]
self.assertEqual(['config', '--format', 'json'], commands[0][-3:])
self.assertEqual(['docker', 'inspect', 'canino_backend_stage', 'canino_frontend_stage'], commands[2])
self.assertEqual(['docker', 'exec', 'canino_frontend_stage', 'cat', '/etc/nginx/conf.d/default.conf'], commands[3])
for call in run.call_args_list:
self.assertTrue(call.kwargs['capture_output'])
self.assertTrue(call.kwargs['check'])
self.assertGreater(call.kwargs['timeout'], 0)
def test_valid_effective_configuration(self):
base, planned, live = fixture()
self.assertEqual([], self.module.verify(base, planned, live, b'nginx\n', b'nginx\n'))
class PipelineWiringTests(unittest.TestCase):
def test_authority_gate_precedes_build(self):
script = (Path(__file__).resolve().parents[2] / 'scripts/release-cache-stage.sh').read_text()
self.assertLess(script.index('python3 scripts/verify-stage-runtime.py'), script.index('"${compose[@]}" --progress plain build'))
if __name__ == '__main__':
unittest.main()
+164
View File
@@ -0,0 +1,164 @@
#!/usr/bin/env python3
"""Fail-closed, read-only comparison of Stage operational configuration."""
import argparse
import json
from pathlib import Path
import posixpath
import re
import subprocess
import sys
def mounts(config, name):
result = []
for mount in config['services'][name].get('volumes', []):
kind = mount['type']
source = mount.get('source', '')
if kind == 'volume':
source = config.get('volumes', {}).get(source, {}).get('name', source)
result.append((kind, posixpath.normpath(source), posixpath.normpath(mount['target']),
not mount.get('read_only', False)))
return sorted(result)
def networks(config, name):
return sorted(config.get('networks', {}).get(key, {}).get('name', key)
for key in config['services'][name].get('networks', {}))
def labels(values):
return {key: value for key, value in values.items() if key.startswith('traefik.')}
def verify(base, planned, live, nginx, live_nginx):
"""Return property/key mismatches, never configuration values."""
errors = []
# Validate resolved Compose environment, not env_file input: environment
# entries can override a provisioned secret. Match provisioner's format.
secrets = [planned['services'][name]['environment'].get('REVALIDATION_SECRET')
for name in ('backend_stage', 'frontend_stage')]
for name, secret in zip(('backend_stage', 'frontend_stage'), secrets):
if not isinstance(secret, str) or not re.fullmatch(r'[A-Za-z0-9_+/=-]{32,256}', secret):
errors.append(name + '.environment.REVALIDATION_SECRET')
if secrets[0] != secrets[1]:
errors.extend(name + '.environment.REVALIDATION_SECRET'
for name in ('backend_stage', 'frontend_stage'))
# Intentional rotation is permitted: do not compare these values to live.
allowed = {'REVALIDATION_SECRET', 'FRONTEND_INTERNAL_URL', 'INTERNAL_API_URL'}
defaults = {'PATH', 'NODE_VERSION', 'YARN_VERSION', 'PRISMA_HIDE_UPDATE_MESSAGE'}
for name in ('backend_stage', 'frontend_stage'):
required_args = {'NPM_REGISTRY': 'https://registry.parsaaghayi.ir/repository/npm/'}
if name == 'frontend_stage':
required_args.update(NEXT_PUBLIC_API_URL='https://stageapi.canina.ir/api',
VITE_API_URL='https://stageapi.canina.ir')
args = planned['services'][name].get('build', {}).get('args', {})
for key, expected_value in required_args.items():
if args.get(key) != expected_value:
errors.append(name + '.build.args.' + key)
original = base['services'][name]['environment']
target = planned['services'][name]['environment']
actual = dict(entry.split('=', 1) for entry in live[name]['Config']['Env'])
for key in sorted(set(original) | set(target) | set(actual)):
if key in allowed:
if ((key in original or key in actual) and key not in target
or key in target and target[key] is None):
errors.append(name + '.environment.' + key)
continue
if key not in original and key not in target and key in defaults:
continue
if (key not in original or key not in target or key not in actual
or original[key] is None or target[key] is None
or original[key] != actual[key] or target[key] != actual[key]):
errors.append(name + '.environment.' + key)
actual_mounts = sorted((m['Type'], posixpath.normpath(
m['Name'] if m['Type'] == 'volume' else m.get('Source', '')),
posixpath.normpath(m['Destination']), m.get('RW')) for m in live[name]['Mounts'])
actual_networks = sorted(live[name]['NetworkSettings']['Networks'])
actual_labels = labels(live[name]['Config'].get('Labels') or {})
for config in (base, planned):
if mounts(config, name) != actual_mounts:
errors.append(name + '.mounts')
for mount in config['services'][name].get('volumes', []):
if mount['type'] == 'bind' and 'propagation' in mount.get('bind', {}):
matches = [m for m in live[name]['Mounts']
if posixpath.normpath(m['Destination']) == posixpath.normpath(mount['target'])]
if (len(matches) != 1 or matches[0].get('Propagation')
!= mount['bind']['propagation']):
errors.append(name + '.mounts')
if mount['type'] == 'volume' and 'nocopy' in mount.get('volume', {}):
source = config.get('volumes', {}).get(mount.get('source'), {}).get(
'name', mount.get('source'))
matches = [m for m in live[name].get('HostConfig', {}).get('Mounts', [])
if m.get('Type') == 'volume' and m.get('Source') == source
and m.get('Target') == mount['target']]
# Some engines expose only HostConfig.Binds; never infer nocopy
# from the runtime Mounts summary, which omits this option.
if (len(matches) != 1 or matches[0].get('VolumeOptions', {}).get('NoCopy')
is not mount['volume']['nocopy']):
errors.append(name + '.mounts')
if networks(config, name) != actual_networks:
errors.append(name + '.networks')
for key, options in config['services'][name].get('networks', {}).items():
network_name = config.get('networks', {}).get(key, {}).get('name', key)
endpoint = live[name]['NetworkSettings']['Networks'].get(network_name, {})
options = options or {}
if not set(options.get('aliases') or []).issubset(set(endpoint.get('Aliases') or [])):
errors.append(name + '.networks')
for option, field in (('ipv4_address', 'IPAddress'),
('ipv6_address', 'GlobalIPv6Address')):
if option in options and options[option] != endpoint.get(field):
errors.append(name + '.networks')
expected = labels(config['services'][name].get('labels', {}))
for key in sorted(set(expected) | set(actual_labels)):
if key not in expected or key not in actual_labels or expected[key] != actual_labels[key]:
errors.append(name + '.labels.' + key)
if nginx != live_nginx:
errors.append('frontend_stage.nginx.bytes')
return sorted(set(errors))
def main(argv=None):
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--base-compose', required=True)
parser.add_argument('--override', action='append', default=[])
parser.add_argument('--project-directory', required=True)
parser.add_argument('--project-name', default='stage')
parser.add_argument('--nginx', required=True)
options = parser.parse_args(argv)
# Subprocess diagnostics may contain interpolated credentials. Capture them
# and never include exceptions, argv, config, or values in diagnostics.
def capture(command):
return subprocess.run(command, capture_output=True, check=True, timeout=30).stdout
try:
compose = ['docker', 'compose', '-p', options.project_name,
'--project-directory', options.project_directory,
'-f', options.base_compose]
base = json.loads(capture(compose + ['config', '--format', 'json']))
planned_command = compose.copy()
for override in options.override:
planned_command.extend(['-f', override])
planned = json.loads(capture(planned_command + ['config', '--format', 'json']))
inspected = json.loads(capture(['docker', 'inspect', 'canino_backend_stage',
'canino_frontend_stage']))
if not isinstance(inspected, list) or len(inspected) != 2:
raise ValueError()
live = dict(zip(('backend_stage', 'frontend_stage'), inspected))
live_nginx = capture(['docker', 'exec', 'canino_frontend_stage', 'cat',
'/etc/nginx/conf.d/default.conf'])
errors = verify(base, planned, live, Path(options.nginx).read_bytes(), live_nginx)
except (OSError, subprocess.SubprocessError, ValueError, KeyError, TypeError, AttributeError):
print('authority.input unavailable or malformed', file=sys.stderr)
return 1
for error in errors:
# Only property/key names are emitted, with control characters removed.
print('authority mismatch: ' + re.sub(r'[^a-zA-Z0-9_.-]', '?', error), file=sys.stderr)
if errors:
return 1
print('Stage runtime authority verified')
return 0
if __name__ == '__main__':
sys.exit(main())