fix(stage): fence runtime authority and bound release transports
Deploy Canina / deploy (push) Failing after 5s
Deploy Canina / deploy (push) Failing after 5s
This commit is contained in:
1 parent
52a7aebb51
commit
2a32f31097
6 files changed
+906
-12
No files matched your search
+158
-12
@@ -5,18 +5,111 @@ set +x
|
||||
umask 077
|
||||
[[ ${GITHUB_ACTIONS:-${GITEA_ACTIONS:-}} == true && ${DEPLOY_SHA:-} =~ ^[0-9a-f]{40}$ ]] || exit 2
|
||||
STAGE_DIR=/opt/services/canina/stage
|
||||
exec 9>/opt/services/canina/stage-backend-deploy.lock
|
||||
flock -n 9 || { printf 'Stage deployment locked\n' >&2; exit 1; }
|
||||
# Alpine runner has BusyBox timeout; supervision needs GNU --foreground.
|
||||
if ! timeout --help 2>&1 | grep -q -- --foreground; then
|
||||
apk add --no-cache coreutils
|
||||
fi
|
||||
timeout --help 2>&1 | grep -q -- --foreground || exit 2
|
||||
# The runner and host share a daemon, not a filesystem inode. The atomic
|
||||
# container name serializes runners; the bind-mounted flock also fences host jobs.
|
||||
# Never reclaim an existing name automatically (including stopped/stale helpers).
|
||||
lock_container_id=
|
||||
release_pid=
|
||||
check_stage_lock() {
|
||||
[[ $(timeout --foreground --kill-after=2s 10 docker inspect -f '{{.State.Running}}' "$lock_container_id") == true ]] || {
|
||||
printf 'Stage host lock lost; aborting release\n' >&2
|
||||
return 1
|
||||
}
|
||||
}
|
||||
supervise_release() {
|
||||
# A separate process group lets lock loss/signals stop the active command too.
|
||||
set -m
|
||||
(set +m; "$@") &
|
||||
release_pid=$!
|
||||
while kill -0 "$release_pid" 2>/dev/null; do
|
||||
check_stage_lock || return 1
|
||||
sleep 1
|
||||
done
|
||||
local status=0
|
||||
wait "$release_pid" || status=$?
|
||||
release_pid=
|
||||
check_stage_lock || return 1
|
||||
return "$status"
|
||||
}
|
||||
cleanup_stage_lock() {
|
||||
local status=$?
|
||||
trap - EXIT
|
||||
if [[ -n $release_pid ]]; then
|
||||
kill -TERM -- "-$release_pid" 2>/dev/null || :
|
||||
sleep 1
|
||||
kill -KILL -- "-$release_pid" 2>/dev/null || :
|
||||
wait "$release_pid" 2>/dev/null || :
|
||||
fi
|
||||
if [[ -n $lock_container_id ]]; then
|
||||
timeout --kill-after=2s 15 docker rm -f "$lock_container_id" >/dev/null || {
|
||||
printf 'Stage lock helper cleanup failed; manual inspection required\n' >&2
|
||||
[[ $status != 0 ]] || status=1
|
||||
}
|
||||
fi
|
||||
exit "$status"
|
||||
}
|
||||
trap cleanup_stage_lock EXIT
|
||||
trap 'exit 130' INT
|
||||
trap 'exit 143' TERM
|
||||
runner_image=$(timeout --foreground --kill-after=2s 10 docker inspect -f '{{.Config.Image}}' gitea_runner)
|
||||
[[ -n $runner_image ]] || exit 1
|
||||
lock_container_id=$(timeout --kill-after=2s 15 docker create --name canina-stage-deploy-lock \
|
||||
--pull=never --network none \
|
||||
--mount type=bind,src=/opt/services/canina,dst=/host-canina \
|
||||
--entrypoint /bin/sh "$runner_image" -c '
|
||||
exec 9>/host-canina/stage-backend-deploy.lock
|
||||
flock -n 9 || exit 1
|
||||
printf "CANINA_STAGE_DEPLOY_LOCK_ACQUIRED\n"
|
||||
while :; do sleep 60; done
|
||||
') || { printf 'Stage deployment locked or helper creation failed\n' >&2; exit 1; }
|
||||
[[ -n $lock_container_id ]] || exit 1
|
||||
timeout --foreground --kill-after=2s 10 docker start "$lock_container_id" >/dev/null
|
||||
lock_acquired=false
|
||||
for lock_attempt in {1..10}; do
|
||||
[[ $(timeout --foreground --kill-after=2s 10 docker inspect -f '{{.State.Running}}' "$lock_container_id") == true ]] || break
|
||||
if [[ $(timeout --foreground --kill-after=2s 10 docker logs "$lock_container_id") == CANINA_STAGE_DEPLOY_LOCK_ACQUIRED ]]; then
|
||||
lock_acquired=true
|
||||
break
|
||||
fi
|
||||
[[ $lock_attempt == 10 ]] || sleep 1
|
||||
done
|
||||
[[ $lock_acquired == true ]] || { printf 'Stage host lock not confirmed\n' >&2; exit 1; }
|
||||
release_stage() {
|
||||
cd "$STAGE_DIR"
|
||||
git fetch origin develop
|
||||
timeout --foreground --kill-after=2s 60 git fetch origin develop
|
||||
[[ $(git rev-parse origin/develop) == "$DEPLOY_SHA" ]] || exit 2
|
||||
# Never reset/clean the operational checkout or overwrite its generated configs.
|
||||
TARGET_DIR="/opt/services/canina/cache-releases/$DEPLOY_SHA"
|
||||
[[ ! -e "$TARGET_DIR" ]] || { printf 'Release directory already exists\n' >&2; exit 2; }
|
||||
mkdir -p "$TARGET_DIR"
|
||||
# Every attempt has its own directory: retain incomplete and active releases.
|
||||
RELEASE_ROOT=/opt/services/canina/cache-releases
|
||||
mkdir -p "$RELEASE_ROOT"
|
||||
TARGET_DIR=$(mktemp -d "$RELEASE_ROOT/$DEPLOY_SHA.attempt.XXXXXXXX")
|
||||
# Archive failure leaves only this isolated attempt, never a reusable release.
|
||||
git archive "$DEPLOY_SHA" | tar -x -C "$TARGET_DIR"
|
||||
[[ $(git rev-parse HEAD) == "$DEPLOY_SHA" ]] || printf 'Operational HEAD intentionally preserved\n'
|
||||
[[ $(git rev-parse "$DEPLOY_SHA^{commit}") == "$DEPLOY_SHA" ]] || exit 2
|
||||
# Record the exact source identity before introducing operational configuration.
|
||||
command -v python3 >/dev/null || apk add --no-cache python3
|
||||
python3 - "$TARGET_DIR" "$DEPLOY_SHA" <<'PY'
|
||||
import hashlib, json, sys
|
||||
from pathlib import Path
|
||||
root = Path(sys.argv[1]); sha = sys.argv[2]
|
||||
files = {str(p.relative_to(root)): hashlib.sha256(p.read_bytes()).hexdigest()
|
||||
for p in sorted(root.rglob('*')) if p.is_file() and not p.is_symlink()}
|
||||
manifest = {'sha': sha, 'files': files}
|
||||
(root / '.release-manifest.json').write_text(json.dumps(manifest, sort_keys=True))
|
||||
loaded = json.loads((root / '.release-manifest.json').read_text())
|
||||
assert loaded['sha'] == sha
|
||||
assert all(hashlib.sha256((root / name).read_bytes()).hexdigest() == digest
|
||||
for name, digest in loaded['files'].items())
|
||||
(root / '.release-ready').write_text(sha + '\n')
|
||||
PY
|
||||
[[ $(cat "$TARGET_DIR/.release-ready") == "$DEPLOY_SHA" ]] || exit 2
|
||||
# The ready marker certifies extraction only, not deployment/readiness.
|
||||
[[ $(git rev-parse HEAD) == "$DEPLOY_SHA" ]] || printf 'Operational HEAD intentionally preserved\n'
|
||||
cd "$TARGET_DIR"
|
||||
python3 scripts/provision-revalidation-env.py --environment stage --repair-empty \
|
||||
--source "$STAGE_DIR/.revalidation.env" --output "$TARGET_DIR/.revalidation.env"
|
||||
@@ -45,11 +138,64 @@ services['frontend_stage']['environment']={'INTERNAL_API_URL':'http://backend_st
|
||||
Path('release.override.json').write_text(json.dumps({'services':services}))
|
||||
PY
|
||||
compose=(docker compose -p stage --project-directory "$STAGE_DIR" -f "$STAGE_DIR/docker-compose.yml" -f "$TARGET_DIR/release.override.json")
|
||||
"${compose[@]}" config --quiet
|
||||
"${compose[@]}" --progress plain build backend_stage frontend_stage
|
||||
"${compose[@]}" up -d --no-deps --force-recreate backend_stage frontend_stage
|
||||
timeout --foreground --kill-after=2s 60 "${compose[@]}" config --quiet
|
||||
python3 scripts/verify-stage-runtime.py --base-compose "$STAGE_DIR/docker-compose.yml" \
|
||||
--override "$TARGET_DIR/release.override.json" --project-directory "$STAGE_DIR" \
|
||||
--nginx "$TARGET_DIR/nginx.conf"
|
||||
check_stage_lock
|
||||
timeout --foreground --kill-after=2s 1800 "${compose[@]}" --progress plain build backend_stage frontend_stage
|
||||
check_stage_lock
|
||||
timeout --foreground --kill-after=2s 120 "${compose[@]}" up -d --no-deps --force-recreate backend_stage frontend_stage
|
||||
for name in canino_backend_stage canino_frontend_stage; do
|
||||
[[ $(docker inspect -f '{{index .Config.Labels "ir.canina.release.sha"}}' "$name") == "$DEPLOY_SHA" ]]
|
||||
docker inspect -f '{{.Name}} image={{.Image}} release={{index .Config.Labels "ir.canina.release.sha"}}' "$name"
|
||||
# Every daemon interaction is bounded, including a hung exec/inspect.
|
||||
[[ $(timeout --foreground --kill-after=2s 10 docker inspect -f '{{index .Config.Labels "ir.canina.release.sha"}}' "$name") == "$DEPLOY_SHA" ]]
|
||||
image=$(timeout --foreground --kill-after=2s 10 docker inspect -f '{{.Image}}' "$name")
|
||||
[[ $(timeout --foreground --kill-after=2s 10 docker image inspect -f '{{index .Config.Labels "org.opencontainers.image.revision"}}' "$image") == "$DEPLOY_SHA" ]]
|
||||
path=/
|
||||
[[ $name != canino_backend_stage ]] || path=/api
|
||||
ready=false
|
||||
for attempt in {1..12}; do
|
||||
[[ $(timeout --foreground --kill-after=2s 10 docker inspect -f '{{.State.Running}} {{.RestartCount}}' "$name") == 'true 0' ]] || { printf 'Container stopped or restarted: %s\n' "$name" >&2; exit 1; }
|
||||
if timeout --foreground --kill-after=2s 10 docker exec "$name" node -e '
|
||||
const http = require("http");
|
||||
const req = http.get({host: "127.0.0.1", port: 3000, path: process.argv[1]}, res => {
|
||||
res.resume(); process.exit(res.statusCode === 200 ? 0 : 1);
|
||||
});
|
||||
req.setTimeout(5000, () => req.destroy(new Error("readiness timeout")));
|
||||
req.on("error", () => process.exit(1));
|
||||
' "$path"; then
|
||||
ready=true
|
||||
break
|
||||
fi
|
||||
[[ $attempt == 12 ]] || sleep 2
|
||||
done
|
||||
[[ $ready == true ]] || { printf 'Readiness timed out: %s\n' "$name" >&2; exit 1; }
|
||||
if [[ $name == canino_frontend_stage ]]; then
|
||||
for host in stage.canina.ir stageadmin.canina.ir; do
|
||||
ready=false
|
||||
for attempt in {1..12}; do
|
||||
if timeout --foreground --kill-after=2s 10 docker exec "$name" node -e '
|
||||
const http = require("http");
|
||||
const req = http.get({host: "127.0.0.1", port: 8080, path: "/", headers: {Host: process.argv[1]}}, res => {
|
||||
res.resume(); process.exit(res.statusCode === 200 ? 0 : 1);
|
||||
});
|
||||
req.setTimeout(5000, () => req.destroy(new Error("nginx readiness timeout")));
|
||||
req.on("error", () => process.exit(1));
|
||||
' "$host"; then
|
||||
ready=true
|
||||
break
|
||||
fi
|
||||
[[ $attempt == 12 ]] || sleep 2
|
||||
done
|
||||
[[ $ready == true ]] || { printf 'Nginx readiness timed out: %s\n' "$host" >&2; exit 1; }
|
||||
done
|
||||
fi
|
||||
# Fence replacement/restart during probing, not just the initial state.
|
||||
[[ $(timeout --foreground --kill-after=2s 10 docker inspect -f '{{.State.Running}} {{.RestartCount}}' "$name") == 'true 0' ]]
|
||||
[[ $(timeout --foreground --kill-after=2s 10 docker inspect -f '{{.Image}}' "$name") == "$image" ]]
|
||||
[[ $(timeout --foreground --kill-after=2s 10 docker inspect -f '{{index .Config.Labels "ir.canina.release.sha"}}' "$name") == "$DEPLOY_SHA" ]]
|
||||
printf '%s image=%s release=%s ready\n' "$name" "$image" "$DEPLOY_SHA"
|
||||
done
|
||||
printf 'Code-only Stage release complete: %s\n' "$DEPLOY_SHA"
|
||||
}
|
||||
supervise_release release_stage
|
||||
Reference in new issue
Block a user