fix(ci): route production code-only release through safe locked deployment without migrations
Deploy Canina / deploy (push) Failing after 10s
Deploy Canina / deploy (push) Failing after 10s
This commit is contained in:
1 parent
9ae839138c
commit
53fccc2d3c
5 files changed
+355
-198
No files matched your search
Executable
+238
@@ -0,0 +1,238 @@
|
||||
#!/usr/bin/env bash
|
||||
# Narrow code-only Production release in the existing CI runner/daemon boundary.
|
||||
set -euo pipefail
|
||||
set +x
|
||||
umask 077
|
||||
[[ ${GITHUB_ACTIONS:-${GITEA_ACTIONS:-}} == true && ${DEPLOY_SHA:-} =~ ^[0-9a-f]{40}$ ]] || exit 2
|
||||
PROD_DIR=/opt/services/canina/prod
|
||||
# Alpine runner has BusyBox timeout; supervision needs GNU --foreground.
|
||||
if ! timeout --help 2>&1 | grep -q -- --foreground; then
|
||||
apk add --no-cache coreutils
|
||||
fi
|
||||
timeout --help 2>&1 | grep -q -- --foreground || exit 2
|
||||
# The runner and host share a daemon, not a filesystem inode. The atomic
|
||||
# container name serializes runners; the bind-mounted flock also fences host jobs.
|
||||
# Never reclaim an existing name automatically (including stopped/stale helpers).
|
||||
lock_container_id=
|
||||
release_pid=
|
||||
check_prod_lock() {
|
||||
[[ $(timeout --foreground --kill-after=2s 10 docker inspect -f '{{.State.Running}}' "$lock_container_id") == true ]] || {
|
||||
printf 'Production host lock lost; aborting release\n' >&2
|
||||
return 1
|
||||
}
|
||||
}
|
||||
supervise_release() {
|
||||
# A separate process group lets lock loss/signals stop the active command too.
|
||||
set -m
|
||||
(set +m; "$@") &
|
||||
release_pid=$!
|
||||
while kill -0 "$release_pid" 2>/dev/null; do
|
||||
check_prod_lock || return 1
|
||||
sleep 1
|
||||
done
|
||||
local status=0
|
||||
wait "$release_pid" || status=$?
|
||||
release_pid=
|
||||
check_prod_lock || return 1
|
||||
return "$status"
|
||||
}
|
||||
cleanup_prod_lock() {
|
||||
local status=$?
|
||||
trap - EXIT
|
||||
if [[ -n $release_pid ]]; then
|
||||
kill -TERM -- "-$release_pid" 2>/dev/null || :
|
||||
sleep 1
|
||||
kill -KILL -- "-$release_pid" 2>/dev/null || :
|
||||
wait "$release_pid" 2>/dev/null || :
|
||||
fi
|
||||
if [[ -n $lock_container_id ]]; then
|
||||
if [[ $status != 0 ]]; then
|
||||
# Never emit raw helper/daemon output (image entrypoints may contain
|
||||
# secrets). Classify only a bounded tail before deleting the owned ID.
|
||||
local helper_log diagnostic=unavailable
|
||||
helper_log=$(timeout --foreground --kill-after=2s 10 docker logs --tail 10 "$lock_container_id" 2>&1 | head -c 4096) || :
|
||||
case "$helper_log" in
|
||||
*'Permission denied'*) diagnostic=permission-denied ;;
|
||||
*'No such file or directory'*) diagnostic=missing-host-lock ;;
|
||||
*'Read-only file system'*) diagnostic=read-only-filesystem ;;
|
||||
*'flock: not found'*) diagnostic=flock-unavailable ;;
|
||||
*CANINA_PRODUCTION_DEPLOY_LOCK_ACQUIRED*) diagnostic=previously-acquired ;;
|
||||
?*) diagnostic=unclassified-output ;;
|
||||
esac
|
||||
printf 'Production lock helper diagnostic: %s\n' "$diagnostic" >&2
|
||||
fi
|
||||
timeout --kill-after=2s 15 docker rm -f "$lock_container_id" >/dev/null || {
|
||||
printf 'Production lock helper cleanup failed; manual inspection required\n' >&2
|
||||
[[ $status != 0 ]] || status=1
|
||||
}
|
||||
fi
|
||||
exit "$status"
|
||||
}
|
||||
trap cleanup_prod_lock EXIT
|
||||
trap 'exit 130' INT
|
||||
trap 'exit 143' TERM
|
||||
runner_image=$(timeout --foreground --kill-after=2s 10 docker inspect -f '{{.Config.Image}}' gitea_runner)
|
||||
[[ -n $runner_image ]] || exit 1
|
||||
lock_container_id=$(timeout --kill-after=2s 15 docker create --name canina-production-deploy-lock \
|
||||
--pull=never --network none --cap-drop=ALL --security-opt=no-new-privileges \
|
||||
--mount type=bind,src=/opt/services/canina,dst=/host-canina,readonly \
|
||||
--entrypoint /bin/sh "$runner_image" -c '
|
||||
# Linux flock permits an exclusive lock on a read-only descriptor. Preserve
|
||||
# daemon userns remapping; require the existing host inode, never create it.
|
||||
exec 9</host-canina/production-deploy.lock
|
||||
flock -n 9 || exit 1
|
||||
printf "CANINA_PRODUCTION_DEPLOY_LOCK_ACQUIRED\n"
|
||||
while :; do sleep 60; done
|
||||
') || { printf 'Production deployment locked or helper creation failed\n' >&2; exit 1; }
|
||||
[[ -n $lock_container_id ]] || exit 1
|
||||
timeout --foreground --kill-after=2s 10 docker start "$lock_container_id" >/dev/null
|
||||
lock_acquired=false
|
||||
for lock_attempt in {1..10}; do
|
||||
[[ $(timeout --foreground --kill-after=2s 10 docker inspect -f '{{.State.Running}}' "$lock_container_id") == true ]] || break
|
||||
if [[ $(timeout --foreground --kill-after=2s 10 docker logs --tail 1 "$lock_container_id" 2>/dev/null) == CANINA_PRODUCTION_DEPLOY_LOCK_ACQUIRED ]]; then
|
||||
lock_acquired=true
|
||||
break
|
||||
fi
|
||||
[[ $lock_attempt == 10 ]] || sleep 1
|
||||
done
|
||||
[[ $lock_acquired == true ]] || { printf 'Production host lock not confirmed\n' >&2; exit 1; }
|
||||
release_prod() {
|
||||
cd "$PROD_DIR"
|
||||
timeout --foreground --kill-after=2s 60 git fetch origin main
|
||||
[[ $(git rev-parse origin/main) == "$DEPLOY_SHA" ]] || exit 2
|
||||
# Never reset/clean the operational checkout or overwrite its generated configs.
|
||||
# Every attempt has its own directory: retain incomplete and active releases.
|
||||
RELEASE_ROOT=/opt/services/canina/cache-releases
|
||||
mkdir -p "$RELEASE_ROOT"
|
||||
TARGET_DIR=$(mktemp -d "$RELEASE_ROOT/$DEPLOY_SHA.attempt.XXXXXXXX")
|
||||
# Archive failure leaves only this isolated attempt, never a reusable release.
|
||||
git archive "$DEPLOY_SHA" | tar -x -C "$TARGET_DIR"
|
||||
[[ $(git rev-parse "$DEPLOY_SHA^{commit}") == "$DEPLOY_SHA" ]] || exit 2
|
||||
# Record the exact source identity before introducing operational configuration.
|
||||
command -v python3 >/dev/null || apk add --no-cache python3
|
||||
python3 - "$TARGET_DIR" "$DEPLOY_SHA" <<'PY'
|
||||
import hashlib, json, sys
|
||||
from pathlib import Path
|
||||
root = Path(sys.argv[1]); sha = sys.argv[2]
|
||||
files = {str(p.relative_to(root)): hashlib.sha256(p.read_bytes()).hexdigest()
|
||||
for p in sorted(root.rglob('*')) if p.is_file() and not p.is_symlink()}
|
||||
manifest = {'sha': sha, 'files': files}
|
||||
(root / '.release-manifest.json').write_text(json.dumps(manifest, sort_keys=True))
|
||||
loaded = json.loads((root / '.release-manifest.json').read_text())
|
||||
assert loaded['sha'] == sha
|
||||
assert all(hashlib.sha256((root / name).read_bytes()).hexdigest() == digest
|
||||
for name, digest in loaded['files'].items())
|
||||
(root / '.release-ready').write_text(sha + '\n')
|
||||
PY
|
||||
[[ $(cat "$TARGET_DIR/.release-ready") == "$DEPLOY_SHA" ]] || exit 2
|
||||
# The ready marker certifies extraction only, not deployment/readiness.
|
||||
[[ $(git rev-parse HEAD) == "$DEPLOY_SHA" ]] || printf 'Operational HEAD intentionally preserved\n'
|
||||
cd "$TARGET_DIR"
|
||||
python3 scripts/provision-revalidation-env.py --environment prod \
|
||||
--source "$PROD_DIR/.revalidation.env" --output "$TARGET_DIR/.revalidation.env"
|
||||
# Build with the existing operational nginx configuration, without changing it.
|
||||
cp "$PROD_DIR/nginx.conf" "$TARGET_DIR/nginx.conf"
|
||||
# Compose uses the existing operational runtime configuration plus a private,
|
||||
# release-specific override. Neither the database nor Redis is recreated.
|
||||
export PROD_DIR TARGET_DIR DEPLOY_SHA
|
||||
python3 - <<'PY'
|
||||
import json, os
|
||||
from pathlib import Path
|
||||
root=Path(os.environ['TARGET_DIR']); sha=os.environ['DEPLOY_SHA']
|
||||
services={}
|
||||
for kind in ('backend','frontend'):
|
||||
name=kind+'_prod'
|
||||
services[name]={
|
||||
'build': {'context': str(root/('backend' if kind=='backend' else '.')),
|
||||
'labels': {'org.opencontainers.image.revision': sha},
|
||||
'args': {'NPM_REGISTRY': 'https://registry.parsaaghayi.ir/repository/npm/'}},
|
||||
'labels': {'ir.canina.release.sha': sha},
|
||||
'env_file': [{'path': str(root/'.revalidation.env'), 'required': True}],
|
||||
}
|
||||
services['backend_prod']['environment']={'FRONTEND_INTERNAL_URL':'http://frontend_prod:3000'}
|
||||
services['backend_prod']['command']=['node','dist/main.js']
|
||||
services['frontend_prod']['environment']={'INTERNAL_API_URL':'http://backend_prod:3000/api', 'NEXT_PUBLIC_SITE_URL':'https://canina.ir'}
|
||||
Path('release.override.json').write_text(json.dumps({'services':services}))
|
||||
PY
|
||||
compose=(docker compose -p prod --project-directory "$PROD_DIR" -f "$PROD_DIR/docker-compose.yml" -f "$TARGET_DIR/release.override.json")
|
||||
timeout --foreground --kill-after=2s 60 "${compose[@]}" config --quiet
|
||||
python3 scripts/verify-stage-runtime.py --base-compose "$PROD_DIR/docker-compose.yml" \
|
||||
--override "$TARGET_DIR/release.override.json" --project-directory "$PROD_DIR" \
|
||||
--project-name prod --environment prod --nginx "$TARGET_DIR/nginx.conf"
|
||||
check_prod_lock
|
||||
timeout --foreground --kill-after=2s 1800 "${compose[@]}" --progress plain build backend_prod frontend_prod
|
||||
check_prod_lock
|
||||
timeout --foreground --kill-after=2s 120 "${compose[@]}" up -d --no-deps --force-recreate backend_prod frontend_prod
|
||||
for name in canino_backend_prod canino_frontend_prod; do
|
||||
# Every daemon interaction is bounded, including a hung exec/inspect.
|
||||
[[ $(timeout --foreground --kill-after=2s 10 docker inspect -f '{{index .Config.Labels "ir.canina.release.sha"}}' "$name") == "$DEPLOY_SHA" ]]
|
||||
image=$(timeout --foreground --kill-after=2s 10 docker inspect -f '{{.Image}}' "$name")
|
||||
[[ $(timeout --foreground --kill-after=2s 10 docker image inspect -f '{{index .Config.Labels "org.opencontainers.image.revision"}}' "$image") == "$DEPLOY_SHA" ]]
|
||||
path=/
|
||||
[[ $name != canino_backend_prod ]] || path='/api/blogs?page=1&limit=1'
|
||||
ready=false
|
||||
for attempt in {1..12}; do
|
||||
[[ $(timeout --foreground --kill-after=2s 10 docker inspect -f '{{.State.Running}} {{.RestartCount}}' "$name") == 'true 0' ]] || { printf 'Container stopped or restarted: %s\n' "$name" >&2; exit 1; }
|
||||
if timeout --foreground --kill-after=2s 10 docker exec "$name" node -e '
|
||||
const http = require("http");
|
||||
const req = http.get({host: "127.0.0.1", port: 3000, path: process.argv[1]}, res => {
|
||||
if (res.statusCode !== 200) { res.resume(); process.exit(1); }
|
||||
if (process.argv[1] === "/") { res.resume(); process.exit(0); }
|
||||
// Existing public DB-backed contract; one row, no new health endpoint.
|
||||
if (!(res.headers["content-type"] || "").includes("application/json")) process.exit(1);
|
||||
let body = "";
|
||||
res.setEncoding("utf8");
|
||||
res.on("data", chunk => {
|
||||
body += chunk;
|
||||
if (Buffer.byteLength(body) > 2 * 1024 * 1024) process.exit(1);
|
||||
});
|
||||
res.on("error", () => process.exit(1));
|
||||
res.on("end", () => {
|
||||
try {
|
||||
const {data, meta} = JSON.parse(body);
|
||||
const valid = Array.isArray(data) && data.length <= 1 && meta &&
|
||||
meta.page === 1 && meta.limit === 1 &&
|
||||
Number.isInteger(meta.total) && meta.total >= data.length &&
|
||||
Number.isInteger(meta.lastPage) && meta.lastPage >= 1;
|
||||
process.exit(valid ? 0 : 1);
|
||||
} catch { process.exit(1); }
|
||||
});
|
||||
});
|
||||
req.setTimeout(5000, () => req.destroy(new Error("readiness timeout")));
|
||||
req.on("error", () => process.exit(1));
|
||||
' "$path"; then
|
||||
ready=true
|
||||
break
|
||||
fi
|
||||
[[ $attempt == 12 ]] || sleep 2
|
||||
done
|
||||
[[ $ready == true ]] || { printf 'Readiness timed out: %s\n' "$name" >&2; exit 1; }
|
||||
if [[ $name == canino_frontend_prod ]]; then
|
||||
for host in canina.ir admin.canina.ir; do
|
||||
ready=false
|
||||
for attempt in {1..12}; do
|
||||
if timeout --foreground --kill-after=2s 10 docker exec "$name" node -e '
|
||||
const http = require("http");
|
||||
const req = http.get({host: "127.0.0.1", port: 8080, path: "/", headers: {Host: process.argv[1]}}, res => {
|
||||
res.resume(); process.exit(res.statusCode === 200 ? 0 : 1);
|
||||
});
|
||||
req.setTimeout(5000, () => req.destroy(new Error("nginx readiness timeout")));
|
||||
req.on("error", () => process.exit(1));
|
||||
' "$host"; then
|
||||
ready=true
|
||||
break
|
||||
fi
|
||||
[[ $attempt == 12 ]] || sleep 2
|
||||
done
|
||||
[[ $ready == true ]] || { printf 'Nginx readiness timed out: %s\n' "$host" >&2; exit 1; }
|
||||
done
|
||||
fi
|
||||
# Fence replacement/restart during probing, not just the initial state.
|
||||
[[ $(timeout --foreground --kill-after=2s 10 docker inspect -f '{{.State.Running}} {{.RestartCount}}' "$name") == 'true 0' ]]
|
||||
[[ $(timeout --foreground --kill-after=2s 10 docker inspect -f '{{.Image}}' "$name") == "$image" ]]
|
||||
[[ $(timeout --foreground --kill-after=2s 10 docker inspect -f '{{index .Config.Labels "ir.canina.release.sha"}}' "$name") == "$DEPLOY_SHA" ]]
|
||||
printf '%s image=%s release=%s ready\n' "$name" "$image" "$DEPLOY_SHA"
|
||||
done
|
||||
printf 'Code-only Production release complete: %s\n' "$DEPLOY_SHA"
|
||||
}
|
||||
supervise_release release_prod
|
||||
Reference in new issue
Block a user