Files
canina/scripts/release-cache-prod.sh
T
2026-10-26 20:30:19 +00:00

239 lines
12 KiB
Bash
Executable File

#!/usr/bin/env bash
# Narrow code-only Production release in the existing CI runner/daemon boundary.
set -euo pipefail
set +x
umask 077
[[ ${GITHUB_ACTIONS:-${GITEA_ACTIONS:-}} == true && ${DEPLOY_SHA:-} =~ ^[0-9a-f]{40}$ ]] || exit 2
PROD_DIR=/opt/services/canina/prod
# Alpine runner has BusyBox timeout; supervision needs GNU --foreground.
if ! timeout --help 2>&1 | grep -q -- --foreground; then
apk add --no-cache coreutils
fi
timeout --help 2>&1 | grep -q -- --foreground || exit 2
# The runner and host share a daemon, not a filesystem inode. The atomic
# container name serializes runners; the bind-mounted flock also fences host jobs.
# Never reclaim an existing name automatically (including stopped/stale helpers).
lock_container_id=
release_pid=
check_prod_lock() {
[[ $(timeout --foreground --kill-after=2s 10 docker inspect -f '{{.State.Running}}' "$lock_container_id") == true ]] || {
printf 'Production host lock lost; aborting release\n' >&2
return 1
}
}
supervise_release() {
# A separate process group lets lock loss/signals stop the active command too.
set -m
(set +m; "$@") &
release_pid=$!
while kill -0 "$release_pid" 2>/dev/null; do
check_prod_lock || return 1
sleep 1
done
local status=0
wait "$release_pid" || status=$?
release_pid=
check_prod_lock || return 1
return "$status"
}
cleanup_prod_lock() {
local status=$?
trap - EXIT
if [[ -n $release_pid ]]; then
kill -TERM -- "-$release_pid" 2>/dev/null || :
sleep 1
kill -KILL -- "-$release_pid" 2>/dev/null || :
wait "$release_pid" 2>/dev/null || :
fi
if [[ -n $lock_container_id ]]; then
if [[ $status != 0 ]]; then
# Never emit raw helper/daemon output (image entrypoints may contain
# secrets). Classify only a bounded tail before deleting the owned ID.
local helper_log diagnostic=unavailable
helper_log=$(timeout --foreground --kill-after=2s 10 docker logs --tail 10 "$lock_container_id" 2>&1 | head -c 4096) || :
case "$helper_log" in
*'Permission denied'*) diagnostic=permission-denied ;;
*'No such file or directory'*) diagnostic=missing-host-lock ;;
*'Read-only file system'*) diagnostic=read-only-filesystem ;;
*'flock: not found'*) diagnostic=flock-unavailable ;;
*CANINA_PRODUCTION_DEPLOY_LOCK_ACQUIRED*) diagnostic=previously-acquired ;;
?*) diagnostic=unclassified-output ;;
esac
printf 'Production lock helper diagnostic: %s\n' "$diagnostic" >&2
fi
timeout --kill-after=2s 15 docker rm -f "$lock_container_id" >/dev/null || {
printf 'Production lock helper cleanup failed; manual inspection required\n' >&2
[[ $status != 0 ]] || status=1
}
fi
exit "$status"
}
trap cleanup_prod_lock EXIT
trap 'exit 130' INT
trap 'exit 143' TERM
runner_image=$(timeout --foreground --kill-after=2s 10 docker inspect -f '{{.Config.Image}}' gitea_runner)
[[ -n $runner_image ]] || exit 1
lock_container_id=$(timeout --kill-after=2s 15 docker create --name canina-production-deploy-lock \
--pull=never --network none --cap-drop=ALL --security-opt=no-new-privileges \
--mount type=bind,src=/opt/services/canina,dst=/host-canina,readonly \
--entrypoint /bin/sh "$runner_image" -c '
# Linux flock permits an exclusive lock on a read-only descriptor. Preserve
# daemon userns remapping; require the existing host inode, never create it.
exec 9</host-canina/production-deploy.lock
flock -n 9 || exit 1
printf "CANINA_PRODUCTION_DEPLOY_LOCK_ACQUIRED\n"
while :; do sleep 60; done
') || { printf 'Production deployment locked or helper creation failed\n' >&2; exit 1; }
[[ -n $lock_container_id ]] || exit 1
timeout --foreground --kill-after=2s 10 docker start "$lock_container_id" >/dev/null
lock_acquired=false
for lock_attempt in {1..10}; do
[[ $(timeout --foreground --kill-after=2s 10 docker inspect -f '{{.State.Running}}' "$lock_container_id") == true ]] || break
if [[ $(timeout --foreground --kill-after=2s 10 docker logs --tail 1 "$lock_container_id" 2>/dev/null) == CANINA_PRODUCTION_DEPLOY_LOCK_ACQUIRED ]]; then
lock_acquired=true
break
fi
[[ $lock_attempt == 10 ]] || sleep 1
done
[[ $lock_acquired == true ]] || { printf 'Production host lock not confirmed\n' >&2; exit 1; }
release_prod() {
cd "$PROD_DIR"
timeout --foreground --kill-after=2s 60 git fetch origin main
[[ $(git rev-parse origin/main) == "$DEPLOY_SHA" ]] || exit 2
# Never reset/clean the operational checkout or overwrite its generated configs.
# Every attempt has its own directory: retain incomplete and active releases.
RELEASE_ROOT=/opt/services/canina/cache-releases
mkdir -p "$RELEASE_ROOT"
TARGET_DIR=$(mktemp -d "$RELEASE_ROOT/$DEPLOY_SHA.attempt.XXXXXXXX")
# Archive failure leaves only this isolated attempt, never a reusable release.
git archive "$DEPLOY_SHA" | tar -x -C "$TARGET_DIR"
[[ $(git rev-parse "$DEPLOY_SHA^{commit}") == "$DEPLOY_SHA" ]] || exit 2
# Record the exact source identity before introducing operational configuration.
command -v python3 >/dev/null || apk add --no-cache python3
python3 - "$TARGET_DIR" "$DEPLOY_SHA" <<'PY'
import hashlib, json, sys
from pathlib import Path
root = Path(sys.argv[1]); sha = sys.argv[2]
files = {str(p.relative_to(root)): hashlib.sha256(p.read_bytes()).hexdigest()
for p in sorted(root.rglob('*')) if p.is_file() and not p.is_symlink()}
manifest = {'sha': sha, 'files': files}
(root / '.release-manifest.json').write_text(json.dumps(manifest, sort_keys=True))
loaded = json.loads((root / '.release-manifest.json').read_text())
assert loaded['sha'] == sha
assert all(hashlib.sha256((root / name).read_bytes()).hexdigest() == digest
for name, digest in loaded['files'].items())
(root / '.release-ready').write_text(sha + '\n')
PY
[[ $(cat "$TARGET_DIR/.release-ready") == "$DEPLOY_SHA" ]] || exit 2
# The ready marker certifies extraction only, not deployment/readiness.
[[ $(git rev-parse HEAD) == "$DEPLOY_SHA" ]] || printf 'Operational HEAD intentionally preserved\n'
cd "$TARGET_DIR"
python3 scripts/provision-revalidation-env.py --environment prod --repair-empty \
--source "$PROD_DIR/.revalidation.env" --output "$TARGET_DIR/.revalidation.env"
# Build with the existing operational nginx configuration, without changing it.
cp "$PROD_DIR/nginx.conf" "$TARGET_DIR/nginx.conf"
# Compose uses the existing operational runtime configuration plus a private,
# release-specific override. Neither the database nor Redis is recreated.
export PROD_DIR TARGET_DIR DEPLOY_SHA
python3 - <<'PY'
import json, os
from pathlib import Path
root=Path(os.environ['TARGET_DIR']); sha=os.environ['DEPLOY_SHA']
services={}
for kind in ('backend','frontend'):
name=kind+'_prod'
services[name]={
'build': {'context': str(root/('backend' if kind=='backend' else '.')),
'labels': {'org.opencontainers.image.revision': sha},
'args': {'NPM_REGISTRY': 'https://registry.parsaaghayi.ir/repository/npm/'}},
'labels': {'ir.canina.release.sha': sha},
'env_file': [{'path': str(root/'.revalidation.env'), 'required': True}],
}
services['backend_prod']['environment']={'FRONTEND_INTERNAL_URL':'http://frontend_prod:3000'}
services['backend_prod']['command']=['node','dist/main.js']
services['frontend_prod']['environment']={'INTERNAL_API_URL':'http://backend_prod:3000/api', 'NEXT_PUBLIC_SITE_URL':'https://canina.ir'}
Path('release.override.json').write_text(json.dumps({'services':services}))
PY
compose=(docker compose -p prod --project-directory "$PROD_DIR" -f "$PROD_DIR/docker-compose.yml" -f "$TARGET_DIR/release.override.json")
timeout --foreground --kill-after=2s 60 "${compose[@]}" config --quiet
python3 scripts/verify-stage-runtime.py --base-compose "$PROD_DIR/docker-compose.yml" \
--override "$TARGET_DIR/release.override.json" --project-directory "$PROD_DIR" \
--project-name prod --environment prod --nginx "$TARGET_DIR/nginx.conf"
check_prod_lock
timeout --foreground --kill-after=2s 1800 "${compose[@]}" --progress plain build backend_prod frontend_prod
check_prod_lock
timeout --foreground --kill-after=2s 120 "${compose[@]}" up -d --no-deps --force-recreate backend_prod frontend_prod
for name in canino_backend_prod canino_frontend_prod; do
# Every daemon interaction is bounded, including a hung exec/inspect.
[[ $(timeout --foreground --kill-after=2s 10 docker inspect -f '{{index .Config.Labels "ir.canina.release.sha"}}' "$name") == "$DEPLOY_SHA" ]]
image=$(timeout --foreground --kill-after=2s 10 docker inspect -f '{{.Image}}' "$name")
[[ $(timeout --foreground --kill-after=2s 10 docker image inspect -f '{{index .Config.Labels "org.opencontainers.image.revision"}}' "$image") == "$DEPLOY_SHA" ]]
path=/
[[ $name != canino_backend_prod ]] || path='/api/blogs?page=1&limit=1'
ready=false
for attempt in {1..12}; do
[[ $(timeout --foreground --kill-after=2s 10 docker inspect -f '{{.State.Running}} {{.RestartCount}}' "$name") == 'true 0' ]] || { printf 'Container stopped or restarted: %s\n' "$name" >&2; exit 1; }
if timeout --foreground --kill-after=2s 10 docker exec "$name" node -e '
const http = require("http");
const req = http.get({host: "127.0.0.1", port: 3000, path: process.argv[1]}, res => {
if (res.statusCode !== 200) { res.resume(); process.exit(1); }
if (process.argv[1] === "/") { res.resume(); process.exit(0); }
// Existing public DB-backed contract; one row, no new health endpoint.
if (!(res.headers["content-type"] || "").includes("application/json")) process.exit(1);
let body = "";
res.setEncoding("utf8");
res.on("data", chunk => {
body += chunk;
if (Buffer.byteLength(body) > 2 * 1024 * 1024) process.exit(1);
});
res.on("error", () => process.exit(1));
res.on("end", () => {
try {
const {data, meta} = JSON.parse(body);
const valid = Array.isArray(data) && data.length <= 1 && meta &&
meta.page === 1 && meta.limit === 1 &&
Number.isInteger(meta.total) && meta.total >= data.length &&
Number.isInteger(meta.lastPage) && meta.lastPage >= 1;
process.exit(valid ? 0 : 1);
} catch { process.exit(1); }
});
});
req.setTimeout(5000, () => req.destroy(new Error("readiness timeout")));
req.on("error", () => process.exit(1));
' "$path"; then
ready=true
break
fi
[[ $attempt == 12 ]] || sleep 2
done
[[ $ready == true ]] || { printf 'Readiness timed out: %s\n' "$name" >&2; exit 1; }
if [[ $name == canino_frontend_prod ]]; then
for host in canina.ir admin.canina.ir; do
ready=false
for attempt in {1..12}; do
if timeout --foreground --kill-after=2s 10 docker exec "$name" node -e '
const http = require("http");
const req = http.get({host: "127.0.0.1", port: 8080, path: "/", headers: {Host: process.argv[1]}}, res => {
res.resume(); process.exit(res.statusCode === 200 ? 0 : 1);
});
req.setTimeout(5000, () => req.destroy(new Error("nginx readiness timeout")));
req.on("error", () => process.exit(1));
' "$host"; then
ready=true
break
fi
[[ $attempt == 12 ]] || sleep 2
done
[[ $ready == true ]] || { printf 'Nginx readiness timed out: %s\n' "$host" >&2; exit 1; }
done
fi
# Fence replacement/restart during probing, not just the initial state.
[[ $(timeout --foreground --kill-after=2s 10 docker inspect -f '{{.State.Running}} {{.RestartCount}}' "$name") == 'true 0' ]]
[[ $(timeout --foreground --kill-after=2s 10 docker inspect -f '{{.Image}}' "$name") == "$image" ]]
[[ $(timeout --foreground --kill-after=2s 10 docker inspect -f '{{index .Config.Labels "ir.canina.release.sha"}}' "$name") == "$DEPLOY_SHA" ]]
printf '%s image=%s release=%s ready\n' "$name" "$image" "$DEPLOY_SHA"
done
printf 'Code-only Production release complete: %s\n' "$DEPLOY_SHA"
}
supervise_release release_prod