219 lines
11 KiB
Bash
219 lines
11 KiB
Bash
#!/usr/bin/env bash
|
|
# Narrow code-only Stage release in the existing CI runner/daemon boundary.
|
|
set -euo pipefail
|
|
set +x
|
|
umask 077
|
|
[[ ${GITHUB_ACTIONS:-${GITEA_ACTIONS:-}} == true && ${DEPLOY_SHA:-} =~ ^[0-9a-f]{40}$ ]] || exit 2
|
|
STAGE_DIR=/opt/services/canina/stage
|
|
# Alpine runner has BusyBox timeout; supervision needs GNU --foreground.
|
|
if ! timeout --help 2>&1 | grep -q -- --foreground; then
|
|
apk add --no-cache coreutils
|
|
fi
|
|
timeout --help 2>&1 | grep -q -- --foreground || exit 2
|
|
# The runner and host share a daemon, not a filesystem inode. The atomic
|
|
# container name serializes runners; the bind-mounted flock also fences host jobs.
|
|
# Never reclaim an existing name automatically (including stopped/stale helpers).
|
|
lock_container_id=
|
|
release_pid=
|
|
check_stage_lock() {
|
|
[[ $(timeout --foreground --kill-after=2s 10 docker inspect -f '{{.State.Running}}' "$lock_container_id") == true ]] || {
|
|
printf 'Stage host lock lost; aborting release\n' >&2
|
|
return 1
|
|
}
|
|
}
|
|
supervise_release() {
|
|
# A separate process group lets lock loss/signals stop the active command too.
|
|
set -m
|
|
(set +m; "$@") &
|
|
release_pid=$!
|
|
while kill -0 "$release_pid" 2>/dev/null; do
|
|
check_stage_lock || return 1
|
|
sleep 1
|
|
done
|
|
local status=0
|
|
wait "$release_pid" || status=$?
|
|
release_pid=
|
|
check_stage_lock || return 1
|
|
return "$status"
|
|
}
|
|
cleanup_stage_lock() {
|
|
local status=$?
|
|
trap - EXIT
|
|
if [[ -n $release_pid ]]; then
|
|
kill -TERM -- "-$release_pid" 2>/dev/null || :
|
|
sleep 1
|
|
kill -KILL -- "-$release_pid" 2>/dev/null || :
|
|
wait "$release_pid" 2>/dev/null || :
|
|
fi
|
|
if [[ -n $lock_container_id ]]; then
|
|
if [[ $status != 0 ]]; then
|
|
# Never emit raw helper/daemon output (image entrypoints may contain
|
|
# secrets). Classify only a bounded tail before deleting the owned ID.
|
|
local helper_log diagnostic=unavailable
|
|
helper_log=$(timeout --foreground --kill-after=2s 10 docker logs --tail 10 "$lock_container_id" 2>&1 | head -c 4096) || :
|
|
case "$helper_log" in
|
|
*'Permission denied'*) diagnostic=permission-denied ;;
|
|
*'No such file or directory'*) diagnostic=missing-host-lock ;;
|
|
*'Read-only file system'*) diagnostic=read-only-filesystem ;;
|
|
*'flock: not found'*) diagnostic=flock-unavailable ;;
|
|
*CANINA_STAGE_DEPLOY_LOCK_ACQUIRED*) diagnostic=previously-acquired ;;
|
|
?*) diagnostic=unclassified-output ;;
|
|
esac
|
|
printf 'Stage lock helper diagnostic: %s\n' "$diagnostic" >&2
|
|
fi
|
|
timeout --kill-after=2s 15 docker rm -f "$lock_container_id" >/dev/null || {
|
|
printf 'Stage lock helper cleanup failed; manual inspection required\n' >&2
|
|
[[ $status != 0 ]] || status=1
|
|
}
|
|
fi
|
|
exit "$status"
|
|
}
|
|
trap cleanup_stage_lock EXIT
|
|
trap 'exit 130' INT
|
|
trap 'exit 143' TERM
|
|
runner_image=$(timeout --foreground --kill-after=2s 10 docker inspect -f '{{.Config.Image}}' gitea_runner)
|
|
[[ -n $runner_image ]] || exit 1
|
|
lock_container_id=$(timeout --kill-after=2s 15 docker create --name canina-stage-deploy-lock \
|
|
--pull=never --network none --cap-drop=ALL --security-opt=no-new-privileges \
|
|
--mount type=bind,src=/opt/services/canina,dst=/host-canina,readonly \
|
|
--entrypoint /bin/sh "$runner_image" -c '
|
|
# Linux flock permits an exclusive lock on a read-only descriptor. Preserve
|
|
# daemon userns remapping; require the existing host inode, never create it.
|
|
exec 9</host-canina/stage-backend-deploy.lock
|
|
flock -n 9 || exit 1
|
|
printf "CANINA_STAGE_DEPLOY_LOCK_ACQUIRED\n"
|
|
while :; do sleep 60; done
|
|
') || { printf 'Stage deployment locked or helper creation failed\n' >&2; exit 1; }
|
|
[[ -n $lock_container_id ]] || exit 1
|
|
timeout --foreground --kill-after=2s 10 docker start "$lock_container_id" >/dev/null
|
|
lock_acquired=false
|
|
for lock_attempt in {1..10}; do
|
|
[[ $(timeout --foreground --kill-after=2s 10 docker inspect -f '{{.State.Running}}' "$lock_container_id") == true ]] || break
|
|
if [[ $(timeout --foreground --kill-after=2s 10 docker logs --tail 1 "$lock_container_id" 2>/dev/null) == CANINA_STAGE_DEPLOY_LOCK_ACQUIRED ]]; then
|
|
lock_acquired=true
|
|
break
|
|
fi
|
|
[[ $lock_attempt == 10 ]] || sleep 1
|
|
done
|
|
[[ $lock_acquired == true ]] || { printf 'Stage host lock not confirmed\n' >&2; exit 1; }
|
|
release_stage() {
|
|
cd "$STAGE_DIR"
|
|
timeout --foreground --kill-after=2s 60 git fetch origin develop
|
|
[[ $(git rev-parse origin/develop) == "$DEPLOY_SHA" ]] || exit 2
|
|
# Never reset/clean the operational checkout or overwrite its generated configs.
|
|
# Every attempt has its own directory: retain incomplete and active releases.
|
|
RELEASE_ROOT=/opt/services/canina/cache-releases
|
|
mkdir -p "$RELEASE_ROOT"
|
|
TARGET_DIR=$(mktemp -d "$RELEASE_ROOT/$DEPLOY_SHA.attempt.XXXXXXXX")
|
|
# Archive failure leaves only this isolated attempt, never a reusable release.
|
|
git archive "$DEPLOY_SHA" | tar -x -C "$TARGET_DIR"
|
|
[[ $(git rev-parse "$DEPLOY_SHA^{commit}") == "$DEPLOY_SHA" ]] || exit 2
|
|
# Record the exact source identity before introducing operational configuration.
|
|
command -v python3 >/dev/null || apk add --no-cache python3
|
|
python3 - "$TARGET_DIR" "$DEPLOY_SHA" <<'PY'
|
|
import hashlib, json, sys
|
|
from pathlib import Path
|
|
root = Path(sys.argv[1]); sha = sys.argv[2]
|
|
files = {str(p.relative_to(root)): hashlib.sha256(p.read_bytes()).hexdigest()
|
|
for p in sorted(root.rglob('*')) if p.is_file() and not p.is_symlink()}
|
|
manifest = {'sha': sha, 'files': files}
|
|
(root / '.release-manifest.json').write_text(json.dumps(manifest, sort_keys=True))
|
|
loaded = json.loads((root / '.release-manifest.json').read_text())
|
|
assert loaded['sha'] == sha
|
|
assert all(hashlib.sha256((root / name).read_bytes()).hexdigest() == digest
|
|
for name, digest in loaded['files'].items())
|
|
(root / '.release-ready').write_text(sha + '\n')
|
|
PY
|
|
[[ $(cat "$TARGET_DIR/.release-ready") == "$DEPLOY_SHA" ]] || exit 2
|
|
# The ready marker certifies extraction only, not deployment/readiness.
|
|
[[ $(git rev-parse HEAD) == "$DEPLOY_SHA" ]] || printf 'Operational HEAD intentionally preserved\n'
|
|
cd "$TARGET_DIR"
|
|
python3 scripts/provision-revalidation-env.py --environment stage --repair-empty \
|
|
--source "$STAGE_DIR/.revalidation.env" --output "$TARGET_DIR/.revalidation.env"
|
|
# Build with the existing operational nginx configuration, without changing it.
|
|
cp "$STAGE_DIR/nginx.conf" "$TARGET_DIR/nginx.conf"
|
|
# Compose uses the existing operational runtime configuration plus a private,
|
|
# release-specific override. Neither the database nor Redis is recreated.
|
|
export STAGE_DIR TARGET_DIR DEPLOY_SHA
|
|
python3 - <<'PY'
|
|
import json, os
|
|
from pathlib import Path
|
|
root=Path(os.environ['TARGET_DIR']); sha=os.environ['DEPLOY_SHA']
|
|
services={}
|
|
for kind in ('backend','frontend'):
|
|
name=kind+'_stage'
|
|
services[name]={
|
|
'build': {'context': str(root/('backend' if kind=='backend' else '.')),
|
|
'labels': {'org.opencontainers.image.revision': sha},
|
|
'args': {'NPM_REGISTRY': 'https://registry.parsaaghayi.ir/repository/npm/'}},
|
|
'labels': {'ir.canina.release.sha': sha},
|
|
'env_file': [{'path': str(root/'.revalidation.env'), 'required': True}],
|
|
}
|
|
services['backend_stage']['environment']={'FRONTEND_INTERNAL_URL':'http://frontend_stage:3000'}
|
|
services['backend_stage']['command']=['node','dist/main.js']
|
|
services['frontend_stage']['environment']={'INTERNAL_API_URL':'http://backend_stage:3000/api'}
|
|
Path('release.override.json').write_text(json.dumps({'services':services}))
|
|
PY
|
|
compose=(docker compose -p stage --project-directory "$STAGE_DIR" -f "$STAGE_DIR/docker-compose.yml" -f "$TARGET_DIR/release.override.json")
|
|
timeout --foreground --kill-after=2s 60 "${compose[@]}" config --quiet
|
|
python3 scripts/verify-stage-runtime.py --base-compose "$STAGE_DIR/docker-compose.yml" \
|
|
--override "$TARGET_DIR/release.override.json" --project-directory "$STAGE_DIR" \
|
|
--nginx "$TARGET_DIR/nginx.conf"
|
|
check_stage_lock
|
|
timeout --foreground --kill-after=2s 1800 "${compose[@]}" --progress plain build backend_stage frontend_stage
|
|
check_stage_lock
|
|
timeout --foreground --kill-after=2s 120 "${compose[@]}" up -d --no-deps --force-recreate backend_stage frontend_stage
|
|
for name in canino_backend_stage canino_frontend_stage; do
|
|
# Every daemon interaction is bounded, including a hung exec/inspect.
|
|
[[ $(timeout --foreground --kill-after=2s 10 docker inspect -f '{{index .Config.Labels "ir.canina.release.sha"}}' "$name") == "$DEPLOY_SHA" ]]
|
|
image=$(timeout --foreground --kill-after=2s 10 docker inspect -f '{{.Image}}' "$name")
|
|
[[ $(timeout --foreground --kill-after=2s 10 docker image inspect -f '{{index .Config.Labels "org.opencontainers.image.revision"}}' "$image") == "$DEPLOY_SHA" ]]
|
|
path=/
|
|
[[ $name != canino_backend_stage ]] || path=/api
|
|
ready=false
|
|
for attempt in {1..12}; do
|
|
[[ $(timeout --foreground --kill-after=2s 10 docker inspect -f '{{.State.Running}} {{.RestartCount}}' "$name") == 'true 0' ]] || { printf 'Container stopped or restarted: %s\n' "$name" >&2; exit 1; }
|
|
if timeout --foreground --kill-after=2s 10 docker exec "$name" node -e '
|
|
const http = require("http");
|
|
const req = http.get({host: "127.0.0.1", port: 3000, path: process.argv[1]}, res => {
|
|
res.resume(); process.exit(res.statusCode === 200 ? 0 : 1);
|
|
});
|
|
req.setTimeout(5000, () => req.destroy(new Error("readiness timeout")));
|
|
req.on("error", () => process.exit(1));
|
|
' "$path"; then
|
|
ready=true
|
|
break
|
|
fi
|
|
[[ $attempt == 12 ]] || sleep 2
|
|
done
|
|
[[ $ready == true ]] || { printf 'Readiness timed out: %s\n' "$name" >&2; exit 1; }
|
|
if [[ $name == canino_frontend_stage ]]; then
|
|
for host in stage.canina.ir stageadmin.canina.ir; do
|
|
ready=false
|
|
for attempt in {1..12}; do
|
|
if timeout --foreground --kill-after=2s 10 docker exec "$name" node -e '
|
|
const http = require("http");
|
|
const req = http.get({host: "127.0.0.1", port: 8080, path: "/", headers: {Host: process.argv[1]}}, res => {
|
|
res.resume(); process.exit(res.statusCode === 200 ? 0 : 1);
|
|
});
|
|
req.setTimeout(5000, () => req.destroy(new Error("nginx readiness timeout")));
|
|
req.on("error", () => process.exit(1));
|
|
' "$host"; then
|
|
ready=true
|
|
break
|
|
fi
|
|
[[ $attempt == 12 ]] || sleep 2
|
|
done
|
|
[[ $ready == true ]] || { printf 'Nginx readiness timed out: %s\n' "$host" >&2; exit 1; }
|
|
done
|
|
fi
|
|
# Fence replacement/restart during probing, not just the initial state.
|
|
[[ $(timeout --foreground --kill-after=2s 10 docker inspect -f '{{.State.Running}} {{.RestartCount}}' "$name") == 'true 0' ]]
|
|
[[ $(timeout --foreground --kill-after=2s 10 docker inspect -f '{{.Image}}' "$name") == "$image" ]]
|
|
[[ $(timeout --foreground --kill-after=2s 10 docker inspect -f '{{index .Config.Labels "ir.canina.release.sha"}}' "$name") == "$DEPLOY_SHA" ]]
|
|
printf '%s image=%s release=%s ready\n' "$name" "$image" "$DEPLOY_SHA"
|
|
done
|
|
printf 'Code-only Stage release complete: %s\n' "$DEPLOY_SHA"
|
|
}
|
|
supervise_release release_stage
|