fix(ci): route production code-only release through safe locked deployment without migrations
Deploy Canina / deploy (push) Failing after 10s

This commit is contained in:
Canina Automation committed 2026-10-26 20:17:27 +00:00
1 parent 9ae839138c
commit 53fccc2d3c
5 files changed
+355 -198

No files matched your search

+17 -181
View File
@@ -1,183 +1,19 @@
#!/bin/bash
set -e
if [ "${1:-develop}" = develop ] && [ "${2:-}" = "" ]; then
SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd)
exec bash "$SCRIPT_DIR/release-cache-stage.sh"
fi
# Explicit no-data-write stage rollout. Run only after testing and pushing HEAD.
if [ "${2:-}" = "stage-backend-only" ]; then
[ "${1:-}" = "develop" ] || exit 2
cd /opt/services/canina/stage
exec 9>/opt/services/canina/stage-backend-deploy.lock
flock -n 9 || { echo "Stage deployment already in progress"; exit 1; }
[ "$(git branch --show-current)" = develop ]
git fetch origin develop
[ "$(git rev-parse HEAD)" = "$(git rev-parse origin/develop)" ]
git diff --quiet -- backend scripts/compose.stage.yml scripts/deploy.sh
# Read tracked compose directly; preserve generated config and nginx.
docker compose -p stage -f scripts/compose.stage.yml --project-directory . build backend_stage
docker compose -p stage -f scripts/compose.stage.yml --project-directory . up -d --no-deps --remove-orphans backend_stage
exit 0
fi
export DOCKER_BUILDKIT=1
export COMPOSE_DOCKER_CLI_BUILD=1
# Ensure nexus is connected to default bridge network so docker build can reach it
docker network connect bridge nexus 2>/dev/null || true
# Nexus npm registry
NEXUS_IP=$(docker inspect -f '{{.NetworkSettings.Networks.bridge.IPAddress}}' nexus 2>/dev/null || true)
if [ -z "$NEXUS_IP" ]; then
NEXUS_IP="172.17.0.3"
fi
NPM_REGISTRY="http://${NEXUS_IP}:8081/repository/bitchpedar/"
BRANCH="${1:-develop}"
echo "Starting Deployment for branch: ${BRANCH}"
if [ "$BRANCH" == "develop" ]; then
COMPOSE_FILE="compose.stage.yml"
TARGET_DIR="/opt/services/canina/stage"
BACKEND_HOST="backend_stage"
APP_HOST="stage.canina.ir"
ADMIN_HOST="stageadmin.canina.ir"
API_URL="https://stageapi.canina.ir"
CONTAINER_PREFIX="canino_backend_stage"
else
COMPOSE_FILE="compose.prod.yml"
TARGET_DIR="/opt/services/canina/prod"
BACKEND_HOST="backend_prod"
APP_HOST="canina.ir"
ADMIN_HOST="admin.canina.ir"
API_URL="https://api.canina.ir"
CONTAINER_PREFIX="canino_backend_prod"
fi
mkdir -p "$TARGET_DIR"
if [ ! -d "$TARGET_DIR/.git" ]; then
echo "Cloning repository into $TARGET_DIR..."
git clone -b "$BRANCH" https://git.parsaaghayi.ir/parsa/canina.git "$TARGET_DIR"
else
echo "Pulling latest changes in $TARGET_DIR..."
cd "$TARGET_DIR"
git fetch origin "$BRANCH"
git reset --hard "origin/$BRANCH"
cd -
fi
cd "$TARGET_DIR"
# Capture the bot key from Docker before recreating the bot; runner mounts may not include the host env file.
if docker inspect canina-bot >/dev/null 2>&1; then
LIVE_BOT_KEY=$(docker inspect -f '{{range .Config.Env}}{{println .}}{{end}}' canina-bot | awk -F= '$1 == "SUPPORT_INTERNAL_KEY" {print substr($0, index($0, "=") + 1); exit}')
if [ -n "$LIVE_BOT_KEY" ]; then
grep -v '^SUPPORT_CHAT_KEY=' .support-chat.env > .support-chat.env.tmp 2>/dev/null || true
printf 'SUPPORT_CHAT_KEY=%s\n' "$LIVE_BOT_KEY" >> .support-chat.env.tmp
mv .support-chat.env.tmp .support-chat.env
chmod 600 .support-chat.env
fi
unset LIVE_BOT_KEY
fi
# Inject support-chat runtime configuration from CI/CD without committing secrets.
# The bot is the runtime owner of the internal key; synchronize only the
# matching value into the colocated backend env file, never into Git or logs.
BOT_ENV="/opt/services/ai/canina-bot/support.env"
if [ -r "$BOT_ENV" ]; then
BOT_KEY=$(awk -F= '$1 == "SUPPORT_INTERNAL_KEY" {print substr($0, index($0, "=") + 1); exit}' "$BOT_ENV")
if [ -n "$BOT_KEY" ]; then
touch .support-chat.env
grep -v '^SUPPORT_CHAT_KEY=' .support-chat.env > .support-chat.env.tmp || true
printf "SUPPORT_CHAT_KEY=%s\n" "$BOT_KEY" >> .support-chat.env.tmp
mv .support-chat.env.tmp .support-chat.env
chmod 600 .support-chat.env
fi
unset BOT_KEY
fi
if [ -n "${SUPPORT_CHAT_KEY:-}" ]; then
touch .support-chat.env
grep -v '^SUPPORT_CHAT_KEY=' .support-chat.env > .support-chat.env.tmp || true
printf 'SUPPORT_CHAT_KEY=%s\n' "$SUPPORT_CHAT_KEY" >> .support-chat.env.tmp
mv .support-chat.env.tmp .support-chat.env
chmod 600 .support-chat.env
fi
if [ ! -s .support-chat.env ]; then
echo "ERROR: missing .support-chat.env / SUPPORT_CHAT_KEY; refusing deployment" >&2
exit 1
fi
# Legacy main path must fail closed rather than deploy an empty secret file.
ENVIRONMENT=prod
[ "$BRANCH" = develop ] && ENVIRONMENT=stage
exec 9>"${TARGET_DIR}/../${ENVIRONMENT}-revalidation-deploy.lock"
flock -n 9 || exit 1
[ -f scripts/provision-revalidation-env.py ] || exit 1
python3 scripts/provision-revalidation-env.py --environment "$ENVIRONMENT" \
--source .revalidation.env --output .revalidation.env.tmp
# The operational source remains unchanged; this validated copy is disposable.
rm .revalidation.env.tmp
# ✅ فایل‌های کانفیگ را از پوشه‌ی scripts مخزن به TARGET_DIR کپی می‌کنیم
echo "Copying configuration files in $TARGET_DIR..."
cp "scripts/$COMPOSE_FILE" "docker-compose.yml"
sed -i 's/- REDIS_PORT: 6379/- REDIS_PORT=6379/g' "docker-compose.yml" 2>/dev/null || true
echo "Patching nginx.conf for environment..."
sed -i "s/__BACKEND_HOST__/${BACKEND_HOST}/g" "nginx.conf"
sed -i "s/__APP_HOST__/${APP_HOST}/g" "nginx.conf"
sed -i "s/__ADMIN_HOST__/${ADMIN_HOST}/g" "nginx.conf"
echo "Using Nexus npm proxy: $NPM_REGISTRY"
NPM_REGISTRY_ARG="--build-arg NPM_REGISTRY=${NPM_REGISTRY}"
echo "Building Docker images..."
docker compose --progress plain build \
--build-arg NEXT_PUBLIC_API_URL="${API_URL}/api" \
--build-arg VITE_API_URL="${API_URL}" \
${NPM_REGISTRY_ARG}
echo "Building support-chat bot image from committed source..."
BOT_IMAGE="canina-support-bot:${BRANCH}"
DOCKER_BUILDKIT=0 docker build --pull=false -f scripts/support-chat/Dockerfile -t "${BOT_IMAGE}" .
BOT_ENV="${TARGET_DIR}/.support-bot.runtime.env"
docker inspect -f '{{range .Config.Env}}{{println .}}{{end}}' canina-bot > "${BOT_ENV}"
chmod 600 "${BOT_ENV}"
echo "Starting containers..."
docker compose up -d --no-deps --force-recreate "backend_${ENVIRONMENT}" "frontend_${ENVIRONMENT}"
echo "Starting support-chat bot container..."
docker rm -f canina-bot >/dev/null 2>&1 || true
docker run -d --name canina-bot --restart unless-stopped --env-file "${BOT_ENV}" -e HERMES_HOME=/opt/data -e HERMES_RUNTIME_DIR=/opt/hermes/tools -e API_SERVER_HOST=0.0.0.0 -e API_SERVER_PORT=8642 --network traefik_public -v canina_bot_data:/opt/data "${BOT_IMAGE}" gateway run
docker network connect --ip 172.22.0.15 vpn_net canina-bot
rm -f "${BOT_ENV}"
echo "Waiting for backend to start..."
sleep 15
# Apply committed Prisma migrations as part of the CI/CD deployment.
# This is intentionally migrate deploy (not db push): it is repeatable,
# fail-closed, and applies only migrations already reviewed and committed.
echo "Applying committed database migrations..."
for migration in \
20261013210253_add_admin_modules \
20261015000000_add_support_conversations \
20261020000000_add_blog_freshness_authorship \
20261022000000_add_orchestration_v4_core \
20261022190000_harden_orchestration_ledger \
20261023170000_prepare_workflow_outbox \
20261024060000_prepare_not_before \
20261024090000_stage_test_principal; do
docker exec -e PRISMA_HIDE_UPDATE_MESSAGE=true "${CONTAINER_PREFIX}" npx prisma migrate resolve --applied "$migration" >/dev/null 2>&1 || true
done
docker exec -e PRISMA_HIDE_UPDATE_MESSAGE=true "${CONTAINER_PREFIX}" npx prisma migrate deploy
# Code-only deployment: seed and destructive data changes are not performed here.
# Auto-cleanup old docker builder cache to keep disk healthy (keeps latest 6GB)
echo "Cleaning up old docker build cache..."
docker builder prune -af --keep-storage 6GB 2>/dev/null || true
docker image prune -f 2>/dev/null || true
echo "Deployment for $BRANCH completed successfully!"
SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd)
case "${1:-develop}:${2:-}" in
develop:) exec bash "$SCRIPT_DIR/release-cache-stage.sh" ;;
main:) exec bash "$SCRIPT_DIR/release-cache-prod.sh" ;;
develop:stage-backend-only)
cd /opt/services/canina/stage
exec 9>/opt/services/canina/stage-backend-deploy.lock
flock -n 9 || exit 1
[ "$(git branch --show-current)" = develop ]
git fetch origin develop
[ "$(git rev-parse HEAD)" = "$(git rev-parse origin/develop)" ]
git diff --quiet -- backend scripts/compose.stage.yml scripts/deploy.sh
docker compose -p stage -f scripts/compose.stage.yml --project-directory . build backend_stage
docker compose -p stage -f scripts/compose.stage.yml --project-directory . up -d --no-deps --remove-orphans backend_stage
;;
*) printf 'Unsupported deployment selection\n' >&2; exit 2 ;;
esac
+238
View File
@@ -0,0 +1,238 @@
#!/usr/bin/env bash
# Narrow code-only Production release in the existing CI runner/daemon boundary.
set -euo pipefail
set +x
umask 077
[[ ${GITHUB_ACTIONS:-${GITEA_ACTIONS:-}} == true && ${DEPLOY_SHA:-} =~ ^[0-9a-f]{40}$ ]] || exit 2
PROD_DIR=/opt/services/canina/prod
# Alpine runner has BusyBox timeout; supervision needs GNU --foreground.
if ! timeout --help 2>&1 | grep -q -- --foreground; then
apk add --no-cache coreutils
fi
timeout --help 2>&1 | grep -q -- --foreground || exit 2
# The runner and host share a daemon, not a filesystem inode. The atomic
# container name serializes runners; the bind-mounted flock also fences host jobs.
# Never reclaim an existing name automatically (including stopped/stale helpers).
lock_container_id=
release_pid=
check_prod_lock() {
[[ $(timeout --foreground --kill-after=2s 10 docker inspect -f '{{.State.Running}}' "$lock_container_id") == true ]] || {
printf 'Production host lock lost; aborting release\n' >&2
return 1
}
}
supervise_release() {
# A separate process group lets lock loss/signals stop the active command too.
set -m
(set +m; "$@") &
release_pid=$!
while kill -0 "$release_pid" 2>/dev/null; do
check_prod_lock || return 1
sleep 1
done
local status=0
wait "$release_pid" || status=$?
release_pid=
check_prod_lock || return 1
return "$status"
}
cleanup_prod_lock() {
local status=$?
trap - EXIT
if [[ -n $release_pid ]]; then
kill -TERM -- "-$release_pid" 2>/dev/null || :
sleep 1
kill -KILL -- "-$release_pid" 2>/dev/null || :
wait "$release_pid" 2>/dev/null || :
fi
if [[ -n $lock_container_id ]]; then
if [[ $status != 0 ]]; then
# Never emit raw helper/daemon output (image entrypoints may contain
# secrets). Classify only a bounded tail before deleting the owned ID.
local helper_log diagnostic=unavailable
helper_log=$(timeout --foreground --kill-after=2s 10 docker logs --tail 10 "$lock_container_id" 2>&1 | head -c 4096) || :
case "$helper_log" in
*'Permission denied'*) diagnostic=permission-denied ;;
*'No such file or directory'*) diagnostic=missing-host-lock ;;
*'Read-only file system'*) diagnostic=read-only-filesystem ;;
*'flock: not found'*) diagnostic=flock-unavailable ;;
*CANINA_PRODUCTION_DEPLOY_LOCK_ACQUIRED*) diagnostic=previously-acquired ;;
?*) diagnostic=unclassified-output ;;
esac
printf 'Production lock helper diagnostic: %s\n' "$diagnostic" >&2
fi
timeout --kill-after=2s 15 docker rm -f "$lock_container_id" >/dev/null || {
printf 'Production lock helper cleanup failed; manual inspection required\n' >&2
[[ $status != 0 ]] || status=1
}
fi
exit "$status"
}
trap cleanup_prod_lock EXIT
trap 'exit 130' INT
trap 'exit 143' TERM
runner_image=$(timeout --foreground --kill-after=2s 10 docker inspect -f '{{.Config.Image}}' gitea_runner)
[[ -n $runner_image ]] || exit 1
lock_container_id=$(timeout --kill-after=2s 15 docker create --name canina-production-deploy-lock \
--pull=never --network none --cap-drop=ALL --security-opt=no-new-privileges \
--mount type=bind,src=/opt/services/canina,dst=/host-canina,readonly \
--entrypoint /bin/sh "$runner_image" -c '
# Linux flock permits an exclusive lock on a read-only descriptor. Preserve
# daemon userns remapping; require the existing host inode, never create it.
exec 9</host-canina/production-deploy.lock
flock -n 9 || exit 1
printf "CANINA_PRODUCTION_DEPLOY_LOCK_ACQUIRED\n"
while :; do sleep 60; done
') || { printf 'Production deployment locked or helper creation failed\n' >&2; exit 1; }
[[ -n $lock_container_id ]] || exit 1
timeout --foreground --kill-after=2s 10 docker start "$lock_container_id" >/dev/null
lock_acquired=false
for lock_attempt in {1..10}; do
[[ $(timeout --foreground --kill-after=2s 10 docker inspect -f '{{.State.Running}}' "$lock_container_id") == true ]] || break
if [[ $(timeout --foreground --kill-after=2s 10 docker logs --tail 1 "$lock_container_id" 2>/dev/null) == CANINA_PRODUCTION_DEPLOY_LOCK_ACQUIRED ]]; then
lock_acquired=true
break
fi
[[ $lock_attempt == 10 ]] || sleep 1
done
[[ $lock_acquired == true ]] || { printf 'Production host lock not confirmed\n' >&2; exit 1; }
release_prod() {
cd "$PROD_DIR"
timeout --foreground --kill-after=2s 60 git fetch origin main
[[ $(git rev-parse origin/main) == "$DEPLOY_SHA" ]] || exit 2
# Never reset/clean the operational checkout or overwrite its generated configs.
# Every attempt has its own directory: retain incomplete and active releases.
RELEASE_ROOT=/opt/services/canina/cache-releases
mkdir -p "$RELEASE_ROOT"
TARGET_DIR=$(mktemp -d "$RELEASE_ROOT/$DEPLOY_SHA.attempt.XXXXXXXX")
# Archive failure leaves only this isolated attempt, never a reusable release.
git archive "$DEPLOY_SHA" | tar -x -C "$TARGET_DIR"
[[ $(git rev-parse "$DEPLOY_SHA^{commit}") == "$DEPLOY_SHA" ]] || exit 2
# Record the exact source identity before introducing operational configuration.
command -v python3 >/dev/null || apk add --no-cache python3
python3 - "$TARGET_DIR" "$DEPLOY_SHA" <<'PY'
import hashlib, json, sys
from pathlib import Path
root = Path(sys.argv[1]); sha = sys.argv[2]
files = {str(p.relative_to(root)): hashlib.sha256(p.read_bytes()).hexdigest()
for p in sorted(root.rglob('*')) if p.is_file() and not p.is_symlink()}
manifest = {'sha': sha, 'files': files}
(root / '.release-manifest.json').write_text(json.dumps(manifest, sort_keys=True))
loaded = json.loads((root / '.release-manifest.json').read_text())
assert loaded['sha'] == sha
assert all(hashlib.sha256((root / name).read_bytes()).hexdigest() == digest
for name, digest in loaded['files'].items())
(root / '.release-ready').write_text(sha + '\n')
PY
[[ $(cat "$TARGET_DIR/.release-ready") == "$DEPLOY_SHA" ]] || exit 2
# The ready marker certifies extraction only, not deployment/readiness.
[[ $(git rev-parse HEAD) == "$DEPLOY_SHA" ]] || printf 'Operational HEAD intentionally preserved\n'
cd "$TARGET_DIR"
python3 scripts/provision-revalidation-env.py --environment prod \
--source "$PROD_DIR/.revalidation.env" --output "$TARGET_DIR/.revalidation.env"
# Build with the existing operational nginx configuration, without changing it.
cp "$PROD_DIR/nginx.conf" "$TARGET_DIR/nginx.conf"
# Compose uses the existing operational runtime configuration plus a private,
# release-specific override. Neither the database nor Redis is recreated.
export PROD_DIR TARGET_DIR DEPLOY_SHA
python3 - <<'PY'
import json, os
from pathlib import Path
root=Path(os.environ['TARGET_DIR']); sha=os.environ['DEPLOY_SHA']
services={}
for kind in ('backend','frontend'):
name=kind+'_prod'
services[name]={
'build': {'context': str(root/('backend' if kind=='backend' else '.')),
'labels': {'org.opencontainers.image.revision': sha},
'args': {'NPM_REGISTRY': 'https://registry.parsaaghayi.ir/repository/npm/'}},
'labels': {'ir.canina.release.sha': sha},
'env_file': [{'path': str(root/'.revalidation.env'), 'required': True}],
}
services['backend_prod']['environment']={'FRONTEND_INTERNAL_URL':'http://frontend_prod:3000'}
services['backend_prod']['command']=['node','dist/main.js']
services['frontend_prod']['environment']={'INTERNAL_API_URL':'http://backend_prod:3000/api', 'NEXT_PUBLIC_SITE_URL':'https://canina.ir'}
Path('release.override.json').write_text(json.dumps({'services':services}))
PY
compose=(docker compose -p prod --project-directory "$PROD_DIR" -f "$PROD_DIR/docker-compose.yml" -f "$TARGET_DIR/release.override.json")
timeout --foreground --kill-after=2s 60 "${compose[@]}" config --quiet
python3 scripts/verify-stage-runtime.py --base-compose "$PROD_DIR/docker-compose.yml" \
--override "$TARGET_DIR/release.override.json" --project-directory "$PROD_DIR" \
--project-name prod --environment prod --nginx "$TARGET_DIR/nginx.conf"
check_prod_lock
timeout --foreground --kill-after=2s 1800 "${compose[@]}" --progress plain build backend_prod frontend_prod
check_prod_lock
timeout --foreground --kill-after=2s 120 "${compose[@]}" up -d --no-deps --force-recreate backend_prod frontend_prod
for name in canino_backend_prod canino_frontend_prod; do
# Every daemon interaction is bounded, including a hung exec/inspect.
[[ $(timeout --foreground --kill-after=2s 10 docker inspect -f '{{index .Config.Labels "ir.canina.release.sha"}}' "$name") == "$DEPLOY_SHA" ]]
image=$(timeout --foreground --kill-after=2s 10 docker inspect -f '{{.Image}}' "$name")
[[ $(timeout --foreground --kill-after=2s 10 docker image inspect -f '{{index .Config.Labels "org.opencontainers.image.revision"}}' "$image") == "$DEPLOY_SHA" ]]
path=/
[[ $name != canino_backend_prod ]] || path='/api/blogs?page=1&limit=1'
ready=false
for attempt in {1..12}; do
[[ $(timeout --foreground --kill-after=2s 10 docker inspect -f '{{.State.Running}} {{.RestartCount}}' "$name") == 'true 0' ]] || { printf 'Container stopped or restarted: %s\n' "$name" >&2; exit 1; }
if timeout --foreground --kill-after=2s 10 docker exec "$name" node -e '
const http = require("http");
const req = http.get({host: "127.0.0.1", port: 3000, path: process.argv[1]}, res => {
if (res.statusCode !== 200) { res.resume(); process.exit(1); }
if (process.argv[1] === "/") { res.resume(); process.exit(0); }
// Existing public DB-backed contract; one row, no new health endpoint.
if (!(res.headers["content-type"] || "").includes("application/json")) process.exit(1);
let body = "";
res.setEncoding("utf8");
res.on("data", chunk => {
body += chunk;
if (Buffer.byteLength(body) > 2 * 1024 * 1024) process.exit(1);
});
res.on("error", () => process.exit(1));
res.on("end", () => {
try {
const {data, meta} = JSON.parse(body);
const valid = Array.isArray(data) && data.length <= 1 && meta &&
meta.page === 1 && meta.limit === 1 &&
Number.isInteger(meta.total) && meta.total >= data.length &&
Number.isInteger(meta.lastPage) && meta.lastPage >= 1;
process.exit(valid ? 0 : 1);
} catch { process.exit(1); }
});
});
req.setTimeout(5000, () => req.destroy(new Error("readiness timeout")));
req.on("error", () => process.exit(1));
' "$path"; then
ready=true
break
fi
[[ $attempt == 12 ]] || sleep 2
done
[[ $ready == true ]] || { printf 'Readiness timed out: %s\n' "$name" >&2; exit 1; }
if [[ $name == canino_frontend_prod ]]; then
for host in canina.ir admin.canina.ir; do
ready=false
for attempt in {1..12}; do
if timeout --foreground --kill-after=2s 10 docker exec "$name" node -e '
const http = require("http");
const req = http.get({host: "127.0.0.1", port: 8080, path: "/", headers: {Host: process.argv[1]}}, res => {
res.resume(); process.exit(res.statusCode === 200 ? 0 : 1);
});
req.setTimeout(5000, () => req.destroy(new Error("nginx readiness timeout")));
req.on("error", () => process.exit(1));
' "$host"; then
ready=true
break
fi
[[ $attempt == 12 ]] || sleep 2
done
[[ $ready == true ]] || { printf 'Nginx readiness timed out: %s\n' "$host" >&2; exit 1; }
done
fi
# Fence replacement/restart during probing, not just the initial state.
[[ $(timeout --foreground --kill-after=2s 10 docker inspect -f '{{.State.Running}} {{.RestartCount}}' "$name") == 'true 0' ]]
[[ $(timeout --foreground --kill-after=2s 10 docker inspect -f '{{.Image}}' "$name") == "$image" ]]
[[ $(timeout --foreground --kill-after=2s 10 docker inspect -f '{{index .Config.Labels "ir.canina.release.sha"}}' "$name") == "$DEPLOY_SHA" ]]
printf '%s image=%s release=%s ready\n' "$name" "$image" "$DEPLOY_SHA"
done
printf 'Code-only Production release complete: %s\n' "$DEPLOY_SHA"
}
supervise_release release_prod
@@ -0,0 +1,75 @@
"""Production CI code-only boundary; no daemon or real credentials required."""
import copy
import importlib.util
from pathlib import Path
import subprocess
import tempfile
import unittest
ROOT = Path(__file__).resolve().parents[2]
class ProductionCI(unittest.TestCase):
def test_main_dispatches_exact_sha_code_only_implementation(self):
source = (ROOT / 'scripts/deploy.sh').read_text()
self.assertIn('release-cache-prod.sh', source)
for unsafe in ('migrate ', 'db push', 'seed', 'canina-bot', 'reset --hard', 'prune '):
self.assertNotIn(unsafe, source)
workflow = (ROOT / '.gitea/workflows/deploy.yml').read_text()
self.assertIn('DEPLOY_SHA: ${{ github.sha }}', workflow)
self.assertIn('/raw/commit/${DEPLOY_SHA}/scripts/release-cache-prod.sh', workflow)
self.assertIn('./deploy.sh "${BRANCH}"', workflow)
with tempfile.TemporaryDirectory() as d:
root = Path(d)
(root / 'deploy.sh').write_text(source)
for branch, environment in [('main', 'prod'), ('develop', 'stage')]:
(root / ('release-cache-' + environment + '.sh')).write_text('printf "%s" "$DEPLOY_SHA"')
result = subprocess.run(['bash', str(root / 'deploy.sh'), branch],
env={'DEPLOY_SHA': 'a' * 40}, capture_output=True, text=True)
self.assertEqual(0, result.returncode)
self.assertEqual('a' * 40, result.stdout)
self.assertNotEqual(0, subprocess.run(['bash', str(root / 'deploy.sh'), 'other'],
capture_output=True).returncode)
def test_prod_release_preserves_authority_and_scopes_services(self):
path = ROOT / 'scripts/release-cache-prod.sh'
self.assertTrue(path.exists(), 'production code-only path missing')
source = path.read_text()
for required in ('git fetch origin main', 'git rev-parse origin/main', 'git archive "$DEPLOY_SHA"',
'production-deploy.lock', 'canina-production-deploy-lock',
'--environment prod', '--source "$PROD_DIR/.revalidation.env"',
'--project-name prod --environment prod', 'check_prod_lock',
'supervise_release', '--force-recreate backend_prod frontend_prod',
'--progress plain build backend_prod frontend_prod',
'http://frontend_prod:3000', 'http://backend_prod:3000/api',
'https://canina.ir', "['node','dist/main.js']"):
self.assertIn(required, source)
for unsafe in ('migrate ', 'db push', 'seed', 'canina-bot', 'reset --hard',
'--remove-orphans', 'prune ', 'backend_stage', 'frontend_stage', '--repair-empty'):
self.assertNotIn(unsafe, source)
self.assertLess(source.index('verify-stage-runtime.py'), source.index('--progress plain build'))
def test_production_runtime_authority_rejects_stage_origin_and_preserves_secrets(self):
spec = importlib.util.spec_from_file_location('authority', ROOT / 'scripts/verify-stage-runtime.py')
module = importlib.util.module_from_spec(spec); spec.loader.exec_module(module)
fixture_spec = importlib.util.spec_from_file_location('fixtures', Path(__file__).with_name('test_stage_runtime_authority.py'))
fixtures = importlib.util.module_from_spec(fixture_spec); fixture_spec.loader.exec_module(fixtures)
base, planned, live = fixtures.fixture()
for config in (base, planned):
config['services'] = {k.replace('_stage', '_prod'): v for k, v in config['services'].items()}
config['services']['frontend_prod']['build']['args'].update(
NEXT_PUBLIC_API_URL='https://api.canina.ir/api', VITE_API_URL='https://api.canina.ir')
live = {k.replace('_stage', '_prod'): v for k, v in live.items()}
planned['services']['frontend_prod']['environment']['NEXT_PUBLIC_SITE_URL'] = 'https://canina.ir'
self.assertEqual([], module.verify(base, planned, live, b'n', b'n', environment='prod'))
planned['services']['frontend_prod']['environment']['NEXT_PUBLIC_SITE_URL'] = 'https://stage.canina.ir'
self.assertIn('frontend_prod.environment.NEXT_PUBLIC_SITE_URL',
module.verify(base, planned, live, b'n', b'n', environment='prod'))
planned['services']['backend_prod']['environment']['OPERATIONAL'] = 'DO-NOT-DISCLOSE'
errors = module.verify(base, planned, live, b'n', b'n', environment='prod')
self.assertIn('backend_prod.environment.OPERATIONAL', errors)
self.assertNotIn('DO-NOT-DISCLOSE', ' '.join(errors))
if __name__ == '__main__':
unittest.main()
+23 -15
View File
@@ -31,27 +31,34 @@ def labels(values):
return {key: value for key, value in values.items() if key.startswith('traefik.')}
def verify(base, planned, live, nginx, live_nginx):
def verify(base, planned, live, nginx, live_nginx, environment="stage"):
if environment not in ("stage", "prod"):
raise ValueError("unsupported environment")
backend, frontend = "backend_" + environment, "frontend_" + environment
"""Return property/key mismatches, never configuration values."""
errors = []
# Validate resolved Compose environment, not env_file input: environment
# entries can override a provisioned secret. Match provisioner's format.
secrets = [planned['services'][name]['environment'].get('REVALIDATION_SECRET')
for name in ('backend_stage', 'frontend_stage')]
for name, secret in zip(('backend_stage', 'frontend_stage'), secrets):
for name in (backend, frontend)]
for name, secret in zip((backend, frontend), secrets):
if not isinstance(secret, str) or not re.fullmatch(r'[A-Za-z0-9_+/=-]{32,256}', secret):
errors.append(name + '.environment.REVALIDATION_SECRET')
if secrets[0] != secrets[1]:
errors.extend(name + '.environment.REVALIDATION_SECRET'
for name in ('backend_stage', 'frontend_stage'))
for name in (backend, frontend))
# Intentional rotation is permitted: do not compare these values to live.
allowed = {'REVALIDATION_SECRET', 'FRONTEND_INTERNAL_URL', 'INTERNAL_API_URL'}
if environment == 'prod':
allowed.add('NEXT_PUBLIC_SITE_URL')
if planned['services'][frontend]['environment'].get('NEXT_PUBLIC_SITE_URL') != 'https://canina.ir':
errors.append(frontend + '.environment.NEXT_PUBLIC_SITE_URL')
defaults = {'PATH', 'NODE_VERSION', 'YARN_VERSION', 'PRISMA_HIDE_UPDATE_MESSAGE'}
for name in ('backend_stage', 'frontend_stage'):
for name in (backend, frontend):
required_args = {'NPM_REGISTRY': 'https://registry.parsaaghayi.ir/repository/npm/'}
if name == 'frontend_stage':
required_args.update(NEXT_PUBLIC_API_URL='https://stageapi.canina.ir/api',
VITE_API_URL='https://stageapi.canina.ir')
if name == frontend:
api_origin = 'https://stageapi.canina.ir' if environment == 'stage' else 'https://api.canina.ir'
required_args.update(NEXT_PUBLIC_API_URL=api_origin + '/api', VITE_API_URL=api_origin)
args = planned['services'][name].get('build', {}).get('args', {})
for key, expected_value in required_args.items():
if args.get(key) != expected_value:
@@ -114,7 +121,7 @@ def verify(base, planned, live, nginx, live_nginx):
if key not in expected or key not in actual_labels or expected[key] != actual_labels[key]:
errors.append(name + '.labels.' + key)
if nginx != live_nginx:
errors.append('frontend_stage.nginx.bytes')
errors.append(frontend + '.nginx.bytes')
return sorted(set(errors))
@@ -125,6 +132,7 @@ def main(argv=None):
parser.add_argument('--project-directory', required=True)
parser.add_argument('--project-name', default='stage')
parser.add_argument('--nginx', required=True)
parser.add_argument('--environment', choices=('stage', 'prod'), default='stage')
options = parser.parse_args(argv)
# Subprocess diagnostics may contain interpolated credentials. Capture them
# and never include exceptions, argv, config, or values in diagnostics.
@@ -140,14 +148,14 @@ def main(argv=None):
for override in options.override:
planned_command.extend(['-f', override])
planned = json.loads(capture(planned_command + ['config', '--format', 'json']))
inspected = json.loads(capture(['docker', 'inspect', 'canino_backend_stage',
'canino_frontend_stage']))
backend, frontend = 'backend_' + options.environment, 'frontend_' + options.environment
inspected = json.loads(capture(['docker', 'inspect', 'canino_' + backend, 'canino_' + frontend]))
if not isinstance(inspected, list) or len(inspected) != 2:
raise ValueError()
live = dict(zip(('backend_stage', 'frontend_stage'), inspected))
live_nginx = capture(['docker', 'exec', 'canino_frontend_stage', 'cat',
live = dict(zip((backend, frontend), inspected))
live_nginx = capture(['docker', 'exec', 'canino_' + frontend, 'cat',
'/etc/nginx/conf.d/default.conf'])
errors = verify(base, planned, live, Path(options.nginx).read_bytes(), live_nginx)
errors = verify(base, planned, live, Path(options.nginx).read_bytes(), live_nginx, environment=options.environment)
except (OSError, subprocess.SubprocessError, ValueError, KeyError, TypeError, AttributeError):
print('authority.input unavailable or malformed', file=sys.stderr)
return 1
@@ -156,7 +164,7 @@ def main(argv=None):
print('authority mismatch: ' + re.sub(r'[^a-zA-Z0-9_.-]', '?', error), file=sys.stderr)
if errors:
return 1
print('Stage runtime authority verified')
print(options.environment + ' runtime authority verified')
return 0