Isolate commit-pinned Stage releases from dirty operational checkout

This commit is contained in:
devops committed 2026-10-23 14:49:52 +00:00
1 parent cc9ec25388
commit e40398a843
2 files changed
+164 -122

No files matched your search

+85 -119
View File
@@ -1,137 +1,103 @@
#!/bin/bash
set -euo pipefail
set +x
# CI/CD-only Stage rollout; Production awaits separate reviewed authorization.
if [ "${GITHUB_ACTIONS:-${GITEA_ACTIONS:-}}" != true ] || [ "${1:-}" != develop ] || [ "$#" != 1 ]; then
echo 'ERROR: refusing deployment outside CI/CD Stage rollout' >&2
exit 2
fi
if [[ ! "${DEPLOY_SHA:-}" =~ ^[0-9a-f]{40}$ ]]; then
echo "ERROR: refusing deployment without an immutable commit SHA" >&2
echo 'ERROR: refusing deployment without an immutable commit SHA' >&2
exit 2
fi
# END ROLLOUT GUARD
exec 9>/opt/services/canina/stage-backend-deploy.lock
flock -n 9 || { echo 'Stage deployment already in progress' >&2; exit 1; }
export DOCKER_BUILDKIT=1
export COMPOSE_DOCKER_CLI_BUILD=1
# Ensure nexus is connected to default bridge network so docker build can reach it
docker network connect bridge nexus 2>/dev/null || true
# Nexus npm registry
NEXUS_IP=$(docker inspect -f '{{.NetworkSettings.Networks.bridge.IPAddress}}' nexus 2>/dev/null || true)
if [ -z "$NEXUS_IP" ]; then
NEXUS_IP="172.17.0.3"
fi
NPM_REGISTRY="http://${NEXUS_IP}:8081/repository/bitchpedar/"
BRANCH="${1:-develop}"
echo "Starting Deployment for branch: ${BRANCH}"
if [ "$BRANCH" == "develop" ]; then
COMPOSE_FILE="compose.stage.yml"
TARGET_DIR="/opt/services/canina/stage"
BACKEND_HOST="backend_stage"
APP_HOST="stage.canina.ir"
ADMIN_HOST="stageadmin.canina.ir"
API_URL="https://stageapi.canina.ir"
CONTAINER_PREFIX="canino_backend_stage"
else
COMPOSE_FILE="compose.prod.yml"
TARGET_DIR="/opt/services/canina/prod"
BACKEND_HOST="backend_prod"
APP_HOST="canina.ir"
ADMIN_HOST="admin.canina.ir"
API_URL="https://api.canina.ir"
CONTAINER_PREFIX="canino_backend_prod"
fi
mkdir -p "$TARGET_DIR"
if [ ! -d "$TARGET_DIR/.git" ]; then
echo "Cloning repository into $TARGET_DIR..."
git clone -b "$BRANCH" https://git.parsaaghayi.ir/parsa/canina.git "$TARGET_DIR"
git -C "$TARGET_DIR" cat-file -e "${DEPLOY_SHA}^{commit}"
git -C "$TARGET_DIR" merge-base --is-ancestor "$DEPLOY_SHA" "origin/$BRANCH"
git -C "$TARGET_DIR" reset --hard "$DEPLOY_SHA"
else
echo "Pulling latest changes in $TARGET_DIR..."
cd "$TARGET_DIR"
git fetch origin "$BRANCH"
git cat-file -e "${DEPLOY_SHA}^{commit}"
git merge-base --is-ancestor "$DEPLOY_SHA" "origin/$BRANCH"
git reset --hard "$DEPLOY_SHA"
cd -
fi
umask 077
export DOCKER_BUILDKIT=1 COMPOSE_DOCKER_CLI_BUILD=1
# Stage is an operational reference, NEVER a checkout/build target.
STAGE_DIR=/opt/services/canina/stage
RELEASE_ROOT=/opt/services/canina/releases
TARGET_DIR="$RELEASE_ROOT/$DEPLOY_SHA"
mkdir -p "$RELEASE_ROOT"
[ ! -L "$RELEASE_ROOT" ] && [ "$(realpath "$RELEASE_ROOT")" = "$RELEASE_ROOT" ]
# Do not overwrite a prior release or retry with stale generated configuration.
mkdir "$TARGET_DIR"
git clone --no-checkout --single-branch --branch develop https://git.parsaaghayi.ir/parsa/canina.git "$TARGET_DIR"
cd "$TARGET_DIR"
# Capture the bot key from Docker before recreating the bot; runner mounts may not include the host env file.
if docker inspect canina-bot >/dev/null 2>&1; then
LIVE_BOT_KEY=$(docker inspect -f '{{range .Config.Env}}{{println .}}{{end}}' canina-bot | awk -F= '$1 == "SUPPORT_INTERNAL_KEY" {print substr($0, index($0, "=") + 1); exit}')
if [ -n "$LIVE_BOT_KEY" ]; then
grep -v '^SUPPORT_CHAT_KEY=' .support-chat.env > .support-chat.env.tmp 2>/dev/null || true
printf 'SUPPORT_CHAT_KEY=%s
' "$LIVE_BOT_KEY" >> .support-chat.env.tmp
mv .support-chat.env.tmp .support-chat.env
chmod 600 .support-chat.env
git cat-file -e "${DEPLOY_SHA}^{commit}"
git merge-base --is-ancestor "$DEPLOY_SHA" origin/develop
git checkout --detach "$DEPLOY_SHA"
[ "$(git rev-parse HEAD)" = "$DEPLOY_SHA" ]
# Keep operational files out of status/accidental commits in this disposable clone.
printf '\n.support-chat.env\n.revalidation.env\n.stage-base.json\n.stage-compose.json\n' >> .git/info/exclude
# Runtime secrets/config must never enter the frontend Docker build context.
printf '\n.support-chat.env\n.revalidation.env\n.stage-*.json\n' >> .dockerignore
# Only these backend/frontend runtime env files are required. Do not copy the
# dirty compose, .env, uploads, bot state, certificates, or arbitrary Stage files.
for name in .support-chat.env .revalidation.env; do
if [ -e "$STAGE_DIR/$name" ] || [ -L "$STAGE_DIR/$name" ]; then
[ -f "$STAGE_DIR/$name" ] && [ ! -L "$STAGE_DIR/$name" ]
[ -z "$(git -C "$STAGE_DIR" ls-files -- "$name")" ]
[ -z "$(git ls-files -- "$name")" ]
install -m 600 "$STAGE_DIR/$name" "$TARGET_DIR/$name"
fi
unset LIVE_BOT_KEY
fi
# Inject support-chat runtime configuration from CI/CD without committing secrets.
# The bot is the runtime owner of the internal key; synchronize only the
# matching value into the colocated backend env file, never into Git or logs.
BOT_ENV="/opt/services/ai/canina-bot/support.env"
if [ -r "$BOT_ENV" ]; then
BOT_KEY=$(awk -F= '$1 == "SUPPORT_INTERNAL_KEY" {print substr($0, index($0, "=") + 1); exit}' "$BOT_ENV")
if [ -n "$BOT_KEY" ]; then
touch .support-chat.env
grep -v '^SUPPORT_CHAT_KEY=' .support-chat.env > .support-chat.env.tmp || true
printf "SUPPORT_CHAT_KEY=%s\n" "$BOT_KEY" >> .support-chat.env.tmp
mv .support-chat.env.tmp .support-chat.env
chmod 600 .support-chat.env
done
[ -s .support-chat.env ] || { echo 'ERROR: missing Stage support runtime env' >&2; exit 1; }
# Never inspect/recreate/mutate the shared support bot or infer keys from it.
# Certs remain the explicit operational absolute bind from tracked compose.
python3 - <<'PY'
from pathlib import Path
p = Path('nginx.conf')
s = p.read_text()
for key, value in {'__BACKEND_HOST__': 'backend_stage', '__APP_HOST__': 'stage.canina.ir', '__ADMIN_HOST__': 'stageadmin.canina.ir'}.items():
s = s.replace(key, value)
p.write_text(s)
PY
# Wrapper also covers every compose call in the sourced rollout helper. Explicit
# project/files/env-file prevent release-SHA projects and ambient .env loading.
docker() {
if [ "${1:-}" = compose ]; then
shift
command docker compose -p stage --env-file /dev/null --project-directory "$TARGET_DIR" -f "$TARGET_DIR/.stage-compose.json" "$@"
else
command docker "$@"
fi
unset BOT_KEY
fi
if [ -n "${SUPPORT_CHAT_KEY:-}" ]; then
touch .support-chat.env
grep -v '^SUPPORT_CHAT_KEY=' .support-chat.env > .support-chat.env.tmp || true
printf 'SUPPORT_CHAT_KEY=%s\n' "$SUPPORT_CHAT_KEY" >> .support-chat.env.tmp
mv .support-chat.env.tmp .support-chat.env
chmod 600 .support-chat.env
fi
if [ ! -s .support-chat.env ]; then
echo "ERROR: missing .support-chat.env / SUPPORT_CHAT_KEY; refusing deployment" >&2
exit 1
fi
# ✅ فایل‌های کانفیگ را از پوشه‌ی scripts مخزن به TARGET_DIR کپی می‌کنیم
echo "Copying configuration files in $TARGET_DIR..."
cp "scripts/$COMPOSE_FILE" "docker-compose.yml"
sed -i 's/- REDIS_PORT: 6379/- REDIS_PORT=6379/g' "docker-compose.yml" 2>/dev/null || true
echo "Patching nginx.conf for environment..."
sed -i "s/__BACKEND_HOST__/${BACKEND_HOST}/g" "nginx.conf"
sed -i "s/__APP_HOST__/${APP_HOST}/g" "nginx.conf"
sed -i "s/__ADMIN_HOST__/${ADMIN_HOST}/g" "nginx.conf"
echo "Using Nexus npm proxy: $NPM_REGISTRY"
NPM_REGISTRY_ARG="--build-arg NPM_REGISTRY=${NPM_REGISTRY}"
echo "Building Docker images..."
docker compose --progress plain build \
--build-arg NEXT_PUBLIC_API_URL="${API_URL}/api" \
--build-arg VITE_API_URL="${API_URL}" \
${NPM_REGISTRY_ARG}
# Never recreate the shared Production support bot during a Stage rollout.
}
# Render privately: config contains env secrets, so never emit it to CI logs.
command docker compose -p stage --env-file /dev/null --project-directory "$TARGET_DIR" -f scripts/compose.stage.yml config --format json > .stage-base.json
python3 - <<'PY'
import json
from pathlib import Path
config = json.loads(Path('.stage-base.json').read_text())
expected = {'canina_stage_db': 'stage_canina_stage_db', 'canina_stage_uploads': 'stage_canina_stage_uploads'}
if set(config['services']) != {'db_stage', 'redis_stage', 'backend_stage', 'frontend_stage'}:
raise SystemExit('ERROR: unexpected Stage service set')
for key, name in expected.items():
if config['volumes'][key].get('name') != name:
raise SystemExit('ERROR: Stage volume identity mismatch')
config['volumes'][key] = {'name': name, 'external': True}
if config['networks']['traefik_public'].get('name') != 'traefik_public' or not config['networks']['traefik_public'].get('external'):
raise SystemExit('ERROR: Stage network identity mismatch')
backend = config['services']['backend_stage']
if backend['container_name'] != 'canino_backend_stage' or config['services']['frontend_stage']['container_name'] != 'canino_frontend_stage':
raise SystemExit('ERROR: Stage container identity mismatch')
mounts = backend.get('volumes', [])
if not any(m.get('type') == 'volume' and m.get('source') == 'canina_stage_uploads' and m.get('target') == '/app/uploads' for m in mounts):
raise SystemExit('ERROR: missing Stage uploads volume')
for service in config['services'].values():
for mount in service.get('volumes', []):
if mount.get('type') == 'bind' and (mount.get('source'), mount.get('target')) != ('/opt/services/proxy/certs/canina', '/app/ssl_certs'):
raise SystemExit('ERROR: unexpected Stage bind mount')
Path('.stage-compose.json').write_text(json.dumps(config))
Path('.stage-base.json').unlink()
PY
# Read-only checks: missing external storage must fail, never provision blank DB.
docker volume inspect stage_canina_stage_db stage_canina_stage_uploads >/dev/null
docker network inspect traefik_public >/dev/null
[ -d /opt/services/proxy/certs/canina ]
# Builds only app images; DB/Redis and support-bot remain untouched.
docker compose --progress plain build backend_stage frontend_stage
source scripts/rollout-stage.sh
stage_rollout
# Auto-cleanup old docker builder cache to keep disk healthy (keeps latest 6GB)
echo "Cleaning up old docker build cache..."
docker builder prune -af --keep-storage 6GB 2>/dev/null || true
docker image prune -f 2>/dev/null || true
echo "Deployment for $BRANCH completed successfully!"
echo 'Deployment for develop completed successfully!'
+79 -3
View File
@@ -14,8 +14,9 @@ class RolloutTests(unittest.TestCase):
workflow=(ROOT/'.gitea/workflows/deploy.yml').read_text()
self.assertIn('DEPLOY_SHA: ${{ github.sha }}',workflow)
self.assertIn('/raw/commit/${DEPLOY_SHA}/scripts/deploy.sh',workflow)
self.assertIn('git reset --hard "$DEPLOY_SHA"',script)
self.assertNotIn('git reset --hard "origin/$BRANCH"',script)
self.assertNotIn('git reset',script)
self.assertIn('git checkout --detach "$DEPLOY_SHA"',script)
self.assertIn('/opt/services/canina/releases',script)
def test_commit_guard_rejects_missing_and_malformed_sha_before_side_effects(self):
guard=(ROOT/'scripts/deploy.sh').read_text().split('# END ROLLOUT GUARD')[0]
@@ -35,6 +36,82 @@ class RolloutTests(unittest.TestCase):
self.assertEqual(run.strip().splitlines()[0].strip(),'set -euo pipefail')
self.assertIn('./deploy.sh "${BRANCH}"',run)
def test_isolated_deploy_preserves_dirty_stage_and_stable_storage(self):
import json, shutil
with tempfile.TemporaryDirectory() as directory:
base = pathlib.Path(directory)
source, stage, releases, tools = [base / n for n in ('source', 'stage', 'releases', 'tools')]
for p in (source, stage, tools): p.mkdir()
def git(*args, cwd=source):
return subprocess.check_output(['git', *args], cwd=cwd, text=True).strip()
git('init', '-b', 'develop')
git('config', 'user.email', 'test@example.invalid')
git('config', 'user.name', 'Test')
shutil.copytree(ROOT/'scripts', source/'scripts')
shutil.copyfile(ROOT/'.dockerignore', source/'.dockerignore')
(source/'nginx.conf').write_text('__BACKEND_HOST__')
git('add', '.')
git('commit', '-qm', 'fixture')
sha = git('rev-parse', 'HEAD')
git('init', '-b', 'develop', cwd=stage)
git('config', 'user.email', 'test@example.invalid', cwd=stage)
git('config', 'user.name', 'Test', cwd=stage)
(stage/'dirty').write_text('original')
git('add', '.', cwd=stage)
git('commit', '-qm', 'stage', cwd=stage)
(stage/'dirty').write_text('staged')
git('add', '.', cwd=stage)
(stage/'dirty').write_text('unstaged')
for n in ('.support-chat.env', '.revalidation.env'):
(stage/n).write_text('TEST_SECRET=never-log-this\n')
before = git('status', '--porcelain', cwd=stage), git('diff', '--binary', 'HEAD', cwd=stage), (stage/'.git/index').read_bytes()
config = {'services': {
'db_stage': {'volumes': [{'type': 'volume', 'source': 'canina_stage_db', 'target': '/var/lib/postgresql/data'}]},
'redis_stage': {},
'backend_stage': {'container_name': 'canino_backend_stage', 'volumes': [
{'type': 'volume', 'source': 'canina_stage_uploads', 'target': '/app/uploads'},
{'type': 'bind', 'source': '/opt/services/proxy/certs/canina', 'target': '/app/ssl_certs'}]},
'frontend_stage': {'container_name': 'canino_frontend_stage'}},
'volumes': {n: {'name': 'stage_'+n} for n in ('canina_stage_db','canina_stage_uploads')},
'networks': {'traefik_public': {'name': 'traefik_public', 'external': True}}}
(base/'config.json').write_text(json.dumps(config))
(tools/'docker').write_text('#!/bin/bash\nprintf "%s\\n" "$*" >> "$TRACE"\nif [[ "$*" == *"config --format json"* ]]; then /bin/cat "$FIXTURE"; fi\nif [[ "$*" == *"run --rm"* && "${FAIL_MIGRATION:-}" == 1 ]]; then exit 42; fi\n')
(tools/'docker').chmod(0o755)
script = (ROOT/'scripts/deploy.sh').read_text()
script = script.replace('/opt/services/canina/stage-backend-deploy.lock', str(base/'lock')).replace('/opt/services/canina/stage', str(stage)).replace('/opt/services/canina/releases', str(releases)).replace('https://git.parsaaghayi.ir/parsa/canina.git', str(source)).replace('[ -d /opt/services/proxy/certs/canina ]', 'true')
deploy = base/'deploy.sh'
deploy.write_text(script)
env = dict(os.environ, GITHUB_ACTIONS='true', DEPLOY_SHA=sha, PATH=str(tools)+':'+os.environ['PATH'], TRACE=str(base/'trace'), FIXTURE=str(base/'config.json'))
for fail in ('1', '0'):
if releases.exists(): shutil.rmtree(releases)
(base/'trace').write_text('')
result = subprocess.run(['bash', str(deploy), 'develop'], env=dict(env, FAIL_MIGRATION=fail), capture_output=True, text=True)
self.assertEqual(result.returncode, 42 if fail == '1' else 0, result.stderr)
trace = (base/'trace').read_text()
compose = [line for line in trace.splitlines() if line.startswith('compose ')]
self.assertTrue(compose)
self.assertTrue(all(line.startswith('compose -p stage --env-file /dev/null --project-directory '+str(releases/sha)) for line in compose))
self.assertNotIn('canina-bot', trace)
self.assertNotIn('remove-orphans', trace)
self.assertNotIn('up -d --no-deps db_stage', trace)
self.assertNotIn('never-log-this', result.stdout+result.stderr)
if fail == '1': self.assertNotIn('up -d', trace)
else:
self.assertLess(trace.index('migrate deploy'), trace.index('up -d --no-deps backend_stage'))
self.assertLess(trace.index('exec -T backend_stage'), trace.index('up -d --no-deps frontend_stage'))
release = releases/sha
self.assertEqual(git('rev-parse', 'HEAD', cwd=release), sha)
self.assertEqual((release/'.support-chat.env').stat().st_mode & 0o777, 0o600)
self.assertIn('.stage-*.json', (release/'.dockerignore').read_text())
self.assertIn('.support-chat.env', (release/'.dockerignore').read_text())
self.assertIn('.revalidation.env', (release/'.dockerignore').read_text())
self.assertEqual((release/'.stage-compose.json').stat().st_mode & 0o777, 0o600)
effective = json.loads((release/'.stage-compose.json').read_text())
self.assertEqual(effective['volumes'], {n: {'name': 'stage_'+n, 'external': True} for n in ('canina_stage_db','canina_stage_uploads')})
self.assertEqual(before, (git('status', '--porcelain', cwd=stage), git('diff', '--binary', 'HEAD', cwd=stage), (stage/'.git/index').read_bytes()))
retry = subprocess.run(['bash', str(deploy), 'develop'], env=env, capture_output=True, text=True)
self.assertNotEqual(retry.returncode, 0)
def test_migration_failure_never_starts_apps(self):
with tempfile.TemporaryDirectory() as d:
docker = pathlib.Path(d)/'docker'
@@ -60,4 +137,3 @@ class RolloutTests(unittest.TestCase):
self.assertIn('exec -T backend_stage node',trace)
self.assertNotIn('up -d --no-deps frontend_stage',trace)
self.assertNotIn('SUCCESS',result.stdout)
if __name__=='__main__': unittest.main()