From e40398a84308ff820d2b1e8d6dd2c0c122ff37c4 Mon Sep 17 00:00:00 2001 From: devops Date: Fri, 23 Oct 2026 14:49:52 +0000 Subject: [PATCH] Isolate commit-pinned Stage releases from dirty operational checkout --- scripts/deploy.sh | 204 ++++++++++++---------------- scripts/tests/test_stage_rollout.py | 82 ++++++++++- 2 files changed, 164 insertions(+), 122 deletions(-) diff --git a/scripts/deploy.sh b/scripts/deploy.sh index 2643ab49..810f82fe 100644 --- a/scripts/deploy.sh +++ b/scripts/deploy.sh @@ -1,137 +1,103 @@ #!/bin/bash set -euo pipefail +set +x # CI/CD-only Stage rollout; Production awaits separate reviewed authorization. if [ "${GITHUB_ACTIONS:-${GITEA_ACTIONS:-}}" != true ] || [ "${1:-}" != develop ] || [ "$#" != 1 ]; then echo 'ERROR: refusing deployment outside CI/CD Stage rollout' >&2 exit 2 fi if [[ ! "${DEPLOY_SHA:-}" =~ ^[0-9a-f]{40}$ ]]; then - echo "ERROR: refusing deployment without an immutable commit SHA" >&2 + echo 'ERROR: refusing deployment without an immutable commit SHA' >&2 exit 2 fi # END ROLLOUT GUARD exec 9>/opt/services/canina/stage-backend-deploy.lock flock -n 9 || { echo 'Stage deployment already in progress' >&2; exit 1; } -export DOCKER_BUILDKIT=1 -export COMPOSE_DOCKER_CLI_BUILD=1 - -# Ensure nexus is connected to default bridge network so docker build can reach it -docker network connect bridge nexus 2>/dev/null || true - -# Nexus npm registry -NEXUS_IP=$(docker inspect -f '{{.NetworkSettings.Networks.bridge.IPAddress}}' nexus 2>/dev/null || true) -if [ -z "$NEXUS_IP" ]; then - NEXUS_IP="172.17.0.3" -fi -NPM_REGISTRY="http://${NEXUS_IP}:8081/repository/bitchpedar/" - -BRANCH="${1:-develop}" -echo "Starting Deployment for branch: ${BRANCH}" - -if [ "$BRANCH" == "develop" ]; then - COMPOSE_FILE="compose.stage.yml" - TARGET_DIR="/opt/services/canina/stage" - BACKEND_HOST="backend_stage" - APP_HOST="stage.canina.ir" - ADMIN_HOST="stageadmin.canina.ir" - API_URL="https://stageapi.canina.ir" - CONTAINER_PREFIX="canino_backend_stage" -else - COMPOSE_FILE="compose.prod.yml" - TARGET_DIR="/opt/services/canina/prod" - BACKEND_HOST="backend_prod" - APP_HOST="canina.ir" - ADMIN_HOST="admin.canina.ir" - API_URL="https://api.canina.ir" - CONTAINER_PREFIX="canino_backend_prod" -fi - -mkdir -p "$TARGET_DIR" - -if [ ! -d "$TARGET_DIR/.git" ]; then - echo "Cloning repository into $TARGET_DIR..." - git clone -b "$BRANCH" https://git.parsaaghayi.ir/parsa/canina.git "$TARGET_DIR" - git -C "$TARGET_DIR" cat-file -e "${DEPLOY_SHA}^{commit}" - git -C "$TARGET_DIR" merge-base --is-ancestor "$DEPLOY_SHA" "origin/$BRANCH" - git -C "$TARGET_DIR" reset --hard "$DEPLOY_SHA" -else - echo "Pulling latest changes in $TARGET_DIR..." - cd "$TARGET_DIR" - git fetch origin "$BRANCH" - git cat-file -e "${DEPLOY_SHA}^{commit}" - git merge-base --is-ancestor "$DEPLOY_SHA" "origin/$BRANCH" - git reset --hard "$DEPLOY_SHA" - cd - -fi - +umask 077 +export DOCKER_BUILDKIT=1 COMPOSE_DOCKER_CLI_BUILD=1 +# Stage is an operational reference, NEVER a checkout/build target. +STAGE_DIR=/opt/services/canina/stage +RELEASE_ROOT=/opt/services/canina/releases +TARGET_DIR="$RELEASE_ROOT/$DEPLOY_SHA" +mkdir -p "$RELEASE_ROOT" +[ ! -L "$RELEASE_ROOT" ] && [ "$(realpath "$RELEASE_ROOT")" = "$RELEASE_ROOT" ] +# Do not overwrite a prior release or retry with stale generated configuration. +mkdir "$TARGET_DIR" +git clone --no-checkout --single-branch --branch develop https://git.parsaaghayi.ir/parsa/canina.git "$TARGET_DIR" cd "$TARGET_DIR" - -# Capture the bot key from Docker before recreating the bot; runner mounts may not include the host env file. -if docker inspect canina-bot >/dev/null 2>&1; then - LIVE_BOT_KEY=$(docker inspect -f '{{range .Config.Env}}{{println .}}{{end}}' canina-bot | awk -F= '$1 == "SUPPORT_INTERNAL_KEY" {print substr($0, index($0, "=") + 1); exit}') - if [ -n "$LIVE_BOT_KEY" ]; then - grep -v '^SUPPORT_CHAT_KEY=' .support-chat.env > .support-chat.env.tmp 2>/dev/null || true - printf 'SUPPORT_CHAT_KEY=%s -' "$LIVE_BOT_KEY" >> .support-chat.env.tmp - mv .support-chat.env.tmp .support-chat.env - chmod 600 .support-chat.env +git cat-file -e "${DEPLOY_SHA}^{commit}" +git merge-base --is-ancestor "$DEPLOY_SHA" origin/develop +git checkout --detach "$DEPLOY_SHA" +[ "$(git rev-parse HEAD)" = "$DEPLOY_SHA" ] +# Keep operational files out of status/accidental commits in this disposable clone. +printf '\n.support-chat.env\n.revalidation.env\n.stage-base.json\n.stage-compose.json\n' >> .git/info/exclude +# Runtime secrets/config must never enter the frontend Docker build context. +printf '\n.support-chat.env\n.revalidation.env\n.stage-*.json\n' >> .dockerignore +# Only these backend/frontend runtime env files are required. Do not copy the +# dirty compose, .env, uploads, bot state, certificates, or arbitrary Stage files. +for name in .support-chat.env .revalidation.env; do + if [ -e "$STAGE_DIR/$name" ] || [ -L "$STAGE_DIR/$name" ]; then + [ -f "$STAGE_DIR/$name" ] && [ ! -L "$STAGE_DIR/$name" ] + [ -z "$(git -C "$STAGE_DIR" ls-files -- "$name")" ] + [ -z "$(git ls-files -- "$name")" ] + install -m 600 "$STAGE_DIR/$name" "$TARGET_DIR/$name" fi - unset LIVE_BOT_KEY -fi - -# Inject support-chat runtime configuration from CI/CD without committing secrets. -# The bot is the runtime owner of the internal key; synchronize only the -# matching value into the colocated backend env file, never into Git or logs. -BOT_ENV="/opt/services/ai/canina-bot/support.env" -if [ -r "$BOT_ENV" ]; then - BOT_KEY=$(awk -F= '$1 == "SUPPORT_INTERNAL_KEY" {print substr($0, index($0, "=") + 1); exit}' "$BOT_ENV") - if [ -n "$BOT_KEY" ]; then - touch .support-chat.env - grep -v '^SUPPORT_CHAT_KEY=' .support-chat.env > .support-chat.env.tmp || true - printf "SUPPORT_CHAT_KEY=%s\n" "$BOT_KEY" >> .support-chat.env.tmp - mv .support-chat.env.tmp .support-chat.env - chmod 600 .support-chat.env +done +[ -s .support-chat.env ] || { echo 'ERROR: missing Stage support runtime env' >&2; exit 1; } +# Never inspect/recreate/mutate the shared support bot or infer keys from it. +# Certs remain the explicit operational absolute bind from tracked compose. +python3 - <<'PY' +from pathlib import Path +p = Path('nginx.conf') +s = p.read_text() +for key, value in {'__BACKEND_HOST__': 'backend_stage', '__APP_HOST__': 'stage.canina.ir', '__ADMIN_HOST__': 'stageadmin.canina.ir'}.items(): + s = s.replace(key, value) +p.write_text(s) +PY +# Wrapper also covers every compose call in the sourced rollout helper. Explicit +# project/files/env-file prevent release-SHA projects and ambient .env loading. +docker() { + if [ "${1:-}" = compose ]; then + shift + command docker compose -p stage --env-file /dev/null --project-directory "$TARGET_DIR" -f "$TARGET_DIR/.stage-compose.json" "$@" + else + command docker "$@" fi - unset BOT_KEY -fi -if [ -n "${SUPPORT_CHAT_KEY:-}" ]; then - touch .support-chat.env - grep -v '^SUPPORT_CHAT_KEY=' .support-chat.env > .support-chat.env.tmp || true - printf 'SUPPORT_CHAT_KEY=%s\n' "$SUPPORT_CHAT_KEY" >> .support-chat.env.tmp - mv .support-chat.env.tmp .support-chat.env - chmod 600 .support-chat.env -fi -if [ ! -s .support-chat.env ]; then - echo "ERROR: missing .support-chat.env / SUPPORT_CHAT_KEY; refusing deployment" >&2 - exit 1 -fi - -# ✅ فایل‌های کانفیگ را از پوشه‌ی scripts مخزن به TARGET_DIR کپی می‌کنیم -echo "Copying configuration files in $TARGET_DIR..." -cp "scripts/$COMPOSE_FILE" "docker-compose.yml" -sed -i 's/- REDIS_PORT: 6379/- REDIS_PORT=6379/g' "docker-compose.yml" 2>/dev/null || true - -echo "Patching nginx.conf for environment..." -sed -i "s/__BACKEND_HOST__/${BACKEND_HOST}/g" "nginx.conf" -sed -i "s/__APP_HOST__/${APP_HOST}/g" "nginx.conf" -sed -i "s/__ADMIN_HOST__/${ADMIN_HOST}/g" "nginx.conf" - -echo "Using Nexus npm proxy: $NPM_REGISTRY" -NPM_REGISTRY_ARG="--build-arg NPM_REGISTRY=${NPM_REGISTRY}" - -echo "Building Docker images..." -docker compose --progress plain build \ - --build-arg NEXT_PUBLIC_API_URL="${API_URL}/api" \ - --build-arg VITE_API_URL="${API_URL}" \ - ${NPM_REGISTRY_ARG} - -# Never recreate the shared Production support bot during a Stage rollout. +} +# Render privately: config contains env secrets, so never emit it to CI logs. +command docker compose -p stage --env-file /dev/null --project-directory "$TARGET_DIR" -f scripts/compose.stage.yml config --format json > .stage-base.json +python3 - <<'PY' +import json +from pathlib import Path +config = json.loads(Path('.stage-base.json').read_text()) +expected = {'canina_stage_db': 'stage_canina_stage_db', 'canina_stage_uploads': 'stage_canina_stage_uploads'} +if set(config['services']) != {'db_stage', 'redis_stage', 'backend_stage', 'frontend_stage'}: + raise SystemExit('ERROR: unexpected Stage service set') +for key, name in expected.items(): + if config['volumes'][key].get('name') != name: + raise SystemExit('ERROR: Stage volume identity mismatch') + config['volumes'][key] = {'name': name, 'external': True} +if config['networks']['traefik_public'].get('name') != 'traefik_public' or not config['networks']['traefik_public'].get('external'): + raise SystemExit('ERROR: Stage network identity mismatch') +backend = config['services']['backend_stage'] +if backend['container_name'] != 'canino_backend_stage' or config['services']['frontend_stage']['container_name'] != 'canino_frontend_stage': + raise SystemExit('ERROR: Stage container identity mismatch') +mounts = backend.get('volumes', []) +if not any(m.get('type') == 'volume' and m.get('source') == 'canina_stage_uploads' and m.get('target') == '/app/uploads' for m in mounts): + raise SystemExit('ERROR: missing Stage uploads volume') +for service in config['services'].values(): + for mount in service.get('volumes', []): + if mount.get('type') == 'bind' and (mount.get('source'), mount.get('target')) != ('/opt/services/proxy/certs/canina', '/app/ssl_certs'): + raise SystemExit('ERROR: unexpected Stage bind mount') +Path('.stage-compose.json').write_text(json.dumps(config)) +Path('.stage-base.json').unlink() +PY +# Read-only checks: missing external storage must fail, never provision blank DB. +docker volume inspect stage_canina_stage_db stage_canina_stage_uploads >/dev/null +docker network inspect traefik_public >/dev/null +[ -d /opt/services/proxy/certs/canina ] +# Builds only app images; DB/Redis and support-bot remain untouched. +docker compose --progress plain build backend_stage frontend_stage source scripts/rollout-stage.sh stage_rollout - -# Auto-cleanup old docker builder cache to keep disk healthy (keeps latest 6GB) -echo "Cleaning up old docker build cache..." -docker builder prune -af --keep-storage 6GB 2>/dev/null || true -docker image prune -f 2>/dev/null || true - -echo "Deployment for $BRANCH completed successfully!" +echo 'Deployment for develop completed successfully!' diff --git a/scripts/tests/test_stage_rollout.py b/scripts/tests/test_stage_rollout.py index b2ad6fe0..fc38cbfa 100644 --- a/scripts/tests/test_stage_rollout.py +++ b/scripts/tests/test_stage_rollout.py @@ -14,8 +14,9 @@ class RolloutTests(unittest.TestCase): workflow=(ROOT/'.gitea/workflows/deploy.yml').read_text() self.assertIn('DEPLOY_SHA: ${{ github.sha }}',workflow) self.assertIn('/raw/commit/${DEPLOY_SHA}/scripts/deploy.sh',workflow) - self.assertIn('git reset --hard "$DEPLOY_SHA"',script) - self.assertNotIn('git reset --hard "origin/$BRANCH"',script) + self.assertNotIn('git reset',script) + self.assertIn('git checkout --detach "$DEPLOY_SHA"',script) + self.assertIn('/opt/services/canina/releases',script) def test_commit_guard_rejects_missing_and_malformed_sha_before_side_effects(self): guard=(ROOT/'scripts/deploy.sh').read_text().split('# END ROLLOUT GUARD')[0] @@ -35,6 +36,82 @@ class RolloutTests(unittest.TestCase): self.assertEqual(run.strip().splitlines()[0].strip(),'set -euo pipefail') self.assertIn('./deploy.sh "${BRANCH}"',run) + def test_isolated_deploy_preserves_dirty_stage_and_stable_storage(self): + import json, shutil + with tempfile.TemporaryDirectory() as directory: + base = pathlib.Path(directory) + source, stage, releases, tools = [base / n for n in ('source', 'stage', 'releases', 'tools')] + for p in (source, stage, tools): p.mkdir() + def git(*args, cwd=source): + return subprocess.check_output(['git', *args], cwd=cwd, text=True).strip() + git('init', '-b', 'develop') + git('config', 'user.email', 'test@example.invalid') + git('config', 'user.name', 'Test') + shutil.copytree(ROOT/'scripts', source/'scripts') + shutil.copyfile(ROOT/'.dockerignore', source/'.dockerignore') + (source/'nginx.conf').write_text('__BACKEND_HOST__') + git('add', '.') + git('commit', '-qm', 'fixture') + sha = git('rev-parse', 'HEAD') + git('init', '-b', 'develop', cwd=stage) + git('config', 'user.email', 'test@example.invalid', cwd=stage) + git('config', 'user.name', 'Test', cwd=stage) + (stage/'dirty').write_text('original') + git('add', '.', cwd=stage) + git('commit', '-qm', 'stage', cwd=stage) + (stage/'dirty').write_text('staged') + git('add', '.', cwd=stage) + (stage/'dirty').write_text('unstaged') + for n in ('.support-chat.env', '.revalidation.env'): + (stage/n).write_text('TEST_SECRET=never-log-this\n') + before = git('status', '--porcelain', cwd=stage), git('diff', '--binary', 'HEAD', cwd=stage), (stage/'.git/index').read_bytes() + config = {'services': { + 'db_stage': {'volumes': [{'type': 'volume', 'source': 'canina_stage_db', 'target': '/var/lib/postgresql/data'}]}, + 'redis_stage': {}, + 'backend_stage': {'container_name': 'canino_backend_stage', 'volumes': [ + {'type': 'volume', 'source': 'canina_stage_uploads', 'target': '/app/uploads'}, + {'type': 'bind', 'source': '/opt/services/proxy/certs/canina', 'target': '/app/ssl_certs'}]}, + 'frontend_stage': {'container_name': 'canino_frontend_stage'}}, + 'volumes': {n: {'name': 'stage_'+n} for n in ('canina_stage_db','canina_stage_uploads')}, + 'networks': {'traefik_public': {'name': 'traefik_public', 'external': True}}} + (base/'config.json').write_text(json.dumps(config)) + (tools/'docker').write_text('#!/bin/bash\nprintf "%s\\n" "$*" >> "$TRACE"\nif [[ "$*" == *"config --format json"* ]]; then /bin/cat "$FIXTURE"; fi\nif [[ "$*" == *"run --rm"* && "${FAIL_MIGRATION:-}" == 1 ]]; then exit 42; fi\n') + (tools/'docker').chmod(0o755) + script = (ROOT/'scripts/deploy.sh').read_text() + script = script.replace('/opt/services/canina/stage-backend-deploy.lock', str(base/'lock')).replace('/opt/services/canina/stage', str(stage)).replace('/opt/services/canina/releases', str(releases)).replace('https://git.parsaaghayi.ir/parsa/canina.git', str(source)).replace('[ -d /opt/services/proxy/certs/canina ]', 'true') + deploy = base/'deploy.sh' + deploy.write_text(script) + env = dict(os.environ, GITHUB_ACTIONS='true', DEPLOY_SHA=sha, PATH=str(tools)+':'+os.environ['PATH'], TRACE=str(base/'trace'), FIXTURE=str(base/'config.json')) + for fail in ('1', '0'): + if releases.exists(): shutil.rmtree(releases) + (base/'trace').write_text('') + result = subprocess.run(['bash', str(deploy), 'develop'], env=dict(env, FAIL_MIGRATION=fail), capture_output=True, text=True) + self.assertEqual(result.returncode, 42 if fail == '1' else 0, result.stderr) + trace = (base/'trace').read_text() + compose = [line for line in trace.splitlines() if line.startswith('compose ')] + self.assertTrue(compose) + self.assertTrue(all(line.startswith('compose -p stage --env-file /dev/null --project-directory '+str(releases/sha)) for line in compose)) + self.assertNotIn('canina-bot', trace) + self.assertNotIn('remove-orphans', trace) + self.assertNotIn('up -d --no-deps db_stage', trace) + self.assertNotIn('never-log-this', result.stdout+result.stderr) + if fail == '1': self.assertNotIn('up -d', trace) + else: + self.assertLess(trace.index('migrate deploy'), trace.index('up -d --no-deps backend_stage')) + self.assertLess(trace.index('exec -T backend_stage'), trace.index('up -d --no-deps frontend_stage')) + release = releases/sha + self.assertEqual(git('rev-parse', 'HEAD', cwd=release), sha) + self.assertEqual((release/'.support-chat.env').stat().st_mode & 0o777, 0o600) + self.assertIn('.stage-*.json', (release/'.dockerignore').read_text()) + self.assertIn('.support-chat.env', (release/'.dockerignore').read_text()) + self.assertIn('.revalidation.env', (release/'.dockerignore').read_text()) + self.assertEqual((release/'.stage-compose.json').stat().st_mode & 0o777, 0o600) + effective = json.loads((release/'.stage-compose.json').read_text()) + self.assertEqual(effective['volumes'], {n: {'name': 'stage_'+n, 'external': True} for n in ('canina_stage_db','canina_stage_uploads')}) + self.assertEqual(before, (git('status', '--porcelain', cwd=stage), git('diff', '--binary', 'HEAD', cwd=stage), (stage/'.git/index').read_bytes())) + retry = subprocess.run(['bash', str(deploy), 'develop'], env=env, capture_output=True, text=True) + self.assertNotEqual(retry.returncode, 0) + def test_migration_failure_never_starts_apps(self): with tempfile.TemporaryDirectory() as d: docker = pathlib.Path(d)/'docker' @@ -60,4 +137,3 @@ class RolloutTests(unittest.TestCase): self.assertIn('exec -T backend_stage node',trace) self.assertNotIn('up -d --no-deps frontend_stage',trace) self.assertNotIn('SUCCESS',result.stdout) -if __name__=='__main__': unittest.main()