Files
canina/.gitea/workflows/recover-production-20261026.yml
T
Canina Incident Recovery 07bc25ca9f
Incident restore production backend 20261026 / recover (push) Failing after 41s
ops: add reviewed backend-only incident rollback CI with read-only preflight
2026-10-26 05:42:15 +00:00

42 lines
1.9 KiB
YAML

name: Incident restore production backend 20261026
on:
push:
branches:
- ops/recover-production-20261026-reviewed
jobs:
recover:
runs-on: canina
steps:
- name: Restore last working backend only through pinned host boundary
shell: bash
env:
DEPLOY_SHA: ${{ github.sha }}
DEPLOY_KEY: ${{ secrets.DEPLOY_KEY }}
DEPLOY_KNOWN_HOSTS: ${{ secrets.DEPLOY_KNOWN_HOSTS }}
run: |
set -euo pipefail
set +x
[[ "$DEPLOY_SHA" =~ ^[0-9a-f]{40}$ ]]
[[ -n "$DEPLOY_KEY" && -n "$DEPLOY_KNOWN_HOSTS" ]]
umask 077
scratch=$(mktemp -d)
trap 'rm -rf "$scratch"' EXIT
printf '%s\n' "$DEPLOY_KEY" > "$scratch/key"
printf '%s\n' "$DEPLOY_KNOWN_HOSTS" > "$scratch/known_hosts"
unset DEPLOY_KEY DEPLOY_KNOWN_HOSTS
ssh-keygen -F '[87.248.133.138]:2234' -f "$scratch/known_hosts" >/dev/null
ssh -F /dev/null -T -p 2234 -i "$scratch/key" -o BatchMode=yes -o IdentitiesOnly=yes -o IdentityAgent=none -o StrictHostKeyChecking=yes -o UserKnownHostsFile="$scratch/known_hosts" -o GlobalKnownHostsFile=/dev/null devops@87.248.133.138 "sudo -n bash -s -- $DEPLOY_SHA" <<'HOST'
set -euo pipefail
set +x
umask 077
[[ $# == 1 && $1 =~ ^[0-9a-f]{40}$ && $(id -u) == 0 && ! -e /.dockerenv ]]
work=$(mktemp -d /opt/services/canina/incident-ci-XXXXXX)
trap 'rm -rf "$work"' EXIT
git clone --quiet --no-checkout /opt/services/canina/stage "$work"
git -C "$work" fetch --quiet https://git.parsaaghayi.ir/parsa/canina.git "$1"
git -C "$work" checkout --quiet --detach "$1"
[[ $(git -C "$work" rev-parse HEAD) == "$1" ]]
python3 "$work/scripts/test-recovery-20261026.py"
GITHUB_ACTIONS=true python3 "$work/scripts/recover-production-20261026.py"
HOST