Incident restore production backend 20261026 / recover (push) Failing after 41s
42 lines
1.9 KiB
YAML
42 lines
1.9 KiB
YAML
name: Incident restore production backend 20261026
|
|
on:
|
|
push:
|
|
branches:
|
|
- ops/recover-production-20261026-reviewed
|
|
jobs:
|
|
recover:
|
|
runs-on: canina
|
|
steps:
|
|
- name: Restore last working backend only through pinned host boundary
|
|
shell: bash
|
|
env:
|
|
DEPLOY_SHA: ${{ github.sha }}
|
|
DEPLOY_KEY: ${{ secrets.DEPLOY_KEY }}
|
|
DEPLOY_KNOWN_HOSTS: ${{ secrets.DEPLOY_KNOWN_HOSTS }}
|
|
run: |
|
|
set -euo pipefail
|
|
set +x
|
|
[[ "$DEPLOY_SHA" =~ ^[0-9a-f]{40}$ ]]
|
|
[[ -n "$DEPLOY_KEY" && -n "$DEPLOY_KNOWN_HOSTS" ]]
|
|
umask 077
|
|
scratch=$(mktemp -d)
|
|
trap 'rm -rf "$scratch"' EXIT
|
|
printf '%s\n' "$DEPLOY_KEY" > "$scratch/key"
|
|
printf '%s\n' "$DEPLOY_KNOWN_HOSTS" > "$scratch/known_hosts"
|
|
unset DEPLOY_KEY DEPLOY_KNOWN_HOSTS
|
|
ssh-keygen -F '[87.248.133.138]:2234' -f "$scratch/known_hosts" >/dev/null
|
|
ssh -F /dev/null -T -p 2234 -i "$scratch/key" -o BatchMode=yes -o IdentitiesOnly=yes -o IdentityAgent=none -o StrictHostKeyChecking=yes -o UserKnownHostsFile="$scratch/known_hosts" -o GlobalKnownHostsFile=/dev/null devops@87.248.133.138 "sudo -n bash -s -- $DEPLOY_SHA" <<'HOST'
|
|
set -euo pipefail
|
|
set +x
|
|
umask 077
|
|
[[ $# == 1 && $1 =~ ^[0-9a-f]{40}$ && $(id -u) == 0 && ! -e /.dockerenv ]]
|
|
work=$(mktemp -d /opt/services/canina/incident-ci-XXXXXX)
|
|
trap 'rm -rf "$work"' EXIT
|
|
git clone --quiet --no-checkout /opt/services/canina/stage "$work"
|
|
git -C "$work" fetch --quiet https://git.parsaaghayi.ir/parsa/canina.git "$1"
|
|
git -C "$work" checkout --quiet --detach "$1"
|
|
[[ $(git -C "$work" rev-parse HEAD) == "$1" ]]
|
|
python3 "$work/scripts/test-recovery-20261026.py"
|
|
GITHUB_ACTIONS=true python3 "$work/scripts/recover-production-20261026.py"
|
|
HOST
|