fix(corporate): restore persisted orchestration APIs with code-only Stage CI
Deploy Canina / deploy (push) Successful in 4m39s

This commit is contained in:
devops committed 2026-11-14 07:32:10 +00:00
1 parent 4d41a5bbf7
commit 85ef01d337
16 files changed
+2832 -1

No files matched your search

+6
View File
@@ -1,6 +1,12 @@
#!/bin/bash
set -e
# CI-only code restoration: isolate Stage before all legacy shared side effects.
if [ "${1:-}" = develop ]; then
SCRIPT_DIR=$(cd -- "$(dirname -- "$0")" && pwd)
exec bash "$SCRIPT_DIR/release-corporate-stage.sh"
fi
# Explicit no-data-write stage rollout. Run only after testing and pushing HEAD.
if [ "${2:-}" = "stage-backend-only" ]; then
[ "${1:-}" = "develop" ] || exit 2
+142
View File
@@ -0,0 +1,142 @@
#!/usr/bin/env bash
# Narrow code-only Stage release in the existing CI runner/daemon boundary.
set -euo pipefail
set +x
umask 077
[[ ${GITHUB_ACTIONS:-${GITEA_ACTIONS:-}} == true && ${DEPLOY_SHA:-} =~ ^[0-9a-f]{40}$ ]] || exit 2
STAGE_DIR=/opt/services/canina/stage
# Alpine runner has BusyBox timeout; supervision needs GNU --foreground.
if ! timeout --help 2>&1 | grep -q -- --foreground; then
apk add --no-cache coreutils
fi
timeout --help 2>&1 | grep -q -- --foreground || exit 2
# The runner and host share a daemon, not a filesystem inode. The atomic
# container name serializes runners; the bind-mounted flock also fences host jobs.
# Never reclaim an existing name automatically (including stopped/stale helpers).
lock_container_id=
release_pid=
check_stage_lock() {
[[ $(timeout --foreground --kill-after=2s 10 docker inspect -f '{{.State.Running}}' "$lock_container_id") == true ]] || {
printf 'Stage host lock lost; aborting release\n' >&2
return 1
}
}
supervise_release() {
# A separate process group lets lock loss/signals stop the active command too.
set -m
(set +m; "$@") &
release_pid=$!
while kill -0 "$release_pid" 2>/dev/null; do
check_stage_lock || return 1
sleep 1
done
local status=0
wait "$release_pid" || status=$?
release_pid=
check_stage_lock || return 1
return "$status"
}
cleanup_stage_lock() {
local status=$?
trap - EXIT
if [[ -n $release_pid ]]; then
kill -TERM -- "-$release_pid" 2>/dev/null || :
sleep 1
kill -KILL -- "-$release_pid" 2>/dev/null || :
wait "$release_pid" 2>/dev/null || :
fi
if [[ -n $lock_container_id ]]; then
if [[ $status != 0 ]]; then
# Never emit raw helper/daemon output (image entrypoints may contain
# secrets). Classify only a bounded tail before deleting the owned ID.
local helper_log diagnostic=unavailable
helper_log=$(timeout --foreground --kill-after=2s 10 docker logs --tail 10 "$lock_container_id" 2>&1 | head -c 4096) || :
case "$helper_log" in
*'Permission denied'*) diagnostic=permission-denied ;;
*'No such file or directory'*) diagnostic=missing-host-lock ;;
*'Read-only file system'*) diagnostic=read-only-filesystem ;;
*'flock: not found'*) diagnostic=flock-unavailable ;;
*CANINA_STAGE_DEPLOY_LOCK_ACQUIRED*) diagnostic=previously-acquired ;;
?*) diagnostic=unclassified-output ;;
esac
printf 'Stage lock helper diagnostic: %s\n' "$diagnostic" >&2
fi
timeout --kill-after=2s 15 docker rm -f "$lock_container_id" >/dev/null || {
printf 'Stage lock helper cleanup failed; manual inspection required\n' >&2
[[ $status != 0 ]] || status=1
}
fi
exit "$status"
}
trap cleanup_stage_lock EXIT
trap 'exit 130' INT
trap 'exit 143' TERM
runner_image=$(timeout --foreground --kill-after=2s 10 docker inspect -f '{{.Config.Image}}' gitea_runner)
[[ -n $runner_image ]] || exit 1
lock_container_id=$(timeout --kill-after=2s 15 docker create --name canina-stage-deploy-lock \
--pull=never --network none --cap-drop=ALL --security-opt=no-new-privileges \
--mount type=bind,src=/opt/services/canina,dst=/host-canina,readonly \
--entrypoint /bin/sh "$runner_image" -c '
# Linux flock permits an exclusive lock on a read-only descriptor. Preserve
# daemon userns remapping; require the existing host inode, never create it.
exec 9</host-canina/stage-backend-deploy.lock
flock -n 9 || exit 1
printf "CANINA_STAGE_DEPLOY_LOCK_ACQUIRED\n"
while :; do sleep 60; done
') || { printf 'Stage deployment locked or helper creation failed\n' >&2; exit 1; }
[[ -n $lock_container_id ]] || exit 1
timeout --foreground --kill-after=2s 10 docker start "$lock_container_id" >/dev/null
lock_acquired=false
for lock_attempt in {1..10}; do
[[ $(timeout --foreground --kill-after=2s 10 docker inspect -f '{{.State.Running}}' "$lock_container_id") == true ]] || break
if [[ $(timeout --foreground --kill-after=2s 10 docker logs --tail 1 "$lock_container_id" 2>/dev/null) == CANINA_STAGE_DEPLOY_LOCK_ACQUIRED ]]; then
lock_acquired=true
break
fi
[[ $lock_attempt == 10 ]] || sleep 1
done
[[ $lock_acquired == true ]] || { printf 'Stage host lock not confirmed\n' >&2; exit 1; }
release_stage() {
cd "$STAGE_DIR"
timeout --foreground --kill-after=2s 60 git fetch origin develop
[[ $(git rev-parse origin/develop) == "$DEPLOY_SHA" ]] || exit 2
RELEASE_ROOT=/opt/services/canina/cache-releases
mkdir -p "$RELEASE_ROOT"
TARGET_DIR=$(mktemp -d "$RELEASE_ROOT/$DEPLOY_SHA.corporate.XXXXXXXX")
git archive "$DEPLOY_SHA" | tar -x -C "$TARGET_DIR"
command -v python3 >/dev/null || apk add --no-cache python3
export TARGET_DIR DEPLOY_SHA
python3 - <<'PY'
import json, os, hashlib
from pathlib import Path
root=Path(os.environ['TARGET_DIR']); sha=os.environ['DEPLOY_SHA']
files={str(p.relative_to(root)):hashlib.sha256(p.read_bytes()).hexdigest() for p in sorted(root.rglob('*')) if p.is_file() and not p.is_symlink()}
(root/'.release-manifest.json').write_text(json.dumps({'sha':sha,'files':files},sort_keys=True))
# Only backend code is changed. Preserve every operational runtime/env file.
service={'build':{'context':str(root/'backend'),'labels':{'org.opencontainers.image.revision':sha},'args':{'NPM_REGISTRY':'https://registry.parsaaghayi.ir/repository/npm/'}},'labels':{'ir.canina.release.sha':sha},'command':['node','dist/main.js']}
(root/'release.override.json').write_text(json.dumps({'services':{'backend_stage':service}}))
PY
compose=(docker compose -p stage --project-directory "$STAGE_DIR" -f "$STAGE_DIR/docker-compose.yml" -f "$TARGET_DIR/release.override.json")
timeout --foreground --kill-after=2s 60 "${compose[@]}" config --quiet
check_stage_lock
timeout --foreground --kill-after=2s 1800 "${compose[@]}" --progress plain build backend_stage
check_stage_lock
# No migrations, seed, DB/Redis/frontend/bot recreation or runtime config edits.
timeout --foreground --kill-after=2s 120 "${compose[@]}" up -d --no-deps --force-recreate backend_stage
ready=false
for attempt in {1..30}; do
check_stage_lock
[[ $(timeout --foreground --kill-after=2s 10 docker inspect -f '{{.State.Running}} {{.RestartCount}}' canino_backend_stage) == 'true 0' ]] || exit 1
if timeout --foreground --kill-after=2s 10 docker exec canino_backend_stage node -e '
const http=require("http");
const req=http.get("http://127.0.0.1:3000/api/admin/hermes/monthly-roadmap",res=>{res.resume();process.exit(res.statusCode===401?0:1)});
req.setTimeout(5000,()=>req.destroy());req.on("error",()=>process.exit(1));'; then ready=true; break; fi
sleep 2
done
[[ $ready == true ]] || exit 1
[[ $(timeout --foreground --kill-after=2s 10 docker inspect -f '{{index .Config.Labels "ir.canina.release.sha"}}' canino_backend_stage) == "$DEPLOY_SHA" ]]
image=$(timeout --foreground --kill-after=2s 10 docker inspect -f '{{.Image}}' canino_backend_stage)
[[ $(timeout --foreground --kill-after=2s 10 docker image inspect -f '{{index .Config.Labels "org.opencontainers.image.revision"}}' "$image") == "$DEPLOY_SHA" ]]
printf 'Corporate Stage backend release complete: %s image=%s (route guarded; authenticated acceptance separate)\n' "$DEPLOY_SHA" "$image"
}
supervise_release release_stage
+16
View File
@@ -0,0 +1,16 @@
const {test} = require('node:test');
const assert = require('node:assert/strict');
const fs = require('node:fs');
const path = require('node:path');
const root = path.join(__dirname, '..');
test('AdminModule registers persisted corporate controller and service', () => {
const module = fs.readFileSync(path.join(root,'backend/src/admin/admin.module.ts'),'utf8');
assert.match(module, /controllers:\s*\[[\s\S]*?CorporateController/);
assert.match(module, /providers:\s*\[[\s\S]*?CorporateService/);
});
test('develop CI dispatches code-only isolated Stage release before legacy side effects', () => {
const deploy = fs.readFileSync(path.join(root,'scripts/deploy.sh'),'utf8');
const dispatch = deploy.indexOf('exec bash "$SCRIPT_DIR/release-corporate-stage.sh"');
assert.ok(dispatch > 0, 'Stage code-only dispatch required');
assert.ok(dispatch < deploy.indexOf('docker network connect'), 'dispatch precedes shared bot and legacy DB changes');
});
+219
View File
@@ -0,0 +1,219 @@
"""Exercise the real release lock shell with a strictly local Docker double."""
import json
import os
from pathlib import Path
import subprocess
import tempfile
import unittest
SCRIPT = Path(__file__).resolve().parents[1] / 'release-corporate-stage.sh'
class StageDeployLockTests(unittest.TestCase):
def exercise(self, mode='ok', body='docker compose build; docker compose up', accelerated=False):
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
docker = root / 'docker'
docker.write_text('''#!/usr/bin/env python3
import json, os, sys, time, signal
from pathlib import Path
args = sys.argv[1:]
root = Path(os.environ['FAKE_ROOT'])
with (root / 'calls').open('a') as out: out.write(json.dumps(args) + '\\n')
mode = os.environ['MODE']
if (mode == 'stubborn-operation' and args[-1] == 'gitea_runner'
or mode == 'stubborn-cleanup' and args[0] == 'rm'):
signal.signal(signal.SIGTERM, signal.SIG_IGN)
time.sleep(6) # Self-bounded even before the production fix.
(root / 'stubborn-completed').touch()
if args[0] == 'inspect':
denied = False
if mode == 'remapped-host' and (root / 'create.json').exists():
create = json.loads((root / 'create.json').read_text())
# uid0 maps to host100000: directory1001:755/file0:644.
# DAC allows read/traversal, not truncation/creation. Model transport
# access only; the separate real-shell test proves actual host flock.
denied = '9>/host-canina/' in create[-1]
print('runner:local' if args[-1] == 'gitea_runner' else ('false' if denied or (root / 'lost').exists() else 'true'))
elif args[0] == 'create':
if mode == 'contended': sys.exit(1)
(root / 'held').touch()
(root / 'create.json').write_text(json.dumps(args))
print('abc123owned')
elif args[0] == 'start':
if mode == 'start-failure': sys.exit(1)
elif args[0] == 'logs':
if mode == 'diagnostic-failure':
print('sh: cannot open lock: Permission denied; SECRET_MUST_NOT_LEAK', file=sys.stderr)
elif mode != 'host-contended': print('CANINA_STAGE_DEPLOY_LOCK_ACQUIRED')
elif args[0] == 'rm':
assert args[-1] == 'abc123owned', 'must only remove owned ID'
(root / 'held').unlink()
elif args[0] == 'compose':
assert (root / 'held').exists(), 'deployment ran without lock'
if mode == 'loss-during-build' and 'build' in args:
(root / 'lost').touch()
def stopped(*_):
with (root / 'calls').open('a') as out: out.write(json.dumps(['child-stopped']) + '\\n')
sys.exit(143)
signal.signal(signal.SIGTERM, stopped)
time.sleep(5)
(root / 'child-completed').touch()
else: sys.exit(99)
''')
docker.chmod(0o755)
(root / 'sleep').write_text('#!/bin/sh\nexec /bin/sleep 0.1\n')
(root / 'sleep').chmod(0o755)
source = SCRIPT.read_text().split('release_stage() {')[0]
if accelerated:
# Keep GNU timeout real; shorten only test-local deadlines.
source = source.replace('--kill-after=2s', '--kill-after=0.1s')
source = source.replace('--foreground --kill-after=0.1s 10 ', '--foreground --kill-after=0.1s 2 ')
source = source.replace('timeout 15 ', 'timeout 2 ')
source = source.replace('--kill-after=0.1s 15 ', '--kill-after=0.1s 2 ')
# Only the isolated lock prefix runs; no operational paths are touched.
result = subprocess.run(['bash', '-c', source + '\nrelease_stage() {\n' + body + '\n}\nsupervise_release release_stage'],
env={**os.environ, 'PATH': str(root) + ':' + os.environ['PATH'],
'GITHUB_ACTIONS': 'true', 'DEPLOY_SHA': 'a' * 40,
'FAKE_ROOT': str(root), 'MODE': mode},
capture_output=True, text=True, timeout=20)
calls = [json.loads(line) for line in (root / 'calls').read_text().splitlines()] if (root / 'calls').exists() else []
if accelerated:
self.assertFalse((root / 'stubborn-completed').exists(),
'TERM-ignoring transport outlived its kill deadline')
return result, calls, (root / 'held').exists()
def test_success_holds_host_lock_through_build_and_deploy(self):
result, calls, held = self.exercise()
self.assertEqual(result.returncode, 0, result.stderr)
create = next(c for c in calls if c[0] == 'create')
self.assertIn('--pull=never', create)
self.assertIn('type=bind,src=/opt/services/canina,dst=/host-canina,readonly', create)
self.assertIn('runner:local', create)
self.assertIn('flock', ' '.join(create))
self.assertIn('/host-canina/stage-backend-deploy.lock', ' '.join(create))
self.assertEqual(calls[-1], ['rm', '-f', 'abc123owned'])
self.assertEqual(len([c for c in calls if c[0] == 'compose']), 2)
self.assertFalse(held)
def test_remapped_uid_can_lock_existing_root_owned_readable_host_file(self):
result, calls, held = self.exercise('remapped-host')
self.assertEqual(result.returncode, 0, result.stderr)
create = next(c for c in calls if c[0] == 'create')
self.assertIn('type=bind,src=/opt/services/canina,dst=/host-canina,readonly', create)
self.assertIn('--cap-drop=ALL', create)
self.assertIn('--security-opt=no-new-privileges', create)
self.assertFalse(any(c.startswith('--userns') or c == '--privileged' for c in create))
self.assertNotIn('9>/host-canina/', create[-1])
self.assertFalse(held)
def test_existing_named_helper_fails_closed_without_removal(self):
result, calls, _ = self.exercise('contended')
self.assertNotEqual(result.returncode, 0)
self.assertFalse(any(c[0] in ('rm', 'compose', 'start') for c in calls))
def test_unconfirmed_host_lock_is_bounded_and_removes_only_owned_id(self):
result, calls, held = self.exercise('host-contended')
self.assertNotEqual(result.returncode, 0)
self.assertFalse(any(c[0] == 'compose' for c in calls))
self.assertEqual(len([c for c in calls if c[:3] == ['logs', '--tail', '1']]), 10)
self.assertEqual(calls[-1], ['rm', '-f', 'abc123owned'])
self.assertFalse(held)
def test_failure_diagnostics_are_sanitized_and_captured_before_owned_cleanup(self):
result, calls, held = self.exercise('diagnostic-failure')
self.assertNotEqual(result.returncode, 0)
self.assertIn('Stage lock helper diagnostic: permission-denied', result.stderr)
self.assertNotIn('SECRET_MUST_NOT_LEAK', result.stderr)
diagnostic = next(i for i, c in enumerate(calls) if c[:3] == ['logs', '--tail', '10'])
self.assertLess(diagnostic, len(calls) - 1)
self.assertFalse(any(c[0] == 'compose' for c in calls))
self.assertFalse(held)
def test_start_failure_cleans_owned_helper(self):
result, calls, held = self.exercise('start-failure')
self.assertNotEqual(result.returncode, 0)
self.assertEqual(calls[-1], ['rm', '-f', 'abc123owned'])
self.assertFalse(held)
self.assertFalse(any(c[0] == 'compose' for c in calls))
def test_deploy_failure_preserves_status_and_releases_owned_helper(self):
result, calls, held = self.exercise(body='docker compose build; exit 42')
self.assertEqual(result.returncode, 42)
self.assertEqual(calls[-1], ['rm', '-f', 'abc123owned'])
self.assertFalse(held)
def test_termination_releases_owned_helper(self):
result, calls, held = self.exercise(body='kill -TERM $$')
self.assertEqual(result.returncode, 143)
self.assertEqual(calls[-1], ['rm', '-f', 'abc123owned'])
self.assertFalse(held)
def test_loss_after_acquisition_before_rollout_prevents_up(self):
result, calls, held = self.exercise(body='touch "$FAKE_ROOT/lost"; /bin/sleep 2; docker compose up')
self.assertNotEqual(result.returncode, 0, result.stderr)
self.assertFalse(any(c[0] == 'compose' for c in calls))
self.assertFalse(held)
def test_loss_during_build_interrupts_child_and_prevents_up(self):
result, calls, held = self.exercise('loss-during-build')
self.assertNotEqual(result.returncode, 0, result.stderr)
self.assertFalse(any(c[0] == 'compose' and 'up' in c for c in calls))
self.assertIn('Stage host lock lost', result.stderr)
self.assertIn(['child-stopped'], calls)
self.assertFalse(held)
def test_loss_interrupts_timeout_wrapped_docker_child(self):
result, calls, held = self.exercise('loss-during-build', body='timeout --foreground 20 docker compose build; docker compose up')
self.assertNotEqual(result.returncode, 0, result.stderr)
self.assertIn(['child-stopped'], calls)
self.assertFalse(any(c[0] == 'compose' and 'up' in c for c in calls))
self.assertFalse(held)
def test_term_ignoring_operational_transport_is_killed(self):
result, calls, held = self.exercise('stubborn-operation', accelerated=True)
self.assertNotEqual(result.returncode, 0)
self.assertFalse(any(c[0] in ('create', 'compose') for c in calls))
self.assertFalse(held)
def test_term_ignoring_cleanup_transport_is_killed_and_fails_closed(self):
result, calls, held = self.exercise('stubborn-cleanup', accelerated=True)
self.assertNotEqual(result.returncode, 0)
self.assertIn('manual inspection required', result.stderr)
self.assertTrue(held, 'failed cleanup must not claim removal')
self.assertEqual(calls[-1], ['rm', '-f', 'abc123owned'])
def test_helper_shell_contends_on_exact_host_inode(self):
import fcntl
import selectors
source = SCRIPT.read_text()
helper = source.split('--entrypoint /bin/sh "$runner_image" -c \'')[1].split("\n ')")[0]
with tempfile.TemporaryDirectory() as tmp:
helper = helper.replace('/host-canina', tmp).replace('sleep 60', 'sleep 0.1')
lock_path = Path(tmp) / 'stage-backend-deploy.lock'
with lock_path.open('w') as host:
fcntl.flock(host, fcntl.LOCK_EX | fcntl.LOCK_NB)
blocked = subprocess.run(['sh', '-c', helper], capture_output=True, text=True, timeout=2)
self.assertNotEqual(blocked.returncode, 0)
self.assertNotIn('ACQUIRED', blocked.stdout)
lock_path.write_text('existing-host-inode\n')
lock_path.chmod(0o444)
with subprocess.Popen(['sh', '-c', helper], stdout=subprocess.PIPE, text=True) as proc:
try:
assert proc.stdout is not None
with selectors.DefaultSelector() as selector:
selector.register(proc.stdout, selectors.EVENT_READ)
self.assertTrue(selector.select(timeout=2), 'helper did not confirm flock')
self.assertEqual(proc.stdout.readline().strip(), 'CANINA_STAGE_DEPLOY_LOCK_ACQUIRED')
self.assertEqual(lock_path.read_text(), 'existing-host-inode\n')
self.assertEqual(lock_path.stat().st_mode & 0o777, 0o444)
with lock_path.open('r') as host:
with self.assertRaises(BlockingIOError):
fcntl.flock(host, fcntl.LOCK_EX | fcntl.LOCK_NB)
finally:
proc.terminate()
proc.wait(timeout=2)
if __name__ == '__main__':
unittest.main()