fix(auth): persist generation-fenced Stage principal cleanup retries

This commit is contained in:
devops committed 2026-10-24 12:59:36 +00:00
1 parent 94fbfe984f
commit 104abf904f
13 files changed
+323 -44

No files matched your search

+15 -10
View File
@@ -242,7 +242,7 @@ export class AdminService {
return this.persistUser(dto);
}
private async persistUser(dto: CreateUserDto, stage?: { session: string; expiresAt: Date }) {
private async persistUser(dto: CreateUserDto, stage?: { session: string; expiresAt: Date; generation?: string }) {
const mobile = normalizeMobile(dto.mobile);
const existingMobile = await this.prisma.user.findUnique({
where: { mobile },
@@ -275,7 +275,7 @@ export class AdminService {
return this.prisma.user.create({
data: {
...(stage ? { id: STAGE_TEST_ID, stageTestGeneration:randomUUID(), stageTestSession: stage.session, authExpiresAt: stage.expiresAt } : {}),
...(stage ? { id: STAGE_TEST_ID, stageTestGeneration:stage.generation || randomUUID(), stageTestSession: stage.session, authExpiresAt: stage.expiresAt } : {}),
firstName: dto.firstName.trim(),
lastName: dto.lastName.trim(),
mobile,
@@ -295,21 +295,22 @@ export class AdminService {
const grant = authorizeStageSession(env, sha);
const existing = await this.prisma.user.findFirst({where:{OR:[{id:STAGE_TEST_ID},{email:STAGE_TEST_EMAIL},{mobile:STAGE_TEST_MOBILE}]}});
if (existing) {
if (existing.id !== STAGE_TEST_ID || existing.email !== STAGE_TEST_EMAIL || existing.mobile !== STAGE_TEST_MOBILE || existing.stageTestSession !== grant.session || existing.role !== 'SUPER_ADMIN' || !existing.authExpiresAt || existing.authExpiresAt.getTime() <= Date.now() || !await bcrypt.compare(password, existing.password || '')) throw new Error('Refusing existing or unowned principal');
if (existing.id !== STAGE_TEST_ID || existing.email !== STAGE_TEST_EMAIL || existing.mobile !== STAGE_TEST_MOBILE || existing.stageTestSession !== grant.session || (env.STAGE_TEST_GENERATION && existing.stageTestGeneration !== env.STAGE_TEST_GENERATION) || existing.role !== 'SUPER_ADMIN' || !existing.authExpiresAt || existing.authExpiresAt.getTime() <= Date.now() || !await bcrypt.compare(password, existing.password || '')) throw new Error('Refusing existing or unowned principal');
return existing; // Never renew TTL on retry.
}
if (password.length < 32) throw new Error('Server generated secret required');
return this.persistUser({firstName:'Stage',lastName:'CI independent reviewer',email:STAGE_TEST_EMAIL,mobile:STAGE_TEST_MOBILE,role:'SUPER_ADMIN',password}, {session:grant.session,expiresAt:new Date(Date.now()+grant.ttl*1000)});
return this.persistUser({firstName:'Stage',lastName:'CI independent reviewer',email:STAGE_TEST_EMAIL,mobile:STAGE_TEST_MOBILE,role:'SUPER_ADMIN',password}, {session:grant.session,expiresAt:new Date(Date.now()+grant.ttl*1000),generation:env.STAGE_TEST_GENERATION});
}
async cleanupStageTestPrincipal(env: NodeJS.ProcessEnv, sha: string) {
async cleanupStageTestPrincipal(env: NodeJS.ProcessEnv, sha: string, onRevoked: () => void = () => undefined) {
const grant = authorizeStageSession(env, sha);
const existing = await this.prisma.user.findUnique({where:{id:STAGE_TEST_ID}});
if (!existing) return;
if (existing.email !== STAGE_TEST_EMAIL || existing.mobile !== STAGE_TEST_MOBILE || existing.stageTestSession !== grant.session || !existing.authExpiresAt) throw new Error('Refusing unowned principal cleanup');
if (!existing) { onRevoked(); return; }
if (existing.email !== STAGE_TEST_EMAIL || existing.mobile !== STAGE_TEST_MOBILE || existing.stageTestSession !== grant.session || !existing.authExpiresAt || (env.STAGE_TEST_GENERATION && existing.stageTestGeneration !== env.STAGE_TEST_GENERATION)) throw new Error('Refusing unowned principal cleanup');
// Revoke before relational cleanup; failed cleanup cannot leave valid tokens.
await this.prisma.user.update({where:{id:STAGE_TEST_ID},data:{authExpiresAt:new Date(0),password:''}});
return this.deleteUserRecords(STAGE_TEST_ID);
await this.prisma.user.update({where:{id:STAGE_TEST_ID,stageTestSession:grant.session,stageTestGeneration:existing.stageTestGeneration},data:{authExpiresAt:new Date(0),password:''}});
onRevoked();
return this.deleteUserRecords(STAGE_TEST_ID, {session:grant.session,generation:existing.stageTestGeneration});
}
async updateUser(id: string, dto: UpdateUserDto, actor?: UserAdminActor) {
@@ -399,8 +400,12 @@ export class AdminService {
return this.deleteUserRecords(id);
}
private async deleteUserRecords(id: string) {
private async deleteUserRecords(id: string, stage?: {session:string; generation:string|null}) {
return this.prisma.$transaction(async (tx) => {
if (stage) {
// Acquire the row lock with an exact fence before deleting any related rows.
await tx.user.update({where:{id,stageTestSession:stage.session,stageTestGeneration:stage.generation},data:{authExpiresAt:new Date(0),password:''}});
}
await tx.userAddress.deleteMany({ where: { userId: id } });
await tx.walletTransaction.deleteMany({ where: { userId: id } });
await tx.pet.deleteMany({ where: { userId: id } });
+18
View File
@@ -0,0 +1,18 @@
import { AdminService } from '../admin/admin.service';
import { STAGE_TEST_ID, STAGE_TEST_EMAIL, STAGE_TEST_MOBILE } from './stage-test-principal';
const sha='a'.repeat(40), generation='8311eb41-23b4-4aab-ade2-e97b90debc11';
const env={APP_ENV:'stage',CI:'true',STAGE_TEST_PRINCIPAL_ENABLED:'true',STAGE_TEST_APPROVAL:'owner-authorized-stage-editorial',STAGE_TEST_SHA:sha,STAGE_TEST_SESSION:'cleanup-session',STAGE_TEST_TTL_SECONDS:'600',STAGE_TEST_GENERATION:generation};
describe('durable cleanup ownership fence',()=>{
it('does not revoke a replacement generation',async()=>{
const update=jest.fn();
const db={user:{findUnique:async()=>({id:STAGE_TEST_ID,email:STAGE_TEST_EMAIL,mobile:STAGE_TEST_MOBILE,stageTestSession:env.STAGE_TEST_SESSION,stageTestGeneration:'replacement',authExpiresAt:new Date()}),update}};
await expect(new AdminService(db as any,undefined as any,undefined as any).cleanupStageTestPrincipal(env,sha)).rejects.toThrow('Refusing');
expect(update).not.toHaveBeenCalled();
});
it('confirms revocation before relational deletion failure',async()=>{
const events:string[]=[];
const db={user:{findUnique:async()=>({id:STAGE_TEST_ID,email:STAGE_TEST_EMAIL,mobile:STAGE_TEST_MOBILE,stageTestSession:env.STAGE_TEST_SESSION,stageTestGeneration:generation,authExpiresAt:new Date()}),update:async()=>events.push('revoke')},$transaction:async()=>{events.push('delete');throw Error('FK failure');}};
await expect((new AdminService(db as any,undefined as any,undefined as any).cleanupStageTestPrincipal as any)(env,sha,()=>events.push('handle-removed'))).rejects.toThrow('FK failure');
expect(events).toEqual(['revoke','handle-removed','delete']);
});
});
@@ -0,0 +1,44 @@
import { PrismaClient } from '@prisma/client';
import { JwtService } from '@nestjs/jwt';
import { AuthService } from './auth.service';
import { RedisService } from '../redis/redis.service';
import { STAGE_TEST_ID } from './stage-test-principal';
import { createServer, Server } from 'http';
import { execFile } from 'child_process';
import { promisify } from 'util';
import { readFileSync, writeFileSync, existsSync } from 'fs';
const enabled=process.env.STAGE_PRINCIPAL_DISPOSABLE_TEST==='true';
(enabled?describe:describe.skip)('actual internal CLI disposable lifecycle',()=>{
const sha='a'.repeat(40), generation='b411eb41-23b4-4aab-ade2-e97b90debc11', session='cli-integration-session';
const handle=`/app/private-stage-test/${session}.json`;
const db=new PrismaClient({log:[]}), redis=new RedisService(); let server:Server;
const env={...process.env,APP_ENV:'stage',CI:'true',STAGE_TEST_PRINCIPAL_ENABLED:'true',STAGE_TEST_APPROVAL:'owner-authorized-stage-editorial',STAGE_TEST_SHA:sha,STAGE_TEST_SESSION:session,STAGE_TEST_TTL_SECONDS:'600',STAGE_TEST_GENERATION:generation};
const cli=(action:string)=>promisify(execFile)(process.execPath,['-r','ts-node/register','src/auth/stage-test-principal.cli.ts',action],{env,timeout:90000});
beforeAll(async()=>{
if(!process.env.DATABASE_URL?.endsWith('/principal_disposable')) throw Error('Disposable database only');
writeFileSync('/app/STAGE_TEST_ARTIFACT_SHA',sha); redis.onModuleInit();
const auth=new AuthService(db as any,new JwtService({secret:process.env.JWT_SECRET,signOptions:{expiresIn:'2h'}}),redis,undefined as any);
server=createServer(async(req,res)=>{
try {let body='';for await(const chunk of req) body+=chunk;
if(req.url!=='/api/auth/admin-login') {res.writeHead(404).end();return;}
const login=await auth.adminLogin(JSON.parse(body));res.setHeader('content-type','application/json');res.end(JSON.stringify(login));
} catch {res.writeHead(401).end();}
});
await new Promise<void>(r=>server.listen(3000,'127.0.0.1',r));
},120000);
afterAll(async()=>{await new Promise<void>(r=>server.close(()=>r()));redis.onModuleDestroy();await db.$disconnect();});
it('logs in using persisted credentials and removes handle on confirmed revocation despite failed delete',async()=>{
await cli('open'); expect(existsSync(handle)).toBe(true);
const credential=JSON.parse(readFileSync(handle,'utf8'));expect(credential.accessToken).toBeTruthy();
expect((await db.user.findUnique({where:{id:STAGE_TEST_ID}}))?.stageTestGeneration).toBe(generation);
await db.$executeRawUnsafe(`CREATE FUNCTION deny_principal_delete() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RAISE EXCEPTION 'test FK deletion failure'; END $$`);
await db.$executeRawUnsafe(`CREATE TRIGGER deny_principal_delete BEFORE DELETE ON users FOR EACH ROW EXECUTE FUNCTION deny_principal_delete()`);
try {
await expect(cli('close')).rejects.toThrow();
expect(existsSync(handle)).toBe(false);
const revoked=await db.user.findUnique({where:{id:STAGE_TEST_ID}});
expect(revoked?.authExpiresAt?.getTime()).toBe(0);expect(revoked?.password).toBe('');
} finally {await db.$executeRawUnsafe(`DROP TRIGGER deny_principal_delete ON users`);await db.$executeRawUnsafe(`DROP FUNCTION deny_principal_delete()`);}
await cli('close');expect(await db.user.findUnique({where:{id:STAGE_TEST_ID}})).toBeNull();
},240000);
});
+5 -4
View File
@@ -10,12 +10,15 @@ async function main() {
const sha = readFileSync('/app/STAGE_TEST_ARTIFACT_SHA', 'utf8').trim();
const grant = authorizeStageSession(process.env, sha);
const action = process.argv[2];
if (action === 'cleanup-capability') { process.stdout.write('stage-generation-cleanup-v1'); return; }
if (!['open','close'].includes(action)) throw new Error('Unknown command');
const root = '/app/private-stage-test';
mkdirSync(root,{recursive:true,mode:0o700});
const st = statSync(root);
if (st.uid !== process.getuid!() || (st.mode & 0o777) !== 0o700) throw new Error('Unsafe private directory');
if (!/^[a-f0-9-]{36}$/.test(process.env.STAGE_TEST_GENERATION || '')) throw new Error('Generation required');
const handle = `${root}/${grant.session}.json`;
const removeHandle = () => { try {unlinkSync(handle);} catch(e:any) {if(e.code !== 'ENOENT') throw e;} };
const audit = (event:string) => {
const fd = openSync(`${root}/audit.jsonl`, constants.O_WRONLY|constants.O_CREAT|constants.O_APPEND|constants.O_NOFOLLOW,0o600);
try { writeFileSync(fd,JSON.stringify({event,sha,session:grant.session,at:new Date().toISOString()})+'\n'); } finally {closeSync(fd);}
@@ -25,8 +28,7 @@ async function main() {
try {
audit(`${action}-requested`);
if(action === 'close') {
await service.cleanupStageTestPrincipal(process.env,sha);
try {unlinkSync(handle);} catch(e:any) {if(e.code !== 'ENOENT') throw e;}
await service.cleanupStageTestPrincipal(process.env,sha,removeHandle);
audit('revoked-and-deleted');
return;
}
@@ -59,8 +61,7 @@ async function main() {
} catch {
audit('failed');
// Leave only a revoked account if relational cleanup fails.
await service.cleanupStageTestPrincipal(process.env,sha).catch(()=>undefined);
try {unlinkSync(handle);} catch { /* no credential content in error */ }
await service.cleanupStageTestPrincipal(process.env,sha,removeHandle).catch(()=>undefined);
throw new Error('Stage test principal operation failed; inspect private audit');
} finally {await db.$disconnect();}
}
@@ -48,5 +48,11 @@ const enabled=process.env.STAGE_PRINCIPAL_DISPOSABLE_TEST === 'true';
await admin.cleanupStageTestPrincipal(grant,sha);
await expect(strategy.validate(req,payload)).rejects.toThrow();
expect(await db.user.findUnique({where:{id:u.id}})).toBeNull();
const replacement=await admin.provisionStageTestPrincipal(grant,sha,password);
expect(replacement.stageTestGeneration).not.toBe(u.stageTestGeneration);
await expect(strategy.validate(req,payload)).rejects.toThrow();
await expect(auth.refreshToken(login.data.refreshToken)).rejects.toThrow();
const fresh=await auth.adminLogin({email:STAGE_TEST_EMAIL,password});
expect((await strategy.validate(req,jwt.verify(fresh.data.accessToken))).id).toBe(u.id);
});
});
@@ -0,0 +1,52 @@
# Durable Stage cleanup verification
This change replaces the SHA-pinned transient timer with a root-owned intent journal
and a fixed host systemd retry worker. No host service is installed or enabled by
the CI credential command. Deployment must first install the reviewed worker as
`/opt/services/canina/stage-principal-cleanup.py` (root:root 0755), install the two
supplied systemd units under `/etc/systemd/system`, daemon-reload, and enable/start
`canina-stage-principal-cleanup.timer`. This is a promotion prerequisite, NOT an
instruction to run these actions during review. The open wrapper refuses to operate
without an enabled and active timer. Never overwrite or delete the 0700 root-owned
`/opt/services/canina/stage-principal-cleanup` journal during deployments.
A journal entry is fsynced before provisioning and contains the exact session,
random generation, creation SHA, bounded TTL and immutable cleanup deadline.
Repeated opens fail rather than replace an intent or renew TTL. Close and the
30-second boot/periodic retry worker retain intents on lock contention, unavailable
containers/DB, and relational deletion failures. Cleanup resolves the current Stage
container ID once, verifies its cleanup capability, and authenticates against its
current artifact SHA while retaining original session/generation ownership. Older
artifacts without generation cleanup support fail closed; never roll back to one
with a pending principal. Persisted expiry remains independently enforced at login,
refresh and every scoped JWT request.
Revocation commits before private-handle removal; relational deletion is subsequent.
A transaction acquires a generation/session-fenced row lock before touching related
records. Failed deletion retains a revoked row and a pending retry intent, but no
credential handle. Wrong ownership/generation never invokes the handle callback.
The CLI requires the generation supplied by the private journal; no broader admin
or production authority was introduced.
Verification performed in a new isolated worktree and disposable Docker network:
- Cleanup fence tests: RED (replacement generation was revoked; callback missing),
then GREEN (2 tests).
- Python simulated worker tests: restart/redeploy, failed-delete retry, lock
contention, immutable container ID/current artifact and old-CLI refusal (4 tests).
- Persisted account lifecycle plus delete/recreate access AND refresh regression.
- Actual CLI via ts-node with real disposable Postgres/Redis and HTTP adminLogin
handler, injected PostgreSQL delete trigger failure: confirmed revocation,
private handle removal, then successful repeated cleanup after trigger removal.
- TypeScript `tsc --noEmit`: passed.
- Full migration chain attempt on fresh disposable database: baseline failure in
20261020000000_add_blog_freshness_authorship (`blogs` missing). Not concealed.
Independent migration test constructed the pre-Stage schema excluding all three
new fields, then applied the EXACT committed 20261024090000 migration with psql
ON_ERROR_STOP before exercising persisted lifecycle. Not db-push-only testing.
No live account, Stage/Production DB mutation, deployment, push, host service change,
or retry of the previous pending-approval operation was performed.
Invocation after normal reviewed deployment and host prerequisite provisioning:
`sudo -n env CI=true STAGE_TEST_APPROVAL=owner-authorized-stage-editorial bash /opt/services/canina/releases/<promoted-sha>/scripts/stage-test-principal-host.sh <promoted-sha> <unique-session> 600 open`
Use the same command with `close` immediately after the authorized test.
+1 -9
View File
@@ -31,15 +31,7 @@ role checks remain unchanged. The canonical default administrator is untouched.
tests. This revokes persisted authentication BEFORE relational deletion and
removes the private credential file. Close is idempotent when absent.
A host systemd timer is armed before creation for independent TTL cleanup even
if CI disappears. Persisted expiry is checked at login, refresh, and every JWT
request, so killing the timer cannot extend access. Failed cleanup leaves revoked
credentials. Cleanup timer is pinned to the creation SHA: finish/close before
redeploy; if redeployed, use the approved CI recovery session with its recorded
ownership rather than changing an unrelated account. A collision with the exact
ID, email or mobile fails closed; an owned retry with the same secret cannot
extend expiry. The test principal is limited in JWT strategy to task reads,
proposal creation, Gate1 decisions and editorial failure. All other paths deny.
A durable root-owned host cleanup journal and boot/periodic systemd retry worker must be provisioned before opening a session. Cleanup follows the current Stage artifact but retains the original exact session and generation fence; persisted expiry cannot be extended. See [cleanup verification and promotion prerequisites](stage-principal-cleanup-verification.md). No live host services are changed by review tests.
## Verification
@@ -0,0 +1,11 @@
[Unit]
Description=Retry exact owned temporary Stage principal cleanup
After=docker.service
Wants=docker.service
[Service]
Type=oneshot
User=root
UMask=0077
ExecStart=/usr/bin/python3 /opt/services/canina/stage-principal-cleanup.py sweep
TimeoutStartSec=120
@@ -0,0 +1,10 @@
[Unit]
Description=Recover temporary Stage principal cleanup after reboot and failures
[Timer]
OnBootSec=30s
OnUnitInactiveSec=30s
Unit=canina-stage-principal-cleanup.service
[Install]
WantedBy=timers.target
+100
View File
@@ -0,0 +1,100 @@
#!/usr/bin/env python3
"""Root-only Stage principal intent journal. No generic command/container inputs."""
import fcntl
import json
import os
from pathlib import Path
import re
import subprocess
import sys
import time
import uuid
ROOT = Path('/opt/services/canina/stage-principal-cleanup')
APPROVAL = 'owner-authorized-stage-editorial'
def validate(i):
if set(i) != {'sha','session','ttl','generation','due'} or not re.fullmatch('[a-f0-9]{40}', i['sha']) or not re.fullmatch('[a-z0-9-]{8,64}', i['session']) or not re.fullmatch('[a-f0-9-]{36}', i['generation']) or not isinstance(i['ttl'],int) or not 60 <= i['ttl'] <= 3600 or not isinstance(i['due'],int):
raise ValueError('Invalid cleanup intent')
class Worker:
def __init__(self, root, run):
self.root, self.run = root, run
def save(self, intent):
validate(intent)
p = self.root / (intent['session']+'.json')
# Exclusive creation forbids reusing a session or extending its deadline.
fd = os.open(p, os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_NOFOLLOW, 0o600)
with os.fdopen(fd,'w') as f:
json.dump(intent,f); f.flush(); os.fsync(f.fileno())
self.sync()
def sync(self):
fd=os.open(self.root,os.O_RDONLY|os.O_DIRECTORY)
try: os.fsync(fd)
finally: os.close(fd)
def sweep(self, now):
with open(self.root/'lock','a') as lock:
try: fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
except BlockingIOError: return
for p in sorted(self.root.glob('*.json')):
try:
fd=os.open(p,os.O_RDONLY|os.O_NOFOLLOW)
with os.fdopen(fd) as f: i=json.load(f)
validate(i)
if p.name != i['session']+'.json': raise ValueError('Intent name mismatch')
if i['due'] <= now and self.run(i,'close'):
p.unlink(); self.sync()
except Exception:
# Keep intent through DB/container/redeploy failures; next tick retries.
print('Stage cleanup pending',file=sys.stderr)
def run(i, action):
# Opens remain SHA pinned; cleanup only uses the CURRENT artifact's narrow CLI.
# Original ownership is the session plus unpredictable account generation.
container=subprocess.check_output(['docker','inspect','--format','{{.Id}}','canino_backend_stage'],text=True,timeout=30).strip()
if not re.fullmatch('[a-f0-9]{64}',container): return False
sha=subprocess.check_output(['docker','exec',container,'cat','/app/STAGE_TEST_ARTIFACT_SHA'],text=True,timeout=30).strip()
if not re.fullmatch('[a-f0-9]{40}',sha) or (action=='open' and sha!=i['sha']):
return False
env={'APP_ENV':'stage','CI':'true','STAGE_TEST_PRINCIPAL_ENABLED':'true','STAGE_TEST_APPROVAL':APPROVAL,'STAGE_TEST_SHA':sha,'STAGE_TEST_SESSION':i['session'],'STAGE_TEST_TTL_SECONDS':str(i['ttl']),'STAGE_TEST_GENERATION':i['generation']}
cmd=['docker','exec']
for k,v in env.items(): cmd+=['-e',k+'='+v]
cli=cmd+[container,'node','dist/auth/stage-test-principal.cli.js']
# Never send a generation-fenced cleanup to an older CLI that ignores fences.
capability=subprocess.check_output(cli+['cleanup-capability'],text=True,timeout=30)
if capability!='stage-generation-cleanup-v1': return False
return subprocess.run(cli+[action],timeout=90).returncode==0
def main():
if os.getuid()!=0 or Path('/.dockerenv').exists(): raise ValueError('Host root only')
os.umask(0o077)
ROOT.mkdir(mode=0o700,exist_ok=True)
st=ROOT.lstat()
if ROOT.is_symlink() or st.st_uid!=0 or st.st_mode & 0o777 != 0o700: raise ValueError('Unsafe journal')
worker=Worker(ROOT,run)
if sys.argv[1:]==['sweep']:
worker.sweep(int(time.time())); return
if len(sys.argv)!=5 or os.environ.get('CI')!='true' or os.environ.get('STAGE_TEST_APPROVAL')!=APPROVAL: raise ValueError('CI grant required')
sha,session,ttl,action=sys.argv[1:]
i={'sha':sha,'session':session,'ttl':int(ttl),'generation':str(uuid.uuid4()),'due':int(time.time())+int(ttl)}
validate(i)
if action not in ('open','close'): raise ValueError('Action denied')
with open(ROOT/'lock','a') as lock:
fcntl.flock(lock,fcntl.LOCK_EX) # contention waits, never discards intent
if action=='open':
# Repeated opens cannot mint a new generation or renew TTL.
worker.save(i)
# Journal is durable before the first possible DB mutation.
if not run(i,'open'):
raise ValueError('Open failed; cleanup remains pending')
else:
p=ROOT/(session+'.json')
fd=os.open(p,os.O_RDONLY|os.O_NOFOLLOW)
with os.fdopen(fd) as f: i=json.load(f)
validate(i)
if run(i,'close'): p.unlink(); worker.sync()
else: raise ValueError('Cleanup pending')
if __name__=='__main__':
try: main()
except Exception:
print('Stage principal denied or pending cleanup',file=sys.stderr); sys.exit(1)
+6 -21
View File
@@ -1,27 +1,12 @@
#!/usr/bin/env bash
# Authenticated CI host-side command only. No production argument or dynamic container name.
# CI-only authority: fixed Stage cleanup worker, no dynamic executable/container.
set -euo pipefail
set +x
[[ $# == 4 && $1 =~ ^[0-9a-f]{40}$ && $2 =~ ^[a-z0-9-]{8,64}$ && $3 =~ ^[0-9]+$ && $4 =~ ^(open|close)$ ]] || exit 2
[[ $3 -ge 60 && $3 -le 3600 && ${STAGE_TEST_APPROVAL:-} == owner-authorized-stage-editorial && ${CI:-} == true ]] || exit 2
sha=$1 session=$2 ttl=$3 action=$4
[[ ! -e /.dockerenv && $(id -u) == 0 ]] || exit 2
exec 9>/opt/services/canina/stage-test-principal.lock
flock -n 9 || exit 2
[[ $(docker exec canino_backend_stage cat /app/STAGE_TEST_ARTIFACT_SHA) == "$sha" ]] || exit 2
[[ $(docker inspect canino_backend_stage --format '{{.Name}}') == /canino_backend_stage ]] || exit 2
run() {
docker exec -e APP_ENV=stage -e CI=true -e STAGE_TEST_PRINCIPAL_ENABLED=true \
-e STAGE_TEST_APPROVAL=owner-authorized-stage-editorial -e STAGE_TEST_SHA="$sha" \
-e STAGE_TEST_SESSION="$session" -e STAGE_TEST_TTL_SECONDS="$ttl" \
canino_backend_stage node dist/auth/stage-test-principal.cli.js "$1"
}
if [[ $action == open ]]; then
# Arm cleanup BEFORE creating the account. systemd keeps this independent of CI loss.
script="/opt/services/canina/releases/$sha/scripts/stage-test-principal-host.sh"
[[ -f $script && ! -L $script ]] || exit 2
systemd-run --quiet --unit="canina-stage-principal-$session" --on-active="${ttl}s" \
--setenv=CI=true --setenv=STAGE_TEST_APPROVAL=owner-authorized-stage-editorial \
/bin/bash "$script" "$sha" "$session" "$ttl" close
fi
run "$action"
# Provisioned through reviewed deployment, not through the credential CI command.
# Fail before any DB mutation unless reboot-persistent retry recovery is enabled.
systemctl is-enabled --quiet canina-stage-principal-cleanup.timer
systemctl is-active --quiet canina-stage-principal-cleanup.timer
exec /usr/bin/python3 /opt/services/canina/stage-principal-cleanup.py "$@"
+10
View File
@@ -0,0 +1,10 @@
#!/usr/bin/env bash
# Run ONLY inside a disposable fixture container with Postgres/Redis test network.
set -euo pipefail
[[ ${STAGE_PRINCIPAL_DISPOSABLE_TEST:-} == true && ${DATABASE_URL:-} == postgresql://postgres@principal-durable-pg:5432/principal_disposable ]] || exit 2
python3 -c 'from pathlib import Path; s=Path("prisma/schema.prisma").read_text(); Path("prisma/pre-stage.prisma").write_text("\n".join(x for x in s.splitlines() if not any(k in x for k in ["authExpiresAt","stageTestSession","stageTestGeneration"])))'
node node_modules/prisma/build/index.js db push --schema prisma/pre-stage.prisma --force-reset --skip-generate
# Executes exact additive SQL, not a final-schema db push substitute.
node node_modules/prisma/build/index.js db execute --schema prisma/schema.prisma --file prisma/migrations/20261024090000_stage_test_principal/migration.sql
node node_modules/prisma/build/index.js generate
node node_modules/jest/bin/jest.js principal --runInBand --testTimeout=240000
+45
View File
@@ -0,0 +1,45 @@
import importlib.util, pathlib, tempfile, unittest, json
spec=importlib.util.spec_from_file_location('cleanup',pathlib.Path(__file__).with_name('stage-principal-cleanup.py'))
m=importlib.util.module_from_spec(spec); spec.loader.exec_module(m)
class CleanupTests(unittest.TestCase):
def test_redeploy_restart_and_failed_delete_retry(self):
with tempfile.TemporaryDirectory() as root:
calls=[]
def run(intent,action):
calls.append((intent['generation'],action)); return len(calls)>1
worker=m.Worker(pathlib.Path(root),run)
intent={'sha':'a'*40,'session':'test-session','ttl':60,'generation':'8311eb41-23b4-4aab-ade2-e97b90debc11','due':0}
worker.save(intent)
worker.sweep(100)
self.assertTrue((pathlib.Path(root)/'test-session.json').exists())
m.Worker(pathlib.Path(root),run).sweep(200)
self.assertFalse((pathlib.Path(root)/'test-session.json').exists())
self.assertEqual(calls,[(intent['generation'],'close')]*2)
def test_contention_preserves_intent(self):
import fcntl
with tempfile.TemporaryDirectory() as root:
worker=m.Worker(pathlib.Path(root),lambda *_:self.fail('must not execute while locked'))
intent={'sha':'a'*40,'session':'test-session','ttl':60,'generation':'8311eb41-23b4-4aab-ade2-e97b90debc11','due':0}
worker.save(intent)
with open(pathlib.Path(root)/'lock','w') as lock:
fcntl.flock(lock,fcntl.LOCK_EX)
worker.sweep(100)
self.assertTrue((pathlib.Path(root)/'test-session.json').exists())
def test_commands_follow_redeploy_but_keep_generation(self):
from unittest.mock import patch
from types import SimpleNamespace
intent={'sha':'a'*40,'session':'test-session','ttl':60,'generation':'8311eb41-23b4-4aab-ade2-e97b90debc11','due':0}
with patch.object(m.subprocess,'check_output',side_effect=['c'*64,'b'*40,'stage-generation-cleanup-v1']), patch.object(m.subprocess,'run',return_value=SimpleNamespace(returncode=0)) as execute:
self.assertTrue(m.run(intent,'close'))
cmd=execute.call_args.args[0]
self.assertIn('STAGE_TEST_SHA='+'b'*40,cmd)
self.assertIn('STAGE_TEST_GENERATION='+intent['generation'],cmd)
self.assertIn('c'*64,cmd)
def test_old_cli_and_redeploy_open_fail_closed(self):
from unittest.mock import patch
intent={'sha':'a'*40,'session':'test-session','ttl':60,'generation':'8311eb41-23b4-4aab-ade2-e97b90debc11','due':0}
with patch.object(m.subprocess,'check_output',side_effect=['c'*64,'b'*40]), patch.object(m.subprocess,'run') as execute:
self.assertFalse(m.run(intent,'open')); execute.assert_not_called()
with patch.object(m.subprocess,'check_output',side_effect=['c'*64,'b'*40,'old']), patch.object(m.subprocess,'run') as execute:
self.assertFalse(m.run(intent,'close')); execute.assert_not_called()
if __name__=='__main__': unittest.main()