fix(auth): persist generation-fenced Stage principal cleanup retries
This commit is contained in:
1 parent
80a2e188cf
commit
70a5449d35
13 files changed
+323
-44
No files matched your search
@@ -242,7 +242,7 @@ export class AdminService {
|
||||
return this.persistUser(dto);
|
||||
}
|
||||
|
||||
private async persistUser(dto: CreateUserDto, stage?: { session: string; expiresAt: Date }) {
|
||||
private async persistUser(dto: CreateUserDto, stage?: { session: string; expiresAt: Date; generation?: string }) {
|
||||
const mobile = normalizeMobile(dto.mobile);
|
||||
const existingMobile = await this.prisma.user.findUnique({
|
||||
where: { mobile },
|
||||
@@ -275,7 +275,7 @@ export class AdminService {
|
||||
|
||||
return this.prisma.user.create({
|
||||
data: {
|
||||
...(stage ? { id: STAGE_TEST_ID, stageTestGeneration:randomUUID(), stageTestSession: stage.session, authExpiresAt: stage.expiresAt } : {}),
|
||||
...(stage ? { id: STAGE_TEST_ID, stageTestGeneration:stage.generation || randomUUID(), stageTestSession: stage.session, authExpiresAt: stage.expiresAt } : {}),
|
||||
firstName: dto.firstName.trim(),
|
||||
lastName: dto.lastName.trim(),
|
||||
mobile,
|
||||
@@ -295,21 +295,22 @@ export class AdminService {
|
||||
const grant = authorizeStageSession(env, sha);
|
||||
const existing = await this.prisma.user.findFirst({where:{OR:[{id:STAGE_TEST_ID},{email:STAGE_TEST_EMAIL},{mobile:STAGE_TEST_MOBILE}]}});
|
||||
if (existing) {
|
||||
if (existing.id !== STAGE_TEST_ID || existing.email !== STAGE_TEST_EMAIL || existing.mobile !== STAGE_TEST_MOBILE || existing.stageTestSession !== grant.session || existing.role !== 'SUPER_ADMIN' || !existing.authExpiresAt || existing.authExpiresAt.getTime() <= Date.now() || !await bcrypt.compare(password, existing.password || '')) throw new Error('Refusing existing or unowned principal');
|
||||
if (existing.id !== STAGE_TEST_ID || existing.email !== STAGE_TEST_EMAIL || existing.mobile !== STAGE_TEST_MOBILE || existing.stageTestSession !== grant.session || (env.STAGE_TEST_GENERATION && existing.stageTestGeneration !== env.STAGE_TEST_GENERATION) || existing.role !== 'SUPER_ADMIN' || !existing.authExpiresAt || existing.authExpiresAt.getTime() <= Date.now() || !await bcrypt.compare(password, existing.password || '')) throw new Error('Refusing existing or unowned principal');
|
||||
return existing; // Never renew TTL on retry.
|
||||
}
|
||||
if (password.length < 32) throw new Error('Server generated secret required');
|
||||
return this.persistUser({firstName:'Stage',lastName:'CI independent reviewer',email:STAGE_TEST_EMAIL,mobile:STAGE_TEST_MOBILE,role:'SUPER_ADMIN',password}, {session:grant.session,expiresAt:new Date(Date.now()+grant.ttl*1000)});
|
||||
return this.persistUser({firstName:'Stage',lastName:'CI independent reviewer',email:STAGE_TEST_EMAIL,mobile:STAGE_TEST_MOBILE,role:'SUPER_ADMIN',password}, {session:grant.session,expiresAt:new Date(Date.now()+grant.ttl*1000),generation:env.STAGE_TEST_GENERATION});
|
||||
}
|
||||
|
||||
async cleanupStageTestPrincipal(env: NodeJS.ProcessEnv, sha: string) {
|
||||
async cleanupStageTestPrincipal(env: NodeJS.ProcessEnv, sha: string, onRevoked: () => void = () => undefined) {
|
||||
const grant = authorizeStageSession(env, sha);
|
||||
const existing = await this.prisma.user.findUnique({where:{id:STAGE_TEST_ID}});
|
||||
if (!existing) return;
|
||||
if (existing.email !== STAGE_TEST_EMAIL || existing.mobile !== STAGE_TEST_MOBILE || existing.stageTestSession !== grant.session || !existing.authExpiresAt) throw new Error('Refusing unowned principal cleanup');
|
||||
if (!existing) { onRevoked(); return; }
|
||||
if (existing.email !== STAGE_TEST_EMAIL || existing.mobile !== STAGE_TEST_MOBILE || existing.stageTestSession !== grant.session || !existing.authExpiresAt || (env.STAGE_TEST_GENERATION && existing.stageTestGeneration !== env.STAGE_TEST_GENERATION)) throw new Error('Refusing unowned principal cleanup');
|
||||
// Revoke before relational cleanup; failed cleanup cannot leave valid tokens.
|
||||
await this.prisma.user.update({where:{id:STAGE_TEST_ID},data:{authExpiresAt:new Date(0),password:''}});
|
||||
return this.deleteUserRecords(STAGE_TEST_ID);
|
||||
await this.prisma.user.update({where:{id:STAGE_TEST_ID,stageTestSession:grant.session,stageTestGeneration:existing.stageTestGeneration},data:{authExpiresAt:new Date(0),password:''}});
|
||||
onRevoked();
|
||||
return this.deleteUserRecords(STAGE_TEST_ID, {session:grant.session,generation:existing.stageTestGeneration});
|
||||
}
|
||||
|
||||
async updateUser(id: string, dto: UpdateUserDto, actor?: UserAdminActor) {
|
||||
@@ -399,8 +400,12 @@ export class AdminService {
|
||||
return this.deleteUserRecords(id);
|
||||
}
|
||||
|
||||
private async deleteUserRecords(id: string) {
|
||||
private async deleteUserRecords(id: string, stage?: {session:string; generation:string|null}) {
|
||||
return this.prisma.$transaction(async (tx) => {
|
||||
if (stage) {
|
||||
// Acquire the row lock with an exact fence before deleting any related rows.
|
||||
await tx.user.update({where:{id,stageTestSession:stage.session,stageTestGeneration:stage.generation},data:{authExpiresAt:new Date(0),password:''}});
|
||||
}
|
||||
await tx.userAddress.deleteMany({ where: { userId: id } });
|
||||
await tx.walletTransaction.deleteMany({ where: { userId: id } });
|
||||
await tx.pet.deleteMany({ where: { userId: id } });
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
import { AdminService } from '../admin/admin.service';
|
||||
import { STAGE_TEST_ID, STAGE_TEST_EMAIL, STAGE_TEST_MOBILE } from './stage-test-principal';
|
||||
const sha='a'.repeat(40), generation='8311eb41-23b4-4aab-ade2-e97b90debc11';
|
||||
const env={APP_ENV:'stage',CI:'true',STAGE_TEST_PRINCIPAL_ENABLED:'true',STAGE_TEST_APPROVAL:'owner-authorized-stage-editorial',STAGE_TEST_SHA:sha,STAGE_TEST_SESSION:'cleanup-session',STAGE_TEST_TTL_SECONDS:'600',STAGE_TEST_GENERATION:generation};
|
||||
describe('durable cleanup ownership fence',()=>{
|
||||
it('does not revoke a replacement generation',async()=>{
|
||||
const update=jest.fn();
|
||||
const db={user:{findUnique:async()=>({id:STAGE_TEST_ID,email:STAGE_TEST_EMAIL,mobile:STAGE_TEST_MOBILE,stageTestSession:env.STAGE_TEST_SESSION,stageTestGeneration:'replacement',authExpiresAt:new Date()}),update}};
|
||||
await expect(new AdminService(db as any,undefined as any,undefined as any).cleanupStageTestPrincipal(env,sha)).rejects.toThrow('Refusing');
|
||||
expect(update).not.toHaveBeenCalled();
|
||||
});
|
||||
it('confirms revocation before relational deletion failure',async()=>{
|
||||
const events:string[]=[];
|
||||
const db={user:{findUnique:async()=>({id:STAGE_TEST_ID,email:STAGE_TEST_EMAIL,mobile:STAGE_TEST_MOBILE,stageTestSession:env.STAGE_TEST_SESSION,stageTestGeneration:generation,authExpiresAt:new Date()}),update:async()=>events.push('revoke')},$transaction:async()=>{events.push('delete');throw Error('FK failure');}};
|
||||
await expect((new AdminService(db as any,undefined as any,undefined as any).cleanupStageTestPrincipal as any)(env,sha,()=>events.push('handle-removed'))).rejects.toThrow('FK failure');
|
||||
expect(events).toEqual(['revoke','handle-removed','delete']);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,44 @@
|
||||
import { PrismaClient } from '@prisma/client';
|
||||
import { JwtService } from '@nestjs/jwt';
|
||||
import { AuthService } from './auth.service';
|
||||
import { RedisService } from '../redis/redis.service';
|
||||
import { STAGE_TEST_ID } from './stage-test-principal';
|
||||
import { createServer, Server } from 'http';
|
||||
import { execFile } from 'child_process';
|
||||
import { promisify } from 'util';
|
||||
import { readFileSync, writeFileSync, existsSync } from 'fs';
|
||||
const enabled=process.env.STAGE_PRINCIPAL_DISPOSABLE_TEST==='true';
|
||||
(enabled?describe:describe.skip)('actual internal CLI disposable lifecycle',()=>{
|
||||
const sha='a'.repeat(40), generation='b411eb41-23b4-4aab-ade2-e97b90debc11', session='cli-integration-session';
|
||||
const handle=`/app/private-stage-test/${session}.json`;
|
||||
const db=new PrismaClient({log:[]}), redis=new RedisService(); let server:Server;
|
||||
const env={...process.env,APP_ENV:'stage',CI:'true',STAGE_TEST_PRINCIPAL_ENABLED:'true',STAGE_TEST_APPROVAL:'owner-authorized-stage-editorial',STAGE_TEST_SHA:sha,STAGE_TEST_SESSION:session,STAGE_TEST_TTL_SECONDS:'600',STAGE_TEST_GENERATION:generation};
|
||||
const cli=(action:string)=>promisify(execFile)(process.execPath,['-r','ts-node/register','src/auth/stage-test-principal.cli.ts',action],{env,timeout:90000});
|
||||
beforeAll(async()=>{
|
||||
if(!process.env.DATABASE_URL?.endsWith('/principal_disposable')) throw Error('Disposable database only');
|
||||
writeFileSync('/app/STAGE_TEST_ARTIFACT_SHA',sha); redis.onModuleInit();
|
||||
const auth=new AuthService(db as any,new JwtService({secret:process.env.JWT_SECRET,signOptions:{expiresIn:'2h'}}),redis,undefined as any);
|
||||
server=createServer(async(req,res)=>{
|
||||
try {let body='';for await(const chunk of req) body+=chunk;
|
||||
if(req.url!=='/api/auth/admin-login') {res.writeHead(404).end();return;}
|
||||
const login=await auth.adminLogin(JSON.parse(body));res.setHeader('content-type','application/json');res.end(JSON.stringify(login));
|
||||
} catch {res.writeHead(401).end();}
|
||||
});
|
||||
await new Promise<void>(r=>server.listen(3000,'127.0.0.1',r));
|
||||
},120000);
|
||||
afterAll(async()=>{await new Promise<void>(r=>server.close(()=>r()));redis.onModuleDestroy();await db.$disconnect();});
|
||||
it('logs in using persisted credentials and removes handle on confirmed revocation despite failed delete',async()=>{
|
||||
await cli('open'); expect(existsSync(handle)).toBe(true);
|
||||
const credential=JSON.parse(readFileSync(handle,'utf8'));expect(credential.accessToken).toBeTruthy();
|
||||
expect((await db.user.findUnique({where:{id:STAGE_TEST_ID}}))?.stageTestGeneration).toBe(generation);
|
||||
await db.$executeRawUnsafe(`CREATE FUNCTION deny_principal_delete() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RAISE EXCEPTION 'test FK deletion failure'; END $$`);
|
||||
await db.$executeRawUnsafe(`CREATE TRIGGER deny_principal_delete BEFORE DELETE ON users FOR EACH ROW EXECUTE FUNCTION deny_principal_delete()`);
|
||||
try {
|
||||
await expect(cli('close')).rejects.toThrow();
|
||||
expect(existsSync(handle)).toBe(false);
|
||||
const revoked=await db.user.findUnique({where:{id:STAGE_TEST_ID}});
|
||||
expect(revoked?.authExpiresAt?.getTime()).toBe(0);expect(revoked?.password).toBe('');
|
||||
} finally {await db.$executeRawUnsafe(`DROP TRIGGER deny_principal_delete ON users`);await db.$executeRawUnsafe(`DROP FUNCTION deny_principal_delete()`);}
|
||||
await cli('close');expect(await db.user.findUnique({where:{id:STAGE_TEST_ID}})).toBeNull();
|
||||
},240000);
|
||||
});
|
||||
@@ -10,12 +10,15 @@ async function main() {
|
||||
const sha = readFileSync('/app/STAGE_TEST_ARTIFACT_SHA', 'utf8').trim();
|
||||
const grant = authorizeStageSession(process.env, sha);
|
||||
const action = process.argv[2];
|
||||
if (action === 'cleanup-capability') { process.stdout.write('stage-generation-cleanup-v1'); return; }
|
||||
if (!['open','close'].includes(action)) throw new Error('Unknown command');
|
||||
const root = '/app/private-stage-test';
|
||||
mkdirSync(root,{recursive:true,mode:0o700});
|
||||
const st = statSync(root);
|
||||
if (st.uid !== process.getuid!() || (st.mode & 0o777) !== 0o700) throw new Error('Unsafe private directory');
|
||||
if (!/^[a-f0-9-]{36}$/.test(process.env.STAGE_TEST_GENERATION || '')) throw new Error('Generation required');
|
||||
const handle = `${root}/${grant.session}.json`;
|
||||
const removeHandle = () => { try {unlinkSync(handle);} catch(e:any) {if(e.code !== 'ENOENT') throw e;} };
|
||||
const audit = (event:string) => {
|
||||
const fd = openSync(`${root}/audit.jsonl`, constants.O_WRONLY|constants.O_CREAT|constants.O_APPEND|constants.O_NOFOLLOW,0o600);
|
||||
try { writeFileSync(fd,JSON.stringify({event,sha,session:grant.session,at:new Date().toISOString()})+'\n'); } finally {closeSync(fd);}
|
||||
@@ -25,8 +28,7 @@ async function main() {
|
||||
try {
|
||||
audit(`${action}-requested`);
|
||||
if(action === 'close') {
|
||||
await service.cleanupStageTestPrincipal(process.env,sha);
|
||||
try {unlinkSync(handle);} catch(e:any) {if(e.code !== 'ENOENT') throw e;}
|
||||
await service.cleanupStageTestPrincipal(process.env,sha,removeHandle);
|
||||
audit('revoked-and-deleted');
|
||||
return;
|
||||
}
|
||||
@@ -59,8 +61,7 @@ async function main() {
|
||||
} catch {
|
||||
audit('failed');
|
||||
// Leave only a revoked account if relational cleanup fails.
|
||||
await service.cleanupStageTestPrincipal(process.env,sha).catch(()=>undefined);
|
||||
try {unlinkSync(handle);} catch { /* no credential content in error */ }
|
||||
await service.cleanupStageTestPrincipal(process.env,sha,removeHandle).catch(()=>undefined);
|
||||
throw new Error('Stage test principal operation failed; inspect private audit');
|
||||
} finally {await db.$disconnect();}
|
||||
}
|
||||
|
||||
@@ -48,5 +48,11 @@ const enabled=process.env.STAGE_PRINCIPAL_DISPOSABLE_TEST === 'true';
|
||||
await admin.cleanupStageTestPrincipal(grant,sha);
|
||||
await expect(strategy.validate(req,payload)).rejects.toThrow();
|
||||
expect(await db.user.findUnique({where:{id:u.id}})).toBeNull();
|
||||
const replacement=await admin.provisionStageTestPrincipal(grant,sha,password);
|
||||
expect(replacement.stageTestGeneration).not.toBe(u.stageTestGeneration);
|
||||
await expect(strategy.validate(req,payload)).rejects.toThrow();
|
||||
await expect(auth.refreshToken(login.data.refreshToken)).rejects.toThrow();
|
||||
const fresh=await auth.adminLogin({email:STAGE_TEST_EMAIL,password});
|
||||
expect((await strategy.validate(req,jwt.verify(fresh.data.accessToken))).id).toBe(u.id);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,52 @@
|
||||
# Durable Stage cleanup verification
|
||||
|
||||
This change replaces the SHA-pinned transient timer with a root-owned intent journal
|
||||
and a fixed host systemd retry worker. No host service is installed or enabled by
|
||||
the CI credential command. Deployment must first install the reviewed worker as
|
||||
`/opt/services/canina/stage-principal-cleanup.py` (root:root 0755), install the two
|
||||
supplied systemd units under `/etc/systemd/system`, daemon-reload, and enable/start
|
||||
`canina-stage-principal-cleanup.timer`. This is a promotion prerequisite, NOT an
|
||||
instruction to run these actions during review. The open wrapper refuses to operate
|
||||
without an enabled and active timer. Never overwrite or delete the 0700 root-owned
|
||||
`/opt/services/canina/stage-principal-cleanup` journal during deployments.
|
||||
|
||||
A journal entry is fsynced before provisioning and contains the exact session,
|
||||
random generation, creation SHA, bounded TTL and immutable cleanup deadline.
|
||||
Repeated opens fail rather than replace an intent or renew TTL. Close and the
|
||||
30-second boot/periodic retry worker retain intents on lock contention, unavailable
|
||||
containers/DB, and relational deletion failures. Cleanup resolves the current Stage
|
||||
container ID once, verifies its cleanup capability, and authenticates against its
|
||||
current artifact SHA while retaining original session/generation ownership. Older
|
||||
artifacts without generation cleanup support fail closed; never roll back to one
|
||||
with a pending principal. Persisted expiry remains independently enforced at login,
|
||||
refresh and every scoped JWT request.
|
||||
|
||||
Revocation commits before private-handle removal; relational deletion is subsequent.
|
||||
A transaction acquires a generation/session-fenced row lock before touching related
|
||||
records. Failed deletion retains a revoked row and a pending retry intent, but no
|
||||
credential handle. Wrong ownership/generation never invokes the handle callback.
|
||||
The CLI requires the generation supplied by the private journal; no broader admin
|
||||
or production authority was introduced.
|
||||
|
||||
Verification performed in a new isolated worktree and disposable Docker network:
|
||||
- Cleanup fence tests: RED (replacement generation was revoked; callback missing),
|
||||
then GREEN (2 tests).
|
||||
- Python simulated worker tests: restart/redeploy, failed-delete retry, lock
|
||||
contention, immutable container ID/current artifact and old-CLI refusal (4 tests).
|
||||
- Persisted account lifecycle plus delete/recreate access AND refresh regression.
|
||||
- Actual CLI via ts-node with real disposable Postgres/Redis and HTTP adminLogin
|
||||
handler, injected PostgreSQL delete trigger failure: confirmed revocation,
|
||||
private handle removal, then successful repeated cleanup after trigger removal.
|
||||
- TypeScript `tsc --noEmit`: passed.
|
||||
- Full migration chain attempt on fresh disposable database: baseline failure in
|
||||
20261020000000_add_blog_freshness_authorship (`blogs` missing). Not concealed.
|
||||
Independent migration test constructed the pre-Stage schema excluding all three
|
||||
new fields, then applied the EXACT committed 20261024090000 migration with psql
|
||||
ON_ERROR_STOP before exercising persisted lifecycle. Not db-push-only testing.
|
||||
|
||||
No live account, Stage/Production DB mutation, deployment, push, host service change,
|
||||
or retry of the previous pending-approval operation was performed.
|
||||
|
||||
Invocation after normal reviewed deployment and host prerequisite provisioning:
|
||||
`sudo -n env CI=true STAGE_TEST_APPROVAL=owner-authorized-stage-editorial bash /opt/services/canina/releases/<promoted-sha>/scripts/stage-test-principal-host.sh <promoted-sha> <unique-session> 600 open`
|
||||
Use the same command with `close` immediately after the authorized test.
|
||||
@@ -31,15 +31,7 @@ role checks remain unchanged. The canonical default administrator is untouched.
|
||||
tests. This revokes persisted authentication BEFORE relational deletion and
|
||||
removes the private credential file. Close is idempotent when absent.
|
||||
|
||||
A host systemd timer is armed before creation for independent TTL cleanup even
|
||||
if CI disappears. Persisted expiry is checked at login, refresh, and every JWT
|
||||
request, so killing the timer cannot extend access. Failed cleanup leaves revoked
|
||||
credentials. Cleanup timer is pinned to the creation SHA: finish/close before
|
||||
redeploy; if redeployed, use the approved CI recovery session with its recorded
|
||||
ownership rather than changing an unrelated account. A collision with the exact
|
||||
ID, email or mobile fails closed; an owned retry with the same secret cannot
|
||||
extend expiry. The test principal is limited in JWT strategy to task reads,
|
||||
proposal creation, Gate1 decisions and editorial failure. All other paths deny.
|
||||
A durable root-owned host cleanup journal and boot/periodic systemd retry worker must be provisioned before opening a session. Cleanup follows the current Stage artifact but retains the original exact session and generation fence; persisted expiry cannot be extended. See [cleanup verification and promotion prerequisites](stage-principal-cleanup-verification.md). No live host services are changed by review tests.
|
||||
|
||||
## Verification
|
||||
|
||||
|
||||
@@ -0,0 +1,11 @@
|
||||
[Unit]
|
||||
Description=Retry exact owned temporary Stage principal cleanup
|
||||
After=docker.service
|
||||
Wants=docker.service
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
User=root
|
||||
UMask=0077
|
||||
ExecStart=/usr/bin/python3 /opt/services/canina/stage-principal-cleanup.py sweep
|
||||
TimeoutStartSec=120
|
||||
@@ -0,0 +1,10 @@
|
||||
[Unit]
|
||||
Description=Recover temporary Stage principal cleanup after reboot and failures
|
||||
|
||||
[Timer]
|
||||
OnBootSec=30s
|
||||
OnUnitInactiveSec=30s
|
||||
Unit=canina-stage-principal-cleanup.service
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
@@ -0,0 +1,100 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Root-only Stage principal intent journal. No generic command/container inputs."""
|
||||
import fcntl
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import uuid
|
||||
|
||||
ROOT = Path('/opt/services/canina/stage-principal-cleanup')
|
||||
APPROVAL = 'owner-authorized-stage-editorial'
|
||||
|
||||
def validate(i):
|
||||
if set(i) != {'sha','session','ttl','generation','due'} or not re.fullmatch('[a-f0-9]{40}', i['sha']) or not re.fullmatch('[a-z0-9-]{8,64}', i['session']) or not re.fullmatch('[a-f0-9-]{36}', i['generation']) or not isinstance(i['ttl'],int) or not 60 <= i['ttl'] <= 3600 or not isinstance(i['due'],int):
|
||||
raise ValueError('Invalid cleanup intent')
|
||||
|
||||
class Worker:
|
||||
def __init__(self, root, run):
|
||||
self.root, self.run = root, run
|
||||
def save(self, intent):
|
||||
validate(intent)
|
||||
p = self.root / (intent['session']+'.json')
|
||||
# Exclusive creation forbids reusing a session or extending its deadline.
|
||||
fd = os.open(p, os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_NOFOLLOW, 0o600)
|
||||
with os.fdopen(fd,'w') as f:
|
||||
json.dump(intent,f); f.flush(); os.fsync(f.fileno())
|
||||
self.sync()
|
||||
def sync(self):
|
||||
fd=os.open(self.root,os.O_RDONLY|os.O_DIRECTORY)
|
||||
try: os.fsync(fd)
|
||||
finally: os.close(fd)
|
||||
def sweep(self, now):
|
||||
with open(self.root/'lock','a') as lock:
|
||||
try: fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
|
||||
except BlockingIOError: return
|
||||
for p in sorted(self.root.glob('*.json')):
|
||||
try:
|
||||
fd=os.open(p,os.O_RDONLY|os.O_NOFOLLOW)
|
||||
with os.fdopen(fd) as f: i=json.load(f)
|
||||
validate(i)
|
||||
if p.name != i['session']+'.json': raise ValueError('Intent name mismatch')
|
||||
if i['due'] <= now and self.run(i,'close'):
|
||||
p.unlink(); self.sync()
|
||||
except Exception:
|
||||
# Keep intent through DB/container/redeploy failures; next tick retries.
|
||||
print('Stage cleanup pending',file=sys.stderr)
|
||||
|
||||
def run(i, action):
|
||||
# Opens remain SHA pinned; cleanup only uses the CURRENT artifact's narrow CLI.
|
||||
# Original ownership is the session plus unpredictable account generation.
|
||||
container=subprocess.check_output(['docker','inspect','--format','{{.Id}}','canino_backend_stage'],text=True,timeout=30).strip()
|
||||
if not re.fullmatch('[a-f0-9]{64}',container): return False
|
||||
sha=subprocess.check_output(['docker','exec',container,'cat','/app/STAGE_TEST_ARTIFACT_SHA'],text=True,timeout=30).strip()
|
||||
if not re.fullmatch('[a-f0-9]{40}',sha) or (action=='open' and sha!=i['sha']):
|
||||
return False
|
||||
env={'APP_ENV':'stage','CI':'true','STAGE_TEST_PRINCIPAL_ENABLED':'true','STAGE_TEST_APPROVAL':APPROVAL,'STAGE_TEST_SHA':sha,'STAGE_TEST_SESSION':i['session'],'STAGE_TEST_TTL_SECONDS':str(i['ttl']),'STAGE_TEST_GENERATION':i['generation']}
|
||||
cmd=['docker','exec']
|
||||
for k,v in env.items(): cmd+=['-e',k+'='+v]
|
||||
cli=cmd+[container,'node','dist/auth/stage-test-principal.cli.js']
|
||||
# Never send a generation-fenced cleanup to an older CLI that ignores fences.
|
||||
capability=subprocess.check_output(cli+['cleanup-capability'],text=True,timeout=30)
|
||||
if capability!='stage-generation-cleanup-v1': return False
|
||||
return subprocess.run(cli+[action],timeout=90).returncode==0
|
||||
|
||||
def main():
|
||||
if os.getuid()!=0 or Path('/.dockerenv').exists(): raise ValueError('Host root only')
|
||||
os.umask(0o077)
|
||||
ROOT.mkdir(mode=0o700,exist_ok=True)
|
||||
st=ROOT.lstat()
|
||||
if ROOT.is_symlink() or st.st_uid!=0 or st.st_mode & 0o777 != 0o700: raise ValueError('Unsafe journal')
|
||||
worker=Worker(ROOT,run)
|
||||
if sys.argv[1:]==['sweep']:
|
||||
worker.sweep(int(time.time())); return
|
||||
if len(sys.argv)!=5 or os.environ.get('CI')!='true' or os.environ.get('STAGE_TEST_APPROVAL')!=APPROVAL: raise ValueError('CI grant required')
|
||||
sha,session,ttl,action=sys.argv[1:]
|
||||
i={'sha':sha,'session':session,'ttl':int(ttl),'generation':str(uuid.uuid4()),'due':int(time.time())+int(ttl)}
|
||||
validate(i)
|
||||
if action not in ('open','close'): raise ValueError('Action denied')
|
||||
with open(ROOT/'lock','a') as lock:
|
||||
fcntl.flock(lock,fcntl.LOCK_EX) # contention waits, never discards intent
|
||||
if action=='open':
|
||||
# Repeated opens cannot mint a new generation or renew TTL.
|
||||
worker.save(i)
|
||||
# Journal is durable before the first possible DB mutation.
|
||||
if not run(i,'open'):
|
||||
raise ValueError('Open failed; cleanup remains pending')
|
||||
else:
|
||||
p=ROOT/(session+'.json')
|
||||
fd=os.open(p,os.O_RDONLY|os.O_NOFOLLOW)
|
||||
with os.fdopen(fd) as f: i=json.load(f)
|
||||
validate(i)
|
||||
if run(i,'close'): p.unlink(); worker.sync()
|
||||
else: raise ValueError('Cleanup pending')
|
||||
if __name__=='__main__':
|
||||
try: main()
|
||||
except Exception:
|
||||
print('Stage principal denied or pending cleanup',file=sys.stderr); sys.exit(1)
|
||||
@@ -1,27 +1,12 @@
|
||||
#!/usr/bin/env bash
|
||||
# Authenticated CI host-side command only. No production argument or dynamic container name.
|
||||
# CI-only authority: fixed Stage cleanup worker, no dynamic executable/container.
|
||||
set -euo pipefail
|
||||
set +x
|
||||
[[ $# == 4 && $1 =~ ^[0-9a-f]{40}$ && $2 =~ ^[a-z0-9-]{8,64}$ && $3 =~ ^[0-9]+$ && $4 =~ ^(open|close)$ ]] || exit 2
|
||||
[[ $3 -ge 60 && $3 -le 3600 && ${STAGE_TEST_APPROVAL:-} == owner-authorized-stage-editorial && ${CI:-} == true ]] || exit 2
|
||||
sha=$1 session=$2 ttl=$3 action=$4
|
||||
[[ ! -e /.dockerenv && $(id -u) == 0 ]] || exit 2
|
||||
exec 9>/opt/services/canina/stage-test-principal.lock
|
||||
flock -n 9 || exit 2
|
||||
[[ $(docker exec canino_backend_stage cat /app/STAGE_TEST_ARTIFACT_SHA) == "$sha" ]] || exit 2
|
||||
[[ $(docker inspect canino_backend_stage --format '{{.Name}}') == /canino_backend_stage ]] || exit 2
|
||||
run() {
|
||||
docker exec -e APP_ENV=stage -e CI=true -e STAGE_TEST_PRINCIPAL_ENABLED=true \
|
||||
-e STAGE_TEST_APPROVAL=owner-authorized-stage-editorial -e STAGE_TEST_SHA="$sha" \
|
||||
-e STAGE_TEST_SESSION="$session" -e STAGE_TEST_TTL_SECONDS="$ttl" \
|
||||
canino_backend_stage node dist/auth/stage-test-principal.cli.js "$1"
|
||||
}
|
||||
if [[ $action == open ]]; then
|
||||
# Arm cleanup BEFORE creating the account. systemd keeps this independent of CI loss.
|
||||
script="/opt/services/canina/releases/$sha/scripts/stage-test-principal-host.sh"
|
||||
[[ -f $script && ! -L $script ]] || exit 2
|
||||
systemd-run --quiet --unit="canina-stage-principal-$session" --on-active="${ttl}s" \
|
||||
--setenv=CI=true --setenv=STAGE_TEST_APPROVAL=owner-authorized-stage-editorial \
|
||||
/bin/bash "$script" "$sha" "$session" "$ttl" close
|
||||
fi
|
||||
run "$action"
|
||||
# Provisioned through reviewed deployment, not through the credential CI command.
|
||||
# Fail before any DB mutation unless reboot-persistent retry recovery is enabled.
|
||||
systemctl is-enabled --quiet canina-stage-principal-cleanup.timer
|
||||
systemctl is-active --quiet canina-stage-principal-cleanup.timer
|
||||
exec /usr/bin/python3 /opt/services/canina/stage-principal-cleanup.py "$@"
|
||||
@@ -0,0 +1,10 @@
|
||||
#!/usr/bin/env bash
|
||||
# Run ONLY inside a disposable fixture container with Postgres/Redis test network.
|
||||
set -euo pipefail
|
||||
[[ ${STAGE_PRINCIPAL_DISPOSABLE_TEST:-} == true && ${DATABASE_URL:-} == postgresql://postgres@principal-durable-pg:5432/principal_disposable ]] || exit 2
|
||||
python3 -c 'from pathlib import Path; s=Path("prisma/schema.prisma").read_text(); Path("prisma/pre-stage.prisma").write_text("\n".join(x for x in s.splitlines() if not any(k in x for k in ["authExpiresAt","stageTestSession","stageTestGeneration"])))'
|
||||
node node_modules/prisma/build/index.js db push --schema prisma/pre-stage.prisma --force-reset --skip-generate
|
||||
# Executes exact additive SQL, not a final-schema db push substitute.
|
||||
node node_modules/prisma/build/index.js db execute --schema prisma/schema.prisma --file prisma/migrations/20261024090000_stage_test_principal/migration.sql
|
||||
node node_modules/prisma/build/index.js generate
|
||||
node node_modules/jest/bin/jest.js principal --runInBand --testTimeout=240000
|
||||
@@ -0,0 +1,45 @@
|
||||
import importlib.util, pathlib, tempfile, unittest, json
|
||||
spec=importlib.util.spec_from_file_location('cleanup',pathlib.Path(__file__).with_name('stage-principal-cleanup.py'))
|
||||
m=importlib.util.module_from_spec(spec); spec.loader.exec_module(m)
|
||||
class CleanupTests(unittest.TestCase):
|
||||
def test_redeploy_restart_and_failed_delete_retry(self):
|
||||
with tempfile.TemporaryDirectory() as root:
|
||||
calls=[]
|
||||
def run(intent,action):
|
||||
calls.append((intent['generation'],action)); return len(calls)>1
|
||||
worker=m.Worker(pathlib.Path(root),run)
|
||||
intent={'sha':'a'*40,'session':'test-session','ttl':60,'generation':'8311eb41-23b4-4aab-ade2-e97b90debc11','due':0}
|
||||
worker.save(intent)
|
||||
worker.sweep(100)
|
||||
self.assertTrue((pathlib.Path(root)/'test-session.json').exists())
|
||||
m.Worker(pathlib.Path(root),run).sweep(200)
|
||||
self.assertFalse((pathlib.Path(root)/'test-session.json').exists())
|
||||
self.assertEqual(calls,[(intent['generation'],'close')]*2)
|
||||
def test_contention_preserves_intent(self):
|
||||
import fcntl
|
||||
with tempfile.TemporaryDirectory() as root:
|
||||
worker=m.Worker(pathlib.Path(root),lambda *_:self.fail('must not execute while locked'))
|
||||
intent={'sha':'a'*40,'session':'test-session','ttl':60,'generation':'8311eb41-23b4-4aab-ade2-e97b90debc11','due':0}
|
||||
worker.save(intent)
|
||||
with open(pathlib.Path(root)/'lock','w') as lock:
|
||||
fcntl.flock(lock,fcntl.LOCK_EX)
|
||||
worker.sweep(100)
|
||||
self.assertTrue((pathlib.Path(root)/'test-session.json').exists())
|
||||
def test_commands_follow_redeploy_but_keep_generation(self):
|
||||
from unittest.mock import patch
|
||||
from types import SimpleNamespace
|
||||
intent={'sha':'a'*40,'session':'test-session','ttl':60,'generation':'8311eb41-23b4-4aab-ade2-e97b90debc11','due':0}
|
||||
with patch.object(m.subprocess,'check_output',side_effect=['c'*64,'b'*40,'stage-generation-cleanup-v1']), patch.object(m.subprocess,'run',return_value=SimpleNamespace(returncode=0)) as execute:
|
||||
self.assertTrue(m.run(intent,'close'))
|
||||
cmd=execute.call_args.args[0]
|
||||
self.assertIn('STAGE_TEST_SHA='+'b'*40,cmd)
|
||||
self.assertIn('STAGE_TEST_GENERATION='+intent['generation'],cmd)
|
||||
self.assertIn('c'*64,cmd)
|
||||
def test_old_cli_and_redeploy_open_fail_closed(self):
|
||||
from unittest.mock import patch
|
||||
intent={'sha':'a'*40,'session':'test-session','ttl':60,'generation':'8311eb41-23b4-4aab-ade2-e97b90debc11','due':0}
|
||||
with patch.object(m.subprocess,'check_output',side_effect=['c'*64,'b'*40]), patch.object(m.subprocess,'run') as execute:
|
||||
self.assertFalse(m.run(intent,'open')); execute.assert_not_called()
|
||||
with patch.object(m.subprocess,'check_output',side_effect=['c'*64,'b'*40,'old']), patch.object(m.subprocess,'run') as execute:
|
||||
self.assertFalse(m.run(intent,'close')); execute.assert_not_called()
|
||||
if __name__=='__main__': unittest.main()
|
||||
Reference in new issue
Block a user