Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
a236ef8b11 |
No files matched your search
@@ -1,41 +0,0 @@
|
||||
name: Owner authorized Stage test principal
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
approval:
|
||||
required: true
|
||||
description: "Exact owner grant: owner-authorized-stage-editorial"
|
||||
session:
|
||||
required: true
|
||||
description: "Unique session (8-64 lowercase letters/digits/hyphens)"
|
||||
ttl:
|
||||
required: true
|
||||
description: "Bounded session TTL, 60-3600 seconds"
|
||||
action:
|
||||
required: true
|
||||
description: "open or close"
|
||||
jobs:
|
||||
principal:
|
||||
if: github.ref == 'refs/heads/develop'
|
||||
runs-on: canina
|
||||
steps:
|
||||
- name: Explicit authenticated Stage CI seed authority
|
||||
shell: bash
|
||||
env:
|
||||
BRANCH: ${{ github.ref_name }}
|
||||
DEPLOY_SHA: ${{ github.sha }}
|
||||
DEPLOY_KEY: ${{ secrets.DEPLOY_KEY }}
|
||||
DEPLOY_KNOWN_HOSTS: ${{ secrets.DEPLOY_KNOWN_HOSTS }}
|
||||
STAGE_TEST_APPROVAL: ${{ inputs.approval }}
|
||||
STAGE_TEST_SESSION: ${{ inputs.session }}
|
||||
STAGE_TEST_TTL_SECONDS: ${{ inputs.ttl }}
|
||||
STAGE_TEST_ACTION: ${{ inputs.action }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
set +x
|
||||
[[ "$DEPLOY_SHA" =~ ^[a-f0-9]{40}$ ]] || exit 2
|
||||
umask 077
|
||||
script=$(mktemp)
|
||||
trap 'rm -f "$script"' EXIT
|
||||
wget --no-proxy -qO "$script" "https://git.parsaaghayi.ir/parsa/canina/raw/commit/$DEPLOY_SHA/scripts/ci-stage-test-principal.sh"
|
||||
bash "$script"
|
||||
@@ -1,30 +0,0 @@
|
||||
name: Bounded Stage v4 credential acceptance window
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- ci/v4-stage-acceptance-timeout-860324e
|
||||
jobs:
|
||||
acceptance-window:
|
||||
if: github.ref == 'refs/heads/ci/v4-stage-acceptance-timeout-860324e'
|
||||
runs-on: canina
|
||||
timeout-minutes: 12
|
||||
steps:
|
||||
- name: Bounded Stage acceptance with guaranteed generation-fenced close
|
||||
shell: bash
|
||||
env:
|
||||
BRANCH: ${{ github.ref_name }}
|
||||
DEPLOY_SHA: ${{ github.sha }}
|
||||
GITHUB_RUN_ATTEMPT: ${{ github.run_attempt }}
|
||||
DEPLOY_KEY: ${{ secrets.DEPLOY_KEY }}
|
||||
DEPLOY_KNOWN_HOSTS: ${{ secrets.DEPLOY_KNOWN_HOSTS }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
set +x
|
||||
[[ "$BRANCH" == ci/v4-stage-acceptance-timeout-860324e && "$DEPLOY_SHA" =~ ^[a-f0-9]{40}$ && "$GITHUB_RUN_ATTEMPT" == 1 ]] || exit 2
|
||||
umask 077
|
||||
scratch=$(mktemp -d)
|
||||
trap 'rm -rf "$scratch"' EXIT
|
||||
for file in ci-stage-acceptance.sh ci-stage-test-principal.sh stage-editorial-acceptance.py; do
|
||||
wget --no-proxy -qO "$scratch/$file" "https://git.parsaaghayi.ir/parsa/canina/raw/commit/$DEPLOY_SHA/scripts/$file"
|
||||
done
|
||||
bash "$scratch/ci-stage-acceptance.sh"
|
||||
+1
-3
@@ -32,10 +32,8 @@ RUN http_proxy=http://172.17.0.1:8888 https_proxy=http://172.17.0.1:8888 \
|
||||
FROM node:20-alpine
|
||||
USER root
|
||||
RUN http_proxy=http://172.17.0.1:8888 https_proxy=http://172.17.0.1:8888 apk add --no-cache openssl || apk add --no-cache openssl || true
|
||||
RUN mkdir -p /app/uploads /app/private-stage-test && chmod 700 /app/private-stage-test && chown node:node /app/uploads /app/private-stage-test
|
||||
RUN mkdir -p /app/uploads && chown node:node /app/uploads
|
||||
WORKDIR /app
|
||||
ARG STAGE_TEST_ARTIFACT_SHA=disabled
|
||||
RUN printf "%s" "$STAGE_TEST_ARTIFACT_SHA" > /app/STAGE_TEST_ARTIFACT_SHA
|
||||
ENV PRISMA_HIDE_UPDATE_MESSAGE=true
|
||||
COPY --chown=node:node --from=builder /app/package*.json ./
|
||||
COPY --chown=node:node --from=builder /app/tsconfig.json ./tsconfig.json
|
||||
|
||||
@@ -1 +0,0 @@
|
||||
ALTER TABLE "users" ADD COLUMN "auth_expires_at" TIMESTAMPTZ, ADD COLUMN "stage_test_session" VARCHAR(64), ADD COLUMN "stage_test_generation" UUID;
|
||||
@@ -16,9 +16,6 @@ model User {
|
||||
mobile String @unique @db.VarChar(15)
|
||||
password String? @db.VarChar(255)
|
||||
role String @default("User_PetOwner") @db.VarChar(30)
|
||||
authExpiresAt DateTime? @map("auth_expires_at") @db.Timestamptz()
|
||||
stageTestGeneration String? @map("stage_test_generation") @db.Uuid
|
||||
stageTestSession String? @map("stage_test_session") @db.VarChar(64)
|
||||
walletBalance Decimal @default(0.00) @map("wallet_balance") @db.Decimal(15, 2)
|
||||
charityDonationTotal Decimal @default(0.00) @map("charity_donation_total") @db.Decimal(15, 2)
|
||||
createdAt DateTime @default(now()) @map("created_at") @db.Timestamptz()
|
||||
|
||||
@@ -63,8 +63,6 @@ export class CouponInput {
|
||||
|
||||
import { ALLOWED_USER_ROLES, assertUserMutationAllowed } from "./user-role-policy";
|
||||
import type { UserAdminActor } from "./user-role-policy";
|
||||
import { randomUUID } from "crypto";
|
||||
import { authorizeStageSession, STAGE_TEST_ID, STAGE_TEST_EMAIL, STAGE_TEST_MOBILE } from "../auth/stage-test-principal";
|
||||
|
||||
@Injectable()
|
||||
export class AdminService {
|
||||
@@ -239,10 +237,6 @@ export class AdminService {
|
||||
|
||||
async createUser(dto: CreateUserDto, actor?: UserAdminActor) {
|
||||
assertUserMutationAllowed(actor, undefined, dto.role);
|
||||
return this.persistUser(dto);
|
||||
}
|
||||
|
||||
private async persistUser(dto: CreateUserDto, stage?: { session: string; expiresAt: Date; generation?: string }) {
|
||||
const mobile = normalizeMobile(dto.mobile);
|
||||
const existingMobile = await this.prisma.user.findUnique({
|
||||
where: { mobile },
|
||||
@@ -275,7 +269,6 @@ export class AdminService {
|
||||
|
||||
return this.prisma.user.create({
|
||||
data: {
|
||||
...(stage ? { id: STAGE_TEST_ID, stageTestGeneration:stage.generation || randomUUID(), stageTestSession: stage.session, authExpiresAt: stage.expiresAt } : {}),
|
||||
firstName: dto.firstName.trim(),
|
||||
lastName: dto.lastName.trim(),
|
||||
mobile,
|
||||
@@ -290,29 +283,6 @@ export class AdminService {
|
||||
});
|
||||
}
|
||||
|
||||
// Explicit CI seed authority, NOT a fabricated SUPER_ADMIN actor. Not controller-exposed.
|
||||
async provisionStageTestPrincipal(env: NodeJS.ProcessEnv, sha: string, password: string) {
|
||||
const grant = authorizeStageSession(env, sha);
|
||||
const existing = await this.prisma.user.findFirst({where:{OR:[{id:STAGE_TEST_ID},{email:STAGE_TEST_EMAIL},{mobile:STAGE_TEST_MOBILE}]}});
|
||||
if (existing) {
|
||||
if (existing.id !== STAGE_TEST_ID || existing.email !== STAGE_TEST_EMAIL || existing.mobile !== STAGE_TEST_MOBILE || existing.stageTestSession !== grant.session || (env.STAGE_TEST_GENERATION && existing.stageTestGeneration !== env.STAGE_TEST_GENERATION) || existing.role !== 'SUPER_ADMIN' || !existing.authExpiresAt || existing.authExpiresAt.getTime() <= Date.now() || !await bcrypt.compare(password, existing.password || '')) throw new Error('Refusing existing or unowned principal');
|
||||
return existing; // Never renew TTL on retry.
|
||||
}
|
||||
if (password.length < 32) throw new Error('Server generated secret required');
|
||||
return this.persistUser({firstName:'Stage',lastName:'CI independent reviewer',email:STAGE_TEST_EMAIL,mobile:STAGE_TEST_MOBILE,role:'SUPER_ADMIN',password}, {session:grant.session,expiresAt:new Date(Date.now()+grant.ttl*1000),generation:env.STAGE_TEST_GENERATION});
|
||||
}
|
||||
|
||||
async cleanupStageTestPrincipal(env: NodeJS.ProcessEnv, sha: string, onRevoked: () => void = () => undefined) {
|
||||
const grant = authorizeStageSession(env, sha);
|
||||
const existing = await this.prisma.user.findUnique({where:{id:STAGE_TEST_ID}});
|
||||
if (!existing) { onRevoked(); return; }
|
||||
if (existing.email !== STAGE_TEST_EMAIL || existing.mobile !== STAGE_TEST_MOBILE || existing.stageTestSession !== grant.session || !existing.authExpiresAt || (env.STAGE_TEST_GENERATION && existing.stageTestGeneration !== env.STAGE_TEST_GENERATION)) throw new Error('Refusing unowned principal cleanup');
|
||||
// Revoke before relational cleanup; failed cleanup cannot leave valid tokens.
|
||||
await this.prisma.user.update({where:{id:STAGE_TEST_ID,stageTestSession:grant.session,stageTestGeneration:existing.stageTestGeneration},data:{authExpiresAt:new Date(0),password:''}});
|
||||
onRevoked();
|
||||
return this.deleteUserRecords(STAGE_TEST_ID, {session:grant.session,generation:existing.stageTestGeneration});
|
||||
}
|
||||
|
||||
async updateUser(id: string, dto: UpdateUserDto, actor?: UserAdminActor) {
|
||||
if (id === "12345678-1234-1234-1234-123456789012") {
|
||||
throw new BadRequestException(
|
||||
@@ -397,15 +367,7 @@ export class AdminService {
|
||||
|
||||
assertUserMutationAllowed(actor, user.role);
|
||||
|
||||
return this.deleteUserRecords(id);
|
||||
}
|
||||
|
||||
private async deleteUserRecords(id: string, stage?: {session:string; generation:string|null}) {
|
||||
return this.prisma.$transaction(async (tx) => {
|
||||
if (stage) {
|
||||
// Acquire the row lock with an exact fence before deleting any related rows.
|
||||
await tx.user.update({where:{id,stageTestSession:stage.session,stageTestGeneration:stage.generation},data:{authExpiresAt:new Date(0),password:''}});
|
||||
}
|
||||
await tx.userAddress.deleteMany({ where: { userId: id } });
|
||||
await tx.walletTransaction.deleteMany({ where: { userId: id } });
|
||||
await tx.pet.deleteMany({ where: { userId: id } });
|
||||
|
||||
@@ -14,7 +14,6 @@ import { SmsService } from "../common/services/sms.service";
|
||||
import * as bcrypt from "bcryptjs";
|
||||
import * as crypto from "crypto";
|
||||
import { normalizeMobile } from "../common/utils/phone.utils";
|
||||
import { assertPrincipalActive } from "./stage-test-principal";
|
||||
import { getJwtRefreshSecret } from "./auth.constants";
|
||||
|
||||
export class RegisterInput {
|
||||
@@ -201,8 +200,7 @@ export class AuthService {
|
||||
});
|
||||
}
|
||||
|
||||
assertPrincipalActive(user);
|
||||
const payload = { sub: user.id, phoneNumber: user.mobile, role: user.role, ...(user.stageTestSession ? {stageTestSession:user.stageTestSession, stageTestGeneration:user.stageTestGeneration} : {}) };
|
||||
const payload = { sub: user.id, phoneNumber: user.mobile, role: user.role };
|
||||
const tokens = await this.generateTokens(payload);
|
||||
|
||||
const { password: _p, ...safeUser } = user;
|
||||
@@ -259,8 +257,7 @@ export class AuthService {
|
||||
},
|
||||
});
|
||||
|
||||
assertPrincipalActive(user);
|
||||
const payload = { sub: user.id, phoneNumber: user.mobile, role: user.role, ...(user.stageTestSession ? {stageTestSession:user.stageTestSession, stageTestGeneration:user.stageTestGeneration} : {}) };
|
||||
const payload = { sub: user.id, phoneNumber: user.mobile, role: user.role };
|
||||
const tokens = await this.generateTokens(payload);
|
||||
|
||||
const { password: _p, ...safeUser } = user;
|
||||
@@ -303,8 +300,7 @@ export class AuthService {
|
||||
});
|
||||
}
|
||||
|
||||
assertPrincipalActive(user);
|
||||
const payload = { sub: user.id, phoneNumber: user.mobile, role: user.role, ...(user.stageTestSession ? {stageTestSession:user.stageTestSession, stageTestGeneration:user.stageTestGeneration} : {}) };
|
||||
const payload = { sub: user.id, phoneNumber: user.mobile, role: user.role };
|
||||
const tokens = await this.generateTokens(payload);
|
||||
|
||||
const { password: _pw, ...safeUser } = user;
|
||||
@@ -330,12 +326,10 @@ export class AuthService {
|
||||
if (dbAdmin && dbAdmin.password && body.password) {
|
||||
const isMatch = await bcrypt.compare(body.password, dbAdmin.password);
|
||||
if (isMatch) {
|
||||
assertPrincipalActive(dbAdmin);
|
||||
const payload = {
|
||||
sub: dbAdmin.id,
|
||||
email: dbAdmin.email,
|
||||
role: dbAdmin.role,
|
||||
...(dbAdmin.stageTestSession ? {stageTestSession:dbAdmin.stageTestSession, stageTestGeneration:dbAdmin.stageTestGeneration} : {}),
|
||||
};
|
||||
const tokens = await this.generateTokens(payload);
|
||||
const { password: _p, ...safeUser } = dbAdmin;
|
||||
@@ -449,14 +443,11 @@ export class AuthService {
|
||||
|
||||
const user = await this.prisma.user.findUnique({ where: { id: userId } });
|
||||
if (user) {
|
||||
assertPrincipalActive(user);
|
||||
if (user.stageTestSession && (!user.stageTestGeneration || decoded.stageTestGeneration !== user.stageTestGeneration || decoded.stageTestSession !== user.stageTestSession)) throw new UnauthorizedException("Stage session revoked");
|
||||
userPayload = {
|
||||
sub: user.id,
|
||||
email: user.email,
|
||||
phoneNumber: user.mobile,
|
||||
role: user.role,
|
||||
...(user.stageTestSession ? {stageTestSession:user.stageTestSession, stageTestGeneration:user.stageTestGeneration} : {}),
|
||||
};
|
||||
const { password: _p, ...restUser } = user;
|
||||
safeUser = restUser;
|
||||
@@ -496,15 +487,12 @@ export class AuthService {
|
||||
email?: string;
|
||||
role?: string;
|
||||
mobile?: string;
|
||||
stageTestSession?: string | null;
|
||||
stageTestGeneration?: string | null;
|
||||
}) {
|
||||
const payload = {
|
||||
sub: user.id,
|
||||
email: user.email,
|
||||
role: user.role,
|
||||
phoneNumber: user.mobile,
|
||||
...(user.stageTestSession ? {stageTestSession:user.stageTestSession, stageTestGeneration:user.stageTestGeneration} : {}),
|
||||
};
|
||||
const tokens = await this.generateTokens(payload);
|
||||
return {
|
||||
|
||||
@@ -1,18 +0,0 @@
|
||||
import { AdminService } from '../admin/admin.service';
|
||||
import { STAGE_TEST_ID, STAGE_TEST_EMAIL, STAGE_TEST_MOBILE } from './stage-test-principal';
|
||||
const sha='a'.repeat(40), generation='8311eb41-23b4-4aab-ade2-e97b90debc11';
|
||||
const env={APP_ENV:'stage',CI:'true',STAGE_TEST_PRINCIPAL_ENABLED:'true',STAGE_TEST_APPROVAL:'owner-authorized-stage-editorial',STAGE_TEST_SHA:sha,STAGE_TEST_SESSION:'cleanup-session',STAGE_TEST_TTL_SECONDS:'600',STAGE_TEST_GENERATION:generation};
|
||||
describe('durable cleanup ownership fence',()=>{
|
||||
it('does not revoke a replacement generation',async()=>{
|
||||
const update=jest.fn();
|
||||
const db={user:{findUnique:async()=>({id:STAGE_TEST_ID,email:STAGE_TEST_EMAIL,mobile:STAGE_TEST_MOBILE,stageTestSession:env.STAGE_TEST_SESSION,stageTestGeneration:'replacement',authExpiresAt:new Date()}),update}};
|
||||
await expect(new AdminService(db as any,undefined as any,undefined as any).cleanupStageTestPrincipal(env,sha)).rejects.toThrow('Refusing');
|
||||
expect(update).not.toHaveBeenCalled();
|
||||
});
|
||||
it('confirms revocation before relational deletion failure',async()=>{
|
||||
const events:string[]=[];
|
||||
const db={user:{findUnique:async()=>({id:STAGE_TEST_ID,email:STAGE_TEST_EMAIL,mobile:STAGE_TEST_MOBILE,stageTestSession:env.STAGE_TEST_SESSION,stageTestGeneration:generation,authExpiresAt:new Date()}),update:async()=>events.push('revoke')},$transaction:async()=>{events.push('delete');throw Error('FK failure');}};
|
||||
await expect((new AdminService(db as any,undefined as any,undefined as any).cleanupStageTestPrincipal as any)(env,sha,()=>events.push('handle-removed'))).rejects.toThrow('FK failure');
|
||||
expect(events).toEqual(['revoke','handle-removed','delete']);
|
||||
});
|
||||
});
|
||||
@@ -2,7 +2,6 @@ import { ExtractJwt, Strategy } from "passport-jwt";
|
||||
import { PassportStrategy } from "@nestjs/passport";
|
||||
import { Injectable, UnauthorizedException } from "@nestjs/common";
|
||||
import { UsersService } from "../users/users.service";
|
||||
import { assertPrincipalActive } from "./stage-test-principal";
|
||||
import { getJwtSecret } from "./auth.constants";
|
||||
|
||||
import { RedisService } from "../redis/redis.service";
|
||||
@@ -13,8 +12,6 @@ export interface JwtPayload {
|
||||
email?: string;
|
||||
role?: string;
|
||||
phoneNumber?: string;
|
||||
stageTestSession?: string;
|
||||
stageTestGeneration?: string;
|
||||
}
|
||||
|
||||
@Injectable()
|
||||
@@ -62,13 +59,6 @@ export class JwtStrategy extends PassportStrategy(Strategy) {
|
||||
if (!user) {
|
||||
throw new UnauthorizedException("کاربر یافت نشد");
|
||||
}
|
||||
try { assertPrincipalActive(user); } catch { throw new UnauthorizedException("Principal expired"); }
|
||||
if (user.stageTestSession) {
|
||||
if (!user.stageTestGeneration || payload.stageTestGeneration !== user.stageTestGeneration || payload.stageTestSession !== user.stageTestSession) throw new UnauthorizedException("Stage session revoked");
|
||||
const path = req.originalUrl?.split('?')[0] || '';
|
||||
const allowed = req.method === 'GET' && /^\/api\/admin\/orchestration\/tasks(?:\/[a-f0-9-]+)?$/.test(path) || req.method === 'POST' && (/^\/api\/admin\/orchestration\/tasks$/.test(path) || /^\/api\/admin\/orchestration\/tasks\/[a-f0-9-]+\/(?:gates\/1\/decide|editorial-failure)$/.test(path));
|
||||
if (!allowed) throw new UnauthorizedException('Stage test scope denied');
|
||||
}
|
||||
return user;
|
||||
}
|
||||
}
|
||||
@@ -1,44 +0,0 @@
|
||||
import { PrismaClient } from '@prisma/client';
|
||||
import { JwtService } from '@nestjs/jwt';
|
||||
import { AuthService } from './auth.service';
|
||||
import { RedisService } from '../redis/redis.service';
|
||||
import { STAGE_TEST_ID } from './stage-test-principal';
|
||||
import { createServer, Server } from 'http';
|
||||
import { execFile } from 'child_process';
|
||||
import { promisify } from 'util';
|
||||
import { readFileSync, writeFileSync, existsSync } from 'fs';
|
||||
const enabled=process.env.STAGE_PRINCIPAL_DISPOSABLE_TEST==='true';
|
||||
(enabled?describe:describe.skip)('actual internal CLI disposable lifecycle',()=>{
|
||||
const sha='a'.repeat(40), generation='b411eb41-23b4-4aab-ade2-e97b90debc11', session='cli-integration-session';
|
||||
const handle=`/app/private-stage-test/${session}.json`;
|
||||
const db=new PrismaClient({log:[]}), redis=new RedisService(); let server:Server;
|
||||
const env={...process.env,APP_ENV:'stage',CI:'true',STAGE_TEST_PRINCIPAL_ENABLED:'true',STAGE_TEST_APPROVAL:'owner-authorized-stage-editorial',STAGE_TEST_SHA:sha,STAGE_TEST_SESSION:session,STAGE_TEST_TTL_SECONDS:'600',STAGE_TEST_GENERATION:generation};
|
||||
const cli=(action:string)=>promisify(execFile)(process.execPath,['-r','ts-node/register','src/auth/stage-test-principal.cli.ts',action],{env,timeout:90000});
|
||||
beforeAll(async()=>{
|
||||
if(!process.env.DATABASE_URL?.endsWith('/principal_disposable')) throw Error('Disposable database only');
|
||||
writeFileSync('/app/STAGE_TEST_ARTIFACT_SHA',sha); redis.onModuleInit();
|
||||
const auth=new AuthService(db as any,new JwtService({secret:process.env.JWT_SECRET,signOptions:{expiresIn:'2h'}}),redis,undefined as any);
|
||||
server=createServer(async(req,res)=>{
|
||||
try {let body='';for await(const chunk of req) body+=chunk;
|
||||
if(req.url!=='/api/auth/admin-login') {res.writeHead(404).end();return;}
|
||||
const login=await auth.adminLogin(JSON.parse(body));res.setHeader('content-type','application/json');res.end(JSON.stringify(login));
|
||||
} catch {res.writeHead(401).end();}
|
||||
});
|
||||
await new Promise<void>(r=>server.listen(3000,'127.0.0.1',r));
|
||||
},120000);
|
||||
afterAll(async()=>{await new Promise<void>(r=>server.close(()=>r()));redis.onModuleDestroy();await db.$disconnect();});
|
||||
it('logs in using persisted credentials and removes handle on confirmed revocation despite failed delete',async()=>{
|
||||
await cli('open'); expect(existsSync(handle)).toBe(true);
|
||||
const credential=JSON.parse(readFileSync(handle,'utf8'));expect(credential.accessToken).toBeTruthy();
|
||||
expect((await db.user.findUnique({where:{id:STAGE_TEST_ID}}))?.stageTestGeneration).toBe(generation);
|
||||
await db.$executeRawUnsafe(`CREATE FUNCTION deny_principal_delete() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RAISE EXCEPTION 'test FK deletion failure'; END $$`);
|
||||
await db.$executeRawUnsafe(`CREATE TRIGGER deny_principal_delete BEFORE DELETE ON users FOR EACH ROW EXECUTE FUNCTION deny_principal_delete()`);
|
||||
try {
|
||||
await expect(cli('close')).rejects.toThrow();
|
||||
expect(existsSync(handle)).toBe(false);
|
||||
const revoked=await db.user.findUnique({where:{id:STAGE_TEST_ID}});
|
||||
expect(revoked?.authExpiresAt?.getTime()).toBe(0);expect(revoked?.password).toBe('');
|
||||
} finally {await db.$executeRawUnsafe(`DROP TRIGGER deny_principal_delete ON users`);await db.$executeRawUnsafe(`DROP FUNCTION deny_principal_delete()`);}
|
||||
await cli('close');expect(await db.user.findUnique({where:{id:STAGE_TEST_ID}})).toBeNull();
|
||||
},240000);
|
||||
});
|
||||
@@ -1,68 +0,0 @@
|
||||
import 'reflect-metadata';
|
||||
import { PrismaClient } from '@prisma/client';
|
||||
import { randomBytes } from 'crypto';
|
||||
import { mkdirSync, openSync, writeFileSync, closeSync, readFileSync, statSync, unlinkSync, constants } from 'fs';
|
||||
import { AdminService } from '../admin/admin.service';
|
||||
import { authorizeStageSession, STAGE_TEST_EMAIL } from './stage-test-principal';
|
||||
|
||||
// CI-only internal seed command; deliberately not registered in AppModule.
|
||||
async function main() {
|
||||
const sha = readFileSync('/app/STAGE_TEST_ARTIFACT_SHA', 'utf8').trim();
|
||||
const grant = authorizeStageSession(process.env, sha);
|
||||
const action = process.argv[2];
|
||||
if (action === 'cleanup-capability') { process.stdout.write('stage-generation-cleanup-v1'); return; }
|
||||
if (!['open','close'].includes(action)) throw new Error('Unknown command');
|
||||
const root = '/app/private-stage-test';
|
||||
mkdirSync(root,{recursive:true,mode:0o700});
|
||||
const st = statSync(root);
|
||||
if (st.uid !== process.getuid!() || (st.mode & 0o777) !== 0o700) throw new Error('Unsafe private directory');
|
||||
if (!/^[a-f0-9-]{36}$/.test(process.env.STAGE_TEST_GENERATION || '')) throw new Error('Generation required');
|
||||
const handle = `${root}/${grant.session}.json`;
|
||||
const removeHandle = () => { try {unlinkSync(handle);} catch(e:any) {if(e.code !== 'ENOENT') throw e;} };
|
||||
const audit = (event:string) => {
|
||||
const fd = openSync(`${root}/audit.jsonl`, constants.O_WRONLY|constants.O_CREAT|constants.O_APPEND|constants.O_NOFOLLOW,0o600);
|
||||
try { writeFileSync(fd,JSON.stringify({event,sha,session:grant.session,at:new Date().toISOString()})+'\n'); } finally {closeSync(fd);}
|
||||
};
|
||||
const db = new PrismaClient({log:[]});
|
||||
const service = new AdminService(db as any,undefined as any,undefined as any);
|
||||
try {
|
||||
audit(`${action}-requested`);
|
||||
if(action === 'close') {
|
||||
await service.cleanupStageTestPrincipal(process.env,sha,removeHandle);
|
||||
audit('revoked-and-deleted');
|
||||
return;
|
||||
}
|
||||
let fd:number;
|
||||
let password:string;
|
||||
try {
|
||||
fd = openSync(handle,constants.O_WRONLY|constants.O_CREAT|constants.O_EXCL|constants.O_NOFOLLOW,0o600);
|
||||
password = randomBytes(48).toString('base64url');
|
||||
try {writeFileSync(fd,JSON.stringify({password}));} finally {closeSync(fd);}
|
||||
} catch(e:any) {
|
||||
if(e.code !== 'EEXIST') throw e;
|
||||
fd = openSync(handle,constants.O_RDONLY|constants.O_NOFOLLOW);
|
||||
try {
|
||||
const {fstatSync} = await import('fs'); const s=fstatSync(fd);
|
||||
if(!s.isFile() || s.uid !== process.getuid!() || (s.mode & 0o777) !== 0o600) throw new Error('Unsafe handle');
|
||||
password=JSON.parse(readFileSync(fd,'utf8')).password;
|
||||
} finally {closeSync(fd);}
|
||||
}
|
||||
const user=await service.provisionStageTestPrincipal(process.env,sha,password);
|
||||
// Use deployed persisted-account HTTP login, never sign tokens here.
|
||||
const response=await fetch('http://127.0.0.1:3000/api/auth/admin-login',{method:'POST',headers:{'content-type':'application/json'},body:JSON.stringify({email:STAGE_TEST_EMAIL,password}),signal:AbortSignal.timeout(15000)});
|
||||
if(!response.ok) throw new Error('Persisted login failed');
|
||||
const result:any=await response.json();
|
||||
if(result.data?.user?.id !== user.id || result.data?.user?.role !== 'SUPER_ADMIN' || !result.data?.accessToken) throw new Error('Persisted login mismatch');
|
||||
fd=openSync(handle,constants.O_WRONLY|constants.O_TRUNC|constants.O_NOFOLLOW);
|
||||
try {writeFileSync(fd,JSON.stringify({password,...result.data,expiresAt:user.authExpiresAt}));} finally {closeSync(fd);}
|
||||
audit('created-and-login-verified');
|
||||
// Only an opaque local path is public. Never upload this file as a CI artifact.
|
||||
process.stdout.write(`Stage credential handle: ${handle}\n`);
|
||||
} catch {
|
||||
audit('failed');
|
||||
// Leave only a revoked account if relational cleanup fails.
|
||||
await service.cleanupStageTestPrincipal(process.env,sha,removeHandle).catch(()=>undefined);
|
||||
throw new Error('Stage test principal operation failed; inspect private audit');
|
||||
} finally {await db.$disconnect();}
|
||||
}
|
||||
main().catch(()=>{process.stderr.write('Stage test principal denied or failed\n');process.exitCode=1;});
|
||||
@@ -1,58 +0,0 @@
|
||||
import { PrismaClient } from '@prisma/client';
|
||||
import { randomBytes } from 'crypto';
|
||||
import { JwtService } from '@nestjs/jwt';
|
||||
import { AdminService } from '../admin/admin.service';
|
||||
import { AuthService } from './auth.service';
|
||||
import { UsersService } from '../users/users.service';
|
||||
import { JwtStrategy } from './jwt.strategy';
|
||||
import { RedisService } from '../redis/redis.service';
|
||||
import { STAGE_TEST_ID, STAGE_TEST_EMAIL, STAGE_TEST_MOBILE } from './stage-test-principal';
|
||||
const enabled=process.env.STAGE_PRINCIPAL_DISPOSABLE_TEST === 'true';
|
||||
(enabled ? describe : describe.skip)('disposable persisted Stage principal lifecycle',()=>{
|
||||
const sha='a'.repeat(40), grant={APP_ENV:'stage',CI:'true',STAGE_TEST_PRINCIPAL_ENABLED:'true',STAGE_TEST_APPROVAL:'owner-authorized-stage-editorial',STAGE_TEST_SHA:sha,STAGE_TEST_SESSION:'integration-session',STAGE_TEST_TTL_SECONDS:'600'};
|
||||
const db=new PrismaClient({log:[]}); const redis=new RedisService();
|
||||
let admin:AdminService,auth:AuthService,strategy:JwtStrategy,password:string;
|
||||
beforeAll(async()=>{
|
||||
// Deliberately require a disposable database name; never point this suite at Stage.
|
||||
if(!process.env.DATABASE_URL?.endsWith('/principal_disposable')) throw new Error('Disposable database required');
|
||||
redis.onModuleInit(); password=randomBytes(48).toString('base64url');
|
||||
admin=new AdminService(db as any,redis,undefined as any);
|
||||
auth=new AuthService(db as any,new JwtService({secret:process.env.JWT_SECRET,signOptions:{expiresIn:'2h'}}),redis,undefined as any);
|
||||
strategy=new JwtStrategy(new UsersService(db as any),redis);
|
||||
});
|
||||
afterAll(async()=>{await admin.cleanupStageTestPrincipal(grant,sha); await db.$disconnect();redis.onModuleDestroy();});
|
||||
it('rejects unowned collision and never changes it',async()=>{
|
||||
const collision=await db.user.create({data:{firstName:'Unowned',lastName:'Account',email:STAGE_TEST_EMAIL,mobile:STAGE_TEST_MOBILE}});
|
||||
try {
|
||||
await expect(admin.provisionStageTestPrincipal(grant,sha,password)).rejects.toThrow('Refusing');
|
||||
expect((await db.user.findUnique({where:{id:collision.id}}))?.role).toBe('User_PetOwner');
|
||||
} finally {await db.user.delete({where:{id:collision.id}});}
|
||||
});
|
||||
it('creates through account service, logs in, verifies signed tokens, never renews retry TTL',async()=>{
|
||||
const u=await admin.provisionStageTestPrincipal(grant,sha,password);
|
||||
expect(u.id).toBe(STAGE_TEST_ID); expect(u.role).toBe('SUPER_ADMIN'); expect(u.password).not.toBe(password);
|
||||
const retry=await admin.provisionStageTestPrincipal(grant,sha,password); expect(retry.authExpiresAt).toEqual(u.authExpiresAt);
|
||||
await expect(admin.provisionStageTestPrincipal({...grant,STAGE_TEST_SESSION:'different-session'},sha,password)).rejects.toThrow('Refusing');
|
||||
const login=await auth.adminLogin({email:STAGE_TEST_EMAIL,password});
|
||||
const jwt=new JwtService({secret:process.env.JWT_SECRET});
|
||||
const payload=jwt.verify(login.data.accessToken); expect(payload.sub).toBe(u.id);expect(payload.role).toBe('SUPER_ADMIN');
|
||||
const req:any={method:'GET',originalUrl:'/api/admin/orchestration/tasks',headers:{authorization:`Bearer ${login.data.accessToken}`}};
|
||||
expect((await strategy.validate(req,payload)).id).toBe(u.id);
|
||||
await expect(strategy.validate({...req,originalUrl:'/api/admin/users'},payload)).rejects.toThrow('scope denied');
|
||||
await expect(strategy.validate({...req,method:'POST',originalUrl:'/api/admin/orchestration/tasks/abcd/gates/2/decide'},payload)).rejects.toThrow('scope denied');
|
||||
// Fixture time travel: persisted expiry is enforced independently of the long-lived signed token.
|
||||
await db.user.update({where:{id:u.id},data:{authExpiresAt:new Date(0)}});
|
||||
await expect(strategy.validate(req,payload)).rejects.toThrow();
|
||||
await expect(auth.adminLogin({email:STAGE_TEST_EMAIL,password})).rejects.toThrow();
|
||||
await expect(auth.refreshToken(login.data.refreshToken)).rejects.toThrow();
|
||||
await admin.cleanupStageTestPrincipal(grant,sha);
|
||||
await expect(strategy.validate(req,payload)).rejects.toThrow();
|
||||
expect(await db.user.findUnique({where:{id:u.id}})).toBeNull();
|
||||
const replacement=await admin.provisionStageTestPrincipal(grant,sha,password);
|
||||
expect(replacement.stageTestGeneration).not.toBe(u.stageTestGeneration);
|
||||
await expect(strategy.validate(req,payload)).rejects.toThrow();
|
||||
await expect(auth.refreshToken(login.data.refreshToken)).rejects.toThrow();
|
||||
const fresh=await auth.adminLogin({email:STAGE_TEST_EMAIL,password});
|
||||
expect((await strategy.validate(req,jwt.verify(fresh.data.accessToken))).id).toBe(u.id);
|
||||
});
|
||||
});
|
||||
@@ -1,13 +0,0 @@
|
||||
import { assertPrincipalActive, authorizeStageSession } from './stage-test-principal';
|
||||
const sha = 'a'.repeat(40);
|
||||
const valid = { APP_ENV: 'stage', STAGE_TEST_PRINCIPAL_ENABLED: 'true', CI: 'true', STAGE_TEST_APPROVAL: 'owner-authorized-stage-editorial', STAGE_TEST_SHA: sha, STAGE_TEST_SESSION: 'editorial-20261024', STAGE_TEST_TTL_SECONDS: '600' };
|
||||
describe('Stage CI authority', () => {
|
||||
it('denies by default and in Production despite opt in', () => {
|
||||
expect(() => authorizeStageSession({}, sha)).toThrow();
|
||||
expect(() => authorizeStageSession({...valid, APP_ENV:'production'}, sha)).toThrow();
|
||||
});
|
||||
it('requires immutable matching SHA and bounded explicit authorization', () => {
|
||||
expect(authorizeStageSession(valid, sha).ttl).toBe(600);
|
||||
for(const patch of [{STAGE_TEST_SHA:'main'}, {STAGE_TEST_SHA:'b'.repeat(40)}, {STAGE_TEST_TTL_SECONDS:'3601'}, {STAGE_TEST_TTL_SECONDS:'0'}, {STAGE_TEST_APPROVAL:''}, {CI:''}]) expect(() => authorizeStageSession({...valid,...patch},sha)).toThrow();
|
||||
});
|
||||
});
|
||||
@@ -1,13 +0,0 @@
|
||||
// Internal seed authority only. No controller, JWT actor, or owner impersonation.
|
||||
export const STAGE_TEST_ID = 'e876e7a0-738f-4e59-b6db-e270512ababc';
|
||||
export const STAGE_TEST_EMAIL = 'stage-editorial-reviewer@ci.canina.invalid';
|
||||
export const STAGE_TEST_MOBILE = '09999999071';
|
||||
export function authorizeStageSession(env: NodeJS.ProcessEnv, artifactSha: string) {
|
||||
if (env.APP_ENV !== 'stage' || env.STAGE_TEST_PRINCIPAL_ENABLED !== 'true' || env.CI !== 'true' || env.STAGE_TEST_APPROVAL !== 'owner-authorized-stage-editorial' || !/^[a-f0-9]{40}$/.test(artifactSha) || env.STAGE_TEST_SHA !== artifactSha || !/^[a-z0-9-]{8,64}$/.test(env.STAGE_TEST_SESSION || '')) throw new Error('Stage CI authority denied');
|
||||
const ttl = Number(env.STAGE_TEST_TTL_SECONDS);
|
||||
if (!Number.isInteger(ttl) || ttl < 60 || ttl > 3600) throw new Error('Bounded TTL required');
|
||||
return {ttl, session:env.STAGE_TEST_SESSION!, sha:artifactSha};
|
||||
}
|
||||
export function assertPrincipalActive(user: { authExpiresAt?: Date | null }, now = Date.now()) {
|
||||
if (user.authExpiresAt && user.authExpiresAt.getTime() <= now) throw new Error('Principal expired');
|
||||
}
|
||||
@@ -1,32 +0,0 @@
|
||||
import { JwtService } from '@nestjs/jwt';
|
||||
import { JwtStrategy } from './jwt.strategy';
|
||||
import { AuthService } from './auth.service';
|
||||
|
||||
describe('Stage credentials remain fenced to their creation session', () => {
|
||||
const id='e876e7a0-738f-4e59-b6db-e270512ababc';
|
||||
const current={id, role:'SUPER_ADMIN', stageTestSession:'new-session', stageTestGeneration:'fresh-generation', authExpiresAt:new Date(Date.now()+600000)};
|
||||
const redis:any={get:jest.fn(async()=> '1'),del:jest.fn(),set:jest.fn()};
|
||||
beforeAll(()=> {process.env.JWT_SECRET='disposable-only-test-secret-at-least-32-characters';process.env.JWT_REFRESH_SECRET='disposable-only-refresh-secret-at-least-32-characters';});
|
||||
it('rejects an old-session access token after fixed account ID is recreated',async()=>{
|
||||
const strategy=new JwtStrategy({findById:async()=>current} as any,{get:async()=>null} as any);
|
||||
const req:any={method:'GET',originalUrl:'/api/admin/orchestration/tasks',headers:{}};
|
||||
await expect(strategy.validate(req,{sub:id,stageTestSession:'old-session'} as any)).rejects.toThrow();
|
||||
await expect(strategy.validate(req,{sub:id})).rejects.toThrow();
|
||||
await expect(strategy.validate(req,{sub:id,stageTestSession:'new-session',stageTestGeneration:'old-generation'} as any)).rejects.toThrow();
|
||||
await expect(strategy.validate(req,{sub:id,stageTestSession:'new-session',stageTestGeneration:'fresh-generation'} as any)).resolves.toMatchObject({id});
|
||||
});
|
||||
it('rejects an old-session refresh token even while its Redis entry remains active',async()=>{
|
||||
const jwt=new JwtService({secret:process.env.JWT_SECRET});
|
||||
const auth=new AuthService({user:{findUnique:async()=>current}} as any,jwt,redis,undefined as any);
|
||||
const token=jwt.sign({sub:id,jti:'old-refresh',role:'SUPER_ADMIN',stageTestSession:'old-session'},{secret:process.env.JWT_REFRESH_SECRET});
|
||||
await expect(auth.refreshToken(token)).rejects.toThrow();
|
||||
});
|
||||
it('includes session fencing in newly issued credentials',async()=>{
|
||||
const jwt=new JwtService({secret:process.env.JWT_SECRET});
|
||||
const auth=new AuthService({} as any,jwt,redis,undefined as any);
|
||||
const result=await auth.refresh(current);
|
||||
expect(jwt.verify(result.data.accessToken).stageTestGeneration).toBe('fresh-generation');
|
||||
expect(jwt.verify(result.data.accessToken).stageTestSession).toBe('new-session');
|
||||
expect(jwt.verify(result.data.refreshToken,{secret:process.env.JWT_REFRESH_SECRET}).stageTestSession).toBe('new-session');
|
||||
});
|
||||
});
|
||||
@@ -1,53 +0,0 @@
|
||||
# Durable Stage cleanup verification
|
||||
|
||||
This change replaces the SHA-pinned transient timer with a root-owned intent journal
|
||||
and a fixed host systemd retry worker. No host service is installed or enabled by
|
||||
the CI credential command. The tracked `deploy.yml` → `ci-host-deploy.sh` →
|
||||
`deploy.sh` path invokes `install-stage-principal-cleanup.py` from the verified
|
||||
immutable release before `stage_rollout`. It atomically installs the root-owned
|
||||
worker (0755) and supplied systemd units (0644), reloads systemd, enables/starts
|
||||
the timer and verifies enabled/active state. Failure aborts rollout. No manual
|
||||
sudo provisioning is required or permitted during review. The open wrapper refuses to operate
|
||||
without an enabled and active timer. Never overwrite or delete the 0700 root-owned
|
||||
`/opt/services/canina/stage-principal-cleanup` journal during deployments.
|
||||
|
||||
A journal entry is fsynced before provisioning and contains the exact session,
|
||||
random generation, creation SHA, bounded TTL and immutable cleanup deadline.
|
||||
Repeated opens fail rather than replace an intent or renew TTL. Close and the
|
||||
30-second boot/periodic retry worker retain intents on lock contention, unavailable
|
||||
containers/DB, and relational deletion failures. Cleanup resolves the current Stage
|
||||
container ID once, verifies its cleanup capability, and authenticates against its
|
||||
current artifact SHA while retaining original session/generation ownership. Older
|
||||
artifacts without generation cleanup support fail closed; never roll back to one
|
||||
with a pending principal. Persisted expiry remains independently enforced at login,
|
||||
refresh and every scoped JWT request.
|
||||
|
||||
Revocation commits before private-handle removal; relational deletion is subsequent.
|
||||
A transaction acquires a generation/session-fenced row lock before touching related
|
||||
records. Failed deletion retains a revoked row and a pending retry intent, but no
|
||||
credential handle. Wrong ownership/generation never invokes the handle callback.
|
||||
The CLI requires the generation supplied by the private journal; no broader admin
|
||||
or production authority was introduced.
|
||||
|
||||
Verification performed in a new isolated worktree and disposable Docker network:
|
||||
- Cleanup fence tests: RED (replacement generation was revoked; callback missing),
|
||||
then GREEN (2 tests).
|
||||
- Python simulated worker tests: restart/redeploy, failed-delete retry, lock
|
||||
contention, immutable container ID/current artifact and old-CLI refusal (4 tests).
|
||||
- Persisted account lifecycle plus delete/recreate access AND refresh regression.
|
||||
- Actual CLI via ts-node with real disposable Postgres/Redis and HTTP adminLogin
|
||||
handler, injected PostgreSQL delete trigger failure: confirmed revocation,
|
||||
private handle removal, then successful repeated cleanup after trigger removal.
|
||||
- TypeScript `tsc --noEmit`: passed.
|
||||
- Full migration chain attempt on fresh disposable database: baseline failure in
|
||||
20261020000000_add_blog_freshness_authorship (`blogs` missing). Not concealed.
|
||||
Independent migration test constructed the pre-Stage schema excluding all three
|
||||
new fields, then applied the EXACT committed 20261024090000 migration with psql
|
||||
ON_ERROR_STOP before exercising persisted lifecycle. Not db-push-only testing.
|
||||
|
||||
No live account, Stage/Production DB mutation, deployment, push, host service change,
|
||||
or retry of the previous pending-approval operation was performed.
|
||||
|
||||
Invocation after normal reviewed deployment and host prerequisite provisioning:
|
||||
`sudo -n env CI=true STAGE_TEST_APPROVAL=owner-authorized-stage-editorial bash /opt/services/canina/releases/<promoted-sha>/scripts/stage-test-principal-host.sh <promoted-sha> <unique-session> 600 open`
|
||||
Use the same command with `close` immediately after the authorized test.
|
||||
@@ -1,44 +0,0 @@
|
||||
# Temporary Stage editorial reviewer (disabled by default)
|
||||
|
||||
Authority: repository write/CI-dispatch permission plus the existing authenticated,
|
||||
pinned Stage deployment SSH boundary. This is a narrowly scoped seed operation,
|
||||
not an API-key actor, owner JWT, or a public bootstrap endpoint. The destructive
|
||||
legacy seed is not used. `AdminService` shares its normal hashing/persistence and
|
||||
relational cleanup implementation with the internal command; normal user mutation
|
||||
role checks remain unchanged. The canonical default administrator is untouched.
|
||||
|
||||
## Activation after independent review
|
||||
|
||||
1. Promote this commit through the normal Git/CI Stage deployment. Do not run the
|
||||
seed command against an old image: the embedded immutable SHA must match.
|
||||
Apply the additive migration through the existing migration deployment step.
|
||||
2. Dispatch **Owner authorized Stage test principal** on develop with `approval`
|
||||
exactly `owner-authorized-stage-editorial`, unique `session` such as
|
||||
`editorial-20261024-a`, `ttl` explicitly 600 (maximum 3600), and `action=open`.
|
||||
The workflow uses existing DEPLOY_KEY / pinned DEPLOY_KNOWN_HOSTS only.
|
||||
3. CI reports only `/app/private-stage-test/<session>.json` inside
|
||||
`canino_backend_stage`. This mode-0600 file in a mode-0700 directory contains
|
||||
the process-generated password and real persisted-account login tokens.
|
||||
Consume it only from an approved in-container CI HTTP process; never print,
|
||||
download, publish as an artifact, or type it into a browser. No owner secret is
|
||||
needed. Audit records contain events, session and SHA, never credentials.
|
||||
4. Use this independent reviewer for Gate1 only; the proposer/PMO must be a
|
||||
distinct actor. Existing self-review checks are not changed. This session does
|
||||
NOT authorize Gate2, publication, release, general admin APIs, worker identity
|
||||
replacement, or budget increases. Approve a separate explicit bounded test
|
||||
budget before running inference; unknown prior reservations remain retained.
|
||||
5. Dispatch the same workflow/session/ttl with `action=close` immediately after
|
||||
tests. This revokes persisted authentication BEFORE relational deletion and
|
||||
removes the private credential file. Close is idempotent when absent.
|
||||
|
||||
A durable root-owned host cleanup journal and boot/periodic systemd retry worker must be provisioned before opening a session. Cleanup follows the current Stage artifact but retains the original exact session and generation fence; persisted expiry cannot be extended. See [cleanup verification and promotion prerequisites](stage-principal-cleanup-verification.md). No live host services are changed by review tests.
|
||||
|
||||
## Verification
|
||||
|
||||
Unit guards and existing independent self-review/role tests run with Jest.
|
||||
`stage-test-principal.integration.spec.ts` is opt-in via
|
||||
`STAGE_PRINCIPAL_DISPOSABLE_TEST=true` and requires a database URL ending in
|
||||
`/principal_disposable`; use a new isolated Postgres and Redis, then remove both.
|
||||
It verifies real account persistence/hash, signed access/refresh login tokens,
|
||||
collision refusal, nonrenewing retries, request scope, expiry, deletion/revocation.
|
||||
Never run it against Stage or Production.
|
||||
@@ -1,44 +0,0 @@
|
||||
# Bounded unattended Stage acceptance
|
||||
|
||||
Parent review/activation only: push exact branch
|
||||
`ci/v4-stage-acceptance-timeout-860324e` once. No push or model execution was
|
||||
performed while preparing this branch. Do not merge this one-shot workflow into
|
||||
develop. The previous session/branch must never be reused.
|
||||
|
||||
Pinned deployed SHA: `860324e3a91b31c2b6d4e784e458f6d120582cb8`.
|
||||
Fixed session: `v4-ci-timeout-860324e-20261024`, TTL600 seconds, acceptance
|
||||
hard window480 seconds. Exact branch plus run_attempt1 required. Durable
|
||||
exclusive session intent rejects subsequent opens even if the CI attempt field
|
||||
is reset. EXIT always attempts existing generation-fenced close; durable timer
|
||||
retains cleanup through runner/SSH loss. Nothing redesigns account provisioning.
|
||||
|
||||
The tracked host Python script requires the exact SHA's successful deploy log,
|
||||
backend artifact and installed editorial package SHA. It reads the Stage private
|
||||
login handle and existing PMO worker credential only into process memory; no
|
||||
public bootstrap, secret stdout, direct DB access or Production writes.
|
||||
|
||||
It copies the exact topic and criteria from the unique FAILED v5 task starting
|
||||
`4a109`, preserves all prior FAILED editorial ledgers (must total unknown60),
|
||||
creates one fixed-request-id20 proposal through PMO transport, reviews Gate1 as
|
||||
the independently authenticated temporary SUPER_ADMIN, queues via PMO, then runs
|
||||
the installed runner once as hermes. Explicit owner authorization for this
|
||||
bounded Stage test is required; this is not independent content QA.
|
||||
|
||||
Runner total330 seconds plus preflight/recovery stays within the480 bound;
|
||||
profile cap120 and CLI30 headroom remain unchanged. If it stops in RUNNING,
|
||||
the script waits for lease expiry and uses the existing narrow editorial-failure
|
||||
HTTP contract, preserving unknown20. No retry or reconciliation/zero-cost claim.
|
||||
Hard process/transport loss can prevent this failure fence; state then requires
|
||||
read-back, not a new attempt. Account close still has durable600 backstop.
|
||||
|
||||
Success requires authenticated status VERIFYING, actual artifact bytes/digest
|
||||
and task/deployed-SHA binding, PENDING_INDEPENDENT_QA, fresh reserved20 and
|
||||
unchanged prior unknown60. No publication, image generation, Production changes,
|
||||
cron, fabricated QA, fabricated cost or provider503 attribution.
|
||||
|
||||
Checks: `python3 scripts/test_stage_editorial_acceptance.py`,
|
||||
`python3 scripts/test_ci_stage_acceptance.py`, existing cleanup tests and runner
|
||||
tests. Transport fixtures exercise open/accept/finally-close and failure-close;
|
||||
they are not live acceptance. Parent must review the SSH sudo streaming Python
|
||||
execution boundary before activation. CI1454 was observed successful for the
|
||||
pinned SHA; that observation is also rechecked by the script before model work.
|
||||
@@ -1,11 +0,0 @@
|
||||
[Unit]
|
||||
Description=Retry exact owned temporary Stage principal cleanup
|
||||
After=docker.service
|
||||
Wants=docker.service
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
User=root
|
||||
UMask=0077
|
||||
ExecStart=/usr/bin/python3 /opt/services/canina/stage-principal-cleanup.py sweep
|
||||
TimeoutStartSec=120
|
||||
@@ -1,10 +0,0 @@
|
||||
[Unit]
|
||||
Description=Recover temporary Stage principal cleanup after reboot and failures
|
||||
|
||||
[Timer]
|
||||
OnBootSec=30s
|
||||
OnUnitInactiveSec=30s
|
||||
Unit=canina-stage-principal-cleanup.service
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
@@ -1,29 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# One bounded Stage acceptance only; never deploy, retry, reconcile or publish.
|
||||
set -euo pipefail
|
||||
set +x
|
||||
[[ $# == 0 && ${GITHUB_ACTIONS:-${GITEA_ACTIONS:-}} == true && ${BRANCH:-} == ci/v4-stage-acceptance-timeout-860324e && ${GITHUB_RUN_ATTEMPT:-} == 1 && ${DEPLOY_SHA:-} =~ ^[a-f0-9]{40}$ ]] || exit 2
|
||||
# Separate workflow source SHA from the immutable, already-deployed Stage artifact.
|
||||
export DEPLOY_SHA=860324e3a91b31c2b6d4e784e458f6d120582cb8
|
||||
export STAGE_TEST_APPROVAL=owner-authorized-stage-editorial
|
||||
export STAGE_TEST_SESSION=v4-ci-timeout-860324e-20261024
|
||||
export STAGE_TEST_TTL_SECONDS=600
|
||||
wrapper="$(dirname "${BASH_SOURCE[0]}")/ci-stage-test-principal.sh"
|
||||
close() {
|
||||
rc=$?
|
||||
trap - EXIT HUP INT TERM
|
||||
export STAGE_TEST_ACTION=close
|
||||
if ! bash "$wrapper"; then
|
||||
printf 'Stage close pending: durable 600-second cleanup intent retained\n' >&2
|
||||
rc=1
|
||||
fi
|
||||
exit "$rc"
|
||||
}
|
||||
# Install before open: failed/partial open must also attempt generation-fenced close.
|
||||
trap close EXIT
|
||||
trap 'exit 130' INT
|
||||
trap 'exit 143' TERM HUP
|
||||
export STAGE_TEST_ACTION=open
|
||||
bash "$wrapper"
|
||||
export STAGE_TEST_ACTION=accept
|
||||
bash "$wrapper"
|
||||
@@ -1,35 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# CI job side only. Never execute host deploy logic in the runner filesystem.
|
||||
set -euo pipefail
|
||||
set +x
|
||||
fail() { printf 'ERROR: %s\n' "$1" >&2; exit 2; }
|
||||
[[ ${GITHUB_ACTIONS:-${GITEA_ACTIONS:-}} == true && $# == 0 ]] || fail 'CI-only Stage operation'
|
||||
case ${BRANCH:-} in
|
||||
develop) ;;
|
||||
ci/v4-stage-acceptance-timeout-860324e)
|
||||
[[ ${DEPLOY_SHA:-} == 860324e3a91b31c2b6d4e784e458f6d120582cb8 && ${STAGE_TEST_SESSION:-} == v4-ci-timeout-860324e-20261024 && ${STAGE_TEST_TTL_SECONDS:-} == 600 ]] || fail 'fixed acceptance scope required'
|
||||
;;
|
||||
*) fail 'Stage branch required; Production is blocked' ;;
|
||||
esac
|
||||
[[ ${STAGE_TEST_ACTION:-} != open || ${BRANCH:-} != ci/v4-stage-acceptance-timeout-860324e || ${GITHUB_RUN_ATTEMPT:-} == 1 ]] || fail 'acceptance reruns blocked'
|
||||
[[ ${DEPLOY_SHA:-} =~ ^[0-9a-f]{40}$ ]] || fail 'immutable DEPLOY_SHA required'
|
||||
# Repository secret DEPLOY_KEY is the historical approved devops SSH credential.
|
||||
# Never fall back to baked-in runner keys, agent forwarding or docker.sock.
|
||||
[[ -n ${DEPLOY_KNOWN_HOSTS:-} ]] || fail 'provision DEPLOY_KNOWN_HOSTS: independently verified [87.248.133.138]:2234 host key; never use ssh-keyscan trust-on-first-use'
|
||||
[[ -n ${DEPLOY_KEY:-} ]] || fail 'provision existing approved DEPLOY_KEY for devops@87.248.133.138:2234'
|
||||
for tool in ssh ssh-keygen mktemp; do command -v "$tool" >/dev/null || fail "job context missing $tool"; done
|
||||
umask 077
|
||||
scratch=$(mktemp -d)
|
||||
trap 'rm -rf "$scratch"' EXIT
|
||||
printf '%s\n' "$DEPLOY_KNOWN_HOSTS" > "$scratch/known_hosts"
|
||||
printf '%s\n' "$DEPLOY_KEY" > "$scratch/key"
|
||||
unset DEPLOY_KEY DEPLOY_KNOWN_HOSTS
|
||||
ssh-keygen -F '[87.248.133.138]:2234' -f "$scratch/known_hosts" >/dev/null || fail 'DEPLOY_KNOWN_HOSTS has no pinned deployment endpoint'
|
||||
[[ ${STAGE_TEST_SESSION:-} =~ ^[a-z0-9-]{8,64}$ && ${STAGE_TEST_TTL_SECONDS:-} =~ ^[0-9]+$ && ${STAGE_TEST_ACTION:-} =~ ^(open|close|accept)$ && ${STAGE_TEST_APPROVAL:-} == owner-authorized-stage-editorial ]] || fail 'explicit owner Stage test grant required'
|
||||
[[ $STAGE_TEST_TTL_SECONDS -ge 60 && $STAGE_TEST_TTL_SECONDS -le 3600 ]] || exit 2
|
||||
if [[ $STAGE_TEST_ACTION == accept ]]; then
|
||||
[[ $BRANCH == ci/v4-stage-acceptance-timeout-860324e && $GITHUB_RUN_ATTEMPT == 1 ]] || fail 'acceptance scope required'
|
||||
ssh -F /dev/null -T -p 2234 -i "$scratch/key" -o BatchMode=yes -o IdentitiesOnly=yes -o IdentityAgent=none -o StrictHostKeyChecking=yes -o UserKnownHostsFile="$scratch/known_hosts" -o GlobalKnownHostsFile=/dev/null -o ConnectTimeout=15 devops@87.248.133.138 "sudo -n timeout --signal=TERM --kill-after=10 480 python3 -" < "$(dirname "${BASH_SOURCE[0]}")/stage-editorial-acceptance.py"
|
||||
exit $?
|
||||
fi
|
||||
ssh -F /dev/null -T -p 2234 -i "$scratch/key" -o BatchMode=yes -o IdentitiesOnly=yes -o IdentityAgent=none -o StrictHostKeyChecking=yes -o UserKnownHostsFile="$scratch/known_hosts" -o GlobalKnownHostsFile=/dev/null -o ConnectTimeout=15 devops@87.248.133.138 "sudo -n env CI=true STAGE_TEST_APPROVAL=owner-authorized-stage-editorial bash /opt/services/canina/releases/$DEPLOY_SHA/scripts/stage-test-principal-host.sh $DEPLOY_SHA $STAGE_TEST_SESSION $STAGE_TEST_TTL_SECONDS $STAGE_TEST_ACTION"
|
||||
@@ -88,8 +88,6 @@ for key, name in expected.items():
|
||||
if config['networks']['traefik_public'].get('name') != 'traefik_public' or not config['networks']['traefik_public'].get('external'):
|
||||
raise SystemExit('ERROR: Stage network identity mismatch')
|
||||
backend = config['services']['backend_stage']
|
||||
import os
|
||||
backend['build'].setdefault('args', {})['STAGE_TEST_ARTIFACT_SHA'] = os.environ['DEPLOY_SHA']
|
||||
if backend['container_name'] != 'canino_backend_stage' or config['services']['frontend_stage']['container_name'] != 'canino_frontend_stage':
|
||||
raise SystemExit('ERROR: Stage container identity mismatch')
|
||||
mounts = backend.get('volumes', [])
|
||||
@@ -112,8 +110,5 @@ docker network inspect traefik_public >/dev/null
|
||||
# Builds only app images; DB/Redis and support-bot remain untouched.
|
||||
docker compose --progress plain build backend_stage frontend_stage
|
||||
source scripts/rollout-stage.sh
|
||||
# Install reboot-persistent cleanup from this immutable CI release before rollout.
|
||||
# This preserves the root-private intent journal and fails closed on timer failure.
|
||||
python3 scripts/install-stage-principal-cleanup.py
|
||||
stage_rollout
|
||||
echo 'Deployment for develop completed successfully!'
|
||||
@@ -1,58 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Install the reviewed retry worker only through the guarded host CI deploy."""
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import tempfile
|
||||
|
||||
|
||||
def install(root, source, run):
|
||||
base = root / 'opt/services/canina'
|
||||
units = root / 'etc/systemd/system'
|
||||
journal = base / 'stage-principal-cleanup'
|
||||
base.mkdir(parents=True, exist_ok=True)
|
||||
units.mkdir(parents=True, exist_ok=True)
|
||||
if journal.is_symlink():
|
||||
raise ValueError('Unsafe journal')
|
||||
journal.mkdir(mode=0o700, exist_ok=True)
|
||||
st = journal.lstat()
|
||||
if not journal.is_dir() or st.st_uid != os.getuid() or st.st_mode & 0o777 != 0o700:
|
||||
raise ValueError('Unsafe journal')
|
||||
# Never touch existing intents. Atomic replacement avoids truncated workers.
|
||||
payloads = [('stage-principal-cleanup.py', base, 0o755)] + [
|
||||
('canina-stage-principal-cleanup.' + suffix, units, 0o644)
|
||||
for suffix in ('service', 'timer')]
|
||||
for name, dest, mode in payloads:
|
||||
fd, temporary = tempfile.mkstemp(prefix='.cleanup-install-', dir=dest)
|
||||
try:
|
||||
with os.fdopen(fd, 'wb') as output:
|
||||
output.write((source / name).read_bytes())
|
||||
os.fchmod(output.fileno(), mode)
|
||||
output.flush()
|
||||
os.fsync(output.fileno())
|
||||
os.replace(temporary, dest / name)
|
||||
directory = os.open(dest, os.O_RDONLY | os.O_DIRECTORY)
|
||||
try:
|
||||
os.fsync(directory)
|
||||
finally:
|
||||
os.close(directory)
|
||||
finally:
|
||||
if os.path.exists(temporary):
|
||||
os.unlink(temporary)
|
||||
for command in [
|
||||
['systemctl', 'daemon-reload'],
|
||||
['systemctl', 'enable', '--now', 'canina-stage-principal-cleanup.timer'],
|
||||
['systemctl', 'is-enabled', '--quiet', 'canina-stage-principal-cleanup.timer'],
|
||||
['systemctl', 'is-active', '--quiet', 'canina-stage-principal-cleanup.timer'],
|
||||
]:
|
||||
run(command)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
if (os.getuid() != 0 or Path('/.dockerenv').exists()
|
||||
or Path('/run/.containerenv').exists()
|
||||
or os.environ.get('GITHUB_ACTIONS', os.environ.get('GITEA_ACTIONS')) != 'true'):
|
||||
raise SystemExit('Guarded host CI deployment required')
|
||||
os.umask(0o077)
|
||||
install(Path('/'), Path(__file__).resolve().parent,
|
||||
lambda command: subprocess.run(command, check=True, timeout=60))
|
||||
@@ -1,115 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Fixed owner-authorized Stage acceptance. No automatic retry or cost reconciliation."""
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import urllib.request
|
||||
import uuid
|
||||
|
||||
SHA='860324e3a91b31c2b6d4e784e458f6d120582cb8'
|
||||
SESSION='v4-ci-timeout-860324e-20261024'
|
||||
REQUEST='6e82f843-45e4-49e3-9164-11a6879e39ba'
|
||||
|
||||
def proposal(prior):
|
||||
if prior.get('state')!='FAILED' or prior.get('version')!=5: raise ValueError('Prior must remain FAILED v5')
|
||||
scope=prior['metadata']['scope']
|
||||
prefix='editorial-research-to-brief-v1\nTopic: '
|
||||
suffix='\nResearch and evidence-backed brief only. No publication, release, production changes or execution authorization.'
|
||||
if not scope.startswith(prefix) or not scope.endswith(suffix): raise ValueError('Exact prior scope required')
|
||||
return dict(requestId=REQUEST,pilotKind='editorial-research-to-brief-v1',operationMode='PREPARE',environment='staging',scope='EDITORIAL_RESEARCH_TO_BRIEF',researchTopic=scope[len(prefix):-len(suffix)],acceptanceCriteria=prior['metadata']['acceptanceCriteria'],commitSha=SHA,budgetLimitCents=20)
|
||||
|
||||
def command(args,timeout=15):
|
||||
return subprocess.check_output(args,stderr=subprocess.DEVNULL,timeout=timeout)
|
||||
|
||||
def http(base,token,route,body=None,method=None):
|
||||
req=urllib.request.Request(base+'/'+route,data=None if body is None else json.dumps(body).encode(),headers={'Authorization':'Bearer '+token,'Content-Type':'application/json'},method=method or ('GET' if body is None else 'POST'))
|
||||
opener=urllib.request.build_opener(urllib.request.ProxyHandler({}),NoRedirect())
|
||||
with opener.open(req,timeout=10) as r:
|
||||
raw=r.read(262145)
|
||||
if len(raw)>262144: raise ValueError('Response bound')
|
||||
return json.loads(raw)
|
||||
|
||||
class NoRedirect(urllib.request.HTTPRedirectHandler):
|
||||
def redirect_request(self,req,fp,code,msg,headers,newurl): return None
|
||||
|
||||
def ci_success(log):
|
||||
return SHA in log and 'Deployment for develop completed successfully!' in log and '🏁 Job succeeded' in log
|
||||
|
||||
def verify_artifact(status,task):
|
||||
artifact=status.get('artifact')
|
||||
if status['state']!='VERIFYING' or not artifact: raise ValueError('Acceptance incomplete')
|
||||
if hashlib.sha256(artifact['bytes'].encode()).hexdigest()!=artifact['sha256']: raise ValueError('Digest mismatch')
|
||||
binding=json.loads(artifact['bytes'])
|
||||
if binding['taskId']!=task or binding['commitSha']!=SHA or status['qaStatus']!='PENDING_INDEPENDENT_QA': raise ValueError('Artifact binding mismatch')
|
||||
return artifact
|
||||
|
||||
def main():
|
||||
if os.getuid()!=0 or Path('/.dockerenv').exists(): raise ValueError('Host only')
|
||||
logs=Path('/opt/services/git/gitea/gitea/gitea/actions_log/parsa/canina')
|
||||
if not any(ci_success(p.read_text()) for p in logs.glob('*/*.log')): raise ValueError('Exact deployed SHA CI success required')
|
||||
actual=command(['docker','exec','canino_backend_stage','cat','/app/STAGE_TEST_ARTIFACT_SHA']).decode().strip()
|
||||
deployed=json.loads(command(['docker','exec','canina-hermes','cat','/opt/data/editorial-v4/deployment.json']))
|
||||
if actual!=SHA or deployed.get('commitSha')!=SHA: raise ValueError('Deployment mismatch')
|
||||
# Strict private backend IP; no public bootstrap endpoint or direct DB writes.
|
||||
networks=json.loads(command(['docker','inspect','--format','{{json .NetworkSettings.Networks}}','canino_backend_stage']))
|
||||
ip=networks['stage_hermes_inventory_backend']['IPAddress']
|
||||
import ipaddress
|
||||
if not ipaddress.ip_address(ip).is_private: raise ValueError('Private backend required')
|
||||
base='http://'+ip+':3000/api'
|
||||
handle=json.loads(command(['docker','exec','canino_backend_stage','cat','/app/private-stage-test/'+SESSION+'.json']))
|
||||
if handle['user']['role']!='SUPER_ADMIN' or handle['user']['id']=='pmo': raise ValueError('Independent reviewer required')
|
||||
admin=handle['accessToken']; del handle
|
||||
worker=command(['docker','exec','--user','hermes','canina-hermes','cat','/opt/data/editorial-v4/credentials/editorial-pmo-token']).decode().strip()
|
||||
a=lambda route,body=None,method=None:http(base,admin,'admin/orchestration/'+route,body,method)
|
||||
w=lambda route,body=None:http(base,worker,'internal/prepare-worker/'+route,body)
|
||||
tasks=a('tasks')
|
||||
prior=[t for t in tasks if t['id'].startswith('4a109')]
|
||||
if len(prior)!=1: raise ValueError('Exact prior task missing')
|
||||
prior=a('tasks/'+prior[0]['id'])
|
||||
# Preserve all old unknown costs and fence previous failed attempts.
|
||||
failed=[t for t in tasks if t.get('key','').startswith('editorial-v1:') and t['state']=='FAILED']
|
||||
frozen={t['id']:a('tasks/'+t['id'])['budgetLedger'] for t in failed}
|
||||
unknown=sum(sum(x['amountCents'] for x in ledger) for ledger in frozen.values())
|
||||
if unknown!=60: raise ValueError('Prior unknown reservation must be 60')
|
||||
p=w('editorial-proposals',proposal(prior))
|
||||
if p['state']!='GATE1_PENDING' or p['version']!=2: raise ValueError('Fresh proposal only; never replay execution')
|
||||
task=p['taskId']; route='tasks/'+task
|
||||
approved=a(route+'/gates/1/decide',{'decision':'APPROVED','expectedVersion':p['version'],'idempotencyKey':SESSION+':gate1','expiresAt':time.strftime('%Y-%m-%dT%H:%M:%SZ',time.gmtime(time.time()+450)),'notes':'Explicit owner-authorized bounded Stage acceptance only; no QA/publication/Production authority.'})
|
||||
if approved['state']!='GATE1_APPROVED': raise ValueError('Gate1 required')
|
||||
ready=w('editorial-queue',{'taskId':task,'expectedVersion':approved['version'],'idempotencyKey':SESSION+':queue'})
|
||||
if ready['state']!='READY': raise ValueError('READY required')
|
||||
# Exactly one process, as hermes. Installed runner has 120-second profile cap,
|
||||
# CLI headroom30 and allowlisted timing evidence. Total330 fits window480.
|
||||
code="import sys,json;sys.path.insert(0,'/opt/data/editorial-v4/runtime-package');import runner;from pathlib import Path;runner.run(Path('/opt/data'),sys.argv[1],total_seconds=330)"
|
||||
try:
|
||||
rc=subprocess.run(['docker','exec','--user','hermes','canina-hermes','python3','-c',code,task],stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL,timeout=430).returncode
|
||||
except subprocess.TimeoutExpired:
|
||||
# No second runner; read persisted state and fence only after its lease expires.
|
||||
rc=1
|
||||
status=w('editorial-proposals/'+task)
|
||||
if status['state']=='RUNNING':
|
||||
# Let last heartbeat lease expire, then fence the artifact-free unknown attempt.
|
||||
time.sleep(31)
|
||||
row=a(route)
|
||||
runs=[r for r in row['runs'] if r['status']=='RUNNING']
|
||||
if len(runs)!=1: raise ValueError('Ambiguous recovery')
|
||||
a(route+'/editorial-failure',{'runId':runs[0]['id'],'expectedVersion':row['version'],'idempotencyKey':SESSION+':stopped','reason':'Bounded acceptance stopped; cost unknown and reservation retained; no retry.'})
|
||||
status=w('editorial-proposals/'+task)
|
||||
for old,ledger in frozen.items():
|
||||
row=a('tasks/'+old)
|
||||
if row['state']!='FAILED' or row['budgetLedger']!=ledger: raise ValueError('Prior fence changed')
|
||||
row=a(route)
|
||||
if sum(x['amountCents'] for x in row['budgetLedger'])!=20 or any(x['kind']=='RECONCILE' for x in row['budgetLedger']): raise ValueError('Reservation changed')
|
||||
if rc: raise ValueError('Acceptance incomplete; fenced unknown20')
|
||||
artifact=verify_artifact(status,task)
|
||||
print(json.dumps({'taskId':task,'state':status['state'],'artifactSha256':artifact['sha256'],'qaStatus':status['qaStatus'],'priorUnknownReservedCents':60,'freshReservedCents':20,'commitSha':SHA}))
|
||||
|
||||
if __name__=='__main__':
|
||||
try: main()
|
||||
except Exception:
|
||||
print('Stage acceptance stopped; no retry or cost reconciliation; read authenticated status',file=sys.stderr)
|
||||
sys.exit(1)
|
||||
@@ -1,100 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Root-only Stage principal intent journal. No generic command/container inputs."""
|
||||
import fcntl
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import uuid
|
||||
|
||||
ROOT = Path('/opt/services/canina/stage-principal-cleanup')
|
||||
APPROVAL = 'owner-authorized-stage-editorial'
|
||||
|
||||
def validate(i):
|
||||
if set(i) != {'sha','session','ttl','generation','due'} or not re.fullmatch('[a-f0-9]{40}', i['sha']) or not re.fullmatch('[a-z0-9-]{8,64}', i['session']) or not re.fullmatch('[a-f0-9-]{36}', i['generation']) or not isinstance(i['ttl'],int) or not 60 <= i['ttl'] <= 3600 or not isinstance(i['due'],int):
|
||||
raise ValueError('Invalid cleanup intent')
|
||||
|
||||
class Worker:
|
||||
def __init__(self, root, run):
|
||||
self.root, self.run = root, run
|
||||
def save(self, intent):
|
||||
validate(intent)
|
||||
p = self.root / (intent['session']+'.json')
|
||||
# Exclusive creation forbids reusing a session or extending its deadline.
|
||||
fd = os.open(p, os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_NOFOLLOW, 0o600)
|
||||
with os.fdopen(fd,'w') as f:
|
||||
json.dump(intent,f); f.flush(); os.fsync(f.fileno())
|
||||
self.sync()
|
||||
def sync(self):
|
||||
fd=os.open(self.root,os.O_RDONLY|os.O_DIRECTORY)
|
||||
try: os.fsync(fd)
|
||||
finally: os.close(fd)
|
||||
def sweep(self, now):
|
||||
with open(self.root/'lock','a') as lock:
|
||||
try: fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
|
||||
except BlockingIOError: return
|
||||
for p in sorted(self.root.glob('*.json')):
|
||||
try:
|
||||
fd=os.open(p,os.O_RDONLY|os.O_NOFOLLOW)
|
||||
with os.fdopen(fd) as f: i=json.load(f)
|
||||
validate(i)
|
||||
if p.name != i['session']+'.json': raise ValueError('Intent name mismatch')
|
||||
if i['due'] <= now and self.run(i,'close'):
|
||||
p.unlink(); self.sync()
|
||||
except Exception:
|
||||
# Keep intent through DB/container/redeploy failures; next tick retries.
|
||||
print('Stage cleanup pending',file=sys.stderr)
|
||||
|
||||
def run(i, action):
|
||||
# Opens remain SHA pinned; cleanup only uses the CURRENT artifact's narrow CLI.
|
||||
# Original ownership is the session plus unpredictable account generation.
|
||||
container=subprocess.check_output(['docker','inspect','--format','{{.Id}}','canino_backend_stage'],text=True,timeout=30).strip()
|
||||
if not re.fullmatch('[a-f0-9]{64}',container): return False
|
||||
sha=subprocess.check_output(['docker','exec',container,'cat','/app/STAGE_TEST_ARTIFACT_SHA'],text=True,timeout=30).strip()
|
||||
if not re.fullmatch('[a-f0-9]{40}',sha) or (action=='open' and sha!=i['sha']):
|
||||
return False
|
||||
env={'APP_ENV':'stage','CI':'true','STAGE_TEST_PRINCIPAL_ENABLED':'true','STAGE_TEST_APPROVAL':APPROVAL,'STAGE_TEST_SHA':sha,'STAGE_TEST_SESSION':i['session'],'STAGE_TEST_TTL_SECONDS':str(i['ttl']),'STAGE_TEST_GENERATION':i['generation']}
|
||||
cmd=['docker','exec']
|
||||
for k,v in env.items(): cmd+=['-e',k+'='+v]
|
||||
cli=cmd+[container,'node','dist/auth/stage-test-principal.cli.js']
|
||||
# Never send a generation-fenced cleanup to an older CLI that ignores fences.
|
||||
capability=subprocess.check_output(cli+['cleanup-capability'],text=True,timeout=30)
|
||||
if capability!='stage-generation-cleanup-v1': return False
|
||||
return subprocess.run(cli+[action],timeout=90).returncode==0
|
||||
|
||||
def main():
|
||||
if os.getuid()!=0 or Path('/.dockerenv').exists(): raise ValueError('Host root only')
|
||||
os.umask(0o077)
|
||||
ROOT.mkdir(mode=0o700,exist_ok=True)
|
||||
st=ROOT.lstat()
|
||||
if ROOT.is_symlink() or st.st_uid!=0 or st.st_mode & 0o777 != 0o700: raise ValueError('Unsafe journal')
|
||||
worker=Worker(ROOT,run)
|
||||
if sys.argv[1:]==['sweep']:
|
||||
worker.sweep(int(time.time())); return
|
||||
if len(sys.argv)!=5 or os.environ.get('CI')!='true' or os.environ.get('STAGE_TEST_APPROVAL')!=APPROVAL: raise ValueError('CI grant required')
|
||||
sha,session,ttl,action=sys.argv[1:]
|
||||
i={'sha':sha,'session':session,'ttl':int(ttl),'generation':str(uuid.uuid4()),'due':int(time.time())+int(ttl)}
|
||||
validate(i)
|
||||
if action not in ('open','close'): raise ValueError('Action denied')
|
||||
with open(ROOT/'lock','a') as lock:
|
||||
fcntl.flock(lock,fcntl.LOCK_EX) # contention waits, never discards intent
|
||||
if action=='open':
|
||||
# Repeated opens cannot mint a new generation or renew TTL.
|
||||
worker.save(i)
|
||||
# Journal is durable before the first possible DB mutation.
|
||||
if not run(i,'open'):
|
||||
raise ValueError('Open failed; cleanup remains pending')
|
||||
else:
|
||||
p=ROOT/(session+'.json')
|
||||
fd=os.open(p,os.O_RDONLY|os.O_NOFOLLOW)
|
||||
with os.fdopen(fd) as f: i=json.load(f)
|
||||
validate(i)
|
||||
if run(i,'close'): p.unlink(); worker.sync()
|
||||
else: raise ValueError('Cleanup pending')
|
||||
if __name__=='__main__':
|
||||
try: main()
|
||||
except Exception:
|
||||
print('Stage principal denied or pending cleanup',file=sys.stderr); sys.exit(1)
|
||||
@@ -1,12 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# CI-only authority: fixed Stage cleanup worker, no dynamic executable/container.
|
||||
set -euo pipefail
|
||||
set +x
|
||||
[[ $# == 4 && $1 =~ ^[0-9a-f]{40}$ && $2 =~ ^[a-z0-9-]{8,64}$ && $3 =~ ^[0-9]+$ && $4 =~ ^(open|close)$ ]] || exit 2
|
||||
[[ $3 -ge 60 && $3 -le 3600 && ${STAGE_TEST_APPROVAL:-} == owner-authorized-stage-editorial && ${CI:-} == true ]] || exit 2
|
||||
[[ ! -e /.dockerenv && $(id -u) == 0 ]] || exit 2
|
||||
# Provisioned through reviewed deployment, not through the credential CI command.
|
||||
# Fail before any DB mutation unless reboot-persistent retry recovery is enabled.
|
||||
systemctl is-enabled --quiet canina-stage-principal-cleanup.timer
|
||||
systemctl is-active --quiet canina-stage-principal-cleanup.timer
|
||||
exec /usr/bin/python3 /opt/services/canina/stage-principal-cleanup.py "$@"
|
||||
@@ -1,10 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# Run ONLY inside a disposable fixture container with Postgres/Redis test network.
|
||||
set -euo pipefail
|
||||
[[ ${STAGE_PRINCIPAL_DISPOSABLE_TEST:-} == true && ${DATABASE_URL:-} == postgresql://postgres@principal-durable-pg:5432/principal_disposable ]] || exit 2
|
||||
python3 -c 'from pathlib import Path; s=Path("prisma/schema.prisma").read_text(); Path("prisma/pre-stage.prisma").write_text("\n".join(x for x in s.splitlines() if not any(k in x for k in ["authExpiresAt","stageTestSession","stageTestGeneration"])))'
|
||||
node node_modules/prisma/build/index.js db push --schema prisma/pre-stage.prisma --force-reset --skip-generate
|
||||
# Executes exact additive SQL, not a final-schema db push substitute.
|
||||
node node_modules/prisma/build/index.js db execute --schema prisma/schema.prisma --file prisma/migrations/20261024090000_stage_test_principal/migration.sql
|
||||
node node_modules/prisma/build/index.js generate
|
||||
node node_modules/jest/bin/jest.js principal --runInBand --testTimeout=240000
|
||||
@@ -1,71 +0,0 @@
|
||||
import os
|
||||
import pathlib
|
||||
import subprocess
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
ROOT = pathlib.Path(__file__).resolve().parent
|
||||
SHA = '860324e3a91b31c2b6d4e784e458f6d120582cb8'
|
||||
|
||||
class AcceptanceTests(unittest.TestCase):
|
||||
def run_driver(self, branch='ci/v4-stage-acceptance-timeout-860324e', fail_open=False, attempt='1', fail_accept=False):
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
p = pathlib.Path(d)
|
||||
(p/'ssh').write_text('#!/bin/bash\nprintf "%s\\n" "${@: -1}" >> "$CALLS"\n[[ "$FAIL_OPEN" != true || "${@: -1}" != *" open" ]] || exit 1\n[[ "$FAIL_ACCEPT" != true || "${@: -1}" != *"480 python3 -" ]]\n')
|
||||
(p/'ssh-keygen').write_text('#!/bin/bash\nexit 0\n')
|
||||
(p/'sleep').write_text('#!/bin/bash\n[[ "$1" == 480 ]]\n')
|
||||
for name in ('ssh','ssh-keygen','sleep'):
|
||||
(p/name).chmod(0o755)
|
||||
env = dict(os.environ, PATH=d+':'+os.environ['PATH'], GITHUB_ACTIONS='true', BRANCH=branch,
|
||||
GITHUB_RUN_ATTEMPT=attempt, DEPLOY_SHA='a'*40, DEPLOY_KEY='NEVER-PRINT-KEY',
|
||||
DEPLOY_KNOWN_HOSTS='test-host-pin', CALLS=str(p/'calls'),
|
||||
FAIL_OPEN=str(fail_open).lower(), FAIL_ACCEPT=str(fail_accept).lower())
|
||||
result = subprocess.run(['bash',str(ROOT/'ci-stage-acceptance.sh')],env=env,capture_output=True,text=True)
|
||||
calls = (p/'calls').read_text().splitlines() if (p/'calls').exists() else []
|
||||
self.assertNotIn('NEVER-PRINT-KEY',result.stdout+result.stderr)
|
||||
return result,calls
|
||||
|
||||
def test_fixed_stage_open_then_finally_close(self):
|
||||
result,calls = self.run_driver()
|
||||
self.assertEqual(result.returncode,0,result.stderr)
|
||||
prefix = 'sudo -n env CI=true STAGE_TEST_APPROVAL=owner-authorized-stage-editorial bash /opt/services/canina/releases/'+SHA+'/scripts/stage-test-principal-host.sh '+SHA+' v4-ci-timeout-860324e-20261024 600 '
|
||||
self.assertEqual(calls[0],prefix+'open')
|
||||
self.assertIn('timeout --signal=TERM --kill-after=10 480 python3 -',calls[1])
|
||||
self.assertEqual(calls[2],prefix+'close')
|
||||
|
||||
def test_failed_acceptance_still_closes(self):
|
||||
result,calls = self.run_driver(fail_accept=True)
|
||||
self.assertNotEqual(result.returncode,0)
|
||||
self.assertEqual(len(calls),3)
|
||||
self.assertTrue(calls[-1].endswith(' close'))
|
||||
|
||||
def test_failed_open_still_closes(self):
|
||||
result,calls = self.run_driver(fail_open=True)
|
||||
self.assertNotEqual(result.returncode,0)
|
||||
self.assertEqual(len(calls),2)
|
||||
self.assertTrue(calls[0].endswith(' open'))
|
||||
self.assertTrue(calls[1].endswith(' close'))
|
||||
|
||||
def test_wrong_branch_and_rerun_have_no_host_effect(self):
|
||||
for branch in ('main','develop','ci/other'):
|
||||
result,calls = self.run_driver(branch=branch)
|
||||
self.assertNotEqual(result.returncode,0)
|
||||
self.assertEqual(calls,[])
|
||||
result,calls = self.run_driver(attempt='2')
|
||||
self.assertNotEqual(result.returncode,0)
|
||||
self.assertEqual(calls,[])
|
||||
|
||||
def test_transport_scope_and_pinned_ssh_contract(self):
|
||||
text = (ROOT/'ci-stage-test-principal.sh').read_text()
|
||||
for value in ('StrictHostKeyChecking=yes','IdentitiesOnly=yes','IdentityAgent=none','GlobalKnownHostsFile=/dev/null'):
|
||||
self.assertIn(value,text)
|
||||
for branch,sha,session,ttl in [('main',SHA,'v4-ci-timeout-860324e-20261024','600'),
|
||||
('ci/v4-stage-acceptance-timeout-860324e','b'*40,'v4-ci-timeout-860324e-20261024','600'),
|
||||
('ci/v4-stage-acceptance-timeout-860324e',SHA,'different-session','600'),
|
||||
('ci/v4-stage-acceptance-timeout-860324e',SHA,'v4-ci-timeout-860324e-20261024','601')]:
|
||||
env = dict(os.environ,GITHUB_ACTIONS='true',BRANCH=branch,DEPLOY_SHA=sha,
|
||||
STAGE_TEST_SESSION=session,STAGE_TEST_TTL_SECONDS=ttl,STAGE_TEST_ACTION='open')
|
||||
result = subprocess.run(['bash',str(ROOT/'ci-stage-test-principal.sh')],env=env,capture_output=True)
|
||||
self.assertNotEqual(result.returncode,0)
|
||||
|
||||
if __name__ == '__main__': unittest.main()
|
||||
@@ -1,39 +0,0 @@
|
||||
import importlib.util
|
||||
from pathlib import Path
|
||||
import unittest
|
||||
|
||||
p=Path(__file__).with_name('stage-editorial-acceptance.py')
|
||||
spec=importlib.util.spec_from_file_location('acceptance',p)
|
||||
assert spec is not None and spec.loader is not None
|
||||
m=importlib.util.module_from_spec(spec)
|
||||
if p.exists(): spec.loader.exec_module(m)
|
||||
|
||||
class AcceptanceTests(unittest.TestCase):
|
||||
def test_prior_contract_copied_exactly(self):
|
||||
prior={'state':'FAILED','version':5,'title':'Editorial research: Persian owner topic','metadata':{'scope':'editorial-research-to-brief-v1\nTopic: Persian owner topic\nResearch and evidence-backed brief only. No publication, release, production changes or execution authorization.','acceptanceCriteria':['Do not invent QA']}}
|
||||
self.assertTrue(hasattr(m,'proposal'), 'missing bounded proposal builder')
|
||||
body=m.proposal(prior)
|
||||
self.assertEqual(body['researchTopic'],'Persian owner topic')
|
||||
self.assertEqual(body['acceptanceCriteria'],prior['metadata']['acceptanceCriteria'])
|
||||
self.assertEqual(body['budgetLimitCents'],20)
|
||||
self.assertEqual(body['commitSha'],m.SHA)
|
||||
def test_wrong_prior_state_denied(self):
|
||||
self.assertTrue(hasattr(m,'proposal'), 'missing bounded proposal builder')
|
||||
with self.assertRaises(ValueError): m.proposal({'state':'READY'})
|
||||
|
||||
class ArtifactTests(unittest.TestCase):
|
||||
def test_digest_and_binding_must_match(self):
|
||||
self.assertTrue(hasattr(m,'verify_artifact'), 'missing artifact verifier')
|
||||
import json,hashlib
|
||||
raw=json.dumps({'taskId':'task','commitSha':m.SHA,'brief':{'candidates':[]}})
|
||||
status={'taskId':'task','state':'VERIFYING','qaStatus':'PENDING_INDEPENDENT_QA','artifact':{'bytes':raw,'sha256':hashlib.sha256(raw.encode()).hexdigest()}}
|
||||
m.verify_artifact(status,'task')
|
||||
status['artifact']['bytes']+=' '
|
||||
with self.assertRaises(ValueError): m.verify_artifact(status,'task')
|
||||
def test_ci_gate_requires_exact_sha_success(self):
|
||||
self.assertTrue(hasattr(m,'ci_success'), 'missing CI exact SHA gate')
|
||||
self.assertFalse(m.ci_success('Deployment for develop completed successfully!'))
|
||||
self.assertFalse(m.ci_success(m.SHA+' Job failed'))
|
||||
self.assertTrue(m.ci_success(m.SHA+' Deployment for develop completed successfully! 🏁 Job succeeded'))
|
||||
|
||||
if __name__=='__main__': unittest.main()
|
||||
@@ -1,45 +0,0 @@
|
||||
import importlib.util, pathlib, tempfile, unittest, json
|
||||
spec=importlib.util.spec_from_file_location('cleanup',pathlib.Path(__file__).with_name('stage-principal-cleanup.py'))
|
||||
m=importlib.util.module_from_spec(spec); spec.loader.exec_module(m)
|
||||
class CleanupTests(unittest.TestCase):
|
||||
def test_redeploy_restart_and_failed_delete_retry(self):
|
||||
with tempfile.TemporaryDirectory() as root:
|
||||
calls=[]
|
||||
def run(intent,action):
|
||||
calls.append((intent['generation'],action)); return len(calls)>1
|
||||
worker=m.Worker(pathlib.Path(root),run)
|
||||
intent={'sha':'a'*40,'session':'test-session','ttl':60,'generation':'8311eb41-23b4-4aab-ade2-e97b90debc11','due':0}
|
||||
worker.save(intent)
|
||||
worker.sweep(100)
|
||||
self.assertTrue((pathlib.Path(root)/'test-session.json').exists())
|
||||
m.Worker(pathlib.Path(root),run).sweep(200)
|
||||
self.assertFalse((pathlib.Path(root)/'test-session.json').exists())
|
||||
self.assertEqual(calls,[(intent['generation'],'close')]*2)
|
||||
def test_contention_preserves_intent(self):
|
||||
import fcntl
|
||||
with tempfile.TemporaryDirectory() as root:
|
||||
worker=m.Worker(pathlib.Path(root),lambda *_:self.fail('must not execute while locked'))
|
||||
intent={'sha':'a'*40,'session':'test-session','ttl':60,'generation':'8311eb41-23b4-4aab-ade2-e97b90debc11','due':0}
|
||||
worker.save(intent)
|
||||
with open(pathlib.Path(root)/'lock','w') as lock:
|
||||
fcntl.flock(lock,fcntl.LOCK_EX)
|
||||
worker.sweep(100)
|
||||
self.assertTrue((pathlib.Path(root)/'test-session.json').exists())
|
||||
def test_commands_follow_redeploy_but_keep_generation(self):
|
||||
from unittest.mock import patch
|
||||
from types import SimpleNamespace
|
||||
intent={'sha':'a'*40,'session':'test-session','ttl':60,'generation':'8311eb41-23b4-4aab-ade2-e97b90debc11','due':0}
|
||||
with patch.object(m.subprocess,'check_output',side_effect=['c'*64,'b'*40,'stage-generation-cleanup-v1']), patch.object(m.subprocess,'run',return_value=SimpleNamespace(returncode=0)) as execute:
|
||||
self.assertTrue(m.run(intent,'close'))
|
||||
cmd=execute.call_args.args[0]
|
||||
self.assertIn('STAGE_TEST_SHA='+'b'*40,cmd)
|
||||
self.assertIn('STAGE_TEST_GENERATION='+intent['generation'],cmd)
|
||||
self.assertIn('c'*64,cmd)
|
||||
def test_old_cli_and_redeploy_open_fail_closed(self):
|
||||
from unittest.mock import patch
|
||||
intent={'sha':'a'*40,'session':'test-session','ttl':60,'generation':'8311eb41-23b4-4aab-ade2-e97b90debc11','due':0}
|
||||
with patch.object(m.subprocess,'check_output',side_effect=['c'*64,'b'*40]), patch.object(m.subprocess,'run') as execute:
|
||||
self.assertFalse(m.run(intent,'open')); execute.assert_not_called()
|
||||
with patch.object(m.subprocess,'check_output',side_effect=['c'*64,'b'*40,'old']), patch.object(m.subprocess,'run') as execute:
|
||||
self.assertFalse(m.run(intent,'close')); execute.assert_not_called()
|
||||
if __name__=='__main__': unittest.main()
|
||||
@@ -1,39 +0,0 @@
|
||||
import pathlib, subprocess, tempfile, unittest, importlib.util
|
||||
HERE=pathlib.Path(__file__).resolve().parent
|
||||
class InstallTests(unittest.TestCase):
|
||||
def test_ci_wires_installer_before_rollout(self):
|
||||
deploy=(HERE/'deploy.sh').read_text()
|
||||
self.assertIn('python3 scripts/install-stage-principal-cleanup.py',deploy)
|
||||
self.assertLess(deploy.index('python3 scripts/install-stage-principal-cleanup.py'),deploy.index('stage_rollout\n'))
|
||||
def test_disposable_install_preserves_intents_and_enables_timer(self):
|
||||
spec=importlib.util.spec_from_file_location('installer',HERE/'install-stage-principal-cleanup.py')
|
||||
m=importlib.util.module_from_spec(spec);spec.loader.exec_module(m)
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root=pathlib.Path(tmp); journal=root/'opt/services/canina/stage-principal-cleanup'
|
||||
journal.mkdir(parents=True,mode=0o700); intent=journal/'owned.json';intent.write_text('pending')
|
||||
commands=[]
|
||||
m.install(root,HERE,lambda cmd:commands.append(cmd))
|
||||
m.install(root,HERE,lambda cmd:commands.append(cmd))
|
||||
self.assertEqual(intent.read_text(),'pending')
|
||||
self.assertEqual((root/'opt/services/canina/stage-principal-cleanup.py').read_bytes(),(HERE/'stage-principal-cleanup.py').read_bytes())
|
||||
for name in ('service','timer'):
|
||||
self.assertEqual((root/f'etc/systemd/system/canina-stage-principal-cleanup.{name}').stat().st_mode & 0o777,0o644)
|
||||
self.assertEqual(commands[:4],[['systemctl','daemon-reload'],['systemctl','enable','--now','canina-stage-principal-cleanup.timer'],['systemctl','is-enabled','--quiet','canina-stage-principal-cleanup.timer'],['systemctl','is-active','--quiet','canina-stage-principal-cleanup.timer']])
|
||||
def test_disposable_host_commands_and_failure_propagation(self):
|
||||
import os
|
||||
from unittest.mock import patch
|
||||
spec=importlib.util.spec_from_file_location('installer',HERE/'install-stage-principal-cleanup.py');m=importlib.util.module_from_spec(spec);spec.loader.exec_module(m)
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root=pathlib.Path(tmp);binary=root/'bin';binary.mkdir();log=root/'commands'
|
||||
ctl=binary/'systemctl';ctl.write_text('#!/bin/sh\nprintf "%s\\n" "$*" >> "$COMMAND_LOG"\n[ "$1" != "$FAIL_COMMAND" ]\n');ctl.chmod(0o755)
|
||||
with patch.dict(os.environ,{'PATH':str(binary)+':'+os.environ['PATH'],'COMMAND_LOG':str(log),'FAIL_COMMAND':'never'}):
|
||||
m.install(root,HERE,lambda cmd:subprocess.run(cmd,check=True))
|
||||
self.assertEqual(log.read_text().splitlines(),['daemon-reload','enable --now canina-stage-principal-cleanup.timer','is-enabled --quiet canina-stage-principal-cleanup.timer','is-active --quiet canina-stage-principal-cleanup.timer'])
|
||||
with patch.dict(os.environ,{'FAIL_COMMAND':'enable'}),self.assertRaises(subprocess.CalledProcessError):
|
||||
m.install(root,HERE,lambda cmd:subprocess.run(cmd,check=True))
|
||||
def test_unsafe_journal_fails_before_commands(self):
|
||||
spec=importlib.util.spec_from_file_location('installer',HERE/'install-stage-principal-cleanup.py');m=importlib.util.module_from_spec(spec);spec.loader.exec_module(m)
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root=pathlib.Path(tmp); journal=root/'opt/services/canina/stage-principal-cleanup';journal.mkdir(parents=True,mode=0o755)
|
||||
with self.assertRaises(ValueError):m.install(root,HERE,lambda _:self.fail('must not enable'))
|
||||
if __name__=='__main__': unittest.main()
|
||||
Reference in new issue
Block a user