Test completed recovery over JWT HTTP Postgres and narrowly admit Stage reviewer
Deploy Canina / deploy (push) Failing after 1m25s

This commit is contained in:
Recovery Review authored and devops committed 2026-10-24 17:35:56 +00:00
1 parent 65063e224a
commit 4413c2ef6c
5 files changed
+156 -2

No files matched your search

+1 -1
View File
@@ -66,7 +66,7 @@ export class JwtStrategy extends PassportStrategy(Strategy) {
if (user.stageTestSession) {
if (!user.stageTestGeneration || payload.stageTestGeneration !== user.stageTestGeneration || payload.stageTestSession !== user.stageTestSession) throw new UnauthorizedException("Stage session revoked");
const path = req.originalUrl?.split('?')[0] || '';
const allowed = req.method === 'GET' && /^\/api\/admin\/orchestration\/tasks(?:\/[a-f0-9-]+)?$/.test(path) || req.method === 'POST' && (/^\/api\/admin\/orchestration\/tasks$/.test(path) || /^\/api\/admin\/orchestration\/tasks\/[a-f0-9-]+\/(?:gates\/1\/decide|editorial-failure)$/.test(path));
const allowed = req.method === 'GET' && /^\/api\/admin\/orchestration\/tasks(?:\/[a-f0-9-]+)?$/.test(path) || req.method === 'POST' && (/^\/api\/admin\/orchestration\/tasks$/.test(path) || /^\/api\/admin\/orchestration\/tasks\/[a-f0-9-]+\/(?:gates\/1\/decide|editorial-failure|editorial-completed-recovery)$/.test(path));
if (!allowed) throw new UnauthorizedException('Stage test scope denied');
}
return user;
@@ -0,0 +1,106 @@
import { Test } from '@nestjs/testing';
import { INestApplication } from '@nestjs/common';
import { PrismaClient } from '@prisma/client';
import { createHash, randomUUID } from 'crypto';
import { sign } from 'jsonwebtoken';
import { PrismaService } from '../prisma/prisma.service';
import { PrepareWorkflowService } from '../orchestration/prepare-workflow.service';
import { OrchestrationController } from '../orchestration/orchestration.controller';
import { OrchestrationService } from '../orchestration/orchestration.service';
import { JwtStrategy } from '../auth/jwt.strategy';
import { UsersService } from '../users/users.service';
import { RedisService } from '../redis/redis.service';
import { WorkerTransportModule } from './worker-transport.module';
import { EditorialExecutionService } from './editorial-execution';
(process.env.EDITORIAL_EXECUTION_DB === '1' ? describe : describe.skip)('completed recovery real JWT HTTP/PostgreSQL (synthetic evidence only)', () => {
const db = new PrismaClient();
const sha = (s: string) => createHash('sha256').update(s).digest('hex');
const workerToken = 'disposable-pmo-'.repeat(5);
let app: INestApplication, base: string, workflow: PrepareWorkflowService;
let reviewer: any, admin: any, jwt: string, adminJwt: string;
const generation = randomUUID(), session = 'disposable-recovery-http';
const brief = () => ({researchDate:'2026-10-24',inputAvailability:[{input:'property',status:'UNAVAILABLE',details:'No authenticated property supplied'}],findings:[{sourceUrl:'https://www.who.int/news',sourceDate:'2026-10-20',finding:'Synthetic contract fixture, not research'}],candidates:[0,1,2].map(i=>({title:'Synthetic '+i,score:80-i,rationale:'Contract fixture only',sourceUrls:['https://www.who.int/news']}))});
const token = (u: any, extra: any = {}) => sign({sub:u.id,stageTestSession:session,stageTestGeneration:generation,...extra},process.env.JWT_SECRET!,{expiresIn:'5m'});
beforeAll(async () => {
const module = await Test.createTestingModule({
imports:[WorkerTransportModule.register({enabled:true,grants:[{id:'pmo',role:'WORKFLOW_WORKER',credentialSha256:sha(workerToken)}],workerIds:['pmo'],stageCiIds:[],artifactDirectory:'/work/tmp/recovery',artifactBaseUrl:'https://isolated.invalid/internal/prepare-worker/artifacts'})],
controllers:[OrchestrationController],
providers:[JwtStrategy,EditorialExecutionService,{provide:OrchestrationService,useValue:{}},{provide:UsersService,useValue:{findById:(id:string)=>db.user.findUnique({where:{id}})}},{provide:RedisService,useValue:{get:async()=>null}}],
}).overrideProvider(PrismaService).useValue(db).compile();
app=module.createNestApplication(); app.setGlobalPrefix('api'); await app.listen(0,'127.0.0.1'); base=await app.getUrl(); workflow=app.get(PrepareWorkflowService);
reviewer=await db.user.create({data:{firstName:'Disposable',lastName:'Reviewer',mobile:'fixture-'+randomUUID().slice(0,7),role:'SUPER_ADMIN',stageTestSession:session,stageTestGeneration:generation,authExpiresAt:new Date(Date.now()+300000)}});
admin=await db.user.create({data:{firstName:'Disposable',lastName:'Admin',mobile:'fixture-'+randomUUID().slice(0,7),role:'Admin',stageTestSession:session,stageTestGeneration:generation,authExpiresAt:new Date(Date.now()+300000)}});
jwt=token(reviewer); adminJwt=token(admin);
});
afterAll(async()=>{await app?.close();await db.$disconnect();});
async function request(path:string,body:any,secret=jwt) {const r=await fetch(base+'/api/'+path,{method:'POST',headers:{authorization:'Bearer '+secret,'content-type':'application/json'},body:JSON.stringify(body)});return {status:r.status,body:await r.json()};}
async function fixture(ttl = 60000) {
const p=await request('internal/prepare-worker/editorial-proposals',{requestId:randomUUID(),pilotKind:'editorial-research-to-brief-v1',operationMode:'PREPARE',environment:'staging',scope:'EDITORIAL_RESEARCH_TO_BRIEF',researchTopic:'Disposable recovery contract',acceptanceCriteria:['Dated evidence'],commitSha:'a'.repeat(40),budgetLimitCents:20},workerToken);
expect(p.status).toBe(201);
let t=await db.orchestrationTask.findUniqueOrThrow({where:{id:p.body.taskId}});
t=await workflow.decide(t.id,1,{decision:'APPROVED',expiresAt:new Date(Date.now()+ttl).toISOString(),expectedVersion:t.version,idempotencyKey:randomUUID()},{id:'independent-approver',role:'SUPER_ADMIN'});
const c=await request('internal/prepare-worker/editorial-claim',{taskId:t.id},workerToken);expect(c.status).toBe(201);
const l=c.body.lease;
await db.prepareWorkflowOutbox.update({where:{taskId:t.id},data:{leaseUntil:new Date(Date.now()-1000)}});
const original=brief(); original.inputAvailability[0].status='missing';
const originalText=JSON.stringify(original);
const evidence:any={taskId:t.id,runId:l.runId,expectedVersion:t.version+1,commitSha:'a'.repeat(40),originalText,sessions:['canina-competitor-worker','canina-content-worker','canina-seo-worker','pmo'].map((profile,i)=>({profile,sessionId:'20261024_120000_aaaaa'+i,outputSha256:i===3?sha(originalText):sha('synthetic '+i),outcome:'COMPLETED'})),outputs:[{retrievalEvents:[{type:'tool_use',name:'web_extract',tool_call_id:'fixture-extract',input:{urls:['https://www.who.int/news']}},{type:'tool_result',name:'web_extract',tool_call_id:'fixture-extract',output:JSON.stringify({results:[{url:'https://www.who.int/news',content:'Synthetic retrieved bytes'}]})}]}],brief:brief(),correction:{mode:'STORED_REPORTS_ONLY',statusChanges:[{index:0,from:'missing',to:'UNAVAILABLE',reason:'Stored text explicitly says no authenticated property supplied'}],omissions:[],reason:'Synthetic stored-reports-only schema correction'}};
return {t,l,evidence,path:'admin/orchestration/tasks/'+t.id+'/editorial-completed-recovery'};
}
const payload=(e:any)=>({runId:e.runId,expectedVersion:e.expectedVersion,evidenceSha256:sha(JSON.stringify(e)),evidence:e});
it('allows narrowly scoped temporary reviewer, atomically ingests same run once, retains cost and pending QA',async()=>{
const {t,l,evidence,path}=await fixture(); const body=payload(evidence);
expect((await request(path,body,adminJwt)).status).toBe(403);
expect((await request(path,body,token(reviewer,{stageTestGeneration:randomUUID()}))).status).toBe(401);
expect((await request(path,body,'not-a-jwt')).status).toBe(401);
expect((await request(path,{...body,actualCents:0})).status).toBe(400);
expect((await request('admin/orchestration/tasks/'+t.id+'/budget/reconcile',{actualCents:0})).status).toBe(401);
const results=await Promise.all([request(path,body),request(path,body)]);
expect(results.map(x=>x.status).sort()).toEqual([201,403]);
expect(results.find(x=>x.status===201)!.body).toMatchObject({state:'VERIFYING',qaStatus:'PENDING_INDEPENDENT_QA',retainedReservationCents:20,costStatus:'upper-bound-reserved'});
const artifacts=await db.orchestrationArtifact.findMany({where:{runId:l.runId}});expect(artifacts).toHaveLength(2);
const stored=artifacts.find(x=>x.kind==='EDITORIAL_RECOVERY_EVIDENCE')!;expect((stored.metadata as any).bytes).toBe(JSON.stringify(evidence));expect(sha((stored.metadata as any).bytes)).toBe(stored.sha256);
const recovered=artifacts.find(x=>x.kind!=='EDITORIAL_RECOVERY_EVIDENCE')!;expect(sha((recovered.metadata as any).bytes)).toBe(recovered.sha256);
expect(JSON.parse((recovered.metadata as any).bytes)).toMatchObject({runId:l.runId,recoveryEvidenceSha256:stored.sha256,qaStatus:'PENDING_INDEPENDENT_QA'});
expect((await db.orchestrationRun.findUniqueOrThrow({where:{id:l.runId}})).status).toBe('AWAITING_QA');
expect(await db.orchestrationAttestation.count({where:{runId:l.runId}})).toBe(0);
expect((await db.orchestrationBudgetLedger.findMany({where:{taskId:t.id}})).map(x=>[x.kind,x.amountCents])).toEqual([['RESERVE',20]]);
expect((await db.prepareWorkflowOutbox.findUniqueOrThrow({where:{taskId:t.id}}))).toMatchObject({status:'DONE',leaseToken:null,leaseUntil:null});
expect((await request('internal/prepare-worker/heartbeat',{taskId:t.id,runId:l.runId,leaseToken:l.leaseToken},workerToken)).status).toBe(403);
expect((await request('internal/prepare-worker/editorial-result',{taskId:t.id,runId:l.runId,leaseToken:l.leaseToken,brief:brief()},workerToken)).status).toBe(403);
await expect(db.orchestrationArtifact.update({where:{id:stored.id},data:{metadata:{bytes:'tampered'}}})).rejects.toThrow('append-only');
});
it('rolls back evidence insertion when canonical database read-back is corrupted',async()=>{
const f=await fixture();
await db.$executeRawUnsafe(`CREATE FUNCTION fixture_corrupt_evidence() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.kind = 'EDITORIAL_RECOVERY_EVIDENCE' THEN NEW.metadata = jsonb_set(NEW.metadata, '{bytes}', '\"corrupt\"'::jsonb); END IF; RETURN NEW; END; $$`);
await db.$executeRawUnsafe(`CREATE TRIGGER fixture_corrupt BEFORE INSERT ON orchestration_artifacts FOR EACH ROW EXECUTE FUNCTION fixture_corrupt_evidence()`);
try {
expect((await request(f.path,payload(f.evidence))).status).toBe(403);
expect(await db.orchestrationArtifact.count({where:{runId:f.l.runId}})).toBe(0);
expect((await db.orchestrationTask.findUniqueOrThrow({where:{id:f.t.id}})).state).toBe('RUNNING');
expect((await db.orchestrationRun.findUniqueOrThrow({where:{id:f.l.runId}})).status).toBe('RUNNING');
expect((await db.prepareWorkflowOutbox.findUniqueOrThrow({where:{taskId:f.t.id}})).leaseToken).toBe(f.l.leaseToken);
expect(await db.orchestrationEvent.count({where:{taskId:f.t.id,eventType:'EDITORIAL_COMPLETED_RUN_RECOVERED'}})).toBe(0);
expect((await db.orchestrationBudgetLedger.findMany({where:{taskId:f.t.id}})).map(x=>[x.kind,x.amountCents])).toEqual([['RESERVE',20]]);
} finally {
await db.$executeRawUnsafe('DROP TRIGGER fixture_corrupt ON orchestration_artifacts');
await db.$executeRawUnsafe('DROP FUNCTION fixture_corrupt_evidence()');
}
});
it('rejects hash/provenance/meaning changes, active leases, expired approvals, cancellation and kills without writes',async()=>{
for(const mode of ['hash','retrieval','rewrite','version','commit','lease','approval','cancel','kill']) {
const f=await fixture(mode==='approval'?1500:60000); const e=structuredClone(f.evidence);
if(mode==='retrieval')e.outputs[0].retrievalEvents[1].tool_call_id='unmatched';
if(mode==='rewrite')e.brief.candidates[0].rationale='Invented replacement';
if(mode==='version')e.expectedVersion++;
if(mode==='commit')e.commitSha='b'.repeat(40);
if(mode==='lease')await db.prepareWorkflowOutbox.update({where:{taskId:f.t.id},data:{leaseUntil:new Date(Date.now()+60000)}});
if(mode==='approval')await new Promise(resolve=>setTimeout(resolve,1600));
if(mode==='cancel')await db.orchestrationTask.update({where:{id:f.t.id},data:{state:'CANCELLED'}});
if(mode==='kill')await db.orchestrationKillSwitch.upsert({where:{scope:'GLOBAL'},create:{scope:'GLOBAL',level:'STOP_AGENTS',active:true,reason:'Disposable test',actorId:'fixture'},update:{active:true,level:'STOP_AGENTS'}});
try {const b=payload(e);if(mode==='hash')b.evidenceSha256='0'.repeat(64); const r=await request(f.path,b);expect([400,403]).toContain(r.status);expect(await db.orchestrationArtifact.count({where:{runId:f.l.runId}})).toBe(0);expect(await db.orchestrationBudgetLedger.count({where:{taskId:f.t.id}})).toBe(1);}
finally {if(mode==='kill')await db.orchestrationKillSwitch.update({where:{scope:'GLOBAL'},data:{active:false}});}
}
});
});