Test completed recovery over JWT HTTP Postgres and narrowly admit Stage reviewer
Deploy Canina / deploy (push) Failing after 1m25s

This commit is contained in:
Recovery Review authored and devops committed 2026-10-24 17:35:56 +00:00
1 parent 65063e224a
commit 4413c2ef6c
5 files changed
+156 -2

No files matched your search

+1 -1
View File
@@ -66,7 +66,7 @@ export class JwtStrategy extends PassportStrategy(Strategy) {
if (user.stageTestSession) {
if (!user.stageTestGeneration || payload.stageTestGeneration !== user.stageTestGeneration || payload.stageTestSession !== user.stageTestSession) throw new UnauthorizedException("Stage session revoked");
const path = req.originalUrl?.split('?')[0] || '';
const allowed = req.method === 'GET' && /^\/api\/admin\/orchestration\/tasks(?:\/[a-f0-9-]+)?$/.test(path) || req.method === 'POST' && (/^\/api\/admin\/orchestration\/tasks$/.test(path) || /^\/api\/admin\/orchestration\/tasks\/[a-f0-9-]+\/(?:gates\/1\/decide|editorial-failure)$/.test(path));
const allowed = req.method === 'GET' && /^\/api\/admin\/orchestration\/tasks(?:\/[a-f0-9-]+)?$/.test(path) || req.method === 'POST' && (/^\/api\/admin\/orchestration\/tasks$/.test(path) || /^\/api\/admin\/orchestration\/tasks\/[a-f0-9-]+\/(?:gates\/1\/decide|editorial-failure|editorial-completed-recovery)$/.test(path));
if (!allowed) throw new UnauthorizedException('Stage test scope denied');
}
return user;
@@ -0,0 +1,106 @@
import { Test } from '@nestjs/testing';
import { INestApplication } from '@nestjs/common';
import { PrismaClient } from '@prisma/client';
import { createHash, randomUUID } from 'crypto';
import { sign } from 'jsonwebtoken';
import { PrismaService } from '../prisma/prisma.service';
import { PrepareWorkflowService } from '../orchestration/prepare-workflow.service';
import { OrchestrationController } from '../orchestration/orchestration.controller';
import { OrchestrationService } from '../orchestration/orchestration.service';
import { JwtStrategy } from '../auth/jwt.strategy';
import { UsersService } from '../users/users.service';
import { RedisService } from '../redis/redis.service';
import { WorkerTransportModule } from './worker-transport.module';
import { EditorialExecutionService } from './editorial-execution';
(process.env.EDITORIAL_EXECUTION_DB === '1' ? describe : describe.skip)('completed recovery real JWT HTTP/PostgreSQL (synthetic evidence only)', () => {
const db = new PrismaClient();
const sha = (s: string) => createHash('sha256').update(s).digest('hex');
const workerToken = 'disposable-pmo-'.repeat(5);
let app: INestApplication, base: string, workflow: PrepareWorkflowService;
let reviewer: any, admin: any, jwt: string, adminJwt: string;
const generation = randomUUID(), session = 'disposable-recovery-http';
const brief = () => ({researchDate:'2026-10-24',inputAvailability:[{input:'property',status:'UNAVAILABLE',details:'No authenticated property supplied'}],findings:[{sourceUrl:'https://www.who.int/news',sourceDate:'2026-10-20',finding:'Synthetic contract fixture, not research'}],candidates:[0,1,2].map(i=>({title:'Synthetic '+i,score:80-i,rationale:'Contract fixture only',sourceUrls:['https://www.who.int/news']}))});
const token = (u: any, extra: any = {}) => sign({sub:u.id,stageTestSession:session,stageTestGeneration:generation,...extra},process.env.JWT_SECRET!,{expiresIn:'5m'});
beforeAll(async () => {
const module = await Test.createTestingModule({
imports:[WorkerTransportModule.register({enabled:true,grants:[{id:'pmo',role:'WORKFLOW_WORKER',credentialSha256:sha(workerToken)}],workerIds:['pmo'],stageCiIds:[],artifactDirectory:'/work/tmp/recovery',artifactBaseUrl:'https://isolated.invalid/internal/prepare-worker/artifacts'})],
controllers:[OrchestrationController],
providers:[JwtStrategy,EditorialExecutionService,{provide:OrchestrationService,useValue:{}},{provide:UsersService,useValue:{findById:(id:string)=>db.user.findUnique({where:{id}})}},{provide:RedisService,useValue:{get:async()=>null}}],
}).overrideProvider(PrismaService).useValue(db).compile();
app=module.createNestApplication(); app.setGlobalPrefix('api'); await app.listen(0,'127.0.0.1'); base=await app.getUrl(); workflow=app.get(PrepareWorkflowService);
reviewer=await db.user.create({data:{firstName:'Disposable',lastName:'Reviewer',mobile:'fixture-'+randomUUID().slice(0,7),role:'SUPER_ADMIN',stageTestSession:session,stageTestGeneration:generation,authExpiresAt:new Date(Date.now()+300000)}});
admin=await db.user.create({data:{firstName:'Disposable',lastName:'Admin',mobile:'fixture-'+randomUUID().slice(0,7),role:'Admin',stageTestSession:session,stageTestGeneration:generation,authExpiresAt:new Date(Date.now()+300000)}});
jwt=token(reviewer); adminJwt=token(admin);
});
afterAll(async()=>{await app?.close();await db.$disconnect();});
async function request(path:string,body:any,secret=jwt) {const r=await fetch(base+'/api/'+path,{method:'POST',headers:{authorization:'Bearer '+secret,'content-type':'application/json'},body:JSON.stringify(body)});return {status:r.status,body:await r.json()};}
async function fixture(ttl = 60000) {
const p=await request('internal/prepare-worker/editorial-proposals',{requestId:randomUUID(),pilotKind:'editorial-research-to-brief-v1',operationMode:'PREPARE',environment:'staging',scope:'EDITORIAL_RESEARCH_TO_BRIEF',researchTopic:'Disposable recovery contract',acceptanceCriteria:['Dated evidence'],commitSha:'a'.repeat(40),budgetLimitCents:20},workerToken);
expect(p.status).toBe(201);
let t=await db.orchestrationTask.findUniqueOrThrow({where:{id:p.body.taskId}});
t=await workflow.decide(t.id,1,{decision:'APPROVED',expiresAt:new Date(Date.now()+ttl).toISOString(),expectedVersion:t.version,idempotencyKey:randomUUID()},{id:'independent-approver',role:'SUPER_ADMIN'});
const c=await request('internal/prepare-worker/editorial-claim',{taskId:t.id},workerToken);expect(c.status).toBe(201);
const l=c.body.lease;
await db.prepareWorkflowOutbox.update({where:{taskId:t.id},data:{leaseUntil:new Date(Date.now()-1000)}});
const original=brief(); original.inputAvailability[0].status='missing';
const originalText=JSON.stringify(original);
const evidence:any={taskId:t.id,runId:l.runId,expectedVersion:t.version+1,commitSha:'a'.repeat(40),originalText,sessions:['canina-competitor-worker','canina-content-worker','canina-seo-worker','pmo'].map((profile,i)=>({profile,sessionId:'20261024_120000_aaaaa'+i,outputSha256:i===3?sha(originalText):sha('synthetic '+i),outcome:'COMPLETED'})),outputs:[{retrievalEvents:[{type:'tool_use',name:'web_extract',tool_call_id:'fixture-extract',input:{urls:['https://www.who.int/news']}},{type:'tool_result',name:'web_extract',tool_call_id:'fixture-extract',output:JSON.stringify({results:[{url:'https://www.who.int/news',content:'Synthetic retrieved bytes'}]})}]}],brief:brief(),correction:{mode:'STORED_REPORTS_ONLY',statusChanges:[{index:0,from:'missing',to:'UNAVAILABLE',reason:'Stored text explicitly says no authenticated property supplied'}],omissions:[],reason:'Synthetic stored-reports-only schema correction'}};
return {t,l,evidence,path:'admin/orchestration/tasks/'+t.id+'/editorial-completed-recovery'};
}
const payload=(e:any)=>({runId:e.runId,expectedVersion:e.expectedVersion,evidenceSha256:sha(JSON.stringify(e)),evidence:e});
it('allows narrowly scoped temporary reviewer, atomically ingests same run once, retains cost and pending QA',async()=>{
const {t,l,evidence,path}=await fixture(); const body=payload(evidence);
expect((await request(path,body,adminJwt)).status).toBe(403);
expect((await request(path,body,token(reviewer,{stageTestGeneration:randomUUID()}))).status).toBe(401);
expect((await request(path,body,'not-a-jwt')).status).toBe(401);
expect((await request(path,{...body,actualCents:0})).status).toBe(400);
expect((await request('admin/orchestration/tasks/'+t.id+'/budget/reconcile',{actualCents:0})).status).toBe(401);
const results=await Promise.all([request(path,body),request(path,body)]);
expect(results.map(x=>x.status).sort()).toEqual([201,403]);
expect(results.find(x=>x.status===201)!.body).toMatchObject({state:'VERIFYING',qaStatus:'PENDING_INDEPENDENT_QA',retainedReservationCents:20,costStatus:'upper-bound-reserved'});
const artifacts=await db.orchestrationArtifact.findMany({where:{runId:l.runId}});expect(artifacts).toHaveLength(2);
const stored=artifacts.find(x=>x.kind==='EDITORIAL_RECOVERY_EVIDENCE')!;expect((stored.metadata as any).bytes).toBe(JSON.stringify(evidence));expect(sha((stored.metadata as any).bytes)).toBe(stored.sha256);
const recovered=artifacts.find(x=>x.kind!=='EDITORIAL_RECOVERY_EVIDENCE')!;expect(sha((recovered.metadata as any).bytes)).toBe(recovered.sha256);
expect(JSON.parse((recovered.metadata as any).bytes)).toMatchObject({runId:l.runId,recoveryEvidenceSha256:stored.sha256,qaStatus:'PENDING_INDEPENDENT_QA'});
expect((await db.orchestrationRun.findUniqueOrThrow({where:{id:l.runId}})).status).toBe('AWAITING_QA');
expect(await db.orchestrationAttestation.count({where:{runId:l.runId}})).toBe(0);
expect((await db.orchestrationBudgetLedger.findMany({where:{taskId:t.id}})).map(x=>[x.kind,x.amountCents])).toEqual([['RESERVE',20]]);
expect((await db.prepareWorkflowOutbox.findUniqueOrThrow({where:{taskId:t.id}}))).toMatchObject({status:'DONE',leaseToken:null,leaseUntil:null});
expect((await request('internal/prepare-worker/heartbeat',{taskId:t.id,runId:l.runId,leaseToken:l.leaseToken},workerToken)).status).toBe(403);
expect((await request('internal/prepare-worker/editorial-result',{taskId:t.id,runId:l.runId,leaseToken:l.leaseToken,brief:brief()},workerToken)).status).toBe(403);
await expect(db.orchestrationArtifact.update({where:{id:stored.id},data:{metadata:{bytes:'tampered'}}})).rejects.toThrow('append-only');
});
it('rolls back evidence insertion when canonical database read-back is corrupted',async()=>{
const f=await fixture();
await db.$executeRawUnsafe(`CREATE FUNCTION fixture_corrupt_evidence() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.kind = 'EDITORIAL_RECOVERY_EVIDENCE' THEN NEW.metadata = jsonb_set(NEW.metadata, '{bytes}', '\"corrupt\"'::jsonb); END IF; RETURN NEW; END; $$`);
await db.$executeRawUnsafe(`CREATE TRIGGER fixture_corrupt BEFORE INSERT ON orchestration_artifacts FOR EACH ROW EXECUTE FUNCTION fixture_corrupt_evidence()`);
try {
expect((await request(f.path,payload(f.evidence))).status).toBe(403);
expect(await db.orchestrationArtifact.count({where:{runId:f.l.runId}})).toBe(0);
expect((await db.orchestrationTask.findUniqueOrThrow({where:{id:f.t.id}})).state).toBe('RUNNING');
expect((await db.orchestrationRun.findUniqueOrThrow({where:{id:f.l.runId}})).status).toBe('RUNNING');
expect((await db.prepareWorkflowOutbox.findUniqueOrThrow({where:{taskId:f.t.id}})).leaseToken).toBe(f.l.leaseToken);
expect(await db.orchestrationEvent.count({where:{taskId:f.t.id,eventType:'EDITORIAL_COMPLETED_RUN_RECOVERED'}})).toBe(0);
expect((await db.orchestrationBudgetLedger.findMany({where:{taskId:f.t.id}})).map(x=>[x.kind,x.amountCents])).toEqual([['RESERVE',20]]);
} finally {
await db.$executeRawUnsafe('DROP TRIGGER fixture_corrupt ON orchestration_artifacts');
await db.$executeRawUnsafe('DROP FUNCTION fixture_corrupt_evidence()');
}
});
it('rejects hash/provenance/meaning changes, active leases, expired approvals, cancellation and kills without writes',async()=>{
for(const mode of ['hash','retrieval','rewrite','version','commit','lease','approval','cancel','kill']) {
const f=await fixture(mode==='approval'?1500:60000); const e=structuredClone(f.evidence);
if(mode==='retrieval')e.outputs[0].retrievalEvents[1].tool_call_id='unmatched';
if(mode==='rewrite')e.brief.candidates[0].rationale='Invented replacement';
if(mode==='version')e.expectedVersion++;
if(mode==='commit')e.commitSha='b'.repeat(40);
if(mode==='lease')await db.prepareWorkflowOutbox.update({where:{taskId:f.t.id},data:{leaseUntil:new Date(Date.now()+60000)}});
if(mode==='approval')await new Promise(resolve=>setTimeout(resolve,1600));
if(mode==='cancel')await db.orchestrationTask.update({where:{id:f.t.id},data:{state:'CANCELLED'}});
if(mode==='kill')await db.orchestrationKillSwitch.upsert({where:{scope:'GLOBAL'},create:{scope:'GLOBAL',level:'STOP_AGENTS',active:true,reason:'Disposable test',actorId:'fixture'},update:{active:true,level:'STOP_AGENTS'}});
try {const b=payload(e);if(mode==='hash')b.evidenceSha256='0'.repeat(64); const r=await request(f.path,b);expect([400,403]).toContain(r.status);expect(await db.orchestrationArtifact.count({where:{runId:f.l.runId}})).toBe(0);expect(await db.orchestrationBudgetLedger.count({where:{taskId:f.t.id}})).toBe(1);}
finally {if(mode==='kill')await db.orchestrationKillSwitch.update({where:{scope:'GLOBAL'},data:{active:false}});}
}
});
});
@@ -0,0 +1,25 @@
#!/bin/sh
# Run only in a private checkout on a Docker host; never consumes application env.
set -eu
cd "$(dirname "$0")"
name="recovery-http-$$"
# Pin the available dependency image once; tests copy dependencies privately.
image=$(docker image inspect "${RECOVERY_TEST_IMAGE:-stage-backend_stage:latest}" --format '{{.Id}}')
cleanup() { docker rm -f "$name-node" "$name-pg" >/dev/null 2>&1 || :; docker network rm "$name" >/dev/null 2>&1 || :; }
trap cleanup EXIT INT TERM
docker network create --internal "$name" >/dev/null
docker run -d --name "$name-pg" --network "$name" --network-alias postgres --tmpfs /var/lib/postgresql/data -e POSTGRES_PASSWORD=disposable -e POSTGRES_DB=recovery postgres:16-alpine >/dev/null
until docker exec "$name-pg" pg_isready -U postgres >/dev/null 2>&1; do sleep 1; done
docker run --rm --name "$name-node" --user 0 --network "$name" --entrypoint sh -v "$PWD:/source:ro" -e DATABASE_URL=postgresql://postgres:disposable@postgres:5432/recovery -e JWT_SECRET=disposable-recovery-jwt-secret-not-a-real-credential -e EDITORIAL_EXECUTION_DB=1 "$image" -c '
set -eu
mkdir -p /work
cp -a /source/. /work/
cp -a /app/node_modules /work/node_modules
cd /work
./node_modules/.bin/prisma generate
./node_modules/.bin/prisma db push --skip-generate
# db push is disposable-schema coverage only; install tracked append-only triggers explicitly.
./node_modules/.bin/prisma db execute --file prisma/migrations/20261022190000_harden_orchestration_ledger/migration.sql --schema prisma/schema.prisma
./node_modules/.bin/jest --runInBand src/orchestration-worker/completed-recovery-http.spec.ts src/orchestration-worker/editorial-execution.spec.ts src/orchestration-worker/editorial-validity.spec.ts src/orchestration-worker/editorial-bridge.spec.ts src/orchestration/editorial-recovery.spec.ts src/orchestration/editorial-recovery-hint.spec.ts
./node_modules/.bin/tsc --noEmit
'
@@ -19,3 +19,13 @@ Do not guess mappings for `research_only`, `qualified`, `pending`, or `unknown_r
Produce private reviewed `recovery-request.json` using the exact existing run and current version from authenticated read-back; never infer a run ID from a session ID. Use an HTTP client that reads the reviewer bearer token from a private credential file, supplies the request via `--data-binary @recovery-request.json`, and does not print credentials. Endpoint above is the exact protocol; no ready-to-submit payload is supplied because original ambiguous statuses have not been corrected. Close the temporary reviewer via the existing generation-fenced cleanup in a finally block, and verify zero matching accounts/handle removal.
No push, deployment, live database mutation, model call, new task, failure terminalization or budget reconciliation was performed.
## Disposable security regression
Run `sh backend/test-completed-recovery-disposable.sh` on an authorized Docker host. It resolves the dependency image (`RECOVERY_TEST_IMAGE`, default `stage-backend_stage:latest`) to its immutable local ID, copies dependencies into a disposable container, creates an internal-only Docker network and tmpfs PostgreSQL16 without published ports, and removes containers/network on exit. No application environment files or database connections are used. Redis blacklist lookup is a deliberately empty test adapter; JWT verification, persisted user/session/generation/expiry lookup, role guards, DTO validation, controller, service, transactions and PostgreSQL triggers are real.
The new `completed-recovery-http.spec.ts` uses synthetic report/retrieval bytes only, never the actual task. It covers temporary reviewer access versus ordinary admin, generation mismatch, invalid JWT, strict DTO and unrelated reconciliation-route denial; concurrent same-run ingestion/replay; evidence and artifact digest read-back; append-only triggers; pending independent QA and unchanged reservation; old lease heartbeat/result rejection; and failed canonical read-back rollback via a disposable INSERT trigger. Rejection cases include unmatched retrieval, rewritten candidates, hash/version/commit mismatch, active lease, expired approval, cancellation and STOP_AGENTS.
Observed verification: 5 suites passed, 1 existing bridge suite skipped (5 gated tests), 23 tests passed, `tsc --noEmit` passed. This is disposable schema coverage using `prisma db push` plus the exact tracked append-only migration, NOT full-chain migration coverage, live acceptance, independent artifact QA, or a claim that unknown cost is zero. The actual unknown 100-cent aggregate reservation is not read or mutated by these tests; each synthetic run preserves its own 20-cent reservation.
The temporary principal JWT allowlist now permits only the additional exact POST suffix `editorial-completed-recovery`; no broad administrative prefix was opened. Parent must independently review/promote the commit and review the actual stored-reports-only correction before invoking the protocol. No ready-to-submit correction for the ambiguous actual statuses is fabricated here.
@@ -10,4 +10,17 @@ Read references/policy.md and references/role.json before acting.
Synthesize research and route all work; own Gate1/Gate2 queue and evidence report. Never author/review in place of workers.
Stage is blocked. This installed skill is not execution evidence. No Production or cron activation.
## Research brief output contract (including stored-report correction)
Return ONLY one JSON object with exactly these keys; no markdown or prose:
`{"researchDate":"YYYY-MM-DD","inputAvailability":[{"input":"string","status":"AVAILABLE|PARTIAL|UNAVAILABLE","details":"string"}],"findings":[{"sourceUrl":"https://...","sourceDate":"YYYY-MM-DD","finding":"string"}],"candidates":[{"title":"string","score":80,"rationale":"string","sourceUrls":["https://..."]}]}`
`status` is exactly one of `AVAILABLE`, `PARTIAL`, `UNAVAILABLE`, never a literal pipe-separated string. Availability means evidence/input availability, not publication authority, QA, qualification or cost. Do not mechanically map ambiguous `research_only`, `qualified`, `pending`, or `unknown_reserved`. Determine availability from the stored report's explicit evidence; if it cannot be grounded, report the gap rather than fabricate a compliant brief.
Limits: 24576 UTF8 bytes; inputAvailability 1..10 (input 120 chars, details 500); findings 1..20 (URL 2000 chars, finding 1000); candidates 3..10 distinct case-insensitive titles (title 180, rationale 1000), score integer 0..100, sourceUrls 1..10. Dates must be valid YYYY-MM-DD, not future; sourceDate <= researchDate. URLs must be HTTPS public hostnames without credentials or explicit ports. Every candidate URL must occur in retained findings.
For an explicitly authorized correction, use ONLY the supplied original PMO final answer and three stored worker reports/tool retrieval events. No search, extraction, delegation, new proposal, retry or publication. A finding URL must belong to the supplied `successfulWorkerRetrievedUrlAllowlist`, computed by matching successful nonempty web_extract results to their original tool-call ID and requested URLs. Search hits or worker prose alone are not retrieval proof. Never invent sources or dates.
For same-run recovery preserve the original researchDate, input names/details, and all retained findings/candidates byte-for-byte; only correct grounded availability statuses or omit whole unsupported findings/candidates. Never silently trim candidate source lists. Supply grounded status-change reasons and explicit original collection/index/reason omissions in the separate reviewed recovery evidence, not extra keys in the brief. If fewer than three unchanged grounded candidates remain, stop and report that schema-only recovery cannot satisfy this contract. No fabricated QA or zero-cost assertion; unknown reservations remain unknown and reserved.
Stage remains gated until the exact run's independent review and authorized ingestion. This installed skill is not execution, ingestion or independent QA evidence. No Production or cron activation.