Test completed recovery over JWT HTTP Postgres and narrowly admit Stage reviewer
Deploy Canina / deploy (push) Failing after 1m25s
Deploy Canina / deploy (push) Failing after 1m25s
This commit is contained in:
1 parent
65063e224a
commit
4413c2ef6c
5 files changed
+156
-2
No files matched your search
@@ -66,7 +66,7 @@ export class JwtStrategy extends PassportStrategy(Strategy) {
|
||||
if (user.stageTestSession) {
|
||||
if (!user.stageTestGeneration || payload.stageTestGeneration !== user.stageTestGeneration || payload.stageTestSession !== user.stageTestSession) throw new UnauthorizedException("Stage session revoked");
|
||||
const path = req.originalUrl?.split('?')[0] || '';
|
||||
const allowed = req.method === 'GET' && /^\/api\/admin\/orchestration\/tasks(?:\/[a-f0-9-]+)?$/.test(path) || req.method === 'POST' && (/^\/api\/admin\/orchestration\/tasks$/.test(path) || /^\/api\/admin\/orchestration\/tasks\/[a-f0-9-]+\/(?:gates\/1\/decide|editorial-failure)$/.test(path));
|
||||
const allowed = req.method === 'GET' && /^\/api\/admin\/orchestration\/tasks(?:\/[a-f0-9-]+)?$/.test(path) || req.method === 'POST' && (/^\/api\/admin\/orchestration\/tasks$/.test(path) || /^\/api\/admin\/orchestration\/tasks\/[a-f0-9-]+\/(?:gates\/1\/decide|editorial-failure|editorial-completed-recovery)$/.test(path));
|
||||
if (!allowed) throw new UnauthorizedException('Stage test scope denied');
|
||||
}
|
||||
return user;
|
||||
|
||||
@@ -0,0 +1,106 @@
|
||||
import { Test } from '@nestjs/testing';
|
||||
import { INestApplication } from '@nestjs/common';
|
||||
import { PrismaClient } from '@prisma/client';
|
||||
import { createHash, randomUUID } from 'crypto';
|
||||
import { sign } from 'jsonwebtoken';
|
||||
import { PrismaService } from '../prisma/prisma.service';
|
||||
import { PrepareWorkflowService } from '../orchestration/prepare-workflow.service';
|
||||
import { OrchestrationController } from '../orchestration/orchestration.controller';
|
||||
import { OrchestrationService } from '../orchestration/orchestration.service';
|
||||
import { JwtStrategy } from '../auth/jwt.strategy';
|
||||
import { UsersService } from '../users/users.service';
|
||||
import { RedisService } from '../redis/redis.service';
|
||||
import { WorkerTransportModule } from './worker-transport.module';
|
||||
import { EditorialExecutionService } from './editorial-execution';
|
||||
|
||||
(process.env.EDITORIAL_EXECUTION_DB === '1' ? describe : describe.skip)('completed recovery real JWT HTTP/PostgreSQL (synthetic evidence only)', () => {
|
||||
const db = new PrismaClient();
|
||||
const sha = (s: string) => createHash('sha256').update(s).digest('hex');
|
||||
const workerToken = 'disposable-pmo-'.repeat(5);
|
||||
let app: INestApplication, base: string, workflow: PrepareWorkflowService;
|
||||
let reviewer: any, admin: any, jwt: string, adminJwt: string;
|
||||
const generation = randomUUID(), session = 'disposable-recovery-http';
|
||||
const brief = () => ({researchDate:'2026-10-24',inputAvailability:[{input:'property',status:'UNAVAILABLE',details:'No authenticated property supplied'}],findings:[{sourceUrl:'https://www.who.int/news',sourceDate:'2026-10-20',finding:'Synthetic contract fixture, not research'}],candidates:[0,1,2].map(i=>({title:'Synthetic '+i,score:80-i,rationale:'Contract fixture only',sourceUrls:['https://www.who.int/news']}))});
|
||||
const token = (u: any, extra: any = {}) => sign({sub:u.id,stageTestSession:session,stageTestGeneration:generation,...extra},process.env.JWT_SECRET!,{expiresIn:'5m'});
|
||||
beforeAll(async () => {
|
||||
const module = await Test.createTestingModule({
|
||||
imports:[WorkerTransportModule.register({enabled:true,grants:[{id:'pmo',role:'WORKFLOW_WORKER',credentialSha256:sha(workerToken)}],workerIds:['pmo'],stageCiIds:[],artifactDirectory:'/work/tmp/recovery',artifactBaseUrl:'https://isolated.invalid/internal/prepare-worker/artifacts'})],
|
||||
controllers:[OrchestrationController],
|
||||
providers:[JwtStrategy,EditorialExecutionService,{provide:OrchestrationService,useValue:{}},{provide:UsersService,useValue:{findById:(id:string)=>db.user.findUnique({where:{id}})}},{provide:RedisService,useValue:{get:async()=>null}}],
|
||||
}).overrideProvider(PrismaService).useValue(db).compile();
|
||||
app=module.createNestApplication(); app.setGlobalPrefix('api'); await app.listen(0,'127.0.0.1'); base=await app.getUrl(); workflow=app.get(PrepareWorkflowService);
|
||||
reviewer=await db.user.create({data:{firstName:'Disposable',lastName:'Reviewer',mobile:'fixture-'+randomUUID().slice(0,7),role:'SUPER_ADMIN',stageTestSession:session,stageTestGeneration:generation,authExpiresAt:new Date(Date.now()+300000)}});
|
||||
admin=await db.user.create({data:{firstName:'Disposable',lastName:'Admin',mobile:'fixture-'+randomUUID().slice(0,7),role:'Admin',stageTestSession:session,stageTestGeneration:generation,authExpiresAt:new Date(Date.now()+300000)}});
|
||||
jwt=token(reviewer); adminJwt=token(admin);
|
||||
});
|
||||
afterAll(async()=>{await app?.close();await db.$disconnect();});
|
||||
async function request(path:string,body:any,secret=jwt) {const r=await fetch(base+'/api/'+path,{method:'POST',headers:{authorization:'Bearer '+secret,'content-type':'application/json'},body:JSON.stringify(body)});return {status:r.status,body:await r.json()};}
|
||||
async function fixture(ttl = 60000) {
|
||||
const p=await request('internal/prepare-worker/editorial-proposals',{requestId:randomUUID(),pilotKind:'editorial-research-to-brief-v1',operationMode:'PREPARE',environment:'staging',scope:'EDITORIAL_RESEARCH_TO_BRIEF',researchTopic:'Disposable recovery contract',acceptanceCriteria:['Dated evidence'],commitSha:'a'.repeat(40),budgetLimitCents:20},workerToken);
|
||||
expect(p.status).toBe(201);
|
||||
let t=await db.orchestrationTask.findUniqueOrThrow({where:{id:p.body.taskId}});
|
||||
t=await workflow.decide(t.id,1,{decision:'APPROVED',expiresAt:new Date(Date.now()+ttl).toISOString(),expectedVersion:t.version,idempotencyKey:randomUUID()},{id:'independent-approver',role:'SUPER_ADMIN'});
|
||||
const c=await request('internal/prepare-worker/editorial-claim',{taskId:t.id},workerToken);expect(c.status).toBe(201);
|
||||
const l=c.body.lease;
|
||||
await db.prepareWorkflowOutbox.update({where:{taskId:t.id},data:{leaseUntil:new Date(Date.now()-1000)}});
|
||||
const original=brief(); original.inputAvailability[0].status='missing';
|
||||
const originalText=JSON.stringify(original);
|
||||
const evidence:any={taskId:t.id,runId:l.runId,expectedVersion:t.version+1,commitSha:'a'.repeat(40),originalText,sessions:['canina-competitor-worker','canina-content-worker','canina-seo-worker','pmo'].map((profile,i)=>({profile,sessionId:'20261024_120000_aaaaa'+i,outputSha256:i===3?sha(originalText):sha('synthetic '+i),outcome:'COMPLETED'})),outputs:[{retrievalEvents:[{type:'tool_use',name:'web_extract',tool_call_id:'fixture-extract',input:{urls:['https://www.who.int/news']}},{type:'tool_result',name:'web_extract',tool_call_id:'fixture-extract',output:JSON.stringify({results:[{url:'https://www.who.int/news',content:'Synthetic retrieved bytes'}]})}]}],brief:brief(),correction:{mode:'STORED_REPORTS_ONLY',statusChanges:[{index:0,from:'missing',to:'UNAVAILABLE',reason:'Stored text explicitly says no authenticated property supplied'}],omissions:[],reason:'Synthetic stored-reports-only schema correction'}};
|
||||
return {t,l,evidence,path:'admin/orchestration/tasks/'+t.id+'/editorial-completed-recovery'};
|
||||
}
|
||||
const payload=(e:any)=>({runId:e.runId,expectedVersion:e.expectedVersion,evidenceSha256:sha(JSON.stringify(e)),evidence:e});
|
||||
it('allows narrowly scoped temporary reviewer, atomically ingests same run once, retains cost and pending QA',async()=>{
|
||||
const {t,l,evidence,path}=await fixture(); const body=payload(evidence);
|
||||
expect((await request(path,body,adminJwt)).status).toBe(403);
|
||||
expect((await request(path,body,token(reviewer,{stageTestGeneration:randomUUID()}))).status).toBe(401);
|
||||
expect((await request(path,body,'not-a-jwt')).status).toBe(401);
|
||||
expect((await request(path,{...body,actualCents:0})).status).toBe(400);
|
||||
expect((await request('admin/orchestration/tasks/'+t.id+'/budget/reconcile',{actualCents:0})).status).toBe(401);
|
||||
const results=await Promise.all([request(path,body),request(path,body)]);
|
||||
expect(results.map(x=>x.status).sort()).toEqual([201,403]);
|
||||
expect(results.find(x=>x.status===201)!.body).toMatchObject({state:'VERIFYING',qaStatus:'PENDING_INDEPENDENT_QA',retainedReservationCents:20,costStatus:'upper-bound-reserved'});
|
||||
const artifacts=await db.orchestrationArtifact.findMany({where:{runId:l.runId}});expect(artifacts).toHaveLength(2);
|
||||
const stored=artifacts.find(x=>x.kind==='EDITORIAL_RECOVERY_EVIDENCE')!;expect((stored.metadata as any).bytes).toBe(JSON.stringify(evidence));expect(sha((stored.metadata as any).bytes)).toBe(stored.sha256);
|
||||
const recovered=artifacts.find(x=>x.kind!=='EDITORIAL_RECOVERY_EVIDENCE')!;expect(sha((recovered.metadata as any).bytes)).toBe(recovered.sha256);
|
||||
expect(JSON.parse((recovered.metadata as any).bytes)).toMatchObject({runId:l.runId,recoveryEvidenceSha256:stored.sha256,qaStatus:'PENDING_INDEPENDENT_QA'});
|
||||
expect((await db.orchestrationRun.findUniqueOrThrow({where:{id:l.runId}})).status).toBe('AWAITING_QA');
|
||||
expect(await db.orchestrationAttestation.count({where:{runId:l.runId}})).toBe(0);
|
||||
expect((await db.orchestrationBudgetLedger.findMany({where:{taskId:t.id}})).map(x=>[x.kind,x.amountCents])).toEqual([['RESERVE',20]]);
|
||||
expect((await db.prepareWorkflowOutbox.findUniqueOrThrow({where:{taskId:t.id}}))).toMatchObject({status:'DONE',leaseToken:null,leaseUntil:null});
|
||||
expect((await request('internal/prepare-worker/heartbeat',{taskId:t.id,runId:l.runId,leaseToken:l.leaseToken},workerToken)).status).toBe(403);
|
||||
expect((await request('internal/prepare-worker/editorial-result',{taskId:t.id,runId:l.runId,leaseToken:l.leaseToken,brief:brief()},workerToken)).status).toBe(403);
|
||||
await expect(db.orchestrationArtifact.update({where:{id:stored.id},data:{metadata:{bytes:'tampered'}}})).rejects.toThrow('append-only');
|
||||
});
|
||||
it('rolls back evidence insertion when canonical database read-back is corrupted',async()=>{
|
||||
const f=await fixture();
|
||||
await db.$executeRawUnsafe(`CREATE FUNCTION fixture_corrupt_evidence() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.kind = 'EDITORIAL_RECOVERY_EVIDENCE' THEN NEW.metadata = jsonb_set(NEW.metadata, '{bytes}', '\"corrupt\"'::jsonb); END IF; RETURN NEW; END; $$`);
|
||||
await db.$executeRawUnsafe(`CREATE TRIGGER fixture_corrupt BEFORE INSERT ON orchestration_artifacts FOR EACH ROW EXECUTE FUNCTION fixture_corrupt_evidence()`);
|
||||
try {
|
||||
expect((await request(f.path,payload(f.evidence))).status).toBe(403);
|
||||
expect(await db.orchestrationArtifact.count({where:{runId:f.l.runId}})).toBe(0);
|
||||
expect((await db.orchestrationTask.findUniqueOrThrow({where:{id:f.t.id}})).state).toBe('RUNNING');
|
||||
expect((await db.orchestrationRun.findUniqueOrThrow({where:{id:f.l.runId}})).status).toBe('RUNNING');
|
||||
expect((await db.prepareWorkflowOutbox.findUniqueOrThrow({where:{taskId:f.t.id}})).leaseToken).toBe(f.l.leaseToken);
|
||||
expect(await db.orchestrationEvent.count({where:{taskId:f.t.id,eventType:'EDITORIAL_COMPLETED_RUN_RECOVERED'}})).toBe(0);
|
||||
expect((await db.orchestrationBudgetLedger.findMany({where:{taskId:f.t.id}})).map(x=>[x.kind,x.amountCents])).toEqual([['RESERVE',20]]);
|
||||
} finally {
|
||||
await db.$executeRawUnsafe('DROP TRIGGER fixture_corrupt ON orchestration_artifacts');
|
||||
await db.$executeRawUnsafe('DROP FUNCTION fixture_corrupt_evidence()');
|
||||
}
|
||||
});
|
||||
it('rejects hash/provenance/meaning changes, active leases, expired approvals, cancellation and kills without writes',async()=>{
|
||||
for(const mode of ['hash','retrieval','rewrite','version','commit','lease','approval','cancel','kill']) {
|
||||
const f=await fixture(mode==='approval'?1500:60000); const e=structuredClone(f.evidence);
|
||||
if(mode==='retrieval')e.outputs[0].retrievalEvents[1].tool_call_id='unmatched';
|
||||
if(mode==='rewrite')e.brief.candidates[0].rationale='Invented replacement';
|
||||
if(mode==='version')e.expectedVersion++;
|
||||
if(mode==='commit')e.commitSha='b'.repeat(40);
|
||||
if(mode==='lease')await db.prepareWorkflowOutbox.update({where:{taskId:f.t.id},data:{leaseUntil:new Date(Date.now()+60000)}});
|
||||
if(mode==='approval')await new Promise(resolve=>setTimeout(resolve,1600));
|
||||
if(mode==='cancel')await db.orchestrationTask.update({where:{id:f.t.id},data:{state:'CANCELLED'}});
|
||||
if(mode==='kill')await db.orchestrationKillSwitch.upsert({where:{scope:'GLOBAL'},create:{scope:'GLOBAL',level:'STOP_AGENTS',active:true,reason:'Disposable test',actorId:'fixture'},update:{active:true,level:'STOP_AGENTS'}});
|
||||
try {const b=payload(e);if(mode==='hash')b.evidenceSha256='0'.repeat(64); const r=await request(f.path,b);expect([400,403]).toContain(r.status);expect(await db.orchestrationArtifact.count({where:{runId:f.l.runId}})).toBe(0);expect(await db.orchestrationBudgetLedger.count({where:{taskId:f.t.id}})).toBe(1);}
|
||||
finally {if(mode==='kill')await db.orchestrationKillSwitch.update({where:{scope:'GLOBAL'},data:{active:false}});}
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,25 @@
|
||||
#!/bin/sh
|
||||
# Run only in a private checkout on a Docker host; never consumes application env.
|
||||
set -eu
|
||||
cd "$(dirname "$0")"
|
||||
name="recovery-http-$$"
|
||||
# Pin the available dependency image once; tests copy dependencies privately.
|
||||
image=$(docker image inspect "${RECOVERY_TEST_IMAGE:-stage-backend_stage:latest}" --format '{{.Id}}')
|
||||
cleanup() { docker rm -f "$name-node" "$name-pg" >/dev/null 2>&1 || :; docker network rm "$name" >/dev/null 2>&1 || :; }
|
||||
trap cleanup EXIT INT TERM
|
||||
docker network create --internal "$name" >/dev/null
|
||||
docker run -d --name "$name-pg" --network "$name" --network-alias postgres --tmpfs /var/lib/postgresql/data -e POSTGRES_PASSWORD=disposable -e POSTGRES_DB=recovery postgres:16-alpine >/dev/null
|
||||
until docker exec "$name-pg" pg_isready -U postgres >/dev/null 2>&1; do sleep 1; done
|
||||
docker run --rm --name "$name-node" --user 0 --network "$name" --entrypoint sh -v "$PWD:/source:ro" -e DATABASE_URL=postgresql://postgres:disposable@postgres:5432/recovery -e JWT_SECRET=disposable-recovery-jwt-secret-not-a-real-credential -e EDITORIAL_EXECUTION_DB=1 "$image" -c '
|
||||
set -eu
|
||||
mkdir -p /work
|
||||
cp -a /source/. /work/
|
||||
cp -a /app/node_modules /work/node_modules
|
||||
cd /work
|
||||
./node_modules/.bin/prisma generate
|
||||
./node_modules/.bin/prisma db push --skip-generate
|
||||
# db push is disposable-schema coverage only; install tracked append-only triggers explicitly.
|
||||
./node_modules/.bin/prisma db execute --file prisma/migrations/20261022190000_harden_orchestration_ledger/migration.sql --schema prisma/schema.prisma
|
||||
./node_modules/.bin/jest --runInBand src/orchestration-worker/completed-recovery-http.spec.ts src/orchestration-worker/editorial-execution.spec.ts src/orchestration-worker/editorial-validity.spec.ts src/orchestration-worker/editorial-bridge.spec.ts src/orchestration/editorial-recovery.spec.ts src/orchestration/editorial-recovery-hint.spec.ts
|
||||
./node_modules/.bin/tsc --noEmit
|
||||
'
|
||||
@@ -19,3 +19,13 @@ Do not guess mappings for `research_only`, `qualified`, `pending`, or `unknown_r
|
||||
Produce private reviewed `recovery-request.json` using the exact existing run and current version from authenticated read-back; never infer a run ID from a session ID. Use an HTTP client that reads the reviewer bearer token from a private credential file, supplies the request via `--data-binary @recovery-request.json`, and does not print credentials. Endpoint above is the exact protocol; no ready-to-submit payload is supplied because original ambiguous statuses have not been corrected. Close the temporary reviewer via the existing generation-fenced cleanup in a finally block, and verify zero matching accounts/handle removal.
|
||||
|
||||
No push, deployment, live database mutation, model call, new task, failure terminalization or budget reconciliation was performed.
|
||||
|
||||
## Disposable security regression
|
||||
|
||||
Run `sh backend/test-completed-recovery-disposable.sh` on an authorized Docker host. It resolves the dependency image (`RECOVERY_TEST_IMAGE`, default `stage-backend_stage:latest`) to its immutable local ID, copies dependencies into a disposable container, creates an internal-only Docker network and tmpfs PostgreSQL16 without published ports, and removes containers/network on exit. No application environment files or database connections are used. Redis blacklist lookup is a deliberately empty test adapter; JWT verification, persisted user/session/generation/expiry lookup, role guards, DTO validation, controller, service, transactions and PostgreSQL triggers are real.
|
||||
|
||||
The new `completed-recovery-http.spec.ts` uses synthetic report/retrieval bytes only, never the actual task. It covers temporary reviewer access versus ordinary admin, generation mismatch, invalid JWT, strict DTO and unrelated reconciliation-route denial; concurrent same-run ingestion/replay; evidence and artifact digest read-back; append-only triggers; pending independent QA and unchanged reservation; old lease heartbeat/result rejection; and failed canonical read-back rollback via a disposable INSERT trigger. Rejection cases include unmatched retrieval, rewritten candidates, hash/version/commit mismatch, active lease, expired approval, cancellation and STOP_AGENTS.
|
||||
|
||||
Observed verification: 5 suites passed, 1 existing bridge suite skipped (5 gated tests), 23 tests passed, `tsc --noEmit` passed. This is disposable schema coverage using `prisma db push` plus the exact tracked append-only migration, NOT full-chain migration coverage, live acceptance, independent artifact QA, or a claim that unknown cost is zero. The actual unknown 100-cent aggregate reservation is not read or mutated by these tests; each synthetic run preserves its own 20-cent reservation.
|
||||
|
||||
The temporary principal JWT allowlist now permits only the additional exact POST suffix `editorial-completed-recovery`; no broad administrative prefix was opened. Parent must independently review/promote the commit and review the actual stored-reports-only correction before invoking the protocol. No ready-to-submit correction for the ambiguous actual statuses is fabricated here.
|
||||
@@ -10,4 +10,17 @@ Read references/policy.md and references/role.json before acting.
|
||||
|
||||
Synthesize research and route all work; own Gate1/Gate2 queue and evidence report. Never author/review in place of workers.
|
||||
|
||||
Stage is blocked. This installed skill is not execution evidence. No Production or cron activation.
|
||||
## Research brief output contract (including stored-report correction)
|
||||
|
||||
Return ONLY one JSON object with exactly these keys; no markdown or prose:
|
||||
`{"researchDate":"YYYY-MM-DD","inputAvailability":[{"input":"string","status":"AVAILABLE|PARTIAL|UNAVAILABLE","details":"string"}],"findings":[{"sourceUrl":"https://...","sourceDate":"YYYY-MM-DD","finding":"string"}],"candidates":[{"title":"string","score":80,"rationale":"string","sourceUrls":["https://..."]}]}`
|
||||
|
||||
`status` is exactly one of `AVAILABLE`, `PARTIAL`, `UNAVAILABLE`, never a literal pipe-separated string. Availability means evidence/input availability, not publication authority, QA, qualification or cost. Do not mechanically map ambiguous `research_only`, `qualified`, `pending`, or `unknown_reserved`. Determine availability from the stored report's explicit evidence; if it cannot be grounded, report the gap rather than fabricate a compliant brief.
|
||||
|
||||
Limits: 24576 UTF8 bytes; inputAvailability 1..10 (input 120 chars, details 500); findings 1..20 (URL 2000 chars, finding 1000); candidates 3..10 distinct case-insensitive titles (title 180, rationale 1000), score integer 0..100, sourceUrls 1..10. Dates must be valid YYYY-MM-DD, not future; sourceDate <= researchDate. URLs must be HTTPS public hostnames without credentials or explicit ports. Every candidate URL must occur in retained findings.
|
||||
|
||||
For an explicitly authorized correction, use ONLY the supplied original PMO final answer and three stored worker reports/tool retrieval events. No search, extraction, delegation, new proposal, retry or publication. A finding URL must belong to the supplied `successfulWorkerRetrievedUrlAllowlist`, computed by matching successful nonempty web_extract results to their original tool-call ID and requested URLs. Search hits or worker prose alone are not retrieval proof. Never invent sources or dates.
|
||||
|
||||
For same-run recovery preserve the original researchDate, input names/details, and all retained findings/candidates byte-for-byte; only correct grounded availability statuses or omit whole unsupported findings/candidates. Never silently trim candidate source lists. Supply grounded status-change reasons and explicit original collection/index/reason omissions in the separate reviewed recovery evidence, not extra keys in the brief. If fewer than three unchanged grounded candidates remain, stop and report that schema-only recovery cannot satisfy this contract. No fabricated QA or zero-cost assertion; unknown reservations remain unknown and reserved.
|
||||
|
||||
Stage remains gated until the exact run's independent review and authorized ingestion. This installed skill is not execution, ingestion or independent QA evidence. No Production or cron activation.
|
||||
Reference in new issue
Block a user