import { Test } from '@nestjs/testing'; import request from 'supertest'; import { OrchestrationController } from './orchestration.controller'; import { OrchestrationService } from './orchestration.service'; import { JwtAuthGuard } from '../auth/jwt-auth.guard'; import { RolesGuard } from '../common/guards/roles.guard'; import { PrismaClient } from '@prisma/client'; import { PrepareWorkflowService } from './prepare-workflow.service'; import { OperationMode, WorkflowState } from './orchestration.domain'; const enabled = process.env.WORKFLOW_DB === '1'; (enabled ? describe : describe.skip)('isolated prepare workflow', () => { const db = new PrismaClient(); const admin = { id: 'proposer', role: 'Admin' }; const reviewer = { id: 'reviewer', role: 'SUPER_ADMIN' }; const worker = { id: 'worker-1', role: 'WORKFLOW_WORKER' }; const ci = { id: 'isolated-ci', role: 'STAGE_CI' }; const service = new PrepareWorkflowService(db as never, { workerIds: ['worker-1', 'worker-2'], stageCiIds: ['isolated-ci'], }); let seq = 0; const dto = () => ({ key: `wf-${Date.now()}-${seq++}`, title: 'Bounded isolated QA tracer', department: 'ENGINEERING', operationMode: OperationMode.PREPARE, budgetLimitCents: 20, metadata: { commitSha: 'a'.repeat(40), token: 'must-not-persist', scope: 'Staging schema audit only', acceptanceCriteria: ['Report all schema failures'], actorId: 'spoof', }, }); async function ready() { let t = await service.create(dto(), admin); t = await service.transition( t.id, { toState: WorkflowState.GATE1_PENDING, expectedVersion: t.version, idempotencyKey: `submit-${t.id}`, }, admin, ); t = await service.decide( t.id, 1, { decision: 'APPROVED', expectedVersion: t.version, idempotencyKey: `approve-${t.id}`, expiresAt: new Date(Date.now() + 60000).toISOString(), }, reviewer, ); return service.transition( t.id, { toState: WorkflowState.READY, expectedVersion: t.version, idempotencyKey: `ready-${t.id}`, }, admin, ); } afterAll(async () => { await db.$disconnect(); }); it('serializes competing owner holds and preserves audited reason, then decides held task', async () => { let t = await service.create(dto(), admin); t = await service.transition(t.id, {toState:'GATE1_PENDING',expectedVersion:t.version,idempotencyKey:`submit-hold-${t.id}`}, admin); const command = {toState:'GATE1_HELD',expectedVersion:t.version,idempotencyKey:`hold-${t.id}`,reason:'Need source review'}; const results = await Promise.allSettled([ service.transition(t.id, command, reviewer), service.transition(t.id, {...command,idempotencyKey:`competing-${t.id}`}, reviewer), ]); expect(results.filter(r=>r.status==='fulfilled')).toHaveLength(1); const held = await db.orchestrationTask.findUniqueOrThrow({where:{id:t.id}}); expect(held.state).toBe('GATE1_HELD'); expect(held.version).toBe(t.version+1); const events = await db.orchestrationEvent.findMany({where:{taskId:t.id,eventType:'PREPARE_GATE1_HELD'}}); expect(events).toHaveLength(1); expect(events[0].actorId).toBe(reviewer.id); expect(JSON.stringify(events[0].payload)).toContain('Need source review'); expect(await db.orchestrationApprovalDecision.count({where:{taskId:t.id}})).toBe(0); expect(await db.prepareWorkflowOutbox.count({where:{taskId:t.id}})).toBe(0); await expect(service.transition(t.id, {...command,expectedVersion:held.version,reason:'changed'},reviewer)).rejects.toMatchObject({status:409}); const decision = await service.decide(t.id,1,{decision:'REJECTED',expectedVersion:held.version,idempotencyKey:`decline-held-${t.id}`,expiresAt:new Date(Date.now()+60000).toISOString()},reviewer); expect(decision.state).toBe('REJECTED'); }); it('persists and projects immutable bounded proposal scope without arbitrary metadata', async () => { const input = dto(); const t = await service.create(input, admin); expect(t.metadata).toMatchObject({ workflowScope: 'STAGE_PREPARE', scope: input.metadata.scope, acceptanceCriteria: input.metadata.acceptanceCriteria, commitSha: input.metadata.commitSha, }); expect(t.metadata).not.toHaveProperty('token'); expect(t.metadata).not.toHaveProperty('actorId'); const submitted = await service.transition( t.id, { toState: 'GATE1_PENDING', expectedVersion: t.version, idempotencyKey: `immutable-${t.id}`, }, admin, ); expect(submitted.metadata).toEqual(t.metadata); expect(service.project(submitted, reviewer).metadata).toEqual({ scope: input.metadata.scope, acceptanceCriteria: input.metadata.acceptanceCriteria, environment: 'staging', commitSha: input.metadata.commitSha, }); }); it.each([ null, [], {}, { commitSha: 'a'.repeat(40), scope: '', acceptanceCriteria: ['ok'] }, { commitSha: 'a'.repeat(40), scope: ' '.repeat(10), acceptanceCriteria: ['ok'], }, { commitSha: 'a'.repeat(40), scope: 'x'.repeat(2001), acceptanceCriteria: ['ok'], }, { commitSha: 'a'.repeat(40), scope: 'ok', acceptanceCriteria: [] }, { commitSha: 'a'.repeat(40), scope: 'ok', acceptanceCriteria: Array(21).fill('ok'), }, { commitSha: 'a'.repeat(40), scope: 'ok', acceptanceCriteria: ['x'.repeat(501)], }, { commitSha: 'a'.repeat(40), scope: 'ok', acceptanceCriteria: [' '] }, { commitSha: 'a'.repeat(40), scope: 'ok', acceptanceCriteria: [123] }, { commitSha: ['a'.repeat(40)], scope: 'ok', acceptanceCriteria: ['ok'] }, { scope: 'ok', acceptanceCriteria: ['ok'] }, ])( 'rejects unsafe or unbounded metadata %# without persisting', async (metadata) => { const input = { ...dto(), metadata }; await expect(service.create(input, admin)).rejects.toMatchObject({ status: 400, }); expect( await db.orchestrationTask.count({ where: { key: input.key } }), ).toBe(0); }, ); it('accepts exact metadata bounds', async () => { const metadata = { commitSha: 'a'.repeat(40), scope: 'x'.repeat(2000), acceptanceCriteria: Array(20).fill('x'.repeat(500)), }; expect( (await service.create({ ...dto(), metadata }, admin)).metadata, ).toMatchObject(metadata); }); it('normalizes racing duplicate proposal keys to conflict without duplicate events', async () => { const input = dto(); const results = await Promise.allSettled([ service.create(input, admin), service.create({ ...input, title: 'Different intent' }, reviewer), ]); expect(results.filter((r) => r.status === 'fulfilled')).toHaveLength(1); const rejected = results.find( (r) => r.status === 'rejected', ) as PromiseRejectedResult; expect(rejected.reason).toMatchObject({ status: 409 }); const persisted = await db.orchestrationTask.findUniqueOrThrow({ where: { key: input.key }, }); expect( await db.orchestrationEvent.count({ where: { taskId: persisted.id } }), ).toBe(1); await expect(service.create(input, admin)).rejects.toMatchObject({ status: 409, }); }); it('enforces create and read contracts over HTTP with real PostgreSQL persistence', async () => { const module = await Test.createTestingModule({ controllers: [OrchestrationController], providers: [ { provide: PrepareWorkflowService, useValue: service }, { provide: OrchestrationService, useValue: { findAll: () => db.orchestrationTask.findMany(), findOne: (id: string) => db.orchestrationTask.findUniqueOrThrow({ where: { id } }), }, }, ], }) .overrideGuard(JwtAuthGuard) .useValue({ canActivate(context: any) { context.switchToHttp().getRequest().user = { sub: admin.id, role: admin.role, }; return true; }, }) .overrideGuard(RolesGuard) .useValue({ canActivate: () => true }) .compile(); const app = module.createNestApplication(); await app.init(); try { const input = dto(); const response = await request(app.getHttpServer()) .post('/admin/orchestration/tasks') .send(input) .expect(201); expect(response.body.metadata).toEqual({ scope: input.metadata.scope, acceptanceCriteria: input.metadata.acceptanceCriteria, commitSha: input.metadata.commitSha, environment: 'staging', }); expect(response.body.availableActions).toEqual(['SUBMIT_GATE1']); const detail = await request(app.getHttpServer()) .get(`/admin/orchestration/tasks/${response.body.id}`) .expect(200); expect(detail.body.metadata).toEqual(response.body.metadata); const list = await request(app.getHttpServer()) .get('/admin/orchestration/tasks') .expect(200); expect( list.body.find((row: any) => row.id === response.body.id).metadata, ).toEqual(response.body.metadata); expect(JSON.stringify(response.body)).not.toMatch( /createdBy|must-not-persist|actorId|workflowScope/, ); await request(app.getHttpServer()) .post('/admin/orchestration/tasks') .send(input) .expect(409); await request(app.getHttpServer()) .post('/admin/orchestration/tasks') .send({ ...input, key: `${input.key}-invalid`, metadata: { scope: 'only stage', acceptanceCriteria: ['QA passes'] }, }) .expect(400); await request(app.getHttpServer()) .post('/admin/orchestration/tasks') .send({ ...input, key: `${input.key}-invalid`, metadata: { ...input.metadata, acceptanceCriteria: [' '] }, }) .expect(400); } finally { await app.close(); } }); it('fences heartbeat and completes QA plus independent Stage evidence only to Gate2 pending', async () => { const t = await ready(); const c = await service.claim(worker, t.id); await expect( service.heartbeat({ ...worker, id: 'worker-2' }, c), ).rejects.toThrow(/lease/i); await service.heartbeat(worker, c); const qa = { ...c, sha256: 'b'.repeat(64), uri: 'https://ci.example/qa', actualCents: 8, }; await service.complete(worker, qa); await expect( service.stageEvidence( { id: 'proposer', role: 'Admin' }, { taskId: t.id, runId: c.runId, sha256: qa.sha256, commitSha: c.commitSha, evidenceSha256: 'c'.repeat(64), uri: 'https://ci.example/stage', }, ), ).rejects.toThrow(/identity/i); await service.stageEvidence(ci, { taskId: t.id, runId: c.runId, sha256: qa.sha256, commitSha: c.commitSha, evidenceSha256: 'c'.repeat(64), uri: 'https://ci.example/stage', }); expect( await db.orchestrationTask.findUnique({ where: { id: t.id } }), ).toMatchObject({ state: 'GATE2_PENDING' }); expect( await db.orchestrationBudgetLedger.aggregate({ where: { taskId: t.id }, _sum: { amountCents: true }, }), ).toMatchObject({ _sum: { amountCents: 8 } }); await expect(service.complete(worker, qa)).rejects.toThrow(/lease|active/i); }); it('bounds expired-lease retries and rejects stale completion without releasing unknown costs', async () => { const t = await service.create({ ...dto(), budgetLimitCents: 60 }, admin); let current = await service.transition( t.id, { toState: 'GATE1_PENDING', expectedVersion: 1, idempotencyKey: `s-${t.id}`, }, admin, ); current = await service.decide( t.id, 1, { decision: 'APPROVED', expectedVersion: current.version, idempotencyKey: `a-${t.id}`, expiresAt: new Date(Date.now() + 60000).toISOString(), }, reviewer, ); await service.transition( t.id, { toState: 'READY', expectedVersion: current.version, idempotencyKey: `q-${t.id}`, }, admin, ); const first = await service.claim(worker, t.id); for (let i = 0; i < 2; i++) { await db.prepareWorkflowOutbox.update({ where: { taskId: t.id }, data: { leaseUntil: new Date(0) }, }); const next = await service.claim(worker, t.id); expect(next.attempt).toBe(i + 2); } await expect( service.complete(worker, { ...first, sha256: 'b'.repeat(64), uri: 'https://ci.example/qa', actualCents: 0, }), ).rejects.toThrow(/lease/i); await db.prepareWorkflowOutbox.update({ where: { taskId: t.id }, data: { leaseUntil: new Date(0) }, }); await expect(service.claim(worker, t.id)).rejects.toThrow(/exhausted/i); expect(await db.orchestrationRun.count({ where: { taskId: t.id } })).toBe( 3, ); expect( await db.orchestrationBudgetLedger.aggregate({ where: { taskId: t.id }, _sum: { amountCents: true }, }), ).toMatchObject({ _sum: { amountCents: 60 } }); }); it('rejects self approval, non-PREPARE and unprovisioned workers', async () => { await expect( service.create( { ...dto(), operationMode: OperationMode.MUTATE_PRODUCTION }, admin, ), ).rejects.toThrow(/PREPARE/); let t = await service.create(dto(), reviewer); t = await service.transition( t.id, { toState: 'GATE1_PENDING', expectedVersion: 1, idempotencyKey: `s-${t.id}`, }, reviewer, ); await expect( service.decide( t.id, 1, { decision: 'APPROVED', expectedVersion: t.version, idempotencyKey: `a-${t.id}`, expiresAt: new Date(Date.now() + 60000).toISOString(), }, reviewer, ), ).rejects.toThrow(/independent/i); await expect( service.claim({ id: 'spoof', role: 'WORKFLOW_WORKER' }), ).rejects.toThrow(/provisioned/i); await expect(service.decide(t.id, 2, {}, reviewer)).rejects.toThrow( /disabled/i, ); }); it('serializes duplicate queue submission and denies claims beyond remaining task budget', async () => { const t = await ready(); await db.orchestrationBudgetLedger.create({ data: { taskId: t.id, amountCents: 1, kind: 'SPEND', actorId: 'fixture', idempotencyKey: `spend-${t.id}`, }, }); await expect(service.claim(worker, t.id)).rejects.toThrow(/budget/i); expect(await db.orchestrationRun.count({ where: { taskId: t.id } })).toBe( 0, ); expect( await db.prepareWorkflowOutbox.count({ where: { taskId: t.id } }), ).toBe(1); }); it('replays submission safely and serializes racing queue requests', async () => { let t = await service.create(dto(), admin); const submit = { toState: 'GATE1_PENDING', expectedVersion: t.version, idempotencyKey: `replay-${t.id}`, }; t = await service.transition(t.id, submit, admin); expect((await service.transition(t.id, submit, admin)).version).toBe( t.version, ); await expect( service.transition(t.id, { ...submit, toState: 'READY' }, admin), ).rejects.toThrow(/mismatch/i); t = await service.decide( t.id, 1, { decision: 'APPROVED', expectedVersion: t.version, idempotencyKey: `r-approve-${t.id}`, expiresAt: new Date(Date.now() + 60000).toISOString(), }, reviewer, ); const results = await Promise.allSettled( ['one', 'two'].map((k) => service.transition( t.id, { toState: 'READY', expectedVersion: t.version, idempotencyKey: `${k}-${t.id}`, }, admin, ), ), ); expect(results.filter((r) => r.status === 'fulfilled')).toHaveLength(1); expect( await db.prepareWorkflowOutbox.count({ where: { taskId: t.id } }), ).toBe(1); }); it('kill switch freezes claims without creating run or reservation', async () => { const t = await ready(); await db.orchestrationKillSwitch.create({ data: { scope: 'GLOBAL', level: 'FREEZE_NEW_WORK', reason: 'isolated test', actorId: 'fixture', }, }); try { await expect(service.claim(worker, t.id)).rejects.toThrow(/kill switch/i); } finally { await db.orchestrationKillSwitch.delete({ where: { scope: 'GLOBAL' } }); } expect(await db.orchestrationRun.count({ where: { taskId: t.id } })).toBe( 0, ); }); it('persists an atomic outbox and exactly one worker wins concurrent claims', async () => { const t = await ready(); const claims = await Promise.all([ service.claim(worker, t.id), service.claim({ id: 'worker-2', role: 'WORKFLOW_WORKER' }, t.id), ]); expect(claims.filter(Boolean)).toHaveLength(1); const claim = claims.find(Boolean)!; expect(claim.taskId).toBe(t.id); expect(claim.attempt).toBe(1); expect(await db.orchestrationRun.count({ where: { taskId: t.id } })).toBe( 1, ); expect( await db.orchestrationBudgetLedger.aggregate({ where: { taskId: t.id }, _sum: { amountCents: true }, }), ).toMatchObject({ _sum: { amountCents: 20 } }); expect( await service.project( await db.orchestrationTask.findUniqueOrThrow({ where: { id: t.id } }), admin, ), ).toMatchObject({ metadata: { scope: 'Staging schema audit only', acceptanceCriteria: ['Report all schema failures'], environment: 'staging', commitSha: 'a'.repeat(40), }, }); }); });