import { OrchestrationController } from './orchestration.controller'; import { DepartmentTasksController } from '../admin/department-tasks.controller'; import { ApprovalsController } from '../admin/approvals.controller'; import { PrismaClient } from '@prisma/client'; import { OrchestrationService } from './orchestration.service'; import { KillSwitchService } from './kill-switch.service'; import { WorkflowState, OperationMode } from './orchestration.domain'; const enabled = process.env.ORCHESTRATION_INTEGRATION === '1'; (enabled ? describe : describe.skip)( 'v4 disposable PostgreSQL invariants', () => { const db = new PrismaClient(); const service = new OrchestrationService( db as never, new KillSwitchService(db as never), ); let n = 0; async function task(state = 'DRAFT', budget = 100) { return db.orchestrationTask.create({ data: { key: `test-${Date.now()}-${n++}`, title: 'test', department: 'TEST', operationMode: 'PREPARE', state, budgetLimitCents: budget, }, }); } afterAll(async () => db.$disconnect()); it('release rejects PREPARE operation class even with gate approval', async()=>{ const t=await task('GATE2_APPROVED'); await expect(service.command(t.id,'RELEASE_AUTHORIZE',{expectedVersion:1,idempotencyKey:`mode-${t.id}`},'ci')).rejects.toThrow(/operation mode/i); }); it('append-only ledger rejects direct rewrites in PostgreSQL', async () => { const t = await task(); const r = await service.recordBudget( t.id, { amountCents: 50, kind: 'RESERVE', idempotencyKey: `immutable-${t.id}`, }, 'a', ); await expect( db.orchestrationBudgetLedger.update({ where: { id: r.id }, data: { amountCents: 0 }, }), ).rejects.toThrow(/append.only/i); await expect( db.orchestrationBudgetLedger.delete({ where: { id: r.id } }), ).rejects.toThrow(/append.only/i); }); it('generic transitions cannot fake run lifecycle or production verification', async () => { for (const [from, to] of [ ['CLAIMED', 'RUNNING'], ['RUNNING', 'VERIFYING'], ['FAILED', 'READY'], ['PRODUCTION_VERIFYING', 'LIVE_VERIFIED'], ]) { const t = await task(from); await expect( service.transition( t.id, { expectedVersion: 1, idempotencyKey: `fake-${t.id}`, toState: to as WorkflowState, }, 'agent', ), ).rejects.toThrow(/dedicated/i); } }); it('exposes command and reconcile HTTP handlers', () => { const c = new OrchestrationController(service); expect(typeof c.command).toBe('function'); expect(typeof c.reconcile).toBe('function'); }); it('highest kill level blocks reads and reconciliation', async () => { const t = await task(); const r = await service.recordBudget( t.id, { kind: 'RESERVE', amountCents: 10, idempotencyKey: `stop-res-${t.id}`, }, 'a', ); await service.setKillSwitch( ['EMERGENCY', 'HALT'].join('_'), 'test', 'ceo', ); try { await expect(service.findOne(t.id)).rejects.toThrow(/kill switch/i); await expect( service.reconcileBudget( t.id, { reservationId: r.id, actualCents: 0, idempotencyKey: `stop-recon-${t.id}`, }, 'a', ), ).rejects.toThrow(/kill switch/i); } finally { await db.orchestrationKillSwitch.deleteMany(); } }); it('legacy arbitrary status and approval endpoints fail closed', async () => { await expect( new DepartmentTasksController(db as never).create({ title: 'bypass', status: 'DONE', }), ).rejects.toThrow(/v4/i); await expect( new DepartmentTasksController(db as never).updateStatus('none', { status: 'DONE', }), ).rejects.toThrow(/v4/i); await expect( new ApprovalsController(db as never).decide('none', { decision: 'APPROVED', } as any), ).rejects.toThrow(/v4/i); }); it('release consumes only matching fresh production Gate2 via dedicated command', async () => { const t = await task('GATE2_PENDING'); const hash = 'f'.repeat(64), commitSha = 'a'.repeat(40); const r = await db.orchestrationRun.create({ data: { taskId: t.id, status: 'SUCCEEDED', commitSha, environment: 'production', idempotencyKey: `rel-run-${t.id}`, }, }); await db.orchestrationArtifact.create({ data: { runId: r.id, kind: 'BUILD', uri: 'https://ci.example/build', sha256: hash, }, }); await db.orchestrationAttestation.create({ data: { runId: r.id, kind: 'TEST', subjectHash: hash, signer: 'ci', predicate: { passed: true }, }, }); const decision = await service.decideGate( t.id, 2, { decision: 'APPROVED', expectedVersion: 1, idempotencyKey: `rel-approve-${t.id}`, expiresAt: new Date(Date.now() + 60000).toISOString(), runId: r.id, artifactHash: hash, commitSha, environment: 'production', }, 'ceo', ); const dto = { expectedVersion: 2, idempotencyKey: `rel-${t.id}`, approvalId: decision.id, runId: r.id, artifactHash: hash, commitSha, environment: 'production', }; await expect( (service as any).command( t.id, 'RELEASE_AUTHORIZE', { ...dto, artifactHash: '0'.repeat(64) }, 'ci', ), ).rejects.toThrow(/binding/i); const result = await (service as any).command( t.id, 'RELEASE_AUTHORIZE', dto, 'ci', ); expect(result.authorized).toBe(true); expect( (await db.orchestrationTask.findUniqueOrThrow({ where: { id: t.id } })) .state, ).toBe('RELEASING'); await expect( (service as any).command( t.id, 'RELEASE_AUTHORIZE', { ...dto, expectedVersion: 3, idempotencyKey: `rel-again-${t.id}` }, 'ci', ), ).rejects.toThrow(); }); it('gate decision replay rejects changed immutable expiry and actor', async () => { const t = await task('GATE1_PENDING'); const dto = { decision: 'APPROVED' as const, expectedVersion: 1, idempotencyKey: `bind-${t.id}`, expiresAt: new Date(Date.now() + 60000).toISOString(), }; await service.decideGate(t.id, 1, dto, 'ceo'); await expect( service.decideGate( t.id, 1, { ...dto, expiresAt: new Date(Date.now() + 120000).toISOString() }, 'ceo', ), ).rejects.toThrow(/Idempotency/i); await expect(service.decideGate(t.id, 1, dto, 'other')).rejects.toThrow( /Idempotency/i, ); }); it('dependencies block admission and reject cycles', async () => { const a = await task(); const b = await task(); await (service as any).command( a.id, 'DEPENDENCY_ADD', { expectedVersion: 1, idempotencyKey: `dep-a-${a.id}`, dependsOnTaskId: b.id, }, 'actor', ); await expect( (service as any).command( b.id, 'DEPENDENCY_ADD', { expectedVersion: 1, idempotencyKey: `dep-b-${b.id}`, dependsOnTaskId: a.id, }, 'actor', ), ).rejects.toThrow(/cycle/i); await db.orchestrationTask.update({ where: { id: a.id }, data: { state: 'GATE1_PENDING' }, }); await service.decideGate( a.id, 1, { decision: 'APPROVED', expectedVersion: 2, idempotencyKey: `dep-approval-${a.id}`, expiresAt: new Date(Date.now() + 60000).toISOString(), }, 'ceo', ); await expect( service.transition( a.id, { toState: WorkflowState.READY, expectedVersion: 3, idempotencyKey: `dep-ready-${a.id}`, }, 'agent', ), ).rejects.toThrow(/dependency/i); }); it('registers artifact and attestation for owned run, blocks unregistered subjects', async () => { const t = await task('CLAIMED'); const base = { expectedVersion: 1, idempotencyKey: `life-${t.id}`, commitSha: 'c'.repeat(40), environment: 'staging', }; const r = await (service as any).command( t.id, 'RUN_CREATE', base, 'agent', ); const hash = 'd'.repeat(64); const a = await (service as any).command( t.id, 'ARTIFACT_ADD', { expectedVersion: 2, idempotencyKey: `art-${t.id}`, runId: r.id, sha256: hash, kind: 'BUILD', uri: 'https://ci.example/artifact', }, 'agent', ); expect(a.sha256).toBe(hash); await expect( (service as any).command( t.id, 'ATTESTATION_ADD', { expectedVersion: 3, idempotencyKey: `bad-att-${t.id}`, runId: r.id, subjectHash: 'e'.repeat(64), kind: 'TEST', predicate: { passed: true }, }, 'ci', ), ).rejects.toThrow(/artifact/i); const att = await (service as any).command( t.id, 'ATTESTATION_ADD', { expectedVersion: 3, idempotencyKey: `att-${t.id}`, runId: r.id, subjectHash: hash, kind: 'TEST', predicate: { passed: true }, }, 'ci', ); expect(att.signer).toBe('ci'); }); it('creates immutable versioned runs with bounded transient retries', async () => { const t = await task('CLAIMED'); const dto = { expectedVersion: 1, idempotencyKey: `run-${t.id}`, commitSha: 'a'.repeat(40), environment: 'staging', }; const r = await (service as any).command( t.id, 'RUN_CREATE', dto, 'agent', ); expect(r.attempt).toBe(1); expect( await (service as any).command(t.id, 'RUN_CREATE', dto, 'agent'), ).toEqual(r); await expect( (service as any).command( t.id, 'RUN_CREATE', { ...dto, commitSha: 'b'.repeat(40) }, 'agent', ), ).rejects.toThrow(/Idempotency/i); await (service as any).command( t.id, 'RUN_FINISH', { expectedVersion: 2, idempotencyKey: `finish-${t.id}`, runId: r.id, status: 'FAILED', retryClass: 'POLICY', }, 'agent', ); await expect( (service as any).command( t.id, 'RUN_CREATE', { ...dto, expectedVersion: 3, idempotencyKey: `retry-${t.id}` }, 'agent', ), ).rejects.toThrow(/retry/i); }); it('selects tier at admission using backend risk and budget', async () => { const t = await task('GATE1_PENDING', 1000); await service.decideGate( t.id, 1, { decision: 'APPROVED', expectedVersion: 1, idempotencyKey: `approve-tier-${t.id}`, expiresAt: new Date(Date.now() + 60000).toISOString(), }, 'ceo', ); await service.transition( t.id, { toState: WorkflowState.READY, expectedVersion: 2, idempotencyKey: `admit-tier-${t.id}`, }, 'agent', ); expect( (await db.orchestrationTask.findUniqueOrThrow({ where: { id: t.id } })) .modelTier, ).toBe('L0'); }); it('STOP_AGENTS prevents claims and running transitions', async () => { await service.setKillSwitch('STOP_AGENTS', 'test', 'ceo'); try { const t = await task('READY'); await expect( service.transition( t.id, { toState: WorkflowState.CLAIMED, expectedVersion: 1, idempotencyKey: `kill-${t.id}`, }, 'agent', ), ).rejects.toThrow(/kill switch/i); } finally { await db.orchestrationKillSwitch.deleteMany(); } }); it('Gate1 admission consumes only unexpired approval', async () => { const t = await task('GATE1_PENDING'); await service.decideGate( t.id, 1, { decision: 'APPROVED', expectedVersion: 1, idempotencyKey: `approve-${t.id}`, expiresAt: new Date(Date.now() + 100).toISOString(), }, 'ceo', ); await new Promise((r) => setTimeout(r, 150)); await expect( service.transition( t.id, { toState: WorkflowState.READY, expectedVersion: 2, idempotencyKey: `admit-${t.id}`, }, 'agent', ), ).rejects.toThrow(/expired/i); }); it('versioned transition replay returns original result and rejects changed payload', async () => { const t = await task(); const dto = { toState: WorkflowState.GATE1_PENDING, expectedVersion: 1, idempotencyKey: `tr-${t.id}`, }; const first = await service.transition(t.id, dto, 'actor'); expect(await service.transition(t.id, dto, 'actor')).toEqual(first); await expect( service.transition(t.id, { ...dto, reason: 'different' }, 'actor'), ).rejects.toThrow(/Idempotency/i); }); it('reconciles a reservation exactly once without erasing spent cost', async () => { const t = await task(); const r = await service.recordBudget( t.id, { amountCents: 80, kind: 'RESERVE', idempotencyKey: `res-${t.id}` }, 'agent', ); const dto = { reservationId: r.id, actualCents: 35, idempotencyKey: `reconcile-${t.id}`, }; await (service as any).reconcileBudget(t.id, dto, 'agent'); await (service as any).reconcileBudget(t.id, dto, 'agent'); await expect( (service as any).reconcileBudget( t.id, { ...dto, idempotencyKey: `second-${t.id}` }, 'agent', ), ).rejects.toThrow(/reconciled/i); expect( ( await db.orchestrationBudgetLedger.aggregate({ where: { taskId: t.id }, _sum: { amountCents: true }, }) )._sum.amountCents, ).toBe(35); await expect( service.recordBudget( t.id, { amountCents: 66, kind: 'RESERVE', idempotencyKey: `too-much-${t.id}`, }, 'agent', ), ).rejects.toThrow(/budget/i); }); it('concurrent budget reservations never exceed hard cap and replay is harmless', async () => { const t = await task(); const result = await Promise.allSettled( Array.from({ length: 8 }, (_, i) => service.recordBudget( t.id, { amountCents: 30, kind: 'RESERVE', idempotencyKey: `reserve-${t.id}-${i}`, }, 'agent', ), ), ); expect(result.filter((x) => x.status === 'fulfilled')).toHaveLength(3); const sum = await db.orchestrationBudgetLedger.aggregate({ where: { taskId: t.id }, _sum: { amountCents: true }, }); expect(sum._sum.amountCents).toBe(90); const first = await db.orchestrationBudgetLedger.findFirstOrThrow({ where: { taskId: t.id }, }); await service.recordBudget( t.id, { amountCents: 30, kind: 'RESERVE', idempotencyKey: first.idempotencyKey, }, 'agent', ); expect( await db.orchestrationBudgetLedger.count({ where: { taskId: t.id } }), ).toBe(3); }); it('generic transition cannot approve gate 1', async () => { const t = await task('GATE1_PENDING'); await expect( service.transition( t.id, { toState: WorkflowState.GATE1_APPROVED, expectedVersion: 1, idempotencyKey: `gate-bypass-${n}`, }, 'ceo', ), ).rejects.toThrow(/gate/i); expect( (await db.orchestrationTask.findUniqueOrThrow({ where: { id: t.id } })) .state, ).toBe('GATE1_PENDING'); }); }, );