578 lines
17 KiB
TypeScript
578 lines
17 KiB
TypeScript
import { OrchestrationController } from './orchestration.controller';
|
|
import { DepartmentTasksController } from '../admin/department-tasks.controller';
|
|
import { ApprovalsController } from '../admin/approvals.controller';
|
|
import { PrismaClient } from '@prisma/client';
|
|
import { OrchestrationService } from './orchestration.service';
|
|
import { KillSwitchService } from './kill-switch.service';
|
|
import { WorkflowState, OperationMode } from './orchestration.domain';
|
|
const enabled = process.env.ORCHESTRATION_INTEGRATION === '1';
|
|
(enabled ? describe : describe.skip)(
|
|
'v4 disposable PostgreSQL invariants',
|
|
() => {
|
|
const db = new PrismaClient();
|
|
const service = new OrchestrationService(
|
|
db as never,
|
|
new KillSwitchService(db as never),
|
|
);
|
|
let n = 0;
|
|
async function task(state = 'DRAFT', budget = 100) {
|
|
return db.orchestrationTask.create({
|
|
data: {
|
|
key: `test-${Date.now()}-${n++}`,
|
|
title: 'test',
|
|
department: 'TEST',
|
|
operationMode: 'PREPARE',
|
|
state,
|
|
budgetLimitCents: budget,
|
|
},
|
|
});
|
|
}
|
|
afterAll(async () => db.$disconnect());
|
|
it('release rejects PREPARE operation class even with gate approval', async()=>{
|
|
const t=await task('GATE2_APPROVED');
|
|
await expect(service.command(t.id,'RELEASE_AUTHORIZE',{expectedVersion:1,idempotencyKey:`mode-${t.id}`},'ci')).rejects.toThrow(/operation mode/i);
|
|
});
|
|
|
|
it('append-only ledger rejects direct rewrites in PostgreSQL', async () => {
|
|
const t = await task();
|
|
const r = await service.recordBudget(
|
|
t.id,
|
|
{
|
|
amountCents: 50,
|
|
kind: 'RESERVE',
|
|
idempotencyKey: `immutable-${t.id}`,
|
|
},
|
|
'a',
|
|
);
|
|
await expect(
|
|
db.orchestrationBudgetLedger.update({
|
|
where: { id: r.id },
|
|
data: { amountCents: 0 },
|
|
}),
|
|
).rejects.toThrow(/append.only/i);
|
|
await expect(
|
|
db.orchestrationBudgetLedger.delete({ where: { id: r.id } }),
|
|
).rejects.toThrow(/append.only/i);
|
|
});
|
|
it('generic transitions cannot fake run lifecycle or production verification', async () => {
|
|
for (const [from, to] of [
|
|
['CLAIMED', 'RUNNING'],
|
|
['RUNNING', 'VERIFYING'],
|
|
['FAILED', 'READY'],
|
|
['PRODUCTION_VERIFYING', 'LIVE_VERIFIED'],
|
|
]) {
|
|
const t = await task(from);
|
|
await expect(
|
|
service.transition(
|
|
t.id,
|
|
{
|
|
expectedVersion: 1,
|
|
idempotencyKey: `fake-${t.id}`,
|
|
toState: to as WorkflowState,
|
|
},
|
|
'agent',
|
|
),
|
|
).rejects.toThrow(/dedicated/i);
|
|
}
|
|
});
|
|
it('exposes command and reconcile HTTP handlers', () => {
|
|
const c = new OrchestrationController(service);
|
|
expect(typeof c.command).toBe('function');
|
|
expect(typeof c.reconcile).toBe('function');
|
|
});
|
|
it('highest kill level blocks reads and reconciliation', async () => {
|
|
const t = await task();
|
|
const r = await service.recordBudget(
|
|
t.id,
|
|
{
|
|
kind: 'RESERVE',
|
|
amountCents: 10,
|
|
idempotencyKey: `stop-res-${t.id}`,
|
|
},
|
|
'a',
|
|
);
|
|
await service.setKillSwitch(
|
|
['EMERGENCY', 'HALT'].join('_'),
|
|
'test',
|
|
'ceo',
|
|
);
|
|
try {
|
|
await expect(service.findOne(t.id)).rejects.toThrow(/kill switch/i);
|
|
await expect(
|
|
service.reconcileBudget(
|
|
t.id,
|
|
{
|
|
reservationId: r.id,
|
|
actualCents: 0,
|
|
idempotencyKey: `stop-recon-${t.id}`,
|
|
},
|
|
'a',
|
|
),
|
|
).rejects.toThrow(/kill switch/i);
|
|
} finally {
|
|
await db.orchestrationKillSwitch.deleteMany();
|
|
}
|
|
});
|
|
it('legacy arbitrary status and approval endpoints fail closed', async () => {
|
|
await expect(
|
|
new DepartmentTasksController(db as never).create({
|
|
title: 'bypass',
|
|
status: 'DONE',
|
|
}),
|
|
).rejects.toThrow(/v4/i);
|
|
await expect(
|
|
new DepartmentTasksController(db as never).updateStatus('none', {
|
|
status: 'DONE',
|
|
}),
|
|
).rejects.toThrow(/v4/i);
|
|
await expect(
|
|
new ApprovalsController(db as never).decide('none', {
|
|
decision: 'APPROVED',
|
|
} as any),
|
|
).rejects.toThrow(/v4/i);
|
|
});
|
|
it('release consumes only matching fresh production Gate2 via dedicated command', async () => {
|
|
const t = await task('GATE2_PENDING');
|
|
const hash = 'f'.repeat(64),
|
|
commitSha = 'a'.repeat(40);
|
|
const r = await db.orchestrationRun.create({
|
|
data: {
|
|
taskId: t.id,
|
|
status: 'SUCCEEDED',
|
|
commitSha,
|
|
environment: 'production',
|
|
idempotencyKey: `rel-run-${t.id}`,
|
|
},
|
|
});
|
|
await db.orchestrationArtifact.create({
|
|
data: {
|
|
runId: r.id,
|
|
kind: 'BUILD',
|
|
uri: 'https://ci.example/build',
|
|
sha256: hash,
|
|
},
|
|
});
|
|
await db.orchestrationAttestation.create({
|
|
data: {
|
|
runId: r.id,
|
|
kind: 'TEST',
|
|
subjectHash: hash,
|
|
signer: 'ci',
|
|
predicate: { passed: true },
|
|
},
|
|
});
|
|
const decision = await service.decideGate(
|
|
t.id,
|
|
2,
|
|
{
|
|
decision: 'APPROVED',
|
|
expectedVersion: 1,
|
|
idempotencyKey: `rel-approve-${t.id}`,
|
|
expiresAt: new Date(Date.now() + 60000).toISOString(),
|
|
runId: r.id,
|
|
artifactHash: hash,
|
|
commitSha,
|
|
environment: 'production',
|
|
},
|
|
'ceo',
|
|
);
|
|
const dto = {
|
|
expectedVersion: 2,
|
|
idempotencyKey: `rel-${t.id}`,
|
|
approvalId: decision.id,
|
|
runId: r.id,
|
|
artifactHash: hash,
|
|
commitSha,
|
|
environment: 'production',
|
|
};
|
|
await expect(
|
|
(service as any).command(
|
|
t.id,
|
|
'RELEASE_AUTHORIZE',
|
|
{ ...dto, artifactHash: '0'.repeat(64) },
|
|
'ci',
|
|
),
|
|
).rejects.toThrow(/binding/i);
|
|
const result = await (service as any).command(
|
|
t.id,
|
|
'RELEASE_AUTHORIZE',
|
|
dto,
|
|
'ci',
|
|
);
|
|
expect(result.authorized).toBe(true);
|
|
expect(
|
|
(await db.orchestrationTask.findUniqueOrThrow({ where: { id: t.id } }))
|
|
.state,
|
|
).toBe('RELEASING');
|
|
await expect(
|
|
(service as any).command(
|
|
t.id,
|
|
'RELEASE_AUTHORIZE',
|
|
{ ...dto, expectedVersion: 3, idempotencyKey: `rel-again-${t.id}` },
|
|
'ci',
|
|
),
|
|
).rejects.toThrow();
|
|
});
|
|
it('gate decision replay rejects changed immutable expiry and actor', async () => {
|
|
const t = await task('GATE1_PENDING');
|
|
const dto = {
|
|
decision: 'APPROVED' as const,
|
|
expectedVersion: 1,
|
|
idempotencyKey: `bind-${t.id}`,
|
|
expiresAt: new Date(Date.now() + 60000).toISOString(),
|
|
};
|
|
await service.decideGate(t.id, 1, dto, 'ceo');
|
|
await expect(
|
|
service.decideGate(
|
|
t.id,
|
|
1,
|
|
{ ...dto, expiresAt: new Date(Date.now() + 120000).toISOString() },
|
|
'ceo',
|
|
),
|
|
).rejects.toThrow(/Idempotency/i);
|
|
await expect(service.decideGate(t.id, 1, dto, 'other')).rejects.toThrow(
|
|
/Idempotency/i,
|
|
);
|
|
});
|
|
it('dependencies block admission and reject cycles', async () => {
|
|
const a = await task();
|
|
const b = await task();
|
|
await (service as any).command(
|
|
a.id,
|
|
'DEPENDENCY_ADD',
|
|
{
|
|
expectedVersion: 1,
|
|
idempotencyKey: `dep-a-${a.id}`,
|
|
dependsOnTaskId: b.id,
|
|
},
|
|
'actor',
|
|
);
|
|
await expect(
|
|
(service as any).command(
|
|
b.id,
|
|
'DEPENDENCY_ADD',
|
|
{
|
|
expectedVersion: 1,
|
|
idempotencyKey: `dep-b-${b.id}`,
|
|
dependsOnTaskId: a.id,
|
|
},
|
|
'actor',
|
|
),
|
|
).rejects.toThrow(/cycle/i);
|
|
await db.orchestrationTask.update({
|
|
where: { id: a.id },
|
|
data: { state: 'GATE1_PENDING' },
|
|
});
|
|
await service.decideGate(
|
|
a.id,
|
|
1,
|
|
{
|
|
decision: 'APPROVED',
|
|
expectedVersion: 2,
|
|
idempotencyKey: `dep-approval-${a.id}`,
|
|
expiresAt: new Date(Date.now() + 60000).toISOString(),
|
|
},
|
|
'ceo',
|
|
);
|
|
await expect(
|
|
service.transition(
|
|
a.id,
|
|
{
|
|
toState: WorkflowState.READY,
|
|
expectedVersion: 3,
|
|
idempotencyKey: `dep-ready-${a.id}`,
|
|
},
|
|
'agent',
|
|
),
|
|
).rejects.toThrow(/dependency/i);
|
|
});
|
|
it('registers artifact and attestation for owned run, blocks unregistered subjects', async () => {
|
|
const t = await task('CLAIMED');
|
|
const base = {
|
|
expectedVersion: 1,
|
|
idempotencyKey: `life-${t.id}`,
|
|
commitSha: 'c'.repeat(40),
|
|
environment: 'staging',
|
|
};
|
|
const r = await (service as any).command(
|
|
t.id,
|
|
'RUN_CREATE',
|
|
base,
|
|
'agent',
|
|
);
|
|
const hash = 'd'.repeat(64);
|
|
const a = await (service as any).command(
|
|
t.id,
|
|
'ARTIFACT_ADD',
|
|
{
|
|
expectedVersion: 2,
|
|
idempotencyKey: `art-${t.id}`,
|
|
runId: r.id,
|
|
sha256: hash,
|
|
kind: 'BUILD',
|
|
uri: 'https://ci.example/artifact',
|
|
},
|
|
'agent',
|
|
);
|
|
expect(a.sha256).toBe(hash);
|
|
await expect(
|
|
(service as any).command(
|
|
t.id,
|
|
'ATTESTATION_ADD',
|
|
{
|
|
expectedVersion: 3,
|
|
idempotencyKey: `bad-att-${t.id}`,
|
|
runId: r.id,
|
|
subjectHash: 'e'.repeat(64),
|
|
kind: 'TEST',
|
|
predicate: { passed: true },
|
|
},
|
|
'ci',
|
|
),
|
|
).rejects.toThrow(/artifact/i);
|
|
const att = await (service as any).command(
|
|
t.id,
|
|
'ATTESTATION_ADD',
|
|
{
|
|
expectedVersion: 3,
|
|
idempotencyKey: `att-${t.id}`,
|
|
runId: r.id,
|
|
subjectHash: hash,
|
|
kind: 'TEST',
|
|
predicate: { passed: true },
|
|
},
|
|
'ci',
|
|
);
|
|
expect(att.signer).toBe('ci');
|
|
});
|
|
it('creates immutable versioned runs with bounded transient retries', async () => {
|
|
const t = await task('CLAIMED');
|
|
const dto = {
|
|
expectedVersion: 1,
|
|
idempotencyKey: `run-${t.id}`,
|
|
commitSha: 'a'.repeat(40),
|
|
environment: 'staging',
|
|
};
|
|
const r = await (service as any).command(
|
|
t.id,
|
|
'RUN_CREATE',
|
|
dto,
|
|
'agent',
|
|
);
|
|
expect(r.attempt).toBe(1);
|
|
expect(
|
|
await (service as any).command(t.id, 'RUN_CREATE', dto, 'agent'),
|
|
).toEqual(r);
|
|
await expect(
|
|
(service as any).command(
|
|
t.id,
|
|
'RUN_CREATE',
|
|
{ ...dto, commitSha: 'b'.repeat(40) },
|
|
'agent',
|
|
),
|
|
).rejects.toThrow(/Idempotency/i);
|
|
await (service as any).command(
|
|
t.id,
|
|
'RUN_FINISH',
|
|
{
|
|
expectedVersion: 2,
|
|
idempotencyKey: `finish-${t.id}`,
|
|
runId: r.id,
|
|
status: 'FAILED',
|
|
retryClass: 'POLICY',
|
|
},
|
|
'agent',
|
|
);
|
|
await expect(
|
|
(service as any).command(
|
|
t.id,
|
|
'RUN_CREATE',
|
|
{ ...dto, expectedVersion: 3, idempotencyKey: `retry-${t.id}` },
|
|
'agent',
|
|
),
|
|
).rejects.toThrow(/retry/i);
|
|
});
|
|
it('selects tier at admission using backend risk and budget', async () => {
|
|
const t = await task('GATE1_PENDING', 1000);
|
|
await service.decideGate(
|
|
t.id,
|
|
1,
|
|
{
|
|
decision: 'APPROVED',
|
|
expectedVersion: 1,
|
|
idempotencyKey: `approve-tier-${t.id}`,
|
|
expiresAt: new Date(Date.now() + 60000).toISOString(),
|
|
},
|
|
'ceo',
|
|
);
|
|
await service.transition(
|
|
t.id,
|
|
{
|
|
toState: WorkflowState.READY,
|
|
expectedVersion: 2,
|
|
idempotencyKey: `admit-tier-${t.id}`,
|
|
},
|
|
'agent',
|
|
);
|
|
expect(
|
|
(await db.orchestrationTask.findUniqueOrThrow({ where: { id: t.id } }))
|
|
.modelTier,
|
|
).toBe('L0');
|
|
});
|
|
it('STOP_AGENTS prevents claims and running transitions', async () => {
|
|
await service.setKillSwitch('STOP_AGENTS', 'test', 'ceo');
|
|
try {
|
|
const t = await task('READY');
|
|
await expect(
|
|
service.transition(
|
|
t.id,
|
|
{
|
|
toState: WorkflowState.CLAIMED,
|
|
expectedVersion: 1,
|
|
idempotencyKey: `kill-${t.id}`,
|
|
},
|
|
'agent',
|
|
),
|
|
).rejects.toThrow(/kill switch/i);
|
|
} finally {
|
|
await db.orchestrationKillSwitch.deleteMany();
|
|
}
|
|
});
|
|
it('Gate1 admission consumes only unexpired approval', async () => {
|
|
const t = await task('GATE1_PENDING');
|
|
await service.decideGate(
|
|
t.id,
|
|
1,
|
|
{
|
|
decision: 'APPROVED',
|
|
expectedVersion: 1,
|
|
idempotencyKey: `approve-${t.id}`,
|
|
expiresAt: new Date(Date.now() + 100).toISOString(),
|
|
},
|
|
'ceo',
|
|
);
|
|
await new Promise((r) => setTimeout(r, 150));
|
|
await expect(
|
|
service.transition(
|
|
t.id,
|
|
{
|
|
toState: WorkflowState.READY,
|
|
expectedVersion: 2,
|
|
idempotencyKey: `admit-${t.id}`,
|
|
},
|
|
'agent',
|
|
),
|
|
).rejects.toThrow(/expired/i);
|
|
});
|
|
it('versioned transition replay returns original result and rejects changed payload', async () => {
|
|
const t = await task();
|
|
const dto = {
|
|
toState: WorkflowState.GATE1_PENDING,
|
|
expectedVersion: 1,
|
|
idempotencyKey: `tr-${t.id}`,
|
|
};
|
|
const first = await service.transition(t.id, dto, 'actor');
|
|
expect(await service.transition(t.id, dto, 'actor')).toEqual(first);
|
|
await expect(
|
|
service.transition(t.id, { ...dto, reason: 'different' }, 'actor'),
|
|
).rejects.toThrow(/Idempotency/i);
|
|
});
|
|
it('reconciles a reservation exactly once without erasing spent cost', async () => {
|
|
const t = await task();
|
|
const r = await service.recordBudget(
|
|
t.id,
|
|
{ amountCents: 80, kind: 'RESERVE', idempotencyKey: `res-${t.id}` },
|
|
'agent',
|
|
);
|
|
const dto = {
|
|
reservationId: r.id,
|
|
actualCents: 35,
|
|
idempotencyKey: `reconcile-${t.id}`,
|
|
};
|
|
await (service as any).reconcileBudget(t.id, dto, 'agent');
|
|
await (service as any).reconcileBudget(t.id, dto, 'agent');
|
|
await expect(
|
|
(service as any).reconcileBudget(
|
|
t.id,
|
|
{ ...dto, idempotencyKey: `second-${t.id}` },
|
|
'agent',
|
|
),
|
|
).rejects.toThrow(/reconciled/i);
|
|
expect(
|
|
(
|
|
await db.orchestrationBudgetLedger.aggregate({
|
|
where: { taskId: t.id },
|
|
_sum: { amountCents: true },
|
|
})
|
|
)._sum.amountCents,
|
|
).toBe(35);
|
|
await expect(
|
|
service.recordBudget(
|
|
t.id,
|
|
{
|
|
amountCents: 66,
|
|
kind: 'RESERVE',
|
|
idempotencyKey: `too-much-${t.id}`,
|
|
},
|
|
'agent',
|
|
),
|
|
).rejects.toThrow(/budget/i);
|
|
});
|
|
it('concurrent budget reservations never exceed hard cap and replay is harmless', async () => {
|
|
const t = await task();
|
|
const result = await Promise.allSettled(
|
|
Array.from({ length: 8 }, (_, i) =>
|
|
service.recordBudget(
|
|
t.id,
|
|
{
|
|
amountCents: 30,
|
|
kind: 'RESERVE',
|
|
idempotencyKey: `reserve-${t.id}-${i}`,
|
|
},
|
|
'agent',
|
|
),
|
|
),
|
|
);
|
|
expect(result.filter((x) => x.status === 'fulfilled')).toHaveLength(3);
|
|
const sum = await db.orchestrationBudgetLedger.aggregate({
|
|
where: { taskId: t.id },
|
|
_sum: { amountCents: true },
|
|
});
|
|
expect(sum._sum.amountCents).toBe(90);
|
|
const first = await db.orchestrationBudgetLedger.findFirstOrThrow({
|
|
where: { taskId: t.id },
|
|
});
|
|
await service.recordBudget(
|
|
t.id,
|
|
{
|
|
amountCents: 30,
|
|
kind: 'RESERVE',
|
|
idempotencyKey: first.idempotencyKey,
|
|
},
|
|
'agent',
|
|
);
|
|
expect(
|
|
await db.orchestrationBudgetLedger.count({ where: { taskId: t.id } }),
|
|
).toBe(3);
|
|
});
|
|
it('generic transition cannot approve gate 1', async () => {
|
|
const t = await task('GATE1_PENDING');
|
|
await expect(
|
|
service.transition(
|
|
t.id,
|
|
{
|
|
toState: WorkflowState.GATE1_APPROVED,
|
|
expectedVersion: 1,
|
|
idempotencyKey: `gate-bypass-${n}`,
|
|
},
|
|
'ceo',
|
|
),
|
|
).rejects.toThrow(/gate/i);
|
|
expect(
|
|
(await db.orchestrationTask.findUniqueOrThrow({ where: { id: t.id } }))
|
|
.state,
|
|
).toBe('GATE1_PENDING');
|
|
});
|
|
},
|
|
);
|