Files
canina/backend/src/orchestration/orchestration.integration.spec.ts
T

578 lines
17 KiB
TypeScript

import { OrchestrationController } from './orchestration.controller';
import { DepartmentTasksController } from '../admin/department-tasks.controller';
import { ApprovalsController } from '../admin/approvals.controller';
import { PrismaClient } from '@prisma/client';
import { OrchestrationService } from './orchestration.service';
import { KillSwitchService } from './kill-switch.service';
import { WorkflowState, OperationMode } from './orchestration.domain';
const enabled = process.env.ORCHESTRATION_INTEGRATION === '1';
(enabled ? describe : describe.skip)(
'v4 disposable PostgreSQL invariants',
() => {
const db = new PrismaClient();
const service = new OrchestrationService(
db as never,
new KillSwitchService(db as never),
);
let n = 0;
async function task(state = 'DRAFT', budget = 100) {
return db.orchestrationTask.create({
data: {
key: `test-${Date.now()}-${n++}`,
title: 'test',
department: 'TEST',
operationMode: 'PREPARE',
state,
budgetLimitCents: budget,
},
});
}
afterAll(async () => db.$disconnect());
it('release rejects PREPARE operation class even with gate approval', async()=>{
const t=await task('GATE2_APPROVED');
await expect(service.command(t.id,'RELEASE_AUTHORIZE',{expectedVersion:1,idempotencyKey:`mode-${t.id}`},'ci')).rejects.toThrow(/operation mode/i);
});
it('append-only ledger rejects direct rewrites in PostgreSQL', async () => {
const t = await task();
const r = await service.recordBudget(
t.id,
{
amountCents: 50,
kind: 'RESERVE',
idempotencyKey: `immutable-${t.id}`,
},
'a',
);
await expect(
db.orchestrationBudgetLedger.update({
where: { id: r.id },
data: { amountCents: 0 },
}),
).rejects.toThrow(/append.only/i);
await expect(
db.orchestrationBudgetLedger.delete({ where: { id: r.id } }),
).rejects.toThrow(/append.only/i);
});
it('generic transitions cannot fake run lifecycle or production verification', async () => {
for (const [from, to] of [
['CLAIMED', 'RUNNING'],
['RUNNING', 'VERIFYING'],
['FAILED', 'READY'],
['PRODUCTION_VERIFYING', 'LIVE_VERIFIED'],
]) {
const t = await task(from);
await expect(
service.transition(
t.id,
{
expectedVersion: 1,
idempotencyKey: `fake-${t.id}`,
toState: to as WorkflowState,
},
'agent',
),
).rejects.toThrow(/dedicated/i);
}
});
it('exposes command and reconcile HTTP handlers', () => {
const c = new OrchestrationController(service);
expect(typeof c.command).toBe('function');
expect(typeof c.reconcile).toBe('function');
});
it('highest kill level blocks reads and reconciliation', async () => {
const t = await task();
const r = await service.recordBudget(
t.id,
{
kind: 'RESERVE',
amountCents: 10,
idempotencyKey: `stop-res-${t.id}`,
},
'a',
);
await service.setKillSwitch(
['EMERGENCY', 'HALT'].join('_'),
'test',
'ceo',
);
try {
await expect(service.findOne(t.id)).rejects.toThrow(/kill switch/i);
await expect(
service.reconcileBudget(
t.id,
{
reservationId: r.id,
actualCents: 0,
idempotencyKey: `stop-recon-${t.id}`,
},
'a',
),
).rejects.toThrow(/kill switch/i);
} finally {
await db.orchestrationKillSwitch.deleteMany();
}
});
it('legacy arbitrary status and approval endpoints fail closed', async () => {
await expect(
new DepartmentTasksController(db as never).create({
title: 'bypass',
status: 'DONE',
}),
).rejects.toThrow(/v4/i);
await expect(
new DepartmentTasksController(db as never).updateStatus('none', {
status: 'DONE',
}),
).rejects.toThrow(/v4/i);
await expect(
new ApprovalsController(db as never).decide('none', {
decision: 'APPROVED',
} as any),
).rejects.toThrow(/v4/i);
});
it('release consumes only matching fresh production Gate2 via dedicated command', async () => {
const t = await task('GATE2_PENDING');
const hash = 'f'.repeat(64),
commitSha = 'a'.repeat(40);
const r = await db.orchestrationRun.create({
data: {
taskId: t.id,
status: 'SUCCEEDED',
commitSha,
environment: 'production',
idempotencyKey: `rel-run-${t.id}`,
},
});
await db.orchestrationArtifact.create({
data: {
runId: r.id,
kind: 'BUILD',
uri: 'https://ci.example/build',
sha256: hash,
},
});
await db.orchestrationAttestation.create({
data: {
runId: r.id,
kind: 'TEST',
subjectHash: hash,
signer: 'ci',
predicate: { passed: true },
},
});
const decision = await service.decideGate(
t.id,
2,
{
decision: 'APPROVED',
expectedVersion: 1,
idempotencyKey: `rel-approve-${t.id}`,
expiresAt: new Date(Date.now() + 60000).toISOString(),
runId: r.id,
artifactHash: hash,
commitSha,
environment: 'production',
},
'ceo',
);
const dto = {
expectedVersion: 2,
idempotencyKey: `rel-${t.id}`,
approvalId: decision.id,
runId: r.id,
artifactHash: hash,
commitSha,
environment: 'production',
};
await expect(
(service as any).command(
t.id,
'RELEASE_AUTHORIZE',
{ ...dto, artifactHash: '0'.repeat(64) },
'ci',
),
).rejects.toThrow(/binding/i);
const result = await (service as any).command(
t.id,
'RELEASE_AUTHORIZE',
dto,
'ci',
);
expect(result.authorized).toBe(true);
expect(
(await db.orchestrationTask.findUniqueOrThrow({ where: { id: t.id } }))
.state,
).toBe('RELEASING');
await expect(
(service as any).command(
t.id,
'RELEASE_AUTHORIZE',
{ ...dto, expectedVersion: 3, idempotencyKey: `rel-again-${t.id}` },
'ci',
),
).rejects.toThrow();
});
it('gate decision replay rejects changed immutable expiry and actor', async () => {
const t = await task('GATE1_PENDING');
const dto = {
decision: 'APPROVED' as const,
expectedVersion: 1,
idempotencyKey: `bind-${t.id}`,
expiresAt: new Date(Date.now() + 60000).toISOString(),
};
await service.decideGate(t.id, 1, dto, 'ceo');
await expect(
service.decideGate(
t.id,
1,
{ ...dto, expiresAt: new Date(Date.now() + 120000).toISOString() },
'ceo',
),
).rejects.toThrow(/Idempotency/i);
await expect(service.decideGate(t.id, 1, dto, 'other')).rejects.toThrow(
/Idempotency/i,
);
});
it('dependencies block admission and reject cycles', async () => {
const a = await task();
const b = await task();
await (service as any).command(
a.id,
'DEPENDENCY_ADD',
{
expectedVersion: 1,
idempotencyKey: `dep-a-${a.id}`,
dependsOnTaskId: b.id,
},
'actor',
);
await expect(
(service as any).command(
b.id,
'DEPENDENCY_ADD',
{
expectedVersion: 1,
idempotencyKey: `dep-b-${b.id}`,
dependsOnTaskId: a.id,
},
'actor',
),
).rejects.toThrow(/cycle/i);
await db.orchestrationTask.update({
where: { id: a.id },
data: { state: 'GATE1_PENDING' },
});
await service.decideGate(
a.id,
1,
{
decision: 'APPROVED',
expectedVersion: 2,
idempotencyKey: `dep-approval-${a.id}`,
expiresAt: new Date(Date.now() + 60000).toISOString(),
},
'ceo',
);
await expect(
service.transition(
a.id,
{
toState: WorkflowState.READY,
expectedVersion: 3,
idempotencyKey: `dep-ready-${a.id}`,
},
'agent',
),
).rejects.toThrow(/dependency/i);
});
it('registers artifact and attestation for owned run, blocks unregistered subjects', async () => {
const t = await task('CLAIMED');
const base = {
expectedVersion: 1,
idempotencyKey: `life-${t.id}`,
commitSha: 'c'.repeat(40),
environment: 'staging',
};
const r = await (service as any).command(
t.id,
'RUN_CREATE',
base,
'agent',
);
const hash = 'd'.repeat(64);
const a = await (service as any).command(
t.id,
'ARTIFACT_ADD',
{
expectedVersion: 2,
idempotencyKey: `art-${t.id}`,
runId: r.id,
sha256: hash,
kind: 'BUILD',
uri: 'https://ci.example/artifact',
},
'agent',
);
expect(a.sha256).toBe(hash);
await expect(
(service as any).command(
t.id,
'ATTESTATION_ADD',
{
expectedVersion: 3,
idempotencyKey: `bad-att-${t.id}`,
runId: r.id,
subjectHash: 'e'.repeat(64),
kind: 'TEST',
predicate: { passed: true },
},
'ci',
),
).rejects.toThrow(/artifact/i);
const att = await (service as any).command(
t.id,
'ATTESTATION_ADD',
{
expectedVersion: 3,
idempotencyKey: `att-${t.id}`,
runId: r.id,
subjectHash: hash,
kind: 'TEST',
predicate: { passed: true },
},
'ci',
);
expect(att.signer).toBe('ci');
});
it('creates immutable versioned runs with bounded transient retries', async () => {
const t = await task('CLAIMED');
const dto = {
expectedVersion: 1,
idempotencyKey: `run-${t.id}`,
commitSha: 'a'.repeat(40),
environment: 'staging',
};
const r = await (service as any).command(
t.id,
'RUN_CREATE',
dto,
'agent',
);
expect(r.attempt).toBe(1);
expect(
await (service as any).command(t.id, 'RUN_CREATE', dto, 'agent'),
).toEqual(r);
await expect(
(service as any).command(
t.id,
'RUN_CREATE',
{ ...dto, commitSha: 'b'.repeat(40) },
'agent',
),
).rejects.toThrow(/Idempotency/i);
await (service as any).command(
t.id,
'RUN_FINISH',
{
expectedVersion: 2,
idempotencyKey: `finish-${t.id}`,
runId: r.id,
status: 'FAILED',
retryClass: 'POLICY',
},
'agent',
);
await expect(
(service as any).command(
t.id,
'RUN_CREATE',
{ ...dto, expectedVersion: 3, idempotencyKey: `retry-${t.id}` },
'agent',
),
).rejects.toThrow(/retry/i);
});
it('selects tier at admission using backend risk and budget', async () => {
const t = await task('GATE1_PENDING', 1000);
await service.decideGate(
t.id,
1,
{
decision: 'APPROVED',
expectedVersion: 1,
idempotencyKey: `approve-tier-${t.id}`,
expiresAt: new Date(Date.now() + 60000).toISOString(),
},
'ceo',
);
await service.transition(
t.id,
{
toState: WorkflowState.READY,
expectedVersion: 2,
idempotencyKey: `admit-tier-${t.id}`,
},
'agent',
);
expect(
(await db.orchestrationTask.findUniqueOrThrow({ where: { id: t.id } }))
.modelTier,
).toBe('L0');
});
it('STOP_AGENTS prevents claims and running transitions', async () => {
await service.setKillSwitch('STOP_AGENTS', 'test', 'ceo');
try {
const t = await task('READY');
await expect(
service.transition(
t.id,
{
toState: WorkflowState.CLAIMED,
expectedVersion: 1,
idempotencyKey: `kill-${t.id}`,
},
'agent',
),
).rejects.toThrow(/kill switch/i);
} finally {
await db.orchestrationKillSwitch.deleteMany();
}
});
it('Gate1 admission consumes only unexpired approval', async () => {
const t = await task('GATE1_PENDING');
await service.decideGate(
t.id,
1,
{
decision: 'APPROVED',
expectedVersion: 1,
idempotencyKey: `approve-${t.id}`,
expiresAt: new Date(Date.now() + 100).toISOString(),
},
'ceo',
);
await new Promise((r) => setTimeout(r, 150));
await expect(
service.transition(
t.id,
{
toState: WorkflowState.READY,
expectedVersion: 2,
idempotencyKey: `admit-${t.id}`,
},
'agent',
),
).rejects.toThrow(/expired/i);
});
it('versioned transition replay returns original result and rejects changed payload', async () => {
const t = await task();
const dto = {
toState: WorkflowState.GATE1_PENDING,
expectedVersion: 1,
idempotencyKey: `tr-${t.id}`,
};
const first = await service.transition(t.id, dto, 'actor');
expect(await service.transition(t.id, dto, 'actor')).toEqual(first);
await expect(
service.transition(t.id, { ...dto, reason: 'different' }, 'actor'),
).rejects.toThrow(/Idempotency/i);
});
it('reconciles a reservation exactly once without erasing spent cost', async () => {
const t = await task();
const r = await service.recordBudget(
t.id,
{ amountCents: 80, kind: 'RESERVE', idempotencyKey: `res-${t.id}` },
'agent',
);
const dto = {
reservationId: r.id,
actualCents: 35,
idempotencyKey: `reconcile-${t.id}`,
};
await (service as any).reconcileBudget(t.id, dto, 'agent');
await (service as any).reconcileBudget(t.id, dto, 'agent');
await expect(
(service as any).reconcileBudget(
t.id,
{ ...dto, idempotencyKey: `second-${t.id}` },
'agent',
),
).rejects.toThrow(/reconciled/i);
expect(
(
await db.orchestrationBudgetLedger.aggregate({
where: { taskId: t.id },
_sum: { amountCents: true },
})
)._sum.amountCents,
).toBe(35);
await expect(
service.recordBudget(
t.id,
{
amountCents: 66,
kind: 'RESERVE',
idempotencyKey: `too-much-${t.id}`,
},
'agent',
),
).rejects.toThrow(/budget/i);
});
it('concurrent budget reservations never exceed hard cap and replay is harmless', async () => {
const t = await task();
const result = await Promise.allSettled(
Array.from({ length: 8 }, (_, i) =>
service.recordBudget(
t.id,
{
amountCents: 30,
kind: 'RESERVE',
idempotencyKey: `reserve-${t.id}-${i}`,
},
'agent',
),
),
);
expect(result.filter((x) => x.status === 'fulfilled')).toHaveLength(3);
const sum = await db.orchestrationBudgetLedger.aggregate({
where: { taskId: t.id },
_sum: { amountCents: true },
});
expect(sum._sum.amountCents).toBe(90);
const first = await db.orchestrationBudgetLedger.findFirstOrThrow({
where: { taskId: t.id },
});
await service.recordBudget(
t.id,
{
amountCents: 30,
kind: 'RESERVE',
idempotencyKey: first.idempotencyKey,
},
'agent',
);
expect(
await db.orchestrationBudgetLedger.count({ where: { taskId: t.id } }),
).toBe(3);
});
it('generic transition cannot approve gate 1', async () => {
const t = await task('GATE1_PENDING');
await expect(
service.transition(
t.id,
{
toState: WorkflowState.GATE1_APPROVED,
expectedVersion: 1,
idempotencyKey: `gate-bypass-${n}`,
},
'ceo',
),
).rejects.toThrow(/gate/i);
expect(
(await db.orchestrationTask.findUniqueOrThrow({ where: { id: t.id } }))
.state,
).toBe('GATE1_PENDING');
});
},
);