Checkpoint recovered orchestration core with tested fail-closed boundaries
This commit is contained in:
1 parent
c7f1d25e98
commit
0952a482fb
18 files changed
+1918
-129
No files matched your search
@@ -0,0 +1,577 @@
|
||||
import { OrchestrationController } from './orchestration.controller';
|
||||
import { DepartmentTasksController } from '../admin/department-tasks.controller';
|
||||
import { ApprovalsController } from '../admin/approvals.controller';
|
||||
import { PrismaClient } from '@prisma/client';
|
||||
import { OrchestrationService } from './orchestration.service';
|
||||
import { KillSwitchService } from './kill-switch.service';
|
||||
import { WorkflowState, OperationMode } from './orchestration.domain';
|
||||
const enabled = process.env.ORCHESTRATION_INTEGRATION === '1';
|
||||
(enabled ? describe : describe.skip)(
|
||||
'v4 disposable PostgreSQL invariants',
|
||||
() => {
|
||||
const db = new PrismaClient();
|
||||
const service = new OrchestrationService(
|
||||
db as never,
|
||||
new KillSwitchService(db as never),
|
||||
);
|
||||
let n = 0;
|
||||
async function task(state = 'DRAFT', budget = 100) {
|
||||
return db.orchestrationTask.create({
|
||||
data: {
|
||||
key: `test-${Date.now()}-${n++}`,
|
||||
title: 'test',
|
||||
department: 'TEST',
|
||||
operationMode: 'PREPARE',
|
||||
state,
|
||||
budgetLimitCents: budget,
|
||||
},
|
||||
});
|
||||
}
|
||||
afterAll(async () => db.$disconnect());
|
||||
it('release rejects PREPARE operation class even with gate approval', async()=>{
|
||||
const t=await task('GATE2_APPROVED');
|
||||
await expect(service.command(t.id,'RELEASE_AUTHORIZE',{expectedVersion:1,idempotencyKey:`mode-${t.id}`},'ci')).rejects.toThrow(/operation mode/i);
|
||||
});
|
||||
|
||||
it('append-only ledger rejects direct rewrites in PostgreSQL', async () => {
|
||||
const t = await task();
|
||||
const r = await service.recordBudget(
|
||||
t.id,
|
||||
{
|
||||
amountCents: 50,
|
||||
kind: 'RESERVE',
|
||||
idempotencyKey: `immutable-${t.id}`,
|
||||
},
|
||||
'a',
|
||||
);
|
||||
await expect(
|
||||
db.orchestrationBudgetLedger.update({
|
||||
where: { id: r.id },
|
||||
data: { amountCents: 0 },
|
||||
}),
|
||||
).rejects.toThrow(/append.only/i);
|
||||
await expect(
|
||||
db.orchestrationBudgetLedger.delete({ where: { id: r.id } }),
|
||||
).rejects.toThrow(/append.only/i);
|
||||
});
|
||||
it('generic transitions cannot fake run lifecycle or production verification', async () => {
|
||||
for (const [from, to] of [
|
||||
['CLAIMED', 'RUNNING'],
|
||||
['RUNNING', 'VERIFYING'],
|
||||
['FAILED', 'READY'],
|
||||
['PRODUCTION_VERIFYING', 'LIVE_VERIFIED'],
|
||||
]) {
|
||||
const t = await task(from);
|
||||
await expect(
|
||||
service.transition(
|
||||
t.id,
|
||||
{
|
||||
expectedVersion: 1,
|
||||
idempotencyKey: `fake-${t.id}`,
|
||||
toState: to as WorkflowState,
|
||||
},
|
||||
'agent',
|
||||
),
|
||||
).rejects.toThrow(/dedicated/i);
|
||||
}
|
||||
});
|
||||
it('exposes command and reconcile HTTP handlers', () => {
|
||||
const c = new OrchestrationController(service);
|
||||
expect(typeof c.command).toBe('function');
|
||||
expect(typeof c.reconcile).toBe('function');
|
||||
});
|
||||
it('highest kill level blocks reads and reconciliation', async () => {
|
||||
const t = await task();
|
||||
const r = await service.recordBudget(
|
||||
t.id,
|
||||
{
|
||||
kind: 'RESERVE',
|
||||
amountCents: 10,
|
||||
idempotencyKey: `stop-res-${t.id}`,
|
||||
},
|
||||
'a',
|
||||
);
|
||||
await service.setKillSwitch(
|
||||
['EMERGENCY', 'HALT'].join('_'),
|
||||
'test',
|
||||
'ceo',
|
||||
);
|
||||
try {
|
||||
await expect(service.findOne(t.id)).rejects.toThrow(/kill switch/i);
|
||||
await expect(
|
||||
service.reconcileBudget(
|
||||
t.id,
|
||||
{
|
||||
reservationId: r.id,
|
||||
actualCents: 0,
|
||||
idempotencyKey: `stop-recon-${t.id}`,
|
||||
},
|
||||
'a',
|
||||
),
|
||||
).rejects.toThrow(/kill switch/i);
|
||||
} finally {
|
||||
await db.orchestrationKillSwitch.deleteMany();
|
||||
}
|
||||
});
|
||||
it('legacy arbitrary status and approval endpoints fail closed', async () => {
|
||||
await expect(
|
||||
new DepartmentTasksController(db as never).create({
|
||||
title: 'bypass',
|
||||
status: 'DONE',
|
||||
}),
|
||||
).rejects.toThrow(/v4/i);
|
||||
await expect(
|
||||
new DepartmentTasksController(db as never).updateStatus('none', {
|
||||
status: 'DONE',
|
||||
}),
|
||||
).rejects.toThrow(/v4/i);
|
||||
await expect(
|
||||
new ApprovalsController(db as never).decide('none', {
|
||||
decision: 'APPROVED',
|
||||
} as any),
|
||||
).rejects.toThrow(/v4/i);
|
||||
});
|
||||
it('release consumes only matching fresh production Gate2 via dedicated command', async () => {
|
||||
const t = await task('GATE2_PENDING');
|
||||
const hash = 'f'.repeat(64),
|
||||
commitSha = 'a'.repeat(40);
|
||||
const r = await db.orchestrationRun.create({
|
||||
data: {
|
||||
taskId: t.id,
|
||||
status: 'SUCCEEDED',
|
||||
commitSha,
|
||||
environment: 'production',
|
||||
idempotencyKey: `rel-run-${t.id}`,
|
||||
},
|
||||
});
|
||||
await db.orchestrationArtifact.create({
|
||||
data: {
|
||||
runId: r.id,
|
||||
kind: 'BUILD',
|
||||
uri: 'https://ci.example/build',
|
||||
sha256: hash,
|
||||
},
|
||||
});
|
||||
await db.orchestrationAttestation.create({
|
||||
data: {
|
||||
runId: r.id,
|
||||
kind: 'TEST',
|
||||
subjectHash: hash,
|
||||
signer: 'ci',
|
||||
predicate: { passed: true },
|
||||
},
|
||||
});
|
||||
const decision = await service.decideGate(
|
||||
t.id,
|
||||
2,
|
||||
{
|
||||
decision: 'APPROVED',
|
||||
expectedVersion: 1,
|
||||
idempotencyKey: `rel-approve-${t.id}`,
|
||||
expiresAt: new Date(Date.now() + 60000).toISOString(),
|
||||
runId: r.id,
|
||||
artifactHash: hash,
|
||||
commitSha,
|
||||
environment: 'production',
|
||||
},
|
||||
'ceo',
|
||||
);
|
||||
const dto = {
|
||||
expectedVersion: 2,
|
||||
idempotencyKey: `rel-${t.id}`,
|
||||
approvalId: decision.id,
|
||||
runId: r.id,
|
||||
artifactHash: hash,
|
||||
commitSha,
|
||||
environment: 'production',
|
||||
};
|
||||
await expect(
|
||||
(service as any).command(
|
||||
t.id,
|
||||
'RELEASE_AUTHORIZE',
|
||||
{ ...dto, artifactHash: '0'.repeat(64) },
|
||||
'ci',
|
||||
),
|
||||
).rejects.toThrow(/binding/i);
|
||||
const result = await (service as any).command(
|
||||
t.id,
|
||||
'RELEASE_AUTHORIZE',
|
||||
dto,
|
||||
'ci',
|
||||
);
|
||||
expect(result.authorized).toBe(true);
|
||||
expect(
|
||||
(await db.orchestrationTask.findUniqueOrThrow({ where: { id: t.id } }))
|
||||
.state,
|
||||
).toBe('RELEASING');
|
||||
await expect(
|
||||
(service as any).command(
|
||||
t.id,
|
||||
'RELEASE_AUTHORIZE',
|
||||
{ ...dto, expectedVersion: 3, idempotencyKey: `rel-again-${t.id}` },
|
||||
'ci',
|
||||
),
|
||||
).rejects.toThrow();
|
||||
});
|
||||
it('gate decision replay rejects changed immutable expiry and actor', async () => {
|
||||
const t = await task('GATE1_PENDING');
|
||||
const dto = {
|
||||
decision: 'APPROVED' as const,
|
||||
expectedVersion: 1,
|
||||
idempotencyKey: `bind-${t.id}`,
|
||||
expiresAt: new Date(Date.now() + 60000).toISOString(),
|
||||
};
|
||||
await service.decideGate(t.id, 1, dto, 'ceo');
|
||||
await expect(
|
||||
service.decideGate(
|
||||
t.id,
|
||||
1,
|
||||
{ ...dto, expiresAt: new Date(Date.now() + 120000).toISOString() },
|
||||
'ceo',
|
||||
),
|
||||
).rejects.toThrow(/Idempotency/i);
|
||||
await expect(service.decideGate(t.id, 1, dto, 'other')).rejects.toThrow(
|
||||
/Idempotency/i,
|
||||
);
|
||||
});
|
||||
it('dependencies block admission and reject cycles', async () => {
|
||||
const a = await task();
|
||||
const b = await task();
|
||||
await (service as any).command(
|
||||
a.id,
|
||||
'DEPENDENCY_ADD',
|
||||
{
|
||||
expectedVersion: 1,
|
||||
idempotencyKey: `dep-a-${a.id}`,
|
||||
dependsOnTaskId: b.id,
|
||||
},
|
||||
'actor',
|
||||
);
|
||||
await expect(
|
||||
(service as any).command(
|
||||
b.id,
|
||||
'DEPENDENCY_ADD',
|
||||
{
|
||||
expectedVersion: 1,
|
||||
idempotencyKey: `dep-b-${b.id}`,
|
||||
dependsOnTaskId: a.id,
|
||||
},
|
||||
'actor',
|
||||
),
|
||||
).rejects.toThrow(/cycle/i);
|
||||
await db.orchestrationTask.update({
|
||||
where: { id: a.id },
|
||||
data: { state: 'GATE1_PENDING' },
|
||||
});
|
||||
await service.decideGate(
|
||||
a.id,
|
||||
1,
|
||||
{
|
||||
decision: 'APPROVED',
|
||||
expectedVersion: 2,
|
||||
idempotencyKey: `dep-approval-${a.id}`,
|
||||
expiresAt: new Date(Date.now() + 60000).toISOString(),
|
||||
},
|
||||
'ceo',
|
||||
);
|
||||
await expect(
|
||||
service.transition(
|
||||
a.id,
|
||||
{
|
||||
toState: WorkflowState.READY,
|
||||
expectedVersion: 3,
|
||||
idempotencyKey: `dep-ready-${a.id}`,
|
||||
},
|
||||
'agent',
|
||||
),
|
||||
).rejects.toThrow(/dependency/i);
|
||||
});
|
||||
it('registers artifact and attestation for owned run, blocks unregistered subjects', async () => {
|
||||
const t = await task('CLAIMED');
|
||||
const base = {
|
||||
expectedVersion: 1,
|
||||
idempotencyKey: `life-${t.id}`,
|
||||
commitSha: 'c'.repeat(40),
|
||||
environment: 'staging',
|
||||
};
|
||||
const r = await (service as any).command(
|
||||
t.id,
|
||||
'RUN_CREATE',
|
||||
base,
|
||||
'agent',
|
||||
);
|
||||
const hash = 'd'.repeat(64);
|
||||
const a = await (service as any).command(
|
||||
t.id,
|
||||
'ARTIFACT_ADD',
|
||||
{
|
||||
expectedVersion: 2,
|
||||
idempotencyKey: `art-${t.id}`,
|
||||
runId: r.id,
|
||||
sha256: hash,
|
||||
kind: 'BUILD',
|
||||
uri: 'https://ci.example/artifact',
|
||||
},
|
||||
'agent',
|
||||
);
|
||||
expect(a.sha256).toBe(hash);
|
||||
await expect(
|
||||
(service as any).command(
|
||||
t.id,
|
||||
'ATTESTATION_ADD',
|
||||
{
|
||||
expectedVersion: 3,
|
||||
idempotencyKey: `bad-att-${t.id}`,
|
||||
runId: r.id,
|
||||
subjectHash: 'e'.repeat(64),
|
||||
kind: 'TEST',
|
||||
predicate: { passed: true },
|
||||
},
|
||||
'ci',
|
||||
),
|
||||
).rejects.toThrow(/artifact/i);
|
||||
const att = await (service as any).command(
|
||||
t.id,
|
||||
'ATTESTATION_ADD',
|
||||
{
|
||||
expectedVersion: 3,
|
||||
idempotencyKey: `att-${t.id}`,
|
||||
runId: r.id,
|
||||
subjectHash: hash,
|
||||
kind: 'TEST',
|
||||
predicate: { passed: true },
|
||||
},
|
||||
'ci',
|
||||
);
|
||||
expect(att.signer).toBe('ci');
|
||||
});
|
||||
it('creates immutable versioned runs with bounded transient retries', async () => {
|
||||
const t = await task('CLAIMED');
|
||||
const dto = {
|
||||
expectedVersion: 1,
|
||||
idempotencyKey: `run-${t.id}`,
|
||||
commitSha: 'a'.repeat(40),
|
||||
environment: 'staging',
|
||||
};
|
||||
const r = await (service as any).command(
|
||||
t.id,
|
||||
'RUN_CREATE',
|
||||
dto,
|
||||
'agent',
|
||||
);
|
||||
expect(r.attempt).toBe(1);
|
||||
expect(
|
||||
await (service as any).command(t.id, 'RUN_CREATE', dto, 'agent'),
|
||||
).toEqual(r);
|
||||
await expect(
|
||||
(service as any).command(
|
||||
t.id,
|
||||
'RUN_CREATE',
|
||||
{ ...dto, commitSha: 'b'.repeat(40) },
|
||||
'agent',
|
||||
),
|
||||
).rejects.toThrow(/Idempotency/i);
|
||||
await (service as any).command(
|
||||
t.id,
|
||||
'RUN_FINISH',
|
||||
{
|
||||
expectedVersion: 2,
|
||||
idempotencyKey: `finish-${t.id}`,
|
||||
runId: r.id,
|
||||
status: 'FAILED',
|
||||
retryClass: 'POLICY',
|
||||
},
|
||||
'agent',
|
||||
);
|
||||
await expect(
|
||||
(service as any).command(
|
||||
t.id,
|
||||
'RUN_CREATE',
|
||||
{ ...dto, expectedVersion: 3, idempotencyKey: `retry-${t.id}` },
|
||||
'agent',
|
||||
),
|
||||
).rejects.toThrow(/retry/i);
|
||||
});
|
||||
it('selects tier at admission using backend risk and budget', async () => {
|
||||
const t = await task('GATE1_PENDING', 1000);
|
||||
await service.decideGate(
|
||||
t.id,
|
||||
1,
|
||||
{
|
||||
decision: 'APPROVED',
|
||||
expectedVersion: 1,
|
||||
idempotencyKey: `approve-tier-${t.id}`,
|
||||
expiresAt: new Date(Date.now() + 60000).toISOString(),
|
||||
},
|
||||
'ceo',
|
||||
);
|
||||
await service.transition(
|
||||
t.id,
|
||||
{
|
||||
toState: WorkflowState.READY,
|
||||
expectedVersion: 2,
|
||||
idempotencyKey: `admit-tier-${t.id}`,
|
||||
},
|
||||
'agent',
|
||||
);
|
||||
expect(
|
||||
(await db.orchestrationTask.findUniqueOrThrow({ where: { id: t.id } }))
|
||||
.modelTier,
|
||||
).toBe('L0');
|
||||
});
|
||||
it('STOP_AGENTS prevents claims and running transitions', async () => {
|
||||
await service.setKillSwitch('STOP_AGENTS', 'test', 'ceo');
|
||||
try {
|
||||
const t = await task('READY');
|
||||
await expect(
|
||||
service.transition(
|
||||
t.id,
|
||||
{
|
||||
toState: WorkflowState.CLAIMED,
|
||||
expectedVersion: 1,
|
||||
idempotencyKey: `kill-${t.id}`,
|
||||
},
|
||||
'agent',
|
||||
),
|
||||
).rejects.toThrow(/kill switch/i);
|
||||
} finally {
|
||||
await db.orchestrationKillSwitch.deleteMany();
|
||||
}
|
||||
});
|
||||
it('Gate1 admission consumes only unexpired approval', async () => {
|
||||
const t = await task('GATE1_PENDING');
|
||||
await service.decideGate(
|
||||
t.id,
|
||||
1,
|
||||
{
|
||||
decision: 'APPROVED',
|
||||
expectedVersion: 1,
|
||||
idempotencyKey: `approve-${t.id}`,
|
||||
expiresAt: new Date(Date.now() + 100).toISOString(),
|
||||
},
|
||||
'ceo',
|
||||
);
|
||||
await new Promise((r) => setTimeout(r, 150));
|
||||
await expect(
|
||||
service.transition(
|
||||
t.id,
|
||||
{
|
||||
toState: WorkflowState.READY,
|
||||
expectedVersion: 2,
|
||||
idempotencyKey: `admit-${t.id}`,
|
||||
},
|
||||
'agent',
|
||||
),
|
||||
).rejects.toThrow(/expired/i);
|
||||
});
|
||||
it('versioned transition replay returns original result and rejects changed payload', async () => {
|
||||
const t = await task();
|
||||
const dto = {
|
||||
toState: WorkflowState.GATE1_PENDING,
|
||||
expectedVersion: 1,
|
||||
idempotencyKey: `tr-${t.id}`,
|
||||
};
|
||||
const first = await service.transition(t.id, dto, 'actor');
|
||||
expect(await service.transition(t.id, dto, 'actor')).toEqual(first);
|
||||
await expect(
|
||||
service.transition(t.id, { ...dto, reason: 'different' }, 'actor'),
|
||||
).rejects.toThrow(/Idempotency/i);
|
||||
});
|
||||
it('reconciles a reservation exactly once without erasing spent cost', async () => {
|
||||
const t = await task();
|
||||
const r = await service.recordBudget(
|
||||
t.id,
|
||||
{ amountCents: 80, kind: 'RESERVE', idempotencyKey: `res-${t.id}` },
|
||||
'agent',
|
||||
);
|
||||
const dto = {
|
||||
reservationId: r.id,
|
||||
actualCents: 35,
|
||||
idempotencyKey: `reconcile-${t.id}`,
|
||||
};
|
||||
await (service as any).reconcileBudget(t.id, dto, 'agent');
|
||||
await (service as any).reconcileBudget(t.id, dto, 'agent');
|
||||
await expect(
|
||||
(service as any).reconcileBudget(
|
||||
t.id,
|
||||
{ ...dto, idempotencyKey: `second-${t.id}` },
|
||||
'agent',
|
||||
),
|
||||
).rejects.toThrow(/reconciled/i);
|
||||
expect(
|
||||
(
|
||||
await db.orchestrationBudgetLedger.aggregate({
|
||||
where: { taskId: t.id },
|
||||
_sum: { amountCents: true },
|
||||
})
|
||||
)._sum.amountCents,
|
||||
).toBe(35);
|
||||
await expect(
|
||||
service.recordBudget(
|
||||
t.id,
|
||||
{
|
||||
amountCents: 66,
|
||||
kind: 'RESERVE',
|
||||
idempotencyKey: `too-much-${t.id}`,
|
||||
},
|
||||
'agent',
|
||||
),
|
||||
).rejects.toThrow(/budget/i);
|
||||
});
|
||||
it('concurrent budget reservations never exceed hard cap and replay is harmless', async () => {
|
||||
const t = await task();
|
||||
const result = await Promise.allSettled(
|
||||
Array.from({ length: 8 }, (_, i) =>
|
||||
service.recordBudget(
|
||||
t.id,
|
||||
{
|
||||
amountCents: 30,
|
||||
kind: 'RESERVE',
|
||||
idempotencyKey: `reserve-${t.id}-${i}`,
|
||||
},
|
||||
'agent',
|
||||
),
|
||||
),
|
||||
);
|
||||
expect(result.filter((x) => x.status === 'fulfilled')).toHaveLength(3);
|
||||
const sum = await db.orchestrationBudgetLedger.aggregate({
|
||||
where: { taskId: t.id },
|
||||
_sum: { amountCents: true },
|
||||
});
|
||||
expect(sum._sum.amountCents).toBe(90);
|
||||
const first = await db.orchestrationBudgetLedger.findFirstOrThrow({
|
||||
where: { taskId: t.id },
|
||||
});
|
||||
await service.recordBudget(
|
||||
t.id,
|
||||
{
|
||||
amountCents: 30,
|
||||
kind: 'RESERVE',
|
||||
idempotencyKey: first.idempotencyKey,
|
||||
},
|
||||
'agent',
|
||||
);
|
||||
expect(
|
||||
await db.orchestrationBudgetLedger.count({ where: { taskId: t.id } }),
|
||||
).toBe(3);
|
||||
});
|
||||
it('generic transition cannot approve gate 1', async () => {
|
||||
const t = await task('GATE1_PENDING');
|
||||
await expect(
|
||||
service.transition(
|
||||
t.id,
|
||||
{
|
||||
toState: WorkflowState.GATE1_APPROVED,
|
||||
expectedVersion: 1,
|
||||
idempotencyKey: `gate-bypass-${n}`,
|
||||
},
|
||||
'ceo',
|
||||
),
|
||||
).rejects.toThrow(/gate/i);
|
||||
expect(
|
||||
(await db.orchestrationTask.findUniqueOrThrow({ where: { id: t.id } }))
|
||||
.state,
|
||||
).toBe('GATE1_PENDING');
|
||||
});
|
||||
},
|
||||
);
|
||||
Reference in new issue
Block a user