Checkpoint recovered orchestration core with tested fail-closed boundaries
This commit is contained in:
1 parent
c7f1d25e98
commit
0952a482fb
18 files changed
+1918
-129
No files matched your search
@@ -1,73 +1,260 @@
|
||||
import { BadRequestException, ConflictException, ForbiddenException } from "@nestjs/common";
|
||||
import { OrchestrationService } from "./orchestration.service";
|
||||
import {
|
||||
BadRequestException,
|
||||
ConflictException,
|
||||
ForbiddenException,
|
||||
} from '@nestjs/common';
|
||||
import { OrchestrationService } from './orchestration.service';
|
||||
|
||||
function mockPrisma() {
|
||||
const tx: any = {
|
||||
orchestrationTask: { findUnique: jest.fn(), create: jest.fn(), updateMany: jest.fn() },
|
||||
$queryRaw: jest.fn().mockResolvedValue([]),
|
||||
orchestrationTask: {
|
||||
findUnique: jest.fn(),
|
||||
create: jest.fn(),
|
||||
updateMany: jest.fn(),
|
||||
},
|
||||
orchestrationRun: { findUnique: jest.fn(), create: jest.fn() },
|
||||
orchestrationArtifact: { findFirst: jest.fn(), create: jest.fn() },
|
||||
orchestrationAttestation: { create: jest.fn() },
|
||||
orchestrationApprovalDecision: { findUnique: jest.fn(), create: jest.fn() },
|
||||
orchestrationEvent: { create: jest.fn() },
|
||||
orchestrationBudgetLedger: { aggregate: jest.fn(), create: jest.fn() },
|
||||
orchestrationEvent: { findUnique: jest.fn(), create: jest.fn() },
|
||||
orchestrationBudgetLedger: {
|
||||
findUnique: jest.fn(),
|
||||
aggregate: jest.fn(),
|
||||
create: jest.fn(),
|
||||
},
|
||||
orchestrationKillSwitch: { findUnique: jest.fn(), upsert: jest.fn() },
|
||||
};
|
||||
tx.$transaction = jest.fn(async (cb: any) => cb(tx));
|
||||
tx.$transaction = jest.fn((cb: any) => Promise.resolve(cb(tx)));
|
||||
return tx;
|
||||
}
|
||||
|
||||
describe("OrchestrationService", () => {
|
||||
describe('OrchestrationService', () => {
|
||||
let prisma: any;
|
||||
let service: OrchestrationService;
|
||||
beforeEach(() => { prisma = mockPrisma(); service = new OrchestrationService(prisma, { assertAllowed: jest.fn() } as never); });
|
||||
beforeEach(() => {
|
||||
prisma = mockPrisma();
|
||||
service = new OrchestrationService(prisma, {
|
||||
assertAllowed: jest.fn(),
|
||||
} as never);
|
||||
});
|
||||
|
||||
it("creates a task and append-only event transactionally", async () => {
|
||||
prisma.orchestrationTask.create.mockResolvedValue({ id: "task-1", state: "DRAFT", version: 1 });
|
||||
const result = await service.createTask({ key: "seo-1", title: "SEO", department: "SEO", operationMode: "PREPARE" as never, budgetLimitCents: 1000 }, "actor-1");
|
||||
it('rejects an unknown kill level before overwriting persisted protection', async () => {
|
||||
await expect(service.setKillSwitch('INVALID_LEVEL','test','admin')).rejects.toThrow(BadRequestException);
|
||||
expect(prisma.orchestrationKillSwitch.upsert).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('creates a task and append-only event transactionally', async () => {
|
||||
prisma.orchestrationTask.create.mockResolvedValue({
|
||||
id: 'task-1',
|
||||
state: 'DRAFT',
|
||||
version: 1,
|
||||
});
|
||||
const result = await service.createTask(
|
||||
{
|
||||
key: 'seo-1',
|
||||
title: 'SEO',
|
||||
department: 'SEO',
|
||||
operationMode: 'PREPARE' as never,
|
||||
budgetLimitCents: 1000,
|
||||
},
|
||||
'actor-1',
|
||||
);
|
||||
expect(prisma.$transaction).toHaveBeenCalledTimes(1);
|
||||
expect(prisma.orchestrationTask.create).toHaveBeenCalledWith({ data: expect.objectContaining({ state: "DRAFT", version: 1 }) });
|
||||
expect(prisma.orchestrationEvent.create).toHaveBeenCalledWith({ data: expect.objectContaining({ eventType: "TASK_CREATED", sequence: 1 }) });
|
||||
expect(result.id).toBe("task-1");
|
||||
expect(prisma.orchestrationTask.create).toHaveBeenCalledWith({
|
||||
data: expect.objectContaining({ state: 'DRAFT', version: 1 }),
|
||||
});
|
||||
expect(prisma.orchestrationEvent.create).toHaveBeenCalledWith({
|
||||
data: expect.objectContaining({ eventType: 'TASK_CREATED', sequence: 1 }),
|
||||
});
|
||||
expect(result.id).toBe('task-1');
|
||||
});
|
||||
|
||||
it("uses compare-and-swap versioning for transitions", async () => {
|
||||
prisma.orchestrationTask.findUnique.mockResolvedValue({ id: "task-1", state: "DRAFT", version: 3 });
|
||||
it('uses compare-and-swap versioning for transitions', async () => {
|
||||
prisma.orchestrationTask.findUnique.mockResolvedValue({
|
||||
id: 'task-1',
|
||||
state: 'DRAFT',
|
||||
version: 3,
|
||||
});
|
||||
prisma.orchestrationTask.updateMany.mockResolvedValue({ count: 0 });
|
||||
await expect(service.transition("task-1", { toState: "GATE1_PENDING" as never, expectedVersion: 3, idempotencyKey: "i-1" }, "actor")).rejects.toThrow(ConflictException);
|
||||
await expect(
|
||||
service.transition(
|
||||
'task-1',
|
||||
{
|
||||
toState: 'GATE1_PENDING' as never,
|
||||
expectedVersion: 3,
|
||||
idempotencyKey: 'i-1',
|
||||
},
|
||||
'actor',
|
||||
),
|
||||
).rejects.toThrow(ConflictException);
|
||||
});
|
||||
|
||||
it("makes gate decisions immutable and idempotent", async () => {
|
||||
prisma.orchestrationApprovalDecision.findUnique.mockResolvedValue({ id: "d1", decision: "APPROVED", gate: 1, taskId: "task-1" });
|
||||
await expect(service.decideGate("task-1", 1, { decision: "APPROVED", expectedVersion: 2, idempotencyKey: "gate-1", expiresAt: new Date(Date.now()+10000).toISOString() }, "ceo")).resolves.toMatchObject({ id: "d1" });
|
||||
it('makes gate decisions immutable and idempotent', async () => {
|
||||
const expiresAt = new Date(Date.now() + 10000).toISOString();
|
||||
prisma.orchestrationApprovalDecision.findUnique.mockResolvedValue({
|
||||
id: 'd1',
|
||||
decision: 'APPROVED',
|
||||
gate: 1,
|
||||
taskId: 'task-1',
|
||||
});
|
||||
prisma.orchestrationEvent.findUnique.mockResolvedValue({
|
||||
payload: {
|
||||
fingerprint: JSON.stringify({
|
||||
taskId: 'task-1',
|
||||
gate: 1,
|
||||
dto: {
|
||||
decision: 'APPROVED',
|
||||
expectedVersion: 2,
|
||||
idempotencyKey: 'gate-1',
|
||||
expiresAt,
|
||||
},
|
||||
actorId: 'ceo',
|
||||
}),
|
||||
},
|
||||
});
|
||||
await expect(
|
||||
service.decideGate(
|
||||
'task-1',
|
||||
1,
|
||||
{
|
||||
decision: 'APPROVED',
|
||||
expectedVersion: 2,
|
||||
idempotencyKey: 'gate-1',
|
||||
expiresAt,
|
||||
},
|
||||
'ceo',
|
||||
),
|
||||
).resolves.toMatchObject({ id: 'd1' });
|
||||
expect(prisma.orchestrationApprovalDecision.create).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("rejects an expired approval", async () => {
|
||||
await expect(service.decideGate("task-1", 1, { decision: "APPROVED", expectedVersion: 2, idempotencyKey: "gate-expired", expiresAt: new Date(0).toISOString() }, "ceo")).rejects.toThrow(BadRequestException);
|
||||
it('rejects an expired approval', async () => {
|
||||
await expect(
|
||||
service.decideGate(
|
||||
'task-1',
|
||||
1,
|
||||
{
|
||||
decision: 'APPROVED',
|
||||
expectedVersion: 2,
|
||||
idempotencyKey: 'gate-expired',
|
||||
expiresAt: new Date(0).toISOString(),
|
||||
},
|
||||
'ceo',
|
||||
),
|
||||
).rejects.toThrow(BadRequestException);
|
||||
});
|
||||
|
||||
it("binds gate 2 approval to run, artifact hash, commit and environment", async () => {
|
||||
it('binds gate 2 approval to run, artifact hash, commit and environment', async () => {
|
||||
prisma.orchestrationApprovalDecision.findUnique.mockResolvedValue(null);
|
||||
prisma.orchestrationTask.findUnique.mockResolvedValue({ id: "task-1", state: "GATE2_PENDING", version: 7 });
|
||||
prisma.orchestrationRun.findUnique.mockResolvedValue({ id: "run-1", taskId: "task-1", commitSha: "abc123", environment: "staging" });
|
||||
prisma.orchestrationArtifact.findFirst.mockResolvedValue({ id: "artifact-1", runId: "run-1", sha256: "f".repeat(64) });
|
||||
prisma.orchestrationTask.findUnique.mockResolvedValue({
|
||||
id: 'task-1',
|
||||
state: 'GATE2_PENDING',
|
||||
version: 7,
|
||||
});
|
||||
prisma.orchestrationRun.findUnique.mockResolvedValue({
|
||||
id: 'run-1',
|
||||
taskId: 'task-1',
|
||||
commitSha: 'abc123',
|
||||
environment: 'staging',
|
||||
status: 'SUCCEEDED',
|
||||
});
|
||||
prisma.orchestrationArtifact.findFirst.mockResolvedValue({
|
||||
id: 'artifact-1',
|
||||
runId: 'run-1',
|
||||
sha256: 'f'.repeat(64),
|
||||
});
|
||||
prisma.orchestrationTask.updateMany.mockResolvedValue({ count: 1 });
|
||||
prisma.orchestrationApprovalDecision.create.mockResolvedValue({ id: "decision-2", gate: 2 });
|
||||
await service.decideGate("task-1", 2, { decision: "APPROVED", expectedVersion: 7, idempotencyKey: "gate-2", expiresAt: new Date(Date.now()+10000).toISOString(), runId: "run-1", artifactHash: "f".repeat(64), commitSha: "abc123", environment: "staging" }, "ceo");
|
||||
expect(prisma.orchestrationApprovalDecision.create).toHaveBeenCalledWith({ data: expect.objectContaining({ runId: "run-1", artifactHash: "f".repeat(64), commitSha: "abc123", environment: "staging" }) });
|
||||
prisma.orchestrationApprovalDecision.create.mockResolvedValue({
|
||||
id: 'decision-2',
|
||||
gate: 2,
|
||||
});
|
||||
await service.decideGate(
|
||||
'task-1',
|
||||
2,
|
||||
{
|
||||
decision: 'APPROVED',
|
||||
expectedVersion: 7,
|
||||
idempotencyKey: 'gate-2',
|
||||
expiresAt: new Date(Date.now() + 10000).toISOString(),
|
||||
runId: 'run-1',
|
||||
artifactHash: 'f'.repeat(64),
|
||||
commitSha: 'abc123',
|
||||
environment: 'staging',
|
||||
},
|
||||
'ceo',
|
||||
);
|
||||
expect(prisma.orchestrationApprovalDecision.create).toHaveBeenCalledWith({
|
||||
data: expect.objectContaining({
|
||||
runId: 'run-1',
|
||||
artifactHash: 'f'.repeat(64),
|
||||
commitSha: 'abc123',
|
||||
environment: 'staging',
|
||||
}),
|
||||
});
|
||||
});
|
||||
|
||||
it("refuses gate 2 when binding does not match the immutable run", async () => {
|
||||
it.each(['RUNNING', 'FAILED', 'CANCELLED'])('refuses Gate 2 approval for an unfinished or unsuccessful %s run', async (status) => {
|
||||
prisma.orchestrationTask.findUnique.mockResolvedValue({id:'task-1',state:'GATE2_PENDING',version:7});
|
||||
prisma.orchestrationRun.findUnique.mockResolvedValue({id:'run-1',taskId:'task-1',commitSha:'abc123',environment:'staging',status});
|
||||
prisma.orchestrationArtifact.findFirst.mockResolvedValue({id:'artifact-1'});
|
||||
prisma.orchestrationTask.updateMany.mockResolvedValue({count:1});
|
||||
await expect(service.decideGate('task-1',2,{
|
||||
decision:'APPROVED',expectedVersion:7,idempotencyKey:'failed-run',
|
||||
expiresAt:new Date(Date.now()+10000).toISOString(),runId:'run-1',
|
||||
artifactHash:'f'.repeat(64),commitSha:'abc123',environment:'staging',
|
||||
},'ceo')).rejects.toThrow(/successful run/i);
|
||||
expect(prisma.orchestrationApprovalDecision.create).not.toHaveBeenCalled();
|
||||
expect(prisma.orchestrationTask.updateMany).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('refuses gate 2 when binding does not match the immutable run', async () => {
|
||||
prisma.orchestrationApprovalDecision.findUnique.mockResolvedValue(null);
|
||||
prisma.orchestrationTask.findUnique.mockResolvedValue({ id: "task-1", state: "GATE2_PENDING", version: 7 });
|
||||
prisma.orchestrationRun.findUnique.mockResolvedValue({ id: "run-1", taskId: "task-1", commitSha: "real", environment: "staging" });
|
||||
await expect(service.decideGate("task-1", 2, { decision: "APPROVED", expectedVersion: 7, idempotencyKey: "gate-bad", expiresAt: new Date(Date.now()+10000).toISOString(), runId: "run-1", artifactHash: "f".repeat(64), commitSha: "wrong", environment: "staging" }, "ceo")).rejects.toThrow(ForbiddenException);
|
||||
prisma.orchestrationTask.findUnique.mockResolvedValue({
|
||||
id: 'task-1',
|
||||
state: 'GATE2_PENDING',
|
||||
version: 7,
|
||||
});
|
||||
prisma.orchestrationRun.findUnique.mockResolvedValue({
|
||||
id: 'run-1',
|
||||
taskId: 'task-1',
|
||||
commitSha: 'real',
|
||||
environment: 'staging',
|
||||
});
|
||||
await expect(
|
||||
service.decideGate(
|
||||
'task-1',
|
||||
2,
|
||||
{
|
||||
decision: 'APPROVED',
|
||||
expectedVersion: 7,
|
||||
idempotencyKey: 'gate-bad',
|
||||
expiresAt: new Date(Date.now() + 10000).toISOString(),
|
||||
runId: 'run-1',
|
||||
artifactHash: 'f'.repeat(64),
|
||||
commitSha: 'wrong',
|
||||
environment: 'staging',
|
||||
},
|
||||
'ceo',
|
||||
),
|
||||
).rejects.toThrow(ForbiddenException);
|
||||
});
|
||||
|
||||
it("enforces hard task budget atomically", async () => {
|
||||
prisma.orchestrationTask.findUnique.mockResolvedValue({ id: "task-1", budgetLimitCents: 1000 });
|
||||
prisma.orchestrationBudgetLedger.aggregate.mockResolvedValue({ _sum: { amountCents: 900 } });
|
||||
await expect(service.recordBudget("task-1", { amountCents: 101, idempotencyKey: "cost-1", kind: "MODEL" }, "agent")).rejects.toThrow(ForbiddenException);
|
||||
it('enforces hard task budget atomically', async () => {
|
||||
prisma.orchestrationTask.findUnique.mockResolvedValue({
|
||||
id: 'task-1',
|
||||
budgetLimitCents: 1000,
|
||||
});
|
||||
prisma.orchestrationBudgetLedger.aggregate.mockResolvedValue({
|
||||
_sum: { amountCents: 900 },
|
||||
});
|
||||
await expect(
|
||||
service.recordBudget(
|
||||
'task-1',
|
||||
{ amountCents: 101, idempotencyKey: 'cost-1', kind: 'SPEND' },
|
||||
'agent',
|
||||
),
|
||||
).rejects.toThrow(ForbiddenException);
|
||||
expect(prisma.orchestrationBudgetLedger.create).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user