Checkpoint recovered orchestration core with tested fail-closed boundaries

This commit is contained in:
devops committed 2026-10-23 08:01:08 +00:00
1 parent c7f1d25e98
commit 0952a482fb
18 files changed
+1918 -129

No files matched your search

@@ -1,73 +1,260 @@
import { BadRequestException, ConflictException, ForbiddenException } from "@nestjs/common";
import { OrchestrationService } from "./orchestration.service";
import {
BadRequestException,
ConflictException,
ForbiddenException,
} from '@nestjs/common';
import { OrchestrationService } from './orchestration.service';
function mockPrisma() {
const tx: any = {
orchestrationTask: { findUnique: jest.fn(), create: jest.fn(), updateMany: jest.fn() },
$queryRaw: jest.fn().mockResolvedValue([]),
orchestrationTask: {
findUnique: jest.fn(),
create: jest.fn(),
updateMany: jest.fn(),
},
orchestrationRun: { findUnique: jest.fn(), create: jest.fn() },
orchestrationArtifact: { findFirst: jest.fn(), create: jest.fn() },
orchestrationAttestation: { create: jest.fn() },
orchestrationApprovalDecision: { findUnique: jest.fn(), create: jest.fn() },
orchestrationEvent: { create: jest.fn() },
orchestrationBudgetLedger: { aggregate: jest.fn(), create: jest.fn() },
orchestrationEvent: { findUnique: jest.fn(), create: jest.fn() },
orchestrationBudgetLedger: {
findUnique: jest.fn(),
aggregate: jest.fn(),
create: jest.fn(),
},
orchestrationKillSwitch: { findUnique: jest.fn(), upsert: jest.fn() },
};
tx.$transaction = jest.fn(async (cb: any) => cb(tx));
tx.$transaction = jest.fn((cb: any) => Promise.resolve(cb(tx)));
return tx;
}
describe("OrchestrationService", () => {
describe('OrchestrationService', () => {
let prisma: any;
let service: OrchestrationService;
beforeEach(() => { prisma = mockPrisma(); service = new OrchestrationService(prisma, { assertAllowed: jest.fn() } as never); });
beforeEach(() => {
prisma = mockPrisma();
service = new OrchestrationService(prisma, {
assertAllowed: jest.fn(),
} as never);
});
it("creates a task and append-only event transactionally", async () => {
prisma.orchestrationTask.create.mockResolvedValue({ id: "task-1", state: "DRAFT", version: 1 });
const result = await service.createTask({ key: "seo-1", title: "SEO", department: "SEO", operationMode: "PREPARE" as never, budgetLimitCents: 1000 }, "actor-1");
it('rejects an unknown kill level before overwriting persisted protection', async () => {
await expect(service.setKillSwitch('INVALID_LEVEL','test','admin')).rejects.toThrow(BadRequestException);
expect(prisma.orchestrationKillSwitch.upsert).not.toHaveBeenCalled();
});
it('creates a task and append-only event transactionally', async () => {
prisma.orchestrationTask.create.mockResolvedValue({
id: 'task-1',
state: 'DRAFT',
version: 1,
});
const result = await service.createTask(
{
key: 'seo-1',
title: 'SEO',
department: 'SEO',
operationMode: 'PREPARE' as never,
budgetLimitCents: 1000,
},
'actor-1',
);
expect(prisma.$transaction).toHaveBeenCalledTimes(1);
expect(prisma.orchestrationTask.create).toHaveBeenCalledWith({ data: expect.objectContaining({ state: "DRAFT", version: 1 }) });
expect(prisma.orchestrationEvent.create).toHaveBeenCalledWith({ data: expect.objectContaining({ eventType: "TASK_CREATED", sequence: 1 }) });
expect(result.id).toBe("task-1");
expect(prisma.orchestrationTask.create).toHaveBeenCalledWith({
data: expect.objectContaining({ state: 'DRAFT', version: 1 }),
});
expect(prisma.orchestrationEvent.create).toHaveBeenCalledWith({
data: expect.objectContaining({ eventType: 'TASK_CREATED', sequence: 1 }),
});
expect(result.id).toBe('task-1');
});
it("uses compare-and-swap versioning for transitions", async () => {
prisma.orchestrationTask.findUnique.mockResolvedValue({ id: "task-1", state: "DRAFT", version: 3 });
it('uses compare-and-swap versioning for transitions', async () => {
prisma.orchestrationTask.findUnique.mockResolvedValue({
id: 'task-1',
state: 'DRAFT',
version: 3,
});
prisma.orchestrationTask.updateMany.mockResolvedValue({ count: 0 });
await expect(service.transition("task-1", { toState: "GATE1_PENDING" as never, expectedVersion: 3, idempotencyKey: "i-1" }, "actor")).rejects.toThrow(ConflictException);
await expect(
service.transition(
'task-1',
{
toState: 'GATE1_PENDING' as never,
expectedVersion: 3,
idempotencyKey: 'i-1',
},
'actor',
),
).rejects.toThrow(ConflictException);
});
it("makes gate decisions immutable and idempotent", async () => {
prisma.orchestrationApprovalDecision.findUnique.mockResolvedValue({ id: "d1", decision: "APPROVED", gate: 1, taskId: "task-1" });
await expect(service.decideGate("task-1", 1, { decision: "APPROVED", expectedVersion: 2, idempotencyKey: "gate-1", expiresAt: new Date(Date.now()+10000).toISOString() }, "ceo")).resolves.toMatchObject({ id: "d1" });
it('makes gate decisions immutable and idempotent', async () => {
const expiresAt = new Date(Date.now() + 10000).toISOString();
prisma.orchestrationApprovalDecision.findUnique.mockResolvedValue({
id: 'd1',
decision: 'APPROVED',
gate: 1,
taskId: 'task-1',
});
prisma.orchestrationEvent.findUnique.mockResolvedValue({
payload: {
fingerprint: JSON.stringify({
taskId: 'task-1',
gate: 1,
dto: {
decision: 'APPROVED',
expectedVersion: 2,
idempotencyKey: 'gate-1',
expiresAt,
},
actorId: 'ceo',
}),
},
});
await expect(
service.decideGate(
'task-1',
1,
{
decision: 'APPROVED',
expectedVersion: 2,
idempotencyKey: 'gate-1',
expiresAt,
},
'ceo',
),
).resolves.toMatchObject({ id: 'd1' });
expect(prisma.orchestrationApprovalDecision.create).not.toHaveBeenCalled();
});
it("rejects an expired approval", async () => {
await expect(service.decideGate("task-1", 1, { decision: "APPROVED", expectedVersion: 2, idempotencyKey: "gate-expired", expiresAt: new Date(0).toISOString() }, "ceo")).rejects.toThrow(BadRequestException);
it('rejects an expired approval', async () => {
await expect(
service.decideGate(
'task-1',
1,
{
decision: 'APPROVED',
expectedVersion: 2,
idempotencyKey: 'gate-expired',
expiresAt: new Date(0).toISOString(),
},
'ceo',
),
).rejects.toThrow(BadRequestException);
});
it("binds gate 2 approval to run, artifact hash, commit and environment", async () => {
it('binds gate 2 approval to run, artifact hash, commit and environment', async () => {
prisma.orchestrationApprovalDecision.findUnique.mockResolvedValue(null);
prisma.orchestrationTask.findUnique.mockResolvedValue({ id: "task-1", state: "GATE2_PENDING", version: 7 });
prisma.orchestrationRun.findUnique.mockResolvedValue({ id: "run-1", taskId: "task-1", commitSha: "abc123", environment: "staging" });
prisma.orchestrationArtifact.findFirst.mockResolvedValue({ id: "artifact-1", runId: "run-1", sha256: "f".repeat(64) });
prisma.orchestrationTask.findUnique.mockResolvedValue({
id: 'task-1',
state: 'GATE2_PENDING',
version: 7,
});
prisma.orchestrationRun.findUnique.mockResolvedValue({
id: 'run-1',
taskId: 'task-1',
commitSha: 'abc123',
environment: 'staging',
status: 'SUCCEEDED',
});
prisma.orchestrationArtifact.findFirst.mockResolvedValue({
id: 'artifact-1',
runId: 'run-1',
sha256: 'f'.repeat(64),
});
prisma.orchestrationTask.updateMany.mockResolvedValue({ count: 1 });
prisma.orchestrationApprovalDecision.create.mockResolvedValue({ id: "decision-2", gate: 2 });
await service.decideGate("task-1", 2, { decision: "APPROVED", expectedVersion: 7, idempotencyKey: "gate-2", expiresAt: new Date(Date.now()+10000).toISOString(), runId: "run-1", artifactHash: "f".repeat(64), commitSha: "abc123", environment: "staging" }, "ceo");
expect(prisma.orchestrationApprovalDecision.create).toHaveBeenCalledWith({ data: expect.objectContaining({ runId: "run-1", artifactHash: "f".repeat(64), commitSha: "abc123", environment: "staging" }) });
prisma.orchestrationApprovalDecision.create.mockResolvedValue({
id: 'decision-2',
gate: 2,
});
await service.decideGate(
'task-1',
2,
{
decision: 'APPROVED',
expectedVersion: 7,
idempotencyKey: 'gate-2',
expiresAt: new Date(Date.now() + 10000).toISOString(),
runId: 'run-1',
artifactHash: 'f'.repeat(64),
commitSha: 'abc123',
environment: 'staging',
},
'ceo',
);
expect(prisma.orchestrationApprovalDecision.create).toHaveBeenCalledWith({
data: expect.objectContaining({
runId: 'run-1',
artifactHash: 'f'.repeat(64),
commitSha: 'abc123',
environment: 'staging',
}),
});
});
it("refuses gate 2 when binding does not match the immutable run", async () => {
it.each(['RUNNING', 'FAILED', 'CANCELLED'])('refuses Gate 2 approval for an unfinished or unsuccessful %s run', async (status) => {
prisma.orchestrationTask.findUnique.mockResolvedValue({id:'task-1',state:'GATE2_PENDING',version:7});
prisma.orchestrationRun.findUnique.mockResolvedValue({id:'run-1',taskId:'task-1',commitSha:'abc123',environment:'staging',status});
prisma.orchestrationArtifact.findFirst.mockResolvedValue({id:'artifact-1'});
prisma.orchestrationTask.updateMany.mockResolvedValue({count:1});
await expect(service.decideGate('task-1',2,{
decision:'APPROVED',expectedVersion:7,idempotencyKey:'failed-run',
expiresAt:new Date(Date.now()+10000).toISOString(),runId:'run-1',
artifactHash:'f'.repeat(64),commitSha:'abc123',environment:'staging',
},'ceo')).rejects.toThrow(/successful run/i);
expect(prisma.orchestrationApprovalDecision.create).not.toHaveBeenCalled();
expect(prisma.orchestrationTask.updateMany).not.toHaveBeenCalled();
});
it('refuses gate 2 when binding does not match the immutable run', async () => {
prisma.orchestrationApprovalDecision.findUnique.mockResolvedValue(null);
prisma.orchestrationTask.findUnique.mockResolvedValue({ id: "task-1", state: "GATE2_PENDING", version: 7 });
prisma.orchestrationRun.findUnique.mockResolvedValue({ id: "run-1", taskId: "task-1", commitSha: "real", environment: "staging" });
await expect(service.decideGate("task-1", 2, { decision: "APPROVED", expectedVersion: 7, idempotencyKey: "gate-bad", expiresAt: new Date(Date.now()+10000).toISOString(), runId: "run-1", artifactHash: "f".repeat(64), commitSha: "wrong", environment: "staging" }, "ceo")).rejects.toThrow(ForbiddenException);
prisma.orchestrationTask.findUnique.mockResolvedValue({
id: 'task-1',
state: 'GATE2_PENDING',
version: 7,
});
prisma.orchestrationRun.findUnique.mockResolvedValue({
id: 'run-1',
taskId: 'task-1',
commitSha: 'real',
environment: 'staging',
});
await expect(
service.decideGate(
'task-1',
2,
{
decision: 'APPROVED',
expectedVersion: 7,
idempotencyKey: 'gate-bad',
expiresAt: new Date(Date.now() + 10000).toISOString(),
runId: 'run-1',
artifactHash: 'f'.repeat(64),
commitSha: 'wrong',
environment: 'staging',
},
'ceo',
),
).rejects.toThrow(ForbiddenException);
});
it("enforces hard task budget atomically", async () => {
prisma.orchestrationTask.findUnique.mockResolvedValue({ id: "task-1", budgetLimitCents: 1000 });
prisma.orchestrationBudgetLedger.aggregate.mockResolvedValue({ _sum: { amountCents: 900 } });
await expect(service.recordBudget("task-1", { amountCents: 101, idempotencyKey: "cost-1", kind: "MODEL" }, "agent")).rejects.toThrow(ForbiddenException);
it('enforces hard task budget atomically', async () => {
prisma.orchestrationTask.findUnique.mockResolvedValue({
id: 'task-1',
budgetLimitCents: 1000,
});
prisma.orchestrationBudgetLedger.aggregate.mockResolvedValue({
_sum: { amountCents: 900 },
});
await expect(
service.recordBudget(
'task-1',
{ amountCents: 101, idempotencyKey: 'cost-1', kind: 'SPEND' },
'agent',
),
).rejects.toThrow(ForbiddenException);
expect(prisma.orchestrationBudgetLedger.create).not.toHaveBeenCalled();
});
});