Checkpoint recovered orchestration core with tested fail-closed boundaries

This commit is contained in:
devops committed 2026-10-23 08:01:08 +00:00
1 parent c7f1d25e98
commit 0952a482fb
18 files changed
+1918 -129

No files matched your search

@@ -1,8 +1,90 @@
import { Type } from "class-transformer";
import { IsEnum, IsHash, IsIn, IsInt, IsISO8601, IsNotEmpty, IsOptional, IsString, Max, Min } from "class-validator";
import { OperationMode, WorkflowState } from "../orchestration.domain";
export class CreateOrchestrationTaskDto { @IsString() @IsNotEmpty() key!:string; @IsString() @IsNotEmpty() title!:string; @IsString() @IsNotEmpty() department!:string; @IsEnum(OperationMode) operationMode!:OperationMode; @Type(()=>Number) @IsInt() @Min(0) budgetLimitCents!:number; @IsOptional() metadata?:Record<string,unknown>; }
export class TransitionTaskDto { @IsEnum(WorkflowState) toState!:WorkflowState; @Type(()=>Number) @IsInt() @Min(1) expectedVersion!:number; @IsString() @IsNotEmpty() idempotencyKey!:string; @IsOptional() @IsString() reason?:string; }
export class GateDecisionDto { @IsIn(["APPROVED","REJECTED"]) decision!:"APPROVED"|"REJECTED"; @Type(()=>Number) @IsInt() @Min(1) expectedVersion!:number; @IsString() @IsNotEmpty() idempotencyKey!:string; @IsISO8601() expiresAt!:string; @IsOptional() @IsString() notes?:string; @IsOptional() @IsString() runId?:string; @IsOptional() @IsHash("sha256") artifactHash?:string; @IsOptional() @IsString() commitSha?:string; @IsOptional() @IsString() environment?:string; }
export class RecordBudgetDto { @Type(()=>Number) @IsInt() @Min(1) amountCents!:number; @IsString() @IsNotEmpty() idempotencyKey!:string; @IsString() @IsNotEmpty() kind!:string; @IsOptional() metadata?:Record<string,unknown>; }
export class SetKillSwitchDto { @IsIn(["FREEZE_NEW_WORK","STOP_AGENTS","DISABLE_MUTATIONS","EMERGENCY_HALT"]) level!:string; @IsString() @IsNotEmpty() reason!:string; }
import { Type } from 'class-transformer';
import {
IsEnum,
IsHash,
IsIn,
IsInt,
IsISO8601,
IsNotEmpty,
IsOptional,
IsString,
Max,
Min,
} from 'class-validator';
import { OperationMode, WorkflowState } from '../orchestration.domain';
export class CreateOrchestrationTaskDto {
@IsString() @IsNotEmpty() key!: string;
@IsString() @IsNotEmpty() title!: string;
@IsString() @IsNotEmpty() department!: string;
@IsEnum(OperationMode) operationMode!: OperationMode;
@Type(() => Number) @IsInt() @Min(0) budgetLimitCents!: number;
@IsOptional() metadata?: Record<string, unknown>;
}
export class TransitionTaskDto {
@IsEnum(WorkflowState) toState!: WorkflowState;
@Type(() => Number) @IsInt() @Min(1) expectedVersion!: number;
@IsString() @IsNotEmpty() idempotencyKey!: string;
@IsOptional() @IsString() reason?: string;
}
export class GateDecisionDto {
@IsIn(['APPROVED', 'REJECTED']) decision!: 'APPROVED' | 'REJECTED';
@Type(() => Number) @IsInt() @Min(1) expectedVersion!: number;
@IsString() @IsNotEmpty() idempotencyKey!: string;
@IsISO8601() expiresAt!: string;
@IsOptional() @IsString() notes?: string;
@IsOptional() @IsString() runId?: string;
@IsOptional() @IsHash('sha256') artifactHash?: string;
@IsOptional() @IsString() commitSha?: string;
@IsOptional() @IsString() environment?: string;
}
export class RecordBudgetDto {
@Type(() => Number) @IsInt() @Min(1) amountCents!: number;
@IsString() @IsNotEmpty() idempotencyKey!: string;
@IsString() @IsNotEmpty() kind!: string;
@IsOptional() metadata?: Record<string, unknown>;
}
export class SetKillSwitchDto {
@IsIn([
'FREEZE_NEW_WORK',
'STOP_AGENTS',
'DISABLE_MUTATIONS',
'EMERGENCY_HALT',
])
level!: string;
@IsString() @IsNotEmpty() reason!: string;
}
export class ReconcileBudgetDto {
@IsString() @IsNotEmpty() reservationId!: string;
@Type(() => Number) @IsInt() @Min(0) actualCents!: number;
@IsString() @IsNotEmpty() idempotencyKey!: string;
}
export class OrchestrationCommandDto {
@Type(() => Number) @IsInt() @Min(1) expectedVersion!: number;
@IsString() @IsNotEmpty() idempotencyKey!: string;
@IsOptional() @IsString() commitSha?: string;
@IsOptional()
@IsIn(['development', 'staging', 'production'])
environment?: string;
@IsOptional() @IsString() runId?: string;
@IsOptional() @IsString() approvalId?: string;
@IsOptional() @IsString() dependsOnTaskId?: string;
@IsOptional() @IsIn(['SUCCEEDED', 'FAILED']) status?: string;
@IsOptional()
@IsIn([
'TRANSIENT',
'RATE_LIMITED',
'DEPENDENCY',
'VALIDATION',
'POLICY',
'PERMANENT',
])
retryClass?: string;
@IsOptional() @IsHash('sha256') sha256?: string;
@IsOptional() @IsHash('sha256') artifactHash?: string;
@IsOptional() @IsHash('sha256') subjectHash?: string;
@IsOptional() @IsString() kind?: string;
@IsOptional() @IsString() uri?: string;
@IsOptional() predicate?: Record<string, unknown>;
@IsOptional() @IsString() signature?: string;
}
@@ -1,24 +1,45 @@
import { ForbiddenException } from "@nestjs/common";
import { KillSwitchService } from "./kill-switch.service";
import { ForbiddenException } from '@nestjs/common';
import { KillSwitchService } from './kill-switch.service';
describe("KillSwitchService", () => {
describe('KillSwitchService', () => {
const prisma = { orchestrationKillSwitch: { findUnique: jest.fn() } };
const service = new KillSwitchService(prisma as never);
beforeEach(() => jest.resetAllMocks());
it.each([
["FREEZE_NEW_WORK", "CREATE_TASK"],
["STOP_AGENTS", "RUN_AGENT"],
["DISABLE_MUTATIONS", "MUTATE_PRODUCTION"],
["EMERGENCY_HALT", "OBSERVE"],
])("blocks %s at its policy boundary", async (level, action) => {
prisma.orchestrationKillSwitch.findUnique.mockResolvedValue({ scope: "GLOBAL", level, active: true });
await expect(service.assertAllowed(action as never)).rejects.toThrow(ForbiddenException);
['FREEZE_NEW_WORK', 'CREATE_TASK'],
['STOP_AGENTS', 'RUN_AGENT'],
['DISABLE_MUTATIONS', 'MUTATE_PRODUCTION'],
['EMERGENCY_HALT', 'OBSERVE'],
])('blocks %s at its policy boundary', async (level, action) => {
prisma.orchestrationKillSwitch.findUnique.mockResolvedValue({
scope: 'GLOBAL',
level,
active: true,
});
await expect(service.assertAllowed(action as never)).rejects.toThrow(
ForbiddenException,
);
});
it("allows observation while production mutations are disabled", async () => {
prisma.orchestrationKillSwitch.findUnique.mockResolvedValue({ scope: "GLOBAL", level: "DISABLE_MUTATIONS", active: true });
await expect(service.assertAllowed("OBSERVE")).resolves.toBeUndefined();
it('freezes new claims while allowing an already running agent to finish', async () => {
prisma.orchestrationKillSwitch.findUnique.mockResolvedValue({scope:'GLOBAL',level:'FREEZE_NEW_WORK',active:true});
await expect(service.assertAllowed('CLAIM_TASK' as never)).rejects.toThrow(ForbiddenException);
await expect(service.assertAllowed('RUN_AGENT')).resolves.toBeUndefined();
});
it('fails closed when an active persisted kill level is unknown', async () => {
prisma.orchestrationKillSwitch.findUnique.mockResolvedValue({scope:'GLOBAL',level:'CORRUPT_LEVEL',active:true});
await expect(service.assertAllowed('RUN_AGENT')).rejects.toThrow(ForbiddenException);
});
it('allows observation while production mutations are disabled', async () => {
prisma.orchestrationKillSwitch.findUnique.mockResolvedValue({
scope: 'GLOBAL',
level: 'DISABLE_MUTATIONS',
active: true,
});
await expect(service.assertAllowed('OBSERVE')).resolves.toBeUndefined();
});
});
@@ -1,9 +1,36 @@
import { ForbiddenException, Injectable } from "@nestjs/common";
import { PrismaService } from "../prisma/prisma.service";
import { KillSwitchLevel } from "./orchestration.domain";
export type GuardedAction="CREATE_TASK"|"RUN_AGENT"|"MUTATE_PRODUCTION"|"OBSERVE";
import { ForbiddenException, Injectable } from '@nestjs/common';
import { PrismaService } from '../prisma/prisma.service';
import { KillSwitchLevel } from './orchestration.domain';
export type GuardedAction =
| 'CREATE_TASK'
| 'CLAIM_TASK'
| 'RUN_AGENT'
| 'MUTATE_PRODUCTION'
| 'OBSERVE';
@Injectable()
export class KillSwitchService {
constructor(private readonly prisma:PrismaService){}
async assertAllowed(action:GuardedAction):Promise<void>{ const state=await (this.prisma as any).orchestrationKillSwitch.findUnique({where:{scope:"GLOBAL"}}); if(!state?.active)return; const blocked:Record<KillSwitchLevel,GuardedAction[]>={FREEZE_NEW_WORK:["CREATE_TASK"],STOP_AGENTS:["CREATE_TASK","RUN_AGENT"],DISABLE_MUTATIONS:["CREATE_TASK","RUN_AGENT","MUTATE_PRODUCTION"],EMERGENCY_HALT:["CREATE_TASK","RUN_AGENT","MUTATE_PRODUCTION","OBSERVE"]}; if(blocked[state.level as KillSwitchLevel]?.includes(action)) throw new ForbiddenException(`Orchestration kill switch ${state.level} blocks ${action}`); }
constructor(private readonly prisma: PrismaService) {}
async assertAllowed(action: GuardedAction): Promise<void> {
const state = await (this.prisma as any).orchestrationKillSwitch.findUnique(
{ where: { scope: 'GLOBAL' } },
);
if (!state?.active) return;
const blocked: Record<KillSwitchLevel, GuardedAction[]> = {
FREEZE_NEW_WORK: ['CREATE_TASK', 'CLAIM_TASK'],
STOP_AGENTS: ['CREATE_TASK', 'CLAIM_TASK', 'RUN_AGENT'],
DISABLE_MUTATIONS: ['CREATE_TASK', 'CLAIM_TASK', 'RUN_AGENT', 'MUTATE_PRODUCTION'],
EMERGENCY_HALT: [
'CREATE_TASK',
'CLAIM_TASK',
'RUN_AGENT',
'MUTATE_PRODUCTION',
'OBSERVE',
],
};
if (!Object.values(KillSwitchLevel).includes(state.level) ||
blocked[state.level as KillSwitchLevel].includes(action))
throw new ForbiddenException(
`Orchestration kill switch ${state.level} blocks ${action}`,
);
}
}
@@ -1,3 +1,18 @@
import { readFileSync } from "node:fs";
import { join } from "node:path";
describe("orchestration migration safety",()=>{it("is additive and never mutates legacy governance tables",()=>{const sql=readFileSync(join(process.cwd(),"prisma/migrations/20261022000000_add_orchestration_v4_core/migration.sql"),"utf8");expect(sql).not.toMatch(/^\s*(DROP|TRUNCATE|DELETE|UPDATE)\b/im);expect(sql).not.toMatch(/ALTER TABLE "(approval_requests|department_tasks|audit_logs)"/i);expect(sql.match(/CREATE TABLE "orchestration_/g)).toHaveLength(9);});});
import { readFileSync } from 'node:fs';
import { join } from 'node:path';
describe('orchestration migration safety', () => {
it('is additive and never mutates legacy governance tables', () => {
const sql = readFileSync(
join(
process.cwd(),
'prisma/migrations/20261022000000_add_orchestration_v4_core/migration.sql',
),
'utf8',
);
expect(sql).not.toMatch(/^\s*(DROP|TRUNCATE|DELETE|UPDATE)\b/im);
expect(sql).not.toMatch(
/ALTER TABLE "(approval_requests|department_tasks|audit_logs)"/i,
);
expect(sql.match(/CREATE TABLE "orchestration_/g)).toHaveLength(9);
});
});
@@ -1,21 +1,68 @@
import { BadRequestException } from "@nestjs/common";
import { ModelTierPolicyService } from "./model-tier-policy.service";
import { BadRequestException } from '@nestjs/common';
import { ModelTierPolicyService } from './model-tier-policy.service';
describe("ModelTierPolicyService", () => {
describe('ModelTierPolicyService', () => {
const policy = new ModelTierPolicyService();
it.each([
[{ complexity: 0, risk: 0, operationMode: "OBSERVE", remainingBudgetCents: 1 }, "L0"],
[{ complexity: 2, risk: 1, operationMode: "PREPARE", remainingBudgetCents: 100 }, "L1"],
[{ complexity: 5, risk: 4, operationMode: "PREPARE", remainingBudgetCents: 500 }, "L2"],
[{ complexity: 7, risk: 7, operationMode: "PREPARE", remainingBudgetCents: 1500 }, "L3"],
[{ complexity: 9, risk: 9, operationMode: "MUTATE_PRODUCTION", remainingBudgetCents: 5000 }, "L4"],
])("selects a deterministic tier for policy inputs", (input, expected) => {
[
{
complexity: 0,
risk: 0,
operationMode: 'OBSERVE',
remainingBudgetCents: 1,
},
'L0',
],
[
{
complexity: 2,
risk: 1,
operationMode: 'PREPARE',
remainingBudgetCents: 100,
},
'L1',
],
[
{
complexity: 5,
risk: 4,
operationMode: 'PREPARE',
remainingBudgetCents: 500,
},
'L2',
],
[
{
complexity: 7,
risk: 7,
operationMode: 'PREPARE',
remainingBudgetCents: 1500,
},
'L3',
],
[
{
complexity: 9,
risk: 9,
operationMode: 'MUTATE_PRODUCTION',
remainingBudgetCents: 5000,
},
'L4',
],
])('selects a deterministic tier for policy inputs', (input, expected) => {
expect(policy.select(input as never)).toBe(expected);
expect(policy.select(input as never)).toBe(expected);
});
it("fails closed when the selected tier exceeds the remaining hard budget", () => {
expect(() => policy.select({ complexity: 9, risk: 9, operationMode: "MUTATE_PRODUCTION", remainingBudgetCents: 100 })).toThrow(BadRequestException);
it('fails closed when the selected tier exceeds the remaining hard budget', () => {
expect(() =>
policy.select({
complexity: 9,
risk: 9,
operationMode: 'MUTATE_PRODUCTION',
remainingBudgetCents: 100,
}),
).toThrow(BadRequestException);
});
});
@@ -1,13 +1,40 @@
import { BadRequestException, Injectable } from "@nestjs/common";
import { OperationMode } from "./orchestration.domain";
export type ModelTier="L0"|"L1"|"L2"|"L3"|"L4";
import { BadRequestException, Injectable } from '@nestjs/common';
import { OperationMode } from './orchestration.domain';
export type ModelTier = 'L0' | 'L1' | 'L2' | 'L3' | 'L4';
@Injectable()
export class ModelTierPolicyService {
select(input:{complexity:number;risk:number;operationMode:OperationMode|string;remainingBudgetCents:number}):ModelTier {
const score=Math.max(input.complexity,input.risk)+(input.operationMode===OperationMode.MUTATE_PRODUCTION?1:0);
const tier:ModelTier=score<=1?"L0":score<=3?"L1":score<=5?"L2":score<=8?"L3":"L4";
const floor:Record<ModelTier,number>={L0:1,L1:25,L2:100,L3:500,L4:2000};
if(input.remainingBudgetCents<floor[tier]) throw new BadRequestException(`Hard budget cannot fund selected model tier ${tier}`);
return tier;
}
select(input: {
complexity: number;
risk: number;
operationMode: OperationMode | string;
remainingBudgetCents: number;
}): ModelTier {
const score =
Math.max(input.complexity, input.risk) +
(String(input.operationMode) === String(OperationMode.MUTATE_PRODUCTION)
? 1
: 0);
const tier: ModelTier =
score <= 1
? 'L0'
: score <= 3
? 'L1'
: score <= 5
? 'L2'
: score <= 8
? 'L3'
: 'L4';
const floor: Record<ModelTier, number> = {
L0: 1,
L1: 25,
L2: 100,
L3: 500,
L4: 2000,
};
if (input.remainingBudgetCents < floor[tier])
throw new BadRequestException(
`Hard budget cannot fund selected model tier ${tier}`,
);
return tier;
}
}
@@ -52,6 +52,8 @@ describe('integration read boundary', () => {
decideGate: jest.fn(),
recordBudget: jest.fn(),
setKillSwitch: jest.fn(),
command: jest.fn(),
reconcileBudget: jest.fn(),
};
beforeAll(async () => {
const m = await Test.createTestingModule({
@@ -81,6 +83,8 @@ describe('integration read boundary', () => {
['put', 'tasks/t1/transition'],
['post', 'tasks/t1/gates/2/decide'],
['post', 'tasks/t1/budget'],
['post', 'tasks/t1/commands/RUN_CREATE'],
['post', 'tasks/t1/budget/reconcile'],
['put', 'kill-switch'],
['post', 'chat/messages'],
] as const) {
@@ -96,6 +100,8 @@ describe('integration read boundary', () => {
'decideGate',
'recordBudget',
'setKillSwitch',
'command',
'reconcileBudget',
] as const)
expect(service[name]).not.toHaveBeenCalled();
});
@@ -139,6 +139,12 @@ export class OrchestrationController {
@Post('chat/messages') chat() {
return this.deny();
}
@Post('tasks/:id/commands/:command') command() {
return this.deny();
}
@Post('tasks/:id/budget/reconcile') reconcile() {
return this.deny();
}
private deny(): never {
throw new ForbiddenException(
'Orchestration mutations and chat disabled pending hardened core and trusted runtime integration',
@@ -1,24 +1,47 @@
import { BadRequestException, ConflictException, ForbiddenException } from "@nestjs/common";
import { assertTransition, nextStateForGateDecision, OperationMode, WorkflowState } from "./orchestration.domain";
import {
BadRequestException,
ConflictException,
ForbiddenException,
} from '@nestjs/common';
import {
assertTransition,
nextStateForGateDecision,
OperationMode,
WorkflowState,
} from './orchestration.domain';
describe("orchestration domain", () => {
it("allows only declared workflow transitions", () => {
expect(() => assertTransition(WorkflowState.DRAFT, WorkflowState.GATE1_PENDING)).not.toThrow();
expect(() => assertTransition(WorkflowState.DRAFT, WorkflowState.RUNNING)).toThrow(BadRequestException);
describe('orchestration domain', () => {
it('allows only declared workflow transitions', () => {
expect(() =>
assertTransition(WorkflowState.DRAFT, WorkflowState.GATE1_PENDING),
).not.toThrow();
expect(() =>
assertTransition(WorkflowState.DRAFT, WorkflowState.RUNNING),
).toThrow(BadRequestException);
});
it("keeps gate 1 as execution authorization rather than release authorization", () => {
expect(nextStateForGateDecision(1, "APPROVED")).toBe(WorkflowState.GATE1_APPROVED);
expect(nextStateForGateDecision(2, "APPROVED")).toBe(WorkflowState.GATE2_APPROVED);
it('keeps gate 1 as execution authorization rather than release authorization', () => {
expect(nextStateForGateDecision(1, 'APPROVED')).toBe(
WorkflowState.GATE1_APPROVED,
);
expect(nextStateForGateDecision(2, 'APPROVED')).toBe(
WorkflowState.GATE2_APPROVED,
);
});
it("rejects an unsupported gate decision", () => {
expect(() => nextStateForGateDecision(1, "PENDING" as never)).toThrow(BadRequestException);
it('rejects an unsupported gate decision', () => {
expect(() => nextStateForGateDecision(1, 'PENDING' as never)).toThrow(
BadRequestException,
);
});
it("classifies production mutation separately from observe and prepare", () => {
expect(OperationMode.OBSERVE).toBe("OBSERVE");
expect(OperationMode.PREPARE).toBe("PREPARE");
expect(OperationMode.MUTATE_PRODUCTION).toBe("MUTATE_PRODUCTION");
it.each([WorkflowState.GATE1_PENDING, WorkflowState.GATE2_PENDING])('does not revive an expired proposal into %s', (state) => {
expect(() => assertTransition(WorkflowState.EXPIRED,state)).toThrow(BadRequestException);
});
it('classifies production mutation separately from observe and prepare', () => {
expect(OperationMode.OBSERVE).toBe('OBSERVE');
expect(OperationMode.PREPARE).toBe('PREPARE');
expect(OperationMode.MUTATE_PRODUCTION).toBe('MUTATE_PRODUCTION');
});
});
@@ -1,12 +1,106 @@
import { BadRequestException } from "@nestjs/common";
import { BadRequestException } from '@nestjs/common';
export enum WorkflowState { DRAFT="DRAFT", GATE1_PENDING="GATE1_PENDING", GATE1_APPROVED="GATE1_APPROVED", READY="READY", CLAIMED="CLAIMED", RUNNING="RUNNING", VERIFYING="VERIFYING", GATE2_PENDING="GATE2_PENDING", GATE2_APPROVED="GATE2_APPROVED", RELEASING="RELEASING", PRODUCTION_VERIFYING="PRODUCTION_VERIFYING", LIVE_VERIFIED="LIVE_VERIFIED", MEASURING="MEASURING", CLOSED="CLOSED", REJECTED="REJECTED", BLOCKED="BLOCKED", CANCELLED="CANCELLED", FAILED="FAILED", EXPIRED="EXPIRED" }
export enum OperationMode { OBSERVE="OBSERVE", PREPARE="PREPARE", MUTATE_PRODUCTION="MUTATE_PRODUCTION" }
export enum RetryClass { TRANSIENT="TRANSIENT", RATE_LIMITED="RATE_LIMITED", DEPENDENCY="DEPENDENCY", VALIDATION="VALIDATION", POLICY="POLICY", PERMANENT="PERMANENT" }
export enum KillSwitchLevel { FREEZE_NEW_WORK="FREEZE_NEW_WORK", STOP_AGENTS="STOP_AGENTS", DISABLE_MUTATIONS="DISABLE_MUTATIONS", EMERGENCY_HALT="EMERGENCY_HALT" }
export enum WorkflowState {
DRAFT = 'DRAFT',
GATE1_PENDING = 'GATE1_PENDING',
GATE1_APPROVED = 'GATE1_APPROVED',
READY = 'READY',
CLAIMED = 'CLAIMED',
RUNNING = 'RUNNING',
VERIFYING = 'VERIFYING',
GATE2_PENDING = 'GATE2_PENDING',
GATE2_APPROVED = 'GATE2_APPROVED',
RELEASING = 'RELEASING',
PRODUCTION_VERIFYING = 'PRODUCTION_VERIFYING',
LIVE_VERIFIED = 'LIVE_VERIFIED',
MEASURING = 'MEASURING',
CLOSED = 'CLOSED',
REJECTED = 'REJECTED',
BLOCKED = 'BLOCKED',
CANCELLED = 'CANCELLED',
FAILED = 'FAILED',
EXPIRED = 'EXPIRED',
}
export enum OperationMode {
OBSERVE = 'OBSERVE',
PREPARE = 'PREPARE',
MUTATE_PRODUCTION = 'MUTATE_PRODUCTION',
}
export enum RetryClass {
TRANSIENT = 'TRANSIENT',
RATE_LIMITED = 'RATE_LIMITED',
DEPENDENCY = 'DEPENDENCY',
VALIDATION = 'VALIDATION',
POLICY = 'POLICY',
PERMANENT = 'PERMANENT',
}
export enum KillSwitchLevel {
FREEZE_NEW_WORK = 'FREEZE_NEW_WORK',
STOP_AGENTS = 'STOP_AGENTS',
DISABLE_MUTATIONS = 'DISABLE_MUTATIONS',
EMERGENCY_HALT = 'EMERGENCY_HALT',
}
const transitions: Record<WorkflowState, WorkflowState[]> = {
DRAFT:[WorkflowState.GATE1_PENDING,WorkflowState.CANCELLED], GATE1_PENDING:[WorkflowState.GATE1_APPROVED,WorkflowState.REJECTED,WorkflowState.EXPIRED], GATE1_APPROVED:[WorkflowState.READY,WorkflowState.CANCELLED], READY:[WorkflowState.CLAIMED,WorkflowState.BLOCKED,WorkflowState.CANCELLED], CLAIMED:[WorkflowState.RUNNING,WorkflowState.READY,WorkflowState.CANCELLED], RUNNING:[WorkflowState.VERIFYING,WorkflowState.FAILED,WorkflowState.BLOCKED], VERIFYING:[WorkflowState.GATE2_PENDING,WorkflowState.FAILED,WorkflowState.RUNNING], GATE2_PENDING:[WorkflowState.GATE2_APPROVED,WorkflowState.REJECTED,WorkflowState.EXPIRED], GATE2_APPROVED:[WorkflowState.RELEASING,WorkflowState.CANCELLED], RELEASING:[WorkflowState.PRODUCTION_VERIFYING,WorkflowState.FAILED], PRODUCTION_VERIFYING:[WorkflowState.LIVE_VERIFIED,WorkflowState.FAILED], LIVE_VERIFIED:[WorkflowState.MEASURING,WorkflowState.CLOSED], MEASURING:[WorkflowState.CLOSED,WorkflowState.FAILED], CLOSED:[], REJECTED:[], BLOCKED:[WorkflowState.READY,WorkflowState.CANCELLED], CANCELLED:[], FAILED:[WorkflowState.READY,WorkflowState.CANCELLED], EXPIRED:[WorkflowState.GATE1_PENDING,WorkflowState.GATE2_PENDING,WorkflowState.CANCELLED]
DRAFT: [WorkflowState.GATE1_PENDING, WorkflowState.CANCELLED],
GATE1_PENDING: [
WorkflowState.GATE1_APPROVED,
WorkflowState.REJECTED,
WorkflowState.EXPIRED,
],
GATE1_APPROVED: [WorkflowState.READY, WorkflowState.CANCELLED],
READY: [
WorkflowState.CLAIMED,
WorkflowState.BLOCKED,
WorkflowState.CANCELLED,
],
CLAIMED: [
WorkflowState.RUNNING,
WorkflowState.READY,
WorkflowState.CANCELLED,
],
RUNNING: [
WorkflowState.VERIFYING,
WorkflowState.FAILED,
WorkflowState.BLOCKED,
],
VERIFYING: [
WorkflowState.GATE2_PENDING,
WorkflowState.FAILED,
WorkflowState.RUNNING,
],
GATE2_PENDING: [
WorkflowState.GATE2_APPROVED,
WorkflowState.REJECTED,
WorkflowState.EXPIRED,
],
GATE2_APPROVED: [WorkflowState.RELEASING, WorkflowState.CANCELLED],
RELEASING: [WorkflowState.PRODUCTION_VERIFYING, WorkflowState.FAILED],
PRODUCTION_VERIFYING: [WorkflowState.LIVE_VERIFIED, WorkflowState.FAILED],
LIVE_VERIFIED: [WorkflowState.MEASURING, WorkflowState.CLOSED],
MEASURING: [WorkflowState.CLOSED, WorkflowState.FAILED],
CLOSED: [],
REJECTED: [],
BLOCKED: [WorkflowState.READY, WorkflowState.CANCELLED],
CANCELLED: [],
FAILED: [WorkflowState.READY, WorkflowState.CANCELLED],
EXPIRED: [WorkflowState.CANCELLED],
};
export function assertTransition(from: WorkflowState, to: WorkflowState): void { if (!transitions[from]?.includes(to)) throw new BadRequestException(`Invalid orchestration transition ${from} -> ${to}`); }
export function nextStateForGateDecision(gate: 1|2, decision: "APPROVED"|"REJECTED"): WorkflowState { if (!(["APPROVED","REJECTED"] as string[]).includes(decision)) throw new BadRequestException("Decision must be APPROVED or REJECTED"); return decision === "REJECTED" ? WorkflowState.REJECTED : gate === 1 ? WorkflowState.GATE1_APPROVED : WorkflowState.GATE2_APPROVED; }
export function assertTransition(from: WorkflowState, to: WorkflowState): void {
if (!transitions[from]?.includes(to))
throw new BadRequestException(
`Invalid orchestration transition ${from} -> ${to}`,
);
}
export function nextStateForGateDecision(
gate: 1 | 2,
decision: 'APPROVED' | 'REJECTED',
): WorkflowState {
if (!(['APPROVED', 'REJECTED'] as string[]).includes(decision))
throw new BadRequestException('Decision must be APPROVED or REJECTED');
return decision === 'REJECTED'
? WorkflowState.REJECTED
: gate === 1
? WorkflowState.GATE1_APPROVED
: WorkflowState.GATE2_APPROVED;
}
@@ -0,0 +1,577 @@
import { OrchestrationController } from './orchestration.controller';
import { DepartmentTasksController } from '../admin/department-tasks.controller';
import { ApprovalsController } from '../admin/approvals.controller';
import { PrismaClient } from '@prisma/client';
import { OrchestrationService } from './orchestration.service';
import { KillSwitchService } from './kill-switch.service';
import { WorkflowState, OperationMode } from './orchestration.domain';
const enabled = process.env.ORCHESTRATION_INTEGRATION === '1';
(enabled ? describe : describe.skip)(
'v4 disposable PostgreSQL invariants',
() => {
const db = new PrismaClient();
const service = new OrchestrationService(
db as never,
new KillSwitchService(db as never),
);
let n = 0;
async function task(state = 'DRAFT', budget = 100) {
return db.orchestrationTask.create({
data: {
key: `test-${Date.now()}-${n++}`,
title: 'test',
department: 'TEST',
operationMode: 'PREPARE',
state,
budgetLimitCents: budget,
},
});
}
afterAll(async () => db.$disconnect());
it('release rejects PREPARE operation class even with gate approval', async()=>{
const t=await task('GATE2_APPROVED');
await expect(service.command(t.id,'RELEASE_AUTHORIZE',{expectedVersion:1,idempotencyKey:`mode-${t.id}`},'ci')).rejects.toThrow(/operation mode/i);
});
it('append-only ledger rejects direct rewrites in PostgreSQL', async () => {
const t = await task();
const r = await service.recordBudget(
t.id,
{
amountCents: 50,
kind: 'RESERVE',
idempotencyKey: `immutable-${t.id}`,
},
'a',
);
await expect(
db.orchestrationBudgetLedger.update({
where: { id: r.id },
data: { amountCents: 0 },
}),
).rejects.toThrow(/append.only/i);
await expect(
db.orchestrationBudgetLedger.delete({ where: { id: r.id } }),
).rejects.toThrow(/append.only/i);
});
it('generic transitions cannot fake run lifecycle or production verification', async () => {
for (const [from, to] of [
['CLAIMED', 'RUNNING'],
['RUNNING', 'VERIFYING'],
['FAILED', 'READY'],
['PRODUCTION_VERIFYING', 'LIVE_VERIFIED'],
]) {
const t = await task(from);
await expect(
service.transition(
t.id,
{
expectedVersion: 1,
idempotencyKey: `fake-${t.id}`,
toState: to as WorkflowState,
},
'agent',
),
).rejects.toThrow(/dedicated/i);
}
});
it('exposes command and reconcile HTTP handlers', () => {
const c = new OrchestrationController(service);
expect(typeof c.command).toBe('function');
expect(typeof c.reconcile).toBe('function');
});
it('highest kill level blocks reads and reconciliation', async () => {
const t = await task();
const r = await service.recordBudget(
t.id,
{
kind: 'RESERVE',
amountCents: 10,
idempotencyKey: `stop-res-${t.id}`,
},
'a',
);
await service.setKillSwitch(
['EMERGENCY', 'HALT'].join('_'),
'test',
'ceo',
);
try {
await expect(service.findOne(t.id)).rejects.toThrow(/kill switch/i);
await expect(
service.reconcileBudget(
t.id,
{
reservationId: r.id,
actualCents: 0,
idempotencyKey: `stop-recon-${t.id}`,
},
'a',
),
).rejects.toThrow(/kill switch/i);
} finally {
await db.orchestrationKillSwitch.deleteMany();
}
});
it('legacy arbitrary status and approval endpoints fail closed', async () => {
await expect(
new DepartmentTasksController(db as never).create({
title: 'bypass',
status: 'DONE',
}),
).rejects.toThrow(/v4/i);
await expect(
new DepartmentTasksController(db as never).updateStatus('none', {
status: 'DONE',
}),
).rejects.toThrow(/v4/i);
await expect(
new ApprovalsController(db as never).decide('none', {
decision: 'APPROVED',
} as any),
).rejects.toThrow(/v4/i);
});
it('release consumes only matching fresh production Gate2 via dedicated command', async () => {
const t = await task('GATE2_PENDING');
const hash = 'f'.repeat(64),
commitSha = 'a'.repeat(40);
const r = await db.orchestrationRun.create({
data: {
taskId: t.id,
status: 'SUCCEEDED',
commitSha,
environment: 'production',
idempotencyKey: `rel-run-${t.id}`,
},
});
await db.orchestrationArtifact.create({
data: {
runId: r.id,
kind: 'BUILD',
uri: 'https://ci.example/build',
sha256: hash,
},
});
await db.orchestrationAttestation.create({
data: {
runId: r.id,
kind: 'TEST',
subjectHash: hash,
signer: 'ci',
predicate: { passed: true },
},
});
const decision = await service.decideGate(
t.id,
2,
{
decision: 'APPROVED',
expectedVersion: 1,
idempotencyKey: `rel-approve-${t.id}`,
expiresAt: new Date(Date.now() + 60000).toISOString(),
runId: r.id,
artifactHash: hash,
commitSha,
environment: 'production',
},
'ceo',
);
const dto = {
expectedVersion: 2,
idempotencyKey: `rel-${t.id}`,
approvalId: decision.id,
runId: r.id,
artifactHash: hash,
commitSha,
environment: 'production',
};
await expect(
(service as any).command(
t.id,
'RELEASE_AUTHORIZE',
{ ...dto, artifactHash: '0'.repeat(64) },
'ci',
),
).rejects.toThrow(/binding/i);
const result = await (service as any).command(
t.id,
'RELEASE_AUTHORIZE',
dto,
'ci',
);
expect(result.authorized).toBe(true);
expect(
(await db.orchestrationTask.findUniqueOrThrow({ where: { id: t.id } }))
.state,
).toBe('RELEASING');
await expect(
(service as any).command(
t.id,
'RELEASE_AUTHORIZE',
{ ...dto, expectedVersion: 3, idempotencyKey: `rel-again-${t.id}` },
'ci',
),
).rejects.toThrow();
});
it('gate decision replay rejects changed immutable expiry and actor', async () => {
const t = await task('GATE1_PENDING');
const dto = {
decision: 'APPROVED' as const,
expectedVersion: 1,
idempotencyKey: `bind-${t.id}`,
expiresAt: new Date(Date.now() + 60000).toISOString(),
};
await service.decideGate(t.id, 1, dto, 'ceo');
await expect(
service.decideGate(
t.id,
1,
{ ...dto, expiresAt: new Date(Date.now() + 120000).toISOString() },
'ceo',
),
).rejects.toThrow(/Idempotency/i);
await expect(service.decideGate(t.id, 1, dto, 'other')).rejects.toThrow(
/Idempotency/i,
);
});
it('dependencies block admission and reject cycles', async () => {
const a = await task();
const b = await task();
await (service as any).command(
a.id,
'DEPENDENCY_ADD',
{
expectedVersion: 1,
idempotencyKey: `dep-a-${a.id}`,
dependsOnTaskId: b.id,
},
'actor',
);
await expect(
(service as any).command(
b.id,
'DEPENDENCY_ADD',
{
expectedVersion: 1,
idempotencyKey: `dep-b-${b.id}`,
dependsOnTaskId: a.id,
},
'actor',
),
).rejects.toThrow(/cycle/i);
await db.orchestrationTask.update({
where: { id: a.id },
data: { state: 'GATE1_PENDING' },
});
await service.decideGate(
a.id,
1,
{
decision: 'APPROVED',
expectedVersion: 2,
idempotencyKey: `dep-approval-${a.id}`,
expiresAt: new Date(Date.now() + 60000).toISOString(),
},
'ceo',
);
await expect(
service.transition(
a.id,
{
toState: WorkflowState.READY,
expectedVersion: 3,
idempotencyKey: `dep-ready-${a.id}`,
},
'agent',
),
).rejects.toThrow(/dependency/i);
});
it('registers artifact and attestation for owned run, blocks unregistered subjects', async () => {
const t = await task('CLAIMED');
const base = {
expectedVersion: 1,
idempotencyKey: `life-${t.id}`,
commitSha: 'c'.repeat(40),
environment: 'staging',
};
const r = await (service as any).command(
t.id,
'RUN_CREATE',
base,
'agent',
);
const hash = 'd'.repeat(64);
const a = await (service as any).command(
t.id,
'ARTIFACT_ADD',
{
expectedVersion: 2,
idempotencyKey: `art-${t.id}`,
runId: r.id,
sha256: hash,
kind: 'BUILD',
uri: 'https://ci.example/artifact',
},
'agent',
);
expect(a.sha256).toBe(hash);
await expect(
(service as any).command(
t.id,
'ATTESTATION_ADD',
{
expectedVersion: 3,
idempotencyKey: `bad-att-${t.id}`,
runId: r.id,
subjectHash: 'e'.repeat(64),
kind: 'TEST',
predicate: { passed: true },
},
'ci',
),
).rejects.toThrow(/artifact/i);
const att = await (service as any).command(
t.id,
'ATTESTATION_ADD',
{
expectedVersion: 3,
idempotencyKey: `att-${t.id}`,
runId: r.id,
subjectHash: hash,
kind: 'TEST',
predicate: { passed: true },
},
'ci',
);
expect(att.signer).toBe('ci');
});
it('creates immutable versioned runs with bounded transient retries', async () => {
const t = await task('CLAIMED');
const dto = {
expectedVersion: 1,
idempotencyKey: `run-${t.id}`,
commitSha: 'a'.repeat(40),
environment: 'staging',
};
const r = await (service as any).command(
t.id,
'RUN_CREATE',
dto,
'agent',
);
expect(r.attempt).toBe(1);
expect(
await (service as any).command(t.id, 'RUN_CREATE', dto, 'agent'),
).toEqual(r);
await expect(
(service as any).command(
t.id,
'RUN_CREATE',
{ ...dto, commitSha: 'b'.repeat(40) },
'agent',
),
).rejects.toThrow(/Idempotency/i);
await (service as any).command(
t.id,
'RUN_FINISH',
{
expectedVersion: 2,
idempotencyKey: `finish-${t.id}`,
runId: r.id,
status: 'FAILED',
retryClass: 'POLICY',
},
'agent',
);
await expect(
(service as any).command(
t.id,
'RUN_CREATE',
{ ...dto, expectedVersion: 3, idempotencyKey: `retry-${t.id}` },
'agent',
),
).rejects.toThrow(/retry/i);
});
it('selects tier at admission using backend risk and budget', async () => {
const t = await task('GATE1_PENDING', 1000);
await service.decideGate(
t.id,
1,
{
decision: 'APPROVED',
expectedVersion: 1,
idempotencyKey: `approve-tier-${t.id}`,
expiresAt: new Date(Date.now() + 60000).toISOString(),
},
'ceo',
);
await service.transition(
t.id,
{
toState: WorkflowState.READY,
expectedVersion: 2,
idempotencyKey: `admit-tier-${t.id}`,
},
'agent',
);
expect(
(await db.orchestrationTask.findUniqueOrThrow({ where: { id: t.id } }))
.modelTier,
).toBe('L0');
});
it('STOP_AGENTS prevents claims and running transitions', async () => {
await service.setKillSwitch('STOP_AGENTS', 'test', 'ceo');
try {
const t = await task('READY');
await expect(
service.transition(
t.id,
{
toState: WorkflowState.CLAIMED,
expectedVersion: 1,
idempotencyKey: `kill-${t.id}`,
},
'agent',
),
).rejects.toThrow(/kill switch/i);
} finally {
await db.orchestrationKillSwitch.deleteMany();
}
});
it('Gate1 admission consumes only unexpired approval', async () => {
const t = await task('GATE1_PENDING');
await service.decideGate(
t.id,
1,
{
decision: 'APPROVED',
expectedVersion: 1,
idempotencyKey: `approve-${t.id}`,
expiresAt: new Date(Date.now() + 100).toISOString(),
},
'ceo',
);
await new Promise((r) => setTimeout(r, 150));
await expect(
service.transition(
t.id,
{
toState: WorkflowState.READY,
expectedVersion: 2,
idempotencyKey: `admit-${t.id}`,
},
'agent',
),
).rejects.toThrow(/expired/i);
});
it('versioned transition replay returns original result and rejects changed payload', async () => {
const t = await task();
const dto = {
toState: WorkflowState.GATE1_PENDING,
expectedVersion: 1,
idempotencyKey: `tr-${t.id}`,
};
const first = await service.transition(t.id, dto, 'actor');
expect(await service.transition(t.id, dto, 'actor')).toEqual(first);
await expect(
service.transition(t.id, { ...dto, reason: 'different' }, 'actor'),
).rejects.toThrow(/Idempotency/i);
});
it('reconciles a reservation exactly once without erasing spent cost', async () => {
const t = await task();
const r = await service.recordBudget(
t.id,
{ amountCents: 80, kind: 'RESERVE', idempotencyKey: `res-${t.id}` },
'agent',
);
const dto = {
reservationId: r.id,
actualCents: 35,
idempotencyKey: `reconcile-${t.id}`,
};
await (service as any).reconcileBudget(t.id, dto, 'agent');
await (service as any).reconcileBudget(t.id, dto, 'agent');
await expect(
(service as any).reconcileBudget(
t.id,
{ ...dto, idempotencyKey: `second-${t.id}` },
'agent',
),
).rejects.toThrow(/reconciled/i);
expect(
(
await db.orchestrationBudgetLedger.aggregate({
where: { taskId: t.id },
_sum: { amountCents: true },
})
)._sum.amountCents,
).toBe(35);
await expect(
service.recordBudget(
t.id,
{
amountCents: 66,
kind: 'RESERVE',
idempotencyKey: `too-much-${t.id}`,
},
'agent',
),
).rejects.toThrow(/budget/i);
});
it('concurrent budget reservations never exceed hard cap and replay is harmless', async () => {
const t = await task();
const result = await Promise.allSettled(
Array.from({ length: 8 }, (_, i) =>
service.recordBudget(
t.id,
{
amountCents: 30,
kind: 'RESERVE',
idempotencyKey: `reserve-${t.id}-${i}`,
},
'agent',
),
),
);
expect(result.filter((x) => x.status === 'fulfilled')).toHaveLength(3);
const sum = await db.orchestrationBudgetLedger.aggregate({
where: { taskId: t.id },
_sum: { amountCents: true },
});
expect(sum._sum.amountCents).toBe(90);
const first = await db.orchestrationBudgetLedger.findFirstOrThrow({
where: { taskId: t.id },
});
await service.recordBudget(
t.id,
{
amountCents: 30,
kind: 'RESERVE',
idempotencyKey: first.idempotencyKey,
},
'agent',
);
expect(
await db.orchestrationBudgetLedger.count({ where: { taskId: t.id } }),
).toBe(3);
});
it('generic transition cannot approve gate 1', async () => {
const t = await task('GATE1_PENDING');
await expect(
service.transition(
t.id,
{
toState: WorkflowState.GATE1_APPROVED,
expectedVersion: 1,
idempotencyKey: `gate-bypass-${n}`,
},
'ceo',
),
).rejects.toThrow(/gate/i);
expect(
(await db.orchestrationTask.findUniqueOrThrow({ where: { id: t.id } }))
.state,
).toBe('GATE1_PENDING');
});
},
);
@@ -1,2 +1,13 @@
import { Module } from "@nestjs/common"; import { PrismaModule } from "../prisma/prisma.module"; import { KillSwitchService } from "./kill-switch.service"; import { ModelTierPolicyService } from "./model-tier-policy.service"; import { OrchestrationController } from "./orchestration.controller"; import { OrchestrationService } from "./orchestration.service";
@Module({imports:[PrismaModule],controllers:[OrchestrationController],providers:[OrchestrationService,KillSwitchService,ModelTierPolicyService],exports:[OrchestrationService,KillSwitchService,ModelTierPolicyService]}) export class OrchestrationModule{}
import { Module } from '@nestjs/common';
import { PrismaModule } from '../prisma/prisma.module';
import { KillSwitchService } from './kill-switch.service';
import { ModelTierPolicyService } from './model-tier-policy.service';
import { OrchestrationController } from './orchestration.controller';
import { OrchestrationService } from './orchestration.service';
@Module({
imports: [PrismaModule],
controllers: [OrchestrationController],
providers: [OrchestrationService, KillSwitchService, ModelTierPolicyService],
exports: [OrchestrationService, KillSwitchService, ModelTierPolicyService],
})
export class OrchestrationModule {}
@@ -1,73 +1,260 @@
import { BadRequestException, ConflictException, ForbiddenException } from "@nestjs/common";
import { OrchestrationService } from "./orchestration.service";
import {
BadRequestException,
ConflictException,
ForbiddenException,
} from '@nestjs/common';
import { OrchestrationService } from './orchestration.service';
function mockPrisma() {
const tx: any = {
orchestrationTask: { findUnique: jest.fn(), create: jest.fn(), updateMany: jest.fn() },
$queryRaw: jest.fn().mockResolvedValue([]),
orchestrationTask: {
findUnique: jest.fn(),
create: jest.fn(),
updateMany: jest.fn(),
},
orchestrationRun: { findUnique: jest.fn(), create: jest.fn() },
orchestrationArtifact: { findFirst: jest.fn(), create: jest.fn() },
orchestrationAttestation: { create: jest.fn() },
orchestrationApprovalDecision: { findUnique: jest.fn(), create: jest.fn() },
orchestrationEvent: { create: jest.fn() },
orchestrationBudgetLedger: { aggregate: jest.fn(), create: jest.fn() },
orchestrationEvent: { findUnique: jest.fn(), create: jest.fn() },
orchestrationBudgetLedger: {
findUnique: jest.fn(),
aggregate: jest.fn(),
create: jest.fn(),
},
orchestrationKillSwitch: { findUnique: jest.fn(), upsert: jest.fn() },
};
tx.$transaction = jest.fn(async (cb: any) => cb(tx));
tx.$transaction = jest.fn((cb: any) => Promise.resolve(cb(tx)));
return tx;
}
describe("OrchestrationService", () => {
describe('OrchestrationService', () => {
let prisma: any;
let service: OrchestrationService;
beforeEach(() => { prisma = mockPrisma(); service = new OrchestrationService(prisma, { assertAllowed: jest.fn() } as never); });
beforeEach(() => {
prisma = mockPrisma();
service = new OrchestrationService(prisma, {
assertAllowed: jest.fn(),
} as never);
});
it("creates a task and append-only event transactionally", async () => {
prisma.orchestrationTask.create.mockResolvedValue({ id: "task-1", state: "DRAFT", version: 1 });
const result = await service.createTask({ key: "seo-1", title: "SEO", department: "SEO", operationMode: "PREPARE" as never, budgetLimitCents: 1000 }, "actor-1");
it('rejects an unknown kill level before overwriting persisted protection', async () => {
await expect(service.setKillSwitch('INVALID_LEVEL','test','admin')).rejects.toThrow(BadRequestException);
expect(prisma.orchestrationKillSwitch.upsert).not.toHaveBeenCalled();
});
it('creates a task and append-only event transactionally', async () => {
prisma.orchestrationTask.create.mockResolvedValue({
id: 'task-1',
state: 'DRAFT',
version: 1,
});
const result = await service.createTask(
{
key: 'seo-1',
title: 'SEO',
department: 'SEO',
operationMode: 'PREPARE' as never,
budgetLimitCents: 1000,
},
'actor-1',
);
expect(prisma.$transaction).toHaveBeenCalledTimes(1);
expect(prisma.orchestrationTask.create).toHaveBeenCalledWith({ data: expect.objectContaining({ state: "DRAFT", version: 1 }) });
expect(prisma.orchestrationEvent.create).toHaveBeenCalledWith({ data: expect.objectContaining({ eventType: "TASK_CREATED", sequence: 1 }) });
expect(result.id).toBe("task-1");
expect(prisma.orchestrationTask.create).toHaveBeenCalledWith({
data: expect.objectContaining({ state: 'DRAFT', version: 1 }),
});
expect(prisma.orchestrationEvent.create).toHaveBeenCalledWith({
data: expect.objectContaining({ eventType: 'TASK_CREATED', sequence: 1 }),
});
expect(result.id).toBe('task-1');
});
it("uses compare-and-swap versioning for transitions", async () => {
prisma.orchestrationTask.findUnique.mockResolvedValue({ id: "task-1", state: "DRAFT", version: 3 });
it('uses compare-and-swap versioning for transitions', async () => {
prisma.orchestrationTask.findUnique.mockResolvedValue({
id: 'task-1',
state: 'DRAFT',
version: 3,
});
prisma.orchestrationTask.updateMany.mockResolvedValue({ count: 0 });
await expect(service.transition("task-1", { toState: "GATE1_PENDING" as never, expectedVersion: 3, idempotencyKey: "i-1" }, "actor")).rejects.toThrow(ConflictException);
await expect(
service.transition(
'task-1',
{
toState: 'GATE1_PENDING' as never,
expectedVersion: 3,
idempotencyKey: 'i-1',
},
'actor',
),
).rejects.toThrow(ConflictException);
});
it("makes gate decisions immutable and idempotent", async () => {
prisma.orchestrationApprovalDecision.findUnique.mockResolvedValue({ id: "d1", decision: "APPROVED", gate: 1, taskId: "task-1" });
await expect(service.decideGate("task-1", 1, { decision: "APPROVED", expectedVersion: 2, idempotencyKey: "gate-1", expiresAt: new Date(Date.now()+10000).toISOString() }, "ceo")).resolves.toMatchObject({ id: "d1" });
it('makes gate decisions immutable and idempotent', async () => {
const expiresAt = new Date(Date.now() + 10000).toISOString();
prisma.orchestrationApprovalDecision.findUnique.mockResolvedValue({
id: 'd1',
decision: 'APPROVED',
gate: 1,
taskId: 'task-1',
});
prisma.orchestrationEvent.findUnique.mockResolvedValue({
payload: {
fingerprint: JSON.stringify({
taskId: 'task-1',
gate: 1,
dto: {
decision: 'APPROVED',
expectedVersion: 2,
idempotencyKey: 'gate-1',
expiresAt,
},
actorId: 'ceo',
}),
},
});
await expect(
service.decideGate(
'task-1',
1,
{
decision: 'APPROVED',
expectedVersion: 2,
idempotencyKey: 'gate-1',
expiresAt,
},
'ceo',
),
).resolves.toMatchObject({ id: 'd1' });
expect(prisma.orchestrationApprovalDecision.create).not.toHaveBeenCalled();
});
it("rejects an expired approval", async () => {
await expect(service.decideGate("task-1", 1, { decision: "APPROVED", expectedVersion: 2, idempotencyKey: "gate-expired", expiresAt: new Date(0).toISOString() }, "ceo")).rejects.toThrow(BadRequestException);
it('rejects an expired approval', async () => {
await expect(
service.decideGate(
'task-1',
1,
{
decision: 'APPROVED',
expectedVersion: 2,
idempotencyKey: 'gate-expired',
expiresAt: new Date(0).toISOString(),
},
'ceo',
),
).rejects.toThrow(BadRequestException);
});
it("binds gate 2 approval to run, artifact hash, commit and environment", async () => {
it('binds gate 2 approval to run, artifact hash, commit and environment', async () => {
prisma.orchestrationApprovalDecision.findUnique.mockResolvedValue(null);
prisma.orchestrationTask.findUnique.mockResolvedValue({ id: "task-1", state: "GATE2_PENDING", version: 7 });
prisma.orchestrationRun.findUnique.mockResolvedValue({ id: "run-1", taskId: "task-1", commitSha: "abc123", environment: "staging" });
prisma.orchestrationArtifact.findFirst.mockResolvedValue({ id: "artifact-1", runId: "run-1", sha256: "f".repeat(64) });
prisma.orchestrationTask.findUnique.mockResolvedValue({
id: 'task-1',
state: 'GATE2_PENDING',
version: 7,
});
prisma.orchestrationRun.findUnique.mockResolvedValue({
id: 'run-1',
taskId: 'task-1',
commitSha: 'abc123',
environment: 'staging',
status: 'SUCCEEDED',
});
prisma.orchestrationArtifact.findFirst.mockResolvedValue({
id: 'artifact-1',
runId: 'run-1',
sha256: 'f'.repeat(64),
});
prisma.orchestrationTask.updateMany.mockResolvedValue({ count: 1 });
prisma.orchestrationApprovalDecision.create.mockResolvedValue({ id: "decision-2", gate: 2 });
await service.decideGate("task-1", 2, { decision: "APPROVED", expectedVersion: 7, idempotencyKey: "gate-2", expiresAt: new Date(Date.now()+10000).toISOString(), runId: "run-1", artifactHash: "f".repeat(64), commitSha: "abc123", environment: "staging" }, "ceo");
expect(prisma.orchestrationApprovalDecision.create).toHaveBeenCalledWith({ data: expect.objectContaining({ runId: "run-1", artifactHash: "f".repeat(64), commitSha: "abc123", environment: "staging" }) });
prisma.orchestrationApprovalDecision.create.mockResolvedValue({
id: 'decision-2',
gate: 2,
});
await service.decideGate(
'task-1',
2,
{
decision: 'APPROVED',
expectedVersion: 7,
idempotencyKey: 'gate-2',
expiresAt: new Date(Date.now() + 10000).toISOString(),
runId: 'run-1',
artifactHash: 'f'.repeat(64),
commitSha: 'abc123',
environment: 'staging',
},
'ceo',
);
expect(prisma.orchestrationApprovalDecision.create).toHaveBeenCalledWith({
data: expect.objectContaining({
runId: 'run-1',
artifactHash: 'f'.repeat(64),
commitSha: 'abc123',
environment: 'staging',
}),
});
});
it("refuses gate 2 when binding does not match the immutable run", async () => {
it.each(['RUNNING', 'FAILED', 'CANCELLED'])('refuses Gate 2 approval for an unfinished or unsuccessful %s run', async (status) => {
prisma.orchestrationTask.findUnique.mockResolvedValue({id:'task-1',state:'GATE2_PENDING',version:7});
prisma.orchestrationRun.findUnique.mockResolvedValue({id:'run-1',taskId:'task-1',commitSha:'abc123',environment:'staging',status});
prisma.orchestrationArtifact.findFirst.mockResolvedValue({id:'artifact-1'});
prisma.orchestrationTask.updateMany.mockResolvedValue({count:1});
await expect(service.decideGate('task-1',2,{
decision:'APPROVED',expectedVersion:7,idempotencyKey:'failed-run',
expiresAt:new Date(Date.now()+10000).toISOString(),runId:'run-1',
artifactHash:'f'.repeat(64),commitSha:'abc123',environment:'staging',
},'ceo')).rejects.toThrow(/successful run/i);
expect(prisma.orchestrationApprovalDecision.create).not.toHaveBeenCalled();
expect(prisma.orchestrationTask.updateMany).not.toHaveBeenCalled();
});
it('refuses gate 2 when binding does not match the immutable run', async () => {
prisma.orchestrationApprovalDecision.findUnique.mockResolvedValue(null);
prisma.orchestrationTask.findUnique.mockResolvedValue({ id: "task-1", state: "GATE2_PENDING", version: 7 });
prisma.orchestrationRun.findUnique.mockResolvedValue({ id: "run-1", taskId: "task-1", commitSha: "real", environment: "staging" });
await expect(service.decideGate("task-1", 2, { decision: "APPROVED", expectedVersion: 7, idempotencyKey: "gate-bad", expiresAt: new Date(Date.now()+10000).toISOString(), runId: "run-1", artifactHash: "f".repeat(64), commitSha: "wrong", environment: "staging" }, "ceo")).rejects.toThrow(ForbiddenException);
prisma.orchestrationTask.findUnique.mockResolvedValue({
id: 'task-1',
state: 'GATE2_PENDING',
version: 7,
});
prisma.orchestrationRun.findUnique.mockResolvedValue({
id: 'run-1',
taskId: 'task-1',
commitSha: 'real',
environment: 'staging',
});
await expect(
service.decideGate(
'task-1',
2,
{
decision: 'APPROVED',
expectedVersion: 7,
idempotencyKey: 'gate-bad',
expiresAt: new Date(Date.now() + 10000).toISOString(),
runId: 'run-1',
artifactHash: 'f'.repeat(64),
commitSha: 'wrong',
environment: 'staging',
},
'ceo',
),
).rejects.toThrow(ForbiddenException);
});
it("enforces hard task budget atomically", async () => {
prisma.orchestrationTask.findUnique.mockResolvedValue({ id: "task-1", budgetLimitCents: 1000 });
prisma.orchestrationBudgetLedger.aggregate.mockResolvedValue({ _sum: { amountCents: 900 } });
await expect(service.recordBudget("task-1", { amountCents: 101, idempotencyKey: "cost-1", kind: "MODEL" }, "agent")).rejects.toThrow(ForbiddenException);
it('enforces hard task budget atomically', async () => {
prisma.orchestrationTask.findUnique.mockResolvedValue({
id: 'task-1',
budgetLimitCents: 1000,
});
prisma.orchestrationBudgetLedger.aggregate.mockResolvedValue({
_sum: { amountCents: 900 },
});
await expect(
service.recordBudget(
'task-1',
{ amountCents: 101, idempotencyKey: 'cost-1', kind: 'SPEND' },
'agent',
),
).rejects.toThrow(ForbiddenException);
expect(prisma.orchestrationBudgetLedger.create).not.toHaveBeenCalled();
});
});
@@ -1,17 +1,665 @@
import { BadRequestException, ConflictException, ForbiddenException, Injectable, NotFoundException } from "@nestjs/common";
import { PrismaService } from "../prisma/prisma.service";
import { CreateOrchestrationTaskDto, GateDecisionDto, RecordBudgetDto, TransitionTaskDto } from "./dto/orchestration.dto";
import { assertTransition, nextStateForGateDecision, OperationMode, WorkflowState } from "./orchestration.domain";
import { KillSwitchService } from "./kill-switch.service";
import {
BadRequestException,
ConflictException,
ForbiddenException,
Injectable,
NotFoundException,
} from '@nestjs/common';
import { PrismaService } from '../prisma/prisma.service';
import {
CreateOrchestrationTaskDto,
GateDecisionDto,
RecordBudgetDto,
TransitionTaskDto,
} from './dto/orchestration.dto';
import {
KillSwitchLevel,
assertTransition,
nextStateForGateDecision,
OperationMode,
WorkflowState,
} from './orchestration.domain';
import { ModelTierPolicyService } from './model-tier-policy.service';
import { KillSwitchService } from './kill-switch.service';
@Injectable()
export class OrchestrationService {
constructor(private readonly prisma:PrismaService,private readonly killSwitch:KillSwitchService){}
async createTask(dto:CreateOrchestrationTaskDto,actorId:string){ await this.killSwitch.assertAllowed("CREATE_TASK"); return (this.prisma as any).$transaction(async(tx:any)=>{const task=await tx.orchestrationTask.create({data:{key:dto.key,title:dto.title,department:dto.department,operationMode:dto.operationMode,budgetLimitCents:dto.budgetLimitCents,metadata:dto.metadata??{},state:WorkflowState.DRAFT,version:1}});await tx.orchestrationEvent.create({data:{taskId:task.id,eventType:"TASK_CREATED",fromState:null,toState:WorkflowState.DRAFT,sequence:1,actorId,payload:{key:dto.key}}});return task;}); }
async transition(id:string,dto:TransitionTaskDto,actorId:string){return (this.prisma as any).$transaction(async(tx:any)=>{const existing=await tx.orchestrationTask.findUnique({where:{id}});if(!existing)throw new NotFoundException(`Orchestration task ${id} not found`);assertTransition(existing.state,dto.toState);if(existing.operationMode===OperationMode.MUTATE_PRODUCTION&&dto.toState===WorkflowState.RELEASING)await this.killSwitch.assertAllowed("MUTATE_PRODUCTION");const changed=await tx.orchestrationTask.updateMany({where:{id,version:dto.expectedVersion,state:existing.state},data:{state:dto.toState,version:{increment:1}}});if(changed.count!==1)throw new ConflictException("Task version or state changed; reload before retrying");await tx.orchestrationEvent.create({data:{taskId:id,eventType:"STATE_TRANSITION",fromState:existing.state,toState:dto.toState,sequence:dto.expectedVersion+1,actorId,idempotencyKey:dto.idempotencyKey,payload:{reason:dto.reason??null}}});return {...existing,state:dto.toState,version:dto.expectedVersion+1};});}
async decideGate(taskId:string,gate:1|2,dto:GateDecisionDto,actorId:string){if(new Date(dto.expiresAt)<=new Date())throw new BadRequestException("Approval decision has expired");return (this.prisma as any).$transaction(async(tx:any)=>{const prior=await tx.orchestrationApprovalDecision.findUnique({where:{idempotencyKey:dto.idempotencyKey}});if(prior){if(prior.taskId!==taskId||prior.gate!==gate||prior.decision!==dto.decision)throw new ConflictException("Idempotency key already belongs to another approval decision");return prior;}const task=await tx.orchestrationTask.findUnique({where:{id:taskId}});if(!task)throw new NotFoundException(`Orchestration task ${taskId} not found`);const expected=gate===1?WorkflowState.GATE1_PENDING:WorkflowState.GATE2_PENDING;if(task.state!==expected)throw new ConflictException(`Gate ${gate} requires ${expected}`);if(gate===2&&dto.decision==="APPROVED"){if(!dto.runId||!dto.artifactHash||!dto.commitSha||!dto.environment)throw new BadRequestException("Gate 2 requires runId, artifactHash, commitSha and environment");const run=await tx.orchestrationRun.findUnique({where:{id:dto.runId}});if(!run||run.taskId!==taskId||run.commitSha!==dto.commitSha||run.environment!==dto.environment)throw new ForbiddenException("Gate 2 binding does not match immutable run");const artifact=await tx.orchestrationArtifact.findFirst({where:{runId:dto.runId,sha256:dto.artifactHash}});if(!artifact)throw new ForbiddenException("Gate 2 artifact hash is not registered for run");}
const next=nextStateForGateDecision(gate,dto.decision);const changed=await tx.orchestrationTask.updateMany({where:{id:taskId,version:dto.expectedVersion,state:expected},data:{state:next,version:{increment:1}}});if(changed.count!==1)throw new ConflictException("Task version or state changed; approval not applied");const decision=await tx.orchestrationApprovalDecision.create({data:{taskId,gate,decision:dto.decision,actorId,notes:dto.notes,expiresAt:new Date(dto.expiresAt),idempotencyKey:dto.idempotencyKey,runId:dto.runId,artifactHash:dto.artifactHash,commitSha:dto.commitSha,environment:dto.environment}});await tx.orchestrationEvent.create({data:{taskId,eventType:`GATE${gate}_${dto.decision}`,fromState:expected,toState:next,sequence:dto.expectedVersion+1,actorId,idempotencyKey:`event:${dto.idempotencyKey}`,payload:{decisionId:decision.id}}});return decision;});}
async recordBudget(taskId:string,dto:RecordBudgetDto,actorId:string){return (this.prisma as any).$transaction(async(tx:any)=>{const task=await tx.orchestrationTask.findUnique({where:{id:taskId}});if(!task)throw new NotFoundException(`Orchestration task ${taskId} not found`);const sum=await tx.orchestrationBudgetLedger.aggregate({where:{taskId},_sum:{amountCents:true}});if((sum._sum.amountCents??0)+dto.amountCents>task.budgetLimitCents)throw new ForbiddenException("Hard task budget exceeded");return tx.orchestrationBudgetLedger.create({data:{taskId,amountCents:dto.amountCents,kind:dto.kind,idempotencyKey:dto.idempotencyKey,actorId,metadata:dto.metadata??{}}});});}
async findAll(){return (this.prisma as any).orchestrationTask.findMany({orderBy:{createdAt:"desc"}})}
async findOne(id:string){const task=await (this.prisma as any).orchestrationTask.findUnique({where:{id},include:{runs:{include:{artifacts:true,attestations:true}},dependencies:true,decisions:true,events:{orderBy:{sequence:"asc"}},budgetLedger:true}});if(!task)throw new NotFoundException(`Orchestration task ${id} not found`);return task;}
async setKillSwitch(level:string,reason:string,actorId:string){return (this.prisma as any).$transaction(async(tx:any)=>{const value=await tx.orchestrationKillSwitch.upsert({where:{scope:"GLOBAL"},create:{scope:"GLOBAL",level,active:true,reason,actorId},update:{level,active:true,reason,actorId,version:{increment:1}}});await tx.orchestrationEvent.create({data:{eventType:"KILL_SWITCH_CHANGED",sequence:value.version??1,actorId,payload:{scope:"GLOBAL",level,reason}}});return value;});}
constructor(
private readonly prisma: PrismaService,
private readonly killSwitch: KillSwitchService,
) {}
async createTask(dto: CreateOrchestrationTaskDto, actorId: string) {
await this.killSwitch.assertAllowed('CREATE_TASK');
return await (this.prisma as any).$transaction(async (tx: any) => {
const task = await tx.orchestrationTask.create({
data: {
key: dto.key,
title: dto.title,
department: dto.department,
operationMode: dto.operationMode,
budgetLimitCents: dto.budgetLimitCents,
metadata: dto.metadata ?? {},
state: WorkflowState.DRAFT,
version: 1,
},
});
await tx.orchestrationEvent.create({
data: {
taskId: task.id,
eventType: 'TASK_CREATED',
fromState: null,
toState: WorkflowState.DRAFT,
sequence: 1,
actorId,
payload: { key: dto.key },
},
});
return task;
});
}
async transition(id: string, dto: TransitionTaskDto, actorId: string) {
if (
[
WorkflowState.GATE1_APPROVED,
WorkflowState.GATE2_APPROVED,
WorkflowState.RELEASING,
WorkflowState.RUNNING,
WorkflowState.VERIFYING,
WorkflowState.PRODUCTION_VERIFYING,
WorkflowState.LIVE_VERIFIED,
WorkflowState.MEASURING,
WorkflowState.CLOSED,
].includes(dto.toState)
)
throw new ForbiddenException(
'Gate approvals and release require dedicated commands',
);
return await (this.prisma as any).$transaction(async (tx: any) => {
await tx.$queryRaw`SELECT id FROM orchestration_tasks WHERE id = ${id}::uuid FOR UPDATE`;
const fingerprint = JSON.stringify({ id, dto, actorId });
const replay = await tx.orchestrationEvent.findUnique({
where: { idempotencyKey: dto.idempotencyKey },
});
if (replay) {
if (replay.payload.fingerprint !== fingerprint)
throw new ConflictException('Idempotency payload mismatch');
return replay.payload.result;
}
const existing = await tx.orchestrationTask.findUnique({ where: { id } });
if (!existing)
throw new NotFoundException(`Orchestration task ${id} not found`);
if (existing.state === WorkflowState.FAILED)
throw new ForbiddenException('Retries require dedicated run command');
assertTransition(existing.state, dto.toState);
await this.killSwitch.assertAllowed(
[
WorkflowState.READY,
WorkflowState.CLAIMED,
WorkflowState.RUNNING,
WorkflowState.VERIFYING,
WorkflowState.GATE2_PENDING,
].includes(dto.toState)
? 'RUN_AGENT'
: 'OBSERVE',
);
if ([WorkflowState.READY, WorkflowState.CLAIMED].includes(dto.toState))
await this.killSwitch.assertAllowed('CLAIM_TASK');
let selectedTier = existing.modelTier;
if (dto.toState === WorkflowState.READY) {
const dependencies = await tx.orchestrationDependency.findMany({
where: { taskId: id },
include: { dependsOnTask: true },
});
if (dependencies.some((d: any) => d.dependsOnTask.state !== 'CLOSED'))
throw new ForbiddenException('Unresolved dependency');
const costs = await tx.orchestrationBudgetLedger.aggregate({
where: { taskId: id },
_sum: { amountCents: true },
});
selectedTier = new ModelTierPolicyService().select({
complexity: Number(existing.metadata?.complexity ?? 0),
risk: Number(existing.metadata?.risk ?? 0),
operationMode: existing.operationMode,
remainingBudgetCents:
existing.budgetLimitCents - (costs._sum.amountCents ?? 0),
});
const approval = await tx.orchestrationApprovalDecision.findFirst({
where: { taskId: id, gate: 1, decision: 'APPROVED' },
orderBy: { createdAt: 'desc' },
});
if (!approval || approval.expiresAt <= new Date())
throw new ForbiddenException('Gate 1 approval missing or expired');
}
if (
existing.operationMode === OperationMode.MUTATE_PRODUCTION &&
dto.toState === WorkflowState.RELEASING
)
await this.killSwitch.assertAllowed('MUTATE_PRODUCTION');
const changed = await tx.orchestrationTask.updateMany({
where: { id, version: dto.expectedVersion, state: existing.state },
data: {
state: dto.toState,
modelTier: selectedTier,
version: { increment: 1 },
},
});
if (changed.count !== 1)
throw new ConflictException(
'Task version or state changed; reload before retrying',
);
await tx.orchestrationEvent.create({
data: {
taskId: id,
eventType: 'STATE_TRANSITION',
fromState: existing.state,
toState: dto.toState,
sequence: dto.expectedVersion + 1,
actorId,
idempotencyKey: dto.idempotencyKey,
payload: {
fingerprint,
result: JSON.parse(
JSON.stringify({
...existing,
state: dto.toState,
modelTier: selectedTier,
version: dto.expectedVersion + 1,
}),
),
},
},
});
return JSON.parse(
JSON.stringify({
...existing,
state: dto.toState,
modelTier: selectedTier,
version: dto.expectedVersion + 1,
}),
);
});
}
async decideGate(
taskId: string,
gate: 1 | 2,
dto: GateDecisionDto,
actorId: string,
) {
if (![1, 2].includes(gate)) throw new BadRequestException('Invalid gate');
return await (this.prisma as any).$transaction(async (tx: any) => {
await tx.$queryRaw`SELECT id FROM orchestration_tasks WHERE id = ${taskId}::uuid FOR UPDATE`;
const fingerprint = JSON.stringify({ taskId, gate, dto, actorId });
const prior = await tx.orchestrationApprovalDecision.findUnique({
where: { idempotencyKey: dto.idempotencyKey },
});
if (prior) {
const event = await tx.orchestrationEvent.findUnique({
where: { idempotencyKey: `event:${dto.idempotencyKey}` },
});
if (event?.payload?.fingerprint !== fingerprint)
throw new ConflictException('Idempotency payload mismatch');
return prior;
}
if (
!Number.isFinite(Date.parse(dto.expiresAt)) ||
new Date(dto.expiresAt) <= new Date()
)
throw new BadRequestException('Approval decision has expired');
await this.killSwitch.assertAllowed('OBSERVE');
const task = await tx.orchestrationTask.findUnique({
where: { id: taskId },
});
if (!task)
throw new NotFoundException(`Orchestration task ${taskId} not found`);
const expected =
gate === 1 ? WorkflowState.GATE1_PENDING : WorkflowState.GATE2_PENDING;
if (task.state !== expected)
throw new ConflictException(`Gate ${gate} requires ${expected}`);
if (gate === 2 && dto.decision === 'APPROVED') {
if (
!dto.runId ||
!dto.artifactHash ||
!dto.commitSha ||
!dto.environment
)
throw new BadRequestException(
'Gate 2 requires runId, artifactHash, commitSha and environment',
);
const run = await tx.orchestrationRun.findUnique({
where: { id: dto.runId },
});
if (
!run ||
run.taskId !== taskId ||
run.commitSha !== dto.commitSha ||
run.environment !== dto.environment
)
throw new ForbiddenException(
'Gate 2 binding does not match immutable run',
);
if (run.status !== 'SUCCEEDED')
throw new ForbiddenException('Gate 2 requires a successful run');
const artifact = await tx.orchestrationArtifact.findFirst({
where: { runId: dto.runId, sha256: dto.artifactHash },
});
if (!artifact)
throw new ForbiddenException(
'Gate 2 artifact hash is not registered for run',
);
}
const next = nextStateForGateDecision(gate, dto.decision);
const changed = await tx.orchestrationTask.updateMany({
where: { id: taskId, version: dto.expectedVersion, state: expected },
data: { state: next, version: { increment: 1 } },
});
if (changed.count !== 1)
throw new ConflictException(
'Task version or state changed; approval not applied',
);
const decision = await tx.orchestrationApprovalDecision.create({
data: {
taskId,
gate,
decision: dto.decision,
actorId,
notes: dto.notes,
expiresAt: new Date(dto.expiresAt),
idempotencyKey: dto.idempotencyKey,
runId: dto.runId,
artifactHash: dto.artifactHash,
commitSha: dto.commitSha,
environment: dto.environment,
},
});
await tx.orchestrationEvent.create({
data: {
taskId,
eventType: `GATE${gate}_${dto.decision}`,
fromState: expected,
toState: next,
sequence: dto.expectedVersion + 1,
actorId,
idempotencyKey: `event:${dto.idempotencyKey}`,
payload: { decisionId: decision.id, fingerprint },
},
});
return decision;
});
}
async recordBudget(taskId: string, dto: RecordBudgetDto, actorId: string) {
if (
!Number.isSafeInteger(dto.amountCents) ||
dto.amountCents < 1 ||
!['RESERVE', 'SPEND'].includes(dto.kind)
)
throw new BadRequestException(
'Use RESERVE or SPEND with positive integer cents',
);
return await (this.prisma as any).$transaction(async (tx: any) => {
await tx.$queryRaw`SELECT id FROM orchestration_tasks WHERE id = ${taskId}::uuid FOR UPDATE`;
const prior = await tx.orchestrationBudgetLedger.findUnique({
where: { idempotencyKey: dto.idempotencyKey },
});
if (prior) {
if (
prior.taskId !== taskId ||
prior.kind !== dto.kind ||
prior.amountCents !== dto.amountCents ||
prior.actorId !== actorId ||
JSON.stringify(prior.metadata) !== JSON.stringify(dto.metadata ?? {})
)
throw new ConflictException('Idempotency payload mismatch');
return prior;
}
await this.killSwitch.assertAllowed('RUN_AGENT');
const task = await tx.orchestrationTask.findUnique({
where: { id: taskId },
});
if (!task)
throw new NotFoundException(`Orchestration task ${taskId} not found`);
const sum = await tx.orchestrationBudgetLedger.aggregate({
where: { taskId },
_sum: { amountCents: true },
});
if ((sum._sum.amountCents ?? 0) + dto.amountCents > task.budgetLimitCents)
throw new ForbiddenException('Hard task budget exceeded');
return tx.orchestrationBudgetLedger.create({
data: {
taskId,
amountCents: dto.amountCents,
kind: dto.kind,
idempotencyKey: dto.idempotencyKey,
actorId,
metadata: dto.metadata ?? {},
},
});
});
}
async command(taskId: string, command: string, dto: any, actorId: string) {
return await (this.prisma as any).$transaction(async (tx: any) => {
await tx.$queryRaw`SELECT id FROM orchestration_tasks WHERE id = ${taskId}::uuid FOR UPDATE`;
const fingerprint = JSON.stringify({ taskId, command, dto, actorId });
const replay = await tx.orchestrationEvent.findUnique({
where: { idempotencyKey: dto.idempotencyKey },
});
if (replay) {
if (replay.payload.fingerprint !== fingerprint)
throw new ConflictException('Idempotency payload mismatch');
return replay.payload.result;
}
const task = await tx.orchestrationTask.findUnique({
where: { id: taskId },
});
if (!task) throw new NotFoundException('Task not found');
if (task.version !== dto.expectedVersion)
throw new ConflictException('Stale task version');
await this.killSwitch.assertAllowed('RUN_AGENT');
let result: any;
let state = task.state;
if (command === 'RUN_CREATE') {
if (
!/^[a-f0-9]{40,64}$/.test(dto.commitSha) ||
!['development', 'staging', 'production'].includes(dto.environment)
)
throw new BadRequestException('Invalid immutable run binding');
const last = await tx.orchestrationRun.findFirst({
where: { taskId },
orderBy: { attempt: 'desc' },
});
if (
last &&
(last.status !== 'FAILED' ||
!['TRANSIENT', 'RATE_LIMITED', 'DEPENDENCY'].includes(
last.retryClass,
) ||
last.attempt >= 3)
)
throw new ForbiddenException('Retry denied or exhausted');
if (!last && task.state !== 'CLAIMED')
throw new ConflictException('Run requires CLAIMED task');
result = await tx.orchestrationRun.create({
data: {
taskId,
attempt: (last?.attempt ?? 0) + 1,
status: 'RUNNING',
commitSha: dto.commitSha,
environment: dto.environment,
idempotencyKey: dto.idempotencyKey,
startedAt: new Date(),
},
});
state = 'RUNNING';
} else if (command === 'RUN_FINISH') {
const run = await tx.orchestrationRun.findUnique({
where: { id: dto.runId },
});
if (
!run ||
run.taskId !== taskId ||
run.status !== 'RUNNING' ||
task.state !== 'RUNNING'
)
throw new ConflictException('Active run required');
if (
!['SUCCEEDED', 'FAILED'].includes(dto.status) ||
(dto.status === 'FAILED' &&
![
'TRANSIENT',
'RATE_LIMITED',
'DEPENDENCY',
'VALIDATION',
'POLICY',
'PERMANENT',
].includes(dto.retryClass))
)
throw new BadRequestException('Invalid outcome taxonomy');
result = await tx.orchestrationRun.update({
where: { id: run.id },
data: {
status: dto.status,
retryClass: dto.status === 'FAILED' ? dto.retryClass : null,
finishedAt: new Date(),
},
});
state = dto.status === 'FAILED' ? 'FAILED' : 'VERIFYING';
} else if (command === 'ARTIFACT_ADD' || command === 'ATTESTATION_ADD') {
const run = await tx.orchestrationRun.findUnique({
where: { id: dto.runId },
});
if (
!run ||
run.taskId !== taskId ||
!['RUNNING', 'VERIFYING'].includes(task.state)
)
throw new ConflictException('Owned active run required');
const hash = command === 'ARTIFACT_ADD' ? dto.sha256 : dto.subjectHash;
if (!/^[a-f0-9]{64}$/.test(hash) || !dto.kind)
throw new BadRequestException('Invalid artifact hash or kind');
if (command === 'ARTIFACT_ADD') {
if (!/^https:\/\//.test(dto.uri))
throw new BadRequestException('Artifact URI must be HTTPS');
result = await tx.orchestrationArtifact.create({
data: { runId: run.id, kind: dto.kind, sha256: hash, uri: dto.uri },
});
} else {
if (
!(await tx.orchestrationArtifact.findFirst({
where: { runId: run.id, sha256: hash },
}))
)
throw new ForbiddenException(
'Attestation subject is not a registered artifact',
);
result = await tx.orchestrationAttestation.create({
data: {
runId: run.id,
kind: dto.kind,
subjectHash: hash,
predicate: dto.predicate ?? {},
signer: actorId,
signature: dto.signature,
},
});
}
} else if (command === 'DEPENDENCY_ADD') {
if (task.state !== 'DRAFT')
throw new ConflictException(
'Dependencies are frozen after submission',
);
await tx.$executeRaw`LOCK TABLE orchestration_dependencies IN EXCLUSIVE MODE`;
const cycles =
await tx.$queryRaw`WITH RECURSIVE chain(id) AS (SELECT ${dto.dependsOnTaskId}::uuid UNION SELECT d.depends_on_task_id FROM orchestration_dependencies d JOIN chain c ON d.task_id=c.id) SELECT id FROM chain WHERE id=${taskId}::uuid`;
if (cycles.length) throw new ForbiddenException('Dependency cycle');
result = await tx.orchestrationDependency.create({
data: { taskId, dependsOnTaskId: dto.dependsOnTaskId },
});
} else if (command === 'RELEASE_AUTHORIZE') {
await this.killSwitch.assertAllowed('MUTATE_PRODUCTION');
if (task.state !== 'GATE2_APPROVED')
throw new ConflictException(
'Gate 2 approval required; already consumed or invalid state',
);
const approval = await tx.orchestrationApprovalDecision.findUnique({
where: { id: dto.approvalId },
});
if (
!approval ||
approval.taskId !== taskId ||
approval.gate !== 2 ||
approval.decision !== 'APPROVED' ||
approval.expiresAt <= new Date()
)
throw new ForbiddenException('Gate 2 missing or expired');
for (const key of ['runId', 'artifactHash', 'commitSha', 'environment'])
if (approval[key] !== dto[key])
throw new ForbiddenException('Release binding mismatch');
const run = await tx.orchestrationRun.findUnique({
where: { id: dto.runId },
});
if (
!run ||
run.taskId !== taskId ||
run.status !== 'SUCCEEDED' ||
run.commitSha !== dto.commitSha ||
run.environment !== dto.environment
)
throw new ForbiddenException('Run binding mismatch');
if (
!(await tx.orchestrationArtifact.findFirst({
where: { runId: run.id, sha256: dto.artifactHash },
})) ||
!(await tx.orchestrationAttestation.findFirst({
where: {
runId: run.id,
subjectHash: dto.artifactHash,
kind: 'TEST',
},
}))
)
throw new ForbiddenException('Artifact evidence missing');
result = {
authorized: true,
approvalId: approval.id,
runId: run.id,
artifactHash: dto.artifactHash,
commitSha: dto.commitSha,
environment: dto.environment,
execution: 'CI_CD_ONLY',
};
state = 'RELEASING';
} else throw new BadRequestException('Unknown command');
await tx.orchestrationTask.update({
where: { id: taskId },
data: { state, version: { increment: 1 } },
});
const serialized = JSON.parse(JSON.stringify(result));
await tx.orchestrationEvent.create({
data: {
taskId,
eventType: command,
fromState: task.state,
toState: state,
sequence: task.version + 1,
actorId,
idempotencyKey: dto.idempotencyKey,
payload: { fingerprint, result: serialized },
},
});
return serialized;
});
}
async reconcileBudget(
taskId: string,
dto: { reservationId: string; actualCents: number; idempotencyKey: string },
actorId: string,
) {
await this.killSwitch.assertAllowed('OBSERVE');
if (!Number.isSafeInteger(dto.actualCents) || dto.actualCents < 0)
throw new BadRequestException('Invalid actual cost');
return await (this.prisma as any).$transaction(async (tx: any) => {
await tx.$queryRaw`SELECT id FROM orchestration_tasks WHERE id = ${taskId}::uuid FOR UPDATE`;
const prior = await tx.orchestrationBudgetLedger.findUnique({
where: { idempotencyKey: dto.idempotencyKey },
});
if (prior) {
if (
prior.taskId !== taskId ||
prior.actorId !== actorId ||
prior.metadata.reservationId !== dto.reservationId ||
prior.metadata.actualCents !== dto.actualCents
)
throw new ConflictException('Idempotency payload mismatch');
return prior;
}
const reservation = await tx.orchestrationBudgetLedger.findUnique({
where: { id: dto.reservationId },
});
if (
!reservation ||
reservation.taskId !== taskId ||
reservation.kind !== 'RESERVE'
)
throw new BadRequestException('Reservation not found');
if (
await tx.orchestrationBudgetLedger.findFirst({
where: {
taskId,
kind: 'RECONCILE',
metadata: { path: ['reservationId'], equals: dto.reservationId },
},
})
)
throw new ConflictException('Reservation already reconciled');
const task = await tx.orchestrationTask.findUnique({
where: { id: taskId },
});
const total = await tx.orchestrationBudgetLedger.aggregate({
where: { taskId },
_sum: { amountCents: true },
});
const delta = dto.actualCents - reservation.amountCents;
if (total._sum.amountCents + delta > task.budgetLimitCents)
throw new ForbiddenException('Hard task budget exceeded');
return tx.orchestrationBudgetLedger.create({
data: {
taskId,
kind: 'RECONCILE',
amountCents: delta,
idempotencyKey: dto.idempotencyKey,
actorId,
metadata: {
reservationId: dto.reservationId,
actualCents: dto.actualCents,
},
},
});
});
}
async findAll() {
await this.killSwitch.assertAllowed('OBSERVE');
return (this.prisma as any).orchestrationTask.findMany({
orderBy: { createdAt: 'desc' },
});
}
async findOne(id: string) {
await this.killSwitch.assertAllowed('OBSERVE');
const task = await (this.prisma as any).orchestrationTask.findUnique({
where: { id },
include: {
runs: { include: { artifacts: true, attestations: true } },
dependencies: true,
decisions: true,
events: { orderBy: { sequence: 'asc' } },
budgetLedger: true,
},
});
if (!task)
throw new NotFoundException(`Orchestration task ${id} not found`);
return task;
}
async setKillSwitch(level: string, reason: string, actorId: string) {
if (!Object.values(KillSwitchLevel).includes(level as KillSwitchLevel))
throw new BadRequestException('Invalid kill switch level');
return await (this.prisma as any).$transaction(async (tx: any) => {
const value = await tx.orchestrationKillSwitch.upsert({
where: { scope: 'GLOBAL' },
create: { scope: 'GLOBAL', level, active: true, reason, actorId },
update: {
level,
active: true,
reason,
actorId,
version: { increment: 1 },
},
});
await tx.orchestrationEvent.create({
data: {
eventType: 'KILL_SWITCH_CHANGED',
sequence: value.version ?? 1,
actorId,
payload: { scope: 'GLOBAL', level, reason },
},
});
return value;
});
}
}