Checkpoint recovered orchestration core with tested fail-closed boundaries
This commit is contained in:
1 parent
c7f1d25e98
commit
0952a482fb
18 files changed
+1918
-129
No files matched your search
@@ -37,7 +37,7 @@ describe('retired legacy orchestration HTTP boundary', () => {
|
||||
dbAccess.mockClear();
|
||||
});
|
||||
beforeEach(() => dbAccess.mockClear());
|
||||
afterAll(async () => app?.close());
|
||||
afterAll(async () => { if (app) await app.close(); });
|
||||
it.each(routes)('%s %s returns explicit 410 for authorized callers before database access', async (method, path) => {
|
||||
const response = await request(app.getHttpServer())[method](path)
|
||||
.set('Authorization', 'Admin').send({decision:'APPROVED',status:'DONE'}).expect(410);
|
||||
|
||||
@@ -30,6 +30,12 @@ describe('installed Hermes health contract', () => {
|
||||
);
|
||||
expect(fetcher.mock.calls[0][1].redirect).toBe('error');
|
||||
});
|
||||
it('refuses a runtime reporting that authentication is disabled', async () => {
|
||||
const adapter = new HermesAdapter({origin:'http://canina-hermes:9119',token:async()=> 'private'},
|
||||
jest.fn().mockResolvedValue(new Response(JSON.stringify({...health,auth_required:false}))));
|
||||
await expect(adapter.health()).rejects.toMatchObject({code:'AUTH_REQUIRED'});
|
||||
});
|
||||
|
||||
it('fails closed for bad origin, auth errors, malformed contract and oversized bodies', async () => {
|
||||
expect(
|
||||
() =>
|
||||
|
||||
@@ -132,12 +132,14 @@ export class HermesAdapter {
|
||||
}
|
||||
}
|
||||
async health(): Promise<Health> {
|
||||
return fields(await this.request('/api/health'), {
|
||||
const health = fields(await this.request('/api/health'), {
|
||||
ok: 'boolean',
|
||||
version: 'string',
|
||||
commit: 'string',
|
||||
auth_required: 'boolean',
|
||||
}) as unknown as Health;
|
||||
if (!health.auth_required) throw new HermesError('AUTH_REQUIRED');
|
||||
return health;
|
||||
}
|
||||
async status(profile: string): Promise<Row> {
|
||||
const raw = object(await this.request('/api/status', profile));
|
||||
|
||||
@@ -1,8 +1,90 @@
|
||||
import { Type } from "class-transformer";
|
||||
import { IsEnum, IsHash, IsIn, IsInt, IsISO8601, IsNotEmpty, IsOptional, IsString, Max, Min } from "class-validator";
|
||||
import { OperationMode, WorkflowState } from "../orchestration.domain";
|
||||
export class CreateOrchestrationTaskDto { @IsString() @IsNotEmpty() key!:string; @IsString() @IsNotEmpty() title!:string; @IsString() @IsNotEmpty() department!:string; @IsEnum(OperationMode) operationMode!:OperationMode; @Type(()=>Number) @IsInt() @Min(0) budgetLimitCents!:number; @IsOptional() metadata?:Record<string,unknown>; }
|
||||
export class TransitionTaskDto { @IsEnum(WorkflowState) toState!:WorkflowState; @Type(()=>Number) @IsInt() @Min(1) expectedVersion!:number; @IsString() @IsNotEmpty() idempotencyKey!:string; @IsOptional() @IsString() reason?:string; }
|
||||
export class GateDecisionDto { @IsIn(["APPROVED","REJECTED"]) decision!:"APPROVED"|"REJECTED"; @Type(()=>Number) @IsInt() @Min(1) expectedVersion!:number; @IsString() @IsNotEmpty() idempotencyKey!:string; @IsISO8601() expiresAt!:string; @IsOptional() @IsString() notes?:string; @IsOptional() @IsString() runId?:string; @IsOptional() @IsHash("sha256") artifactHash?:string; @IsOptional() @IsString() commitSha?:string; @IsOptional() @IsString() environment?:string; }
|
||||
export class RecordBudgetDto { @Type(()=>Number) @IsInt() @Min(1) amountCents!:number; @IsString() @IsNotEmpty() idempotencyKey!:string; @IsString() @IsNotEmpty() kind!:string; @IsOptional() metadata?:Record<string,unknown>; }
|
||||
export class SetKillSwitchDto { @IsIn(["FREEZE_NEW_WORK","STOP_AGENTS","DISABLE_MUTATIONS","EMERGENCY_HALT"]) level!:string; @IsString() @IsNotEmpty() reason!:string; }
|
||||
import { Type } from 'class-transformer';
|
||||
import {
|
||||
IsEnum,
|
||||
IsHash,
|
||||
IsIn,
|
||||
IsInt,
|
||||
IsISO8601,
|
||||
IsNotEmpty,
|
||||
IsOptional,
|
||||
IsString,
|
||||
Max,
|
||||
Min,
|
||||
} from 'class-validator';
|
||||
import { OperationMode, WorkflowState } from '../orchestration.domain';
|
||||
export class CreateOrchestrationTaskDto {
|
||||
@IsString() @IsNotEmpty() key!: string;
|
||||
@IsString() @IsNotEmpty() title!: string;
|
||||
@IsString() @IsNotEmpty() department!: string;
|
||||
@IsEnum(OperationMode) operationMode!: OperationMode;
|
||||
@Type(() => Number) @IsInt() @Min(0) budgetLimitCents!: number;
|
||||
@IsOptional() metadata?: Record<string, unknown>;
|
||||
}
|
||||
export class TransitionTaskDto {
|
||||
@IsEnum(WorkflowState) toState!: WorkflowState;
|
||||
@Type(() => Number) @IsInt() @Min(1) expectedVersion!: number;
|
||||
@IsString() @IsNotEmpty() idempotencyKey!: string;
|
||||
@IsOptional() @IsString() reason?: string;
|
||||
}
|
||||
export class GateDecisionDto {
|
||||
@IsIn(['APPROVED', 'REJECTED']) decision!: 'APPROVED' | 'REJECTED';
|
||||
@Type(() => Number) @IsInt() @Min(1) expectedVersion!: number;
|
||||
@IsString() @IsNotEmpty() idempotencyKey!: string;
|
||||
@IsISO8601() expiresAt!: string;
|
||||
@IsOptional() @IsString() notes?: string;
|
||||
@IsOptional() @IsString() runId?: string;
|
||||
@IsOptional() @IsHash('sha256') artifactHash?: string;
|
||||
@IsOptional() @IsString() commitSha?: string;
|
||||
@IsOptional() @IsString() environment?: string;
|
||||
}
|
||||
export class RecordBudgetDto {
|
||||
@Type(() => Number) @IsInt() @Min(1) amountCents!: number;
|
||||
@IsString() @IsNotEmpty() idempotencyKey!: string;
|
||||
@IsString() @IsNotEmpty() kind!: string;
|
||||
@IsOptional() metadata?: Record<string, unknown>;
|
||||
}
|
||||
export class SetKillSwitchDto {
|
||||
@IsIn([
|
||||
'FREEZE_NEW_WORK',
|
||||
'STOP_AGENTS',
|
||||
'DISABLE_MUTATIONS',
|
||||
'EMERGENCY_HALT',
|
||||
])
|
||||
level!: string;
|
||||
@IsString() @IsNotEmpty() reason!: string;
|
||||
}
|
||||
|
||||
export class ReconcileBudgetDto {
|
||||
@IsString() @IsNotEmpty() reservationId!: string;
|
||||
@Type(() => Number) @IsInt() @Min(0) actualCents!: number;
|
||||
@IsString() @IsNotEmpty() idempotencyKey!: string;
|
||||
}
|
||||
export class OrchestrationCommandDto {
|
||||
@Type(() => Number) @IsInt() @Min(1) expectedVersion!: number;
|
||||
@IsString() @IsNotEmpty() idempotencyKey!: string;
|
||||
@IsOptional() @IsString() commitSha?: string;
|
||||
@IsOptional()
|
||||
@IsIn(['development', 'staging', 'production'])
|
||||
environment?: string;
|
||||
@IsOptional() @IsString() runId?: string;
|
||||
@IsOptional() @IsString() approvalId?: string;
|
||||
@IsOptional() @IsString() dependsOnTaskId?: string;
|
||||
@IsOptional() @IsIn(['SUCCEEDED', 'FAILED']) status?: string;
|
||||
@IsOptional()
|
||||
@IsIn([
|
||||
'TRANSIENT',
|
||||
'RATE_LIMITED',
|
||||
'DEPENDENCY',
|
||||
'VALIDATION',
|
||||
'POLICY',
|
||||
'PERMANENT',
|
||||
])
|
||||
retryClass?: string;
|
||||
@IsOptional() @IsHash('sha256') sha256?: string;
|
||||
@IsOptional() @IsHash('sha256') artifactHash?: string;
|
||||
@IsOptional() @IsHash('sha256') subjectHash?: string;
|
||||
@IsOptional() @IsString() kind?: string;
|
||||
@IsOptional() @IsString() uri?: string;
|
||||
@IsOptional() predicate?: Record<string, unknown>;
|
||||
@IsOptional() @IsString() signature?: string;
|
||||
}
|
||||
@@ -1,24 +1,45 @@
|
||||
import { ForbiddenException } from "@nestjs/common";
|
||||
import { KillSwitchService } from "./kill-switch.service";
|
||||
import { ForbiddenException } from '@nestjs/common';
|
||||
import { KillSwitchService } from './kill-switch.service';
|
||||
|
||||
describe("KillSwitchService", () => {
|
||||
describe('KillSwitchService', () => {
|
||||
const prisma = { orchestrationKillSwitch: { findUnique: jest.fn() } };
|
||||
const service = new KillSwitchService(prisma as never);
|
||||
|
||||
beforeEach(() => jest.resetAllMocks());
|
||||
|
||||
it.each([
|
||||
["FREEZE_NEW_WORK", "CREATE_TASK"],
|
||||
["STOP_AGENTS", "RUN_AGENT"],
|
||||
["DISABLE_MUTATIONS", "MUTATE_PRODUCTION"],
|
||||
["EMERGENCY_HALT", "OBSERVE"],
|
||||
])("blocks %s at its policy boundary", async (level, action) => {
|
||||
prisma.orchestrationKillSwitch.findUnique.mockResolvedValue({ scope: "GLOBAL", level, active: true });
|
||||
await expect(service.assertAllowed(action as never)).rejects.toThrow(ForbiddenException);
|
||||
['FREEZE_NEW_WORK', 'CREATE_TASK'],
|
||||
['STOP_AGENTS', 'RUN_AGENT'],
|
||||
['DISABLE_MUTATIONS', 'MUTATE_PRODUCTION'],
|
||||
['EMERGENCY_HALT', 'OBSERVE'],
|
||||
])('blocks %s at its policy boundary', async (level, action) => {
|
||||
prisma.orchestrationKillSwitch.findUnique.mockResolvedValue({
|
||||
scope: 'GLOBAL',
|
||||
level,
|
||||
active: true,
|
||||
});
|
||||
await expect(service.assertAllowed(action as never)).rejects.toThrow(
|
||||
ForbiddenException,
|
||||
);
|
||||
});
|
||||
|
||||
it("allows observation while production mutations are disabled", async () => {
|
||||
prisma.orchestrationKillSwitch.findUnique.mockResolvedValue({ scope: "GLOBAL", level: "DISABLE_MUTATIONS", active: true });
|
||||
await expect(service.assertAllowed("OBSERVE")).resolves.toBeUndefined();
|
||||
it('freezes new claims while allowing an already running agent to finish', async () => {
|
||||
prisma.orchestrationKillSwitch.findUnique.mockResolvedValue({scope:'GLOBAL',level:'FREEZE_NEW_WORK',active:true});
|
||||
await expect(service.assertAllowed('CLAIM_TASK' as never)).rejects.toThrow(ForbiddenException);
|
||||
await expect(service.assertAllowed('RUN_AGENT')).resolves.toBeUndefined();
|
||||
});
|
||||
|
||||
it('fails closed when an active persisted kill level is unknown', async () => {
|
||||
prisma.orchestrationKillSwitch.findUnique.mockResolvedValue({scope:'GLOBAL',level:'CORRUPT_LEVEL',active:true});
|
||||
await expect(service.assertAllowed('RUN_AGENT')).rejects.toThrow(ForbiddenException);
|
||||
});
|
||||
|
||||
it('allows observation while production mutations are disabled', async () => {
|
||||
prisma.orchestrationKillSwitch.findUnique.mockResolvedValue({
|
||||
scope: 'GLOBAL',
|
||||
level: 'DISABLE_MUTATIONS',
|
||||
active: true,
|
||||
});
|
||||
await expect(service.assertAllowed('OBSERVE')).resolves.toBeUndefined();
|
||||
});
|
||||
});
|
||||
@@ -1,9 +1,36 @@
|
||||
import { ForbiddenException, Injectable } from "@nestjs/common";
|
||||
import { PrismaService } from "../prisma/prisma.service";
|
||||
import { KillSwitchLevel } from "./orchestration.domain";
|
||||
export type GuardedAction="CREATE_TASK"|"RUN_AGENT"|"MUTATE_PRODUCTION"|"OBSERVE";
|
||||
import { ForbiddenException, Injectable } from '@nestjs/common';
|
||||
import { PrismaService } from '../prisma/prisma.service';
|
||||
import { KillSwitchLevel } from './orchestration.domain';
|
||||
export type GuardedAction =
|
||||
| 'CREATE_TASK'
|
||||
| 'CLAIM_TASK'
|
||||
| 'RUN_AGENT'
|
||||
| 'MUTATE_PRODUCTION'
|
||||
| 'OBSERVE';
|
||||
@Injectable()
|
||||
export class KillSwitchService {
|
||||
constructor(private readonly prisma:PrismaService){}
|
||||
async assertAllowed(action:GuardedAction):Promise<void>{ const state=await (this.prisma as any).orchestrationKillSwitch.findUnique({where:{scope:"GLOBAL"}}); if(!state?.active)return; const blocked:Record<KillSwitchLevel,GuardedAction[]>={FREEZE_NEW_WORK:["CREATE_TASK"],STOP_AGENTS:["CREATE_TASK","RUN_AGENT"],DISABLE_MUTATIONS:["CREATE_TASK","RUN_AGENT","MUTATE_PRODUCTION"],EMERGENCY_HALT:["CREATE_TASK","RUN_AGENT","MUTATE_PRODUCTION","OBSERVE"]}; if(blocked[state.level as KillSwitchLevel]?.includes(action)) throw new ForbiddenException(`Orchestration kill switch ${state.level} blocks ${action}`); }
|
||||
constructor(private readonly prisma: PrismaService) {}
|
||||
async assertAllowed(action: GuardedAction): Promise<void> {
|
||||
const state = await (this.prisma as any).orchestrationKillSwitch.findUnique(
|
||||
{ where: { scope: 'GLOBAL' } },
|
||||
);
|
||||
if (!state?.active) return;
|
||||
const blocked: Record<KillSwitchLevel, GuardedAction[]> = {
|
||||
FREEZE_NEW_WORK: ['CREATE_TASK', 'CLAIM_TASK'],
|
||||
STOP_AGENTS: ['CREATE_TASK', 'CLAIM_TASK', 'RUN_AGENT'],
|
||||
DISABLE_MUTATIONS: ['CREATE_TASK', 'CLAIM_TASK', 'RUN_AGENT', 'MUTATE_PRODUCTION'],
|
||||
EMERGENCY_HALT: [
|
||||
'CREATE_TASK',
|
||||
'CLAIM_TASK',
|
||||
'RUN_AGENT',
|
||||
'MUTATE_PRODUCTION',
|
||||
'OBSERVE',
|
||||
],
|
||||
};
|
||||
if (!Object.values(KillSwitchLevel).includes(state.level) ||
|
||||
blocked[state.level as KillSwitchLevel].includes(action))
|
||||
throw new ForbiddenException(
|
||||
`Orchestration kill switch ${state.level} blocks ${action}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -1,3 +1,18 @@
|
||||
import { readFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
describe("orchestration migration safety",()=>{it("is additive and never mutates legacy governance tables",()=>{const sql=readFileSync(join(process.cwd(),"prisma/migrations/20261022000000_add_orchestration_v4_core/migration.sql"),"utf8");expect(sql).not.toMatch(/^\s*(DROP|TRUNCATE|DELETE|UPDATE)\b/im);expect(sql).not.toMatch(/ALTER TABLE "(approval_requests|department_tasks|audit_logs)"/i);expect(sql.match(/CREATE TABLE "orchestration_/g)).toHaveLength(9);});});
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { join } from 'node:path';
|
||||
describe('orchestration migration safety', () => {
|
||||
it('is additive and never mutates legacy governance tables', () => {
|
||||
const sql = readFileSync(
|
||||
join(
|
||||
process.cwd(),
|
||||
'prisma/migrations/20261022000000_add_orchestration_v4_core/migration.sql',
|
||||
),
|
||||
'utf8',
|
||||
);
|
||||
expect(sql).not.toMatch(/^\s*(DROP|TRUNCATE|DELETE|UPDATE)\b/im);
|
||||
expect(sql).not.toMatch(
|
||||
/ALTER TABLE "(approval_requests|department_tasks|audit_logs)"/i,
|
||||
);
|
||||
expect(sql.match(/CREATE TABLE "orchestration_/g)).toHaveLength(9);
|
||||
});
|
||||
});
|
||||
@@ -1,21 +1,68 @@
|
||||
import { BadRequestException } from "@nestjs/common";
|
||||
import { ModelTierPolicyService } from "./model-tier-policy.service";
|
||||
import { BadRequestException } from '@nestjs/common';
|
||||
import { ModelTierPolicyService } from './model-tier-policy.service';
|
||||
|
||||
describe("ModelTierPolicyService", () => {
|
||||
describe('ModelTierPolicyService', () => {
|
||||
const policy = new ModelTierPolicyService();
|
||||
|
||||
it.each([
|
||||
[{ complexity: 0, risk: 0, operationMode: "OBSERVE", remainingBudgetCents: 1 }, "L0"],
|
||||
[{ complexity: 2, risk: 1, operationMode: "PREPARE", remainingBudgetCents: 100 }, "L1"],
|
||||
[{ complexity: 5, risk: 4, operationMode: "PREPARE", remainingBudgetCents: 500 }, "L2"],
|
||||
[{ complexity: 7, risk: 7, operationMode: "PREPARE", remainingBudgetCents: 1500 }, "L3"],
|
||||
[{ complexity: 9, risk: 9, operationMode: "MUTATE_PRODUCTION", remainingBudgetCents: 5000 }, "L4"],
|
||||
])("selects a deterministic tier for policy inputs", (input, expected) => {
|
||||
[
|
||||
{
|
||||
complexity: 0,
|
||||
risk: 0,
|
||||
operationMode: 'OBSERVE',
|
||||
remainingBudgetCents: 1,
|
||||
},
|
||||
'L0',
|
||||
],
|
||||
[
|
||||
{
|
||||
complexity: 2,
|
||||
risk: 1,
|
||||
operationMode: 'PREPARE',
|
||||
remainingBudgetCents: 100,
|
||||
},
|
||||
'L1',
|
||||
],
|
||||
[
|
||||
{
|
||||
complexity: 5,
|
||||
risk: 4,
|
||||
operationMode: 'PREPARE',
|
||||
remainingBudgetCents: 500,
|
||||
},
|
||||
'L2',
|
||||
],
|
||||
[
|
||||
{
|
||||
complexity: 7,
|
||||
risk: 7,
|
||||
operationMode: 'PREPARE',
|
||||
remainingBudgetCents: 1500,
|
||||
},
|
||||
'L3',
|
||||
],
|
||||
[
|
||||
{
|
||||
complexity: 9,
|
||||
risk: 9,
|
||||
operationMode: 'MUTATE_PRODUCTION',
|
||||
remainingBudgetCents: 5000,
|
||||
},
|
||||
'L4',
|
||||
],
|
||||
])('selects a deterministic tier for policy inputs', (input, expected) => {
|
||||
expect(policy.select(input as never)).toBe(expected);
|
||||
expect(policy.select(input as never)).toBe(expected);
|
||||
});
|
||||
|
||||
it("fails closed when the selected tier exceeds the remaining hard budget", () => {
|
||||
expect(() => policy.select({ complexity: 9, risk: 9, operationMode: "MUTATE_PRODUCTION", remainingBudgetCents: 100 })).toThrow(BadRequestException);
|
||||
it('fails closed when the selected tier exceeds the remaining hard budget', () => {
|
||||
expect(() =>
|
||||
policy.select({
|
||||
complexity: 9,
|
||||
risk: 9,
|
||||
operationMode: 'MUTATE_PRODUCTION',
|
||||
remainingBudgetCents: 100,
|
||||
}),
|
||||
).toThrow(BadRequestException);
|
||||
});
|
||||
});
|
||||
@@ -1,13 +1,40 @@
|
||||
import { BadRequestException, Injectable } from "@nestjs/common";
|
||||
import { OperationMode } from "./orchestration.domain";
|
||||
export type ModelTier="L0"|"L1"|"L2"|"L3"|"L4";
|
||||
import { BadRequestException, Injectable } from '@nestjs/common';
|
||||
import { OperationMode } from './orchestration.domain';
|
||||
export type ModelTier = 'L0' | 'L1' | 'L2' | 'L3' | 'L4';
|
||||
@Injectable()
|
||||
export class ModelTierPolicyService {
|
||||
select(input:{complexity:number;risk:number;operationMode:OperationMode|string;remainingBudgetCents:number}):ModelTier {
|
||||
const score=Math.max(input.complexity,input.risk)+(input.operationMode===OperationMode.MUTATE_PRODUCTION?1:0);
|
||||
const tier:ModelTier=score<=1?"L0":score<=3?"L1":score<=5?"L2":score<=8?"L3":"L4";
|
||||
const floor:Record<ModelTier,number>={L0:1,L1:25,L2:100,L3:500,L4:2000};
|
||||
if(input.remainingBudgetCents<floor[tier]) throw new BadRequestException(`Hard budget cannot fund selected model tier ${tier}`);
|
||||
return tier;
|
||||
}
|
||||
select(input: {
|
||||
complexity: number;
|
||||
risk: number;
|
||||
operationMode: OperationMode | string;
|
||||
remainingBudgetCents: number;
|
||||
}): ModelTier {
|
||||
const score =
|
||||
Math.max(input.complexity, input.risk) +
|
||||
(String(input.operationMode) === String(OperationMode.MUTATE_PRODUCTION)
|
||||
? 1
|
||||
: 0);
|
||||
const tier: ModelTier =
|
||||
score <= 1
|
||||
? 'L0'
|
||||
: score <= 3
|
||||
? 'L1'
|
||||
: score <= 5
|
||||
? 'L2'
|
||||
: score <= 8
|
||||
? 'L3'
|
||||
: 'L4';
|
||||
const floor: Record<ModelTier, number> = {
|
||||
L0: 1,
|
||||
L1: 25,
|
||||
L2: 100,
|
||||
L3: 500,
|
||||
L4: 2000,
|
||||
};
|
||||
if (input.remainingBudgetCents < floor[tier])
|
||||
throw new BadRequestException(
|
||||
`Hard budget cannot fund selected model tier ${tier}`,
|
||||
);
|
||||
return tier;
|
||||
}
|
||||
}
|
||||
@@ -52,6 +52,8 @@ describe('integration read boundary', () => {
|
||||
decideGate: jest.fn(),
|
||||
recordBudget: jest.fn(),
|
||||
setKillSwitch: jest.fn(),
|
||||
command: jest.fn(),
|
||||
reconcileBudget: jest.fn(),
|
||||
};
|
||||
beforeAll(async () => {
|
||||
const m = await Test.createTestingModule({
|
||||
@@ -81,6 +83,8 @@ describe('integration read boundary', () => {
|
||||
['put', 'tasks/t1/transition'],
|
||||
['post', 'tasks/t1/gates/2/decide'],
|
||||
['post', 'tasks/t1/budget'],
|
||||
['post', 'tasks/t1/commands/RUN_CREATE'],
|
||||
['post', 'tasks/t1/budget/reconcile'],
|
||||
['put', 'kill-switch'],
|
||||
['post', 'chat/messages'],
|
||||
] as const) {
|
||||
@@ -96,6 +100,8 @@ describe('integration read boundary', () => {
|
||||
'decideGate',
|
||||
'recordBudget',
|
||||
'setKillSwitch',
|
||||
'command',
|
||||
'reconcileBudget',
|
||||
] as const)
|
||||
expect(service[name]).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
@@ -139,6 +139,12 @@ export class OrchestrationController {
|
||||
@Post('chat/messages') chat() {
|
||||
return this.deny();
|
||||
}
|
||||
@Post('tasks/:id/commands/:command') command() {
|
||||
return this.deny();
|
||||
}
|
||||
@Post('tasks/:id/budget/reconcile') reconcile() {
|
||||
return this.deny();
|
||||
}
|
||||
private deny(): never {
|
||||
throw new ForbiddenException(
|
||||
'Orchestration mutations and chat disabled pending hardened core and trusted runtime integration',
|
||||
|
||||
@@ -1,24 +1,47 @@
|
||||
import { BadRequestException, ConflictException, ForbiddenException } from "@nestjs/common";
|
||||
import { assertTransition, nextStateForGateDecision, OperationMode, WorkflowState } from "./orchestration.domain";
|
||||
import {
|
||||
BadRequestException,
|
||||
ConflictException,
|
||||
ForbiddenException,
|
||||
} from '@nestjs/common';
|
||||
import {
|
||||
assertTransition,
|
||||
nextStateForGateDecision,
|
||||
OperationMode,
|
||||
WorkflowState,
|
||||
} from './orchestration.domain';
|
||||
|
||||
describe("orchestration domain", () => {
|
||||
it("allows only declared workflow transitions", () => {
|
||||
expect(() => assertTransition(WorkflowState.DRAFT, WorkflowState.GATE1_PENDING)).not.toThrow();
|
||||
expect(() => assertTransition(WorkflowState.DRAFT, WorkflowState.RUNNING)).toThrow(BadRequestException);
|
||||
describe('orchestration domain', () => {
|
||||
it('allows only declared workflow transitions', () => {
|
||||
expect(() =>
|
||||
assertTransition(WorkflowState.DRAFT, WorkflowState.GATE1_PENDING),
|
||||
).not.toThrow();
|
||||
expect(() =>
|
||||
assertTransition(WorkflowState.DRAFT, WorkflowState.RUNNING),
|
||||
).toThrow(BadRequestException);
|
||||
});
|
||||
|
||||
it("keeps gate 1 as execution authorization rather than release authorization", () => {
|
||||
expect(nextStateForGateDecision(1, "APPROVED")).toBe(WorkflowState.GATE1_APPROVED);
|
||||
expect(nextStateForGateDecision(2, "APPROVED")).toBe(WorkflowState.GATE2_APPROVED);
|
||||
it('keeps gate 1 as execution authorization rather than release authorization', () => {
|
||||
expect(nextStateForGateDecision(1, 'APPROVED')).toBe(
|
||||
WorkflowState.GATE1_APPROVED,
|
||||
);
|
||||
expect(nextStateForGateDecision(2, 'APPROVED')).toBe(
|
||||
WorkflowState.GATE2_APPROVED,
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects an unsupported gate decision", () => {
|
||||
expect(() => nextStateForGateDecision(1, "PENDING" as never)).toThrow(BadRequestException);
|
||||
it('rejects an unsupported gate decision', () => {
|
||||
expect(() => nextStateForGateDecision(1, 'PENDING' as never)).toThrow(
|
||||
BadRequestException,
|
||||
);
|
||||
});
|
||||
|
||||
it("classifies production mutation separately from observe and prepare", () => {
|
||||
expect(OperationMode.OBSERVE).toBe("OBSERVE");
|
||||
expect(OperationMode.PREPARE).toBe("PREPARE");
|
||||
expect(OperationMode.MUTATE_PRODUCTION).toBe("MUTATE_PRODUCTION");
|
||||
it.each([WorkflowState.GATE1_PENDING, WorkflowState.GATE2_PENDING])('does not revive an expired proposal into %s', (state) => {
|
||||
expect(() => assertTransition(WorkflowState.EXPIRED,state)).toThrow(BadRequestException);
|
||||
});
|
||||
|
||||
it('classifies production mutation separately from observe and prepare', () => {
|
||||
expect(OperationMode.OBSERVE).toBe('OBSERVE');
|
||||
expect(OperationMode.PREPARE).toBe('PREPARE');
|
||||
expect(OperationMode.MUTATE_PRODUCTION).toBe('MUTATE_PRODUCTION');
|
||||
});
|
||||
});
|
||||
@@ -1,12 +1,106 @@
|
||||
import { BadRequestException } from "@nestjs/common";
|
||||
import { BadRequestException } from '@nestjs/common';
|
||||
|
||||
export enum WorkflowState { DRAFT="DRAFT", GATE1_PENDING="GATE1_PENDING", GATE1_APPROVED="GATE1_APPROVED", READY="READY", CLAIMED="CLAIMED", RUNNING="RUNNING", VERIFYING="VERIFYING", GATE2_PENDING="GATE2_PENDING", GATE2_APPROVED="GATE2_APPROVED", RELEASING="RELEASING", PRODUCTION_VERIFYING="PRODUCTION_VERIFYING", LIVE_VERIFIED="LIVE_VERIFIED", MEASURING="MEASURING", CLOSED="CLOSED", REJECTED="REJECTED", BLOCKED="BLOCKED", CANCELLED="CANCELLED", FAILED="FAILED", EXPIRED="EXPIRED" }
|
||||
export enum OperationMode { OBSERVE="OBSERVE", PREPARE="PREPARE", MUTATE_PRODUCTION="MUTATE_PRODUCTION" }
|
||||
export enum RetryClass { TRANSIENT="TRANSIENT", RATE_LIMITED="RATE_LIMITED", DEPENDENCY="DEPENDENCY", VALIDATION="VALIDATION", POLICY="POLICY", PERMANENT="PERMANENT" }
|
||||
export enum KillSwitchLevel { FREEZE_NEW_WORK="FREEZE_NEW_WORK", STOP_AGENTS="STOP_AGENTS", DISABLE_MUTATIONS="DISABLE_MUTATIONS", EMERGENCY_HALT="EMERGENCY_HALT" }
|
||||
export enum WorkflowState {
|
||||
DRAFT = 'DRAFT',
|
||||
GATE1_PENDING = 'GATE1_PENDING',
|
||||
GATE1_APPROVED = 'GATE1_APPROVED',
|
||||
READY = 'READY',
|
||||
CLAIMED = 'CLAIMED',
|
||||
RUNNING = 'RUNNING',
|
||||
VERIFYING = 'VERIFYING',
|
||||
GATE2_PENDING = 'GATE2_PENDING',
|
||||
GATE2_APPROVED = 'GATE2_APPROVED',
|
||||
RELEASING = 'RELEASING',
|
||||
PRODUCTION_VERIFYING = 'PRODUCTION_VERIFYING',
|
||||
LIVE_VERIFIED = 'LIVE_VERIFIED',
|
||||
MEASURING = 'MEASURING',
|
||||
CLOSED = 'CLOSED',
|
||||
REJECTED = 'REJECTED',
|
||||
BLOCKED = 'BLOCKED',
|
||||
CANCELLED = 'CANCELLED',
|
||||
FAILED = 'FAILED',
|
||||
EXPIRED = 'EXPIRED',
|
||||
}
|
||||
export enum OperationMode {
|
||||
OBSERVE = 'OBSERVE',
|
||||
PREPARE = 'PREPARE',
|
||||
MUTATE_PRODUCTION = 'MUTATE_PRODUCTION',
|
||||
}
|
||||
export enum RetryClass {
|
||||
TRANSIENT = 'TRANSIENT',
|
||||
RATE_LIMITED = 'RATE_LIMITED',
|
||||
DEPENDENCY = 'DEPENDENCY',
|
||||
VALIDATION = 'VALIDATION',
|
||||
POLICY = 'POLICY',
|
||||
PERMANENT = 'PERMANENT',
|
||||
}
|
||||
export enum KillSwitchLevel {
|
||||
FREEZE_NEW_WORK = 'FREEZE_NEW_WORK',
|
||||
STOP_AGENTS = 'STOP_AGENTS',
|
||||
DISABLE_MUTATIONS = 'DISABLE_MUTATIONS',
|
||||
EMERGENCY_HALT = 'EMERGENCY_HALT',
|
||||
}
|
||||
|
||||
const transitions: Record<WorkflowState, WorkflowState[]> = {
|
||||
DRAFT:[WorkflowState.GATE1_PENDING,WorkflowState.CANCELLED], GATE1_PENDING:[WorkflowState.GATE1_APPROVED,WorkflowState.REJECTED,WorkflowState.EXPIRED], GATE1_APPROVED:[WorkflowState.READY,WorkflowState.CANCELLED], READY:[WorkflowState.CLAIMED,WorkflowState.BLOCKED,WorkflowState.CANCELLED], CLAIMED:[WorkflowState.RUNNING,WorkflowState.READY,WorkflowState.CANCELLED], RUNNING:[WorkflowState.VERIFYING,WorkflowState.FAILED,WorkflowState.BLOCKED], VERIFYING:[WorkflowState.GATE2_PENDING,WorkflowState.FAILED,WorkflowState.RUNNING], GATE2_PENDING:[WorkflowState.GATE2_APPROVED,WorkflowState.REJECTED,WorkflowState.EXPIRED], GATE2_APPROVED:[WorkflowState.RELEASING,WorkflowState.CANCELLED], RELEASING:[WorkflowState.PRODUCTION_VERIFYING,WorkflowState.FAILED], PRODUCTION_VERIFYING:[WorkflowState.LIVE_VERIFIED,WorkflowState.FAILED], LIVE_VERIFIED:[WorkflowState.MEASURING,WorkflowState.CLOSED], MEASURING:[WorkflowState.CLOSED,WorkflowState.FAILED], CLOSED:[], REJECTED:[], BLOCKED:[WorkflowState.READY,WorkflowState.CANCELLED], CANCELLED:[], FAILED:[WorkflowState.READY,WorkflowState.CANCELLED], EXPIRED:[WorkflowState.GATE1_PENDING,WorkflowState.GATE2_PENDING,WorkflowState.CANCELLED]
|
||||
DRAFT: [WorkflowState.GATE1_PENDING, WorkflowState.CANCELLED],
|
||||
GATE1_PENDING: [
|
||||
WorkflowState.GATE1_APPROVED,
|
||||
WorkflowState.REJECTED,
|
||||
WorkflowState.EXPIRED,
|
||||
],
|
||||
GATE1_APPROVED: [WorkflowState.READY, WorkflowState.CANCELLED],
|
||||
READY: [
|
||||
WorkflowState.CLAIMED,
|
||||
WorkflowState.BLOCKED,
|
||||
WorkflowState.CANCELLED,
|
||||
],
|
||||
CLAIMED: [
|
||||
WorkflowState.RUNNING,
|
||||
WorkflowState.READY,
|
||||
WorkflowState.CANCELLED,
|
||||
],
|
||||
RUNNING: [
|
||||
WorkflowState.VERIFYING,
|
||||
WorkflowState.FAILED,
|
||||
WorkflowState.BLOCKED,
|
||||
],
|
||||
VERIFYING: [
|
||||
WorkflowState.GATE2_PENDING,
|
||||
WorkflowState.FAILED,
|
||||
WorkflowState.RUNNING,
|
||||
],
|
||||
GATE2_PENDING: [
|
||||
WorkflowState.GATE2_APPROVED,
|
||||
WorkflowState.REJECTED,
|
||||
WorkflowState.EXPIRED,
|
||||
],
|
||||
GATE2_APPROVED: [WorkflowState.RELEASING, WorkflowState.CANCELLED],
|
||||
RELEASING: [WorkflowState.PRODUCTION_VERIFYING, WorkflowState.FAILED],
|
||||
PRODUCTION_VERIFYING: [WorkflowState.LIVE_VERIFIED, WorkflowState.FAILED],
|
||||
LIVE_VERIFIED: [WorkflowState.MEASURING, WorkflowState.CLOSED],
|
||||
MEASURING: [WorkflowState.CLOSED, WorkflowState.FAILED],
|
||||
CLOSED: [],
|
||||
REJECTED: [],
|
||||
BLOCKED: [WorkflowState.READY, WorkflowState.CANCELLED],
|
||||
CANCELLED: [],
|
||||
FAILED: [WorkflowState.READY, WorkflowState.CANCELLED],
|
||||
EXPIRED: [WorkflowState.CANCELLED],
|
||||
};
|
||||
export function assertTransition(from: WorkflowState, to: WorkflowState): void { if (!transitions[from]?.includes(to)) throw new BadRequestException(`Invalid orchestration transition ${from} -> ${to}`); }
|
||||
export function nextStateForGateDecision(gate: 1|2, decision: "APPROVED"|"REJECTED"): WorkflowState { if (!(["APPROVED","REJECTED"] as string[]).includes(decision)) throw new BadRequestException("Decision must be APPROVED or REJECTED"); return decision === "REJECTED" ? WorkflowState.REJECTED : gate === 1 ? WorkflowState.GATE1_APPROVED : WorkflowState.GATE2_APPROVED; }
|
||||
export function assertTransition(from: WorkflowState, to: WorkflowState): void {
|
||||
if (!transitions[from]?.includes(to))
|
||||
throw new BadRequestException(
|
||||
`Invalid orchestration transition ${from} -> ${to}`,
|
||||
);
|
||||
}
|
||||
export function nextStateForGateDecision(
|
||||
gate: 1 | 2,
|
||||
decision: 'APPROVED' | 'REJECTED',
|
||||
): WorkflowState {
|
||||
if (!(['APPROVED', 'REJECTED'] as string[]).includes(decision))
|
||||
throw new BadRequestException('Decision must be APPROVED or REJECTED');
|
||||
return decision === 'REJECTED'
|
||||
? WorkflowState.REJECTED
|
||||
: gate === 1
|
||||
? WorkflowState.GATE1_APPROVED
|
||||
: WorkflowState.GATE2_APPROVED;
|
||||
}
|
||||
@@ -0,0 +1,577 @@
|
||||
import { OrchestrationController } from './orchestration.controller';
|
||||
import { DepartmentTasksController } from '../admin/department-tasks.controller';
|
||||
import { ApprovalsController } from '../admin/approvals.controller';
|
||||
import { PrismaClient } from '@prisma/client';
|
||||
import { OrchestrationService } from './orchestration.service';
|
||||
import { KillSwitchService } from './kill-switch.service';
|
||||
import { WorkflowState, OperationMode } from './orchestration.domain';
|
||||
const enabled = process.env.ORCHESTRATION_INTEGRATION === '1';
|
||||
(enabled ? describe : describe.skip)(
|
||||
'v4 disposable PostgreSQL invariants',
|
||||
() => {
|
||||
const db = new PrismaClient();
|
||||
const service = new OrchestrationService(
|
||||
db as never,
|
||||
new KillSwitchService(db as never),
|
||||
);
|
||||
let n = 0;
|
||||
async function task(state = 'DRAFT', budget = 100) {
|
||||
return db.orchestrationTask.create({
|
||||
data: {
|
||||
key: `test-${Date.now()}-${n++}`,
|
||||
title: 'test',
|
||||
department: 'TEST',
|
||||
operationMode: 'PREPARE',
|
||||
state,
|
||||
budgetLimitCents: budget,
|
||||
},
|
||||
});
|
||||
}
|
||||
afterAll(async () => db.$disconnect());
|
||||
it('release rejects PREPARE operation class even with gate approval', async()=>{
|
||||
const t=await task('GATE2_APPROVED');
|
||||
await expect(service.command(t.id,'RELEASE_AUTHORIZE',{expectedVersion:1,idempotencyKey:`mode-${t.id}`},'ci')).rejects.toThrow(/operation mode/i);
|
||||
});
|
||||
|
||||
it('append-only ledger rejects direct rewrites in PostgreSQL', async () => {
|
||||
const t = await task();
|
||||
const r = await service.recordBudget(
|
||||
t.id,
|
||||
{
|
||||
amountCents: 50,
|
||||
kind: 'RESERVE',
|
||||
idempotencyKey: `immutable-${t.id}`,
|
||||
},
|
||||
'a',
|
||||
);
|
||||
await expect(
|
||||
db.orchestrationBudgetLedger.update({
|
||||
where: { id: r.id },
|
||||
data: { amountCents: 0 },
|
||||
}),
|
||||
).rejects.toThrow(/append.only/i);
|
||||
await expect(
|
||||
db.orchestrationBudgetLedger.delete({ where: { id: r.id } }),
|
||||
).rejects.toThrow(/append.only/i);
|
||||
});
|
||||
it('generic transitions cannot fake run lifecycle or production verification', async () => {
|
||||
for (const [from, to] of [
|
||||
['CLAIMED', 'RUNNING'],
|
||||
['RUNNING', 'VERIFYING'],
|
||||
['FAILED', 'READY'],
|
||||
['PRODUCTION_VERIFYING', 'LIVE_VERIFIED'],
|
||||
]) {
|
||||
const t = await task(from);
|
||||
await expect(
|
||||
service.transition(
|
||||
t.id,
|
||||
{
|
||||
expectedVersion: 1,
|
||||
idempotencyKey: `fake-${t.id}`,
|
||||
toState: to as WorkflowState,
|
||||
},
|
||||
'agent',
|
||||
),
|
||||
).rejects.toThrow(/dedicated/i);
|
||||
}
|
||||
});
|
||||
it('exposes command and reconcile HTTP handlers', () => {
|
||||
const c = new OrchestrationController(service);
|
||||
expect(typeof c.command).toBe('function');
|
||||
expect(typeof c.reconcile).toBe('function');
|
||||
});
|
||||
it('highest kill level blocks reads and reconciliation', async () => {
|
||||
const t = await task();
|
||||
const r = await service.recordBudget(
|
||||
t.id,
|
||||
{
|
||||
kind: 'RESERVE',
|
||||
amountCents: 10,
|
||||
idempotencyKey: `stop-res-${t.id}`,
|
||||
},
|
||||
'a',
|
||||
);
|
||||
await service.setKillSwitch(
|
||||
['EMERGENCY', 'HALT'].join('_'),
|
||||
'test',
|
||||
'ceo',
|
||||
);
|
||||
try {
|
||||
await expect(service.findOne(t.id)).rejects.toThrow(/kill switch/i);
|
||||
await expect(
|
||||
service.reconcileBudget(
|
||||
t.id,
|
||||
{
|
||||
reservationId: r.id,
|
||||
actualCents: 0,
|
||||
idempotencyKey: `stop-recon-${t.id}`,
|
||||
},
|
||||
'a',
|
||||
),
|
||||
).rejects.toThrow(/kill switch/i);
|
||||
} finally {
|
||||
await db.orchestrationKillSwitch.deleteMany();
|
||||
}
|
||||
});
|
||||
it('legacy arbitrary status and approval endpoints fail closed', async () => {
|
||||
await expect(
|
||||
new DepartmentTasksController(db as never).create({
|
||||
title: 'bypass',
|
||||
status: 'DONE',
|
||||
}),
|
||||
).rejects.toThrow(/v4/i);
|
||||
await expect(
|
||||
new DepartmentTasksController(db as never).updateStatus('none', {
|
||||
status: 'DONE',
|
||||
}),
|
||||
).rejects.toThrow(/v4/i);
|
||||
await expect(
|
||||
new ApprovalsController(db as never).decide('none', {
|
||||
decision: 'APPROVED',
|
||||
} as any),
|
||||
).rejects.toThrow(/v4/i);
|
||||
});
|
||||
it('release consumes only matching fresh production Gate2 via dedicated command', async () => {
|
||||
const t = await task('GATE2_PENDING');
|
||||
const hash = 'f'.repeat(64),
|
||||
commitSha = 'a'.repeat(40);
|
||||
const r = await db.orchestrationRun.create({
|
||||
data: {
|
||||
taskId: t.id,
|
||||
status: 'SUCCEEDED',
|
||||
commitSha,
|
||||
environment: 'production',
|
||||
idempotencyKey: `rel-run-${t.id}`,
|
||||
},
|
||||
});
|
||||
await db.orchestrationArtifact.create({
|
||||
data: {
|
||||
runId: r.id,
|
||||
kind: 'BUILD',
|
||||
uri: 'https://ci.example/build',
|
||||
sha256: hash,
|
||||
},
|
||||
});
|
||||
await db.orchestrationAttestation.create({
|
||||
data: {
|
||||
runId: r.id,
|
||||
kind: 'TEST',
|
||||
subjectHash: hash,
|
||||
signer: 'ci',
|
||||
predicate: { passed: true },
|
||||
},
|
||||
});
|
||||
const decision = await service.decideGate(
|
||||
t.id,
|
||||
2,
|
||||
{
|
||||
decision: 'APPROVED',
|
||||
expectedVersion: 1,
|
||||
idempotencyKey: `rel-approve-${t.id}`,
|
||||
expiresAt: new Date(Date.now() + 60000).toISOString(),
|
||||
runId: r.id,
|
||||
artifactHash: hash,
|
||||
commitSha,
|
||||
environment: 'production',
|
||||
},
|
||||
'ceo',
|
||||
);
|
||||
const dto = {
|
||||
expectedVersion: 2,
|
||||
idempotencyKey: `rel-${t.id}`,
|
||||
approvalId: decision.id,
|
||||
runId: r.id,
|
||||
artifactHash: hash,
|
||||
commitSha,
|
||||
environment: 'production',
|
||||
};
|
||||
await expect(
|
||||
(service as any).command(
|
||||
t.id,
|
||||
'RELEASE_AUTHORIZE',
|
||||
{ ...dto, artifactHash: '0'.repeat(64) },
|
||||
'ci',
|
||||
),
|
||||
).rejects.toThrow(/binding/i);
|
||||
const result = await (service as any).command(
|
||||
t.id,
|
||||
'RELEASE_AUTHORIZE',
|
||||
dto,
|
||||
'ci',
|
||||
);
|
||||
expect(result.authorized).toBe(true);
|
||||
expect(
|
||||
(await db.orchestrationTask.findUniqueOrThrow({ where: { id: t.id } }))
|
||||
.state,
|
||||
).toBe('RELEASING');
|
||||
await expect(
|
||||
(service as any).command(
|
||||
t.id,
|
||||
'RELEASE_AUTHORIZE',
|
||||
{ ...dto, expectedVersion: 3, idempotencyKey: `rel-again-${t.id}` },
|
||||
'ci',
|
||||
),
|
||||
).rejects.toThrow();
|
||||
});
|
||||
it('gate decision replay rejects changed immutable expiry and actor', async () => {
|
||||
const t = await task('GATE1_PENDING');
|
||||
const dto = {
|
||||
decision: 'APPROVED' as const,
|
||||
expectedVersion: 1,
|
||||
idempotencyKey: `bind-${t.id}`,
|
||||
expiresAt: new Date(Date.now() + 60000).toISOString(),
|
||||
};
|
||||
await service.decideGate(t.id, 1, dto, 'ceo');
|
||||
await expect(
|
||||
service.decideGate(
|
||||
t.id,
|
||||
1,
|
||||
{ ...dto, expiresAt: new Date(Date.now() + 120000).toISOString() },
|
||||
'ceo',
|
||||
),
|
||||
).rejects.toThrow(/Idempotency/i);
|
||||
await expect(service.decideGate(t.id, 1, dto, 'other')).rejects.toThrow(
|
||||
/Idempotency/i,
|
||||
);
|
||||
});
|
||||
it('dependencies block admission and reject cycles', async () => {
|
||||
const a = await task();
|
||||
const b = await task();
|
||||
await (service as any).command(
|
||||
a.id,
|
||||
'DEPENDENCY_ADD',
|
||||
{
|
||||
expectedVersion: 1,
|
||||
idempotencyKey: `dep-a-${a.id}`,
|
||||
dependsOnTaskId: b.id,
|
||||
},
|
||||
'actor',
|
||||
);
|
||||
await expect(
|
||||
(service as any).command(
|
||||
b.id,
|
||||
'DEPENDENCY_ADD',
|
||||
{
|
||||
expectedVersion: 1,
|
||||
idempotencyKey: `dep-b-${b.id}`,
|
||||
dependsOnTaskId: a.id,
|
||||
},
|
||||
'actor',
|
||||
),
|
||||
).rejects.toThrow(/cycle/i);
|
||||
await db.orchestrationTask.update({
|
||||
where: { id: a.id },
|
||||
data: { state: 'GATE1_PENDING' },
|
||||
});
|
||||
await service.decideGate(
|
||||
a.id,
|
||||
1,
|
||||
{
|
||||
decision: 'APPROVED',
|
||||
expectedVersion: 2,
|
||||
idempotencyKey: `dep-approval-${a.id}`,
|
||||
expiresAt: new Date(Date.now() + 60000).toISOString(),
|
||||
},
|
||||
'ceo',
|
||||
);
|
||||
await expect(
|
||||
service.transition(
|
||||
a.id,
|
||||
{
|
||||
toState: WorkflowState.READY,
|
||||
expectedVersion: 3,
|
||||
idempotencyKey: `dep-ready-${a.id}`,
|
||||
},
|
||||
'agent',
|
||||
),
|
||||
).rejects.toThrow(/dependency/i);
|
||||
});
|
||||
it('registers artifact and attestation for owned run, blocks unregistered subjects', async () => {
|
||||
const t = await task('CLAIMED');
|
||||
const base = {
|
||||
expectedVersion: 1,
|
||||
idempotencyKey: `life-${t.id}`,
|
||||
commitSha: 'c'.repeat(40),
|
||||
environment: 'staging',
|
||||
};
|
||||
const r = await (service as any).command(
|
||||
t.id,
|
||||
'RUN_CREATE',
|
||||
base,
|
||||
'agent',
|
||||
);
|
||||
const hash = 'd'.repeat(64);
|
||||
const a = await (service as any).command(
|
||||
t.id,
|
||||
'ARTIFACT_ADD',
|
||||
{
|
||||
expectedVersion: 2,
|
||||
idempotencyKey: `art-${t.id}`,
|
||||
runId: r.id,
|
||||
sha256: hash,
|
||||
kind: 'BUILD',
|
||||
uri: 'https://ci.example/artifact',
|
||||
},
|
||||
'agent',
|
||||
);
|
||||
expect(a.sha256).toBe(hash);
|
||||
await expect(
|
||||
(service as any).command(
|
||||
t.id,
|
||||
'ATTESTATION_ADD',
|
||||
{
|
||||
expectedVersion: 3,
|
||||
idempotencyKey: `bad-att-${t.id}`,
|
||||
runId: r.id,
|
||||
subjectHash: 'e'.repeat(64),
|
||||
kind: 'TEST',
|
||||
predicate: { passed: true },
|
||||
},
|
||||
'ci',
|
||||
),
|
||||
).rejects.toThrow(/artifact/i);
|
||||
const att = await (service as any).command(
|
||||
t.id,
|
||||
'ATTESTATION_ADD',
|
||||
{
|
||||
expectedVersion: 3,
|
||||
idempotencyKey: `att-${t.id}`,
|
||||
runId: r.id,
|
||||
subjectHash: hash,
|
||||
kind: 'TEST',
|
||||
predicate: { passed: true },
|
||||
},
|
||||
'ci',
|
||||
);
|
||||
expect(att.signer).toBe('ci');
|
||||
});
|
||||
it('creates immutable versioned runs with bounded transient retries', async () => {
|
||||
const t = await task('CLAIMED');
|
||||
const dto = {
|
||||
expectedVersion: 1,
|
||||
idempotencyKey: `run-${t.id}`,
|
||||
commitSha: 'a'.repeat(40),
|
||||
environment: 'staging',
|
||||
};
|
||||
const r = await (service as any).command(
|
||||
t.id,
|
||||
'RUN_CREATE',
|
||||
dto,
|
||||
'agent',
|
||||
);
|
||||
expect(r.attempt).toBe(1);
|
||||
expect(
|
||||
await (service as any).command(t.id, 'RUN_CREATE', dto, 'agent'),
|
||||
).toEqual(r);
|
||||
await expect(
|
||||
(service as any).command(
|
||||
t.id,
|
||||
'RUN_CREATE',
|
||||
{ ...dto, commitSha: 'b'.repeat(40) },
|
||||
'agent',
|
||||
),
|
||||
).rejects.toThrow(/Idempotency/i);
|
||||
await (service as any).command(
|
||||
t.id,
|
||||
'RUN_FINISH',
|
||||
{
|
||||
expectedVersion: 2,
|
||||
idempotencyKey: `finish-${t.id}`,
|
||||
runId: r.id,
|
||||
status: 'FAILED',
|
||||
retryClass: 'POLICY',
|
||||
},
|
||||
'agent',
|
||||
);
|
||||
await expect(
|
||||
(service as any).command(
|
||||
t.id,
|
||||
'RUN_CREATE',
|
||||
{ ...dto, expectedVersion: 3, idempotencyKey: `retry-${t.id}` },
|
||||
'agent',
|
||||
),
|
||||
).rejects.toThrow(/retry/i);
|
||||
});
|
||||
it('selects tier at admission using backend risk and budget', async () => {
|
||||
const t = await task('GATE1_PENDING', 1000);
|
||||
await service.decideGate(
|
||||
t.id,
|
||||
1,
|
||||
{
|
||||
decision: 'APPROVED',
|
||||
expectedVersion: 1,
|
||||
idempotencyKey: `approve-tier-${t.id}`,
|
||||
expiresAt: new Date(Date.now() + 60000).toISOString(),
|
||||
},
|
||||
'ceo',
|
||||
);
|
||||
await service.transition(
|
||||
t.id,
|
||||
{
|
||||
toState: WorkflowState.READY,
|
||||
expectedVersion: 2,
|
||||
idempotencyKey: `admit-tier-${t.id}`,
|
||||
},
|
||||
'agent',
|
||||
);
|
||||
expect(
|
||||
(await db.orchestrationTask.findUniqueOrThrow({ where: { id: t.id } }))
|
||||
.modelTier,
|
||||
).toBe('L0');
|
||||
});
|
||||
it('STOP_AGENTS prevents claims and running transitions', async () => {
|
||||
await service.setKillSwitch('STOP_AGENTS', 'test', 'ceo');
|
||||
try {
|
||||
const t = await task('READY');
|
||||
await expect(
|
||||
service.transition(
|
||||
t.id,
|
||||
{
|
||||
toState: WorkflowState.CLAIMED,
|
||||
expectedVersion: 1,
|
||||
idempotencyKey: `kill-${t.id}`,
|
||||
},
|
||||
'agent',
|
||||
),
|
||||
).rejects.toThrow(/kill switch/i);
|
||||
} finally {
|
||||
await db.orchestrationKillSwitch.deleteMany();
|
||||
}
|
||||
});
|
||||
it('Gate1 admission consumes only unexpired approval', async () => {
|
||||
const t = await task('GATE1_PENDING');
|
||||
await service.decideGate(
|
||||
t.id,
|
||||
1,
|
||||
{
|
||||
decision: 'APPROVED',
|
||||
expectedVersion: 1,
|
||||
idempotencyKey: `approve-${t.id}`,
|
||||
expiresAt: new Date(Date.now() + 100).toISOString(),
|
||||
},
|
||||
'ceo',
|
||||
);
|
||||
await new Promise((r) => setTimeout(r, 150));
|
||||
await expect(
|
||||
service.transition(
|
||||
t.id,
|
||||
{
|
||||
toState: WorkflowState.READY,
|
||||
expectedVersion: 2,
|
||||
idempotencyKey: `admit-${t.id}`,
|
||||
},
|
||||
'agent',
|
||||
),
|
||||
).rejects.toThrow(/expired/i);
|
||||
});
|
||||
it('versioned transition replay returns original result and rejects changed payload', async () => {
|
||||
const t = await task();
|
||||
const dto = {
|
||||
toState: WorkflowState.GATE1_PENDING,
|
||||
expectedVersion: 1,
|
||||
idempotencyKey: `tr-${t.id}`,
|
||||
};
|
||||
const first = await service.transition(t.id, dto, 'actor');
|
||||
expect(await service.transition(t.id, dto, 'actor')).toEqual(first);
|
||||
await expect(
|
||||
service.transition(t.id, { ...dto, reason: 'different' }, 'actor'),
|
||||
).rejects.toThrow(/Idempotency/i);
|
||||
});
|
||||
it('reconciles a reservation exactly once without erasing spent cost', async () => {
|
||||
const t = await task();
|
||||
const r = await service.recordBudget(
|
||||
t.id,
|
||||
{ amountCents: 80, kind: 'RESERVE', idempotencyKey: `res-${t.id}` },
|
||||
'agent',
|
||||
);
|
||||
const dto = {
|
||||
reservationId: r.id,
|
||||
actualCents: 35,
|
||||
idempotencyKey: `reconcile-${t.id}`,
|
||||
};
|
||||
await (service as any).reconcileBudget(t.id, dto, 'agent');
|
||||
await (service as any).reconcileBudget(t.id, dto, 'agent');
|
||||
await expect(
|
||||
(service as any).reconcileBudget(
|
||||
t.id,
|
||||
{ ...dto, idempotencyKey: `second-${t.id}` },
|
||||
'agent',
|
||||
),
|
||||
).rejects.toThrow(/reconciled/i);
|
||||
expect(
|
||||
(
|
||||
await db.orchestrationBudgetLedger.aggregate({
|
||||
where: { taskId: t.id },
|
||||
_sum: { amountCents: true },
|
||||
})
|
||||
)._sum.amountCents,
|
||||
).toBe(35);
|
||||
await expect(
|
||||
service.recordBudget(
|
||||
t.id,
|
||||
{
|
||||
amountCents: 66,
|
||||
kind: 'RESERVE',
|
||||
idempotencyKey: `too-much-${t.id}`,
|
||||
},
|
||||
'agent',
|
||||
),
|
||||
).rejects.toThrow(/budget/i);
|
||||
});
|
||||
it('concurrent budget reservations never exceed hard cap and replay is harmless', async () => {
|
||||
const t = await task();
|
||||
const result = await Promise.allSettled(
|
||||
Array.from({ length: 8 }, (_, i) =>
|
||||
service.recordBudget(
|
||||
t.id,
|
||||
{
|
||||
amountCents: 30,
|
||||
kind: 'RESERVE',
|
||||
idempotencyKey: `reserve-${t.id}-${i}`,
|
||||
},
|
||||
'agent',
|
||||
),
|
||||
),
|
||||
);
|
||||
expect(result.filter((x) => x.status === 'fulfilled')).toHaveLength(3);
|
||||
const sum = await db.orchestrationBudgetLedger.aggregate({
|
||||
where: { taskId: t.id },
|
||||
_sum: { amountCents: true },
|
||||
});
|
||||
expect(sum._sum.amountCents).toBe(90);
|
||||
const first = await db.orchestrationBudgetLedger.findFirstOrThrow({
|
||||
where: { taskId: t.id },
|
||||
});
|
||||
await service.recordBudget(
|
||||
t.id,
|
||||
{
|
||||
amountCents: 30,
|
||||
kind: 'RESERVE',
|
||||
idempotencyKey: first.idempotencyKey,
|
||||
},
|
||||
'agent',
|
||||
);
|
||||
expect(
|
||||
await db.orchestrationBudgetLedger.count({ where: { taskId: t.id } }),
|
||||
).toBe(3);
|
||||
});
|
||||
it('generic transition cannot approve gate 1', async () => {
|
||||
const t = await task('GATE1_PENDING');
|
||||
await expect(
|
||||
service.transition(
|
||||
t.id,
|
||||
{
|
||||
toState: WorkflowState.GATE1_APPROVED,
|
||||
expectedVersion: 1,
|
||||
idempotencyKey: `gate-bypass-${n}`,
|
||||
},
|
||||
'ceo',
|
||||
),
|
||||
).rejects.toThrow(/gate/i);
|
||||
expect(
|
||||
(await db.orchestrationTask.findUniqueOrThrow({ where: { id: t.id } }))
|
||||
.state,
|
||||
).toBe('GATE1_PENDING');
|
||||
});
|
||||
},
|
||||
);
|
||||
@@ -1,2 +1,13 @@
|
||||
import { Module } from "@nestjs/common"; import { PrismaModule } from "../prisma/prisma.module"; import { KillSwitchService } from "./kill-switch.service"; import { ModelTierPolicyService } from "./model-tier-policy.service"; import { OrchestrationController } from "./orchestration.controller"; import { OrchestrationService } from "./orchestration.service";
|
||||
@Module({imports:[PrismaModule],controllers:[OrchestrationController],providers:[OrchestrationService,KillSwitchService,ModelTierPolicyService],exports:[OrchestrationService,KillSwitchService,ModelTierPolicyService]}) export class OrchestrationModule{}
|
||||
import { Module } from '@nestjs/common';
|
||||
import { PrismaModule } from '../prisma/prisma.module';
|
||||
import { KillSwitchService } from './kill-switch.service';
|
||||
import { ModelTierPolicyService } from './model-tier-policy.service';
|
||||
import { OrchestrationController } from './orchestration.controller';
|
||||
import { OrchestrationService } from './orchestration.service';
|
||||
@Module({
|
||||
imports: [PrismaModule],
|
||||
controllers: [OrchestrationController],
|
||||
providers: [OrchestrationService, KillSwitchService, ModelTierPolicyService],
|
||||
exports: [OrchestrationService, KillSwitchService, ModelTierPolicyService],
|
||||
})
|
||||
export class OrchestrationModule {}
|
||||
@@ -1,73 +1,260 @@
|
||||
import { BadRequestException, ConflictException, ForbiddenException } from "@nestjs/common";
|
||||
import { OrchestrationService } from "./orchestration.service";
|
||||
import {
|
||||
BadRequestException,
|
||||
ConflictException,
|
||||
ForbiddenException,
|
||||
} from '@nestjs/common';
|
||||
import { OrchestrationService } from './orchestration.service';
|
||||
|
||||
function mockPrisma() {
|
||||
const tx: any = {
|
||||
orchestrationTask: { findUnique: jest.fn(), create: jest.fn(), updateMany: jest.fn() },
|
||||
$queryRaw: jest.fn().mockResolvedValue([]),
|
||||
orchestrationTask: {
|
||||
findUnique: jest.fn(),
|
||||
create: jest.fn(),
|
||||
updateMany: jest.fn(),
|
||||
},
|
||||
orchestrationRun: { findUnique: jest.fn(), create: jest.fn() },
|
||||
orchestrationArtifact: { findFirst: jest.fn(), create: jest.fn() },
|
||||
orchestrationAttestation: { create: jest.fn() },
|
||||
orchestrationApprovalDecision: { findUnique: jest.fn(), create: jest.fn() },
|
||||
orchestrationEvent: { create: jest.fn() },
|
||||
orchestrationBudgetLedger: { aggregate: jest.fn(), create: jest.fn() },
|
||||
orchestrationEvent: { findUnique: jest.fn(), create: jest.fn() },
|
||||
orchestrationBudgetLedger: {
|
||||
findUnique: jest.fn(),
|
||||
aggregate: jest.fn(),
|
||||
create: jest.fn(),
|
||||
},
|
||||
orchestrationKillSwitch: { findUnique: jest.fn(), upsert: jest.fn() },
|
||||
};
|
||||
tx.$transaction = jest.fn(async (cb: any) => cb(tx));
|
||||
tx.$transaction = jest.fn((cb: any) => Promise.resolve(cb(tx)));
|
||||
return tx;
|
||||
}
|
||||
|
||||
describe("OrchestrationService", () => {
|
||||
describe('OrchestrationService', () => {
|
||||
let prisma: any;
|
||||
let service: OrchestrationService;
|
||||
beforeEach(() => { prisma = mockPrisma(); service = new OrchestrationService(prisma, { assertAllowed: jest.fn() } as never); });
|
||||
beforeEach(() => {
|
||||
prisma = mockPrisma();
|
||||
service = new OrchestrationService(prisma, {
|
||||
assertAllowed: jest.fn(),
|
||||
} as never);
|
||||
});
|
||||
|
||||
it("creates a task and append-only event transactionally", async () => {
|
||||
prisma.orchestrationTask.create.mockResolvedValue({ id: "task-1", state: "DRAFT", version: 1 });
|
||||
const result = await service.createTask({ key: "seo-1", title: "SEO", department: "SEO", operationMode: "PREPARE" as never, budgetLimitCents: 1000 }, "actor-1");
|
||||
it('rejects an unknown kill level before overwriting persisted protection', async () => {
|
||||
await expect(service.setKillSwitch('INVALID_LEVEL','test','admin')).rejects.toThrow(BadRequestException);
|
||||
expect(prisma.orchestrationKillSwitch.upsert).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('creates a task and append-only event transactionally', async () => {
|
||||
prisma.orchestrationTask.create.mockResolvedValue({
|
||||
id: 'task-1',
|
||||
state: 'DRAFT',
|
||||
version: 1,
|
||||
});
|
||||
const result = await service.createTask(
|
||||
{
|
||||
key: 'seo-1',
|
||||
title: 'SEO',
|
||||
department: 'SEO',
|
||||
operationMode: 'PREPARE' as never,
|
||||
budgetLimitCents: 1000,
|
||||
},
|
||||
'actor-1',
|
||||
);
|
||||
expect(prisma.$transaction).toHaveBeenCalledTimes(1);
|
||||
expect(prisma.orchestrationTask.create).toHaveBeenCalledWith({ data: expect.objectContaining({ state: "DRAFT", version: 1 }) });
|
||||
expect(prisma.orchestrationEvent.create).toHaveBeenCalledWith({ data: expect.objectContaining({ eventType: "TASK_CREATED", sequence: 1 }) });
|
||||
expect(result.id).toBe("task-1");
|
||||
expect(prisma.orchestrationTask.create).toHaveBeenCalledWith({
|
||||
data: expect.objectContaining({ state: 'DRAFT', version: 1 }),
|
||||
});
|
||||
expect(prisma.orchestrationEvent.create).toHaveBeenCalledWith({
|
||||
data: expect.objectContaining({ eventType: 'TASK_CREATED', sequence: 1 }),
|
||||
});
|
||||
expect(result.id).toBe('task-1');
|
||||
});
|
||||
|
||||
it("uses compare-and-swap versioning for transitions", async () => {
|
||||
prisma.orchestrationTask.findUnique.mockResolvedValue({ id: "task-1", state: "DRAFT", version: 3 });
|
||||
it('uses compare-and-swap versioning for transitions', async () => {
|
||||
prisma.orchestrationTask.findUnique.mockResolvedValue({
|
||||
id: 'task-1',
|
||||
state: 'DRAFT',
|
||||
version: 3,
|
||||
});
|
||||
prisma.orchestrationTask.updateMany.mockResolvedValue({ count: 0 });
|
||||
await expect(service.transition("task-1", { toState: "GATE1_PENDING" as never, expectedVersion: 3, idempotencyKey: "i-1" }, "actor")).rejects.toThrow(ConflictException);
|
||||
await expect(
|
||||
service.transition(
|
||||
'task-1',
|
||||
{
|
||||
toState: 'GATE1_PENDING' as never,
|
||||
expectedVersion: 3,
|
||||
idempotencyKey: 'i-1',
|
||||
},
|
||||
'actor',
|
||||
),
|
||||
).rejects.toThrow(ConflictException);
|
||||
});
|
||||
|
||||
it("makes gate decisions immutable and idempotent", async () => {
|
||||
prisma.orchestrationApprovalDecision.findUnique.mockResolvedValue({ id: "d1", decision: "APPROVED", gate: 1, taskId: "task-1" });
|
||||
await expect(service.decideGate("task-1", 1, { decision: "APPROVED", expectedVersion: 2, idempotencyKey: "gate-1", expiresAt: new Date(Date.now()+10000).toISOString() }, "ceo")).resolves.toMatchObject({ id: "d1" });
|
||||
it('makes gate decisions immutable and idempotent', async () => {
|
||||
const expiresAt = new Date(Date.now() + 10000).toISOString();
|
||||
prisma.orchestrationApprovalDecision.findUnique.mockResolvedValue({
|
||||
id: 'd1',
|
||||
decision: 'APPROVED',
|
||||
gate: 1,
|
||||
taskId: 'task-1',
|
||||
});
|
||||
prisma.orchestrationEvent.findUnique.mockResolvedValue({
|
||||
payload: {
|
||||
fingerprint: JSON.stringify({
|
||||
taskId: 'task-1',
|
||||
gate: 1,
|
||||
dto: {
|
||||
decision: 'APPROVED',
|
||||
expectedVersion: 2,
|
||||
idempotencyKey: 'gate-1',
|
||||
expiresAt,
|
||||
},
|
||||
actorId: 'ceo',
|
||||
}),
|
||||
},
|
||||
});
|
||||
await expect(
|
||||
service.decideGate(
|
||||
'task-1',
|
||||
1,
|
||||
{
|
||||
decision: 'APPROVED',
|
||||
expectedVersion: 2,
|
||||
idempotencyKey: 'gate-1',
|
||||
expiresAt,
|
||||
},
|
||||
'ceo',
|
||||
),
|
||||
).resolves.toMatchObject({ id: 'd1' });
|
||||
expect(prisma.orchestrationApprovalDecision.create).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("rejects an expired approval", async () => {
|
||||
await expect(service.decideGate("task-1", 1, { decision: "APPROVED", expectedVersion: 2, idempotencyKey: "gate-expired", expiresAt: new Date(0).toISOString() }, "ceo")).rejects.toThrow(BadRequestException);
|
||||
it('rejects an expired approval', async () => {
|
||||
await expect(
|
||||
service.decideGate(
|
||||
'task-1',
|
||||
1,
|
||||
{
|
||||
decision: 'APPROVED',
|
||||
expectedVersion: 2,
|
||||
idempotencyKey: 'gate-expired',
|
||||
expiresAt: new Date(0).toISOString(),
|
||||
},
|
||||
'ceo',
|
||||
),
|
||||
).rejects.toThrow(BadRequestException);
|
||||
});
|
||||
|
||||
it("binds gate 2 approval to run, artifact hash, commit and environment", async () => {
|
||||
it('binds gate 2 approval to run, artifact hash, commit and environment', async () => {
|
||||
prisma.orchestrationApprovalDecision.findUnique.mockResolvedValue(null);
|
||||
prisma.orchestrationTask.findUnique.mockResolvedValue({ id: "task-1", state: "GATE2_PENDING", version: 7 });
|
||||
prisma.orchestrationRun.findUnique.mockResolvedValue({ id: "run-1", taskId: "task-1", commitSha: "abc123", environment: "staging" });
|
||||
prisma.orchestrationArtifact.findFirst.mockResolvedValue({ id: "artifact-1", runId: "run-1", sha256: "f".repeat(64) });
|
||||
prisma.orchestrationTask.findUnique.mockResolvedValue({
|
||||
id: 'task-1',
|
||||
state: 'GATE2_PENDING',
|
||||
version: 7,
|
||||
});
|
||||
prisma.orchestrationRun.findUnique.mockResolvedValue({
|
||||
id: 'run-1',
|
||||
taskId: 'task-1',
|
||||
commitSha: 'abc123',
|
||||
environment: 'staging',
|
||||
status: 'SUCCEEDED',
|
||||
});
|
||||
prisma.orchestrationArtifact.findFirst.mockResolvedValue({
|
||||
id: 'artifact-1',
|
||||
runId: 'run-1',
|
||||
sha256: 'f'.repeat(64),
|
||||
});
|
||||
prisma.orchestrationTask.updateMany.mockResolvedValue({ count: 1 });
|
||||
prisma.orchestrationApprovalDecision.create.mockResolvedValue({ id: "decision-2", gate: 2 });
|
||||
await service.decideGate("task-1", 2, { decision: "APPROVED", expectedVersion: 7, idempotencyKey: "gate-2", expiresAt: new Date(Date.now()+10000).toISOString(), runId: "run-1", artifactHash: "f".repeat(64), commitSha: "abc123", environment: "staging" }, "ceo");
|
||||
expect(prisma.orchestrationApprovalDecision.create).toHaveBeenCalledWith({ data: expect.objectContaining({ runId: "run-1", artifactHash: "f".repeat(64), commitSha: "abc123", environment: "staging" }) });
|
||||
prisma.orchestrationApprovalDecision.create.mockResolvedValue({
|
||||
id: 'decision-2',
|
||||
gate: 2,
|
||||
});
|
||||
await service.decideGate(
|
||||
'task-1',
|
||||
2,
|
||||
{
|
||||
decision: 'APPROVED',
|
||||
expectedVersion: 7,
|
||||
idempotencyKey: 'gate-2',
|
||||
expiresAt: new Date(Date.now() + 10000).toISOString(),
|
||||
runId: 'run-1',
|
||||
artifactHash: 'f'.repeat(64),
|
||||
commitSha: 'abc123',
|
||||
environment: 'staging',
|
||||
},
|
||||
'ceo',
|
||||
);
|
||||
expect(prisma.orchestrationApprovalDecision.create).toHaveBeenCalledWith({
|
||||
data: expect.objectContaining({
|
||||
runId: 'run-1',
|
||||
artifactHash: 'f'.repeat(64),
|
||||
commitSha: 'abc123',
|
||||
environment: 'staging',
|
||||
}),
|
||||
});
|
||||
});
|
||||
|
||||
it("refuses gate 2 when binding does not match the immutable run", async () => {
|
||||
it.each(['RUNNING', 'FAILED', 'CANCELLED'])('refuses Gate 2 approval for an unfinished or unsuccessful %s run', async (status) => {
|
||||
prisma.orchestrationTask.findUnique.mockResolvedValue({id:'task-1',state:'GATE2_PENDING',version:7});
|
||||
prisma.orchestrationRun.findUnique.mockResolvedValue({id:'run-1',taskId:'task-1',commitSha:'abc123',environment:'staging',status});
|
||||
prisma.orchestrationArtifact.findFirst.mockResolvedValue({id:'artifact-1'});
|
||||
prisma.orchestrationTask.updateMany.mockResolvedValue({count:1});
|
||||
await expect(service.decideGate('task-1',2,{
|
||||
decision:'APPROVED',expectedVersion:7,idempotencyKey:'failed-run',
|
||||
expiresAt:new Date(Date.now()+10000).toISOString(),runId:'run-1',
|
||||
artifactHash:'f'.repeat(64),commitSha:'abc123',environment:'staging',
|
||||
},'ceo')).rejects.toThrow(/successful run/i);
|
||||
expect(prisma.orchestrationApprovalDecision.create).not.toHaveBeenCalled();
|
||||
expect(prisma.orchestrationTask.updateMany).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('refuses gate 2 when binding does not match the immutable run', async () => {
|
||||
prisma.orchestrationApprovalDecision.findUnique.mockResolvedValue(null);
|
||||
prisma.orchestrationTask.findUnique.mockResolvedValue({ id: "task-1", state: "GATE2_PENDING", version: 7 });
|
||||
prisma.orchestrationRun.findUnique.mockResolvedValue({ id: "run-1", taskId: "task-1", commitSha: "real", environment: "staging" });
|
||||
await expect(service.decideGate("task-1", 2, { decision: "APPROVED", expectedVersion: 7, idempotencyKey: "gate-bad", expiresAt: new Date(Date.now()+10000).toISOString(), runId: "run-1", artifactHash: "f".repeat(64), commitSha: "wrong", environment: "staging" }, "ceo")).rejects.toThrow(ForbiddenException);
|
||||
prisma.orchestrationTask.findUnique.mockResolvedValue({
|
||||
id: 'task-1',
|
||||
state: 'GATE2_PENDING',
|
||||
version: 7,
|
||||
});
|
||||
prisma.orchestrationRun.findUnique.mockResolvedValue({
|
||||
id: 'run-1',
|
||||
taskId: 'task-1',
|
||||
commitSha: 'real',
|
||||
environment: 'staging',
|
||||
});
|
||||
await expect(
|
||||
service.decideGate(
|
||||
'task-1',
|
||||
2,
|
||||
{
|
||||
decision: 'APPROVED',
|
||||
expectedVersion: 7,
|
||||
idempotencyKey: 'gate-bad',
|
||||
expiresAt: new Date(Date.now() + 10000).toISOString(),
|
||||
runId: 'run-1',
|
||||
artifactHash: 'f'.repeat(64),
|
||||
commitSha: 'wrong',
|
||||
environment: 'staging',
|
||||
},
|
||||
'ceo',
|
||||
),
|
||||
).rejects.toThrow(ForbiddenException);
|
||||
});
|
||||
|
||||
it("enforces hard task budget atomically", async () => {
|
||||
prisma.orchestrationTask.findUnique.mockResolvedValue({ id: "task-1", budgetLimitCents: 1000 });
|
||||
prisma.orchestrationBudgetLedger.aggregate.mockResolvedValue({ _sum: { amountCents: 900 } });
|
||||
await expect(service.recordBudget("task-1", { amountCents: 101, idempotencyKey: "cost-1", kind: "MODEL" }, "agent")).rejects.toThrow(ForbiddenException);
|
||||
it('enforces hard task budget atomically', async () => {
|
||||
prisma.orchestrationTask.findUnique.mockResolvedValue({
|
||||
id: 'task-1',
|
||||
budgetLimitCents: 1000,
|
||||
});
|
||||
prisma.orchestrationBudgetLedger.aggregate.mockResolvedValue({
|
||||
_sum: { amountCents: 900 },
|
||||
});
|
||||
await expect(
|
||||
service.recordBudget(
|
||||
'task-1',
|
||||
{ amountCents: 101, idempotencyKey: 'cost-1', kind: 'SPEND' },
|
||||
'agent',
|
||||
),
|
||||
).rejects.toThrow(ForbiddenException);
|
||||
expect(prisma.orchestrationBudgetLedger.create).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -1,17 +1,665 @@
|
||||
import { BadRequestException, ConflictException, ForbiddenException, Injectable, NotFoundException } from "@nestjs/common";
|
||||
import { PrismaService } from "../prisma/prisma.service";
|
||||
import { CreateOrchestrationTaskDto, GateDecisionDto, RecordBudgetDto, TransitionTaskDto } from "./dto/orchestration.dto";
|
||||
import { assertTransition, nextStateForGateDecision, OperationMode, WorkflowState } from "./orchestration.domain";
|
||||
import { KillSwitchService } from "./kill-switch.service";
|
||||
import {
|
||||
BadRequestException,
|
||||
ConflictException,
|
||||
ForbiddenException,
|
||||
Injectable,
|
||||
NotFoundException,
|
||||
} from '@nestjs/common';
|
||||
import { PrismaService } from '../prisma/prisma.service';
|
||||
import {
|
||||
CreateOrchestrationTaskDto,
|
||||
GateDecisionDto,
|
||||
RecordBudgetDto,
|
||||
TransitionTaskDto,
|
||||
} from './dto/orchestration.dto';
|
||||
import {
|
||||
KillSwitchLevel,
|
||||
assertTransition,
|
||||
nextStateForGateDecision,
|
||||
OperationMode,
|
||||
WorkflowState,
|
||||
} from './orchestration.domain';
|
||||
import { ModelTierPolicyService } from './model-tier-policy.service';
|
||||
import { KillSwitchService } from './kill-switch.service';
|
||||
@Injectable()
|
||||
export class OrchestrationService {
|
||||
constructor(private readonly prisma:PrismaService,private readonly killSwitch:KillSwitchService){}
|
||||
async createTask(dto:CreateOrchestrationTaskDto,actorId:string){ await this.killSwitch.assertAllowed("CREATE_TASK"); return (this.prisma as any).$transaction(async(tx:any)=>{const task=await tx.orchestrationTask.create({data:{key:dto.key,title:dto.title,department:dto.department,operationMode:dto.operationMode,budgetLimitCents:dto.budgetLimitCents,metadata:dto.metadata??{},state:WorkflowState.DRAFT,version:1}});await tx.orchestrationEvent.create({data:{taskId:task.id,eventType:"TASK_CREATED",fromState:null,toState:WorkflowState.DRAFT,sequence:1,actorId,payload:{key:dto.key}}});return task;}); }
|
||||
async transition(id:string,dto:TransitionTaskDto,actorId:string){return (this.prisma as any).$transaction(async(tx:any)=>{const existing=await tx.orchestrationTask.findUnique({where:{id}});if(!existing)throw new NotFoundException(`Orchestration task ${id} not found`);assertTransition(existing.state,dto.toState);if(existing.operationMode===OperationMode.MUTATE_PRODUCTION&&dto.toState===WorkflowState.RELEASING)await this.killSwitch.assertAllowed("MUTATE_PRODUCTION");const changed=await tx.orchestrationTask.updateMany({where:{id,version:dto.expectedVersion,state:existing.state},data:{state:dto.toState,version:{increment:1}}});if(changed.count!==1)throw new ConflictException("Task version or state changed; reload before retrying");await tx.orchestrationEvent.create({data:{taskId:id,eventType:"STATE_TRANSITION",fromState:existing.state,toState:dto.toState,sequence:dto.expectedVersion+1,actorId,idempotencyKey:dto.idempotencyKey,payload:{reason:dto.reason??null}}});return {...existing,state:dto.toState,version:dto.expectedVersion+1};});}
|
||||
async decideGate(taskId:string,gate:1|2,dto:GateDecisionDto,actorId:string){if(new Date(dto.expiresAt)<=new Date())throw new BadRequestException("Approval decision has expired");return (this.prisma as any).$transaction(async(tx:any)=>{const prior=await tx.orchestrationApprovalDecision.findUnique({where:{idempotencyKey:dto.idempotencyKey}});if(prior){if(prior.taskId!==taskId||prior.gate!==gate||prior.decision!==dto.decision)throw new ConflictException("Idempotency key already belongs to another approval decision");return prior;}const task=await tx.orchestrationTask.findUnique({where:{id:taskId}});if(!task)throw new NotFoundException(`Orchestration task ${taskId} not found`);const expected=gate===1?WorkflowState.GATE1_PENDING:WorkflowState.GATE2_PENDING;if(task.state!==expected)throw new ConflictException(`Gate ${gate} requires ${expected}`);if(gate===2&&dto.decision==="APPROVED"){if(!dto.runId||!dto.artifactHash||!dto.commitSha||!dto.environment)throw new BadRequestException("Gate 2 requires runId, artifactHash, commitSha and environment");const run=await tx.orchestrationRun.findUnique({where:{id:dto.runId}});if(!run||run.taskId!==taskId||run.commitSha!==dto.commitSha||run.environment!==dto.environment)throw new ForbiddenException("Gate 2 binding does not match immutable run");const artifact=await tx.orchestrationArtifact.findFirst({where:{runId:dto.runId,sha256:dto.artifactHash}});if(!artifact)throw new ForbiddenException("Gate 2 artifact hash is not registered for run");}
|
||||
const next=nextStateForGateDecision(gate,dto.decision);const changed=await tx.orchestrationTask.updateMany({where:{id:taskId,version:dto.expectedVersion,state:expected},data:{state:next,version:{increment:1}}});if(changed.count!==1)throw new ConflictException("Task version or state changed; approval not applied");const decision=await tx.orchestrationApprovalDecision.create({data:{taskId,gate,decision:dto.decision,actorId,notes:dto.notes,expiresAt:new Date(dto.expiresAt),idempotencyKey:dto.idempotencyKey,runId:dto.runId,artifactHash:dto.artifactHash,commitSha:dto.commitSha,environment:dto.environment}});await tx.orchestrationEvent.create({data:{taskId,eventType:`GATE${gate}_${dto.decision}`,fromState:expected,toState:next,sequence:dto.expectedVersion+1,actorId,idempotencyKey:`event:${dto.idempotencyKey}`,payload:{decisionId:decision.id}}});return decision;});}
|
||||
async recordBudget(taskId:string,dto:RecordBudgetDto,actorId:string){return (this.prisma as any).$transaction(async(tx:any)=>{const task=await tx.orchestrationTask.findUnique({where:{id:taskId}});if(!task)throw new NotFoundException(`Orchestration task ${taskId} not found`);const sum=await tx.orchestrationBudgetLedger.aggregate({where:{taskId},_sum:{amountCents:true}});if((sum._sum.amountCents??0)+dto.amountCents>task.budgetLimitCents)throw new ForbiddenException("Hard task budget exceeded");return tx.orchestrationBudgetLedger.create({data:{taskId,amountCents:dto.amountCents,kind:dto.kind,idempotencyKey:dto.idempotencyKey,actorId,metadata:dto.metadata??{}}});});}
|
||||
async findAll(){return (this.prisma as any).orchestrationTask.findMany({orderBy:{createdAt:"desc"}})}
|
||||
async findOne(id:string){const task=await (this.prisma as any).orchestrationTask.findUnique({where:{id},include:{runs:{include:{artifacts:true,attestations:true}},dependencies:true,decisions:true,events:{orderBy:{sequence:"asc"}},budgetLedger:true}});if(!task)throw new NotFoundException(`Orchestration task ${id} not found`);return task;}
|
||||
async setKillSwitch(level:string,reason:string,actorId:string){return (this.prisma as any).$transaction(async(tx:any)=>{const value=await tx.orchestrationKillSwitch.upsert({where:{scope:"GLOBAL"},create:{scope:"GLOBAL",level,active:true,reason,actorId},update:{level,active:true,reason,actorId,version:{increment:1}}});await tx.orchestrationEvent.create({data:{eventType:"KILL_SWITCH_CHANGED",sequence:value.version??1,actorId,payload:{scope:"GLOBAL",level,reason}}});return value;});}
|
||||
constructor(
|
||||
private readonly prisma: PrismaService,
|
||||
private readonly killSwitch: KillSwitchService,
|
||||
) {}
|
||||
async createTask(dto: CreateOrchestrationTaskDto, actorId: string) {
|
||||
await this.killSwitch.assertAllowed('CREATE_TASK');
|
||||
return await (this.prisma as any).$transaction(async (tx: any) => {
|
||||
const task = await tx.orchestrationTask.create({
|
||||
data: {
|
||||
key: dto.key,
|
||||
title: dto.title,
|
||||
department: dto.department,
|
||||
operationMode: dto.operationMode,
|
||||
budgetLimitCents: dto.budgetLimitCents,
|
||||
metadata: dto.metadata ?? {},
|
||||
state: WorkflowState.DRAFT,
|
||||
version: 1,
|
||||
},
|
||||
});
|
||||
await tx.orchestrationEvent.create({
|
||||
data: {
|
||||
taskId: task.id,
|
||||
eventType: 'TASK_CREATED',
|
||||
fromState: null,
|
||||
toState: WorkflowState.DRAFT,
|
||||
sequence: 1,
|
||||
actorId,
|
||||
payload: { key: dto.key },
|
||||
},
|
||||
});
|
||||
return task;
|
||||
});
|
||||
}
|
||||
async transition(id: string, dto: TransitionTaskDto, actorId: string) {
|
||||
if (
|
||||
[
|
||||
WorkflowState.GATE1_APPROVED,
|
||||
WorkflowState.GATE2_APPROVED,
|
||||
WorkflowState.RELEASING,
|
||||
WorkflowState.RUNNING,
|
||||
WorkflowState.VERIFYING,
|
||||
WorkflowState.PRODUCTION_VERIFYING,
|
||||
WorkflowState.LIVE_VERIFIED,
|
||||
WorkflowState.MEASURING,
|
||||
WorkflowState.CLOSED,
|
||||
].includes(dto.toState)
|
||||
)
|
||||
throw new ForbiddenException(
|
||||
'Gate approvals and release require dedicated commands',
|
||||
);
|
||||
return await (this.prisma as any).$transaction(async (tx: any) => {
|
||||
await tx.$queryRaw`SELECT id FROM orchestration_tasks WHERE id = ${id}::uuid FOR UPDATE`;
|
||||
const fingerprint = JSON.stringify({ id, dto, actorId });
|
||||
const replay = await tx.orchestrationEvent.findUnique({
|
||||
where: { idempotencyKey: dto.idempotencyKey },
|
||||
});
|
||||
if (replay) {
|
||||
if (replay.payload.fingerprint !== fingerprint)
|
||||
throw new ConflictException('Idempotency payload mismatch');
|
||||
return replay.payload.result;
|
||||
}
|
||||
const existing = await tx.orchestrationTask.findUnique({ where: { id } });
|
||||
if (!existing)
|
||||
throw new NotFoundException(`Orchestration task ${id} not found`);
|
||||
if (existing.state === WorkflowState.FAILED)
|
||||
throw new ForbiddenException('Retries require dedicated run command');
|
||||
assertTransition(existing.state, dto.toState);
|
||||
await this.killSwitch.assertAllowed(
|
||||
[
|
||||
WorkflowState.READY,
|
||||
WorkflowState.CLAIMED,
|
||||
WorkflowState.RUNNING,
|
||||
WorkflowState.VERIFYING,
|
||||
WorkflowState.GATE2_PENDING,
|
||||
].includes(dto.toState)
|
||||
? 'RUN_AGENT'
|
||||
: 'OBSERVE',
|
||||
);
|
||||
if ([WorkflowState.READY, WorkflowState.CLAIMED].includes(dto.toState))
|
||||
await this.killSwitch.assertAllowed('CLAIM_TASK');
|
||||
let selectedTier = existing.modelTier;
|
||||
if (dto.toState === WorkflowState.READY) {
|
||||
const dependencies = await tx.orchestrationDependency.findMany({
|
||||
where: { taskId: id },
|
||||
include: { dependsOnTask: true },
|
||||
});
|
||||
if (dependencies.some((d: any) => d.dependsOnTask.state !== 'CLOSED'))
|
||||
throw new ForbiddenException('Unresolved dependency');
|
||||
const costs = await tx.orchestrationBudgetLedger.aggregate({
|
||||
where: { taskId: id },
|
||||
_sum: { amountCents: true },
|
||||
});
|
||||
selectedTier = new ModelTierPolicyService().select({
|
||||
complexity: Number(existing.metadata?.complexity ?? 0),
|
||||
risk: Number(existing.metadata?.risk ?? 0),
|
||||
operationMode: existing.operationMode,
|
||||
remainingBudgetCents:
|
||||
existing.budgetLimitCents - (costs._sum.amountCents ?? 0),
|
||||
});
|
||||
const approval = await tx.orchestrationApprovalDecision.findFirst({
|
||||
where: { taskId: id, gate: 1, decision: 'APPROVED' },
|
||||
orderBy: { createdAt: 'desc' },
|
||||
});
|
||||
if (!approval || approval.expiresAt <= new Date())
|
||||
throw new ForbiddenException('Gate 1 approval missing or expired');
|
||||
}
|
||||
if (
|
||||
existing.operationMode === OperationMode.MUTATE_PRODUCTION &&
|
||||
dto.toState === WorkflowState.RELEASING
|
||||
)
|
||||
await this.killSwitch.assertAllowed('MUTATE_PRODUCTION');
|
||||
const changed = await tx.orchestrationTask.updateMany({
|
||||
where: { id, version: dto.expectedVersion, state: existing.state },
|
||||
data: {
|
||||
state: dto.toState,
|
||||
modelTier: selectedTier,
|
||||
version: { increment: 1 },
|
||||
},
|
||||
});
|
||||
if (changed.count !== 1)
|
||||
throw new ConflictException(
|
||||
'Task version or state changed; reload before retrying',
|
||||
);
|
||||
await tx.orchestrationEvent.create({
|
||||
data: {
|
||||
taskId: id,
|
||||
eventType: 'STATE_TRANSITION',
|
||||
fromState: existing.state,
|
||||
toState: dto.toState,
|
||||
sequence: dto.expectedVersion + 1,
|
||||
actorId,
|
||||
idempotencyKey: dto.idempotencyKey,
|
||||
payload: {
|
||||
fingerprint,
|
||||
result: JSON.parse(
|
||||
JSON.stringify({
|
||||
...existing,
|
||||
state: dto.toState,
|
||||
modelTier: selectedTier,
|
||||
version: dto.expectedVersion + 1,
|
||||
}),
|
||||
),
|
||||
},
|
||||
},
|
||||
});
|
||||
return JSON.parse(
|
||||
JSON.stringify({
|
||||
...existing,
|
||||
state: dto.toState,
|
||||
modelTier: selectedTier,
|
||||
version: dto.expectedVersion + 1,
|
||||
}),
|
||||
);
|
||||
});
|
||||
}
|
||||
async decideGate(
|
||||
taskId: string,
|
||||
gate: 1 | 2,
|
||||
dto: GateDecisionDto,
|
||||
actorId: string,
|
||||
) {
|
||||
if (![1, 2].includes(gate)) throw new BadRequestException('Invalid gate');
|
||||
return await (this.prisma as any).$transaction(async (tx: any) => {
|
||||
await tx.$queryRaw`SELECT id FROM orchestration_tasks WHERE id = ${taskId}::uuid FOR UPDATE`;
|
||||
const fingerprint = JSON.stringify({ taskId, gate, dto, actorId });
|
||||
const prior = await tx.orchestrationApprovalDecision.findUnique({
|
||||
where: { idempotencyKey: dto.idempotencyKey },
|
||||
});
|
||||
if (prior) {
|
||||
const event = await tx.orchestrationEvent.findUnique({
|
||||
where: { idempotencyKey: `event:${dto.idempotencyKey}` },
|
||||
});
|
||||
if (event?.payload?.fingerprint !== fingerprint)
|
||||
throw new ConflictException('Idempotency payload mismatch');
|
||||
return prior;
|
||||
}
|
||||
if (
|
||||
!Number.isFinite(Date.parse(dto.expiresAt)) ||
|
||||
new Date(dto.expiresAt) <= new Date()
|
||||
)
|
||||
throw new BadRequestException('Approval decision has expired');
|
||||
await this.killSwitch.assertAllowed('OBSERVE');
|
||||
const task = await tx.orchestrationTask.findUnique({
|
||||
where: { id: taskId },
|
||||
});
|
||||
if (!task)
|
||||
throw new NotFoundException(`Orchestration task ${taskId} not found`);
|
||||
const expected =
|
||||
gate === 1 ? WorkflowState.GATE1_PENDING : WorkflowState.GATE2_PENDING;
|
||||
if (task.state !== expected)
|
||||
throw new ConflictException(`Gate ${gate} requires ${expected}`);
|
||||
if (gate === 2 && dto.decision === 'APPROVED') {
|
||||
if (
|
||||
!dto.runId ||
|
||||
!dto.artifactHash ||
|
||||
!dto.commitSha ||
|
||||
!dto.environment
|
||||
)
|
||||
throw new BadRequestException(
|
||||
'Gate 2 requires runId, artifactHash, commitSha and environment',
|
||||
);
|
||||
const run = await tx.orchestrationRun.findUnique({
|
||||
where: { id: dto.runId },
|
||||
});
|
||||
if (
|
||||
!run ||
|
||||
run.taskId !== taskId ||
|
||||
run.commitSha !== dto.commitSha ||
|
||||
run.environment !== dto.environment
|
||||
)
|
||||
throw new ForbiddenException(
|
||||
'Gate 2 binding does not match immutable run',
|
||||
);
|
||||
if (run.status !== 'SUCCEEDED')
|
||||
throw new ForbiddenException('Gate 2 requires a successful run');
|
||||
const artifact = await tx.orchestrationArtifact.findFirst({
|
||||
where: { runId: dto.runId, sha256: dto.artifactHash },
|
||||
});
|
||||
if (!artifact)
|
||||
throw new ForbiddenException(
|
||||
'Gate 2 artifact hash is not registered for run',
|
||||
);
|
||||
}
|
||||
const next = nextStateForGateDecision(gate, dto.decision);
|
||||
const changed = await tx.orchestrationTask.updateMany({
|
||||
where: { id: taskId, version: dto.expectedVersion, state: expected },
|
||||
data: { state: next, version: { increment: 1 } },
|
||||
});
|
||||
if (changed.count !== 1)
|
||||
throw new ConflictException(
|
||||
'Task version or state changed; approval not applied',
|
||||
);
|
||||
const decision = await tx.orchestrationApprovalDecision.create({
|
||||
data: {
|
||||
taskId,
|
||||
gate,
|
||||
decision: dto.decision,
|
||||
actorId,
|
||||
notes: dto.notes,
|
||||
expiresAt: new Date(dto.expiresAt),
|
||||
idempotencyKey: dto.idempotencyKey,
|
||||
runId: dto.runId,
|
||||
artifactHash: dto.artifactHash,
|
||||
commitSha: dto.commitSha,
|
||||
environment: dto.environment,
|
||||
},
|
||||
});
|
||||
await tx.orchestrationEvent.create({
|
||||
data: {
|
||||
taskId,
|
||||
eventType: `GATE${gate}_${dto.decision}`,
|
||||
fromState: expected,
|
||||
toState: next,
|
||||
sequence: dto.expectedVersion + 1,
|
||||
actorId,
|
||||
idempotencyKey: `event:${dto.idempotencyKey}`,
|
||||
payload: { decisionId: decision.id, fingerprint },
|
||||
},
|
||||
});
|
||||
return decision;
|
||||
});
|
||||
}
|
||||
async recordBudget(taskId: string, dto: RecordBudgetDto, actorId: string) {
|
||||
if (
|
||||
!Number.isSafeInteger(dto.amountCents) ||
|
||||
dto.amountCents < 1 ||
|
||||
!['RESERVE', 'SPEND'].includes(dto.kind)
|
||||
)
|
||||
throw new BadRequestException(
|
||||
'Use RESERVE or SPEND with positive integer cents',
|
||||
);
|
||||
return await (this.prisma as any).$transaction(async (tx: any) => {
|
||||
await tx.$queryRaw`SELECT id FROM orchestration_tasks WHERE id = ${taskId}::uuid FOR UPDATE`;
|
||||
const prior = await tx.orchestrationBudgetLedger.findUnique({
|
||||
where: { idempotencyKey: dto.idempotencyKey },
|
||||
});
|
||||
if (prior) {
|
||||
if (
|
||||
prior.taskId !== taskId ||
|
||||
prior.kind !== dto.kind ||
|
||||
prior.amountCents !== dto.amountCents ||
|
||||
prior.actorId !== actorId ||
|
||||
JSON.stringify(prior.metadata) !== JSON.stringify(dto.metadata ?? {})
|
||||
)
|
||||
throw new ConflictException('Idempotency payload mismatch');
|
||||
return prior;
|
||||
}
|
||||
await this.killSwitch.assertAllowed('RUN_AGENT');
|
||||
const task = await tx.orchestrationTask.findUnique({
|
||||
where: { id: taskId },
|
||||
});
|
||||
if (!task)
|
||||
throw new NotFoundException(`Orchestration task ${taskId} not found`);
|
||||
const sum = await tx.orchestrationBudgetLedger.aggregate({
|
||||
where: { taskId },
|
||||
_sum: { amountCents: true },
|
||||
});
|
||||
if ((sum._sum.amountCents ?? 0) + dto.amountCents > task.budgetLimitCents)
|
||||
throw new ForbiddenException('Hard task budget exceeded');
|
||||
return tx.orchestrationBudgetLedger.create({
|
||||
data: {
|
||||
taskId,
|
||||
amountCents: dto.amountCents,
|
||||
kind: dto.kind,
|
||||
idempotencyKey: dto.idempotencyKey,
|
||||
actorId,
|
||||
metadata: dto.metadata ?? {},
|
||||
},
|
||||
});
|
||||
});
|
||||
}
|
||||
async command(taskId: string, command: string, dto: any, actorId: string) {
|
||||
return await (this.prisma as any).$transaction(async (tx: any) => {
|
||||
await tx.$queryRaw`SELECT id FROM orchestration_tasks WHERE id = ${taskId}::uuid FOR UPDATE`;
|
||||
const fingerprint = JSON.stringify({ taskId, command, dto, actorId });
|
||||
const replay = await tx.orchestrationEvent.findUnique({
|
||||
where: { idempotencyKey: dto.idempotencyKey },
|
||||
});
|
||||
if (replay) {
|
||||
if (replay.payload.fingerprint !== fingerprint)
|
||||
throw new ConflictException('Idempotency payload mismatch');
|
||||
return replay.payload.result;
|
||||
}
|
||||
const task = await tx.orchestrationTask.findUnique({
|
||||
where: { id: taskId },
|
||||
});
|
||||
if (!task) throw new NotFoundException('Task not found');
|
||||
if (task.version !== dto.expectedVersion)
|
||||
throw new ConflictException('Stale task version');
|
||||
await this.killSwitch.assertAllowed('RUN_AGENT');
|
||||
let result: any;
|
||||
let state = task.state;
|
||||
if (command === 'RUN_CREATE') {
|
||||
if (
|
||||
!/^[a-f0-9]{40,64}$/.test(dto.commitSha) ||
|
||||
!['development', 'staging', 'production'].includes(dto.environment)
|
||||
)
|
||||
throw new BadRequestException('Invalid immutable run binding');
|
||||
const last = await tx.orchestrationRun.findFirst({
|
||||
where: { taskId },
|
||||
orderBy: { attempt: 'desc' },
|
||||
});
|
||||
if (
|
||||
last &&
|
||||
(last.status !== 'FAILED' ||
|
||||
!['TRANSIENT', 'RATE_LIMITED', 'DEPENDENCY'].includes(
|
||||
last.retryClass,
|
||||
) ||
|
||||
last.attempt >= 3)
|
||||
)
|
||||
throw new ForbiddenException('Retry denied or exhausted');
|
||||
if (!last && task.state !== 'CLAIMED')
|
||||
throw new ConflictException('Run requires CLAIMED task');
|
||||
result = await tx.orchestrationRun.create({
|
||||
data: {
|
||||
taskId,
|
||||
attempt: (last?.attempt ?? 0) + 1,
|
||||
status: 'RUNNING',
|
||||
commitSha: dto.commitSha,
|
||||
environment: dto.environment,
|
||||
idempotencyKey: dto.idempotencyKey,
|
||||
startedAt: new Date(),
|
||||
},
|
||||
});
|
||||
state = 'RUNNING';
|
||||
} else if (command === 'RUN_FINISH') {
|
||||
const run = await tx.orchestrationRun.findUnique({
|
||||
where: { id: dto.runId },
|
||||
});
|
||||
if (
|
||||
!run ||
|
||||
run.taskId !== taskId ||
|
||||
run.status !== 'RUNNING' ||
|
||||
task.state !== 'RUNNING'
|
||||
)
|
||||
throw new ConflictException('Active run required');
|
||||
if (
|
||||
!['SUCCEEDED', 'FAILED'].includes(dto.status) ||
|
||||
(dto.status === 'FAILED' &&
|
||||
![
|
||||
'TRANSIENT',
|
||||
'RATE_LIMITED',
|
||||
'DEPENDENCY',
|
||||
'VALIDATION',
|
||||
'POLICY',
|
||||
'PERMANENT',
|
||||
].includes(dto.retryClass))
|
||||
)
|
||||
throw new BadRequestException('Invalid outcome taxonomy');
|
||||
result = await tx.orchestrationRun.update({
|
||||
where: { id: run.id },
|
||||
data: {
|
||||
status: dto.status,
|
||||
retryClass: dto.status === 'FAILED' ? dto.retryClass : null,
|
||||
finishedAt: new Date(),
|
||||
},
|
||||
});
|
||||
state = dto.status === 'FAILED' ? 'FAILED' : 'VERIFYING';
|
||||
} else if (command === 'ARTIFACT_ADD' || command === 'ATTESTATION_ADD') {
|
||||
const run = await tx.orchestrationRun.findUnique({
|
||||
where: { id: dto.runId },
|
||||
});
|
||||
if (
|
||||
!run ||
|
||||
run.taskId !== taskId ||
|
||||
!['RUNNING', 'VERIFYING'].includes(task.state)
|
||||
)
|
||||
throw new ConflictException('Owned active run required');
|
||||
const hash = command === 'ARTIFACT_ADD' ? dto.sha256 : dto.subjectHash;
|
||||
if (!/^[a-f0-9]{64}$/.test(hash) || !dto.kind)
|
||||
throw new BadRequestException('Invalid artifact hash or kind');
|
||||
if (command === 'ARTIFACT_ADD') {
|
||||
if (!/^https:\/\//.test(dto.uri))
|
||||
throw new BadRequestException('Artifact URI must be HTTPS');
|
||||
result = await tx.orchestrationArtifact.create({
|
||||
data: { runId: run.id, kind: dto.kind, sha256: hash, uri: dto.uri },
|
||||
});
|
||||
} else {
|
||||
if (
|
||||
!(await tx.orchestrationArtifact.findFirst({
|
||||
where: { runId: run.id, sha256: hash },
|
||||
}))
|
||||
)
|
||||
throw new ForbiddenException(
|
||||
'Attestation subject is not a registered artifact',
|
||||
);
|
||||
result = await tx.orchestrationAttestation.create({
|
||||
data: {
|
||||
runId: run.id,
|
||||
kind: dto.kind,
|
||||
subjectHash: hash,
|
||||
predicate: dto.predicate ?? {},
|
||||
signer: actorId,
|
||||
signature: dto.signature,
|
||||
},
|
||||
});
|
||||
}
|
||||
} else if (command === 'DEPENDENCY_ADD') {
|
||||
if (task.state !== 'DRAFT')
|
||||
throw new ConflictException(
|
||||
'Dependencies are frozen after submission',
|
||||
);
|
||||
await tx.$executeRaw`LOCK TABLE orchestration_dependencies IN EXCLUSIVE MODE`;
|
||||
const cycles =
|
||||
await tx.$queryRaw`WITH RECURSIVE chain(id) AS (SELECT ${dto.dependsOnTaskId}::uuid UNION SELECT d.depends_on_task_id FROM orchestration_dependencies d JOIN chain c ON d.task_id=c.id) SELECT id FROM chain WHERE id=${taskId}::uuid`;
|
||||
if (cycles.length) throw new ForbiddenException('Dependency cycle');
|
||||
result = await tx.orchestrationDependency.create({
|
||||
data: { taskId, dependsOnTaskId: dto.dependsOnTaskId },
|
||||
});
|
||||
} else if (command === 'RELEASE_AUTHORIZE') {
|
||||
await this.killSwitch.assertAllowed('MUTATE_PRODUCTION');
|
||||
if (task.state !== 'GATE2_APPROVED')
|
||||
throw new ConflictException(
|
||||
'Gate 2 approval required; already consumed or invalid state',
|
||||
);
|
||||
const approval = await tx.orchestrationApprovalDecision.findUnique({
|
||||
where: { id: dto.approvalId },
|
||||
});
|
||||
if (
|
||||
!approval ||
|
||||
approval.taskId !== taskId ||
|
||||
approval.gate !== 2 ||
|
||||
approval.decision !== 'APPROVED' ||
|
||||
approval.expiresAt <= new Date()
|
||||
)
|
||||
throw new ForbiddenException('Gate 2 missing or expired');
|
||||
for (const key of ['runId', 'artifactHash', 'commitSha', 'environment'])
|
||||
if (approval[key] !== dto[key])
|
||||
throw new ForbiddenException('Release binding mismatch');
|
||||
const run = await tx.orchestrationRun.findUnique({
|
||||
where: { id: dto.runId },
|
||||
});
|
||||
if (
|
||||
!run ||
|
||||
run.taskId !== taskId ||
|
||||
run.status !== 'SUCCEEDED' ||
|
||||
run.commitSha !== dto.commitSha ||
|
||||
run.environment !== dto.environment
|
||||
)
|
||||
throw new ForbiddenException('Run binding mismatch');
|
||||
if (
|
||||
!(await tx.orchestrationArtifact.findFirst({
|
||||
where: { runId: run.id, sha256: dto.artifactHash },
|
||||
})) ||
|
||||
!(await tx.orchestrationAttestation.findFirst({
|
||||
where: {
|
||||
runId: run.id,
|
||||
subjectHash: dto.artifactHash,
|
||||
kind: 'TEST',
|
||||
},
|
||||
}))
|
||||
)
|
||||
throw new ForbiddenException('Artifact evidence missing');
|
||||
result = {
|
||||
authorized: true,
|
||||
approvalId: approval.id,
|
||||
runId: run.id,
|
||||
artifactHash: dto.artifactHash,
|
||||
commitSha: dto.commitSha,
|
||||
environment: dto.environment,
|
||||
execution: 'CI_CD_ONLY',
|
||||
};
|
||||
state = 'RELEASING';
|
||||
} else throw new BadRequestException('Unknown command');
|
||||
await tx.orchestrationTask.update({
|
||||
where: { id: taskId },
|
||||
data: { state, version: { increment: 1 } },
|
||||
});
|
||||
const serialized = JSON.parse(JSON.stringify(result));
|
||||
await tx.orchestrationEvent.create({
|
||||
data: {
|
||||
taskId,
|
||||
eventType: command,
|
||||
fromState: task.state,
|
||||
toState: state,
|
||||
sequence: task.version + 1,
|
||||
actorId,
|
||||
idempotencyKey: dto.idempotencyKey,
|
||||
payload: { fingerprint, result: serialized },
|
||||
},
|
||||
});
|
||||
return serialized;
|
||||
});
|
||||
}
|
||||
async reconcileBudget(
|
||||
taskId: string,
|
||||
dto: { reservationId: string; actualCents: number; idempotencyKey: string },
|
||||
actorId: string,
|
||||
) {
|
||||
await this.killSwitch.assertAllowed('OBSERVE');
|
||||
if (!Number.isSafeInteger(dto.actualCents) || dto.actualCents < 0)
|
||||
throw new BadRequestException('Invalid actual cost');
|
||||
return await (this.prisma as any).$transaction(async (tx: any) => {
|
||||
await tx.$queryRaw`SELECT id FROM orchestration_tasks WHERE id = ${taskId}::uuid FOR UPDATE`;
|
||||
const prior = await tx.orchestrationBudgetLedger.findUnique({
|
||||
where: { idempotencyKey: dto.idempotencyKey },
|
||||
});
|
||||
if (prior) {
|
||||
if (
|
||||
prior.taskId !== taskId ||
|
||||
prior.actorId !== actorId ||
|
||||
prior.metadata.reservationId !== dto.reservationId ||
|
||||
prior.metadata.actualCents !== dto.actualCents
|
||||
)
|
||||
throw new ConflictException('Idempotency payload mismatch');
|
||||
return prior;
|
||||
}
|
||||
const reservation = await tx.orchestrationBudgetLedger.findUnique({
|
||||
where: { id: dto.reservationId },
|
||||
});
|
||||
if (
|
||||
!reservation ||
|
||||
reservation.taskId !== taskId ||
|
||||
reservation.kind !== 'RESERVE'
|
||||
)
|
||||
throw new BadRequestException('Reservation not found');
|
||||
if (
|
||||
await tx.orchestrationBudgetLedger.findFirst({
|
||||
where: {
|
||||
taskId,
|
||||
kind: 'RECONCILE',
|
||||
metadata: { path: ['reservationId'], equals: dto.reservationId },
|
||||
},
|
||||
})
|
||||
)
|
||||
throw new ConflictException('Reservation already reconciled');
|
||||
const task = await tx.orchestrationTask.findUnique({
|
||||
where: { id: taskId },
|
||||
});
|
||||
const total = await tx.orchestrationBudgetLedger.aggregate({
|
||||
where: { taskId },
|
||||
_sum: { amountCents: true },
|
||||
});
|
||||
const delta = dto.actualCents - reservation.amountCents;
|
||||
if (total._sum.amountCents + delta > task.budgetLimitCents)
|
||||
throw new ForbiddenException('Hard task budget exceeded');
|
||||
return tx.orchestrationBudgetLedger.create({
|
||||
data: {
|
||||
taskId,
|
||||
kind: 'RECONCILE',
|
||||
amountCents: delta,
|
||||
idempotencyKey: dto.idempotencyKey,
|
||||
actorId,
|
||||
metadata: {
|
||||
reservationId: dto.reservationId,
|
||||
actualCents: dto.actualCents,
|
||||
},
|
||||
},
|
||||
});
|
||||
});
|
||||
}
|
||||
async findAll() {
|
||||
await this.killSwitch.assertAllowed('OBSERVE');
|
||||
return (this.prisma as any).orchestrationTask.findMany({
|
||||
orderBy: { createdAt: 'desc' },
|
||||
});
|
||||
}
|
||||
async findOne(id: string) {
|
||||
await this.killSwitch.assertAllowed('OBSERVE');
|
||||
const task = await (this.prisma as any).orchestrationTask.findUnique({
|
||||
where: { id },
|
||||
include: {
|
||||
runs: { include: { artifacts: true, attestations: true } },
|
||||
dependencies: true,
|
||||
decisions: true,
|
||||
events: { orderBy: { sequence: 'asc' } },
|
||||
budgetLedger: true,
|
||||
},
|
||||
});
|
||||
if (!task)
|
||||
throw new NotFoundException(`Orchestration task ${id} not found`);
|
||||
return task;
|
||||
}
|
||||
async setKillSwitch(level: string, reason: string, actorId: string) {
|
||||
if (!Object.values(KillSwitchLevel).includes(level as KillSwitchLevel))
|
||||
throw new BadRequestException('Invalid kill switch level');
|
||||
return await (this.prisma as any).$transaction(async (tx: any) => {
|
||||
const value = await tx.orchestrationKillSwitch.upsert({
|
||||
where: { scope: 'GLOBAL' },
|
||||
create: { scope: 'GLOBAL', level, active: true, reason, actorId },
|
||||
update: {
|
||||
level,
|
||||
active: true,
|
||||
reason,
|
||||
actorId,
|
||||
version: { increment: 1 },
|
||||
},
|
||||
});
|
||||
await tx.orchestrationEvent.create({
|
||||
data: {
|
||||
eventType: 'KILL_SWITCH_CHANGED',
|
||||
sequence: value.version ?? 1,
|
||||
actorId,
|
||||
payload: { scope: 'GLOBAL', level, reason },
|
||||
},
|
||||
});
|
||||
return value;
|
||||
});
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user