From 0952a482fbec8ffd4ef5a6833f25ddb819892ac2 Mon Sep 17 00:00:00 2001 From: devops Date: Fri, 23 Oct 2026 08:01:08 +0000 Subject: [PATCH] Checkpoint recovered orchestration core with tested fail-closed boundaries --- .../migration.sql | 10 + .../admin/legacy-orchestration.http.spec.ts | 2 +- .../hermes-management/hermes.adapter.spec.ts | 6 + .../src/hermes-management/hermes.adapter.ts | 4 +- .../orchestration/dto/orchestration.dto.ts | 98 ++- .../orchestration/kill-switch.service.spec.ts | 47 +- .../src/orchestration/kill-switch.service.ts | 39 +- .../orchestration/migration-safety.spec.ts | 21 +- .../model-tier-policy.service.spec.ts | 69 +- .../model-tier-policy.service.ts | 47 +- .../orchestration-read.http.spec.ts | 6 + .../orchestration/orchestration.controller.ts | 6 + .../orchestration.domain.spec.ts | 53 +- .../src/orchestration/orchestration.domain.ts | 110 ++- .../orchestration.integration.spec.ts | 577 +++++++++++++++ .../src/orchestration/orchestration.module.ts | 15 +- .../orchestration.service.spec.ts | 261 ++++++- .../orchestration/orchestration.service.ts | 676 +++++++++++++++++- 18 files changed, 1918 insertions(+), 129 deletions(-) create mode 100644 backend/prisma/migrations/20261022190000_harden_orchestration_ledger/migration.sql create mode 100644 backend/src/orchestration/orchestration.integration.spec.ts diff --git a/backend/prisma/migrations/20261022190000_harden_orchestration_ledger/migration.sql b/backend/prisma/migrations/20261022190000_harden_orchestration_ledger/migration.sql new file mode 100644 index 00000000..e2ac0411 --- /dev/null +++ b/backend/prisma/migrations/20261022190000_harden_orchestration_ledger/migration.sql @@ -0,0 +1,10 @@ +-- Immutable evidence and accounting; corrections are new entries. +CREATE FUNCTION orchestration_append_only() RETURNS trigger LANGUAGE plpgsql AS $$ +BEGIN RAISE EXCEPTION 'Orchestration append-only record cannot be changed'; END; +$$; +CREATE TRIGGER orchestration_budget_immutable BEFORE UPDATE OR DELETE ON orchestration_budget_ledger FOR EACH ROW EXECUTE FUNCTION orchestration_append_only(); +CREATE TRIGGER orchestration_events_immutable BEFORE UPDATE OR DELETE ON orchestration_events FOR EACH ROW EXECUTE FUNCTION orchestration_append_only(); +CREATE TRIGGER orchestration_approvals_immutable BEFORE UPDATE OR DELETE ON orchestration_approval_decisions FOR EACH ROW EXECUTE FUNCTION orchestration_append_only(); +CREATE TRIGGER orchestration_artifacts_immutable BEFORE UPDATE OR DELETE ON orchestration_artifacts FOR EACH ROW EXECUTE FUNCTION orchestration_append_only(); +CREATE TRIGGER orchestration_attestations_immutable BEFORE UPDATE OR DELETE ON orchestration_attestations FOR EACH ROW EXECUTE FUNCTION orchestration_append_only(); +ALTER TABLE orchestration_tasks ADD CONSTRAINT orchestration_budget_nonnegative CHECK (budget_limit_cents >= 0); diff --git a/backend/src/admin/legacy-orchestration.http.spec.ts b/backend/src/admin/legacy-orchestration.http.spec.ts index af779f33..953d24d3 100644 --- a/backend/src/admin/legacy-orchestration.http.spec.ts +++ b/backend/src/admin/legacy-orchestration.http.spec.ts @@ -37,7 +37,7 @@ describe('retired legacy orchestration HTTP boundary', () => { dbAccess.mockClear(); }); beforeEach(() => dbAccess.mockClear()); - afterAll(async () => app?.close()); + afterAll(async () => { if (app) await app.close(); }); it.each(routes)('%s %s returns explicit 410 for authorized callers before database access', async (method, path) => { const response = await request(app.getHttpServer())[method](path) .set('Authorization', 'Admin').send({decision:'APPROVED',status:'DONE'}).expect(410); diff --git a/backend/src/hermes-management/hermes.adapter.spec.ts b/backend/src/hermes-management/hermes.adapter.spec.ts index cffce5af..0945dd9c 100644 --- a/backend/src/hermes-management/hermes.adapter.spec.ts +++ b/backend/src/hermes-management/hermes.adapter.spec.ts @@ -30,6 +30,12 @@ describe('installed Hermes health contract', () => { ); expect(fetcher.mock.calls[0][1].redirect).toBe('error'); }); + it('refuses a runtime reporting that authentication is disabled', async () => { + const adapter = new HermesAdapter({origin:'http://canina-hermes:9119',token:async()=> 'private'}, + jest.fn().mockResolvedValue(new Response(JSON.stringify({...health,auth_required:false})))); + await expect(adapter.health()).rejects.toMatchObject({code:'AUTH_REQUIRED'}); + }); + it('fails closed for bad origin, auth errors, malformed contract and oversized bodies', async () => { expect( () => diff --git a/backend/src/hermes-management/hermes.adapter.ts b/backend/src/hermes-management/hermes.adapter.ts index df19b69e..842aae2d 100644 --- a/backend/src/hermes-management/hermes.adapter.ts +++ b/backend/src/hermes-management/hermes.adapter.ts @@ -132,12 +132,14 @@ export class HermesAdapter { } } async health(): Promise { - return fields(await this.request('/api/health'), { + const health = fields(await this.request('/api/health'), { ok: 'boolean', version: 'string', commit: 'string', auth_required: 'boolean', }) as unknown as Health; + if (!health.auth_required) throw new HermesError('AUTH_REQUIRED'); + return health; } async status(profile: string): Promise { const raw = object(await this.request('/api/status', profile)); diff --git a/backend/src/orchestration/dto/orchestration.dto.ts b/backend/src/orchestration/dto/orchestration.dto.ts index 96df2f19..1bca5f57 100644 --- a/backend/src/orchestration/dto/orchestration.dto.ts +++ b/backend/src/orchestration/dto/orchestration.dto.ts @@ -1,8 +1,90 @@ -import { Type } from "class-transformer"; -import { IsEnum, IsHash, IsIn, IsInt, IsISO8601, IsNotEmpty, IsOptional, IsString, Max, Min } from "class-validator"; -import { OperationMode, WorkflowState } from "../orchestration.domain"; -export class CreateOrchestrationTaskDto { @IsString() @IsNotEmpty() key!:string; @IsString() @IsNotEmpty() title!:string; @IsString() @IsNotEmpty() department!:string; @IsEnum(OperationMode) operationMode!:OperationMode; @Type(()=>Number) @IsInt() @Min(0) budgetLimitCents!:number; @IsOptional() metadata?:Record; } -export class TransitionTaskDto { @IsEnum(WorkflowState) toState!:WorkflowState; @Type(()=>Number) @IsInt() @Min(1) expectedVersion!:number; @IsString() @IsNotEmpty() idempotencyKey!:string; @IsOptional() @IsString() reason?:string; } -export class GateDecisionDto { @IsIn(["APPROVED","REJECTED"]) decision!:"APPROVED"|"REJECTED"; @Type(()=>Number) @IsInt() @Min(1) expectedVersion!:number; @IsString() @IsNotEmpty() idempotencyKey!:string; @IsISO8601() expiresAt!:string; @IsOptional() @IsString() notes?:string; @IsOptional() @IsString() runId?:string; @IsOptional() @IsHash("sha256") artifactHash?:string; @IsOptional() @IsString() commitSha?:string; @IsOptional() @IsString() environment?:string; } -export class RecordBudgetDto { @Type(()=>Number) @IsInt() @Min(1) amountCents!:number; @IsString() @IsNotEmpty() idempotencyKey!:string; @IsString() @IsNotEmpty() kind!:string; @IsOptional() metadata?:Record; } -export class SetKillSwitchDto { @IsIn(["FREEZE_NEW_WORK","STOP_AGENTS","DISABLE_MUTATIONS","EMERGENCY_HALT"]) level!:string; @IsString() @IsNotEmpty() reason!:string; } +import { Type } from 'class-transformer'; +import { + IsEnum, + IsHash, + IsIn, + IsInt, + IsISO8601, + IsNotEmpty, + IsOptional, + IsString, + Max, + Min, +} from 'class-validator'; +import { OperationMode, WorkflowState } from '../orchestration.domain'; +export class CreateOrchestrationTaskDto { + @IsString() @IsNotEmpty() key!: string; + @IsString() @IsNotEmpty() title!: string; + @IsString() @IsNotEmpty() department!: string; + @IsEnum(OperationMode) operationMode!: OperationMode; + @Type(() => Number) @IsInt() @Min(0) budgetLimitCents!: number; + @IsOptional() metadata?: Record; +} +export class TransitionTaskDto { + @IsEnum(WorkflowState) toState!: WorkflowState; + @Type(() => Number) @IsInt() @Min(1) expectedVersion!: number; + @IsString() @IsNotEmpty() idempotencyKey!: string; + @IsOptional() @IsString() reason?: string; +} +export class GateDecisionDto { + @IsIn(['APPROVED', 'REJECTED']) decision!: 'APPROVED' | 'REJECTED'; + @Type(() => Number) @IsInt() @Min(1) expectedVersion!: number; + @IsString() @IsNotEmpty() idempotencyKey!: string; + @IsISO8601() expiresAt!: string; + @IsOptional() @IsString() notes?: string; + @IsOptional() @IsString() runId?: string; + @IsOptional() @IsHash('sha256') artifactHash?: string; + @IsOptional() @IsString() commitSha?: string; + @IsOptional() @IsString() environment?: string; +} +export class RecordBudgetDto { + @Type(() => Number) @IsInt() @Min(1) amountCents!: number; + @IsString() @IsNotEmpty() idempotencyKey!: string; + @IsString() @IsNotEmpty() kind!: string; + @IsOptional() metadata?: Record; +} +export class SetKillSwitchDto { + @IsIn([ + 'FREEZE_NEW_WORK', + 'STOP_AGENTS', + 'DISABLE_MUTATIONS', + 'EMERGENCY_HALT', + ]) + level!: string; + @IsString() @IsNotEmpty() reason!: string; +} + +export class ReconcileBudgetDto { + @IsString() @IsNotEmpty() reservationId!: string; + @Type(() => Number) @IsInt() @Min(0) actualCents!: number; + @IsString() @IsNotEmpty() idempotencyKey!: string; +} +export class OrchestrationCommandDto { + @Type(() => Number) @IsInt() @Min(1) expectedVersion!: number; + @IsString() @IsNotEmpty() idempotencyKey!: string; + @IsOptional() @IsString() commitSha?: string; + @IsOptional() + @IsIn(['development', 'staging', 'production']) + environment?: string; + @IsOptional() @IsString() runId?: string; + @IsOptional() @IsString() approvalId?: string; + @IsOptional() @IsString() dependsOnTaskId?: string; + @IsOptional() @IsIn(['SUCCEEDED', 'FAILED']) status?: string; + @IsOptional() + @IsIn([ + 'TRANSIENT', + 'RATE_LIMITED', + 'DEPENDENCY', + 'VALIDATION', + 'POLICY', + 'PERMANENT', + ]) + retryClass?: string; + @IsOptional() @IsHash('sha256') sha256?: string; + @IsOptional() @IsHash('sha256') artifactHash?: string; + @IsOptional() @IsHash('sha256') subjectHash?: string; + @IsOptional() @IsString() kind?: string; + @IsOptional() @IsString() uri?: string; + @IsOptional() predicate?: Record; + @IsOptional() @IsString() signature?: string; +} diff --git a/backend/src/orchestration/kill-switch.service.spec.ts b/backend/src/orchestration/kill-switch.service.spec.ts index 968cbcd7..6d8b2e74 100644 --- a/backend/src/orchestration/kill-switch.service.spec.ts +++ b/backend/src/orchestration/kill-switch.service.spec.ts @@ -1,24 +1,45 @@ -import { ForbiddenException } from "@nestjs/common"; -import { KillSwitchService } from "./kill-switch.service"; +import { ForbiddenException } from '@nestjs/common'; +import { KillSwitchService } from './kill-switch.service'; -describe("KillSwitchService", () => { +describe('KillSwitchService', () => { const prisma = { orchestrationKillSwitch: { findUnique: jest.fn() } }; const service = new KillSwitchService(prisma as never); beforeEach(() => jest.resetAllMocks()); it.each([ - ["FREEZE_NEW_WORK", "CREATE_TASK"], - ["STOP_AGENTS", "RUN_AGENT"], - ["DISABLE_MUTATIONS", "MUTATE_PRODUCTION"], - ["EMERGENCY_HALT", "OBSERVE"], - ])("blocks %s at its policy boundary", async (level, action) => { - prisma.orchestrationKillSwitch.findUnique.mockResolvedValue({ scope: "GLOBAL", level, active: true }); - await expect(service.assertAllowed(action as never)).rejects.toThrow(ForbiddenException); + ['FREEZE_NEW_WORK', 'CREATE_TASK'], + ['STOP_AGENTS', 'RUN_AGENT'], + ['DISABLE_MUTATIONS', 'MUTATE_PRODUCTION'], + ['EMERGENCY_HALT', 'OBSERVE'], + ])('blocks %s at its policy boundary', async (level, action) => { + prisma.orchestrationKillSwitch.findUnique.mockResolvedValue({ + scope: 'GLOBAL', + level, + active: true, + }); + await expect(service.assertAllowed(action as never)).rejects.toThrow( + ForbiddenException, + ); }); - it("allows observation while production mutations are disabled", async () => { - prisma.orchestrationKillSwitch.findUnique.mockResolvedValue({ scope: "GLOBAL", level: "DISABLE_MUTATIONS", active: true }); - await expect(service.assertAllowed("OBSERVE")).resolves.toBeUndefined(); + it('freezes new claims while allowing an already running agent to finish', async () => { + prisma.orchestrationKillSwitch.findUnique.mockResolvedValue({scope:'GLOBAL',level:'FREEZE_NEW_WORK',active:true}); + await expect(service.assertAllowed('CLAIM_TASK' as never)).rejects.toThrow(ForbiddenException); + await expect(service.assertAllowed('RUN_AGENT')).resolves.toBeUndefined(); + }); + + it('fails closed when an active persisted kill level is unknown', async () => { + prisma.orchestrationKillSwitch.findUnique.mockResolvedValue({scope:'GLOBAL',level:'CORRUPT_LEVEL',active:true}); + await expect(service.assertAllowed('RUN_AGENT')).rejects.toThrow(ForbiddenException); + }); + + it('allows observation while production mutations are disabled', async () => { + prisma.orchestrationKillSwitch.findUnique.mockResolvedValue({ + scope: 'GLOBAL', + level: 'DISABLE_MUTATIONS', + active: true, + }); + await expect(service.assertAllowed('OBSERVE')).resolves.toBeUndefined(); }); }); diff --git a/backend/src/orchestration/kill-switch.service.ts b/backend/src/orchestration/kill-switch.service.ts index 5e8f0d9f..8cb433f6 100644 --- a/backend/src/orchestration/kill-switch.service.ts +++ b/backend/src/orchestration/kill-switch.service.ts @@ -1,9 +1,36 @@ -import { ForbiddenException, Injectable } from "@nestjs/common"; -import { PrismaService } from "../prisma/prisma.service"; -import { KillSwitchLevel } from "./orchestration.domain"; -export type GuardedAction="CREATE_TASK"|"RUN_AGENT"|"MUTATE_PRODUCTION"|"OBSERVE"; +import { ForbiddenException, Injectable } from '@nestjs/common'; +import { PrismaService } from '../prisma/prisma.service'; +import { KillSwitchLevel } from './orchestration.domain'; +export type GuardedAction = + | 'CREATE_TASK' + | 'CLAIM_TASK' + | 'RUN_AGENT' + | 'MUTATE_PRODUCTION' + | 'OBSERVE'; @Injectable() export class KillSwitchService { - constructor(private readonly prisma:PrismaService){} - async assertAllowed(action:GuardedAction):Promise{ const state=await (this.prisma as any).orchestrationKillSwitch.findUnique({where:{scope:"GLOBAL"}}); if(!state?.active)return; const blocked:Record={FREEZE_NEW_WORK:["CREATE_TASK"],STOP_AGENTS:["CREATE_TASK","RUN_AGENT"],DISABLE_MUTATIONS:["CREATE_TASK","RUN_AGENT","MUTATE_PRODUCTION"],EMERGENCY_HALT:["CREATE_TASK","RUN_AGENT","MUTATE_PRODUCTION","OBSERVE"]}; if(blocked[state.level as KillSwitchLevel]?.includes(action)) throw new ForbiddenException(`Orchestration kill switch ${state.level} blocks ${action}`); } + constructor(private readonly prisma: PrismaService) {} + async assertAllowed(action: GuardedAction): Promise { + const state = await (this.prisma as any).orchestrationKillSwitch.findUnique( + { where: { scope: 'GLOBAL' } }, + ); + if (!state?.active) return; + const blocked: Record = { + FREEZE_NEW_WORK: ['CREATE_TASK', 'CLAIM_TASK'], + STOP_AGENTS: ['CREATE_TASK', 'CLAIM_TASK', 'RUN_AGENT'], + DISABLE_MUTATIONS: ['CREATE_TASK', 'CLAIM_TASK', 'RUN_AGENT', 'MUTATE_PRODUCTION'], + EMERGENCY_HALT: [ + 'CREATE_TASK', + 'CLAIM_TASK', + 'RUN_AGENT', + 'MUTATE_PRODUCTION', + 'OBSERVE', + ], + }; + if (!Object.values(KillSwitchLevel).includes(state.level) || + blocked[state.level as KillSwitchLevel].includes(action)) + throw new ForbiddenException( + `Orchestration kill switch ${state.level} blocks ${action}`, + ); + } } diff --git a/backend/src/orchestration/migration-safety.spec.ts b/backend/src/orchestration/migration-safety.spec.ts index 7f43f408..33570558 100644 --- a/backend/src/orchestration/migration-safety.spec.ts +++ b/backend/src/orchestration/migration-safety.spec.ts @@ -1,3 +1,18 @@ -import { readFileSync } from "node:fs"; -import { join } from "node:path"; -describe("orchestration migration safety",()=>{it("is additive and never mutates legacy governance tables",()=>{const sql=readFileSync(join(process.cwd(),"prisma/migrations/20261022000000_add_orchestration_v4_core/migration.sql"),"utf8");expect(sql).not.toMatch(/^\s*(DROP|TRUNCATE|DELETE|UPDATE)\b/im);expect(sql).not.toMatch(/ALTER TABLE "(approval_requests|department_tasks|audit_logs)"/i);expect(sql.match(/CREATE TABLE "orchestration_/g)).toHaveLength(9);});}); +import { readFileSync } from 'node:fs'; +import { join } from 'node:path'; +describe('orchestration migration safety', () => { + it('is additive and never mutates legacy governance tables', () => { + const sql = readFileSync( + join( + process.cwd(), + 'prisma/migrations/20261022000000_add_orchestration_v4_core/migration.sql', + ), + 'utf8', + ); + expect(sql).not.toMatch(/^\s*(DROP|TRUNCATE|DELETE|UPDATE)\b/im); + expect(sql).not.toMatch( + /ALTER TABLE "(approval_requests|department_tasks|audit_logs)"/i, + ); + expect(sql.match(/CREATE TABLE "orchestration_/g)).toHaveLength(9); + }); +}); diff --git a/backend/src/orchestration/model-tier-policy.service.spec.ts b/backend/src/orchestration/model-tier-policy.service.spec.ts index 8b78a695..3e01020a 100644 --- a/backend/src/orchestration/model-tier-policy.service.spec.ts +++ b/backend/src/orchestration/model-tier-policy.service.spec.ts @@ -1,21 +1,68 @@ -import { BadRequestException } from "@nestjs/common"; -import { ModelTierPolicyService } from "./model-tier-policy.service"; +import { BadRequestException } from '@nestjs/common'; +import { ModelTierPolicyService } from './model-tier-policy.service'; -describe("ModelTierPolicyService", () => { +describe('ModelTierPolicyService', () => { const policy = new ModelTierPolicyService(); it.each([ - [{ complexity: 0, risk: 0, operationMode: "OBSERVE", remainingBudgetCents: 1 }, "L0"], - [{ complexity: 2, risk: 1, operationMode: "PREPARE", remainingBudgetCents: 100 }, "L1"], - [{ complexity: 5, risk: 4, operationMode: "PREPARE", remainingBudgetCents: 500 }, "L2"], - [{ complexity: 7, risk: 7, operationMode: "PREPARE", remainingBudgetCents: 1500 }, "L3"], - [{ complexity: 9, risk: 9, operationMode: "MUTATE_PRODUCTION", remainingBudgetCents: 5000 }, "L4"], - ])("selects a deterministic tier for policy inputs", (input, expected) => { + [ + { + complexity: 0, + risk: 0, + operationMode: 'OBSERVE', + remainingBudgetCents: 1, + }, + 'L0', + ], + [ + { + complexity: 2, + risk: 1, + operationMode: 'PREPARE', + remainingBudgetCents: 100, + }, + 'L1', + ], + [ + { + complexity: 5, + risk: 4, + operationMode: 'PREPARE', + remainingBudgetCents: 500, + }, + 'L2', + ], + [ + { + complexity: 7, + risk: 7, + operationMode: 'PREPARE', + remainingBudgetCents: 1500, + }, + 'L3', + ], + [ + { + complexity: 9, + risk: 9, + operationMode: 'MUTATE_PRODUCTION', + remainingBudgetCents: 5000, + }, + 'L4', + ], + ])('selects a deterministic tier for policy inputs', (input, expected) => { expect(policy.select(input as never)).toBe(expected); expect(policy.select(input as never)).toBe(expected); }); - it("fails closed when the selected tier exceeds the remaining hard budget", () => { - expect(() => policy.select({ complexity: 9, risk: 9, operationMode: "MUTATE_PRODUCTION", remainingBudgetCents: 100 })).toThrow(BadRequestException); + it('fails closed when the selected tier exceeds the remaining hard budget', () => { + expect(() => + policy.select({ + complexity: 9, + risk: 9, + operationMode: 'MUTATE_PRODUCTION', + remainingBudgetCents: 100, + }), + ).toThrow(BadRequestException); }); }); diff --git a/backend/src/orchestration/model-tier-policy.service.ts b/backend/src/orchestration/model-tier-policy.service.ts index 0ef12900..40a85040 100644 --- a/backend/src/orchestration/model-tier-policy.service.ts +++ b/backend/src/orchestration/model-tier-policy.service.ts @@ -1,13 +1,40 @@ -import { BadRequestException, Injectable } from "@nestjs/common"; -import { OperationMode } from "./orchestration.domain"; -export type ModelTier="L0"|"L1"|"L2"|"L3"|"L4"; +import { BadRequestException, Injectable } from '@nestjs/common'; +import { OperationMode } from './orchestration.domain'; +export type ModelTier = 'L0' | 'L1' | 'L2' | 'L3' | 'L4'; @Injectable() export class ModelTierPolicyService { - select(input:{complexity:number;risk:number;operationMode:OperationMode|string;remainingBudgetCents:number}):ModelTier { - const score=Math.max(input.complexity,input.risk)+(input.operationMode===OperationMode.MUTATE_PRODUCTION?1:0); - const tier:ModelTier=score<=1?"L0":score<=3?"L1":score<=5?"L2":score<=8?"L3":"L4"; - const floor:Record={L0:1,L1:25,L2:100,L3:500,L4:2000}; - if(input.remainingBudgetCents = { + L0: 1, + L1: 25, + L2: 100, + L3: 500, + L4: 2000, + }; + if (input.remainingBudgetCents < floor[tier]) + throw new BadRequestException( + `Hard budget cannot fund selected model tier ${tier}`, + ); + return tier; + } } diff --git a/backend/src/orchestration/orchestration-read.http.spec.ts b/backend/src/orchestration/orchestration-read.http.spec.ts index 9d1537b6..76898cac 100644 --- a/backend/src/orchestration/orchestration-read.http.spec.ts +++ b/backend/src/orchestration/orchestration-read.http.spec.ts @@ -52,6 +52,8 @@ describe('integration read boundary', () => { decideGate: jest.fn(), recordBudget: jest.fn(), setKillSwitch: jest.fn(), + command: jest.fn(), + reconcileBudget: jest.fn(), }; beforeAll(async () => { const m = await Test.createTestingModule({ @@ -81,6 +83,8 @@ describe('integration read boundary', () => { ['put', 'tasks/t1/transition'], ['post', 'tasks/t1/gates/2/decide'], ['post', 'tasks/t1/budget'], + ['post', 'tasks/t1/commands/RUN_CREATE'], + ['post', 'tasks/t1/budget/reconcile'], ['put', 'kill-switch'], ['post', 'chat/messages'], ] as const) { @@ -96,6 +100,8 @@ describe('integration read boundary', () => { 'decideGate', 'recordBudget', 'setKillSwitch', + 'command', + 'reconcileBudget', ] as const) expect(service[name]).not.toHaveBeenCalled(); }); diff --git a/backend/src/orchestration/orchestration.controller.ts b/backend/src/orchestration/orchestration.controller.ts index 223c756a..b78f988f 100644 --- a/backend/src/orchestration/orchestration.controller.ts +++ b/backend/src/orchestration/orchestration.controller.ts @@ -139,6 +139,12 @@ export class OrchestrationController { @Post('chat/messages') chat() { return this.deny(); } + @Post('tasks/:id/commands/:command') command() { + return this.deny(); + } + @Post('tasks/:id/budget/reconcile') reconcile() { + return this.deny(); + } private deny(): never { throw new ForbiddenException( 'Orchestration mutations and chat disabled pending hardened core and trusted runtime integration', diff --git a/backend/src/orchestration/orchestration.domain.spec.ts b/backend/src/orchestration/orchestration.domain.spec.ts index 1c828088..84ffb73a 100644 --- a/backend/src/orchestration/orchestration.domain.spec.ts +++ b/backend/src/orchestration/orchestration.domain.spec.ts @@ -1,24 +1,47 @@ -import { BadRequestException, ConflictException, ForbiddenException } from "@nestjs/common"; -import { assertTransition, nextStateForGateDecision, OperationMode, WorkflowState } from "./orchestration.domain"; +import { + BadRequestException, + ConflictException, + ForbiddenException, +} from '@nestjs/common'; +import { + assertTransition, + nextStateForGateDecision, + OperationMode, + WorkflowState, +} from './orchestration.domain'; -describe("orchestration domain", () => { - it("allows only declared workflow transitions", () => { - expect(() => assertTransition(WorkflowState.DRAFT, WorkflowState.GATE1_PENDING)).not.toThrow(); - expect(() => assertTransition(WorkflowState.DRAFT, WorkflowState.RUNNING)).toThrow(BadRequestException); +describe('orchestration domain', () => { + it('allows only declared workflow transitions', () => { + expect(() => + assertTransition(WorkflowState.DRAFT, WorkflowState.GATE1_PENDING), + ).not.toThrow(); + expect(() => + assertTransition(WorkflowState.DRAFT, WorkflowState.RUNNING), + ).toThrow(BadRequestException); }); - it("keeps gate 1 as execution authorization rather than release authorization", () => { - expect(nextStateForGateDecision(1, "APPROVED")).toBe(WorkflowState.GATE1_APPROVED); - expect(nextStateForGateDecision(2, "APPROVED")).toBe(WorkflowState.GATE2_APPROVED); + it('keeps gate 1 as execution authorization rather than release authorization', () => { + expect(nextStateForGateDecision(1, 'APPROVED')).toBe( + WorkflowState.GATE1_APPROVED, + ); + expect(nextStateForGateDecision(2, 'APPROVED')).toBe( + WorkflowState.GATE2_APPROVED, + ); }); - it("rejects an unsupported gate decision", () => { - expect(() => nextStateForGateDecision(1, "PENDING" as never)).toThrow(BadRequestException); + it('rejects an unsupported gate decision', () => { + expect(() => nextStateForGateDecision(1, 'PENDING' as never)).toThrow( + BadRequestException, + ); }); - it("classifies production mutation separately from observe and prepare", () => { - expect(OperationMode.OBSERVE).toBe("OBSERVE"); - expect(OperationMode.PREPARE).toBe("PREPARE"); - expect(OperationMode.MUTATE_PRODUCTION).toBe("MUTATE_PRODUCTION"); + it.each([WorkflowState.GATE1_PENDING, WorkflowState.GATE2_PENDING])('does not revive an expired proposal into %s', (state) => { + expect(() => assertTransition(WorkflowState.EXPIRED,state)).toThrow(BadRequestException); + }); + + it('classifies production mutation separately from observe and prepare', () => { + expect(OperationMode.OBSERVE).toBe('OBSERVE'); + expect(OperationMode.PREPARE).toBe('PREPARE'); + expect(OperationMode.MUTATE_PRODUCTION).toBe('MUTATE_PRODUCTION'); }); }); diff --git a/backend/src/orchestration/orchestration.domain.ts b/backend/src/orchestration/orchestration.domain.ts index 41ba5081..e395060b 100644 --- a/backend/src/orchestration/orchestration.domain.ts +++ b/backend/src/orchestration/orchestration.domain.ts @@ -1,12 +1,106 @@ -import { BadRequestException } from "@nestjs/common"; +import { BadRequestException } from '@nestjs/common'; -export enum WorkflowState { DRAFT="DRAFT", GATE1_PENDING="GATE1_PENDING", GATE1_APPROVED="GATE1_APPROVED", READY="READY", CLAIMED="CLAIMED", RUNNING="RUNNING", VERIFYING="VERIFYING", GATE2_PENDING="GATE2_PENDING", GATE2_APPROVED="GATE2_APPROVED", RELEASING="RELEASING", PRODUCTION_VERIFYING="PRODUCTION_VERIFYING", LIVE_VERIFIED="LIVE_VERIFIED", MEASURING="MEASURING", CLOSED="CLOSED", REJECTED="REJECTED", BLOCKED="BLOCKED", CANCELLED="CANCELLED", FAILED="FAILED", EXPIRED="EXPIRED" } -export enum OperationMode { OBSERVE="OBSERVE", PREPARE="PREPARE", MUTATE_PRODUCTION="MUTATE_PRODUCTION" } -export enum RetryClass { TRANSIENT="TRANSIENT", RATE_LIMITED="RATE_LIMITED", DEPENDENCY="DEPENDENCY", VALIDATION="VALIDATION", POLICY="POLICY", PERMANENT="PERMANENT" } -export enum KillSwitchLevel { FREEZE_NEW_WORK="FREEZE_NEW_WORK", STOP_AGENTS="STOP_AGENTS", DISABLE_MUTATIONS="DISABLE_MUTATIONS", EMERGENCY_HALT="EMERGENCY_HALT" } +export enum WorkflowState { + DRAFT = 'DRAFT', + GATE1_PENDING = 'GATE1_PENDING', + GATE1_APPROVED = 'GATE1_APPROVED', + READY = 'READY', + CLAIMED = 'CLAIMED', + RUNNING = 'RUNNING', + VERIFYING = 'VERIFYING', + GATE2_PENDING = 'GATE2_PENDING', + GATE2_APPROVED = 'GATE2_APPROVED', + RELEASING = 'RELEASING', + PRODUCTION_VERIFYING = 'PRODUCTION_VERIFYING', + LIVE_VERIFIED = 'LIVE_VERIFIED', + MEASURING = 'MEASURING', + CLOSED = 'CLOSED', + REJECTED = 'REJECTED', + BLOCKED = 'BLOCKED', + CANCELLED = 'CANCELLED', + FAILED = 'FAILED', + EXPIRED = 'EXPIRED', +} +export enum OperationMode { + OBSERVE = 'OBSERVE', + PREPARE = 'PREPARE', + MUTATE_PRODUCTION = 'MUTATE_PRODUCTION', +} +export enum RetryClass { + TRANSIENT = 'TRANSIENT', + RATE_LIMITED = 'RATE_LIMITED', + DEPENDENCY = 'DEPENDENCY', + VALIDATION = 'VALIDATION', + POLICY = 'POLICY', + PERMANENT = 'PERMANENT', +} +export enum KillSwitchLevel { + FREEZE_NEW_WORK = 'FREEZE_NEW_WORK', + STOP_AGENTS = 'STOP_AGENTS', + DISABLE_MUTATIONS = 'DISABLE_MUTATIONS', + EMERGENCY_HALT = 'EMERGENCY_HALT', +} const transitions: Record = { - DRAFT:[WorkflowState.GATE1_PENDING,WorkflowState.CANCELLED], GATE1_PENDING:[WorkflowState.GATE1_APPROVED,WorkflowState.REJECTED,WorkflowState.EXPIRED], GATE1_APPROVED:[WorkflowState.READY,WorkflowState.CANCELLED], READY:[WorkflowState.CLAIMED,WorkflowState.BLOCKED,WorkflowState.CANCELLED], CLAIMED:[WorkflowState.RUNNING,WorkflowState.READY,WorkflowState.CANCELLED], RUNNING:[WorkflowState.VERIFYING,WorkflowState.FAILED,WorkflowState.BLOCKED], VERIFYING:[WorkflowState.GATE2_PENDING,WorkflowState.FAILED,WorkflowState.RUNNING], GATE2_PENDING:[WorkflowState.GATE2_APPROVED,WorkflowState.REJECTED,WorkflowState.EXPIRED], GATE2_APPROVED:[WorkflowState.RELEASING,WorkflowState.CANCELLED], RELEASING:[WorkflowState.PRODUCTION_VERIFYING,WorkflowState.FAILED], PRODUCTION_VERIFYING:[WorkflowState.LIVE_VERIFIED,WorkflowState.FAILED], LIVE_VERIFIED:[WorkflowState.MEASURING,WorkflowState.CLOSED], MEASURING:[WorkflowState.CLOSED,WorkflowState.FAILED], CLOSED:[], REJECTED:[], BLOCKED:[WorkflowState.READY,WorkflowState.CANCELLED], CANCELLED:[], FAILED:[WorkflowState.READY,WorkflowState.CANCELLED], EXPIRED:[WorkflowState.GATE1_PENDING,WorkflowState.GATE2_PENDING,WorkflowState.CANCELLED] + DRAFT: [WorkflowState.GATE1_PENDING, WorkflowState.CANCELLED], + GATE1_PENDING: [ + WorkflowState.GATE1_APPROVED, + WorkflowState.REJECTED, + WorkflowState.EXPIRED, + ], + GATE1_APPROVED: [WorkflowState.READY, WorkflowState.CANCELLED], + READY: [ + WorkflowState.CLAIMED, + WorkflowState.BLOCKED, + WorkflowState.CANCELLED, + ], + CLAIMED: [ + WorkflowState.RUNNING, + WorkflowState.READY, + WorkflowState.CANCELLED, + ], + RUNNING: [ + WorkflowState.VERIFYING, + WorkflowState.FAILED, + WorkflowState.BLOCKED, + ], + VERIFYING: [ + WorkflowState.GATE2_PENDING, + WorkflowState.FAILED, + WorkflowState.RUNNING, + ], + GATE2_PENDING: [ + WorkflowState.GATE2_APPROVED, + WorkflowState.REJECTED, + WorkflowState.EXPIRED, + ], + GATE2_APPROVED: [WorkflowState.RELEASING, WorkflowState.CANCELLED], + RELEASING: [WorkflowState.PRODUCTION_VERIFYING, WorkflowState.FAILED], + PRODUCTION_VERIFYING: [WorkflowState.LIVE_VERIFIED, WorkflowState.FAILED], + LIVE_VERIFIED: [WorkflowState.MEASURING, WorkflowState.CLOSED], + MEASURING: [WorkflowState.CLOSED, WorkflowState.FAILED], + CLOSED: [], + REJECTED: [], + BLOCKED: [WorkflowState.READY, WorkflowState.CANCELLED], + CANCELLED: [], + FAILED: [WorkflowState.READY, WorkflowState.CANCELLED], + EXPIRED: [WorkflowState.CANCELLED], }; -export function assertTransition(from: WorkflowState, to: WorkflowState): void { if (!transitions[from]?.includes(to)) throw new BadRequestException(`Invalid orchestration transition ${from} -> ${to}`); } -export function nextStateForGateDecision(gate: 1|2, decision: "APPROVED"|"REJECTED"): WorkflowState { if (!(["APPROVED","REJECTED"] as string[]).includes(decision)) throw new BadRequestException("Decision must be APPROVED or REJECTED"); return decision === "REJECTED" ? WorkflowState.REJECTED : gate === 1 ? WorkflowState.GATE1_APPROVED : WorkflowState.GATE2_APPROVED; } +export function assertTransition(from: WorkflowState, to: WorkflowState): void { + if (!transitions[from]?.includes(to)) + throw new BadRequestException( + `Invalid orchestration transition ${from} -> ${to}`, + ); +} +export function nextStateForGateDecision( + gate: 1 | 2, + decision: 'APPROVED' | 'REJECTED', +): WorkflowState { + if (!(['APPROVED', 'REJECTED'] as string[]).includes(decision)) + throw new BadRequestException('Decision must be APPROVED or REJECTED'); + return decision === 'REJECTED' + ? WorkflowState.REJECTED + : gate === 1 + ? WorkflowState.GATE1_APPROVED + : WorkflowState.GATE2_APPROVED; +} diff --git a/backend/src/orchestration/orchestration.integration.spec.ts b/backend/src/orchestration/orchestration.integration.spec.ts new file mode 100644 index 00000000..ff08f657 --- /dev/null +++ b/backend/src/orchestration/orchestration.integration.spec.ts @@ -0,0 +1,577 @@ +import { OrchestrationController } from './orchestration.controller'; +import { DepartmentTasksController } from '../admin/department-tasks.controller'; +import { ApprovalsController } from '../admin/approvals.controller'; +import { PrismaClient } from '@prisma/client'; +import { OrchestrationService } from './orchestration.service'; +import { KillSwitchService } from './kill-switch.service'; +import { WorkflowState, OperationMode } from './orchestration.domain'; +const enabled = process.env.ORCHESTRATION_INTEGRATION === '1'; +(enabled ? describe : describe.skip)( + 'v4 disposable PostgreSQL invariants', + () => { + const db = new PrismaClient(); + const service = new OrchestrationService( + db as never, + new KillSwitchService(db as never), + ); + let n = 0; + async function task(state = 'DRAFT', budget = 100) { + return db.orchestrationTask.create({ + data: { + key: `test-${Date.now()}-${n++}`, + title: 'test', + department: 'TEST', + operationMode: 'PREPARE', + state, + budgetLimitCents: budget, + }, + }); + } + afterAll(async () => db.$disconnect()); + it('release rejects PREPARE operation class even with gate approval', async()=>{ + const t=await task('GATE2_APPROVED'); + await expect(service.command(t.id,'RELEASE_AUTHORIZE',{expectedVersion:1,idempotencyKey:`mode-${t.id}`},'ci')).rejects.toThrow(/operation mode/i); + }); + + it('append-only ledger rejects direct rewrites in PostgreSQL', async () => { + const t = await task(); + const r = await service.recordBudget( + t.id, + { + amountCents: 50, + kind: 'RESERVE', + idempotencyKey: `immutable-${t.id}`, + }, + 'a', + ); + await expect( + db.orchestrationBudgetLedger.update({ + where: { id: r.id }, + data: { amountCents: 0 }, + }), + ).rejects.toThrow(/append.only/i); + await expect( + db.orchestrationBudgetLedger.delete({ where: { id: r.id } }), + ).rejects.toThrow(/append.only/i); + }); + it('generic transitions cannot fake run lifecycle or production verification', async () => { + for (const [from, to] of [ + ['CLAIMED', 'RUNNING'], + ['RUNNING', 'VERIFYING'], + ['FAILED', 'READY'], + ['PRODUCTION_VERIFYING', 'LIVE_VERIFIED'], + ]) { + const t = await task(from); + await expect( + service.transition( + t.id, + { + expectedVersion: 1, + idempotencyKey: `fake-${t.id}`, + toState: to as WorkflowState, + }, + 'agent', + ), + ).rejects.toThrow(/dedicated/i); + } + }); + it('exposes command and reconcile HTTP handlers', () => { + const c = new OrchestrationController(service); + expect(typeof c.command).toBe('function'); + expect(typeof c.reconcile).toBe('function'); + }); + it('highest kill level blocks reads and reconciliation', async () => { + const t = await task(); + const r = await service.recordBudget( + t.id, + { + kind: 'RESERVE', + amountCents: 10, + idempotencyKey: `stop-res-${t.id}`, + }, + 'a', + ); + await service.setKillSwitch( + ['EMERGENCY', 'HALT'].join('_'), + 'test', + 'ceo', + ); + try { + await expect(service.findOne(t.id)).rejects.toThrow(/kill switch/i); + await expect( + service.reconcileBudget( + t.id, + { + reservationId: r.id, + actualCents: 0, + idempotencyKey: `stop-recon-${t.id}`, + }, + 'a', + ), + ).rejects.toThrow(/kill switch/i); + } finally { + await db.orchestrationKillSwitch.deleteMany(); + } + }); + it('legacy arbitrary status and approval endpoints fail closed', async () => { + await expect( + new DepartmentTasksController(db as never).create({ + title: 'bypass', + status: 'DONE', + }), + ).rejects.toThrow(/v4/i); + await expect( + new DepartmentTasksController(db as never).updateStatus('none', { + status: 'DONE', + }), + ).rejects.toThrow(/v4/i); + await expect( + new ApprovalsController(db as never).decide('none', { + decision: 'APPROVED', + } as any), + ).rejects.toThrow(/v4/i); + }); + it('release consumes only matching fresh production Gate2 via dedicated command', async () => { + const t = await task('GATE2_PENDING'); + const hash = 'f'.repeat(64), + commitSha = 'a'.repeat(40); + const r = await db.orchestrationRun.create({ + data: { + taskId: t.id, + status: 'SUCCEEDED', + commitSha, + environment: 'production', + idempotencyKey: `rel-run-${t.id}`, + }, + }); + await db.orchestrationArtifact.create({ + data: { + runId: r.id, + kind: 'BUILD', + uri: 'https://ci.example/build', + sha256: hash, + }, + }); + await db.orchestrationAttestation.create({ + data: { + runId: r.id, + kind: 'TEST', + subjectHash: hash, + signer: 'ci', + predicate: { passed: true }, + }, + }); + const decision = await service.decideGate( + t.id, + 2, + { + decision: 'APPROVED', + expectedVersion: 1, + idempotencyKey: `rel-approve-${t.id}`, + expiresAt: new Date(Date.now() + 60000).toISOString(), + runId: r.id, + artifactHash: hash, + commitSha, + environment: 'production', + }, + 'ceo', + ); + const dto = { + expectedVersion: 2, + idempotencyKey: `rel-${t.id}`, + approvalId: decision.id, + runId: r.id, + artifactHash: hash, + commitSha, + environment: 'production', + }; + await expect( + (service as any).command( + t.id, + 'RELEASE_AUTHORIZE', + { ...dto, artifactHash: '0'.repeat(64) }, + 'ci', + ), + ).rejects.toThrow(/binding/i); + const result = await (service as any).command( + t.id, + 'RELEASE_AUTHORIZE', + dto, + 'ci', + ); + expect(result.authorized).toBe(true); + expect( + (await db.orchestrationTask.findUniqueOrThrow({ where: { id: t.id } })) + .state, + ).toBe('RELEASING'); + await expect( + (service as any).command( + t.id, + 'RELEASE_AUTHORIZE', + { ...dto, expectedVersion: 3, idempotencyKey: `rel-again-${t.id}` }, + 'ci', + ), + ).rejects.toThrow(); + }); + it('gate decision replay rejects changed immutable expiry and actor', async () => { + const t = await task('GATE1_PENDING'); + const dto = { + decision: 'APPROVED' as const, + expectedVersion: 1, + idempotencyKey: `bind-${t.id}`, + expiresAt: new Date(Date.now() + 60000).toISOString(), + }; + await service.decideGate(t.id, 1, dto, 'ceo'); + await expect( + service.decideGate( + t.id, + 1, + { ...dto, expiresAt: new Date(Date.now() + 120000).toISOString() }, + 'ceo', + ), + ).rejects.toThrow(/Idempotency/i); + await expect(service.decideGate(t.id, 1, dto, 'other')).rejects.toThrow( + /Idempotency/i, + ); + }); + it('dependencies block admission and reject cycles', async () => { + const a = await task(); + const b = await task(); + await (service as any).command( + a.id, + 'DEPENDENCY_ADD', + { + expectedVersion: 1, + idempotencyKey: `dep-a-${a.id}`, + dependsOnTaskId: b.id, + }, + 'actor', + ); + await expect( + (service as any).command( + b.id, + 'DEPENDENCY_ADD', + { + expectedVersion: 1, + idempotencyKey: `dep-b-${b.id}`, + dependsOnTaskId: a.id, + }, + 'actor', + ), + ).rejects.toThrow(/cycle/i); + await db.orchestrationTask.update({ + where: { id: a.id }, + data: { state: 'GATE1_PENDING' }, + }); + await service.decideGate( + a.id, + 1, + { + decision: 'APPROVED', + expectedVersion: 2, + idempotencyKey: `dep-approval-${a.id}`, + expiresAt: new Date(Date.now() + 60000).toISOString(), + }, + 'ceo', + ); + await expect( + service.transition( + a.id, + { + toState: WorkflowState.READY, + expectedVersion: 3, + idempotencyKey: `dep-ready-${a.id}`, + }, + 'agent', + ), + ).rejects.toThrow(/dependency/i); + }); + it('registers artifact and attestation for owned run, blocks unregistered subjects', async () => { + const t = await task('CLAIMED'); + const base = { + expectedVersion: 1, + idempotencyKey: `life-${t.id}`, + commitSha: 'c'.repeat(40), + environment: 'staging', + }; + const r = await (service as any).command( + t.id, + 'RUN_CREATE', + base, + 'agent', + ); + const hash = 'd'.repeat(64); + const a = await (service as any).command( + t.id, + 'ARTIFACT_ADD', + { + expectedVersion: 2, + idempotencyKey: `art-${t.id}`, + runId: r.id, + sha256: hash, + kind: 'BUILD', + uri: 'https://ci.example/artifact', + }, + 'agent', + ); + expect(a.sha256).toBe(hash); + await expect( + (service as any).command( + t.id, + 'ATTESTATION_ADD', + { + expectedVersion: 3, + idempotencyKey: `bad-att-${t.id}`, + runId: r.id, + subjectHash: 'e'.repeat(64), + kind: 'TEST', + predicate: { passed: true }, + }, + 'ci', + ), + ).rejects.toThrow(/artifact/i); + const att = await (service as any).command( + t.id, + 'ATTESTATION_ADD', + { + expectedVersion: 3, + idempotencyKey: `att-${t.id}`, + runId: r.id, + subjectHash: hash, + kind: 'TEST', + predicate: { passed: true }, + }, + 'ci', + ); + expect(att.signer).toBe('ci'); + }); + it('creates immutable versioned runs with bounded transient retries', async () => { + const t = await task('CLAIMED'); + const dto = { + expectedVersion: 1, + idempotencyKey: `run-${t.id}`, + commitSha: 'a'.repeat(40), + environment: 'staging', + }; + const r = await (service as any).command( + t.id, + 'RUN_CREATE', + dto, + 'agent', + ); + expect(r.attempt).toBe(1); + expect( + await (service as any).command(t.id, 'RUN_CREATE', dto, 'agent'), + ).toEqual(r); + await expect( + (service as any).command( + t.id, + 'RUN_CREATE', + { ...dto, commitSha: 'b'.repeat(40) }, + 'agent', + ), + ).rejects.toThrow(/Idempotency/i); + await (service as any).command( + t.id, + 'RUN_FINISH', + { + expectedVersion: 2, + idempotencyKey: `finish-${t.id}`, + runId: r.id, + status: 'FAILED', + retryClass: 'POLICY', + }, + 'agent', + ); + await expect( + (service as any).command( + t.id, + 'RUN_CREATE', + { ...dto, expectedVersion: 3, idempotencyKey: `retry-${t.id}` }, + 'agent', + ), + ).rejects.toThrow(/retry/i); + }); + it('selects tier at admission using backend risk and budget', async () => { + const t = await task('GATE1_PENDING', 1000); + await service.decideGate( + t.id, + 1, + { + decision: 'APPROVED', + expectedVersion: 1, + idempotencyKey: `approve-tier-${t.id}`, + expiresAt: new Date(Date.now() + 60000).toISOString(), + }, + 'ceo', + ); + await service.transition( + t.id, + { + toState: WorkflowState.READY, + expectedVersion: 2, + idempotencyKey: `admit-tier-${t.id}`, + }, + 'agent', + ); + expect( + (await db.orchestrationTask.findUniqueOrThrow({ where: { id: t.id } })) + .modelTier, + ).toBe('L0'); + }); + it('STOP_AGENTS prevents claims and running transitions', async () => { + await service.setKillSwitch('STOP_AGENTS', 'test', 'ceo'); + try { + const t = await task('READY'); + await expect( + service.transition( + t.id, + { + toState: WorkflowState.CLAIMED, + expectedVersion: 1, + idempotencyKey: `kill-${t.id}`, + }, + 'agent', + ), + ).rejects.toThrow(/kill switch/i); + } finally { + await db.orchestrationKillSwitch.deleteMany(); + } + }); + it('Gate1 admission consumes only unexpired approval', async () => { + const t = await task('GATE1_PENDING'); + await service.decideGate( + t.id, + 1, + { + decision: 'APPROVED', + expectedVersion: 1, + idempotencyKey: `approve-${t.id}`, + expiresAt: new Date(Date.now() + 100).toISOString(), + }, + 'ceo', + ); + await new Promise((r) => setTimeout(r, 150)); + await expect( + service.transition( + t.id, + { + toState: WorkflowState.READY, + expectedVersion: 2, + idempotencyKey: `admit-${t.id}`, + }, + 'agent', + ), + ).rejects.toThrow(/expired/i); + }); + it('versioned transition replay returns original result and rejects changed payload', async () => { + const t = await task(); + const dto = { + toState: WorkflowState.GATE1_PENDING, + expectedVersion: 1, + idempotencyKey: `tr-${t.id}`, + }; + const first = await service.transition(t.id, dto, 'actor'); + expect(await service.transition(t.id, dto, 'actor')).toEqual(first); + await expect( + service.transition(t.id, { ...dto, reason: 'different' }, 'actor'), + ).rejects.toThrow(/Idempotency/i); + }); + it('reconciles a reservation exactly once without erasing spent cost', async () => { + const t = await task(); + const r = await service.recordBudget( + t.id, + { amountCents: 80, kind: 'RESERVE', idempotencyKey: `res-${t.id}` }, + 'agent', + ); + const dto = { + reservationId: r.id, + actualCents: 35, + idempotencyKey: `reconcile-${t.id}`, + }; + await (service as any).reconcileBudget(t.id, dto, 'agent'); + await (service as any).reconcileBudget(t.id, dto, 'agent'); + await expect( + (service as any).reconcileBudget( + t.id, + { ...dto, idempotencyKey: `second-${t.id}` }, + 'agent', + ), + ).rejects.toThrow(/reconciled/i); + expect( + ( + await db.orchestrationBudgetLedger.aggregate({ + where: { taskId: t.id }, + _sum: { amountCents: true }, + }) + )._sum.amountCents, + ).toBe(35); + await expect( + service.recordBudget( + t.id, + { + amountCents: 66, + kind: 'RESERVE', + idempotencyKey: `too-much-${t.id}`, + }, + 'agent', + ), + ).rejects.toThrow(/budget/i); + }); + it('concurrent budget reservations never exceed hard cap and replay is harmless', async () => { + const t = await task(); + const result = await Promise.allSettled( + Array.from({ length: 8 }, (_, i) => + service.recordBudget( + t.id, + { + amountCents: 30, + kind: 'RESERVE', + idempotencyKey: `reserve-${t.id}-${i}`, + }, + 'agent', + ), + ), + ); + expect(result.filter((x) => x.status === 'fulfilled')).toHaveLength(3); + const sum = await db.orchestrationBudgetLedger.aggregate({ + where: { taskId: t.id }, + _sum: { amountCents: true }, + }); + expect(sum._sum.amountCents).toBe(90); + const first = await db.orchestrationBudgetLedger.findFirstOrThrow({ + where: { taskId: t.id }, + }); + await service.recordBudget( + t.id, + { + amountCents: 30, + kind: 'RESERVE', + idempotencyKey: first.idempotencyKey, + }, + 'agent', + ); + expect( + await db.orchestrationBudgetLedger.count({ where: { taskId: t.id } }), + ).toBe(3); + }); + it('generic transition cannot approve gate 1', async () => { + const t = await task('GATE1_PENDING'); + await expect( + service.transition( + t.id, + { + toState: WorkflowState.GATE1_APPROVED, + expectedVersion: 1, + idempotencyKey: `gate-bypass-${n}`, + }, + 'ceo', + ), + ).rejects.toThrow(/gate/i); + expect( + (await db.orchestrationTask.findUniqueOrThrow({ where: { id: t.id } })) + .state, + ).toBe('GATE1_PENDING'); + }); + }, +); diff --git a/backend/src/orchestration/orchestration.module.ts b/backend/src/orchestration/orchestration.module.ts index cd0d1e63..9d3d64b2 100644 --- a/backend/src/orchestration/orchestration.module.ts +++ b/backend/src/orchestration/orchestration.module.ts @@ -1,2 +1,13 @@ -import { Module } from "@nestjs/common"; import { PrismaModule } from "../prisma/prisma.module"; import { KillSwitchService } from "./kill-switch.service"; import { ModelTierPolicyService } from "./model-tier-policy.service"; import { OrchestrationController } from "./orchestration.controller"; import { OrchestrationService } from "./orchestration.service"; -@Module({imports:[PrismaModule],controllers:[OrchestrationController],providers:[OrchestrationService,KillSwitchService,ModelTierPolicyService],exports:[OrchestrationService,KillSwitchService,ModelTierPolicyService]}) export class OrchestrationModule{} +import { Module } from '@nestjs/common'; +import { PrismaModule } from '../prisma/prisma.module'; +import { KillSwitchService } from './kill-switch.service'; +import { ModelTierPolicyService } from './model-tier-policy.service'; +import { OrchestrationController } from './orchestration.controller'; +import { OrchestrationService } from './orchestration.service'; +@Module({ + imports: [PrismaModule], + controllers: [OrchestrationController], + providers: [OrchestrationService, KillSwitchService, ModelTierPolicyService], + exports: [OrchestrationService, KillSwitchService, ModelTierPolicyService], +}) +export class OrchestrationModule {} diff --git a/backend/src/orchestration/orchestration.service.spec.ts b/backend/src/orchestration/orchestration.service.spec.ts index d0cab9bf..941e998b 100644 --- a/backend/src/orchestration/orchestration.service.spec.ts +++ b/backend/src/orchestration/orchestration.service.spec.ts @@ -1,73 +1,260 @@ -import { BadRequestException, ConflictException, ForbiddenException } from "@nestjs/common"; -import { OrchestrationService } from "./orchestration.service"; +import { + BadRequestException, + ConflictException, + ForbiddenException, +} from '@nestjs/common'; +import { OrchestrationService } from './orchestration.service'; function mockPrisma() { const tx: any = { - orchestrationTask: { findUnique: jest.fn(), create: jest.fn(), updateMany: jest.fn() }, + $queryRaw: jest.fn().mockResolvedValue([]), + orchestrationTask: { + findUnique: jest.fn(), + create: jest.fn(), + updateMany: jest.fn(), + }, orchestrationRun: { findUnique: jest.fn(), create: jest.fn() }, orchestrationArtifact: { findFirst: jest.fn(), create: jest.fn() }, orchestrationAttestation: { create: jest.fn() }, orchestrationApprovalDecision: { findUnique: jest.fn(), create: jest.fn() }, - orchestrationEvent: { create: jest.fn() }, - orchestrationBudgetLedger: { aggregate: jest.fn(), create: jest.fn() }, + orchestrationEvent: { findUnique: jest.fn(), create: jest.fn() }, + orchestrationBudgetLedger: { + findUnique: jest.fn(), + aggregate: jest.fn(), + create: jest.fn(), + }, orchestrationKillSwitch: { findUnique: jest.fn(), upsert: jest.fn() }, }; - tx.$transaction = jest.fn(async (cb: any) => cb(tx)); + tx.$transaction = jest.fn((cb: any) => Promise.resolve(cb(tx))); return tx; } -describe("OrchestrationService", () => { +describe('OrchestrationService', () => { let prisma: any; let service: OrchestrationService; - beforeEach(() => { prisma = mockPrisma(); service = new OrchestrationService(prisma, { assertAllowed: jest.fn() } as never); }); + beforeEach(() => { + prisma = mockPrisma(); + service = new OrchestrationService(prisma, { + assertAllowed: jest.fn(), + } as never); + }); - it("creates a task and append-only event transactionally", async () => { - prisma.orchestrationTask.create.mockResolvedValue({ id: "task-1", state: "DRAFT", version: 1 }); - const result = await service.createTask({ key: "seo-1", title: "SEO", department: "SEO", operationMode: "PREPARE" as never, budgetLimitCents: 1000 }, "actor-1"); + it('rejects an unknown kill level before overwriting persisted protection', async () => { + await expect(service.setKillSwitch('INVALID_LEVEL','test','admin')).rejects.toThrow(BadRequestException); + expect(prisma.orchestrationKillSwitch.upsert).not.toHaveBeenCalled(); + }); + + it('creates a task and append-only event transactionally', async () => { + prisma.orchestrationTask.create.mockResolvedValue({ + id: 'task-1', + state: 'DRAFT', + version: 1, + }); + const result = await service.createTask( + { + key: 'seo-1', + title: 'SEO', + department: 'SEO', + operationMode: 'PREPARE' as never, + budgetLimitCents: 1000, + }, + 'actor-1', + ); expect(prisma.$transaction).toHaveBeenCalledTimes(1); - expect(prisma.orchestrationTask.create).toHaveBeenCalledWith({ data: expect.objectContaining({ state: "DRAFT", version: 1 }) }); - expect(prisma.orchestrationEvent.create).toHaveBeenCalledWith({ data: expect.objectContaining({ eventType: "TASK_CREATED", sequence: 1 }) }); - expect(result.id).toBe("task-1"); + expect(prisma.orchestrationTask.create).toHaveBeenCalledWith({ + data: expect.objectContaining({ state: 'DRAFT', version: 1 }), + }); + expect(prisma.orchestrationEvent.create).toHaveBeenCalledWith({ + data: expect.objectContaining({ eventType: 'TASK_CREATED', sequence: 1 }), + }); + expect(result.id).toBe('task-1'); }); - it("uses compare-and-swap versioning for transitions", async () => { - prisma.orchestrationTask.findUnique.mockResolvedValue({ id: "task-1", state: "DRAFT", version: 3 }); + it('uses compare-and-swap versioning for transitions', async () => { + prisma.orchestrationTask.findUnique.mockResolvedValue({ + id: 'task-1', + state: 'DRAFT', + version: 3, + }); prisma.orchestrationTask.updateMany.mockResolvedValue({ count: 0 }); - await expect(service.transition("task-1", { toState: "GATE1_PENDING" as never, expectedVersion: 3, idempotencyKey: "i-1" }, "actor")).rejects.toThrow(ConflictException); + await expect( + service.transition( + 'task-1', + { + toState: 'GATE1_PENDING' as never, + expectedVersion: 3, + idempotencyKey: 'i-1', + }, + 'actor', + ), + ).rejects.toThrow(ConflictException); }); - it("makes gate decisions immutable and idempotent", async () => { - prisma.orchestrationApprovalDecision.findUnique.mockResolvedValue({ id: "d1", decision: "APPROVED", gate: 1, taskId: "task-1" }); - await expect(service.decideGate("task-1", 1, { decision: "APPROVED", expectedVersion: 2, idempotencyKey: "gate-1", expiresAt: new Date(Date.now()+10000).toISOString() }, "ceo")).resolves.toMatchObject({ id: "d1" }); + it('makes gate decisions immutable and idempotent', async () => { + const expiresAt = new Date(Date.now() + 10000).toISOString(); + prisma.orchestrationApprovalDecision.findUnique.mockResolvedValue({ + id: 'd1', + decision: 'APPROVED', + gate: 1, + taskId: 'task-1', + }); + prisma.orchestrationEvent.findUnique.mockResolvedValue({ + payload: { + fingerprint: JSON.stringify({ + taskId: 'task-1', + gate: 1, + dto: { + decision: 'APPROVED', + expectedVersion: 2, + idempotencyKey: 'gate-1', + expiresAt, + }, + actorId: 'ceo', + }), + }, + }); + await expect( + service.decideGate( + 'task-1', + 1, + { + decision: 'APPROVED', + expectedVersion: 2, + idempotencyKey: 'gate-1', + expiresAt, + }, + 'ceo', + ), + ).resolves.toMatchObject({ id: 'd1' }); expect(prisma.orchestrationApprovalDecision.create).not.toHaveBeenCalled(); }); - it("rejects an expired approval", async () => { - await expect(service.decideGate("task-1", 1, { decision: "APPROVED", expectedVersion: 2, idempotencyKey: "gate-expired", expiresAt: new Date(0).toISOString() }, "ceo")).rejects.toThrow(BadRequestException); + it('rejects an expired approval', async () => { + await expect( + service.decideGate( + 'task-1', + 1, + { + decision: 'APPROVED', + expectedVersion: 2, + idempotencyKey: 'gate-expired', + expiresAt: new Date(0).toISOString(), + }, + 'ceo', + ), + ).rejects.toThrow(BadRequestException); }); - it("binds gate 2 approval to run, artifact hash, commit and environment", async () => { + it('binds gate 2 approval to run, artifact hash, commit and environment', async () => { prisma.orchestrationApprovalDecision.findUnique.mockResolvedValue(null); - prisma.orchestrationTask.findUnique.mockResolvedValue({ id: "task-1", state: "GATE2_PENDING", version: 7 }); - prisma.orchestrationRun.findUnique.mockResolvedValue({ id: "run-1", taskId: "task-1", commitSha: "abc123", environment: "staging" }); - prisma.orchestrationArtifact.findFirst.mockResolvedValue({ id: "artifact-1", runId: "run-1", sha256: "f".repeat(64) }); + prisma.orchestrationTask.findUnique.mockResolvedValue({ + id: 'task-1', + state: 'GATE2_PENDING', + version: 7, + }); + prisma.orchestrationRun.findUnique.mockResolvedValue({ + id: 'run-1', + taskId: 'task-1', + commitSha: 'abc123', + environment: 'staging', + status: 'SUCCEEDED', + }); + prisma.orchestrationArtifact.findFirst.mockResolvedValue({ + id: 'artifact-1', + runId: 'run-1', + sha256: 'f'.repeat(64), + }); prisma.orchestrationTask.updateMany.mockResolvedValue({ count: 1 }); - prisma.orchestrationApprovalDecision.create.mockResolvedValue({ id: "decision-2", gate: 2 }); - await service.decideGate("task-1", 2, { decision: "APPROVED", expectedVersion: 7, idempotencyKey: "gate-2", expiresAt: new Date(Date.now()+10000).toISOString(), runId: "run-1", artifactHash: "f".repeat(64), commitSha: "abc123", environment: "staging" }, "ceo"); - expect(prisma.orchestrationApprovalDecision.create).toHaveBeenCalledWith({ data: expect.objectContaining({ runId: "run-1", artifactHash: "f".repeat(64), commitSha: "abc123", environment: "staging" }) }); + prisma.orchestrationApprovalDecision.create.mockResolvedValue({ + id: 'decision-2', + gate: 2, + }); + await service.decideGate( + 'task-1', + 2, + { + decision: 'APPROVED', + expectedVersion: 7, + idempotencyKey: 'gate-2', + expiresAt: new Date(Date.now() + 10000).toISOString(), + runId: 'run-1', + artifactHash: 'f'.repeat(64), + commitSha: 'abc123', + environment: 'staging', + }, + 'ceo', + ); + expect(prisma.orchestrationApprovalDecision.create).toHaveBeenCalledWith({ + data: expect.objectContaining({ + runId: 'run-1', + artifactHash: 'f'.repeat(64), + commitSha: 'abc123', + environment: 'staging', + }), + }); }); - it("refuses gate 2 when binding does not match the immutable run", async () => { + it.each(['RUNNING', 'FAILED', 'CANCELLED'])('refuses Gate 2 approval for an unfinished or unsuccessful %s run', async (status) => { + prisma.orchestrationTask.findUnique.mockResolvedValue({id:'task-1',state:'GATE2_PENDING',version:7}); + prisma.orchestrationRun.findUnique.mockResolvedValue({id:'run-1',taskId:'task-1',commitSha:'abc123',environment:'staging',status}); + prisma.orchestrationArtifact.findFirst.mockResolvedValue({id:'artifact-1'}); + prisma.orchestrationTask.updateMany.mockResolvedValue({count:1}); + await expect(service.decideGate('task-1',2,{ + decision:'APPROVED',expectedVersion:7,idempotencyKey:'failed-run', + expiresAt:new Date(Date.now()+10000).toISOString(),runId:'run-1', + artifactHash:'f'.repeat(64),commitSha:'abc123',environment:'staging', + },'ceo')).rejects.toThrow(/successful run/i); + expect(prisma.orchestrationApprovalDecision.create).not.toHaveBeenCalled(); + expect(prisma.orchestrationTask.updateMany).not.toHaveBeenCalled(); + }); + + it('refuses gate 2 when binding does not match the immutable run', async () => { prisma.orchestrationApprovalDecision.findUnique.mockResolvedValue(null); - prisma.orchestrationTask.findUnique.mockResolvedValue({ id: "task-1", state: "GATE2_PENDING", version: 7 }); - prisma.orchestrationRun.findUnique.mockResolvedValue({ id: "run-1", taskId: "task-1", commitSha: "real", environment: "staging" }); - await expect(service.decideGate("task-1", 2, { decision: "APPROVED", expectedVersion: 7, idempotencyKey: "gate-bad", expiresAt: new Date(Date.now()+10000).toISOString(), runId: "run-1", artifactHash: "f".repeat(64), commitSha: "wrong", environment: "staging" }, "ceo")).rejects.toThrow(ForbiddenException); + prisma.orchestrationTask.findUnique.mockResolvedValue({ + id: 'task-1', + state: 'GATE2_PENDING', + version: 7, + }); + prisma.orchestrationRun.findUnique.mockResolvedValue({ + id: 'run-1', + taskId: 'task-1', + commitSha: 'real', + environment: 'staging', + }); + await expect( + service.decideGate( + 'task-1', + 2, + { + decision: 'APPROVED', + expectedVersion: 7, + idempotencyKey: 'gate-bad', + expiresAt: new Date(Date.now() + 10000).toISOString(), + runId: 'run-1', + artifactHash: 'f'.repeat(64), + commitSha: 'wrong', + environment: 'staging', + }, + 'ceo', + ), + ).rejects.toThrow(ForbiddenException); }); - it("enforces hard task budget atomically", async () => { - prisma.orchestrationTask.findUnique.mockResolvedValue({ id: "task-1", budgetLimitCents: 1000 }); - prisma.orchestrationBudgetLedger.aggregate.mockResolvedValue({ _sum: { amountCents: 900 } }); - await expect(service.recordBudget("task-1", { amountCents: 101, idempotencyKey: "cost-1", kind: "MODEL" }, "agent")).rejects.toThrow(ForbiddenException); + it('enforces hard task budget atomically', async () => { + prisma.orchestrationTask.findUnique.mockResolvedValue({ + id: 'task-1', + budgetLimitCents: 1000, + }); + prisma.orchestrationBudgetLedger.aggregate.mockResolvedValue({ + _sum: { amountCents: 900 }, + }); + await expect( + service.recordBudget( + 'task-1', + { amountCents: 101, idempotencyKey: 'cost-1', kind: 'SPEND' }, + 'agent', + ), + ).rejects.toThrow(ForbiddenException); expect(prisma.orchestrationBudgetLedger.create).not.toHaveBeenCalled(); }); }); diff --git a/backend/src/orchestration/orchestration.service.ts b/backend/src/orchestration/orchestration.service.ts index 2a8ef43b..7ef7bb0b 100644 --- a/backend/src/orchestration/orchestration.service.ts +++ b/backend/src/orchestration/orchestration.service.ts @@ -1,17 +1,665 @@ -import { BadRequestException, ConflictException, ForbiddenException, Injectable, NotFoundException } from "@nestjs/common"; -import { PrismaService } from "../prisma/prisma.service"; -import { CreateOrchestrationTaskDto, GateDecisionDto, RecordBudgetDto, TransitionTaskDto } from "./dto/orchestration.dto"; -import { assertTransition, nextStateForGateDecision, OperationMode, WorkflowState } from "./orchestration.domain"; -import { KillSwitchService } from "./kill-switch.service"; +import { + BadRequestException, + ConflictException, + ForbiddenException, + Injectable, + NotFoundException, +} from '@nestjs/common'; +import { PrismaService } from '../prisma/prisma.service'; +import { + CreateOrchestrationTaskDto, + GateDecisionDto, + RecordBudgetDto, + TransitionTaskDto, +} from './dto/orchestration.dto'; +import { + KillSwitchLevel, + assertTransition, + nextStateForGateDecision, + OperationMode, + WorkflowState, +} from './orchestration.domain'; +import { ModelTierPolicyService } from './model-tier-policy.service'; +import { KillSwitchService } from './kill-switch.service'; @Injectable() export class OrchestrationService { - constructor(private readonly prisma:PrismaService,private readonly killSwitch:KillSwitchService){} - async createTask(dto:CreateOrchestrationTaskDto,actorId:string){ await this.killSwitch.assertAllowed("CREATE_TASK"); return (this.prisma as any).$transaction(async(tx:any)=>{const task=await tx.orchestrationTask.create({data:{key:dto.key,title:dto.title,department:dto.department,operationMode:dto.operationMode,budgetLimitCents:dto.budgetLimitCents,metadata:dto.metadata??{},state:WorkflowState.DRAFT,version:1}});await tx.orchestrationEvent.create({data:{taskId:task.id,eventType:"TASK_CREATED",fromState:null,toState:WorkflowState.DRAFT,sequence:1,actorId,payload:{key:dto.key}}});return task;}); } - async transition(id:string,dto:TransitionTaskDto,actorId:string){return (this.prisma as any).$transaction(async(tx:any)=>{const existing=await tx.orchestrationTask.findUnique({where:{id}});if(!existing)throw new NotFoundException(`Orchestration task ${id} not found`);assertTransition(existing.state,dto.toState);if(existing.operationMode===OperationMode.MUTATE_PRODUCTION&&dto.toState===WorkflowState.RELEASING)await this.killSwitch.assertAllowed("MUTATE_PRODUCTION");const changed=await tx.orchestrationTask.updateMany({where:{id,version:dto.expectedVersion,state:existing.state},data:{state:dto.toState,version:{increment:1}}});if(changed.count!==1)throw new ConflictException("Task version or state changed; reload before retrying");await tx.orchestrationEvent.create({data:{taskId:id,eventType:"STATE_TRANSITION",fromState:existing.state,toState:dto.toState,sequence:dto.expectedVersion+1,actorId,idempotencyKey:dto.idempotencyKey,payload:{reason:dto.reason??null}}});return {...existing,state:dto.toState,version:dto.expectedVersion+1};});} - async decideGate(taskId:string,gate:1|2,dto:GateDecisionDto,actorId:string){if(new Date(dto.expiresAt)<=new Date())throw new BadRequestException("Approval decision has expired");return (this.prisma as any).$transaction(async(tx:any)=>{const prior=await tx.orchestrationApprovalDecision.findUnique({where:{idempotencyKey:dto.idempotencyKey}});if(prior){if(prior.taskId!==taskId||prior.gate!==gate||prior.decision!==dto.decision)throw new ConflictException("Idempotency key already belongs to another approval decision");return prior;}const task=await tx.orchestrationTask.findUnique({where:{id:taskId}});if(!task)throw new NotFoundException(`Orchestration task ${taskId} not found`);const expected=gate===1?WorkflowState.GATE1_PENDING:WorkflowState.GATE2_PENDING;if(task.state!==expected)throw new ConflictException(`Gate ${gate} requires ${expected}`);if(gate===2&&dto.decision==="APPROVED"){if(!dto.runId||!dto.artifactHash||!dto.commitSha||!dto.environment)throw new BadRequestException("Gate 2 requires runId, artifactHash, commitSha and environment");const run=await tx.orchestrationRun.findUnique({where:{id:dto.runId}});if(!run||run.taskId!==taskId||run.commitSha!==dto.commitSha||run.environment!==dto.environment)throw new ForbiddenException("Gate 2 binding does not match immutable run");const artifact=await tx.orchestrationArtifact.findFirst({where:{runId:dto.runId,sha256:dto.artifactHash}});if(!artifact)throw new ForbiddenException("Gate 2 artifact hash is not registered for run");} - const next=nextStateForGateDecision(gate,dto.decision);const changed=await tx.orchestrationTask.updateMany({where:{id:taskId,version:dto.expectedVersion,state:expected},data:{state:next,version:{increment:1}}});if(changed.count!==1)throw new ConflictException("Task version or state changed; approval not applied");const decision=await tx.orchestrationApprovalDecision.create({data:{taskId,gate,decision:dto.decision,actorId,notes:dto.notes,expiresAt:new Date(dto.expiresAt),idempotencyKey:dto.idempotencyKey,runId:dto.runId,artifactHash:dto.artifactHash,commitSha:dto.commitSha,environment:dto.environment}});await tx.orchestrationEvent.create({data:{taskId,eventType:`GATE${gate}_${dto.decision}`,fromState:expected,toState:next,sequence:dto.expectedVersion+1,actorId,idempotencyKey:`event:${dto.idempotencyKey}`,payload:{decisionId:decision.id}}});return decision;});} - async recordBudget(taskId:string,dto:RecordBudgetDto,actorId:string){return (this.prisma as any).$transaction(async(tx:any)=>{const task=await tx.orchestrationTask.findUnique({where:{id:taskId}});if(!task)throw new NotFoundException(`Orchestration task ${taskId} not found`);const sum=await tx.orchestrationBudgetLedger.aggregate({where:{taskId},_sum:{amountCents:true}});if((sum._sum.amountCents??0)+dto.amountCents>task.budgetLimitCents)throw new ForbiddenException("Hard task budget exceeded");return tx.orchestrationBudgetLedger.create({data:{taskId,amountCents:dto.amountCents,kind:dto.kind,idempotencyKey:dto.idempotencyKey,actorId,metadata:dto.metadata??{}}});});} - async findAll(){return (this.prisma as any).orchestrationTask.findMany({orderBy:{createdAt:"desc"}})} - async findOne(id:string){const task=await (this.prisma as any).orchestrationTask.findUnique({where:{id},include:{runs:{include:{artifacts:true,attestations:true}},dependencies:true,decisions:true,events:{orderBy:{sequence:"asc"}},budgetLedger:true}});if(!task)throw new NotFoundException(`Orchestration task ${id} not found`);return task;} - async setKillSwitch(level:string,reason:string,actorId:string){return (this.prisma as any).$transaction(async(tx:any)=>{const value=await tx.orchestrationKillSwitch.upsert({where:{scope:"GLOBAL"},create:{scope:"GLOBAL",level,active:true,reason,actorId},update:{level,active:true,reason,actorId,version:{increment:1}}});await tx.orchestrationEvent.create({data:{eventType:"KILL_SWITCH_CHANGED",sequence:value.version??1,actorId,payload:{scope:"GLOBAL",level,reason}}});return value;});} + constructor( + private readonly prisma: PrismaService, + private readonly killSwitch: KillSwitchService, + ) {} + async createTask(dto: CreateOrchestrationTaskDto, actorId: string) { + await this.killSwitch.assertAllowed('CREATE_TASK'); + return await (this.prisma as any).$transaction(async (tx: any) => { + const task = await tx.orchestrationTask.create({ + data: { + key: dto.key, + title: dto.title, + department: dto.department, + operationMode: dto.operationMode, + budgetLimitCents: dto.budgetLimitCents, + metadata: dto.metadata ?? {}, + state: WorkflowState.DRAFT, + version: 1, + }, + }); + await tx.orchestrationEvent.create({ + data: { + taskId: task.id, + eventType: 'TASK_CREATED', + fromState: null, + toState: WorkflowState.DRAFT, + sequence: 1, + actorId, + payload: { key: dto.key }, + }, + }); + return task; + }); + } + async transition(id: string, dto: TransitionTaskDto, actorId: string) { + if ( + [ + WorkflowState.GATE1_APPROVED, + WorkflowState.GATE2_APPROVED, + WorkflowState.RELEASING, + WorkflowState.RUNNING, + WorkflowState.VERIFYING, + WorkflowState.PRODUCTION_VERIFYING, + WorkflowState.LIVE_VERIFIED, + WorkflowState.MEASURING, + WorkflowState.CLOSED, + ].includes(dto.toState) + ) + throw new ForbiddenException( + 'Gate approvals and release require dedicated commands', + ); + return await (this.prisma as any).$transaction(async (tx: any) => { + await tx.$queryRaw`SELECT id FROM orchestration_tasks WHERE id = ${id}::uuid FOR UPDATE`; + const fingerprint = JSON.stringify({ id, dto, actorId }); + const replay = await tx.orchestrationEvent.findUnique({ + where: { idempotencyKey: dto.idempotencyKey }, + }); + if (replay) { + if (replay.payload.fingerprint !== fingerprint) + throw new ConflictException('Idempotency payload mismatch'); + return replay.payload.result; + } + const existing = await tx.orchestrationTask.findUnique({ where: { id } }); + if (!existing) + throw new NotFoundException(`Orchestration task ${id} not found`); + if (existing.state === WorkflowState.FAILED) + throw new ForbiddenException('Retries require dedicated run command'); + assertTransition(existing.state, dto.toState); + await this.killSwitch.assertAllowed( + [ + WorkflowState.READY, + WorkflowState.CLAIMED, + WorkflowState.RUNNING, + WorkflowState.VERIFYING, + WorkflowState.GATE2_PENDING, + ].includes(dto.toState) + ? 'RUN_AGENT' + : 'OBSERVE', + ); + if ([WorkflowState.READY, WorkflowState.CLAIMED].includes(dto.toState)) + await this.killSwitch.assertAllowed('CLAIM_TASK'); + let selectedTier = existing.modelTier; + if (dto.toState === WorkflowState.READY) { + const dependencies = await tx.orchestrationDependency.findMany({ + where: { taskId: id }, + include: { dependsOnTask: true }, + }); + if (dependencies.some((d: any) => d.dependsOnTask.state !== 'CLOSED')) + throw new ForbiddenException('Unresolved dependency'); + const costs = await tx.orchestrationBudgetLedger.aggregate({ + where: { taskId: id }, + _sum: { amountCents: true }, + }); + selectedTier = new ModelTierPolicyService().select({ + complexity: Number(existing.metadata?.complexity ?? 0), + risk: Number(existing.metadata?.risk ?? 0), + operationMode: existing.operationMode, + remainingBudgetCents: + existing.budgetLimitCents - (costs._sum.amountCents ?? 0), + }); + const approval = await tx.orchestrationApprovalDecision.findFirst({ + where: { taskId: id, gate: 1, decision: 'APPROVED' }, + orderBy: { createdAt: 'desc' }, + }); + if (!approval || approval.expiresAt <= new Date()) + throw new ForbiddenException('Gate 1 approval missing or expired'); + } + if ( + existing.operationMode === OperationMode.MUTATE_PRODUCTION && + dto.toState === WorkflowState.RELEASING + ) + await this.killSwitch.assertAllowed('MUTATE_PRODUCTION'); + const changed = await tx.orchestrationTask.updateMany({ + where: { id, version: dto.expectedVersion, state: existing.state }, + data: { + state: dto.toState, + modelTier: selectedTier, + version: { increment: 1 }, + }, + }); + if (changed.count !== 1) + throw new ConflictException( + 'Task version or state changed; reload before retrying', + ); + await tx.orchestrationEvent.create({ + data: { + taskId: id, + eventType: 'STATE_TRANSITION', + fromState: existing.state, + toState: dto.toState, + sequence: dto.expectedVersion + 1, + actorId, + idempotencyKey: dto.idempotencyKey, + payload: { + fingerprint, + result: JSON.parse( + JSON.stringify({ + ...existing, + state: dto.toState, + modelTier: selectedTier, + version: dto.expectedVersion + 1, + }), + ), + }, + }, + }); + return JSON.parse( + JSON.stringify({ + ...existing, + state: dto.toState, + modelTier: selectedTier, + version: dto.expectedVersion + 1, + }), + ); + }); + } + async decideGate( + taskId: string, + gate: 1 | 2, + dto: GateDecisionDto, + actorId: string, + ) { + if (![1, 2].includes(gate)) throw new BadRequestException('Invalid gate'); + return await (this.prisma as any).$transaction(async (tx: any) => { + await tx.$queryRaw`SELECT id FROM orchestration_tasks WHERE id = ${taskId}::uuid FOR UPDATE`; + const fingerprint = JSON.stringify({ taskId, gate, dto, actorId }); + const prior = await tx.orchestrationApprovalDecision.findUnique({ + where: { idempotencyKey: dto.idempotencyKey }, + }); + if (prior) { + const event = await tx.orchestrationEvent.findUnique({ + where: { idempotencyKey: `event:${dto.idempotencyKey}` }, + }); + if (event?.payload?.fingerprint !== fingerprint) + throw new ConflictException('Idempotency payload mismatch'); + return prior; + } + if ( + !Number.isFinite(Date.parse(dto.expiresAt)) || + new Date(dto.expiresAt) <= new Date() + ) + throw new BadRequestException('Approval decision has expired'); + await this.killSwitch.assertAllowed('OBSERVE'); + const task = await tx.orchestrationTask.findUnique({ + where: { id: taskId }, + }); + if (!task) + throw new NotFoundException(`Orchestration task ${taskId} not found`); + const expected = + gate === 1 ? WorkflowState.GATE1_PENDING : WorkflowState.GATE2_PENDING; + if (task.state !== expected) + throw new ConflictException(`Gate ${gate} requires ${expected}`); + if (gate === 2 && dto.decision === 'APPROVED') { + if ( + !dto.runId || + !dto.artifactHash || + !dto.commitSha || + !dto.environment + ) + throw new BadRequestException( + 'Gate 2 requires runId, artifactHash, commitSha and environment', + ); + const run = await tx.orchestrationRun.findUnique({ + where: { id: dto.runId }, + }); + if ( + !run || + run.taskId !== taskId || + run.commitSha !== dto.commitSha || + run.environment !== dto.environment + ) + throw new ForbiddenException( + 'Gate 2 binding does not match immutable run', + ); + if (run.status !== 'SUCCEEDED') + throw new ForbiddenException('Gate 2 requires a successful run'); + const artifact = await tx.orchestrationArtifact.findFirst({ + where: { runId: dto.runId, sha256: dto.artifactHash }, + }); + if (!artifact) + throw new ForbiddenException( + 'Gate 2 artifact hash is not registered for run', + ); + } + const next = nextStateForGateDecision(gate, dto.decision); + const changed = await tx.orchestrationTask.updateMany({ + where: { id: taskId, version: dto.expectedVersion, state: expected }, + data: { state: next, version: { increment: 1 } }, + }); + if (changed.count !== 1) + throw new ConflictException( + 'Task version or state changed; approval not applied', + ); + const decision = await tx.orchestrationApprovalDecision.create({ + data: { + taskId, + gate, + decision: dto.decision, + actorId, + notes: dto.notes, + expiresAt: new Date(dto.expiresAt), + idempotencyKey: dto.idempotencyKey, + runId: dto.runId, + artifactHash: dto.artifactHash, + commitSha: dto.commitSha, + environment: dto.environment, + }, + }); + await tx.orchestrationEvent.create({ + data: { + taskId, + eventType: `GATE${gate}_${dto.decision}`, + fromState: expected, + toState: next, + sequence: dto.expectedVersion + 1, + actorId, + idempotencyKey: `event:${dto.idempotencyKey}`, + payload: { decisionId: decision.id, fingerprint }, + }, + }); + return decision; + }); + } + async recordBudget(taskId: string, dto: RecordBudgetDto, actorId: string) { + if ( + !Number.isSafeInteger(dto.amountCents) || + dto.amountCents < 1 || + !['RESERVE', 'SPEND'].includes(dto.kind) + ) + throw new BadRequestException( + 'Use RESERVE or SPEND with positive integer cents', + ); + return await (this.prisma as any).$transaction(async (tx: any) => { + await tx.$queryRaw`SELECT id FROM orchestration_tasks WHERE id = ${taskId}::uuid FOR UPDATE`; + const prior = await tx.orchestrationBudgetLedger.findUnique({ + where: { idempotencyKey: dto.idempotencyKey }, + }); + if (prior) { + if ( + prior.taskId !== taskId || + prior.kind !== dto.kind || + prior.amountCents !== dto.amountCents || + prior.actorId !== actorId || + JSON.stringify(prior.metadata) !== JSON.stringify(dto.metadata ?? {}) + ) + throw new ConflictException('Idempotency payload mismatch'); + return prior; + } + await this.killSwitch.assertAllowed('RUN_AGENT'); + const task = await tx.orchestrationTask.findUnique({ + where: { id: taskId }, + }); + if (!task) + throw new NotFoundException(`Orchestration task ${taskId} not found`); + const sum = await tx.orchestrationBudgetLedger.aggregate({ + where: { taskId }, + _sum: { amountCents: true }, + }); + if ((sum._sum.amountCents ?? 0) + dto.amountCents > task.budgetLimitCents) + throw new ForbiddenException('Hard task budget exceeded'); + return tx.orchestrationBudgetLedger.create({ + data: { + taskId, + amountCents: dto.amountCents, + kind: dto.kind, + idempotencyKey: dto.idempotencyKey, + actorId, + metadata: dto.metadata ?? {}, + }, + }); + }); + } + async command(taskId: string, command: string, dto: any, actorId: string) { + return await (this.prisma as any).$transaction(async (tx: any) => { + await tx.$queryRaw`SELECT id FROM orchestration_tasks WHERE id = ${taskId}::uuid FOR UPDATE`; + const fingerprint = JSON.stringify({ taskId, command, dto, actorId }); + const replay = await tx.orchestrationEvent.findUnique({ + where: { idempotencyKey: dto.idempotencyKey }, + }); + if (replay) { + if (replay.payload.fingerprint !== fingerprint) + throw new ConflictException('Idempotency payload mismatch'); + return replay.payload.result; + } + const task = await tx.orchestrationTask.findUnique({ + where: { id: taskId }, + }); + if (!task) throw new NotFoundException('Task not found'); + if (task.version !== dto.expectedVersion) + throw new ConflictException('Stale task version'); + await this.killSwitch.assertAllowed('RUN_AGENT'); + let result: any; + let state = task.state; + if (command === 'RUN_CREATE') { + if ( + !/^[a-f0-9]{40,64}$/.test(dto.commitSha) || + !['development', 'staging', 'production'].includes(dto.environment) + ) + throw new BadRequestException('Invalid immutable run binding'); + const last = await tx.orchestrationRun.findFirst({ + where: { taskId }, + orderBy: { attempt: 'desc' }, + }); + if ( + last && + (last.status !== 'FAILED' || + !['TRANSIENT', 'RATE_LIMITED', 'DEPENDENCY'].includes( + last.retryClass, + ) || + last.attempt >= 3) + ) + throw new ForbiddenException('Retry denied or exhausted'); + if (!last && task.state !== 'CLAIMED') + throw new ConflictException('Run requires CLAIMED task'); + result = await tx.orchestrationRun.create({ + data: { + taskId, + attempt: (last?.attempt ?? 0) + 1, + status: 'RUNNING', + commitSha: dto.commitSha, + environment: dto.environment, + idempotencyKey: dto.idempotencyKey, + startedAt: new Date(), + }, + }); + state = 'RUNNING'; + } else if (command === 'RUN_FINISH') { + const run = await tx.orchestrationRun.findUnique({ + where: { id: dto.runId }, + }); + if ( + !run || + run.taskId !== taskId || + run.status !== 'RUNNING' || + task.state !== 'RUNNING' + ) + throw new ConflictException('Active run required'); + if ( + !['SUCCEEDED', 'FAILED'].includes(dto.status) || + (dto.status === 'FAILED' && + ![ + 'TRANSIENT', + 'RATE_LIMITED', + 'DEPENDENCY', + 'VALIDATION', + 'POLICY', + 'PERMANENT', + ].includes(dto.retryClass)) + ) + throw new BadRequestException('Invalid outcome taxonomy'); + result = await tx.orchestrationRun.update({ + where: { id: run.id }, + data: { + status: dto.status, + retryClass: dto.status === 'FAILED' ? dto.retryClass : null, + finishedAt: new Date(), + }, + }); + state = dto.status === 'FAILED' ? 'FAILED' : 'VERIFYING'; + } else if (command === 'ARTIFACT_ADD' || command === 'ATTESTATION_ADD') { + const run = await tx.orchestrationRun.findUnique({ + where: { id: dto.runId }, + }); + if ( + !run || + run.taskId !== taskId || + !['RUNNING', 'VERIFYING'].includes(task.state) + ) + throw new ConflictException('Owned active run required'); + const hash = command === 'ARTIFACT_ADD' ? dto.sha256 : dto.subjectHash; + if (!/^[a-f0-9]{64}$/.test(hash) || !dto.kind) + throw new BadRequestException('Invalid artifact hash or kind'); + if (command === 'ARTIFACT_ADD') { + if (!/^https:\/\//.test(dto.uri)) + throw new BadRequestException('Artifact URI must be HTTPS'); + result = await tx.orchestrationArtifact.create({ + data: { runId: run.id, kind: dto.kind, sha256: hash, uri: dto.uri }, + }); + } else { + if ( + !(await tx.orchestrationArtifact.findFirst({ + where: { runId: run.id, sha256: hash }, + })) + ) + throw new ForbiddenException( + 'Attestation subject is not a registered artifact', + ); + result = await tx.orchestrationAttestation.create({ + data: { + runId: run.id, + kind: dto.kind, + subjectHash: hash, + predicate: dto.predicate ?? {}, + signer: actorId, + signature: dto.signature, + }, + }); + } + } else if (command === 'DEPENDENCY_ADD') { + if (task.state !== 'DRAFT') + throw new ConflictException( + 'Dependencies are frozen after submission', + ); + await tx.$executeRaw`LOCK TABLE orchestration_dependencies IN EXCLUSIVE MODE`; + const cycles = + await tx.$queryRaw`WITH RECURSIVE chain(id) AS (SELECT ${dto.dependsOnTaskId}::uuid UNION SELECT d.depends_on_task_id FROM orchestration_dependencies d JOIN chain c ON d.task_id=c.id) SELECT id FROM chain WHERE id=${taskId}::uuid`; + if (cycles.length) throw new ForbiddenException('Dependency cycle'); + result = await tx.orchestrationDependency.create({ + data: { taskId, dependsOnTaskId: dto.dependsOnTaskId }, + }); + } else if (command === 'RELEASE_AUTHORIZE') { + await this.killSwitch.assertAllowed('MUTATE_PRODUCTION'); + if (task.state !== 'GATE2_APPROVED') + throw new ConflictException( + 'Gate 2 approval required; already consumed or invalid state', + ); + const approval = await tx.orchestrationApprovalDecision.findUnique({ + where: { id: dto.approvalId }, + }); + if ( + !approval || + approval.taskId !== taskId || + approval.gate !== 2 || + approval.decision !== 'APPROVED' || + approval.expiresAt <= new Date() + ) + throw new ForbiddenException('Gate 2 missing or expired'); + for (const key of ['runId', 'artifactHash', 'commitSha', 'environment']) + if (approval[key] !== dto[key]) + throw new ForbiddenException('Release binding mismatch'); + const run = await tx.orchestrationRun.findUnique({ + where: { id: dto.runId }, + }); + if ( + !run || + run.taskId !== taskId || + run.status !== 'SUCCEEDED' || + run.commitSha !== dto.commitSha || + run.environment !== dto.environment + ) + throw new ForbiddenException('Run binding mismatch'); + if ( + !(await tx.orchestrationArtifact.findFirst({ + where: { runId: run.id, sha256: dto.artifactHash }, + })) || + !(await tx.orchestrationAttestation.findFirst({ + where: { + runId: run.id, + subjectHash: dto.artifactHash, + kind: 'TEST', + }, + })) + ) + throw new ForbiddenException('Artifact evidence missing'); + result = { + authorized: true, + approvalId: approval.id, + runId: run.id, + artifactHash: dto.artifactHash, + commitSha: dto.commitSha, + environment: dto.environment, + execution: 'CI_CD_ONLY', + }; + state = 'RELEASING'; + } else throw new BadRequestException('Unknown command'); + await tx.orchestrationTask.update({ + where: { id: taskId }, + data: { state, version: { increment: 1 } }, + }); + const serialized = JSON.parse(JSON.stringify(result)); + await tx.orchestrationEvent.create({ + data: { + taskId, + eventType: command, + fromState: task.state, + toState: state, + sequence: task.version + 1, + actorId, + idempotencyKey: dto.idempotencyKey, + payload: { fingerprint, result: serialized }, + }, + }); + return serialized; + }); + } + async reconcileBudget( + taskId: string, + dto: { reservationId: string; actualCents: number; idempotencyKey: string }, + actorId: string, + ) { + await this.killSwitch.assertAllowed('OBSERVE'); + if (!Number.isSafeInteger(dto.actualCents) || dto.actualCents < 0) + throw new BadRequestException('Invalid actual cost'); + return await (this.prisma as any).$transaction(async (tx: any) => { + await tx.$queryRaw`SELECT id FROM orchestration_tasks WHERE id = ${taskId}::uuid FOR UPDATE`; + const prior = await tx.orchestrationBudgetLedger.findUnique({ + where: { idempotencyKey: dto.idempotencyKey }, + }); + if (prior) { + if ( + prior.taskId !== taskId || + prior.actorId !== actorId || + prior.metadata.reservationId !== dto.reservationId || + prior.metadata.actualCents !== dto.actualCents + ) + throw new ConflictException('Idempotency payload mismatch'); + return prior; + } + const reservation = await tx.orchestrationBudgetLedger.findUnique({ + where: { id: dto.reservationId }, + }); + if ( + !reservation || + reservation.taskId !== taskId || + reservation.kind !== 'RESERVE' + ) + throw new BadRequestException('Reservation not found'); + if ( + await tx.orchestrationBudgetLedger.findFirst({ + where: { + taskId, + kind: 'RECONCILE', + metadata: { path: ['reservationId'], equals: dto.reservationId }, + }, + }) + ) + throw new ConflictException('Reservation already reconciled'); + const task = await tx.orchestrationTask.findUnique({ + where: { id: taskId }, + }); + const total = await tx.orchestrationBudgetLedger.aggregate({ + where: { taskId }, + _sum: { amountCents: true }, + }); + const delta = dto.actualCents - reservation.amountCents; + if (total._sum.amountCents + delta > task.budgetLimitCents) + throw new ForbiddenException('Hard task budget exceeded'); + return tx.orchestrationBudgetLedger.create({ + data: { + taskId, + kind: 'RECONCILE', + amountCents: delta, + idempotencyKey: dto.idempotencyKey, + actorId, + metadata: { + reservationId: dto.reservationId, + actualCents: dto.actualCents, + }, + }, + }); + }); + } + async findAll() { + await this.killSwitch.assertAllowed('OBSERVE'); + return (this.prisma as any).orchestrationTask.findMany({ + orderBy: { createdAt: 'desc' }, + }); + } + async findOne(id: string) { + await this.killSwitch.assertAllowed('OBSERVE'); + const task = await (this.prisma as any).orchestrationTask.findUnique({ + where: { id }, + include: { + runs: { include: { artifacts: true, attestations: true } }, + dependencies: true, + decisions: true, + events: { orderBy: { sequence: 'asc' } }, + budgetLedger: true, + }, + }); + if (!task) + throw new NotFoundException(`Orchestration task ${id} not found`); + return task; + } + async setKillSwitch(level: string, reason: string, actorId: string) { + if (!Object.values(KillSwitchLevel).includes(level as KillSwitchLevel)) + throw new BadRequestException('Invalid kill switch level'); + return await (this.prisma as any).$transaction(async (tx: any) => { + const value = await tx.orchestrationKillSwitch.upsert({ + where: { scope: 'GLOBAL' }, + create: { scope: 'GLOBAL', level, active: true, reason, actorId }, + update: { + level, + active: true, + reason, + actorId, + version: { increment: 1 }, + }, + }); + await tx.orchestrationEvent.create({ + data: { + eventType: 'KILL_SWITCH_CHANGED', + sequence: value.version ?? 1, + actorId, + payload: { scope: 'GLOBAL', level, reason }, + }, + }); + return value; + }); + } }