feat(orchestration): add v4 additive backend core and policy tests

This commit is contained in:
devops committed 2026-10-22 18:15:32 +00:00
1 parent eb17c8e284
commit e5c83e87fe
18 files changed
+586 -14

No files matched your search

+13
View File
@@ -0,0 +1,13 @@
# Prisma migration baseline and Orchestration v4 rollout
The production databases predate complete migration tracking for the legacy governance tables (`approval_requests`, `department_tasks`, and `audit_logs`). Do **not** run `prisma db push`, reset, or edit an applied migration.
## Safe baseline procedure (per environment, during a CI/CD maintenance step)
1. Back up the database and verify restore capability.
2. Run `prisma migrate status` and compare the live schema to the reviewed migrations.
3. For migrations whose objects already exist in that environment, record them without executing SQL using `prisma migrate resolve --applied <migration_name>`. This is an environment-specific reconciliation step; never guess which migrations are present.
4. Review `20261022000000_add_orchestration_v4_core/migration.sql`. It is additive: it creates only `orchestration_*` tables, indexes, and foreign keys. It does not alter or copy legacy governance data.
5. Apply through CI/CD only with `prisma migrate deploy`; validate counts and application health. No production deployment is part of this change.
Rollback is application-first: stop use of the new module while retaining its append-only records. Do not drop tables in an automated rollback. Any later removal requires a separately reviewed archival migration.
@@ -0,0 +1,211 @@
-- CreateTable
CREATE TABLE "orchestration_tasks" (
"id" UUID NOT NULL,
"key" VARCHAR(120) NOT NULL,
"title" VARCHAR(255) NOT NULL,
"department" VARCHAR(50) NOT NULL,
"state" VARCHAR(40) NOT NULL DEFAULT 'DRAFT',
"operation_mode" VARCHAR(30) NOT NULL,
"model_tier" VARCHAR(2),
"budget_limit_cents" INTEGER NOT NULL,
"version" INTEGER NOT NULL DEFAULT 1,
"metadata" JSONB NOT NULL DEFAULT '{}',
"created_at" TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
"updated_at" TIMESTAMPTZ NOT NULL,
CONSTRAINT "orchestration_tasks_pkey" PRIMARY KEY ("id")
);
-- CreateTable
CREATE TABLE "orchestration_runs" (
"id" UUID NOT NULL,
"task_id" UUID NOT NULL,
"attempt" INTEGER NOT NULL DEFAULT 1,
"status" VARCHAR(30) NOT NULL DEFAULT 'PENDING',
"commit_sha" VARCHAR(64) NOT NULL,
"environment" VARCHAR(50) NOT NULL,
"retry_class" VARCHAR(30),
"idempotency_key" VARCHAR(150) NOT NULL,
"started_at" TIMESTAMPTZ,
"finished_at" TIMESTAMPTZ,
"created_at" TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
CONSTRAINT "orchestration_runs_pkey" PRIMARY KEY ("id")
);
-- CreateTable
CREATE TABLE "orchestration_artifacts" (
"id" UUID NOT NULL,
"run_id" UUID NOT NULL,
"kind" VARCHAR(50) NOT NULL,
"uri" TEXT NOT NULL,
"sha256" CHAR(64) NOT NULL,
"size_bytes" BIGINT,
"metadata" JSONB NOT NULL DEFAULT '{}',
"created_at" TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
CONSTRAINT "orchestration_artifacts_pkey" PRIMARY KEY ("id")
);
-- CreateTable
CREATE TABLE "orchestration_attestations" (
"id" UUID NOT NULL,
"run_id" UUID NOT NULL,
"kind" VARCHAR(50) NOT NULL,
"subject_hash" CHAR(64) NOT NULL,
"predicate" JSONB NOT NULL,
"signer" VARCHAR(150) NOT NULL,
"signature" TEXT,
"created_at" TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
CONSTRAINT "orchestration_attestations_pkey" PRIMARY KEY ("id")
);
-- CreateTable
CREATE TABLE "orchestration_dependencies" (
"id" UUID NOT NULL,
"task_id" UUID NOT NULL,
"depends_on_task_id" UUID NOT NULL,
"kind" VARCHAR(30) NOT NULL DEFAULT 'BLOCKS',
"created_at" TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
CONSTRAINT "orchestration_dependencies_pkey" PRIMARY KEY ("id")
);
-- CreateTable
CREATE TABLE "orchestration_approval_decisions" (
"id" UUID NOT NULL,
"task_id" UUID NOT NULL,
"gate" INTEGER NOT NULL,
"decision" VARCHAR(20) NOT NULL,
"actor_id" VARCHAR(100) NOT NULL,
"notes" TEXT,
"expires_at" TIMESTAMPTZ NOT NULL,
"idempotency_key" VARCHAR(150) NOT NULL,
"run_id" UUID,
"artifact_hash" CHAR(64),
"commit_sha" VARCHAR(64),
"environment" VARCHAR(50),
"created_at" TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
CONSTRAINT "orchestration_approval_decisions_pkey" PRIMARY KEY ("id")
);
-- CreateTable
CREATE TABLE "orchestration_events" (
"id" UUID NOT NULL,
"task_id" UUID,
"event_type" VARCHAR(60) NOT NULL,
"from_state" VARCHAR(40),
"to_state" VARCHAR(40),
"sequence" INTEGER NOT NULL,
"actor_id" VARCHAR(100) NOT NULL,
"idempotency_key" VARCHAR(150),
"payload" JSONB NOT NULL DEFAULT '{}',
"created_at" TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
CONSTRAINT "orchestration_events_pkey" PRIMARY KEY ("id")
);
-- CreateTable
CREATE TABLE "orchestration_budget_ledger" (
"id" UUID NOT NULL,
"task_id" UUID NOT NULL,
"amount_cents" INTEGER NOT NULL,
"kind" VARCHAR(40) NOT NULL,
"idempotency_key" VARCHAR(150) NOT NULL,
"actor_id" VARCHAR(100) NOT NULL,
"metadata" JSONB NOT NULL DEFAULT '{}',
"created_at" TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
CONSTRAINT "orchestration_budget_ledger_pkey" PRIMARY KEY ("id")
);
-- CreateTable
CREATE TABLE "orchestration_kill_switches" (
"scope" VARCHAR(80) NOT NULL,
"level" VARCHAR(30) NOT NULL,
"active" BOOLEAN NOT NULL DEFAULT true,
"reason" TEXT NOT NULL,
"actor_id" VARCHAR(100) NOT NULL,
"version" INTEGER NOT NULL DEFAULT 1,
"updated_at" TIMESTAMPTZ NOT NULL,
"created_at" TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
CONSTRAINT "orchestration_kill_switches_pkey" PRIMARY KEY ("scope")
);
-- CreateIndex
CREATE UNIQUE INDEX "orchestration_tasks_key_key" ON "orchestration_tasks"("key");
-- CreateIndex
CREATE INDEX "orchestration_tasks_state_created_at_idx" ON "orchestration_tasks"("state", "created_at");
-- CreateIndex
CREATE INDEX "orchestration_tasks_department_state_idx" ON "orchestration_tasks"("department", "state");
-- CreateIndex
CREATE UNIQUE INDEX "orchestration_runs_idempotency_key_key" ON "orchestration_runs"("idempotency_key");
-- CreateIndex
CREATE INDEX "orchestration_runs_task_id_status_idx" ON "orchestration_runs"("task_id", "status");
-- CreateIndex
CREATE UNIQUE INDEX "orchestration_runs_task_id_attempt_key" ON "orchestration_runs"("task_id", "attempt");
-- CreateIndex
CREATE UNIQUE INDEX "orchestration_artifacts_run_id_sha256_key" ON "orchestration_artifacts"("run_id", "sha256");
-- CreateIndex
CREATE UNIQUE INDEX "orchestration_attestations_run_id_kind_subject_hash_signer_key" ON "orchestration_attestations"("run_id", "kind", "subject_hash", "signer");
-- CreateIndex
CREATE UNIQUE INDEX "orchestration_dependencies_task_id_depends_on_task_id_key" ON "orchestration_dependencies"("task_id", "depends_on_task_id");
-- CreateIndex
CREATE UNIQUE INDEX "orchestration_approval_decisions_idempotency_key_key" ON "orchestration_approval_decisions"("idempotency_key");
-- CreateIndex
CREATE INDEX "orchestration_approval_decisions_task_id_gate_created_at_idx" ON "orchestration_approval_decisions"("task_id", "gate", "created_at");
-- CreateIndex
CREATE UNIQUE INDEX "orchestration_events_idempotency_key_key" ON "orchestration_events"("idempotency_key");
-- CreateIndex
CREATE INDEX "orchestration_events_event_type_created_at_idx" ON "orchestration_events"("event_type", "created_at");
-- CreateIndex
CREATE UNIQUE INDEX "orchestration_events_task_id_sequence_key" ON "orchestration_events"("task_id", "sequence");
-- CreateIndex
CREATE UNIQUE INDEX "orchestration_budget_ledger_idempotency_key_key" ON "orchestration_budget_ledger"("idempotency_key");
-- CreateIndex
CREATE INDEX "orchestration_budget_ledger_task_id_created_at_idx" ON "orchestration_budget_ledger"("task_id", "created_at");
-- AddForeignKey
ALTER TABLE "orchestration_runs" ADD CONSTRAINT "orchestration_runs_task_id_fkey" FOREIGN KEY ("task_id") REFERENCES "orchestration_tasks"("id") ON DELETE RESTRICT ON UPDATE CASCADE;
-- AddForeignKey
ALTER TABLE "orchestration_artifacts" ADD CONSTRAINT "orchestration_artifacts_run_id_fkey" FOREIGN KEY ("run_id") REFERENCES "orchestration_runs"("id") ON DELETE RESTRICT ON UPDATE CASCADE;
-- AddForeignKey
ALTER TABLE "orchestration_attestations" ADD CONSTRAINT "orchestration_attestations_run_id_fkey" FOREIGN KEY ("run_id") REFERENCES "orchestration_runs"("id") ON DELETE RESTRICT ON UPDATE CASCADE;
-- AddForeignKey
ALTER TABLE "orchestration_dependencies" ADD CONSTRAINT "orchestration_dependencies_task_id_fkey" FOREIGN KEY ("task_id") REFERENCES "orchestration_tasks"("id") ON DELETE RESTRICT ON UPDATE CASCADE;
-- AddForeignKey
ALTER TABLE "orchestration_dependencies" ADD CONSTRAINT "orchestration_dependencies_depends_on_task_id_fkey" FOREIGN KEY ("depends_on_task_id") REFERENCES "orchestration_tasks"("id") ON DELETE RESTRICT ON UPDATE CASCADE;
-- AddForeignKey
ALTER TABLE "orchestration_approval_decisions" ADD CONSTRAINT "orchestration_approval_decisions_task_id_fkey" FOREIGN KEY ("task_id") REFERENCES "orchestration_tasks"("id") ON DELETE RESTRICT ON UPDATE CASCADE;
-- AddForeignKey
ALTER TABLE "orchestration_approval_decisions" ADD CONSTRAINT "orchestration_approval_decisions_run_id_fkey" FOREIGN KEY ("run_id") REFERENCES "orchestration_runs"("id") ON DELETE RESTRICT ON UPDATE CASCADE;
-- AddForeignKey
ALTER TABLE "orchestration_events" ADD CONSTRAINT "orchestration_events_task_id_fkey" FOREIGN KEY ("task_id") REFERENCES "orchestration_tasks"("id") ON DELETE RESTRICT ON UPDATE CASCADE;
-- AddForeignKey
ALTER TABLE "orchestration_budget_ledger" ADD CONSTRAINT "orchestration_budget_ledger_task_id_fkey" FOREIGN KEY ("task_id") REFERENCES "orchestration_tasks"("id") ON DELETE RESTRICT ON UPDATE CASCADE;
+141
View File
@@ -1091,3 +1091,144 @@ model DepartmentTask {
@@index([createdAt])
@@map("department_tasks")
}
// Orchestration v4 additive models. Legacy governance tables remain untouched.
model OrchestrationTask {
id String @id @default(uuid()) @db.Uuid
key String @unique @db.VarChar(120)
title String @db.VarChar(255)
department String @db.VarChar(50)
state String @default("DRAFT") @db.VarChar(40)
operationMode String @map("operation_mode") @db.VarChar(30)
modelTier String? @map("model_tier") @db.VarChar(2)
budgetLimitCents Int @map("budget_limit_cents")
version Int @default(1)
metadata Json @default("{}") @db.JsonB
createdAt DateTime @default(now()) @map("created_at") @db.Timestamptz()
updatedAt DateTime @updatedAt @map("updated_at") @db.Timestamptz()
runs OrchestrationRun[]
dependencies OrchestrationDependency[] @relation("task_dependencies")
dependedOnBy OrchestrationDependency[] @relation("task_dependents")
decisions OrchestrationApprovalDecision[]
events OrchestrationEvent[]
budgetLedger OrchestrationBudgetLedger[]
@@index([state, createdAt])
@@index([department, state])
@@map("orchestration_tasks")
}
model OrchestrationRun {
id String @id @default(uuid()) @db.Uuid
taskId String @map("task_id") @db.Uuid
attempt Int @default(1)
status String @default("PENDING") @db.VarChar(30)
commitSha String @map("commit_sha") @db.VarChar(64)
environment String @db.VarChar(50)
retryClass String? @map("retry_class") @db.VarChar(30)
idempotencyKey String @unique @map("idempotency_key") @db.VarChar(150)
startedAt DateTime? @map("started_at") @db.Timestamptz()
finishedAt DateTime? @map("finished_at") @db.Timestamptz()
createdAt DateTime @default(now()) @map("created_at") @db.Timestamptz()
task OrchestrationTask @relation(fields:[taskId],references:[id],onDelete:Restrict)
artifacts OrchestrationArtifact[]
attestations OrchestrationAttestation[]
approvals OrchestrationApprovalDecision[]
@@unique([taskId, attempt])
@@index([taskId, status])
@@map("orchestration_runs")
}
model OrchestrationArtifact {
id String @id @default(uuid()) @db.Uuid
runId String @map("run_id") @db.Uuid
kind String @db.VarChar(50)
uri String @db.Text
sha256 String @db.Char(64)
sizeBytes BigInt? @map("size_bytes")
metadata Json @default("{}") @db.JsonB
createdAt DateTime @default(now()) @map("created_at") @db.Timestamptz()
run OrchestrationRun @relation(fields:[runId],references:[id],onDelete:Restrict)
@@unique([runId, sha256])
@@map("orchestration_artifacts")
}
model OrchestrationAttestation {
id String @id @default(uuid()) @db.Uuid
runId String @map("run_id") @db.Uuid
kind String @db.VarChar(50)
subjectHash String @map("subject_hash") @db.Char(64)
predicate Json @db.JsonB
signer String @db.VarChar(150)
signature String? @db.Text
createdAt DateTime @default(now()) @map("created_at") @db.Timestamptz()
run OrchestrationRun @relation(fields:[runId],references:[id],onDelete:Restrict)
@@unique([runId, kind, subjectHash, signer])
@@map("orchestration_attestations")
}
model OrchestrationDependency {
id String @id @default(uuid()) @db.Uuid
taskId String @map("task_id") @db.Uuid
dependsOnTaskId String @map("depends_on_task_id") @db.Uuid
kind String @default("BLOCKS") @db.VarChar(30)
createdAt DateTime @default(now()) @map("created_at") @db.Timestamptz()
task OrchestrationTask @relation("task_dependencies",fields:[taskId],references:[id],onDelete:Restrict)
dependsOnTask OrchestrationTask @relation("task_dependents",fields:[dependsOnTaskId],references:[id],onDelete:Restrict)
@@unique([taskId, dependsOnTaskId])
@@map("orchestration_dependencies")
}
model OrchestrationApprovalDecision {
id String @id @default(uuid()) @db.Uuid
taskId String @map("task_id") @db.Uuid
gate Int
decision String @db.VarChar(20)
actorId String @map("actor_id") @db.VarChar(100)
notes String? @db.Text
expiresAt DateTime @map("expires_at") @db.Timestamptz()
idempotencyKey String @unique @map("idempotency_key") @db.VarChar(150)
runId String? @map("run_id") @db.Uuid
artifactHash String? @map("artifact_hash") @db.Char(64)
commitSha String? @map("commit_sha") @db.VarChar(64)
environment String? @db.VarChar(50)
createdAt DateTime @default(now()) @map("created_at") @db.Timestamptz()
task OrchestrationTask @relation(fields:[taskId],references:[id],onDelete:Restrict)
run OrchestrationRun? @relation(fields:[runId],references:[id],onDelete:Restrict)
@@index([taskId, gate, createdAt])
@@map("orchestration_approval_decisions")
}
model OrchestrationEvent {
id String @id @default(uuid()) @db.Uuid
taskId String? @map("task_id") @db.Uuid
eventType String @map("event_type") @db.VarChar(60)
fromState String? @map("from_state") @db.VarChar(40)
toState String? @map("to_state") @db.VarChar(40)
sequence Int
actorId String @map("actor_id") @db.VarChar(100)
idempotencyKey String? @unique @map("idempotency_key") @db.VarChar(150)
payload Json @default("{}") @db.JsonB
createdAt DateTime @default(now()) @map("created_at") @db.Timestamptz()
task OrchestrationTask? @relation(fields:[taskId],references:[id],onDelete:Restrict)
@@unique([taskId, sequence])
@@index([eventType, createdAt])
@@map("orchestration_events")
}
model OrchestrationBudgetLedger {
id String @id @default(uuid()) @db.Uuid
taskId String @map("task_id") @db.Uuid
amountCents Int @map("amount_cents")
kind String @db.VarChar(40)
idempotencyKey String @unique @map("idempotency_key") @db.VarChar(150)
actorId String @map("actor_id") @db.VarChar(100)
metadata Json @default("{}") @db.JsonB
createdAt DateTime @default(now()) @map("created_at") @db.Timestamptz()
task OrchestrationTask @relation(fields:[taskId],references:[id],onDelete:Restrict)
@@index([taskId, createdAt])
@@map("orchestration_budget_ledger")
}
model OrchestrationKillSwitch {
scope String @id @db.VarChar(80)
level String @db.VarChar(30)
active Boolean @default(true)
reason String @db.Text
actorId String @map("actor_id") @db.VarChar(100)
version Int @default(1)
updatedAt DateTime @updatedAt @map("updated_at") @db.Timestamptz()
createdAt DateTime @default(now()) @map("created_at") @db.Timestamptz()
@@map("orchestration_kill_switches")
}
@@ -3,7 +3,6 @@ import {
Get,
Post,
Put,
Delete,
Body,
Param,
Query,
@@ -255,10 +254,4 @@ export class ApprovalsController {
};
}
@Delete(":id")
@ApiOperation({ summary: "حذف آیتم از کارتابل" })
async remove(@Param("id") id: string) {
await (this.prisma as any).approvalRequest.delete({ where: { id } });
return { success: true, message: "Deleted successfully" };
}
}
@@ -3,7 +3,6 @@ import {
Get,
Post,
Put,
Delete,
Body,
Param,
Query,
@@ -208,10 +207,4 @@ export class DepartmentTasksController {
return { success: true, data: updated };
}
@Delete(":id")
@ApiOperation({ summary: "حذف تسک" })
async remove(@Param("id") id: string) {
await (this.prisma as any).departmentTask.delete({ where: { id } });
return { success: true, message: "Deleted successfully" };
}
}
+2
View File
@@ -37,11 +37,13 @@ import { DoctorsModule } from "./doctors/doctors.module";
import { FaqModule } from "./faq/faq.module";
import { MenuModule } from "./menu/menu.module";
import { RevalidationModule } from "./common/revalidation/revalidation.module";
import { OrchestrationModule } from "./orchestration/orchestration.module";
import { Request, Response, NextFunction } from "express";
@Module({
imports: [
OrchestrationModule,
SupportChatModule,
RevalidationModule,
SmsModule,
@@ -0,0 +1,8 @@
import { Type } from "class-transformer";
import { IsEnum, IsHash, IsIn, IsInt, IsISO8601, IsNotEmpty, IsOptional, IsString, Max, Min } from "class-validator";
import { OperationMode, WorkflowState } from "../orchestration.domain";
export class CreateOrchestrationTaskDto { @IsString() @IsNotEmpty() key!:string; @IsString() @IsNotEmpty() title!:string; @IsString() @IsNotEmpty() department!:string; @IsEnum(OperationMode) operationMode!:OperationMode; @Type(()=>Number) @IsInt() @Min(0) budgetLimitCents!:number; @IsOptional() metadata?:Record<string,unknown>; }
export class TransitionTaskDto { @IsEnum(WorkflowState) toState!:WorkflowState; @Type(()=>Number) @IsInt() @Min(1) expectedVersion!:number; @IsString() @IsNotEmpty() idempotencyKey!:string; @IsOptional() @IsString() reason?:string; }
export class GateDecisionDto { @IsIn(["APPROVED","REJECTED"]) decision!:"APPROVED"|"REJECTED"; @Type(()=>Number) @IsInt() @Min(1) expectedVersion!:number; @IsString() @IsNotEmpty() idempotencyKey!:string; @IsISO8601() expiresAt!:string; @IsOptional() @IsString() notes?:string; @IsOptional() @IsString() runId?:string; @IsOptional() @IsHash("sha256") artifactHash?:string; @IsOptional() @IsString() commitSha?:string; @IsOptional() @IsString() environment?:string; }
export class RecordBudgetDto { @Type(()=>Number) @IsInt() @Min(1) amountCents!:number; @IsString() @IsNotEmpty() idempotencyKey!:string; @IsString() @IsNotEmpty() kind!:string; @IsOptional() metadata?:Record<string,unknown>; }
export class SetKillSwitchDto { @IsIn(["FREEZE_NEW_WORK","STOP_AGENTS","DISABLE_MUTATIONS","EMERGENCY_HALT"]) level!:string; @IsString() @IsNotEmpty() reason!:string; }
@@ -0,0 +1,24 @@
import { ForbiddenException } from "@nestjs/common";
import { KillSwitchService } from "./kill-switch.service";
describe("KillSwitchService", () => {
const prisma = { orchestrationKillSwitch: { findUnique: jest.fn() } };
const service = new KillSwitchService(prisma as never);
beforeEach(() => jest.resetAllMocks());
it.each([
["FREEZE_NEW_WORK", "CREATE_TASK"],
["STOP_AGENTS", "RUN_AGENT"],
["DISABLE_MUTATIONS", "MUTATE_PRODUCTION"],
["EMERGENCY_HALT", "OBSERVE"],
])("blocks %s at its policy boundary", async (level, action) => {
prisma.orchestrationKillSwitch.findUnique.mockResolvedValue({ scope: "GLOBAL", level, active: true });
await expect(service.assertAllowed(action as never)).rejects.toThrow(ForbiddenException);
});
it("allows observation while production mutations are disabled", async () => {
prisma.orchestrationKillSwitch.findUnique.mockResolvedValue({ scope: "GLOBAL", level: "DISABLE_MUTATIONS", active: true });
await expect(service.assertAllowed("OBSERVE")).resolves.toBeUndefined();
});
});
@@ -0,0 +1,9 @@
import { ForbiddenException, Injectable } from "@nestjs/common";
import { PrismaService } from "../prisma/prisma.service";
import { KillSwitchLevel } from "./orchestration.domain";
export type GuardedAction="CREATE_TASK"|"RUN_AGENT"|"MUTATE_PRODUCTION"|"OBSERVE";
@Injectable()
export class KillSwitchService {
constructor(private readonly prisma:PrismaService){}
async assertAllowed(action:GuardedAction):Promise<void>{ const state=await (this.prisma as any).orchestrationKillSwitch.findUnique({where:{scope:"GLOBAL"}}); if(!state?.active)return; const blocked:Record<KillSwitchLevel,GuardedAction[]>={FREEZE_NEW_WORK:["CREATE_TASK"],STOP_AGENTS:["CREATE_TASK","RUN_AGENT"],DISABLE_MUTATIONS:["CREATE_TASK","RUN_AGENT","MUTATE_PRODUCTION"],EMERGENCY_HALT:["CREATE_TASK","RUN_AGENT","MUTATE_PRODUCTION","OBSERVE"]}; if(blocked[state.level as KillSwitchLevel]?.includes(action)) throw new ForbiddenException(`Orchestration kill switch ${state.level} blocks ${action}`); }
}
@@ -0,0 +1,3 @@
import { readFileSync } from "node:fs";
import { join } from "node:path";
describe("orchestration migration safety",()=>{it("is additive and never mutates legacy governance tables",()=>{const sql=readFileSync(join(process.cwd(),"prisma/migrations/20261022000000_add_orchestration_v4_core/migration.sql"),"utf8");expect(sql).not.toMatch(/^\s*(DROP|TRUNCATE|DELETE|UPDATE)\b/im);expect(sql).not.toMatch(/ALTER TABLE "(approval_requests|department_tasks|audit_logs)"/i);expect(sql.match(/CREATE TABLE "orchestration_/g)).toHaveLength(9);});});
@@ -0,0 +1,21 @@
import { BadRequestException } from "@nestjs/common";
import { ModelTierPolicyService } from "./model-tier-policy.service";
describe("ModelTierPolicyService", () => {
const policy = new ModelTierPolicyService();
it.each([
[{ complexity: 0, risk: 0, operationMode: "OBSERVE", remainingBudgetCents: 1 }, "L0"],
[{ complexity: 2, risk: 1, operationMode: "PREPARE", remainingBudgetCents: 100 }, "L1"],
[{ complexity: 5, risk: 4, operationMode: "PREPARE", remainingBudgetCents: 500 }, "L2"],
[{ complexity: 7, risk: 7, operationMode: "PREPARE", remainingBudgetCents: 1500 }, "L3"],
[{ complexity: 9, risk: 9, operationMode: "MUTATE_PRODUCTION", remainingBudgetCents: 5000 }, "L4"],
])("selects a deterministic tier for policy inputs", (input, expected) => {
expect(policy.select(input as never)).toBe(expected);
expect(policy.select(input as never)).toBe(expected);
});
it("fails closed when the selected tier exceeds the remaining hard budget", () => {
expect(() => policy.select({ complexity: 9, risk: 9, operationMode: "MUTATE_PRODUCTION", remainingBudgetCents: 100 })).toThrow(BadRequestException);
});
});
@@ -0,0 +1,13 @@
import { BadRequestException, Injectable } from "@nestjs/common";
import { OperationMode } from "./orchestration.domain";
export type ModelTier="L0"|"L1"|"L2"|"L3"|"L4";
@Injectable()
export class ModelTierPolicyService {
select(input:{complexity:number;risk:number;operationMode:OperationMode|string;remainingBudgetCents:number}):ModelTier {
const score=Math.max(input.complexity,input.risk)+(input.operationMode===OperationMode.MUTATE_PRODUCTION?1:0);
const tier:ModelTier=score<=1?"L0":score<=3?"L1":score<=5?"L2":score<=8?"L3":"L4";
const floor:Record<ModelTier,number>={L0:1,L1:25,L2:100,L3:500,L4:2000};
if(input.remainingBudgetCents<floor[tier]) throw new BadRequestException(`Hard budget cannot fund selected model tier ${tier}`);
return tier;
}
}
@@ -0,0 +1,13 @@
import { Body, Controller, Get, Param, Post, Put, Req, UseGuards } from "@nestjs/common";
import { ApiBearerAuth, ApiTags } from "@nestjs/swagger";
import { JwtAuthGuard } from "../auth/jwt-auth.guard"; import { RolesGuard } from "../common/guards/roles.guard"; import { Roles } from "../common/decorators/roles.decorator";
import { CreateOrchestrationTaskDto, GateDecisionDto, RecordBudgetDto, SetKillSwitchDto, TransitionTaskDto } from "./dto/orchestration.dto"; import { OrchestrationService } from "./orchestration.service";
@ApiTags("Admin - Orchestration v4") @ApiBearerAuth() @UseGuards(JwtAuthGuard,RolesGuard) @Roles("Admin","SUPER_ADMIN") @Controller("admin/orchestration")
export class OrchestrationController { constructor(private readonly service:OrchestrationService){} private actor(req:any){return String(req.user?.id??req.user?.sub??"SYSTEM")}
@Get("tasks") all(){return this.service.findAll()} @Get("tasks/:id") one(@Param("id")id:string){return this.service.findOne(id)}
@Post("tasks") create(@Body()dto:CreateOrchestrationTaskDto,@Req()req:any){return this.service.createTask(dto,this.actor(req))}
@Put("tasks/:id/transition") transition(@Param("id")id:string,@Body()dto:TransitionTaskDto,@Req()req:any){return this.service.transition(id,dto,this.actor(req))}
@Post("tasks/:id/gates/:gate/decide") decide(@Param("id")id:string,@Param("gate")gate:string,@Body()dto:GateDecisionDto,@Req()req:any){if(gate!=="1"&&gate!=="2")throw new Error("gate must be 1 or 2");return this.service.decideGate(id,Number(gate) as 1|2,dto,this.actor(req))}
@Post("tasks/:id/budget") budget(@Param("id")id:string,@Body()dto:RecordBudgetDto,@Req()req:any){return this.service.recordBudget(id,dto,this.actor(req))}
@Put("kill-switch") kill(@Body()dto:SetKillSwitchDto,@Req()req:any){return this.service.setKillSwitch(dto.level,dto.reason,this.actor(req))}
}
@@ -0,0 +1,24 @@
import { BadRequestException, ConflictException, ForbiddenException } from "@nestjs/common";
import { assertTransition, nextStateForGateDecision, OperationMode, WorkflowState } from "./orchestration.domain";
describe("orchestration domain", () => {
it("allows only declared workflow transitions", () => {
expect(() => assertTransition(WorkflowState.DRAFT, WorkflowState.GATE1_PENDING)).not.toThrow();
expect(() => assertTransition(WorkflowState.DRAFT, WorkflowState.RUNNING)).toThrow(BadRequestException);
});
it("keeps gate 1 as execution authorization rather than release authorization", () => {
expect(nextStateForGateDecision(1, "APPROVED")).toBe(WorkflowState.GATE1_APPROVED);
expect(nextStateForGateDecision(2, "APPROVED")).toBe(WorkflowState.GATE2_APPROVED);
});
it("rejects an unsupported gate decision", () => {
expect(() => nextStateForGateDecision(1, "PENDING" as never)).toThrow(BadRequestException);
});
it("classifies production mutation separately from observe and prepare", () => {
expect(OperationMode.OBSERVE).toBe("OBSERVE");
expect(OperationMode.PREPARE).toBe("PREPARE");
expect(OperationMode.MUTATE_PRODUCTION).toBe("MUTATE_PRODUCTION");
});
});
@@ -0,0 +1,12 @@
import { BadRequestException } from "@nestjs/common";
export enum WorkflowState { DRAFT="DRAFT", GATE1_PENDING="GATE1_PENDING", GATE1_APPROVED="GATE1_APPROVED", READY="READY", CLAIMED="CLAIMED", RUNNING="RUNNING", VERIFYING="VERIFYING", GATE2_PENDING="GATE2_PENDING", GATE2_APPROVED="GATE2_APPROVED", RELEASING="RELEASING", PRODUCTION_VERIFYING="PRODUCTION_VERIFYING", LIVE_VERIFIED="LIVE_VERIFIED", MEASURING="MEASURING", CLOSED="CLOSED", REJECTED="REJECTED", BLOCKED="BLOCKED", CANCELLED="CANCELLED", FAILED="FAILED", EXPIRED="EXPIRED" }
export enum OperationMode { OBSERVE="OBSERVE", PREPARE="PREPARE", MUTATE_PRODUCTION="MUTATE_PRODUCTION" }
export enum RetryClass { TRANSIENT="TRANSIENT", RATE_LIMITED="RATE_LIMITED", DEPENDENCY="DEPENDENCY", VALIDATION="VALIDATION", POLICY="POLICY", PERMANENT="PERMANENT" }
export enum KillSwitchLevel { FREEZE_NEW_WORK="FREEZE_NEW_WORK", STOP_AGENTS="STOP_AGENTS", DISABLE_MUTATIONS="DISABLE_MUTATIONS", EMERGENCY_HALT="EMERGENCY_HALT" }
const transitions: Record<WorkflowState, WorkflowState[]> = {
DRAFT:[WorkflowState.GATE1_PENDING,WorkflowState.CANCELLED], GATE1_PENDING:[WorkflowState.GATE1_APPROVED,WorkflowState.REJECTED,WorkflowState.EXPIRED], GATE1_APPROVED:[WorkflowState.READY,WorkflowState.CANCELLED], READY:[WorkflowState.CLAIMED,WorkflowState.BLOCKED,WorkflowState.CANCELLED], CLAIMED:[WorkflowState.RUNNING,WorkflowState.READY,WorkflowState.CANCELLED], RUNNING:[WorkflowState.VERIFYING,WorkflowState.FAILED,WorkflowState.BLOCKED], VERIFYING:[WorkflowState.GATE2_PENDING,WorkflowState.FAILED,WorkflowState.RUNNING], GATE2_PENDING:[WorkflowState.GATE2_APPROVED,WorkflowState.REJECTED,WorkflowState.EXPIRED], GATE2_APPROVED:[WorkflowState.RELEASING,WorkflowState.CANCELLED], RELEASING:[WorkflowState.PRODUCTION_VERIFYING,WorkflowState.FAILED], PRODUCTION_VERIFYING:[WorkflowState.LIVE_VERIFIED,WorkflowState.FAILED], LIVE_VERIFIED:[WorkflowState.MEASURING,WorkflowState.CLOSED], MEASURING:[WorkflowState.CLOSED,WorkflowState.FAILED], CLOSED:[], REJECTED:[], BLOCKED:[WorkflowState.READY,WorkflowState.CANCELLED], CANCELLED:[], FAILED:[WorkflowState.READY,WorkflowState.CANCELLED], EXPIRED:[WorkflowState.GATE1_PENDING,WorkflowState.GATE2_PENDING,WorkflowState.CANCELLED]
};
export function assertTransition(from: WorkflowState, to: WorkflowState): void { if (!transitions[from]?.includes(to)) throw new BadRequestException(`Invalid orchestration transition ${from} -> ${to}`); }
export function nextStateForGateDecision(gate: 1|2, decision: "APPROVED"|"REJECTED"): WorkflowState { if (!(["APPROVED","REJECTED"] as string[]).includes(decision)) throw new BadRequestException("Decision must be APPROVED or REJECTED"); return decision === "REJECTED" ? WorkflowState.REJECTED : gate === 1 ? WorkflowState.GATE1_APPROVED : WorkflowState.GATE2_APPROVED; }
@@ -0,0 +1,2 @@
import { Module } from "@nestjs/common"; import { PrismaModule } from "../prisma/prisma.module"; import { KillSwitchService } from "./kill-switch.service"; import { ModelTierPolicyService } from "./model-tier-policy.service"; import { OrchestrationController } from "./orchestration.controller"; import { OrchestrationService } from "./orchestration.service";
@Module({imports:[PrismaModule],controllers:[OrchestrationController],providers:[OrchestrationService,KillSwitchService,ModelTierPolicyService],exports:[OrchestrationService,KillSwitchService,ModelTierPolicyService]}) export class OrchestrationModule{}
@@ -0,0 +1,73 @@
import { BadRequestException, ConflictException, ForbiddenException } from "@nestjs/common";
import { OrchestrationService } from "./orchestration.service";
function mockPrisma() {
const tx: any = {
orchestrationTask: { findUnique: jest.fn(), create: jest.fn(), updateMany: jest.fn() },
orchestrationRun: { findUnique: jest.fn(), create: jest.fn() },
orchestrationArtifact: { findFirst: jest.fn(), create: jest.fn() },
orchestrationAttestation: { create: jest.fn() },
orchestrationApprovalDecision: { findUnique: jest.fn(), create: jest.fn() },
orchestrationEvent: { create: jest.fn() },
orchestrationBudgetLedger: { aggregate: jest.fn(), create: jest.fn() },
orchestrationKillSwitch: { findUnique: jest.fn(), upsert: jest.fn() },
};
tx.$transaction = jest.fn(async (cb: any) => cb(tx));
return tx;
}
describe("OrchestrationService", () => {
let prisma: any;
let service: OrchestrationService;
beforeEach(() => { prisma = mockPrisma(); service = new OrchestrationService(prisma, { assertAllowed: jest.fn() } as never); });
it("creates a task and append-only event transactionally", async () => {
prisma.orchestrationTask.create.mockResolvedValue({ id: "task-1", state: "DRAFT", version: 1 });
const result = await service.createTask({ key: "seo-1", title: "SEO", department: "SEO", operationMode: "PREPARE" as never, budgetLimitCents: 1000 }, "actor-1");
expect(prisma.$transaction).toHaveBeenCalledTimes(1);
expect(prisma.orchestrationTask.create).toHaveBeenCalledWith({ data: expect.objectContaining({ state: "DRAFT", version: 1 }) });
expect(prisma.orchestrationEvent.create).toHaveBeenCalledWith({ data: expect.objectContaining({ eventType: "TASK_CREATED", sequence: 1 }) });
expect(result.id).toBe("task-1");
});
it("uses compare-and-swap versioning for transitions", async () => {
prisma.orchestrationTask.findUnique.mockResolvedValue({ id: "task-1", state: "DRAFT", version: 3 });
prisma.orchestrationTask.updateMany.mockResolvedValue({ count: 0 });
await expect(service.transition("task-1", { toState: "GATE1_PENDING" as never, expectedVersion: 3, idempotencyKey: "i-1" }, "actor")).rejects.toThrow(ConflictException);
});
it("makes gate decisions immutable and idempotent", async () => {
prisma.orchestrationApprovalDecision.findUnique.mockResolvedValue({ id: "d1", decision: "APPROVED", gate: 1, taskId: "task-1" });
await expect(service.decideGate("task-1", 1, { decision: "APPROVED", expectedVersion: 2, idempotencyKey: "gate-1", expiresAt: new Date(Date.now()+10000).toISOString() }, "ceo")).resolves.toMatchObject({ id: "d1" });
expect(prisma.orchestrationApprovalDecision.create).not.toHaveBeenCalled();
});
it("rejects an expired approval", async () => {
await expect(service.decideGate("task-1", 1, { decision: "APPROVED", expectedVersion: 2, idempotencyKey: "gate-expired", expiresAt: new Date(0).toISOString() }, "ceo")).rejects.toThrow(BadRequestException);
});
it("binds gate 2 approval to run, artifact hash, commit and environment", async () => {
prisma.orchestrationApprovalDecision.findUnique.mockResolvedValue(null);
prisma.orchestrationTask.findUnique.mockResolvedValue({ id: "task-1", state: "GATE2_PENDING", version: 7 });
prisma.orchestrationRun.findUnique.mockResolvedValue({ id: "run-1", taskId: "task-1", commitSha: "abc123", environment: "staging" });
prisma.orchestrationArtifact.findFirst.mockResolvedValue({ id: "artifact-1", runId: "run-1", sha256: "f".repeat(64) });
prisma.orchestrationTask.updateMany.mockResolvedValue({ count: 1 });
prisma.orchestrationApprovalDecision.create.mockResolvedValue({ id: "decision-2", gate: 2 });
await service.decideGate("task-1", 2, { decision: "APPROVED", expectedVersion: 7, idempotencyKey: "gate-2", expiresAt: new Date(Date.now()+10000).toISOString(), runId: "run-1", artifactHash: "f".repeat(64), commitSha: "abc123", environment: "staging" }, "ceo");
expect(prisma.orchestrationApprovalDecision.create).toHaveBeenCalledWith({ data: expect.objectContaining({ runId: "run-1", artifactHash: "f".repeat(64), commitSha: "abc123", environment: "staging" }) });
});
it("refuses gate 2 when binding does not match the immutable run", async () => {
prisma.orchestrationApprovalDecision.findUnique.mockResolvedValue(null);
prisma.orchestrationTask.findUnique.mockResolvedValue({ id: "task-1", state: "GATE2_PENDING", version: 7 });
prisma.orchestrationRun.findUnique.mockResolvedValue({ id: "run-1", taskId: "task-1", commitSha: "real", environment: "staging" });
await expect(service.decideGate("task-1", 2, { decision: "APPROVED", expectedVersion: 7, idempotencyKey: "gate-bad", expiresAt: new Date(Date.now()+10000).toISOString(), runId: "run-1", artifactHash: "f".repeat(64), commitSha: "wrong", environment: "staging" }, "ceo")).rejects.toThrow(ForbiddenException);
});
it("enforces hard task budget atomically", async () => {
prisma.orchestrationTask.findUnique.mockResolvedValue({ id: "task-1", budgetLimitCents: 1000 });
prisma.orchestrationBudgetLedger.aggregate.mockResolvedValue({ _sum: { amountCents: 900 } });
await expect(service.recordBudget("task-1", { amountCents: 101, idempotencyKey: "cost-1", kind: "MODEL" }, "agent")).rejects.toThrow(ForbiddenException);
expect(prisma.orchestrationBudgetLedger.create).not.toHaveBeenCalled();
});
});
@@ -0,0 +1,17 @@
import { BadRequestException, ConflictException, ForbiddenException, Injectable, NotFoundException } from "@nestjs/common";
import { PrismaService } from "../prisma/prisma.service";
import { CreateOrchestrationTaskDto, GateDecisionDto, RecordBudgetDto, TransitionTaskDto } from "./dto/orchestration.dto";
import { assertTransition, nextStateForGateDecision, OperationMode, WorkflowState } from "./orchestration.domain";
import { KillSwitchService } from "./kill-switch.service";
@Injectable()
export class OrchestrationService {
constructor(private readonly prisma:PrismaService,private readonly killSwitch:KillSwitchService){}
async createTask(dto:CreateOrchestrationTaskDto,actorId:string){ await this.killSwitch.assertAllowed("CREATE_TASK"); return (this.prisma as any).$transaction(async(tx:any)=>{const task=await tx.orchestrationTask.create({data:{key:dto.key,title:dto.title,department:dto.department,operationMode:dto.operationMode,budgetLimitCents:dto.budgetLimitCents,metadata:dto.metadata??{},state:WorkflowState.DRAFT,version:1}});await tx.orchestrationEvent.create({data:{taskId:task.id,eventType:"TASK_CREATED",fromState:null,toState:WorkflowState.DRAFT,sequence:1,actorId,payload:{key:dto.key}}});return task;}); }
async transition(id:string,dto:TransitionTaskDto,actorId:string){return (this.prisma as any).$transaction(async(tx:any)=>{const existing=await tx.orchestrationTask.findUnique({where:{id}});if(!existing)throw new NotFoundException(`Orchestration task ${id} not found`);assertTransition(existing.state,dto.toState);if(existing.operationMode===OperationMode.MUTATE_PRODUCTION&&dto.toState===WorkflowState.RELEASING)await this.killSwitch.assertAllowed("MUTATE_PRODUCTION");const changed=await tx.orchestrationTask.updateMany({where:{id,version:dto.expectedVersion,state:existing.state},data:{state:dto.toState,version:{increment:1}}});if(changed.count!==1)throw new ConflictException("Task version or state changed; reload before retrying");await tx.orchestrationEvent.create({data:{taskId:id,eventType:"STATE_TRANSITION",fromState:existing.state,toState:dto.toState,sequence:dto.expectedVersion+1,actorId,idempotencyKey:dto.idempotencyKey,payload:{reason:dto.reason??null}}});return {...existing,state:dto.toState,version:dto.expectedVersion+1};});}
async decideGate(taskId:string,gate:1|2,dto:GateDecisionDto,actorId:string){if(new Date(dto.expiresAt)<=new Date())throw new BadRequestException("Approval decision has expired");return (this.prisma as any).$transaction(async(tx:any)=>{const prior=await tx.orchestrationApprovalDecision.findUnique({where:{idempotencyKey:dto.idempotencyKey}});if(prior){if(prior.taskId!==taskId||prior.gate!==gate||prior.decision!==dto.decision)throw new ConflictException("Idempotency key already belongs to another approval decision");return prior;}const task=await tx.orchestrationTask.findUnique({where:{id:taskId}});if(!task)throw new NotFoundException(`Orchestration task ${taskId} not found`);const expected=gate===1?WorkflowState.GATE1_PENDING:WorkflowState.GATE2_PENDING;if(task.state!==expected)throw new ConflictException(`Gate ${gate} requires ${expected}`);if(gate===2&&dto.decision==="APPROVED"){if(!dto.runId||!dto.artifactHash||!dto.commitSha||!dto.environment)throw new BadRequestException("Gate 2 requires runId, artifactHash, commitSha and environment");const run=await tx.orchestrationRun.findUnique({where:{id:dto.runId}});if(!run||run.taskId!==taskId||run.commitSha!==dto.commitSha||run.environment!==dto.environment)throw new ForbiddenException("Gate 2 binding does not match immutable run");const artifact=await tx.orchestrationArtifact.findFirst({where:{runId:dto.runId,sha256:dto.artifactHash}});if(!artifact)throw new ForbiddenException("Gate 2 artifact hash is not registered for run");}
const next=nextStateForGateDecision(gate,dto.decision);const changed=await tx.orchestrationTask.updateMany({where:{id:taskId,version:dto.expectedVersion,state:expected},data:{state:next,version:{increment:1}}});if(changed.count!==1)throw new ConflictException("Task version or state changed; approval not applied");const decision=await tx.orchestrationApprovalDecision.create({data:{taskId,gate,decision:dto.decision,actorId,notes:dto.notes,expiresAt:new Date(dto.expiresAt),idempotencyKey:dto.idempotencyKey,runId:dto.runId,artifactHash:dto.artifactHash,commitSha:dto.commitSha,environment:dto.environment}});await tx.orchestrationEvent.create({data:{taskId,eventType:`GATE${gate}_${dto.decision}`,fromState:expected,toState:next,sequence:dto.expectedVersion+1,actorId,idempotencyKey:`event:${dto.idempotencyKey}`,payload:{decisionId:decision.id}}});return decision;});}
async recordBudget(taskId:string,dto:RecordBudgetDto,actorId:string){return (this.prisma as any).$transaction(async(tx:any)=>{const task=await tx.orchestrationTask.findUnique({where:{id:taskId}});if(!task)throw new NotFoundException(`Orchestration task ${taskId} not found`);const sum=await tx.orchestrationBudgetLedger.aggregate({where:{taskId},_sum:{amountCents:true}});if((sum._sum.amountCents??0)+dto.amountCents>task.budgetLimitCents)throw new ForbiddenException("Hard task budget exceeded");return tx.orchestrationBudgetLedger.create({data:{taskId,amountCents:dto.amountCents,kind:dto.kind,idempotencyKey:dto.idempotencyKey,actorId,metadata:dto.metadata??{}}});});}
async findAll(){return (this.prisma as any).orchestrationTask.findMany({orderBy:{createdAt:"desc"}})}
async findOne(id:string){const task=await (this.prisma as any).orchestrationTask.findUnique({where:{id},include:{runs:{include:{artifacts:true,attestations:true}},dependencies:true,decisions:true,events:{orderBy:{sequence:"asc"}},budgetLedger:true}});if(!task)throw new NotFoundException(`Orchestration task ${id} not found`);return task;}
async setKillSwitch(level:string,reason:string,actorId:string){return (this.prisma as any).$transaction(async(tx:any)=>{const value=await tx.orchestrationKillSwitch.upsert({where:{scope:"GLOBAL"},create:{scope:"GLOBAL",level,active:true,reason,actorId},update:{level,active:true,reason,actorId,version:{increment:1}}});await tx.orchestrationEvent.create({data:{eventType:"KILL_SWITCH_CHANGED",sequence:value.version??1,actorId,payload:{scope:"GLOBAL",level,reason}}});return value;});}
}