Checkpoint recovered orchestration core with tested fail-closed boundaries
This commit is contained in:
1 parent
c7f1d25e98
commit
0952a482fb
18 files changed
+1918
-129
No files matched your search
@@ -1,17 +1,665 @@
|
||||
import { BadRequestException, ConflictException, ForbiddenException, Injectable, NotFoundException } from "@nestjs/common";
|
||||
import { PrismaService } from "../prisma/prisma.service";
|
||||
import { CreateOrchestrationTaskDto, GateDecisionDto, RecordBudgetDto, TransitionTaskDto } from "./dto/orchestration.dto";
|
||||
import { assertTransition, nextStateForGateDecision, OperationMode, WorkflowState } from "./orchestration.domain";
|
||||
import { KillSwitchService } from "./kill-switch.service";
|
||||
import {
|
||||
BadRequestException,
|
||||
ConflictException,
|
||||
ForbiddenException,
|
||||
Injectable,
|
||||
NotFoundException,
|
||||
} from '@nestjs/common';
|
||||
import { PrismaService } from '../prisma/prisma.service';
|
||||
import {
|
||||
CreateOrchestrationTaskDto,
|
||||
GateDecisionDto,
|
||||
RecordBudgetDto,
|
||||
TransitionTaskDto,
|
||||
} from './dto/orchestration.dto';
|
||||
import {
|
||||
KillSwitchLevel,
|
||||
assertTransition,
|
||||
nextStateForGateDecision,
|
||||
OperationMode,
|
||||
WorkflowState,
|
||||
} from './orchestration.domain';
|
||||
import { ModelTierPolicyService } from './model-tier-policy.service';
|
||||
import { KillSwitchService } from './kill-switch.service';
|
||||
@Injectable()
|
||||
export class OrchestrationService {
|
||||
constructor(private readonly prisma:PrismaService,private readonly killSwitch:KillSwitchService){}
|
||||
async createTask(dto:CreateOrchestrationTaskDto,actorId:string){ await this.killSwitch.assertAllowed("CREATE_TASK"); return (this.prisma as any).$transaction(async(tx:any)=>{const task=await tx.orchestrationTask.create({data:{key:dto.key,title:dto.title,department:dto.department,operationMode:dto.operationMode,budgetLimitCents:dto.budgetLimitCents,metadata:dto.metadata??{},state:WorkflowState.DRAFT,version:1}});await tx.orchestrationEvent.create({data:{taskId:task.id,eventType:"TASK_CREATED",fromState:null,toState:WorkflowState.DRAFT,sequence:1,actorId,payload:{key:dto.key}}});return task;}); }
|
||||
async transition(id:string,dto:TransitionTaskDto,actorId:string){return (this.prisma as any).$transaction(async(tx:any)=>{const existing=await tx.orchestrationTask.findUnique({where:{id}});if(!existing)throw new NotFoundException(`Orchestration task ${id} not found`);assertTransition(existing.state,dto.toState);if(existing.operationMode===OperationMode.MUTATE_PRODUCTION&&dto.toState===WorkflowState.RELEASING)await this.killSwitch.assertAllowed("MUTATE_PRODUCTION");const changed=await tx.orchestrationTask.updateMany({where:{id,version:dto.expectedVersion,state:existing.state},data:{state:dto.toState,version:{increment:1}}});if(changed.count!==1)throw new ConflictException("Task version or state changed; reload before retrying");await tx.orchestrationEvent.create({data:{taskId:id,eventType:"STATE_TRANSITION",fromState:existing.state,toState:dto.toState,sequence:dto.expectedVersion+1,actorId,idempotencyKey:dto.idempotencyKey,payload:{reason:dto.reason??null}}});return {...existing,state:dto.toState,version:dto.expectedVersion+1};});}
|
||||
async decideGate(taskId:string,gate:1|2,dto:GateDecisionDto,actorId:string){if(new Date(dto.expiresAt)<=new Date())throw new BadRequestException("Approval decision has expired");return (this.prisma as any).$transaction(async(tx:any)=>{const prior=await tx.orchestrationApprovalDecision.findUnique({where:{idempotencyKey:dto.idempotencyKey}});if(prior){if(prior.taskId!==taskId||prior.gate!==gate||prior.decision!==dto.decision)throw new ConflictException("Idempotency key already belongs to another approval decision");return prior;}const task=await tx.orchestrationTask.findUnique({where:{id:taskId}});if(!task)throw new NotFoundException(`Orchestration task ${taskId} not found`);const expected=gate===1?WorkflowState.GATE1_PENDING:WorkflowState.GATE2_PENDING;if(task.state!==expected)throw new ConflictException(`Gate ${gate} requires ${expected}`);if(gate===2&&dto.decision==="APPROVED"){if(!dto.runId||!dto.artifactHash||!dto.commitSha||!dto.environment)throw new BadRequestException("Gate 2 requires runId, artifactHash, commitSha and environment");const run=await tx.orchestrationRun.findUnique({where:{id:dto.runId}});if(!run||run.taskId!==taskId||run.commitSha!==dto.commitSha||run.environment!==dto.environment)throw new ForbiddenException("Gate 2 binding does not match immutable run");const artifact=await tx.orchestrationArtifact.findFirst({where:{runId:dto.runId,sha256:dto.artifactHash}});if(!artifact)throw new ForbiddenException("Gate 2 artifact hash is not registered for run");}
|
||||
const next=nextStateForGateDecision(gate,dto.decision);const changed=await tx.orchestrationTask.updateMany({where:{id:taskId,version:dto.expectedVersion,state:expected},data:{state:next,version:{increment:1}}});if(changed.count!==1)throw new ConflictException("Task version or state changed; approval not applied");const decision=await tx.orchestrationApprovalDecision.create({data:{taskId,gate,decision:dto.decision,actorId,notes:dto.notes,expiresAt:new Date(dto.expiresAt),idempotencyKey:dto.idempotencyKey,runId:dto.runId,artifactHash:dto.artifactHash,commitSha:dto.commitSha,environment:dto.environment}});await tx.orchestrationEvent.create({data:{taskId,eventType:`GATE${gate}_${dto.decision}`,fromState:expected,toState:next,sequence:dto.expectedVersion+1,actorId,idempotencyKey:`event:${dto.idempotencyKey}`,payload:{decisionId:decision.id}}});return decision;});}
|
||||
async recordBudget(taskId:string,dto:RecordBudgetDto,actorId:string){return (this.prisma as any).$transaction(async(tx:any)=>{const task=await tx.orchestrationTask.findUnique({where:{id:taskId}});if(!task)throw new NotFoundException(`Orchestration task ${taskId} not found`);const sum=await tx.orchestrationBudgetLedger.aggregate({where:{taskId},_sum:{amountCents:true}});if((sum._sum.amountCents??0)+dto.amountCents>task.budgetLimitCents)throw new ForbiddenException("Hard task budget exceeded");return tx.orchestrationBudgetLedger.create({data:{taskId,amountCents:dto.amountCents,kind:dto.kind,idempotencyKey:dto.idempotencyKey,actorId,metadata:dto.metadata??{}}});});}
|
||||
async findAll(){return (this.prisma as any).orchestrationTask.findMany({orderBy:{createdAt:"desc"}})}
|
||||
async findOne(id:string){const task=await (this.prisma as any).orchestrationTask.findUnique({where:{id},include:{runs:{include:{artifacts:true,attestations:true}},dependencies:true,decisions:true,events:{orderBy:{sequence:"asc"}},budgetLedger:true}});if(!task)throw new NotFoundException(`Orchestration task ${id} not found`);return task;}
|
||||
async setKillSwitch(level:string,reason:string,actorId:string){return (this.prisma as any).$transaction(async(tx:any)=>{const value=await tx.orchestrationKillSwitch.upsert({where:{scope:"GLOBAL"},create:{scope:"GLOBAL",level,active:true,reason,actorId},update:{level,active:true,reason,actorId,version:{increment:1}}});await tx.orchestrationEvent.create({data:{eventType:"KILL_SWITCH_CHANGED",sequence:value.version??1,actorId,payload:{scope:"GLOBAL",level,reason}}});return value;});}
|
||||
constructor(
|
||||
private readonly prisma: PrismaService,
|
||||
private readonly killSwitch: KillSwitchService,
|
||||
) {}
|
||||
async createTask(dto: CreateOrchestrationTaskDto, actorId: string) {
|
||||
await this.killSwitch.assertAllowed('CREATE_TASK');
|
||||
return await (this.prisma as any).$transaction(async (tx: any) => {
|
||||
const task = await tx.orchestrationTask.create({
|
||||
data: {
|
||||
key: dto.key,
|
||||
title: dto.title,
|
||||
department: dto.department,
|
||||
operationMode: dto.operationMode,
|
||||
budgetLimitCents: dto.budgetLimitCents,
|
||||
metadata: dto.metadata ?? {},
|
||||
state: WorkflowState.DRAFT,
|
||||
version: 1,
|
||||
},
|
||||
});
|
||||
await tx.orchestrationEvent.create({
|
||||
data: {
|
||||
taskId: task.id,
|
||||
eventType: 'TASK_CREATED',
|
||||
fromState: null,
|
||||
toState: WorkflowState.DRAFT,
|
||||
sequence: 1,
|
||||
actorId,
|
||||
payload: { key: dto.key },
|
||||
},
|
||||
});
|
||||
return task;
|
||||
});
|
||||
}
|
||||
async transition(id: string, dto: TransitionTaskDto, actorId: string) {
|
||||
if (
|
||||
[
|
||||
WorkflowState.GATE1_APPROVED,
|
||||
WorkflowState.GATE2_APPROVED,
|
||||
WorkflowState.RELEASING,
|
||||
WorkflowState.RUNNING,
|
||||
WorkflowState.VERIFYING,
|
||||
WorkflowState.PRODUCTION_VERIFYING,
|
||||
WorkflowState.LIVE_VERIFIED,
|
||||
WorkflowState.MEASURING,
|
||||
WorkflowState.CLOSED,
|
||||
].includes(dto.toState)
|
||||
)
|
||||
throw new ForbiddenException(
|
||||
'Gate approvals and release require dedicated commands',
|
||||
);
|
||||
return await (this.prisma as any).$transaction(async (tx: any) => {
|
||||
await tx.$queryRaw`SELECT id FROM orchestration_tasks WHERE id = ${id}::uuid FOR UPDATE`;
|
||||
const fingerprint = JSON.stringify({ id, dto, actorId });
|
||||
const replay = await tx.orchestrationEvent.findUnique({
|
||||
where: { idempotencyKey: dto.idempotencyKey },
|
||||
});
|
||||
if (replay) {
|
||||
if (replay.payload.fingerprint !== fingerprint)
|
||||
throw new ConflictException('Idempotency payload mismatch');
|
||||
return replay.payload.result;
|
||||
}
|
||||
const existing = await tx.orchestrationTask.findUnique({ where: { id } });
|
||||
if (!existing)
|
||||
throw new NotFoundException(`Orchestration task ${id} not found`);
|
||||
if (existing.state === WorkflowState.FAILED)
|
||||
throw new ForbiddenException('Retries require dedicated run command');
|
||||
assertTransition(existing.state, dto.toState);
|
||||
await this.killSwitch.assertAllowed(
|
||||
[
|
||||
WorkflowState.READY,
|
||||
WorkflowState.CLAIMED,
|
||||
WorkflowState.RUNNING,
|
||||
WorkflowState.VERIFYING,
|
||||
WorkflowState.GATE2_PENDING,
|
||||
].includes(dto.toState)
|
||||
? 'RUN_AGENT'
|
||||
: 'OBSERVE',
|
||||
);
|
||||
if ([WorkflowState.READY, WorkflowState.CLAIMED].includes(dto.toState))
|
||||
await this.killSwitch.assertAllowed('CLAIM_TASK');
|
||||
let selectedTier = existing.modelTier;
|
||||
if (dto.toState === WorkflowState.READY) {
|
||||
const dependencies = await tx.orchestrationDependency.findMany({
|
||||
where: { taskId: id },
|
||||
include: { dependsOnTask: true },
|
||||
});
|
||||
if (dependencies.some((d: any) => d.dependsOnTask.state !== 'CLOSED'))
|
||||
throw new ForbiddenException('Unresolved dependency');
|
||||
const costs = await tx.orchestrationBudgetLedger.aggregate({
|
||||
where: { taskId: id },
|
||||
_sum: { amountCents: true },
|
||||
});
|
||||
selectedTier = new ModelTierPolicyService().select({
|
||||
complexity: Number(existing.metadata?.complexity ?? 0),
|
||||
risk: Number(existing.metadata?.risk ?? 0),
|
||||
operationMode: existing.operationMode,
|
||||
remainingBudgetCents:
|
||||
existing.budgetLimitCents - (costs._sum.amountCents ?? 0),
|
||||
});
|
||||
const approval = await tx.orchestrationApprovalDecision.findFirst({
|
||||
where: { taskId: id, gate: 1, decision: 'APPROVED' },
|
||||
orderBy: { createdAt: 'desc' },
|
||||
});
|
||||
if (!approval || approval.expiresAt <= new Date())
|
||||
throw new ForbiddenException('Gate 1 approval missing or expired');
|
||||
}
|
||||
if (
|
||||
existing.operationMode === OperationMode.MUTATE_PRODUCTION &&
|
||||
dto.toState === WorkflowState.RELEASING
|
||||
)
|
||||
await this.killSwitch.assertAllowed('MUTATE_PRODUCTION');
|
||||
const changed = await tx.orchestrationTask.updateMany({
|
||||
where: { id, version: dto.expectedVersion, state: existing.state },
|
||||
data: {
|
||||
state: dto.toState,
|
||||
modelTier: selectedTier,
|
||||
version: { increment: 1 },
|
||||
},
|
||||
});
|
||||
if (changed.count !== 1)
|
||||
throw new ConflictException(
|
||||
'Task version or state changed; reload before retrying',
|
||||
);
|
||||
await tx.orchestrationEvent.create({
|
||||
data: {
|
||||
taskId: id,
|
||||
eventType: 'STATE_TRANSITION',
|
||||
fromState: existing.state,
|
||||
toState: dto.toState,
|
||||
sequence: dto.expectedVersion + 1,
|
||||
actorId,
|
||||
idempotencyKey: dto.idempotencyKey,
|
||||
payload: {
|
||||
fingerprint,
|
||||
result: JSON.parse(
|
||||
JSON.stringify({
|
||||
...existing,
|
||||
state: dto.toState,
|
||||
modelTier: selectedTier,
|
||||
version: dto.expectedVersion + 1,
|
||||
}),
|
||||
),
|
||||
},
|
||||
},
|
||||
});
|
||||
return JSON.parse(
|
||||
JSON.stringify({
|
||||
...existing,
|
||||
state: dto.toState,
|
||||
modelTier: selectedTier,
|
||||
version: dto.expectedVersion + 1,
|
||||
}),
|
||||
);
|
||||
});
|
||||
}
|
||||
async decideGate(
|
||||
taskId: string,
|
||||
gate: 1 | 2,
|
||||
dto: GateDecisionDto,
|
||||
actorId: string,
|
||||
) {
|
||||
if (![1, 2].includes(gate)) throw new BadRequestException('Invalid gate');
|
||||
return await (this.prisma as any).$transaction(async (tx: any) => {
|
||||
await tx.$queryRaw`SELECT id FROM orchestration_tasks WHERE id = ${taskId}::uuid FOR UPDATE`;
|
||||
const fingerprint = JSON.stringify({ taskId, gate, dto, actorId });
|
||||
const prior = await tx.orchestrationApprovalDecision.findUnique({
|
||||
where: { idempotencyKey: dto.idempotencyKey },
|
||||
});
|
||||
if (prior) {
|
||||
const event = await tx.orchestrationEvent.findUnique({
|
||||
where: { idempotencyKey: `event:${dto.idempotencyKey}` },
|
||||
});
|
||||
if (event?.payload?.fingerprint !== fingerprint)
|
||||
throw new ConflictException('Idempotency payload mismatch');
|
||||
return prior;
|
||||
}
|
||||
if (
|
||||
!Number.isFinite(Date.parse(dto.expiresAt)) ||
|
||||
new Date(dto.expiresAt) <= new Date()
|
||||
)
|
||||
throw new BadRequestException('Approval decision has expired');
|
||||
await this.killSwitch.assertAllowed('OBSERVE');
|
||||
const task = await tx.orchestrationTask.findUnique({
|
||||
where: { id: taskId },
|
||||
});
|
||||
if (!task)
|
||||
throw new NotFoundException(`Orchestration task ${taskId} not found`);
|
||||
const expected =
|
||||
gate === 1 ? WorkflowState.GATE1_PENDING : WorkflowState.GATE2_PENDING;
|
||||
if (task.state !== expected)
|
||||
throw new ConflictException(`Gate ${gate} requires ${expected}`);
|
||||
if (gate === 2 && dto.decision === 'APPROVED') {
|
||||
if (
|
||||
!dto.runId ||
|
||||
!dto.artifactHash ||
|
||||
!dto.commitSha ||
|
||||
!dto.environment
|
||||
)
|
||||
throw new BadRequestException(
|
||||
'Gate 2 requires runId, artifactHash, commitSha and environment',
|
||||
);
|
||||
const run = await tx.orchestrationRun.findUnique({
|
||||
where: { id: dto.runId },
|
||||
});
|
||||
if (
|
||||
!run ||
|
||||
run.taskId !== taskId ||
|
||||
run.commitSha !== dto.commitSha ||
|
||||
run.environment !== dto.environment
|
||||
)
|
||||
throw new ForbiddenException(
|
||||
'Gate 2 binding does not match immutable run',
|
||||
);
|
||||
if (run.status !== 'SUCCEEDED')
|
||||
throw new ForbiddenException('Gate 2 requires a successful run');
|
||||
const artifact = await tx.orchestrationArtifact.findFirst({
|
||||
where: { runId: dto.runId, sha256: dto.artifactHash },
|
||||
});
|
||||
if (!artifact)
|
||||
throw new ForbiddenException(
|
||||
'Gate 2 artifact hash is not registered for run',
|
||||
);
|
||||
}
|
||||
const next = nextStateForGateDecision(gate, dto.decision);
|
||||
const changed = await tx.orchestrationTask.updateMany({
|
||||
where: { id: taskId, version: dto.expectedVersion, state: expected },
|
||||
data: { state: next, version: { increment: 1 } },
|
||||
});
|
||||
if (changed.count !== 1)
|
||||
throw new ConflictException(
|
||||
'Task version or state changed; approval not applied',
|
||||
);
|
||||
const decision = await tx.orchestrationApprovalDecision.create({
|
||||
data: {
|
||||
taskId,
|
||||
gate,
|
||||
decision: dto.decision,
|
||||
actorId,
|
||||
notes: dto.notes,
|
||||
expiresAt: new Date(dto.expiresAt),
|
||||
idempotencyKey: dto.idempotencyKey,
|
||||
runId: dto.runId,
|
||||
artifactHash: dto.artifactHash,
|
||||
commitSha: dto.commitSha,
|
||||
environment: dto.environment,
|
||||
},
|
||||
});
|
||||
await tx.orchestrationEvent.create({
|
||||
data: {
|
||||
taskId,
|
||||
eventType: `GATE${gate}_${dto.decision}`,
|
||||
fromState: expected,
|
||||
toState: next,
|
||||
sequence: dto.expectedVersion + 1,
|
||||
actorId,
|
||||
idempotencyKey: `event:${dto.idempotencyKey}`,
|
||||
payload: { decisionId: decision.id, fingerprint },
|
||||
},
|
||||
});
|
||||
return decision;
|
||||
});
|
||||
}
|
||||
async recordBudget(taskId: string, dto: RecordBudgetDto, actorId: string) {
|
||||
if (
|
||||
!Number.isSafeInteger(dto.amountCents) ||
|
||||
dto.amountCents < 1 ||
|
||||
!['RESERVE', 'SPEND'].includes(dto.kind)
|
||||
)
|
||||
throw new BadRequestException(
|
||||
'Use RESERVE or SPEND with positive integer cents',
|
||||
);
|
||||
return await (this.prisma as any).$transaction(async (tx: any) => {
|
||||
await tx.$queryRaw`SELECT id FROM orchestration_tasks WHERE id = ${taskId}::uuid FOR UPDATE`;
|
||||
const prior = await tx.orchestrationBudgetLedger.findUnique({
|
||||
where: { idempotencyKey: dto.idempotencyKey },
|
||||
});
|
||||
if (prior) {
|
||||
if (
|
||||
prior.taskId !== taskId ||
|
||||
prior.kind !== dto.kind ||
|
||||
prior.amountCents !== dto.amountCents ||
|
||||
prior.actorId !== actorId ||
|
||||
JSON.stringify(prior.metadata) !== JSON.stringify(dto.metadata ?? {})
|
||||
)
|
||||
throw new ConflictException('Idempotency payload mismatch');
|
||||
return prior;
|
||||
}
|
||||
await this.killSwitch.assertAllowed('RUN_AGENT');
|
||||
const task = await tx.orchestrationTask.findUnique({
|
||||
where: { id: taskId },
|
||||
});
|
||||
if (!task)
|
||||
throw new NotFoundException(`Orchestration task ${taskId} not found`);
|
||||
const sum = await tx.orchestrationBudgetLedger.aggregate({
|
||||
where: { taskId },
|
||||
_sum: { amountCents: true },
|
||||
});
|
||||
if ((sum._sum.amountCents ?? 0) + dto.amountCents > task.budgetLimitCents)
|
||||
throw new ForbiddenException('Hard task budget exceeded');
|
||||
return tx.orchestrationBudgetLedger.create({
|
||||
data: {
|
||||
taskId,
|
||||
amountCents: dto.amountCents,
|
||||
kind: dto.kind,
|
||||
idempotencyKey: dto.idempotencyKey,
|
||||
actorId,
|
||||
metadata: dto.metadata ?? {},
|
||||
},
|
||||
});
|
||||
});
|
||||
}
|
||||
async command(taskId: string, command: string, dto: any, actorId: string) {
|
||||
return await (this.prisma as any).$transaction(async (tx: any) => {
|
||||
await tx.$queryRaw`SELECT id FROM orchestration_tasks WHERE id = ${taskId}::uuid FOR UPDATE`;
|
||||
const fingerprint = JSON.stringify({ taskId, command, dto, actorId });
|
||||
const replay = await tx.orchestrationEvent.findUnique({
|
||||
where: { idempotencyKey: dto.idempotencyKey },
|
||||
});
|
||||
if (replay) {
|
||||
if (replay.payload.fingerprint !== fingerprint)
|
||||
throw new ConflictException('Idempotency payload mismatch');
|
||||
return replay.payload.result;
|
||||
}
|
||||
const task = await tx.orchestrationTask.findUnique({
|
||||
where: { id: taskId },
|
||||
});
|
||||
if (!task) throw new NotFoundException('Task not found');
|
||||
if (task.version !== dto.expectedVersion)
|
||||
throw new ConflictException('Stale task version');
|
||||
await this.killSwitch.assertAllowed('RUN_AGENT');
|
||||
let result: any;
|
||||
let state = task.state;
|
||||
if (command === 'RUN_CREATE') {
|
||||
if (
|
||||
!/^[a-f0-9]{40,64}$/.test(dto.commitSha) ||
|
||||
!['development', 'staging', 'production'].includes(dto.environment)
|
||||
)
|
||||
throw new BadRequestException('Invalid immutable run binding');
|
||||
const last = await tx.orchestrationRun.findFirst({
|
||||
where: { taskId },
|
||||
orderBy: { attempt: 'desc' },
|
||||
});
|
||||
if (
|
||||
last &&
|
||||
(last.status !== 'FAILED' ||
|
||||
!['TRANSIENT', 'RATE_LIMITED', 'DEPENDENCY'].includes(
|
||||
last.retryClass,
|
||||
) ||
|
||||
last.attempt >= 3)
|
||||
)
|
||||
throw new ForbiddenException('Retry denied or exhausted');
|
||||
if (!last && task.state !== 'CLAIMED')
|
||||
throw new ConflictException('Run requires CLAIMED task');
|
||||
result = await tx.orchestrationRun.create({
|
||||
data: {
|
||||
taskId,
|
||||
attempt: (last?.attempt ?? 0) + 1,
|
||||
status: 'RUNNING',
|
||||
commitSha: dto.commitSha,
|
||||
environment: dto.environment,
|
||||
idempotencyKey: dto.idempotencyKey,
|
||||
startedAt: new Date(),
|
||||
},
|
||||
});
|
||||
state = 'RUNNING';
|
||||
} else if (command === 'RUN_FINISH') {
|
||||
const run = await tx.orchestrationRun.findUnique({
|
||||
where: { id: dto.runId },
|
||||
});
|
||||
if (
|
||||
!run ||
|
||||
run.taskId !== taskId ||
|
||||
run.status !== 'RUNNING' ||
|
||||
task.state !== 'RUNNING'
|
||||
)
|
||||
throw new ConflictException('Active run required');
|
||||
if (
|
||||
!['SUCCEEDED', 'FAILED'].includes(dto.status) ||
|
||||
(dto.status === 'FAILED' &&
|
||||
![
|
||||
'TRANSIENT',
|
||||
'RATE_LIMITED',
|
||||
'DEPENDENCY',
|
||||
'VALIDATION',
|
||||
'POLICY',
|
||||
'PERMANENT',
|
||||
].includes(dto.retryClass))
|
||||
)
|
||||
throw new BadRequestException('Invalid outcome taxonomy');
|
||||
result = await tx.orchestrationRun.update({
|
||||
where: { id: run.id },
|
||||
data: {
|
||||
status: dto.status,
|
||||
retryClass: dto.status === 'FAILED' ? dto.retryClass : null,
|
||||
finishedAt: new Date(),
|
||||
},
|
||||
});
|
||||
state = dto.status === 'FAILED' ? 'FAILED' : 'VERIFYING';
|
||||
} else if (command === 'ARTIFACT_ADD' || command === 'ATTESTATION_ADD') {
|
||||
const run = await tx.orchestrationRun.findUnique({
|
||||
where: { id: dto.runId },
|
||||
});
|
||||
if (
|
||||
!run ||
|
||||
run.taskId !== taskId ||
|
||||
!['RUNNING', 'VERIFYING'].includes(task.state)
|
||||
)
|
||||
throw new ConflictException('Owned active run required');
|
||||
const hash = command === 'ARTIFACT_ADD' ? dto.sha256 : dto.subjectHash;
|
||||
if (!/^[a-f0-9]{64}$/.test(hash) || !dto.kind)
|
||||
throw new BadRequestException('Invalid artifact hash or kind');
|
||||
if (command === 'ARTIFACT_ADD') {
|
||||
if (!/^https:\/\//.test(dto.uri))
|
||||
throw new BadRequestException('Artifact URI must be HTTPS');
|
||||
result = await tx.orchestrationArtifact.create({
|
||||
data: { runId: run.id, kind: dto.kind, sha256: hash, uri: dto.uri },
|
||||
});
|
||||
} else {
|
||||
if (
|
||||
!(await tx.orchestrationArtifact.findFirst({
|
||||
where: { runId: run.id, sha256: hash },
|
||||
}))
|
||||
)
|
||||
throw new ForbiddenException(
|
||||
'Attestation subject is not a registered artifact',
|
||||
);
|
||||
result = await tx.orchestrationAttestation.create({
|
||||
data: {
|
||||
runId: run.id,
|
||||
kind: dto.kind,
|
||||
subjectHash: hash,
|
||||
predicate: dto.predicate ?? {},
|
||||
signer: actorId,
|
||||
signature: dto.signature,
|
||||
},
|
||||
});
|
||||
}
|
||||
} else if (command === 'DEPENDENCY_ADD') {
|
||||
if (task.state !== 'DRAFT')
|
||||
throw new ConflictException(
|
||||
'Dependencies are frozen after submission',
|
||||
);
|
||||
await tx.$executeRaw`LOCK TABLE orchestration_dependencies IN EXCLUSIVE MODE`;
|
||||
const cycles =
|
||||
await tx.$queryRaw`WITH RECURSIVE chain(id) AS (SELECT ${dto.dependsOnTaskId}::uuid UNION SELECT d.depends_on_task_id FROM orchestration_dependencies d JOIN chain c ON d.task_id=c.id) SELECT id FROM chain WHERE id=${taskId}::uuid`;
|
||||
if (cycles.length) throw new ForbiddenException('Dependency cycle');
|
||||
result = await tx.orchestrationDependency.create({
|
||||
data: { taskId, dependsOnTaskId: dto.dependsOnTaskId },
|
||||
});
|
||||
} else if (command === 'RELEASE_AUTHORIZE') {
|
||||
await this.killSwitch.assertAllowed('MUTATE_PRODUCTION');
|
||||
if (task.state !== 'GATE2_APPROVED')
|
||||
throw new ConflictException(
|
||||
'Gate 2 approval required; already consumed or invalid state',
|
||||
);
|
||||
const approval = await tx.orchestrationApprovalDecision.findUnique({
|
||||
where: { id: dto.approvalId },
|
||||
});
|
||||
if (
|
||||
!approval ||
|
||||
approval.taskId !== taskId ||
|
||||
approval.gate !== 2 ||
|
||||
approval.decision !== 'APPROVED' ||
|
||||
approval.expiresAt <= new Date()
|
||||
)
|
||||
throw new ForbiddenException('Gate 2 missing or expired');
|
||||
for (const key of ['runId', 'artifactHash', 'commitSha', 'environment'])
|
||||
if (approval[key] !== dto[key])
|
||||
throw new ForbiddenException('Release binding mismatch');
|
||||
const run = await tx.orchestrationRun.findUnique({
|
||||
where: { id: dto.runId },
|
||||
});
|
||||
if (
|
||||
!run ||
|
||||
run.taskId !== taskId ||
|
||||
run.status !== 'SUCCEEDED' ||
|
||||
run.commitSha !== dto.commitSha ||
|
||||
run.environment !== dto.environment
|
||||
)
|
||||
throw new ForbiddenException('Run binding mismatch');
|
||||
if (
|
||||
!(await tx.orchestrationArtifact.findFirst({
|
||||
where: { runId: run.id, sha256: dto.artifactHash },
|
||||
})) ||
|
||||
!(await tx.orchestrationAttestation.findFirst({
|
||||
where: {
|
||||
runId: run.id,
|
||||
subjectHash: dto.artifactHash,
|
||||
kind: 'TEST',
|
||||
},
|
||||
}))
|
||||
)
|
||||
throw new ForbiddenException('Artifact evidence missing');
|
||||
result = {
|
||||
authorized: true,
|
||||
approvalId: approval.id,
|
||||
runId: run.id,
|
||||
artifactHash: dto.artifactHash,
|
||||
commitSha: dto.commitSha,
|
||||
environment: dto.environment,
|
||||
execution: 'CI_CD_ONLY',
|
||||
};
|
||||
state = 'RELEASING';
|
||||
} else throw new BadRequestException('Unknown command');
|
||||
await tx.orchestrationTask.update({
|
||||
where: { id: taskId },
|
||||
data: { state, version: { increment: 1 } },
|
||||
});
|
||||
const serialized = JSON.parse(JSON.stringify(result));
|
||||
await tx.orchestrationEvent.create({
|
||||
data: {
|
||||
taskId,
|
||||
eventType: command,
|
||||
fromState: task.state,
|
||||
toState: state,
|
||||
sequence: task.version + 1,
|
||||
actorId,
|
||||
idempotencyKey: dto.idempotencyKey,
|
||||
payload: { fingerprint, result: serialized },
|
||||
},
|
||||
});
|
||||
return serialized;
|
||||
});
|
||||
}
|
||||
async reconcileBudget(
|
||||
taskId: string,
|
||||
dto: { reservationId: string; actualCents: number; idempotencyKey: string },
|
||||
actorId: string,
|
||||
) {
|
||||
await this.killSwitch.assertAllowed('OBSERVE');
|
||||
if (!Number.isSafeInteger(dto.actualCents) || dto.actualCents < 0)
|
||||
throw new BadRequestException('Invalid actual cost');
|
||||
return await (this.prisma as any).$transaction(async (tx: any) => {
|
||||
await tx.$queryRaw`SELECT id FROM orchestration_tasks WHERE id = ${taskId}::uuid FOR UPDATE`;
|
||||
const prior = await tx.orchestrationBudgetLedger.findUnique({
|
||||
where: { idempotencyKey: dto.idempotencyKey },
|
||||
});
|
||||
if (prior) {
|
||||
if (
|
||||
prior.taskId !== taskId ||
|
||||
prior.actorId !== actorId ||
|
||||
prior.metadata.reservationId !== dto.reservationId ||
|
||||
prior.metadata.actualCents !== dto.actualCents
|
||||
)
|
||||
throw new ConflictException('Idempotency payload mismatch');
|
||||
return prior;
|
||||
}
|
||||
const reservation = await tx.orchestrationBudgetLedger.findUnique({
|
||||
where: { id: dto.reservationId },
|
||||
});
|
||||
if (
|
||||
!reservation ||
|
||||
reservation.taskId !== taskId ||
|
||||
reservation.kind !== 'RESERVE'
|
||||
)
|
||||
throw new BadRequestException('Reservation not found');
|
||||
if (
|
||||
await tx.orchestrationBudgetLedger.findFirst({
|
||||
where: {
|
||||
taskId,
|
||||
kind: 'RECONCILE',
|
||||
metadata: { path: ['reservationId'], equals: dto.reservationId },
|
||||
},
|
||||
})
|
||||
)
|
||||
throw new ConflictException('Reservation already reconciled');
|
||||
const task = await tx.orchestrationTask.findUnique({
|
||||
where: { id: taskId },
|
||||
});
|
||||
const total = await tx.orchestrationBudgetLedger.aggregate({
|
||||
where: { taskId },
|
||||
_sum: { amountCents: true },
|
||||
});
|
||||
const delta = dto.actualCents - reservation.amountCents;
|
||||
if (total._sum.amountCents + delta > task.budgetLimitCents)
|
||||
throw new ForbiddenException('Hard task budget exceeded');
|
||||
return tx.orchestrationBudgetLedger.create({
|
||||
data: {
|
||||
taskId,
|
||||
kind: 'RECONCILE',
|
||||
amountCents: delta,
|
||||
idempotencyKey: dto.idempotencyKey,
|
||||
actorId,
|
||||
metadata: {
|
||||
reservationId: dto.reservationId,
|
||||
actualCents: dto.actualCents,
|
||||
},
|
||||
},
|
||||
});
|
||||
});
|
||||
}
|
||||
async findAll() {
|
||||
await this.killSwitch.assertAllowed('OBSERVE');
|
||||
return (this.prisma as any).orchestrationTask.findMany({
|
||||
orderBy: { createdAt: 'desc' },
|
||||
});
|
||||
}
|
||||
async findOne(id: string) {
|
||||
await this.killSwitch.assertAllowed('OBSERVE');
|
||||
const task = await (this.prisma as any).orchestrationTask.findUnique({
|
||||
where: { id },
|
||||
include: {
|
||||
runs: { include: { artifacts: true, attestations: true } },
|
||||
dependencies: true,
|
||||
decisions: true,
|
||||
events: { orderBy: { sequence: 'asc' } },
|
||||
budgetLedger: true,
|
||||
},
|
||||
});
|
||||
if (!task)
|
||||
throw new NotFoundException(`Orchestration task ${id} not found`);
|
||||
return task;
|
||||
}
|
||||
async setKillSwitch(level: string, reason: string, actorId: string) {
|
||||
if (!Object.values(KillSwitchLevel).includes(level as KillSwitchLevel))
|
||||
throw new BadRequestException('Invalid kill switch level');
|
||||
return await (this.prisma as any).$transaction(async (tx: any) => {
|
||||
const value = await tx.orchestrationKillSwitch.upsert({
|
||||
where: { scope: 'GLOBAL' },
|
||||
create: { scope: 'GLOBAL', level, active: true, reason, actorId },
|
||||
update: {
|
||||
level,
|
||||
active: true,
|
||||
reason,
|
||||
actorId,
|
||||
version: { increment: 1 },
|
||||
},
|
||||
});
|
||||
await tx.orchestrationEvent.create({
|
||||
data: {
|
||||
eventType: 'KILL_SWITCH_CHANGED',
|
||||
sequence: value.version ?? 1,
|
||||
actorId,
|
||||
payload: { scope: 'GLOBAL', level, reason },
|
||||
},
|
||||
});
|
||||
return value;
|
||||
});
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user