668 lines
24 KiB
TypeScript
668 lines
24 KiB
TypeScript
import {
|
|
BadRequestException,
|
|
ConflictException,
|
|
ForbiddenException,
|
|
Injectable,
|
|
NotFoundException,
|
|
} from '@nestjs/common';
|
|
import { PrismaService } from '../prisma/prisma.service';
|
|
import {
|
|
CreateOrchestrationTaskDto,
|
|
GateDecisionDto,
|
|
RecordBudgetDto,
|
|
TransitionTaskDto,
|
|
} from './dto/orchestration.dto';
|
|
import {
|
|
KillSwitchLevel,
|
|
assertTransition,
|
|
nextStateForGateDecision,
|
|
OperationMode,
|
|
WorkflowState,
|
|
} from './orchestration.domain';
|
|
import { ModelTierPolicyService } from './model-tier-policy.service';
|
|
import { KillSwitchService } from './kill-switch.service';
|
|
@Injectable()
|
|
export class OrchestrationService {
|
|
constructor(
|
|
private readonly prisma: PrismaService,
|
|
private readonly killSwitch: KillSwitchService,
|
|
) {}
|
|
async createTask(dto: CreateOrchestrationTaskDto, actorId: string) {
|
|
await this.killSwitch.assertAllowed('CREATE_TASK');
|
|
return await (this.prisma as any).$transaction(async (tx: any) => {
|
|
const task = await tx.orchestrationTask.create({
|
|
data: {
|
|
key: dto.key,
|
|
title: dto.title,
|
|
department: dto.department,
|
|
operationMode: dto.operationMode,
|
|
budgetLimitCents: dto.budgetLimitCents,
|
|
metadata: dto.metadata ?? {},
|
|
state: WorkflowState.DRAFT,
|
|
version: 1,
|
|
},
|
|
});
|
|
await tx.orchestrationEvent.create({
|
|
data: {
|
|
taskId: task.id,
|
|
eventType: 'TASK_CREATED',
|
|
fromState: null,
|
|
toState: WorkflowState.DRAFT,
|
|
sequence: 1,
|
|
actorId,
|
|
payload: { key: dto.key },
|
|
},
|
|
});
|
|
return task;
|
|
});
|
|
}
|
|
async transition(id: string, dto: TransitionTaskDto, actorId: string) {
|
|
if (
|
|
[
|
|
WorkflowState.GATE1_APPROVED,
|
|
WorkflowState.GATE2_APPROVED,
|
|
WorkflowState.RELEASING,
|
|
WorkflowState.RUNNING,
|
|
WorkflowState.VERIFYING,
|
|
WorkflowState.PRODUCTION_VERIFYING,
|
|
WorkflowState.LIVE_VERIFIED,
|
|
WorkflowState.MEASURING,
|
|
WorkflowState.CLOSED,
|
|
].includes(dto.toState)
|
|
)
|
|
throw new ForbiddenException(
|
|
'Gate approvals and release require dedicated commands',
|
|
);
|
|
return await (this.prisma as any).$transaction(async (tx: any) => {
|
|
await tx.$queryRaw`SELECT id FROM orchestration_tasks WHERE id = ${id}::uuid FOR UPDATE`;
|
|
const fingerprint = JSON.stringify({ id, dto, actorId });
|
|
const replay = await tx.orchestrationEvent.findUnique({
|
|
where: { idempotencyKey: dto.idempotencyKey },
|
|
});
|
|
if (replay) {
|
|
if (replay.payload.fingerprint !== fingerprint)
|
|
throw new ConflictException('Idempotency payload mismatch');
|
|
return replay.payload.result;
|
|
}
|
|
const existing = await tx.orchestrationTask.findUnique({ where: { id } });
|
|
if (!existing)
|
|
throw new NotFoundException(`Orchestration task ${id} not found`);
|
|
if (existing.state === WorkflowState.FAILED)
|
|
throw new ForbiddenException('Retries require dedicated run command');
|
|
assertTransition(existing.state, dto.toState);
|
|
await this.killSwitch.assertAllowed(
|
|
[
|
|
WorkflowState.READY,
|
|
WorkflowState.CLAIMED,
|
|
WorkflowState.RUNNING,
|
|
WorkflowState.VERIFYING,
|
|
WorkflowState.GATE2_PENDING,
|
|
].includes(dto.toState)
|
|
? 'RUN_AGENT'
|
|
: 'OBSERVE',
|
|
);
|
|
if ([WorkflowState.READY, WorkflowState.CLAIMED].includes(dto.toState))
|
|
await this.killSwitch.assertAllowed('CLAIM_TASK');
|
|
let selectedTier = existing.modelTier;
|
|
if (dto.toState === WorkflowState.READY) {
|
|
const dependencies = await tx.orchestrationDependency.findMany({
|
|
where: { taskId: id },
|
|
include: { dependsOnTask: true },
|
|
});
|
|
if (dependencies.some((d: any) => d.dependsOnTask.state !== 'CLOSED'))
|
|
throw new ForbiddenException('Unresolved dependency');
|
|
const costs = await tx.orchestrationBudgetLedger.aggregate({
|
|
where: { taskId: id },
|
|
_sum: { amountCents: true },
|
|
});
|
|
selectedTier = new ModelTierPolicyService().select({
|
|
complexity: Number(existing.metadata?.complexity ?? 0),
|
|
risk: Number(existing.metadata?.risk ?? 0),
|
|
operationMode: existing.operationMode,
|
|
remainingBudgetCents:
|
|
existing.budgetLimitCents - (costs._sum.amountCents ?? 0),
|
|
});
|
|
const approval = await tx.orchestrationApprovalDecision.findFirst({
|
|
where: { taskId: id, gate: 1, decision: 'APPROVED' },
|
|
orderBy: { createdAt: 'desc' },
|
|
});
|
|
if (!approval || approval.expiresAt <= new Date())
|
|
throw new ForbiddenException('Gate 1 approval missing or expired');
|
|
}
|
|
if (
|
|
existing.operationMode === OperationMode.MUTATE_PRODUCTION &&
|
|
dto.toState === WorkflowState.RELEASING
|
|
)
|
|
await this.killSwitch.assertAllowed('MUTATE_PRODUCTION');
|
|
const changed = await tx.orchestrationTask.updateMany({
|
|
where: { id, version: dto.expectedVersion, state: existing.state },
|
|
data: {
|
|
state: dto.toState,
|
|
modelTier: selectedTier,
|
|
version: { increment: 1 },
|
|
},
|
|
});
|
|
if (changed.count !== 1)
|
|
throw new ConflictException(
|
|
'Task version or state changed; reload before retrying',
|
|
);
|
|
await tx.orchestrationEvent.create({
|
|
data: {
|
|
taskId: id,
|
|
eventType: 'STATE_TRANSITION',
|
|
fromState: existing.state,
|
|
toState: dto.toState,
|
|
sequence: dto.expectedVersion + 1,
|
|
actorId,
|
|
idempotencyKey: dto.idempotencyKey,
|
|
payload: {
|
|
fingerprint,
|
|
result: JSON.parse(
|
|
JSON.stringify({
|
|
...existing,
|
|
state: dto.toState,
|
|
modelTier: selectedTier,
|
|
version: dto.expectedVersion + 1,
|
|
}),
|
|
),
|
|
},
|
|
},
|
|
});
|
|
return JSON.parse(
|
|
JSON.stringify({
|
|
...existing,
|
|
state: dto.toState,
|
|
modelTier: selectedTier,
|
|
version: dto.expectedVersion + 1,
|
|
}),
|
|
);
|
|
});
|
|
}
|
|
async decideGate(
|
|
taskId: string,
|
|
gate: 1 | 2,
|
|
dto: GateDecisionDto,
|
|
actorId: string,
|
|
) {
|
|
if (![1, 2].includes(gate)) throw new BadRequestException('Invalid gate');
|
|
return await (this.prisma as any).$transaction(async (tx: any) => {
|
|
await tx.$queryRaw`SELECT id FROM orchestration_tasks WHERE id = ${taskId}::uuid FOR UPDATE`;
|
|
const fingerprint = JSON.stringify({ taskId, gate, dto, actorId });
|
|
const prior = await tx.orchestrationApprovalDecision.findUnique({
|
|
where: { idempotencyKey: dto.idempotencyKey },
|
|
});
|
|
if (prior) {
|
|
const event = await tx.orchestrationEvent.findUnique({
|
|
where: { idempotencyKey: `event:${dto.idempotencyKey}` },
|
|
});
|
|
if (event?.payload?.fingerprint !== fingerprint)
|
|
throw new ConflictException('Idempotency payload mismatch');
|
|
return prior;
|
|
}
|
|
if (
|
|
typeof dto.expiresAt !== 'string' ||
|
|
!Number.isFinite(Date.parse(dto.expiresAt)) ||
|
|
new Date(dto.expiresAt) <= new Date()
|
|
)
|
|
throw new BadRequestException('Approval decision has expired');
|
|
await this.killSwitch.assertAllowed('OBSERVE');
|
|
const task = await tx.orchestrationTask.findUnique({
|
|
where: { id: taskId },
|
|
});
|
|
if (!task)
|
|
throw new NotFoundException(`Orchestration task ${taskId} not found`);
|
|
const expected =
|
|
gate === 1 ? WorkflowState.GATE1_PENDING : WorkflowState.GATE2_PENDING;
|
|
if (task.state !== expected)
|
|
throw new ConflictException(`Gate ${gate} requires ${expected}`);
|
|
if (gate === 2 && dto.decision === 'APPROVED') {
|
|
if (
|
|
!dto.runId ||
|
|
!dto.artifactHash ||
|
|
!dto.commitSha ||
|
|
!dto.environment
|
|
)
|
|
throw new BadRequestException(
|
|
'Gate 2 requires runId, artifactHash, commitSha and environment',
|
|
);
|
|
const run = await tx.orchestrationRun.findUnique({
|
|
where: { id: dto.runId },
|
|
});
|
|
if (
|
|
!run ||
|
|
run.taskId !== taskId ||
|
|
run.commitSha !== dto.commitSha ||
|
|
run.environment !== dto.environment
|
|
)
|
|
throw new ForbiddenException(
|
|
'Gate 2 binding does not match immutable run',
|
|
);
|
|
if (run.status !== 'SUCCEEDED')
|
|
throw new ForbiddenException('Gate 2 requires a successful run');
|
|
const artifact = await tx.orchestrationArtifact.findFirst({
|
|
where: { runId: dto.runId, sha256: dto.artifactHash },
|
|
});
|
|
if (!artifact)
|
|
throw new ForbiddenException(
|
|
'Gate 2 artifact hash is not registered for run',
|
|
);
|
|
}
|
|
const next = nextStateForGateDecision(gate, dto.decision);
|
|
const changed = await tx.orchestrationTask.updateMany({
|
|
where: { id: taskId, version: dto.expectedVersion, state: expected },
|
|
data: { state: next, version: { increment: 1 } },
|
|
});
|
|
if (changed.count !== 1)
|
|
throw new ConflictException(
|
|
'Task version or state changed; approval not applied',
|
|
);
|
|
const decision = await tx.orchestrationApprovalDecision.create({
|
|
data: {
|
|
taskId,
|
|
gate,
|
|
decision: dto.decision,
|
|
actorId,
|
|
notes: dto.notes,
|
|
expiresAt: new Date(dto.expiresAt),
|
|
idempotencyKey: dto.idempotencyKey,
|
|
runId: dto.runId,
|
|
artifactHash: dto.artifactHash,
|
|
commitSha: dto.commitSha,
|
|
environment: dto.environment,
|
|
},
|
|
});
|
|
await tx.orchestrationEvent.create({
|
|
data: {
|
|
taskId,
|
|
eventType: `GATE${gate}_${dto.decision}`,
|
|
fromState: expected,
|
|
toState: next,
|
|
sequence: dto.expectedVersion + 1,
|
|
actorId,
|
|
idempotencyKey: `event:${dto.idempotencyKey}`,
|
|
payload: { decisionId: decision.id, fingerprint },
|
|
},
|
|
});
|
|
return decision;
|
|
});
|
|
}
|
|
async recordBudget(taskId: string, dto: RecordBudgetDto, actorId: string) {
|
|
if (
|
|
!Number.isSafeInteger(dto.amountCents) ||
|
|
dto.amountCents < 1 ||
|
|
!['RESERVE', 'SPEND'].includes(dto.kind)
|
|
)
|
|
throw new BadRequestException(
|
|
'Use RESERVE or SPEND with positive integer cents',
|
|
);
|
|
return await (this.prisma as any).$transaction(async (tx: any) => {
|
|
await tx.$queryRaw`SELECT id FROM orchestration_tasks WHERE id = ${taskId}::uuid FOR UPDATE`;
|
|
const prior = await tx.orchestrationBudgetLedger.findUnique({
|
|
where: { idempotencyKey: dto.idempotencyKey },
|
|
});
|
|
if (prior) {
|
|
if (
|
|
prior.taskId !== taskId ||
|
|
prior.kind !== dto.kind ||
|
|
prior.amountCents !== dto.amountCents ||
|
|
prior.actorId !== actorId ||
|
|
JSON.stringify(prior.metadata) !== JSON.stringify(dto.metadata ?? {})
|
|
)
|
|
throw new ConflictException('Idempotency payload mismatch');
|
|
return prior;
|
|
}
|
|
await this.killSwitch.assertAllowed('RUN_AGENT');
|
|
const task = await tx.orchestrationTask.findUnique({
|
|
where: { id: taskId },
|
|
});
|
|
if (!task)
|
|
throw new NotFoundException(`Orchestration task ${taskId} not found`);
|
|
const sum = await tx.orchestrationBudgetLedger.aggregate({
|
|
where: { taskId },
|
|
_sum: { amountCents: true },
|
|
});
|
|
if ((sum._sum.amountCents ?? 0) + dto.amountCents > task.budgetLimitCents)
|
|
throw new ForbiddenException('Hard task budget exceeded');
|
|
return tx.orchestrationBudgetLedger.create({
|
|
data: {
|
|
taskId,
|
|
amountCents: dto.amountCents,
|
|
kind: dto.kind,
|
|
idempotencyKey: dto.idempotencyKey,
|
|
actorId,
|
|
metadata: dto.metadata ?? {},
|
|
},
|
|
});
|
|
});
|
|
}
|
|
async command(taskId: string, command: string, dto: any, actorId: string) {
|
|
return await (this.prisma as any).$transaction(async (tx: any) => {
|
|
await tx.$queryRaw`SELECT id FROM orchestration_tasks WHERE id = ${taskId}::uuid FOR UPDATE`;
|
|
const fingerprint = JSON.stringify({ taskId, command, dto, actorId });
|
|
const replay = await tx.orchestrationEvent.findUnique({
|
|
where: { idempotencyKey: dto.idempotencyKey },
|
|
});
|
|
if (replay) {
|
|
if (replay.payload.fingerprint !== fingerprint)
|
|
throw new ConflictException('Idempotency payload mismatch');
|
|
return replay.payload.result;
|
|
}
|
|
const task = await tx.orchestrationTask.findUnique({
|
|
where: { id: taskId },
|
|
});
|
|
if (!task) throw new NotFoundException('Task not found');
|
|
if (task.version !== dto.expectedVersion)
|
|
throw new ConflictException('Stale task version');
|
|
await this.killSwitch.assertAllowed('RUN_AGENT');
|
|
let result: any;
|
|
let state = task.state;
|
|
if (command === 'RUN_CREATE') {
|
|
if (
|
|
!/^[a-f0-9]{40,64}$/.test(dto.commitSha) ||
|
|
!['development', 'staging', 'production'].includes(dto.environment)
|
|
)
|
|
throw new BadRequestException('Invalid immutable run binding');
|
|
const last = await tx.orchestrationRun.findFirst({
|
|
where: { taskId },
|
|
orderBy: { attempt: 'desc' },
|
|
});
|
|
if (
|
|
last &&
|
|
(last.status !== 'FAILED' ||
|
|
!['TRANSIENT', 'RATE_LIMITED', 'DEPENDENCY'].includes(
|
|
last.retryClass,
|
|
) ||
|
|
last.attempt >= 3)
|
|
)
|
|
throw new ForbiddenException('Retry denied or exhausted');
|
|
if (!last && task.state !== 'CLAIMED')
|
|
throw new ConflictException('Run requires CLAIMED task');
|
|
result = await tx.orchestrationRun.create({
|
|
data: {
|
|
taskId,
|
|
attempt: (last?.attempt ?? 0) + 1,
|
|
status: 'RUNNING',
|
|
commitSha: dto.commitSha,
|
|
environment: dto.environment,
|
|
idempotencyKey: dto.idempotencyKey,
|
|
startedAt: new Date(),
|
|
},
|
|
});
|
|
state = 'RUNNING';
|
|
} else if (command === 'RUN_FINISH') {
|
|
const run = await tx.orchestrationRun.findUnique({
|
|
where: { id: dto.runId },
|
|
});
|
|
if (
|
|
!run ||
|
|
run.taskId !== taskId ||
|
|
run.status !== 'RUNNING' ||
|
|
task.state !== 'RUNNING'
|
|
)
|
|
throw new ConflictException('Active run required');
|
|
if (
|
|
!['SUCCEEDED', 'FAILED'].includes(dto.status) ||
|
|
(dto.status === 'FAILED' &&
|
|
![
|
|
'TRANSIENT',
|
|
'RATE_LIMITED',
|
|
'DEPENDENCY',
|
|
'VALIDATION',
|
|
'POLICY',
|
|
'PERMANENT',
|
|
].includes(dto.retryClass))
|
|
)
|
|
throw new BadRequestException('Invalid outcome taxonomy');
|
|
result = await tx.orchestrationRun.update({
|
|
where: { id: run.id },
|
|
data: {
|
|
status: dto.status,
|
|
retryClass: dto.status === 'FAILED' ? dto.retryClass : null,
|
|
finishedAt: new Date(),
|
|
},
|
|
});
|
|
state = dto.status === 'FAILED' ? 'FAILED' : 'VERIFYING';
|
|
} else if (command === 'ARTIFACT_ADD' || command === 'ATTESTATION_ADD') {
|
|
const run = await tx.orchestrationRun.findUnique({
|
|
where: { id: dto.runId },
|
|
});
|
|
if (
|
|
!run ||
|
|
run.taskId !== taskId ||
|
|
!['RUNNING', 'VERIFYING'].includes(task.state)
|
|
)
|
|
throw new ConflictException('Owned active run required');
|
|
const hash = command === 'ARTIFACT_ADD' ? dto.sha256 : dto.subjectHash;
|
|
if (!/^[a-f0-9]{64}$/.test(hash) || !dto.kind)
|
|
throw new BadRequestException('Invalid artifact hash or kind');
|
|
if (command === 'ARTIFACT_ADD') {
|
|
if (!/^https:\/\//.test(dto.uri))
|
|
throw new BadRequestException('Artifact URI must be HTTPS');
|
|
result = await tx.orchestrationArtifact.create({
|
|
data: { runId: run.id, kind: dto.kind, sha256: hash, uri: dto.uri },
|
|
});
|
|
} else {
|
|
if (
|
|
!(await tx.orchestrationArtifact.findFirst({
|
|
where: { runId: run.id, sha256: hash },
|
|
}))
|
|
)
|
|
throw new ForbiddenException(
|
|
'Attestation subject is not a registered artifact',
|
|
);
|
|
result = await tx.orchestrationAttestation.create({
|
|
data: {
|
|
runId: run.id,
|
|
kind: dto.kind,
|
|
subjectHash: hash,
|
|
predicate: dto.predicate ?? {},
|
|
signer: actorId,
|
|
signature: dto.signature,
|
|
},
|
|
});
|
|
}
|
|
} else if (command === 'DEPENDENCY_ADD') {
|
|
if (task.state !== 'DRAFT')
|
|
throw new ConflictException(
|
|
'Dependencies are frozen after submission',
|
|
);
|
|
await tx.$executeRaw`LOCK TABLE orchestration_dependencies IN EXCLUSIVE MODE`;
|
|
const cycles =
|
|
await tx.$queryRaw`WITH RECURSIVE chain(id) AS (SELECT ${dto.dependsOnTaskId}::uuid UNION SELECT d.depends_on_task_id FROM orchestration_dependencies d JOIN chain c ON d.task_id=c.id) SELECT id FROM chain WHERE id=${taskId}::uuid`;
|
|
if (cycles.length) throw new ForbiddenException('Dependency cycle');
|
|
result = await tx.orchestrationDependency.create({
|
|
data: { taskId, dependsOnTaskId: dto.dependsOnTaskId },
|
|
});
|
|
} else if (command === 'RELEASE_AUTHORIZE') {
|
|
await this.killSwitch.assertAllowed('MUTATE_PRODUCTION');
|
|
if (task.state !== 'GATE2_APPROVED')
|
|
throw new ConflictException(
|
|
'Gate 2 approval required; already consumed or invalid state',
|
|
);
|
|
const approval = await tx.orchestrationApprovalDecision.findUnique({
|
|
where: { id: dto.approvalId },
|
|
});
|
|
if (
|
|
!approval ||
|
|
approval.taskId !== taskId ||
|
|
approval.gate !== 2 ||
|
|
approval.decision !== 'APPROVED' ||
|
|
approval.expiresAt <= new Date()
|
|
)
|
|
throw new ForbiddenException('Gate 2 missing or expired');
|
|
for (const key of ['runId', 'artifactHash', 'commitSha', 'environment'])
|
|
if (approval[key] !== dto[key])
|
|
throw new ForbiddenException('Release binding mismatch');
|
|
const run = await tx.orchestrationRun.findUnique({
|
|
where: { id: dto.runId },
|
|
});
|
|
if (
|
|
!run ||
|
|
run.taskId !== taskId ||
|
|
run.status !== 'SUCCEEDED' ||
|
|
run.commitSha !== dto.commitSha ||
|
|
run.environment !== dto.environment
|
|
)
|
|
throw new ForbiddenException('Run binding mismatch');
|
|
if (
|
|
!(await tx.orchestrationArtifact.findFirst({
|
|
where: { runId: run.id, sha256: dto.artifactHash },
|
|
})) ||
|
|
!(await tx.orchestrationAttestation.findFirst({
|
|
where: {
|
|
runId: run.id,
|
|
subjectHash: dto.artifactHash,
|
|
kind: 'TEST',
|
|
},
|
|
}))
|
|
)
|
|
throw new ForbiddenException('Artifact evidence missing');
|
|
result = {
|
|
authorized: true,
|
|
approvalId: approval.id,
|
|
runId: run.id,
|
|
artifactHash: dto.artifactHash,
|
|
commitSha: dto.commitSha,
|
|
environment: dto.environment,
|
|
execution: 'CI_CD_ONLY',
|
|
};
|
|
state = 'RELEASING';
|
|
} else throw new BadRequestException('Unknown command');
|
|
await tx.orchestrationTask.update({
|
|
where: { id: taskId },
|
|
data: { state, version: { increment: 1 } },
|
|
});
|
|
const serialized = JSON.parse(JSON.stringify(result));
|
|
await tx.orchestrationEvent.create({
|
|
data: {
|
|
taskId,
|
|
eventType: command,
|
|
fromState: task.state,
|
|
toState: state,
|
|
sequence: task.version + 1,
|
|
actorId,
|
|
idempotencyKey: dto.idempotencyKey,
|
|
payload: { fingerprint, result: serialized },
|
|
},
|
|
});
|
|
return serialized;
|
|
});
|
|
}
|
|
async reconcileBudget(
|
|
taskId: string,
|
|
dto: { reservationId: string; actualCents: number; idempotencyKey: string },
|
|
actorId: string,
|
|
) {
|
|
await this.killSwitch.assertAllowed('OBSERVE');
|
|
if (!Number.isSafeInteger(dto.actualCents) || dto.actualCents < 0)
|
|
throw new BadRequestException('Invalid actual cost');
|
|
return await (this.prisma as any).$transaction(async (tx: any) => {
|
|
await tx.$queryRaw`SELECT id FROM orchestration_tasks WHERE id = ${taskId}::uuid FOR UPDATE`;
|
|
const prior = await tx.orchestrationBudgetLedger.findUnique({
|
|
where: { idempotencyKey: dto.idempotencyKey },
|
|
});
|
|
if (prior) {
|
|
if (
|
|
prior.taskId !== taskId ||
|
|
prior.actorId !== actorId ||
|
|
prior.metadata.reservationId !== dto.reservationId ||
|
|
prior.metadata.actualCents !== dto.actualCents
|
|
)
|
|
throw new ConflictException('Idempotency payload mismatch');
|
|
return prior;
|
|
}
|
|
const reservation = await tx.orchestrationBudgetLedger.findUnique({
|
|
where: { id: dto.reservationId },
|
|
});
|
|
if (
|
|
!reservation ||
|
|
reservation.taskId !== taskId ||
|
|
reservation.kind !== 'RESERVE'
|
|
)
|
|
throw new BadRequestException('Reservation not found');
|
|
if (
|
|
await tx.orchestrationBudgetLedger.findFirst({
|
|
where: {
|
|
taskId,
|
|
kind: 'RECONCILE',
|
|
metadata: { path: ['reservationId'], equals: dto.reservationId },
|
|
},
|
|
})
|
|
)
|
|
throw new ConflictException('Reservation already reconciled');
|
|
const task = await tx.orchestrationTask.findUnique({
|
|
where: { id: taskId },
|
|
});
|
|
const total = await tx.orchestrationBudgetLedger.aggregate({
|
|
where: { taskId },
|
|
_sum: { amountCents: true },
|
|
});
|
|
const delta = dto.actualCents - reservation.amountCents;
|
|
if (total._sum.amountCents + delta > task.budgetLimitCents)
|
|
throw new ForbiddenException('Hard task budget exceeded');
|
|
return tx.orchestrationBudgetLedger.create({
|
|
data: {
|
|
taskId,
|
|
kind: 'RECONCILE',
|
|
amountCents: delta,
|
|
idempotencyKey: dto.idempotencyKey,
|
|
actorId,
|
|
metadata: {
|
|
reservationId: dto.reservationId,
|
|
actualCents: dto.actualCents,
|
|
},
|
|
},
|
|
});
|
|
});
|
|
}
|
|
async findAll() {
|
|
await this.killSwitch.assertAllowed('OBSERVE');
|
|
return (this.prisma as any).orchestrationTask.findMany({
|
|
orderBy: { createdAt: 'desc' },
|
|
include: { decisions: true },
|
|
});
|
|
}
|
|
async findOne(id: string) {
|
|
await this.killSwitch.assertAllowed('OBSERVE');
|
|
const task = await (this.prisma as any).orchestrationTask.findUnique({
|
|
where: { id },
|
|
include: {
|
|
runs: { include: { artifacts: true, attestations: true } },
|
|
dependencies: true,
|
|
decisions: true,
|
|
events: { orderBy: { sequence: 'asc' } },
|
|
budgetLedger: true,
|
|
},
|
|
});
|
|
if (!task)
|
|
throw new NotFoundException(`Orchestration task ${id} not found`);
|
|
return task;
|
|
}
|
|
async setKillSwitch(level: string, reason: string, actorId: string) {
|
|
if (!Object.values(KillSwitchLevel).includes(level as KillSwitchLevel))
|
|
throw new BadRequestException('Invalid kill switch level');
|
|
return await (this.prisma as any).$transaction(async (tx: any) => {
|
|
const value = await tx.orchestrationKillSwitch.upsert({
|
|
where: { scope: 'GLOBAL' },
|
|
create: { scope: 'GLOBAL', level, active: true, reason, actorId },
|
|
update: {
|
|
level,
|
|
active: true,
|
|
reason,
|
|
actorId,
|
|
version: { increment: 1 },
|
|
},
|
|
});
|
|
await tx.orchestrationEvent.create({
|
|
data: {
|
|
eventType: 'KILL_SWITCH_CHANGED',
|
|
sequence: value.version ?? 1,
|
|
actorId,
|
|
payload: { scope: 'GLOBAL', level, reason },
|
|
},
|
|
});
|
|
return value;
|
|
});
|
|
}
|
|
}
|