Files
canina/backend/src/orchestration/orchestration.service.ts
T
2026-10-24 09:53:07 +00:00

668 lines
24 KiB
TypeScript

import {
BadRequestException,
ConflictException,
ForbiddenException,
Injectable,
NotFoundException,
} from '@nestjs/common';
import { PrismaService } from '../prisma/prisma.service';
import {
CreateOrchestrationTaskDto,
GateDecisionDto,
RecordBudgetDto,
TransitionTaskDto,
} from './dto/orchestration.dto';
import {
KillSwitchLevel,
assertTransition,
nextStateForGateDecision,
OperationMode,
WorkflowState,
} from './orchestration.domain';
import { ModelTierPolicyService } from './model-tier-policy.service';
import { KillSwitchService } from './kill-switch.service';
@Injectable()
export class OrchestrationService {
constructor(
private readonly prisma: PrismaService,
private readonly killSwitch: KillSwitchService,
) {}
async createTask(dto: CreateOrchestrationTaskDto, actorId: string) {
await this.killSwitch.assertAllowed('CREATE_TASK');
return await (this.prisma as any).$transaction(async (tx: any) => {
const task = await tx.orchestrationTask.create({
data: {
key: dto.key,
title: dto.title,
department: dto.department,
operationMode: dto.operationMode,
budgetLimitCents: dto.budgetLimitCents,
metadata: dto.metadata ?? {},
state: WorkflowState.DRAFT,
version: 1,
},
});
await tx.orchestrationEvent.create({
data: {
taskId: task.id,
eventType: 'TASK_CREATED',
fromState: null,
toState: WorkflowState.DRAFT,
sequence: 1,
actorId,
payload: { key: dto.key },
},
});
return task;
});
}
async transition(id: string, dto: TransitionTaskDto, actorId: string) {
if (
[
WorkflowState.GATE1_APPROVED,
WorkflowState.GATE2_APPROVED,
WorkflowState.RELEASING,
WorkflowState.RUNNING,
WorkflowState.VERIFYING,
WorkflowState.PRODUCTION_VERIFYING,
WorkflowState.LIVE_VERIFIED,
WorkflowState.MEASURING,
WorkflowState.CLOSED,
].includes(dto.toState)
)
throw new ForbiddenException(
'Gate approvals and release require dedicated commands',
);
return await (this.prisma as any).$transaction(async (tx: any) => {
await tx.$queryRaw`SELECT id FROM orchestration_tasks WHERE id = ${id}::uuid FOR UPDATE`;
const fingerprint = JSON.stringify({ id, dto, actorId });
const replay = await tx.orchestrationEvent.findUnique({
where: { idempotencyKey: dto.idempotencyKey },
});
if (replay) {
if (replay.payload.fingerprint !== fingerprint)
throw new ConflictException('Idempotency payload mismatch');
return replay.payload.result;
}
const existing = await tx.orchestrationTask.findUnique({ where: { id } });
if (!existing)
throw new NotFoundException(`Orchestration task ${id} not found`);
if (existing.state === WorkflowState.FAILED)
throw new ForbiddenException('Retries require dedicated run command');
assertTransition(existing.state, dto.toState);
await this.killSwitch.assertAllowed(
[
WorkflowState.READY,
WorkflowState.CLAIMED,
WorkflowState.RUNNING,
WorkflowState.VERIFYING,
WorkflowState.GATE2_PENDING,
].includes(dto.toState)
? 'RUN_AGENT'
: 'OBSERVE',
);
if ([WorkflowState.READY, WorkflowState.CLAIMED].includes(dto.toState))
await this.killSwitch.assertAllowed('CLAIM_TASK');
let selectedTier = existing.modelTier;
if (dto.toState === WorkflowState.READY) {
const dependencies = await tx.orchestrationDependency.findMany({
where: { taskId: id },
include: { dependsOnTask: true },
});
if (dependencies.some((d: any) => d.dependsOnTask.state !== 'CLOSED'))
throw new ForbiddenException('Unresolved dependency');
const costs = await tx.orchestrationBudgetLedger.aggregate({
where: { taskId: id },
_sum: { amountCents: true },
});
selectedTier = new ModelTierPolicyService().select({
complexity: Number(existing.metadata?.complexity ?? 0),
risk: Number(existing.metadata?.risk ?? 0),
operationMode: existing.operationMode,
remainingBudgetCents:
existing.budgetLimitCents - (costs._sum.amountCents ?? 0),
});
const approval = await tx.orchestrationApprovalDecision.findFirst({
where: { taskId: id, gate: 1, decision: 'APPROVED' },
orderBy: { createdAt: 'desc' },
});
if (!approval || approval.expiresAt <= new Date())
throw new ForbiddenException('Gate 1 approval missing or expired');
}
if (
existing.operationMode === OperationMode.MUTATE_PRODUCTION &&
dto.toState === WorkflowState.RELEASING
)
await this.killSwitch.assertAllowed('MUTATE_PRODUCTION');
const changed = await tx.orchestrationTask.updateMany({
where: { id, version: dto.expectedVersion, state: existing.state },
data: {
state: dto.toState,
modelTier: selectedTier,
version: { increment: 1 },
},
});
if (changed.count !== 1)
throw new ConflictException(
'Task version or state changed; reload before retrying',
);
await tx.orchestrationEvent.create({
data: {
taskId: id,
eventType: 'STATE_TRANSITION',
fromState: existing.state,
toState: dto.toState,
sequence: dto.expectedVersion + 1,
actorId,
idempotencyKey: dto.idempotencyKey,
payload: {
fingerprint,
result: JSON.parse(
JSON.stringify({
...existing,
state: dto.toState,
modelTier: selectedTier,
version: dto.expectedVersion + 1,
}),
),
},
},
});
return JSON.parse(
JSON.stringify({
...existing,
state: dto.toState,
modelTier: selectedTier,
version: dto.expectedVersion + 1,
}),
);
});
}
async decideGate(
taskId: string,
gate: 1 | 2,
dto: GateDecisionDto,
actorId: string,
) {
if (![1, 2].includes(gate)) throw new BadRequestException('Invalid gate');
return await (this.prisma as any).$transaction(async (tx: any) => {
await tx.$queryRaw`SELECT id FROM orchestration_tasks WHERE id = ${taskId}::uuid FOR UPDATE`;
const fingerprint = JSON.stringify({ taskId, gate, dto, actorId });
const prior = await tx.orchestrationApprovalDecision.findUnique({
where: { idempotencyKey: dto.idempotencyKey },
});
if (prior) {
const event = await tx.orchestrationEvent.findUnique({
where: { idempotencyKey: `event:${dto.idempotencyKey}` },
});
if (event?.payload?.fingerprint !== fingerprint)
throw new ConflictException('Idempotency payload mismatch');
return prior;
}
if (
typeof dto.expiresAt !== 'string' ||
!Number.isFinite(Date.parse(dto.expiresAt)) ||
new Date(dto.expiresAt) <= new Date()
)
throw new BadRequestException('Approval decision has expired');
await this.killSwitch.assertAllowed('OBSERVE');
const task = await tx.orchestrationTask.findUnique({
where: { id: taskId },
});
if (!task)
throw new NotFoundException(`Orchestration task ${taskId} not found`);
const expected =
gate === 1 ? WorkflowState.GATE1_PENDING : WorkflowState.GATE2_PENDING;
if (task.state !== expected)
throw new ConflictException(`Gate ${gate} requires ${expected}`);
if (gate === 2 && dto.decision === 'APPROVED') {
if (
!dto.runId ||
!dto.artifactHash ||
!dto.commitSha ||
!dto.environment
)
throw new BadRequestException(
'Gate 2 requires runId, artifactHash, commitSha and environment',
);
const run = await tx.orchestrationRun.findUnique({
where: { id: dto.runId },
});
if (
!run ||
run.taskId !== taskId ||
run.commitSha !== dto.commitSha ||
run.environment !== dto.environment
)
throw new ForbiddenException(
'Gate 2 binding does not match immutable run',
);
if (run.status !== 'SUCCEEDED')
throw new ForbiddenException('Gate 2 requires a successful run');
const artifact = await tx.orchestrationArtifact.findFirst({
where: { runId: dto.runId, sha256: dto.artifactHash },
});
if (!artifact)
throw new ForbiddenException(
'Gate 2 artifact hash is not registered for run',
);
}
const next = nextStateForGateDecision(gate, dto.decision);
const changed = await tx.orchestrationTask.updateMany({
where: { id: taskId, version: dto.expectedVersion, state: expected },
data: { state: next, version: { increment: 1 } },
});
if (changed.count !== 1)
throw new ConflictException(
'Task version or state changed; approval not applied',
);
const decision = await tx.orchestrationApprovalDecision.create({
data: {
taskId,
gate,
decision: dto.decision,
actorId,
notes: dto.notes,
expiresAt: new Date(dto.expiresAt),
idempotencyKey: dto.idempotencyKey,
runId: dto.runId,
artifactHash: dto.artifactHash,
commitSha: dto.commitSha,
environment: dto.environment,
},
});
await tx.orchestrationEvent.create({
data: {
taskId,
eventType: `GATE${gate}_${dto.decision}`,
fromState: expected,
toState: next,
sequence: dto.expectedVersion + 1,
actorId,
idempotencyKey: `event:${dto.idempotencyKey}`,
payload: { decisionId: decision.id, fingerprint },
},
});
return decision;
});
}
async recordBudget(taskId: string, dto: RecordBudgetDto, actorId: string) {
if (
!Number.isSafeInteger(dto.amountCents) ||
dto.amountCents < 1 ||
!['RESERVE', 'SPEND'].includes(dto.kind)
)
throw new BadRequestException(
'Use RESERVE or SPEND with positive integer cents',
);
return await (this.prisma as any).$transaction(async (tx: any) => {
await tx.$queryRaw`SELECT id FROM orchestration_tasks WHERE id = ${taskId}::uuid FOR UPDATE`;
const prior = await tx.orchestrationBudgetLedger.findUnique({
where: { idempotencyKey: dto.idempotencyKey },
});
if (prior) {
if (
prior.taskId !== taskId ||
prior.kind !== dto.kind ||
prior.amountCents !== dto.amountCents ||
prior.actorId !== actorId ||
JSON.stringify(prior.metadata) !== JSON.stringify(dto.metadata ?? {})
)
throw new ConflictException('Idempotency payload mismatch');
return prior;
}
await this.killSwitch.assertAllowed('RUN_AGENT');
const task = await tx.orchestrationTask.findUnique({
where: { id: taskId },
});
if (!task)
throw new NotFoundException(`Orchestration task ${taskId} not found`);
const sum = await tx.orchestrationBudgetLedger.aggregate({
where: { taskId },
_sum: { amountCents: true },
});
if ((sum._sum.amountCents ?? 0) + dto.amountCents > task.budgetLimitCents)
throw new ForbiddenException('Hard task budget exceeded');
return tx.orchestrationBudgetLedger.create({
data: {
taskId,
amountCents: dto.amountCents,
kind: dto.kind,
idempotencyKey: dto.idempotencyKey,
actorId,
metadata: dto.metadata ?? {},
},
});
});
}
async command(taskId: string, command: string, dto: any, actorId: string) {
return await (this.prisma as any).$transaction(async (tx: any) => {
await tx.$queryRaw`SELECT id FROM orchestration_tasks WHERE id = ${taskId}::uuid FOR UPDATE`;
const fingerprint = JSON.stringify({ taskId, command, dto, actorId });
const replay = await tx.orchestrationEvent.findUnique({
where: { idempotencyKey: dto.idempotencyKey },
});
if (replay) {
if (replay.payload.fingerprint !== fingerprint)
throw new ConflictException('Idempotency payload mismatch');
return replay.payload.result;
}
const task = await tx.orchestrationTask.findUnique({
where: { id: taskId },
});
if (!task) throw new NotFoundException('Task not found');
if (task.version !== dto.expectedVersion)
throw new ConflictException('Stale task version');
await this.killSwitch.assertAllowed('RUN_AGENT');
let result: any;
let state = task.state;
if (command === 'RUN_CREATE') {
if (
!/^[a-f0-9]{40,64}$/.test(dto.commitSha) ||
!['development', 'staging', 'production'].includes(dto.environment)
)
throw new BadRequestException('Invalid immutable run binding');
const last = await tx.orchestrationRun.findFirst({
where: { taskId },
orderBy: { attempt: 'desc' },
});
if (
last &&
(last.status !== 'FAILED' ||
!['TRANSIENT', 'RATE_LIMITED', 'DEPENDENCY'].includes(
last.retryClass,
) ||
last.attempt >= 3)
)
throw new ForbiddenException('Retry denied or exhausted');
if (!last && task.state !== 'CLAIMED')
throw new ConflictException('Run requires CLAIMED task');
result = await tx.orchestrationRun.create({
data: {
taskId,
attempt: (last?.attempt ?? 0) + 1,
status: 'RUNNING',
commitSha: dto.commitSha,
environment: dto.environment,
idempotencyKey: dto.idempotencyKey,
startedAt: new Date(),
},
});
state = 'RUNNING';
} else if (command === 'RUN_FINISH') {
const run = await tx.orchestrationRun.findUnique({
where: { id: dto.runId },
});
if (
!run ||
run.taskId !== taskId ||
run.status !== 'RUNNING' ||
task.state !== 'RUNNING'
)
throw new ConflictException('Active run required');
if (
!['SUCCEEDED', 'FAILED'].includes(dto.status) ||
(dto.status === 'FAILED' &&
![
'TRANSIENT',
'RATE_LIMITED',
'DEPENDENCY',
'VALIDATION',
'POLICY',
'PERMANENT',
].includes(dto.retryClass))
)
throw new BadRequestException('Invalid outcome taxonomy');
result = await tx.orchestrationRun.update({
where: { id: run.id },
data: {
status: dto.status,
retryClass: dto.status === 'FAILED' ? dto.retryClass : null,
finishedAt: new Date(),
},
});
state = dto.status === 'FAILED' ? 'FAILED' : 'VERIFYING';
} else if (command === 'ARTIFACT_ADD' || command === 'ATTESTATION_ADD') {
const run = await tx.orchestrationRun.findUnique({
where: { id: dto.runId },
});
if (
!run ||
run.taskId !== taskId ||
!['RUNNING', 'VERIFYING'].includes(task.state)
)
throw new ConflictException('Owned active run required');
const hash = command === 'ARTIFACT_ADD' ? dto.sha256 : dto.subjectHash;
if (!/^[a-f0-9]{64}$/.test(hash) || !dto.kind)
throw new BadRequestException('Invalid artifact hash or kind');
if (command === 'ARTIFACT_ADD') {
if (!/^https:\/\//.test(dto.uri))
throw new BadRequestException('Artifact URI must be HTTPS');
result = await tx.orchestrationArtifact.create({
data: { runId: run.id, kind: dto.kind, sha256: hash, uri: dto.uri },
});
} else {
if (
!(await tx.orchestrationArtifact.findFirst({
where: { runId: run.id, sha256: hash },
}))
)
throw new ForbiddenException(
'Attestation subject is not a registered artifact',
);
result = await tx.orchestrationAttestation.create({
data: {
runId: run.id,
kind: dto.kind,
subjectHash: hash,
predicate: dto.predicate ?? {},
signer: actorId,
signature: dto.signature,
},
});
}
} else if (command === 'DEPENDENCY_ADD') {
if (task.state !== 'DRAFT')
throw new ConflictException(
'Dependencies are frozen after submission',
);
await tx.$executeRaw`LOCK TABLE orchestration_dependencies IN EXCLUSIVE MODE`;
const cycles =
await tx.$queryRaw`WITH RECURSIVE chain(id) AS (SELECT ${dto.dependsOnTaskId}::uuid UNION SELECT d.depends_on_task_id FROM orchestration_dependencies d JOIN chain c ON d.task_id=c.id) SELECT id FROM chain WHERE id=${taskId}::uuid`;
if (cycles.length) throw new ForbiddenException('Dependency cycle');
result = await tx.orchestrationDependency.create({
data: { taskId, dependsOnTaskId: dto.dependsOnTaskId },
});
} else if (command === 'RELEASE_AUTHORIZE') {
await this.killSwitch.assertAllowed('MUTATE_PRODUCTION');
if (task.state !== 'GATE2_APPROVED')
throw new ConflictException(
'Gate 2 approval required; already consumed or invalid state',
);
const approval = await tx.orchestrationApprovalDecision.findUnique({
where: { id: dto.approvalId },
});
if (
!approval ||
approval.taskId !== taskId ||
approval.gate !== 2 ||
approval.decision !== 'APPROVED' ||
approval.expiresAt <= new Date()
)
throw new ForbiddenException('Gate 2 missing or expired');
for (const key of ['runId', 'artifactHash', 'commitSha', 'environment'])
if (approval[key] !== dto[key])
throw new ForbiddenException('Release binding mismatch');
const run = await tx.orchestrationRun.findUnique({
where: { id: dto.runId },
});
if (
!run ||
run.taskId !== taskId ||
run.status !== 'SUCCEEDED' ||
run.commitSha !== dto.commitSha ||
run.environment !== dto.environment
)
throw new ForbiddenException('Run binding mismatch');
if (
!(await tx.orchestrationArtifact.findFirst({
where: { runId: run.id, sha256: dto.artifactHash },
})) ||
!(await tx.orchestrationAttestation.findFirst({
where: {
runId: run.id,
subjectHash: dto.artifactHash,
kind: 'TEST',
},
}))
)
throw new ForbiddenException('Artifact evidence missing');
result = {
authorized: true,
approvalId: approval.id,
runId: run.id,
artifactHash: dto.artifactHash,
commitSha: dto.commitSha,
environment: dto.environment,
execution: 'CI_CD_ONLY',
};
state = 'RELEASING';
} else throw new BadRequestException('Unknown command');
await tx.orchestrationTask.update({
where: { id: taskId },
data: { state, version: { increment: 1 } },
});
const serialized = JSON.parse(JSON.stringify(result));
await tx.orchestrationEvent.create({
data: {
taskId,
eventType: command,
fromState: task.state,
toState: state,
sequence: task.version + 1,
actorId,
idempotencyKey: dto.idempotencyKey,
payload: { fingerprint, result: serialized },
},
});
return serialized;
});
}
async reconcileBudget(
taskId: string,
dto: { reservationId: string; actualCents: number; idempotencyKey: string },
actorId: string,
) {
await this.killSwitch.assertAllowed('OBSERVE');
if (!Number.isSafeInteger(dto.actualCents) || dto.actualCents < 0)
throw new BadRequestException('Invalid actual cost');
return await (this.prisma as any).$transaction(async (tx: any) => {
await tx.$queryRaw`SELECT id FROM orchestration_tasks WHERE id = ${taskId}::uuid FOR UPDATE`;
const prior = await tx.orchestrationBudgetLedger.findUnique({
where: { idempotencyKey: dto.idempotencyKey },
});
if (prior) {
if (
prior.taskId !== taskId ||
prior.actorId !== actorId ||
prior.metadata.reservationId !== dto.reservationId ||
prior.metadata.actualCents !== dto.actualCents
)
throw new ConflictException('Idempotency payload mismatch');
return prior;
}
const reservation = await tx.orchestrationBudgetLedger.findUnique({
where: { id: dto.reservationId },
});
if (
!reservation ||
reservation.taskId !== taskId ||
reservation.kind !== 'RESERVE'
)
throw new BadRequestException('Reservation not found');
if (
await tx.orchestrationBudgetLedger.findFirst({
where: {
taskId,
kind: 'RECONCILE',
metadata: { path: ['reservationId'], equals: dto.reservationId },
},
})
)
throw new ConflictException('Reservation already reconciled');
const task = await tx.orchestrationTask.findUnique({
where: { id: taskId },
});
const total = await tx.orchestrationBudgetLedger.aggregate({
where: { taskId },
_sum: { amountCents: true },
});
const delta = dto.actualCents - reservation.amountCents;
if (total._sum.amountCents + delta > task.budgetLimitCents)
throw new ForbiddenException('Hard task budget exceeded');
return tx.orchestrationBudgetLedger.create({
data: {
taskId,
kind: 'RECONCILE',
amountCents: delta,
idempotencyKey: dto.idempotencyKey,
actorId,
metadata: {
reservationId: dto.reservationId,
actualCents: dto.actualCents,
},
},
});
});
}
async findAll() {
await this.killSwitch.assertAllowed('OBSERVE');
return (this.prisma as any).orchestrationTask.findMany({
orderBy: { createdAt: 'desc' },
include: { decisions: true },
});
}
async findOne(id: string) {
await this.killSwitch.assertAllowed('OBSERVE');
const task = await (this.prisma as any).orchestrationTask.findUnique({
where: { id },
include: {
runs: { include: { artifacts: true, attestations: true } },
dependencies: true,
decisions: true,
events: { orderBy: { sequence: 'asc' } },
budgetLedger: true,
},
});
if (!task)
throw new NotFoundException(`Orchestration task ${id} not found`);
return task;
}
async setKillSwitch(level: string, reason: string, actorId: string) {
if (!Object.values(KillSwitchLevel).includes(level as KillSwitchLevel))
throw new BadRequestException('Invalid kill switch level');
return await (this.prisma as any).$transaction(async (tx: any) => {
const value = await tx.orchestrationKillSwitch.upsert({
where: { scope: 'GLOBAL' },
create: { scope: 'GLOBAL', level, active: true, reason, actorId },
update: {
level,
active: true,
reason,
actorId,
version: { increment: 1 },
},
});
await tx.orchestrationEvent.create({
data: {
eventType: 'KILL_SWITCH_CHANGED',
sequence: value.version ?? 1,
actorId,
payload: { scope: 'GLOBAL', level, reason },
},
});
return value;
});
}
}