import { BadRequestException, ConflictException, ForbiddenException, Injectable, NotFoundException, } from '@nestjs/common'; import { PrismaService } from '../prisma/prisma.service'; import { CreateOrchestrationTaskDto, GateDecisionDto, RecordBudgetDto, TransitionTaskDto, } from './dto/orchestration.dto'; import { KillSwitchLevel, assertTransition, nextStateForGateDecision, OperationMode, WorkflowState, } from './orchestration.domain'; import { ModelTierPolicyService } from './model-tier-policy.service'; import { KillSwitchService } from './kill-switch.service'; @Injectable() export class OrchestrationService { constructor( private readonly prisma: PrismaService, private readonly killSwitch: KillSwitchService, ) {} async createTask(dto: CreateOrchestrationTaskDto, actorId: string) { await this.killSwitch.assertAllowed('CREATE_TASK'); return await (this.prisma as any).$transaction(async (tx: any) => { const task = await tx.orchestrationTask.create({ data: { key: dto.key, title: dto.title, department: dto.department, operationMode: dto.operationMode, budgetLimitCents: dto.budgetLimitCents, metadata: dto.metadata ?? {}, state: WorkflowState.DRAFT, version: 1, }, }); await tx.orchestrationEvent.create({ data: { taskId: task.id, eventType: 'TASK_CREATED', fromState: null, toState: WorkflowState.DRAFT, sequence: 1, actorId, payload: { key: dto.key }, }, }); return task; }); } async transition(id: string, dto: TransitionTaskDto, actorId: string) { if ( [ WorkflowState.GATE1_APPROVED, WorkflowState.GATE2_APPROVED, WorkflowState.RELEASING, WorkflowState.RUNNING, WorkflowState.VERIFYING, WorkflowState.PRODUCTION_VERIFYING, WorkflowState.LIVE_VERIFIED, WorkflowState.MEASURING, WorkflowState.CLOSED, ].includes(dto.toState) ) throw new ForbiddenException( 'Gate approvals and release require dedicated commands', ); return await (this.prisma as any).$transaction(async (tx: any) => { await tx.$queryRaw`SELECT id FROM orchestration_tasks WHERE id = ${id}::uuid FOR UPDATE`; const fingerprint = JSON.stringify({ id, dto, actorId }); const replay = await tx.orchestrationEvent.findUnique({ where: { idempotencyKey: dto.idempotencyKey }, }); if (replay) { if (replay.payload.fingerprint !== fingerprint) throw new ConflictException('Idempotency payload mismatch'); return replay.payload.result; } const existing = await tx.orchestrationTask.findUnique({ where: { id } }); if (!existing) throw new NotFoundException(`Orchestration task ${id} not found`); if (existing.state === WorkflowState.FAILED) throw new ForbiddenException('Retries require dedicated run command'); assertTransition(existing.state, dto.toState); await this.killSwitch.assertAllowed( [ WorkflowState.READY, WorkflowState.CLAIMED, WorkflowState.RUNNING, WorkflowState.VERIFYING, WorkflowState.GATE2_PENDING, ].includes(dto.toState) ? 'RUN_AGENT' : 'OBSERVE', ); if ([WorkflowState.READY, WorkflowState.CLAIMED].includes(dto.toState)) await this.killSwitch.assertAllowed('CLAIM_TASK'); let selectedTier = existing.modelTier; if (dto.toState === WorkflowState.READY) { const dependencies = await tx.orchestrationDependency.findMany({ where: { taskId: id }, include: { dependsOnTask: true }, }); if (dependencies.some((d: any) => d.dependsOnTask.state !== 'CLOSED')) throw new ForbiddenException('Unresolved dependency'); const costs = await tx.orchestrationBudgetLedger.aggregate({ where: { taskId: id }, _sum: { amountCents: true }, }); selectedTier = new ModelTierPolicyService().select({ complexity: Number(existing.metadata?.complexity ?? 0), risk: Number(existing.metadata?.risk ?? 0), operationMode: existing.operationMode, remainingBudgetCents: existing.budgetLimitCents - (costs._sum.amountCents ?? 0), }); const approval = await tx.orchestrationApprovalDecision.findFirst({ where: { taskId: id, gate: 1, decision: 'APPROVED' }, orderBy: { createdAt: 'desc' }, }); if (!approval || approval.expiresAt <= new Date()) throw new ForbiddenException('Gate 1 approval missing or expired'); } if ( existing.operationMode === OperationMode.MUTATE_PRODUCTION && dto.toState === WorkflowState.RELEASING ) await this.killSwitch.assertAllowed('MUTATE_PRODUCTION'); const changed = await tx.orchestrationTask.updateMany({ where: { id, version: dto.expectedVersion, state: existing.state }, data: { state: dto.toState, modelTier: selectedTier, version: { increment: 1 }, }, }); if (changed.count !== 1) throw new ConflictException( 'Task version or state changed; reload before retrying', ); await tx.orchestrationEvent.create({ data: { taskId: id, eventType: 'STATE_TRANSITION', fromState: existing.state, toState: dto.toState, sequence: dto.expectedVersion + 1, actorId, idempotencyKey: dto.idempotencyKey, payload: { fingerprint, result: JSON.parse( JSON.stringify({ ...existing, state: dto.toState, modelTier: selectedTier, version: dto.expectedVersion + 1, }), ), }, }, }); return JSON.parse( JSON.stringify({ ...existing, state: dto.toState, modelTier: selectedTier, version: dto.expectedVersion + 1, }), ); }); } async decideGate( taskId: string, gate: 1 | 2, dto: GateDecisionDto, actorId: string, ) { if (![1, 2].includes(gate)) throw new BadRequestException('Invalid gate'); return await (this.prisma as any).$transaction(async (tx: any) => { await tx.$queryRaw`SELECT id FROM orchestration_tasks WHERE id = ${taskId}::uuid FOR UPDATE`; const fingerprint = JSON.stringify({ taskId, gate, dto, actorId }); const prior = await tx.orchestrationApprovalDecision.findUnique({ where: { idempotencyKey: dto.idempotencyKey }, }); if (prior) { const event = await tx.orchestrationEvent.findUnique({ where: { idempotencyKey: `event:${dto.idempotencyKey}` }, }); if (event?.payload?.fingerprint !== fingerprint) throw new ConflictException('Idempotency payload mismatch'); return prior; } if ( !Number.isFinite(Date.parse(dto.expiresAt)) || new Date(dto.expiresAt) <= new Date() ) throw new BadRequestException('Approval decision has expired'); await this.killSwitch.assertAllowed('OBSERVE'); const task = await tx.orchestrationTask.findUnique({ where: { id: taskId }, }); if (!task) throw new NotFoundException(`Orchestration task ${taskId} not found`); const expected = gate === 1 ? WorkflowState.GATE1_PENDING : WorkflowState.GATE2_PENDING; if (task.state !== expected) throw new ConflictException(`Gate ${gate} requires ${expected}`); if (gate === 2 && dto.decision === 'APPROVED') { if ( !dto.runId || !dto.artifactHash || !dto.commitSha || !dto.environment ) throw new BadRequestException( 'Gate 2 requires runId, artifactHash, commitSha and environment', ); const run = await tx.orchestrationRun.findUnique({ where: { id: dto.runId }, }); if ( !run || run.taskId !== taskId || run.commitSha !== dto.commitSha || run.environment !== dto.environment ) throw new ForbiddenException( 'Gate 2 binding does not match immutable run', ); if (run.status !== 'SUCCEEDED') throw new ForbiddenException('Gate 2 requires a successful run'); const artifact = await tx.orchestrationArtifact.findFirst({ where: { runId: dto.runId, sha256: dto.artifactHash }, }); if (!artifact) throw new ForbiddenException( 'Gate 2 artifact hash is not registered for run', ); } const next = nextStateForGateDecision(gate, dto.decision); const changed = await tx.orchestrationTask.updateMany({ where: { id: taskId, version: dto.expectedVersion, state: expected }, data: { state: next, version: { increment: 1 } }, }); if (changed.count !== 1) throw new ConflictException( 'Task version or state changed; approval not applied', ); const decision = await tx.orchestrationApprovalDecision.create({ data: { taskId, gate, decision: dto.decision, actorId, notes: dto.notes, expiresAt: new Date(dto.expiresAt), idempotencyKey: dto.idempotencyKey, runId: dto.runId, artifactHash: dto.artifactHash, commitSha: dto.commitSha, environment: dto.environment, }, }); await tx.orchestrationEvent.create({ data: { taskId, eventType: `GATE${gate}_${dto.decision}`, fromState: expected, toState: next, sequence: dto.expectedVersion + 1, actorId, idempotencyKey: `event:${dto.idempotencyKey}`, payload: { decisionId: decision.id, fingerprint }, }, }); return decision; }); } async recordBudget(taskId: string, dto: RecordBudgetDto, actorId: string) { if ( !Number.isSafeInteger(dto.amountCents) || dto.amountCents < 1 || !['RESERVE', 'SPEND'].includes(dto.kind) ) throw new BadRequestException( 'Use RESERVE or SPEND with positive integer cents', ); return await (this.prisma as any).$transaction(async (tx: any) => { await tx.$queryRaw`SELECT id FROM orchestration_tasks WHERE id = ${taskId}::uuid FOR UPDATE`; const prior = await tx.orchestrationBudgetLedger.findUnique({ where: { idempotencyKey: dto.idempotencyKey }, }); if (prior) { if ( prior.taskId !== taskId || prior.kind !== dto.kind || prior.amountCents !== dto.amountCents || prior.actorId !== actorId || JSON.stringify(prior.metadata) !== JSON.stringify(dto.metadata ?? {}) ) throw new ConflictException('Idempotency payload mismatch'); return prior; } await this.killSwitch.assertAllowed('RUN_AGENT'); const task = await tx.orchestrationTask.findUnique({ where: { id: taskId }, }); if (!task) throw new NotFoundException(`Orchestration task ${taskId} not found`); const sum = await tx.orchestrationBudgetLedger.aggregate({ where: { taskId }, _sum: { amountCents: true }, }); if ((sum._sum.amountCents ?? 0) + dto.amountCents > task.budgetLimitCents) throw new ForbiddenException('Hard task budget exceeded'); return tx.orchestrationBudgetLedger.create({ data: { taskId, amountCents: dto.amountCents, kind: dto.kind, idempotencyKey: dto.idempotencyKey, actorId, metadata: dto.metadata ?? {}, }, }); }); } async command(taskId: string, command: string, dto: any, actorId: string) { return await (this.prisma as any).$transaction(async (tx: any) => { await tx.$queryRaw`SELECT id FROM orchestration_tasks WHERE id = ${taskId}::uuid FOR UPDATE`; const fingerprint = JSON.stringify({ taskId, command, dto, actorId }); const replay = await tx.orchestrationEvent.findUnique({ where: { idempotencyKey: dto.idempotencyKey }, }); if (replay) { if (replay.payload.fingerprint !== fingerprint) throw new ConflictException('Idempotency payload mismatch'); return replay.payload.result; } const task = await tx.orchestrationTask.findUnique({ where: { id: taskId }, }); if (!task) throw new NotFoundException('Task not found'); if (task.version !== dto.expectedVersion) throw new ConflictException('Stale task version'); await this.killSwitch.assertAllowed('RUN_AGENT'); let result: any; let state = task.state; if (command === 'RUN_CREATE') { if ( !/^[a-f0-9]{40,64}$/.test(dto.commitSha) || !['development', 'staging', 'production'].includes(dto.environment) ) throw new BadRequestException('Invalid immutable run binding'); const last = await tx.orchestrationRun.findFirst({ where: { taskId }, orderBy: { attempt: 'desc' }, }); if ( last && (last.status !== 'FAILED' || !['TRANSIENT', 'RATE_LIMITED', 'DEPENDENCY'].includes( last.retryClass, ) || last.attempt >= 3) ) throw new ForbiddenException('Retry denied or exhausted'); if (!last && task.state !== 'CLAIMED') throw new ConflictException('Run requires CLAIMED task'); result = await tx.orchestrationRun.create({ data: { taskId, attempt: (last?.attempt ?? 0) + 1, status: 'RUNNING', commitSha: dto.commitSha, environment: dto.environment, idempotencyKey: dto.idempotencyKey, startedAt: new Date(), }, }); state = 'RUNNING'; } else if (command === 'RUN_FINISH') { const run = await tx.orchestrationRun.findUnique({ where: { id: dto.runId }, }); if ( !run || run.taskId !== taskId || run.status !== 'RUNNING' || task.state !== 'RUNNING' ) throw new ConflictException('Active run required'); if ( !['SUCCEEDED', 'FAILED'].includes(dto.status) || (dto.status === 'FAILED' && ![ 'TRANSIENT', 'RATE_LIMITED', 'DEPENDENCY', 'VALIDATION', 'POLICY', 'PERMANENT', ].includes(dto.retryClass)) ) throw new BadRequestException('Invalid outcome taxonomy'); result = await tx.orchestrationRun.update({ where: { id: run.id }, data: { status: dto.status, retryClass: dto.status === 'FAILED' ? dto.retryClass : null, finishedAt: new Date(), }, }); state = dto.status === 'FAILED' ? 'FAILED' : 'VERIFYING'; } else if (command === 'ARTIFACT_ADD' || command === 'ATTESTATION_ADD') { const run = await tx.orchestrationRun.findUnique({ where: { id: dto.runId }, }); if ( !run || run.taskId !== taskId || !['RUNNING', 'VERIFYING'].includes(task.state) ) throw new ConflictException('Owned active run required'); const hash = command === 'ARTIFACT_ADD' ? dto.sha256 : dto.subjectHash; if (!/^[a-f0-9]{64}$/.test(hash) || !dto.kind) throw new BadRequestException('Invalid artifact hash or kind'); if (command === 'ARTIFACT_ADD') { if (!/^https:\/\//.test(dto.uri)) throw new BadRequestException('Artifact URI must be HTTPS'); result = await tx.orchestrationArtifact.create({ data: { runId: run.id, kind: dto.kind, sha256: hash, uri: dto.uri }, }); } else { if ( !(await tx.orchestrationArtifact.findFirst({ where: { runId: run.id, sha256: hash }, })) ) throw new ForbiddenException( 'Attestation subject is not a registered artifact', ); result = await tx.orchestrationAttestation.create({ data: { runId: run.id, kind: dto.kind, subjectHash: hash, predicate: dto.predicate ?? {}, signer: actorId, signature: dto.signature, }, }); } } else if (command === 'DEPENDENCY_ADD') { if (task.state !== 'DRAFT') throw new ConflictException( 'Dependencies are frozen after submission', ); await tx.$executeRaw`LOCK TABLE orchestration_dependencies IN EXCLUSIVE MODE`; const cycles = await tx.$queryRaw`WITH RECURSIVE chain(id) AS (SELECT ${dto.dependsOnTaskId}::uuid UNION SELECT d.depends_on_task_id FROM orchestration_dependencies d JOIN chain c ON d.task_id=c.id) SELECT id FROM chain WHERE id=${taskId}::uuid`; if (cycles.length) throw new ForbiddenException('Dependency cycle'); result = await tx.orchestrationDependency.create({ data: { taskId, dependsOnTaskId: dto.dependsOnTaskId }, }); } else if (command === 'RELEASE_AUTHORIZE') { await this.killSwitch.assertAllowed('MUTATE_PRODUCTION'); if (task.state !== 'GATE2_APPROVED') throw new ConflictException( 'Gate 2 approval required; already consumed or invalid state', ); const approval = await tx.orchestrationApprovalDecision.findUnique({ where: { id: dto.approvalId }, }); if ( !approval || approval.taskId !== taskId || approval.gate !== 2 || approval.decision !== 'APPROVED' || approval.expiresAt <= new Date() ) throw new ForbiddenException('Gate 2 missing or expired'); for (const key of ['runId', 'artifactHash', 'commitSha', 'environment']) if (approval[key] !== dto[key]) throw new ForbiddenException('Release binding mismatch'); const run = await tx.orchestrationRun.findUnique({ where: { id: dto.runId }, }); if ( !run || run.taskId !== taskId || run.status !== 'SUCCEEDED' || run.commitSha !== dto.commitSha || run.environment !== dto.environment ) throw new ForbiddenException('Run binding mismatch'); if ( !(await tx.orchestrationArtifact.findFirst({ where: { runId: run.id, sha256: dto.artifactHash }, })) || !(await tx.orchestrationAttestation.findFirst({ where: { runId: run.id, subjectHash: dto.artifactHash, kind: 'TEST', }, })) ) throw new ForbiddenException('Artifact evidence missing'); result = { authorized: true, approvalId: approval.id, runId: run.id, artifactHash: dto.artifactHash, commitSha: dto.commitSha, environment: dto.environment, execution: 'CI_CD_ONLY', }; state = 'RELEASING'; } else throw new BadRequestException('Unknown command'); await tx.orchestrationTask.update({ where: { id: taskId }, data: { state, version: { increment: 1 } }, }); const serialized = JSON.parse(JSON.stringify(result)); await tx.orchestrationEvent.create({ data: { taskId, eventType: command, fromState: task.state, toState: state, sequence: task.version + 1, actorId, idempotencyKey: dto.idempotencyKey, payload: { fingerprint, result: serialized }, }, }); return serialized; }); } async reconcileBudget( taskId: string, dto: { reservationId: string; actualCents: number; idempotencyKey: string }, actorId: string, ) { await this.killSwitch.assertAllowed('OBSERVE'); if (!Number.isSafeInteger(dto.actualCents) || dto.actualCents < 0) throw new BadRequestException('Invalid actual cost'); return await (this.prisma as any).$transaction(async (tx: any) => { await tx.$queryRaw`SELECT id FROM orchestration_tasks WHERE id = ${taskId}::uuid FOR UPDATE`; const prior = await tx.orchestrationBudgetLedger.findUnique({ where: { idempotencyKey: dto.idempotencyKey }, }); if (prior) { if ( prior.taskId !== taskId || prior.actorId !== actorId || prior.metadata.reservationId !== dto.reservationId || prior.metadata.actualCents !== dto.actualCents ) throw new ConflictException('Idempotency payload mismatch'); return prior; } const reservation = await tx.orchestrationBudgetLedger.findUnique({ where: { id: dto.reservationId }, }); if ( !reservation || reservation.taskId !== taskId || reservation.kind !== 'RESERVE' ) throw new BadRequestException('Reservation not found'); if ( await tx.orchestrationBudgetLedger.findFirst({ where: { taskId, kind: 'RECONCILE', metadata: { path: ['reservationId'], equals: dto.reservationId }, }, }) ) throw new ConflictException('Reservation already reconciled'); const task = await tx.orchestrationTask.findUnique({ where: { id: taskId }, }); const total = await tx.orchestrationBudgetLedger.aggregate({ where: { taskId }, _sum: { amountCents: true }, }); const delta = dto.actualCents - reservation.amountCents; if (total._sum.amountCents + delta > task.budgetLimitCents) throw new ForbiddenException('Hard task budget exceeded'); return tx.orchestrationBudgetLedger.create({ data: { taskId, kind: 'RECONCILE', amountCents: delta, idempotencyKey: dto.idempotencyKey, actorId, metadata: { reservationId: dto.reservationId, actualCents: dto.actualCents, }, }, }); }); } async findAll() { await this.killSwitch.assertAllowed('OBSERVE'); return (this.prisma as any).orchestrationTask.findMany({ orderBy: { createdAt: 'desc' }, }); } async findOne(id: string) { await this.killSwitch.assertAllowed('OBSERVE'); const task = await (this.prisma as any).orchestrationTask.findUnique({ where: { id }, include: { runs: { include: { artifacts: true, attestations: true } }, dependencies: true, decisions: true, events: { orderBy: { sequence: 'asc' } }, budgetLedger: true, }, }); if (!task) throw new NotFoundException(`Orchestration task ${id} not found`); return task; } async setKillSwitch(level: string, reason: string, actorId: string) { if (!Object.values(KillSwitchLevel).includes(level as KillSwitchLevel)) throw new BadRequestException('Invalid kill switch level'); return await (this.prisma as any).$transaction(async (tx: any) => { const value = await tx.orchestrationKillSwitch.upsert({ where: { scope: 'GLOBAL' }, create: { scope: 'GLOBAL', level, active: true, reason, actorId }, update: { level, active: true, reason, actorId, version: { increment: 1 }, }, }); await tx.orchestrationEvent.create({ data: { eventType: 'KILL_SWITCH_CHANGED', sequence: value.version ?? 1, actorId, payload: { scope: 'GLOBAL', level, reason }, }, }); return value; }); } }