Add bounded Stage PREPARE workflow with transactional leased outbox
This commit is contained in:
1 parent
4b17a6fcc3
commit
09e8a890e0
9 files changed
+1150
-17
No files matched your search
@@ -0,0 +1,517 @@
|
||||
import {
|
||||
Injectable,
|
||||
Inject,
|
||||
ForbiddenException,
|
||||
BadRequestException,
|
||||
ConflictException,
|
||||
NotFoundException,
|
||||
} from '@nestjs/common';
|
||||
import { randomUUID } from 'crypto';
|
||||
import { PrismaService } from '../prisma/prisma.service';
|
||||
export const PREPARE_WORKFLOW_CONFIG = 'PREPARE_WORKFLOW_CONFIG';
|
||||
export interface WorkflowIdentity {
|
||||
id: string;
|
||||
role: string;
|
||||
}
|
||||
export interface WorkflowConfig {
|
||||
workerIds: string[];
|
||||
stageCiIds: string[];
|
||||
}
|
||||
|
||||
/** Bounded Stage preparation only. No release executor, production scope or browser worker identities. */
|
||||
@Injectable()
|
||||
export class PrepareWorkflowService {
|
||||
constructor(
|
||||
@Inject(PrismaService) private readonly db: any,
|
||||
@Inject(PREPARE_WORKFLOW_CONFIG) private readonly config: WorkflowConfig,
|
||||
) {}
|
||||
private identity(a: WorkflowIdentity, roles: string[]) {
|
||||
if (
|
||||
!a?.id ||
|
||||
typeof a.id !== 'string' ||
|
||||
a.id.length > 100 ||
|
||||
!roles.includes(a.role)
|
||||
)
|
||||
throw new ForbiddenException('Authenticated workflow identity required');
|
||||
}
|
||||
private worker(a: WorkflowIdentity) {
|
||||
this.identity(a, ['WORKFLOW_WORKER']);
|
||||
if (!this.config.workerIds.includes(a.id))
|
||||
throw new ForbiddenException('Worker identity not provisioned');
|
||||
}
|
||||
private async allowed(tx: any, claiming = false) {
|
||||
const rows =
|
||||
await tx.$queryRaw`SELECT level, active FROM orchestration_kill_switches WHERE scope = 'GLOBAL' FOR SHARE`;
|
||||
if (rows[0]?.active && !(rows[0].level === 'FREEZE_NEW_WORK' && !claiming))
|
||||
throw new ForbiddenException('Kill switch blocks workflow');
|
||||
}
|
||||
private async task(tx: any, id: string) {
|
||||
await tx.$queryRaw`SELECT id FROM orchestration_tasks WHERE id = ${id}::uuid FOR UPDATE`;
|
||||
const t = await tx.orchestrationTask.findUnique({ where: { id } });
|
||||
if (!t) throw new NotFoundException('Task not found');
|
||||
if (t.operationMode !== 'PREPARE' || t.metadata?.scope !== 'STAGE_PREPARE')
|
||||
throw new ForbiddenException('Not a bounded Stage proposal');
|
||||
return t;
|
||||
}
|
||||
private async event(
|
||||
tx: any,
|
||||
t: any,
|
||||
state: string,
|
||||
a: WorkflowIdentity,
|
||||
key: string,
|
||||
type: string,
|
||||
fingerprint: string,
|
||||
) {
|
||||
const result = await tx.orchestrationTask.update({
|
||||
where: { id: t.id },
|
||||
data: { state, version: { increment: 1 } },
|
||||
});
|
||||
await tx.orchestrationEvent.create({
|
||||
data: {
|
||||
taskId: t.id,
|
||||
sequence: result.version,
|
||||
actorId: a.id,
|
||||
eventType: type,
|
||||
fromState: t.state,
|
||||
toState: state,
|
||||
idempotencyKey: key,
|
||||
payload: { fingerprint },
|
||||
},
|
||||
});
|
||||
return result;
|
||||
}
|
||||
private async replay(tx: any, key: string, fingerprint: string) {
|
||||
const e = await tx.orchestrationEvent.findUnique({
|
||||
where: { idempotencyKey: key },
|
||||
});
|
||||
if (e && e.payload?.fingerprint !== fingerprint)
|
||||
throw new ConflictException('Idempotency payload mismatch');
|
||||
return e;
|
||||
}
|
||||
private version(t: any, dto: any) {
|
||||
if (
|
||||
!Number.isSafeInteger(dto.expectedVersion) ||
|
||||
dto.expectedVersion !== t.version
|
||||
)
|
||||
throw new ConflictException('Stale task version');
|
||||
if (
|
||||
typeof dto.idempotencyKey !== 'string' ||
|
||||
!dto.idempotencyKey.length ||
|
||||
dto.idempotencyKey.length > 120
|
||||
)
|
||||
throw new BadRequestException('Invalid idempotency key');
|
||||
}
|
||||
async create(dto: any, a: WorkflowIdentity) {
|
||||
this.identity(a, ['Admin', 'SUPER_ADMIN']);
|
||||
if (
|
||||
dto.operationMode !== 'PREPARE' ||
|
||||
!/^[a-f0-9]{40}$/.test(dto.metadata?.commitSha ?? '') ||
|
||||
!Number.isSafeInteger(dto.budgetLimitCents) ||
|
||||
dto.budgetLimitCents < 20 ||
|
||||
dto.budgetLimitCents > 10000
|
||||
)
|
||||
throw new BadRequestException(
|
||||
'PREPARE requires immutable 40hex commit and budget 20..10000 cents',
|
||||
);
|
||||
if (
|
||||
!dto.key ||
|
||||
dto.key.length > 120 ||
|
||||
!dto.title ||
|
||||
dto.title.length > 255 ||
|
||||
!dto.department ||
|
||||
dto.department.length > 50
|
||||
)
|
||||
throw new BadRequestException('Invalid proposal fields');
|
||||
return this.db.$transaction(async (tx: any) => {
|
||||
await this.allowed(tx, true);
|
||||
const t = await tx.orchestrationTask.create({
|
||||
data: {
|
||||
key: dto.key,
|
||||
title: dto.title,
|
||||
department: dto.department,
|
||||
operationMode: 'PREPARE',
|
||||
budgetLimitCents: dto.budgetLimitCents,
|
||||
metadata: {
|
||||
scope: 'STAGE_PREPARE',
|
||||
environment: 'staging',
|
||||
commitSha: dto.metadata.commitSha,
|
||||
createdBy: a.id,
|
||||
},
|
||||
state: 'DRAFT',
|
||||
},
|
||||
});
|
||||
await tx.orchestrationEvent.create({
|
||||
data: {
|
||||
taskId: t.id,
|
||||
sequence: 1,
|
||||
eventType: 'PREPARE_PROPOSED',
|
||||
actorId: a.id,
|
||||
toState: 'DRAFT',
|
||||
},
|
||||
});
|
||||
return t;
|
||||
});
|
||||
}
|
||||
async transition(id: string, dto: any, a: WorkflowIdentity) {
|
||||
this.identity(a, ['Admin', 'SUPER_ADMIN']);
|
||||
return this.db.$transaction(async (tx: any) => {
|
||||
await this.allowed(tx, true);
|
||||
const t = await this.task(tx, id);
|
||||
const fingerprint = JSON.stringify({ id, dto, a });
|
||||
if (await this.replay(tx, dto.idempotencyKey, fingerprint)) return t;
|
||||
this.version(t, dto);
|
||||
if (t.metadata.createdBy !== a.id && a.role !== 'SUPER_ADMIN')
|
||||
throw new ForbiddenException('Proposal owner required');
|
||||
if (
|
||||
!(
|
||||
(t.state === 'DRAFT' && dto.toState === 'GATE1_PENDING') ||
|
||||
(t.state === 'GATE1_APPROVED' && dto.toState === 'READY')
|
||||
)
|
||||
)
|
||||
throw new ForbiddenException(
|
||||
'Only submission and approved queueing are available',
|
||||
);
|
||||
if (dto.toState === 'READY') {
|
||||
const approval = await tx.orchestrationApprovalDecision.findFirst({
|
||||
where: { taskId: id, gate: 1, decision: 'APPROVED' },
|
||||
orderBy: { createdAt: 'desc' },
|
||||
});
|
||||
if (!approval || approval.expiresAt <= new Date())
|
||||
throw new ForbiddenException('Gate1 expired');
|
||||
await tx.prepareWorkflowOutbox.create({ data: { taskId: id } });
|
||||
}
|
||||
return this.event(
|
||||
tx,
|
||||
t,
|
||||
dto.toState,
|
||||
a,
|
||||
dto.idempotencyKey,
|
||||
'PREPARE_TRANSITION',
|
||||
fingerprint,
|
||||
);
|
||||
});
|
||||
}
|
||||
async decide(id: string, gate: number, dto: any, a: WorkflowIdentity) {
|
||||
this.identity(a, ['SUPER_ADMIN']);
|
||||
if (gate !== 1)
|
||||
throw new ForbiddenException(
|
||||
'Gate2 decisions and release remain disabled',
|
||||
);
|
||||
if (
|
||||
!['APPROVED', 'REJECTED'].includes(dto.decision) ||
|
||||
!Number.isFinite(Date.parse(dto.expiresAt)) ||
|
||||
Date.parse(dto.expiresAt) <= Date.now() ||
|
||||
Date.parse(dto.expiresAt) > Date.now() + 86400000
|
||||
)
|
||||
throw new BadRequestException(
|
||||
'Decision requires future expiry within 24 hours',
|
||||
);
|
||||
return this.db.$transaction(async (tx: any) => {
|
||||
await this.allowed(tx);
|
||||
const t = await this.task(tx, id);
|
||||
const fingerprint = JSON.stringify({ id, gate, dto, a });
|
||||
if (await this.replay(tx, dto.idempotencyKey, fingerprint)) return t;
|
||||
this.version(t, dto);
|
||||
if (t.metadata.createdBy === a.id)
|
||||
throw new ForbiddenException('Independent reviewer required');
|
||||
if (t.state !== 'GATE1_PENDING')
|
||||
throw new ConflictException('Gate1 pending required');
|
||||
await tx.orchestrationApprovalDecision.create({
|
||||
data: {
|
||||
taskId: id,
|
||||
gate: 1,
|
||||
decision: dto.decision,
|
||||
actorId: a.id,
|
||||
expiresAt: new Date(dto.expiresAt),
|
||||
idempotencyKey: `decision:${dto.idempotencyKey}`,
|
||||
},
|
||||
});
|
||||
return this.event(
|
||||
tx,
|
||||
t,
|
||||
dto.decision === 'APPROVED' ? 'GATE1_APPROVED' : 'REJECTED',
|
||||
a,
|
||||
dto.idempotencyKey,
|
||||
'PREPARE_GATE1',
|
||||
fingerprint,
|
||||
);
|
||||
});
|
||||
}
|
||||
async claim(a: WorkflowIdentity, id?: string): Promise<any> {
|
||||
this.worker(a);
|
||||
return this.db.$transaction(async (tx: any) => {
|
||||
await this.allowed(tx, true);
|
||||
// Lock order is task then outbox for every worker operation. SKIP LOCKED permits competing workers.
|
||||
const rows: any[] = id
|
||||
? await tx.$queryRaw`SELECT t.id FROM orchestration_tasks t JOIN prepare_workflow_outbox o ON o.task_id=t.id WHERE t.id=${id}::uuid AND o.status='PENDING' AND (o.lease_until IS NULL OR o.lease_until < now()) ORDER BY o.created_at FOR UPDATE OF t SKIP LOCKED LIMIT 1`
|
||||
: await tx.$queryRaw`SELECT t.id FROM orchestration_tasks t JOIN prepare_workflow_outbox o ON o.task_id=t.id WHERE o.status='PENDING' AND (o.lease_until IS NULL OR o.lease_until < now()) ORDER BY o.created_at FOR UPDATE OF t SKIP LOCKED LIMIT 1`;
|
||||
if (!rows.length) return null;
|
||||
const t = await this.task(tx, rows[0].id);
|
||||
const o = await tx.prepareWorkflowOutbox.findUnique({
|
||||
where: { taskId: t.id },
|
||||
});
|
||||
const approval = await tx.orchestrationApprovalDecision.findFirst({
|
||||
where: { taskId: t.id, gate: 1, decision: 'APPROVED' },
|
||||
orderBy: { createdAt: 'desc' },
|
||||
});
|
||||
if (!approval || approval.expiresAt <= new Date())
|
||||
throw new ForbiddenException('Gate1 expired');
|
||||
if (o.attempts >= 3) throw new ForbiddenException('Attempts exhausted');
|
||||
const costs = await tx.orchestrationBudgetLedger.aggregate({
|
||||
where: { taskId: t.id },
|
||||
_sum: { amountCents: true },
|
||||
});
|
||||
if ((costs._sum.amountCents ?? 0) + 20 > t.budgetLimitCents)
|
||||
throw new ForbiddenException('Hard task budget exceeded');
|
||||
if (o.runId)
|
||||
await tx.orchestrationRun.update({
|
||||
where: { id: o.runId },
|
||||
data: {
|
||||
status: 'FAILED',
|
||||
retryClass: 'TRANSIENT',
|
||||
finishedAt: new Date(),
|
||||
},
|
||||
});
|
||||
const token = randomUUID(),
|
||||
attempt = o.attempts + 1;
|
||||
const run = await tx.orchestrationRun.create({
|
||||
data: {
|
||||
taskId: t.id,
|
||||
attempt,
|
||||
status: 'RUNNING',
|
||||
environment: 'staging',
|
||||
commitSha: t.metadata.commitSha,
|
||||
idempotencyKey: `prepare-run:${o.id}:${attempt}`,
|
||||
startedAt: new Date(),
|
||||
},
|
||||
});
|
||||
const reservation = await tx.orchestrationBudgetLedger.create({
|
||||
data: {
|
||||
taskId: t.id,
|
||||
amountCents: 20,
|
||||
kind: 'RESERVE',
|
||||
actorId: a.id,
|
||||
idempotencyKey: `prepare-reserve:${run.id}`,
|
||||
},
|
||||
});
|
||||
const leaseUntil = new Date(Date.now() + 30000);
|
||||
await tx.prepareWorkflowOutbox.update({
|
||||
where: { id: o.id },
|
||||
data: {
|
||||
attempts: attempt,
|
||||
leaseOwner: a.id,
|
||||
leaseToken: token,
|
||||
leaseUntil,
|
||||
heartbeatAt: new Date(),
|
||||
runId: run.id,
|
||||
reservationId: reservation.id,
|
||||
},
|
||||
});
|
||||
await this.event(
|
||||
tx,
|
||||
t,
|
||||
'RUNNING',
|
||||
a,
|
||||
`prepare-claim:${run.id}`,
|
||||
'PREPARE_CLAIMED',
|
||||
token,
|
||||
);
|
||||
return {
|
||||
taskId: t.id,
|
||||
runId: run.id,
|
||||
attempt,
|
||||
leaseToken: token,
|
||||
leaseUntil,
|
||||
commitSha: run.commitSha,
|
||||
environment: 'staging',
|
||||
maxCostCents: 20,
|
||||
};
|
||||
});
|
||||
}
|
||||
private async lease(tx: any, a: WorkflowIdentity, dto: any) {
|
||||
const t = await this.task(tx, dto.taskId);
|
||||
const o = await tx.prepareWorkflowOutbox.findUnique({
|
||||
where: { taskId: t.id },
|
||||
});
|
||||
if (
|
||||
!o ||
|
||||
o.status !== 'PENDING' ||
|
||||
o.leaseOwner !== a.id ||
|
||||
o.leaseToken !== dto.leaseToken ||
|
||||
o.runId !== dto.runId ||
|
||||
!o.leaseUntil ||
|
||||
o.leaseUntil <= new Date() ||
|
||||
t.state !== 'RUNNING'
|
||||
)
|
||||
throw new ForbiddenException('Active owned lease required');
|
||||
return { t, o };
|
||||
}
|
||||
async heartbeat(a: WorkflowIdentity, dto: any) {
|
||||
this.worker(a);
|
||||
return this.db.$transaction(async (tx: any) => {
|
||||
await this.allowed(tx);
|
||||
const { o } = await this.lease(tx, a, dto);
|
||||
return tx.prepareWorkflowOutbox.update({
|
||||
where: { id: o.id },
|
||||
data: {
|
||||
heartbeatAt: new Date(),
|
||||
leaseUntil: new Date(Date.now() + 30000),
|
||||
},
|
||||
select: { leaseUntil: true },
|
||||
});
|
||||
});
|
||||
}
|
||||
async complete(a: WorkflowIdentity, dto: any) {
|
||||
this.worker(a);
|
||||
if (
|
||||
!/^[a-f0-9]{64}$/.test(dto.sha256 ?? '') ||
|
||||
!/^https:\/\//.test(dto.uri ?? '') ||
|
||||
!Number.isSafeInteger(dto.actualCents) ||
|
||||
dto.actualCents < 0 ||
|
||||
dto.actualCents > 20
|
||||
)
|
||||
throw new BadRequestException(
|
||||
'QA hash, HTTPS URI and cost 0..20 required',
|
||||
);
|
||||
return this.db.$transaction(async (tx: any) => {
|
||||
await this.allowed(tx);
|
||||
const { t, o } = await this.lease(tx, a, dto);
|
||||
await tx.orchestrationArtifact.create({
|
||||
data: { runId: o.runId, kind: 'QA', sha256: dto.sha256, uri: dto.uri },
|
||||
});
|
||||
await tx.orchestrationAttestation.create({
|
||||
data: {
|
||||
runId: o.runId,
|
||||
kind: 'QA',
|
||||
subjectHash: dto.sha256,
|
||||
signer: a.id,
|
||||
predicate: { passed: true, scope: 'STAGE_PREPARE' },
|
||||
},
|
||||
});
|
||||
await tx.orchestrationRun.update({
|
||||
where: { id: o.runId },
|
||||
data: { status: 'SUCCEEDED', finishedAt: new Date() },
|
||||
});
|
||||
await tx.orchestrationBudgetLedger.create({
|
||||
data: {
|
||||
taskId: t.id,
|
||||
kind: 'RECONCILE',
|
||||
amountCents: dto.actualCents - 20,
|
||||
actorId: a.id,
|
||||
idempotencyKey: `prepare-cost:${o.runId}`,
|
||||
metadata: {
|
||||
reservationId: o.reservationId,
|
||||
actualCents: dto.actualCents,
|
||||
},
|
||||
},
|
||||
});
|
||||
await tx.prepareWorkflowOutbox.update({
|
||||
where: { id: o.id },
|
||||
data: { status: 'DONE', leaseToken: null, leaseUntil: null },
|
||||
});
|
||||
return this.event(
|
||||
tx,
|
||||
t,
|
||||
'VERIFYING',
|
||||
a,
|
||||
`prepare-qa:${o.runId}`,
|
||||
'PREPARE_QA_SUCCEEDED',
|
||||
dto.sha256,
|
||||
);
|
||||
});
|
||||
}
|
||||
async stageEvidence(a: WorkflowIdentity, dto: any) {
|
||||
this.identity(a, ['STAGE_CI']);
|
||||
if (!this.config.stageCiIds.includes(a.id))
|
||||
throw new ForbiddenException('Stage CI identity not provisioned');
|
||||
if (
|
||||
!/^[a-f0-9]{64}$/.test(dto.evidenceSha256 ?? '') ||
|
||||
!/^https:\/\//.test(dto.uri ?? '')
|
||||
)
|
||||
throw new BadRequestException(
|
||||
'Stage evidence hash and HTTPS URI required',
|
||||
);
|
||||
return this.db.$transaction(async (tx: any) => {
|
||||
await this.allowed(tx);
|
||||
const t = await this.task(tx, dto.taskId);
|
||||
const run = await tx.orchestrationRun.findUnique({
|
||||
where: { id: dto.runId },
|
||||
});
|
||||
if (
|
||||
t.state !== 'VERIFYING' ||
|
||||
!run ||
|
||||
run.taskId !== t.id ||
|
||||
run.environment !== 'staging' ||
|
||||
run.status !== 'SUCCEEDED' ||
|
||||
run.commitSha !== dto.commitSha ||
|
||||
t.metadata.commitSha !== dto.commitSha
|
||||
)
|
||||
throw new ForbiddenException(
|
||||
'Immutable successful Stage run binding required',
|
||||
);
|
||||
const qa = await tx.orchestrationAttestation.findFirst({
|
||||
where: { runId: run.id, kind: 'QA', subjectHash: dto.sha256 },
|
||||
});
|
||||
if (!qa || qa.signer === a.id)
|
||||
throw new ForbiddenException('Independent QA evidence required');
|
||||
await tx.orchestrationArtifact.create({
|
||||
data: {
|
||||
runId: run.id,
|
||||
kind: 'STAGE_EVIDENCE',
|
||||
sha256: dto.evidenceSha256,
|
||||
uri: dto.uri,
|
||||
},
|
||||
});
|
||||
await tx.orchestrationAttestation.create({
|
||||
data: {
|
||||
runId: run.id,
|
||||
kind: 'STAGE',
|
||||
subjectHash: dto.sha256,
|
||||
signer: a.id,
|
||||
predicate: {
|
||||
evidenceSha256: dto.evidenceSha256,
|
||||
commitSha: dto.commitSha,
|
||||
environment: 'staging',
|
||||
},
|
||||
},
|
||||
});
|
||||
return this.event(
|
||||
tx,
|
||||
t,
|
||||
'GATE2_PENDING',
|
||||
a,
|
||||
`prepare-stage:${run.id}`,
|
||||
'STAGE_EVIDENCE_RECORDED',
|
||||
dto.evidenceSha256,
|
||||
);
|
||||
});
|
||||
}
|
||||
project(t: any, a: WorkflowIdentity) {
|
||||
const safe =
|
||||
t.operationMode === 'PREPARE' && t.metadata?.scope === 'STAGE_PREPARE';
|
||||
const owner =
|
||||
a?.id && (t.metadata?.createdBy === a.id || a.role === 'SUPER_ADMIN');
|
||||
const actions: string[] = [];
|
||||
if (safe && owner && t.state === 'DRAFT') actions.push('SUBMIT_GATE1');
|
||||
if (safe && owner && t.state === 'GATE1_APPROVED')
|
||||
actions.push('QUEUE_PREPARE');
|
||||
if (
|
||||
safe &&
|
||||
a?.role === 'SUPER_ADMIN' &&
|
||||
t.metadata?.createdBy !== a.id &&
|
||||
t.state === 'GATE1_PENDING'
|
||||
)
|
||||
actions.push('APPROVE_GATE1', 'REJECT_GATE1');
|
||||
return {
|
||||
...t,
|
||||
metadata: safe
|
||||
? {
|
||||
scope: 'STAGE_PREPARE',
|
||||
environment: 'staging',
|
||||
commitSha: t.metadata.commitSha,
|
||||
}
|
||||
: {},
|
||||
availableActions: actions,
|
||||
};
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user