diff --git a/backend/prisma/migrations/20261023170000_prepare_workflow_outbox/migration.sql b/backend/prisma/migrations/20261023170000_prepare_workflow_outbox/migration.sql new file mode 100644 index 00000000..f0fe463a --- /dev/null +++ b/backend/prisma/migrations/20261023170000_prepare_workflow_outbox/migration.sql @@ -0,0 +1,10 @@ +CREATE TABLE prepare_workflow_outbox ( + id uuid PRIMARY KEY, task_id uuid NOT NULL UNIQUE REFERENCES orchestration_tasks(id) ON DELETE RESTRICT, + status varchar(20) NOT NULL DEFAULT 'PENDING' CHECK (status IN ('PENDING','DONE')), + attempts integer NOT NULL DEFAULT 0 CHECK (attempts BETWEEN 0 AND 3), + lease_owner varchar(100), lease_token uuid, lease_until timestamptz, heartbeat_at timestamptz, + run_id uuid REFERENCES orchestration_runs(id) ON DELETE RESTRICT, + reservation_id uuid REFERENCES orchestration_budget_ledger(id) ON DELETE RESTRICT, + created_at timestamptz NOT NULL DEFAULT now() +); +CREATE INDEX prepare_workflow_outbox_status_lease_until_idx ON prepare_workflow_outbox(status,lease_until); diff --git a/backend/prisma/schema.prisma b/backend/prisma/schema.prisma index 40c5bd31..4d68d328 100644 --- a/backend/prisma/schema.prisma +++ b/backend/prisma/schema.prisma @@ -1232,3 +1232,20 @@ model OrchestrationKillSwitch { createdAt DateTime @default(now()) @map("created_at") @db.Timestamptz() @@map("orchestration_kill_switches") } + +// Transactional dispatch intent. Lease tokens are server-only fencing credentials. +model PrepareWorkflowOutbox { + id String @id @default(uuid()) @db.Uuid + taskId String @unique @map("task_id") @db.Uuid + status String @default("PENDING") @db.VarChar(20) + attempts Int @default(0) + leaseOwner String? @map("lease_owner") @db.VarChar(100) + leaseToken String? @map("lease_token") @db.Uuid + leaseUntil DateTime? @map("lease_until") @db.Timestamptz() + heartbeatAt DateTime? @map("heartbeat_at") @db.Timestamptz() + runId String? @map("run_id") @db.Uuid + reservationId String? @map("reservation_id") @db.Uuid + createdAt DateTime @default(now()) @map("created_at") @db.Timestamptz() + @@index([status, leaseUntil]) + @@map("prepare_workflow_outbox") +} diff --git a/backend/src/orchestration/PREPARE_CONTRACT.md b/backend/src/orchestration/PREPARE_CONTRACT.md new file mode 100644 index 00000000..27da11cf --- /dev/null +++ b/backend/src/orchestration/PREPARE_CONTRACT.md @@ -0,0 +1,37 @@ +# Bounded Stage PREPARE workflow foundation + +This slice is NOT a runtime executor, deployment mechanism, or full v4 completion. It enables authenticated proposal/Gate1/queue APIs and provides internal worker/CI service methods. Default worker/CI identity allowlists are empty: no browser or production execution route exists. + +## Panel HTTP contract +All routes require existing JWT authentication with `Admin` or `SUPER_ADMIN`. Actor is exclusively `req.user.id ?? req.user.sub`, role exclusively `req.user.role`. Top-level unknown mutation fields return 400; client actor/profile fields are never trusted. + +Base `/admin/orchestration`: + +- `GET /tasks`: task array. +- `GET /tasks/:id`: task object. +- `GET /overview`: existing inventory items plus capabilities `{mutations:true, prepare:true, release:false, gate2Decisions:false, chat:false, runtime:false}`, reason `bounded_stage_prepare_only`. +- `POST /tasks`: existing CreateOrchestrationTaskDto `{key,title,department,operationMode:"PREPARE",budgetLimitCents,metadata:{commitSha}}`; commitSha is lowercase 40hex; budget is integer 20..10000 cents. Returns 201 projected task, state DRAFT/version1. Metadata input scope/profile/actor/other arbitrary keys are discarded. Key max120, title max255, department max50. Proposal key is unique; create replay should fetch existing task rather than retry blindly. +- `PUT /tasks/:id/transition`: `{toState:"GATE1_PENDING"|"READY",expectedVersion,idempotencyKey,reason?}`. Only DRAFT->GATE1_PENDING and GATE1_APPROVED->READY. Owner or SUPER_ADMIN. Returns 200 projected task. READY atomically creates dispatch outbox. Requires unexpired Gate1. +- `POST /tasks/:id/gates/1/decide`: existing GateDecisionDto `{decision:"APPROVED"|"REJECTED",expectedVersion,idempotencyKey,expiresAt,notes?}`. SUPER_ADMIN independent from proposer. Future expiry <=24h. Returns 201 **projected task**, not decision object. +- Gate2 decision, budget writes/reconcile, command routes (including release), chat and kill-switch HTTP mutations remain 403. Runtime resource reads remain 503. + +Task projection keeps existing scalar fields and safe nested runs/artifacts, decisions, events and ledger. Adds only `metadata:{scope:"STAGE_PREPARE",environment:"staging",commitSha}` for bounded proposals, otherwise `{}`. Never includes createdBy, payloads, URI, signatures, tokens or lease credentials. + +`availableActions:string[]` is state/identity-derived: DRAFT owner -> `SUBMIT_GATE1`; GATE1_PENDING independent SUPER_ADMIN -> `APPROVE_GATE1`,`REJECT_GATE1`; GATE1_APPROVED owner/SUPER_ADMIN -> `QUEUE_PREPARE`; all worker/verification/Gate2 and legacy states -> `[]`. Actions indicate workflow eligibility, not a guarantee: current kill switch, approval expiry and concurrent versions are rechecked transactionally on mutation. Panel must not infer actions from state or send actor identity. + +Common errors: 400 invalid DTO/binding; 403 wrong scope/identity/independence/kill/budget/expiry; 409 stale version or idempotency payload mismatch. Transition/decision replay returns current persisted task and does not duplicate side effects. + +## Internal service contracts (NOT browser endpoints) +`PrepareWorkflowService` is exported by OrchestrationModule. Trusted backend composition must override `PREPARE_WORKFLOW_CONFIG` with explicit `workerIds` and `stageCiIds`, default both empty. Do not populate from request bodies or user-selected runtime profiles. Worker identities require role WORKFLOW_WORKER AND ID allowlist; Stage identities require STAGE_CI AND separate ID allowlist. Credential/session provisioning and transport remain the integrating runtime owner's responsibility. + +- `claim(identity, taskId?) -> null | {taskId,runId,attempt,leaseToken,leaseUntil,commitSha,environment:"staging",maxCostCents:20}`. Transactional task row lock with SKIP LOCKED, immutable run binding, 30s lease, max3 attempts. Each attempt reserves **a fixed hard upper bound of 20 cents**, not an estimated or measured departmental cost. Unknown cost on expired attempts remains reserved conservatively. +- `heartbeat(identity,{taskId,runId,leaseToken}) -> {leaseUntil}`. Same owner/token, unexpired active lease required. +- `complete(identity,{taskId,runId,leaseToken,sha256,uri,actualCents}) -> task`. QA success ingestion only; lowercase sha256, HTTPS URI, actual cost integer0..20 from trusted worker. Atomically records immutable QA artifact/attestation, SUCCEEDED run, reservation reconciliation, DONE outbox and VERIFYING task. Expired/stale/foreign token cannot finish. Not replayable after success; caller must read persisted task/run after response loss. +- `stageEvidence(identity,{taskId,runId,sha256,commitSha,evidenceSha256,uri}) -> task`. Independently allowlisted Stage CI identity, successful exact staging commit/run, registered QA subject, independent QA signer. Atomically records Stage artifact/attestation and reaches GATE2_PENDING. No release or Gate2 approval. Does not perform or claim deployment itself. Caller supplies actual trusted CI evidence, not browser assertions. + +## Verification and explicit gaps +Real isolated PostgreSQL16, no published ports/no live volumes. Added migrations applied with ON_ERROR_STOP. TDD RED logs cover missing workflow behavior, heartbeat/QA behavior and HTTP enablement. Full backend regression excluding existing legacy release-containing orchestration.integration.spec: 106 suites /851 tests including seven real-DB workflow cases, build exit0. No RELEASE_AUTHORIZE service operation executed. + +DB regressions: concurrent claim winner, queue race/replay mismatch, hard budget rollback, stale lease fencing, max3 attempts, conservative expired costs, kill switch, self approval, trusted IDs, QA reconciliation and Stage evidence to Gate2 pending. Test fixture artifact URIs/hashes are not real deployment evidence. + +Remaining: actual authenticated runtime worker transport/executor and pinned CI evidence transport; shared department/company budget model and locking (NOT implemented or represented by fake totals); retry backoff/dead-letter UX/cancellation; hard total execution wall-time; durable request replay of completion/evidence; duplicate proposal-key HTTP conflict normalization; kill-switch concurrent insertion/version locking; migration rehearsal with full legacy schema; real Stage CI artifact/deployment verification. No push/deploy/live DB/runtime edits. Production release remains denied at HTTP boundary; unsafe legacy core release method is deliberately untouched and must never be called by this slice. diff --git a/backend/src/orchestration/orchestration-read.http.spec.ts b/backend/src/orchestration/orchestration-read.http.spec.ts index 76898cac..b32fb984 100644 --- a/backend/src/orchestration/orchestration-read.http.spec.ts +++ b/backend/src/orchestration/orchestration-read.http.spec.ts @@ -92,7 +92,7 @@ describe('integration read boundary', () => { [method]('/admin/orchestration/' + path) .set('Authorization', 'Admin') .send({}) - .expect(403); + .expect(path === 'tasks' || path.endsWith('/transition') ? 400 : 403); } for (const name of [ 'createTask', diff --git a/backend/src/orchestration/orchestration.controller.ts b/backend/src/orchestration/orchestration.controller.ts index b78f988f..8acf3079 100644 --- a/backend/src/orchestration/orchestration.controller.ts +++ b/backend/src/orchestration/orchestration.controller.ts @@ -1,5 +1,9 @@ import { Controller, + Body, + Req, + Optional, + ValidationPipe, Get, Param, Post, @@ -13,6 +17,20 @@ import { JwtAuthGuard } from '../auth/jwt-auth.guard'; import { RolesGuard } from '../common/guards/roles.guard'; import { Roles } from '../common/decorators/roles.decorator'; import { OrchestrationService } from './orchestration.service'; +import { + PrepareWorkflowService, + WorkflowIdentity, +} from './prepare-workflow.service'; +import { + CreateOrchestrationTaskDto, + TransitionTaskDto, + GateDecisionDto, +} from './dto/orchestration.dto'; +const workflowValidation = new ValidationPipe({ + transform: true, + whitelist: true, + forbidNonWhitelisted: true, +}); // Explicit response allowlists: never serialize arbitrary JSON, signatures, URIs or BigInt. function pick(row: any, keys: string[]) { @@ -73,12 +91,37 @@ function project(row: any) { @Roles('Admin', 'SUPER_ADMIN') @Controller('admin/orchestration') export class OrchestrationController { - constructor(private readonly service: OrchestrationService) {} - @Get('tasks') async all() { - return (await this.service.findAll()).map(project); + constructor( + private readonly service: OrchestrationService, + @Optional() private readonly workflow?: PrepareWorkflowService, + ) {} + private actor(req: any): WorkflowIdentity { + const id = req?.user?.id ?? req?.user?.sub; + if ( + typeof id !== 'string' || + !id || + !['Admin', 'SUPER_ADMIN'].includes(req?.user?.role) + ) + throw new ForbiddenException('Authenticated workflow identity required'); + return { id, role: req.user.role }; } - @Get('tasks/:id') async one(@Param('id') id: string) { - return project(await this.service.findOne(id)); + private safe(row: any, req: any) { + const view = project(row); + if (!this.workflow) return view; + const w = this.workflow.project(row, this.actor(req)); + return { + ...view, + metadata: w.metadata, + availableActions: w.availableActions, + }; + } + @Get('tasks') async all(@Req() req: any) { + return (await this.service.findAll()).map((row: any) => + this.safe(row, req), + ); + } + @Get('tasks/:id') async one(@Param('id') id: string, @Req() req: any) { + return this.safe(await this.service.findOne(id), req); } @Get('overview') async overview() { const tasks = await this.service.findAll(); @@ -87,8 +130,17 @@ export class OrchestrationController { { id: 'tasks', title: 'Persisted task inventory', count: tasks.length }, ], measuredAt: new Date().toISOString(), - capabilities: { mutations: false, chat: false, runtime: false }, - reason: 'security_hardening_pending', + capabilities: { + mutations: !!this.workflow, + prepare: !!this.workflow, + release: false, + gate2Decisions: false, + chat: false, + runtime: false, + }, + reason: this.workflow + ? 'bounded_stage_prepare_only' + : 'security_hardening_pending', }; } @Get('releases') async releases() { @@ -120,15 +172,40 @@ export class OrchestrationController { 'No persisted model policy read contract available', ); } - // The committed legacy core is not safe to expose. No env switch or role may bypass this boundary. - @Post('tasks') create() { - return this.deny(); + // Only the separate bounded PREPARE service is exposed; legacy commands remain denied. + @Post('tasks') async create( + @Body(workflowValidation) dto: CreateOrchestrationTaskDto, + @Req() req: any, + ) { + if (!this.workflow) return this.deny(); + return this.safe(await this.workflow.create(dto, this.actor(req)), req); } - @Put('tasks/:id/transition') transition() { - return this.deny(); + @Put('tasks/:id/transition') async transition( + @Param('id') id: string, + @Body(workflowValidation) dto: TransitionTaskDto, + @Req() req: any, + ) { + if (!this.workflow) return this.deny(); + return this.safe( + await this.workflow.transition(id, dto, this.actor(req)), + req, + ); } - @Post('tasks/:id/gates/:gate/decide') decide() { - return this.deny(); + @Post('tasks/:id/gates/:gate/decide') async decide( + @Param('id') id: string, + @Param('gate') gate: string, + @Body() body: unknown, + @Req() req: any, + ) { + if (!this.workflow || gate !== '1') return this.deny(); + const dto = await workflowValidation.transform(body, { + type: 'body', + metatype: GateDecisionDto, + }); + return this.safe( + await this.workflow.decide(id, 1, dto, this.actor(req)), + req, + ); } @Post('tasks/:id/budget') budget() { return this.deny(); diff --git a/backend/src/orchestration/orchestration.module.ts b/backend/src/orchestration/orchestration.module.ts index 9d3d64b2..6bd7ab74 100644 --- a/backend/src/orchestration/orchestration.module.ts +++ b/backend/src/orchestration/orchestration.module.ts @@ -1,3 +1,7 @@ +import { + PrepareWorkflowService, + PREPARE_WORKFLOW_CONFIG, +} from './prepare-workflow.service'; import { Module } from '@nestjs/common'; import { PrismaModule } from '../prisma/prisma.module'; import { KillSwitchService } from './kill-switch.service'; @@ -7,7 +11,21 @@ import { OrchestrationService } from './orchestration.service'; @Module({ imports: [PrismaModule], controllers: [OrchestrationController], - providers: [OrchestrationService, KillSwitchService, ModelTierPolicyService], - exports: [OrchestrationService, KillSwitchService, ModelTierPolicyService], + providers: [ + OrchestrationService, + KillSwitchService, + ModelTierPolicyService, + PrepareWorkflowService, + { + provide: PREPARE_WORKFLOW_CONFIG, + useValue: { workerIds: [], stageCiIds: [] }, + }, + ], + exports: [ + OrchestrationService, + KillSwitchService, + ModelTierPolicyService, + PrepareWorkflowService, + ], }) export class OrchestrationModule {} diff --git a/backend/src/orchestration/prepare-workflow.http.spec.ts b/backend/src/orchestration/prepare-workflow.http.spec.ts new file mode 100644 index 00000000..c6ccf658 --- /dev/null +++ b/backend/src/orchestration/prepare-workflow.http.spec.ts @@ -0,0 +1,133 @@ +import { Test } from '@nestjs/testing'; +import { INestApplication } from '@nestjs/common'; +import request from 'supertest'; +import { OrchestrationController } from './orchestration.controller'; +import { OrchestrationService } from './orchestration.service'; +import { PrepareWorkflowService } from './prepare-workflow.service'; +import { JwtAuthGuard } from '../auth/jwt-auth.guard'; +import { RolesGuard } from '../common/guards/roles.guard'; + +describe('bounded workflow HTTP contracts', () => { + let app: INestApplication; + const t = { + id: 't', + key: 'k', + state: 'DRAFT', + version: 1, + metadata: { + scope: 'STAGE_PREPARE', + environment: 'staging', + commitSha: 'a'.repeat(40), + token: 'secret', + }, + }; + const workflow = { + create: jest.fn(async () => t), + transition: jest.fn(async () => t), + decide: jest.fn(async () => t), + project: jest.fn((row: any) => ({ + ...row, + metadata: { + scope: 'STAGE_PREPARE', + environment: 'staging', + commitSha: 'a'.repeat(40), + }, + availableActions: ['SUBMIT_GATE1'], + })), + }; + const core = { + findAll: async () => [t], + findOne: async () => t, + command: jest.fn(), + }; + beforeAll(async () => { + const m = await Test.createTestingModule({ + controllers: [OrchestrationController], + providers: [ + { provide: OrchestrationService, useValue: core }, + { provide: PrepareWorkflowService, useValue: workflow }, + RolesGuard, + ], + }) + .overrideGuard(JwtAuthGuard) + .useValue({ + canActivate(c: any) { + const req = c.switchToHttp().getRequest(); + if (!req.headers.authorization) return false; + req.user = { + sub: 'verified-subject', + role: req.headers.authorization, + }; + return true; + }, + }) + .compile(); + app = m.createNestApplication(); + await app.init(); + }); + afterAll(async () => { + await app.close(); + }); + it('accepts existing proposal DTO, derives actor from JWT sub and returns projected task', async () => { + const dto = { + key: 'k', + title: 'T', + department: 'ENGINEERING', + operationMode: 'PREPARE', + budgetLimitCents: 20, + metadata: { commitSha: 'a'.repeat(40) }, + }; + const r = await request(app.getHttpServer()) + .post('/admin/orchestration/tasks') + .set('Authorization', 'Admin') + .send(dto) + .expect(201); + expect(workflow.create).toHaveBeenCalledWith(dto, { + id: 'verified-subject', + role: 'Admin', + }); + expect(r.body.availableActions).toEqual(['SUBMIT_GATE1']); + expect(JSON.stringify(r.body)).not.toMatch(/token|secret/); + await request(app.getHttpServer()) + .post('/admin/orchestration/tasks') + .set('Authorization', 'Admin') + .send({ ...dto, actorId: 'browser' }) + .expect(400); + }); + it('routes fixed transition/gate contracts and rejects invalid gates plus release without invoking core', async () => { + await request(app.getHttpServer()) + .put('/admin/orchestration/tasks/t/transition') + .set('Authorization', 'Admin') + .send({ + toState: 'GATE1_PENDING', + expectedVersion: 1, + idempotencyKey: 'submit', + }) + .expect(200); + await request(app.getHttpServer()) + .post('/admin/orchestration/tasks/t/gates/1/decide') + .set('Authorization', 'SUPER_ADMIN') + .send({ + decision: 'APPROVED', + expectedVersion: 2, + idempotencyKey: 'approval', + expiresAt: new Date(Date.now() + 60000).toISOString(), + }) + .expect(201); + expect(workflow.decide).toHaveBeenCalledWith('t', 1, expect.any(Object), { + id: 'verified-subject', + role: 'SUPER_ADMIN', + }); + await request(app.getHttpServer()) + .post('/admin/orchestration/tasks/t/gates/2/decide') + .set('Authorization', 'SUPER_ADMIN') + .send({}) + .expect(403); + await request(app.getHttpServer()) + .post('/admin/orchestration/tasks/t/commands/RELEASE_AUTHORIZE') + .set('Authorization', 'SUPER_ADMIN') + .send({}) + .expect(403); + expect(core.command).not.toHaveBeenCalled(); + }); +}); diff --git a/backend/src/orchestration/prepare-workflow.service.ts b/backend/src/orchestration/prepare-workflow.service.ts new file mode 100644 index 00000000..403984c6 --- /dev/null +++ b/backend/src/orchestration/prepare-workflow.service.ts @@ -0,0 +1,517 @@ +import { + Injectable, + Inject, + ForbiddenException, + BadRequestException, + ConflictException, + NotFoundException, +} from '@nestjs/common'; +import { randomUUID } from 'crypto'; +import { PrismaService } from '../prisma/prisma.service'; +export const PREPARE_WORKFLOW_CONFIG = 'PREPARE_WORKFLOW_CONFIG'; +export interface WorkflowIdentity { + id: string; + role: string; +} +export interface WorkflowConfig { + workerIds: string[]; + stageCiIds: string[]; +} + +/** Bounded Stage preparation only. No release executor, production scope or browser worker identities. */ +@Injectable() +export class PrepareWorkflowService { + constructor( + @Inject(PrismaService) private readonly db: any, + @Inject(PREPARE_WORKFLOW_CONFIG) private readonly config: WorkflowConfig, + ) {} + private identity(a: WorkflowIdentity, roles: string[]) { + if ( + !a?.id || + typeof a.id !== 'string' || + a.id.length > 100 || + !roles.includes(a.role) + ) + throw new ForbiddenException('Authenticated workflow identity required'); + } + private worker(a: WorkflowIdentity) { + this.identity(a, ['WORKFLOW_WORKER']); + if (!this.config.workerIds.includes(a.id)) + throw new ForbiddenException('Worker identity not provisioned'); + } + private async allowed(tx: any, claiming = false) { + const rows = + await tx.$queryRaw`SELECT level, active FROM orchestration_kill_switches WHERE scope = 'GLOBAL' FOR SHARE`; + if (rows[0]?.active && !(rows[0].level === 'FREEZE_NEW_WORK' && !claiming)) + throw new ForbiddenException('Kill switch blocks workflow'); + } + private async task(tx: any, id: string) { + await tx.$queryRaw`SELECT id FROM orchestration_tasks WHERE id = ${id}::uuid FOR UPDATE`; + const t = await tx.orchestrationTask.findUnique({ where: { id } }); + if (!t) throw new NotFoundException('Task not found'); + if (t.operationMode !== 'PREPARE' || t.metadata?.scope !== 'STAGE_PREPARE') + throw new ForbiddenException('Not a bounded Stage proposal'); + return t; + } + private async event( + tx: any, + t: any, + state: string, + a: WorkflowIdentity, + key: string, + type: string, + fingerprint: string, + ) { + const result = await tx.orchestrationTask.update({ + where: { id: t.id }, + data: { state, version: { increment: 1 } }, + }); + await tx.orchestrationEvent.create({ + data: { + taskId: t.id, + sequence: result.version, + actorId: a.id, + eventType: type, + fromState: t.state, + toState: state, + idempotencyKey: key, + payload: { fingerprint }, + }, + }); + return result; + } + private async replay(tx: any, key: string, fingerprint: string) { + const e = await tx.orchestrationEvent.findUnique({ + where: { idempotencyKey: key }, + }); + if (e && e.payload?.fingerprint !== fingerprint) + throw new ConflictException('Idempotency payload mismatch'); + return e; + } + private version(t: any, dto: any) { + if ( + !Number.isSafeInteger(dto.expectedVersion) || + dto.expectedVersion !== t.version + ) + throw new ConflictException('Stale task version'); + if ( + typeof dto.idempotencyKey !== 'string' || + !dto.idempotencyKey.length || + dto.idempotencyKey.length > 120 + ) + throw new BadRequestException('Invalid idempotency key'); + } + async create(dto: any, a: WorkflowIdentity) { + this.identity(a, ['Admin', 'SUPER_ADMIN']); + if ( + dto.operationMode !== 'PREPARE' || + !/^[a-f0-9]{40}$/.test(dto.metadata?.commitSha ?? '') || + !Number.isSafeInteger(dto.budgetLimitCents) || + dto.budgetLimitCents < 20 || + dto.budgetLimitCents > 10000 + ) + throw new BadRequestException( + 'PREPARE requires immutable 40hex commit and budget 20..10000 cents', + ); + if ( + !dto.key || + dto.key.length > 120 || + !dto.title || + dto.title.length > 255 || + !dto.department || + dto.department.length > 50 + ) + throw new BadRequestException('Invalid proposal fields'); + return this.db.$transaction(async (tx: any) => { + await this.allowed(tx, true); + const t = await tx.orchestrationTask.create({ + data: { + key: dto.key, + title: dto.title, + department: dto.department, + operationMode: 'PREPARE', + budgetLimitCents: dto.budgetLimitCents, + metadata: { + scope: 'STAGE_PREPARE', + environment: 'staging', + commitSha: dto.metadata.commitSha, + createdBy: a.id, + }, + state: 'DRAFT', + }, + }); + await tx.orchestrationEvent.create({ + data: { + taskId: t.id, + sequence: 1, + eventType: 'PREPARE_PROPOSED', + actorId: a.id, + toState: 'DRAFT', + }, + }); + return t; + }); + } + async transition(id: string, dto: any, a: WorkflowIdentity) { + this.identity(a, ['Admin', 'SUPER_ADMIN']); + return this.db.$transaction(async (tx: any) => { + await this.allowed(tx, true); + const t = await this.task(tx, id); + const fingerprint = JSON.stringify({ id, dto, a }); + if (await this.replay(tx, dto.idempotencyKey, fingerprint)) return t; + this.version(t, dto); + if (t.metadata.createdBy !== a.id && a.role !== 'SUPER_ADMIN') + throw new ForbiddenException('Proposal owner required'); + if ( + !( + (t.state === 'DRAFT' && dto.toState === 'GATE1_PENDING') || + (t.state === 'GATE1_APPROVED' && dto.toState === 'READY') + ) + ) + throw new ForbiddenException( + 'Only submission and approved queueing are available', + ); + if (dto.toState === 'READY') { + const approval = await tx.orchestrationApprovalDecision.findFirst({ + where: { taskId: id, gate: 1, decision: 'APPROVED' }, + orderBy: { createdAt: 'desc' }, + }); + if (!approval || approval.expiresAt <= new Date()) + throw new ForbiddenException('Gate1 expired'); + await tx.prepareWorkflowOutbox.create({ data: { taskId: id } }); + } + return this.event( + tx, + t, + dto.toState, + a, + dto.idempotencyKey, + 'PREPARE_TRANSITION', + fingerprint, + ); + }); + } + async decide(id: string, gate: number, dto: any, a: WorkflowIdentity) { + this.identity(a, ['SUPER_ADMIN']); + if (gate !== 1) + throw new ForbiddenException( + 'Gate2 decisions and release remain disabled', + ); + if ( + !['APPROVED', 'REJECTED'].includes(dto.decision) || + !Number.isFinite(Date.parse(dto.expiresAt)) || + Date.parse(dto.expiresAt) <= Date.now() || + Date.parse(dto.expiresAt) > Date.now() + 86400000 + ) + throw new BadRequestException( + 'Decision requires future expiry within 24 hours', + ); + return this.db.$transaction(async (tx: any) => { + await this.allowed(tx); + const t = await this.task(tx, id); + const fingerprint = JSON.stringify({ id, gate, dto, a }); + if (await this.replay(tx, dto.idempotencyKey, fingerprint)) return t; + this.version(t, dto); + if (t.metadata.createdBy === a.id) + throw new ForbiddenException('Independent reviewer required'); + if (t.state !== 'GATE1_PENDING') + throw new ConflictException('Gate1 pending required'); + await tx.orchestrationApprovalDecision.create({ + data: { + taskId: id, + gate: 1, + decision: dto.decision, + actorId: a.id, + expiresAt: new Date(dto.expiresAt), + idempotencyKey: `decision:${dto.idempotencyKey}`, + }, + }); + return this.event( + tx, + t, + dto.decision === 'APPROVED' ? 'GATE1_APPROVED' : 'REJECTED', + a, + dto.idempotencyKey, + 'PREPARE_GATE1', + fingerprint, + ); + }); + } + async claim(a: WorkflowIdentity, id?: string): Promise { + this.worker(a); + return this.db.$transaction(async (tx: any) => { + await this.allowed(tx, true); + // Lock order is task then outbox for every worker operation. SKIP LOCKED permits competing workers. + const rows: any[] = id + ? await tx.$queryRaw`SELECT t.id FROM orchestration_tasks t JOIN prepare_workflow_outbox o ON o.task_id=t.id WHERE t.id=${id}::uuid AND o.status='PENDING' AND (o.lease_until IS NULL OR o.lease_until < now()) ORDER BY o.created_at FOR UPDATE OF t SKIP LOCKED LIMIT 1` + : await tx.$queryRaw`SELECT t.id FROM orchestration_tasks t JOIN prepare_workflow_outbox o ON o.task_id=t.id WHERE o.status='PENDING' AND (o.lease_until IS NULL OR o.lease_until < now()) ORDER BY o.created_at FOR UPDATE OF t SKIP LOCKED LIMIT 1`; + if (!rows.length) return null; + const t = await this.task(tx, rows[0].id); + const o = await tx.prepareWorkflowOutbox.findUnique({ + where: { taskId: t.id }, + }); + const approval = await tx.orchestrationApprovalDecision.findFirst({ + where: { taskId: t.id, gate: 1, decision: 'APPROVED' }, + orderBy: { createdAt: 'desc' }, + }); + if (!approval || approval.expiresAt <= new Date()) + throw new ForbiddenException('Gate1 expired'); + if (o.attempts >= 3) throw new ForbiddenException('Attempts exhausted'); + const costs = await tx.orchestrationBudgetLedger.aggregate({ + where: { taskId: t.id }, + _sum: { amountCents: true }, + }); + if ((costs._sum.amountCents ?? 0) + 20 > t.budgetLimitCents) + throw new ForbiddenException('Hard task budget exceeded'); + if (o.runId) + await tx.orchestrationRun.update({ + where: { id: o.runId }, + data: { + status: 'FAILED', + retryClass: 'TRANSIENT', + finishedAt: new Date(), + }, + }); + const token = randomUUID(), + attempt = o.attempts + 1; + const run = await tx.orchestrationRun.create({ + data: { + taskId: t.id, + attempt, + status: 'RUNNING', + environment: 'staging', + commitSha: t.metadata.commitSha, + idempotencyKey: `prepare-run:${o.id}:${attempt}`, + startedAt: new Date(), + }, + }); + const reservation = await tx.orchestrationBudgetLedger.create({ + data: { + taskId: t.id, + amountCents: 20, + kind: 'RESERVE', + actorId: a.id, + idempotencyKey: `prepare-reserve:${run.id}`, + }, + }); + const leaseUntil = new Date(Date.now() + 30000); + await tx.prepareWorkflowOutbox.update({ + where: { id: o.id }, + data: { + attempts: attempt, + leaseOwner: a.id, + leaseToken: token, + leaseUntil, + heartbeatAt: new Date(), + runId: run.id, + reservationId: reservation.id, + }, + }); + await this.event( + tx, + t, + 'RUNNING', + a, + `prepare-claim:${run.id}`, + 'PREPARE_CLAIMED', + token, + ); + return { + taskId: t.id, + runId: run.id, + attempt, + leaseToken: token, + leaseUntil, + commitSha: run.commitSha, + environment: 'staging', + maxCostCents: 20, + }; + }); + } + private async lease(tx: any, a: WorkflowIdentity, dto: any) { + const t = await this.task(tx, dto.taskId); + const o = await tx.prepareWorkflowOutbox.findUnique({ + where: { taskId: t.id }, + }); + if ( + !o || + o.status !== 'PENDING' || + o.leaseOwner !== a.id || + o.leaseToken !== dto.leaseToken || + o.runId !== dto.runId || + !o.leaseUntil || + o.leaseUntil <= new Date() || + t.state !== 'RUNNING' + ) + throw new ForbiddenException('Active owned lease required'); + return { t, o }; + } + async heartbeat(a: WorkflowIdentity, dto: any) { + this.worker(a); + return this.db.$transaction(async (tx: any) => { + await this.allowed(tx); + const { o } = await this.lease(tx, a, dto); + return tx.prepareWorkflowOutbox.update({ + where: { id: o.id }, + data: { + heartbeatAt: new Date(), + leaseUntil: new Date(Date.now() + 30000), + }, + select: { leaseUntil: true }, + }); + }); + } + async complete(a: WorkflowIdentity, dto: any) { + this.worker(a); + if ( + !/^[a-f0-9]{64}$/.test(dto.sha256 ?? '') || + !/^https:\/\//.test(dto.uri ?? '') || + !Number.isSafeInteger(dto.actualCents) || + dto.actualCents < 0 || + dto.actualCents > 20 + ) + throw new BadRequestException( + 'QA hash, HTTPS URI and cost 0..20 required', + ); + return this.db.$transaction(async (tx: any) => { + await this.allowed(tx); + const { t, o } = await this.lease(tx, a, dto); + await tx.orchestrationArtifact.create({ + data: { runId: o.runId, kind: 'QA', sha256: dto.sha256, uri: dto.uri }, + }); + await tx.orchestrationAttestation.create({ + data: { + runId: o.runId, + kind: 'QA', + subjectHash: dto.sha256, + signer: a.id, + predicate: { passed: true, scope: 'STAGE_PREPARE' }, + }, + }); + await tx.orchestrationRun.update({ + where: { id: o.runId }, + data: { status: 'SUCCEEDED', finishedAt: new Date() }, + }); + await tx.orchestrationBudgetLedger.create({ + data: { + taskId: t.id, + kind: 'RECONCILE', + amountCents: dto.actualCents - 20, + actorId: a.id, + idempotencyKey: `prepare-cost:${o.runId}`, + metadata: { + reservationId: o.reservationId, + actualCents: dto.actualCents, + }, + }, + }); + await tx.prepareWorkflowOutbox.update({ + where: { id: o.id }, + data: { status: 'DONE', leaseToken: null, leaseUntil: null }, + }); + return this.event( + tx, + t, + 'VERIFYING', + a, + `prepare-qa:${o.runId}`, + 'PREPARE_QA_SUCCEEDED', + dto.sha256, + ); + }); + } + async stageEvidence(a: WorkflowIdentity, dto: any) { + this.identity(a, ['STAGE_CI']); + if (!this.config.stageCiIds.includes(a.id)) + throw new ForbiddenException('Stage CI identity not provisioned'); + if ( + !/^[a-f0-9]{64}$/.test(dto.evidenceSha256 ?? '') || + !/^https:\/\//.test(dto.uri ?? '') + ) + throw new BadRequestException( + 'Stage evidence hash and HTTPS URI required', + ); + return this.db.$transaction(async (tx: any) => { + await this.allowed(tx); + const t = await this.task(tx, dto.taskId); + const run = await tx.orchestrationRun.findUnique({ + where: { id: dto.runId }, + }); + if ( + t.state !== 'VERIFYING' || + !run || + run.taskId !== t.id || + run.environment !== 'staging' || + run.status !== 'SUCCEEDED' || + run.commitSha !== dto.commitSha || + t.metadata.commitSha !== dto.commitSha + ) + throw new ForbiddenException( + 'Immutable successful Stage run binding required', + ); + const qa = await tx.orchestrationAttestation.findFirst({ + where: { runId: run.id, kind: 'QA', subjectHash: dto.sha256 }, + }); + if (!qa || qa.signer === a.id) + throw new ForbiddenException('Independent QA evidence required'); + await tx.orchestrationArtifact.create({ + data: { + runId: run.id, + kind: 'STAGE_EVIDENCE', + sha256: dto.evidenceSha256, + uri: dto.uri, + }, + }); + await tx.orchestrationAttestation.create({ + data: { + runId: run.id, + kind: 'STAGE', + subjectHash: dto.sha256, + signer: a.id, + predicate: { + evidenceSha256: dto.evidenceSha256, + commitSha: dto.commitSha, + environment: 'staging', + }, + }, + }); + return this.event( + tx, + t, + 'GATE2_PENDING', + a, + `prepare-stage:${run.id}`, + 'STAGE_EVIDENCE_RECORDED', + dto.evidenceSha256, + ); + }); + } + project(t: any, a: WorkflowIdentity) { + const safe = + t.operationMode === 'PREPARE' && t.metadata?.scope === 'STAGE_PREPARE'; + const owner = + a?.id && (t.metadata?.createdBy === a.id || a.role === 'SUPER_ADMIN'); + const actions: string[] = []; + if (safe && owner && t.state === 'DRAFT') actions.push('SUBMIT_GATE1'); + if (safe && owner && t.state === 'GATE1_APPROVED') + actions.push('QUEUE_PREPARE'); + if ( + safe && + a?.role === 'SUPER_ADMIN' && + t.metadata?.createdBy !== a.id && + t.state === 'GATE1_PENDING' + ) + actions.push('APPROVE_GATE1', 'REJECT_GATE1'); + return { + ...t, + metadata: safe + ? { + scope: 'STAGE_PREPARE', + environment: 'staging', + commitSha: t.metadata.commitSha, + } + : {}, + availableActions: actions, + }; + } +} diff --git a/backend/src/orchestration/workflow-db.spec.ts b/backend/src/orchestration/workflow-db.spec.ts new file mode 100644 index 00000000..8d98004f --- /dev/null +++ b/backend/src/orchestration/workflow-db.spec.ts @@ -0,0 +1,324 @@ +import { PrismaClient } from '@prisma/client'; +import { PrepareWorkflowService } from './prepare-workflow.service'; +import { OperationMode, WorkflowState } from './orchestration.domain'; + +const enabled = process.env.WORKFLOW_DB === '1'; +(enabled ? describe : describe.skip)('isolated prepare workflow', () => { + const db = new PrismaClient(); + const admin = { id: 'proposer', role: 'Admin' }; + const reviewer = { id: 'reviewer', role: 'SUPER_ADMIN' }; + const worker = { id: 'worker-1', role: 'WORKFLOW_WORKER' }; + const ci = { id: 'isolated-ci', role: 'STAGE_CI' }; + const service = new PrepareWorkflowService(db as never, { + workerIds: ['worker-1', 'worker-2'], + stageCiIds: ['isolated-ci'], + }); + let seq = 0; + const dto = () => ({ + key: `wf-${Date.now()}-${seq++}`, + title: 'Bounded isolated QA tracer', + department: 'ENGINEERING', + operationMode: OperationMode.PREPARE, + budgetLimitCents: 20, + metadata: { + commitSha: 'a'.repeat(40), + token: 'must-not-persist', + scope: 'production', + actorId: 'spoof', + }, + }); + async function ready() { + let t = await service.create(dto(), admin); + t = await service.transition( + t.id, + { + toState: WorkflowState.GATE1_PENDING, + expectedVersion: t.version, + idempotencyKey: `submit-${t.id}`, + }, + admin, + ); + t = await service.decide( + t.id, + 1, + { + decision: 'APPROVED', + expectedVersion: t.version, + idempotencyKey: `approve-${t.id}`, + expiresAt: new Date(Date.now() + 60000).toISOString(), + }, + reviewer, + ); + return service.transition( + t.id, + { + toState: WorkflowState.READY, + expectedVersion: t.version, + idempotencyKey: `ready-${t.id}`, + }, + admin, + ); + } + afterAll(async () => { + await db.$disconnect(); + }); + it('fences heartbeat and completes QA plus independent Stage evidence only to Gate2 pending', async () => { + const t = await ready(); + const c = await service.claim(worker, t.id); + await expect( + service.heartbeat({ ...worker, id: 'worker-2' }, c), + ).rejects.toThrow(/lease/i); + await service.heartbeat(worker, c); + const qa = { + ...c, + sha256: 'b'.repeat(64), + uri: 'https://ci.example/qa', + actualCents: 8, + }; + await service.complete(worker, qa); + await expect( + service.stageEvidence( + { id: 'proposer', role: 'Admin' }, + { + taskId: t.id, + runId: c.runId, + sha256: qa.sha256, + commitSha: c.commitSha, + evidenceSha256: 'c'.repeat(64), + uri: 'https://ci.example/stage', + }, + ), + ).rejects.toThrow(/identity/i); + await service.stageEvidence(ci, { + taskId: t.id, + runId: c.runId, + sha256: qa.sha256, + commitSha: c.commitSha, + evidenceSha256: 'c'.repeat(64), + uri: 'https://ci.example/stage', + }); + expect( + await db.orchestrationTask.findUnique({ where: { id: t.id } }), + ).toMatchObject({ state: 'GATE2_PENDING' }); + expect( + await db.orchestrationBudgetLedger.aggregate({ + where: { taskId: t.id }, + _sum: { amountCents: true }, + }), + ).toMatchObject({ _sum: { amountCents: 8 } }); + await expect(service.complete(worker, qa)).rejects.toThrow(/lease|active/i); + }); + it('bounds expired-lease retries and rejects stale completion without releasing unknown costs', async () => { + const t = await service.create({ ...dto(), budgetLimitCents: 60 }, admin); + let current = await service.transition( + t.id, + { + toState: 'GATE1_PENDING', + expectedVersion: 1, + idempotencyKey: `s-${t.id}`, + }, + admin, + ); + current = await service.decide( + t.id, + 1, + { + decision: 'APPROVED', + expectedVersion: current.version, + idempotencyKey: `a-${t.id}`, + expiresAt: new Date(Date.now() + 60000).toISOString(), + }, + reviewer, + ); + await service.transition( + t.id, + { + toState: 'READY', + expectedVersion: current.version, + idempotencyKey: `q-${t.id}`, + }, + admin, + ); + const first = await service.claim(worker, t.id); + for (let i = 0; i < 2; i++) { + await db.prepareWorkflowOutbox.update({ + where: { taskId: t.id }, + data: { leaseUntil: new Date(0) }, + }); + const next = await service.claim(worker, t.id); + expect(next.attempt).toBe(i + 2); + } + await expect( + service.complete(worker, { + ...first, + sha256: 'b'.repeat(64), + uri: 'https://ci.example/qa', + actualCents: 0, + }), + ).rejects.toThrow(/lease/i); + await db.prepareWorkflowOutbox.update({ + where: { taskId: t.id }, + data: { leaseUntil: new Date(0) }, + }); + await expect(service.claim(worker, t.id)).rejects.toThrow(/exhausted/i); + expect(await db.orchestrationRun.count({ where: { taskId: t.id } })).toBe( + 3, + ); + expect( + await db.orchestrationBudgetLedger.aggregate({ + where: { taskId: t.id }, + _sum: { amountCents: true }, + }), + ).toMatchObject({ _sum: { amountCents: 60 } }); + }); + it('rejects self approval, non-PREPARE and unprovisioned workers', async () => { + await expect( + service.create( + { ...dto(), operationMode: OperationMode.MUTATE_PRODUCTION }, + admin, + ), + ).rejects.toThrow(/PREPARE/); + let t = await service.create(dto(), reviewer); + t = await service.transition( + t.id, + { + toState: 'GATE1_PENDING', + expectedVersion: 1, + idempotencyKey: `s-${t.id}`, + }, + reviewer, + ); + await expect( + service.decide( + t.id, + 1, + { + decision: 'APPROVED', + expectedVersion: t.version, + idempotencyKey: `a-${t.id}`, + expiresAt: new Date(Date.now() + 60000).toISOString(), + }, + reviewer, + ), + ).rejects.toThrow(/independent/i); + await expect( + service.claim({ id: 'spoof', role: 'WORKFLOW_WORKER' }), + ).rejects.toThrow(/provisioned/i); + await expect(service.decide(t.id, 2, {}, reviewer)).rejects.toThrow( + /disabled/i, + ); + }); + it('serializes duplicate queue submission and denies claims beyond remaining task budget', async () => { + const t = await ready(); + await db.orchestrationBudgetLedger.create({ + data: { + taskId: t.id, + amountCents: 1, + kind: 'SPEND', + actorId: 'fixture', + idempotencyKey: `spend-${t.id}`, + }, + }); + await expect(service.claim(worker, t.id)).rejects.toThrow(/budget/i); + expect(await db.orchestrationRun.count({ where: { taskId: t.id } })).toBe( + 0, + ); + expect( + await db.prepareWorkflowOutbox.count({ where: { taskId: t.id } }), + ).toBe(1); + }); + it('replays submission safely and serializes racing queue requests', async () => { + let t = await service.create(dto(), admin); + const submit = { + toState: 'GATE1_PENDING', + expectedVersion: t.version, + idempotencyKey: `replay-${t.id}`, + }; + t = await service.transition(t.id, submit, admin); + expect((await service.transition(t.id, submit, admin)).version).toBe( + t.version, + ); + await expect( + service.transition(t.id, { ...submit, toState: 'READY' }, admin), + ).rejects.toThrow(/mismatch/i); + t = await service.decide( + t.id, + 1, + { + decision: 'APPROVED', + expectedVersion: t.version, + idempotencyKey: `r-approve-${t.id}`, + expiresAt: new Date(Date.now() + 60000).toISOString(), + }, + reviewer, + ); + const results = await Promise.allSettled( + ['one', 'two'].map((k) => + service.transition( + t.id, + { + toState: 'READY', + expectedVersion: t.version, + idempotencyKey: `${k}-${t.id}`, + }, + admin, + ), + ), + ); + expect(results.filter((r) => r.status === 'fulfilled')).toHaveLength(1); + expect( + await db.prepareWorkflowOutbox.count({ where: { taskId: t.id } }), + ).toBe(1); + }); + it('kill switch freezes claims without creating run or reservation', async () => { + const t = await ready(); + await db.orchestrationKillSwitch.create({ + data: { + scope: 'GLOBAL', + level: 'FREEZE_NEW_WORK', + reason: 'isolated test', + actorId: 'fixture', + }, + }); + try { + await expect(service.claim(worker, t.id)).rejects.toThrow(/kill switch/i); + } finally { + await db.orchestrationKillSwitch.delete({ where: { scope: 'GLOBAL' } }); + } + expect(await db.orchestrationRun.count({ where: { taskId: t.id } })).toBe( + 0, + ); + }); + it('persists an atomic outbox and exactly one worker wins concurrent claims', async () => { + const t = await ready(); + const claims = await Promise.all([ + service.claim(worker, t.id), + service.claim({ id: 'worker-2', role: 'WORKFLOW_WORKER' }, t.id), + ]); + expect(claims.filter(Boolean)).toHaveLength(1); + const claim = claims.find(Boolean)!; + expect(claim.taskId).toBe(t.id); + expect(claim.attempt).toBe(1); + expect(await db.orchestrationRun.count({ where: { taskId: t.id } })).toBe( + 1, + ); + expect( + await db.orchestrationBudgetLedger.aggregate({ + where: { taskId: t.id }, + _sum: { amountCents: true }, + }), + ).toMatchObject({ _sum: { amountCents: 20 } }); + expect( + await service.project( + await db.orchestrationTask.findUniqueOrThrow({ where: { id: t.id } }), + admin, + ), + ).toMatchObject({ + metadata: { + scope: 'STAGE_PREPARE', + environment: 'staging', + commitSha: 'a'.repeat(40), + }, + }); + }); +});