Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
d43aa02b01 | ||
|
|
5b4d099d80 |
No files matched your search
@@ -0,0 +1,10 @@
|
||||
CREATE TABLE prepare_workflow_outbox (
|
||||
id uuid PRIMARY KEY, task_id uuid NOT NULL UNIQUE REFERENCES orchestration_tasks(id) ON DELETE RESTRICT,
|
||||
status varchar(20) NOT NULL DEFAULT 'PENDING' CHECK (status IN ('PENDING','DONE')),
|
||||
attempts integer NOT NULL DEFAULT 0 CHECK (attempts BETWEEN 0 AND 3),
|
||||
lease_owner varchar(100), lease_token uuid, lease_until timestamptz, heartbeat_at timestamptz,
|
||||
run_id uuid REFERENCES orchestration_runs(id) ON DELETE RESTRICT,
|
||||
reservation_id uuid REFERENCES orchestration_budget_ledger(id) ON DELETE RESTRICT,
|
||||
created_at timestamptz NOT NULL DEFAULT now()
|
||||
);
|
||||
CREATE INDEX prepare_workflow_outbox_status_lease_until_idx ON prepare_workflow_outbox(status,lease_until);
|
||||
@@ -1232,3 +1232,20 @@ model OrchestrationKillSwitch {
|
||||
createdAt DateTime @default(now()) @map("created_at") @db.Timestamptz()
|
||||
@@map("orchestration_kill_switches")
|
||||
}
|
||||
|
||||
// Transactional dispatch intent. Lease tokens are server-only fencing credentials.
|
||||
model PrepareWorkflowOutbox {
|
||||
id String @id @default(uuid()) @db.Uuid
|
||||
taskId String @unique @map("task_id") @db.Uuid
|
||||
status String @default("PENDING") @db.VarChar(20)
|
||||
attempts Int @default(0)
|
||||
leaseOwner String? @map("lease_owner") @db.VarChar(100)
|
||||
leaseToken String? @map("lease_token") @db.Uuid
|
||||
leaseUntil DateTime? @map("lease_until") @db.Timestamptz()
|
||||
heartbeatAt DateTime? @map("heartbeat_at") @db.Timestamptz()
|
||||
runId String? @map("run_id") @db.Uuid
|
||||
reservationId String? @map("reservation_id") @db.Uuid
|
||||
createdAt DateTime @default(now()) @map("created_at") @db.Timestamptz()
|
||||
@@index([status, leaseUntil])
|
||||
@@map("prepare_workflow_outbox")
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
# Bounded Stage PREPARE workflow foundation
|
||||
|
||||
This slice is NOT a runtime executor, deployment mechanism, or full v4 completion. It enables authenticated proposal/Gate1/queue APIs and provides internal worker/CI service methods. Default worker/CI identity allowlists are empty: no browser or production execution route exists.
|
||||
|
||||
## Panel HTTP contract
|
||||
All routes require existing JWT authentication with `Admin` or `SUPER_ADMIN`. Actor is exclusively `req.user.id ?? req.user.sub`, role exclusively `req.user.role`. Top-level unknown mutation fields return 400; client actor/profile fields are never trusted.
|
||||
|
||||
Base `/admin/orchestration`:
|
||||
|
||||
- `GET /tasks`: task array.
|
||||
- `GET /tasks/:id`: task object.
|
||||
- `GET /overview`: existing inventory items plus capabilities `{mutations:true, prepare:true, release:false, gate2Decisions:false, chat:false, runtime:false}`, reason `bounded_stage_prepare_only`.
|
||||
- `POST /tasks`: existing CreateOrchestrationTaskDto `{key,title,department,operationMode:"PREPARE",budgetLimitCents,metadata:{commitSha,scope,acceptanceCriteria}}`; commitSha is required lowercase 40hex; scope is a non-blank string of at most 2000 characters; acceptanceCriteria is 1..20 non-blank strings of at most 500 characters each; budget is integer 20..10000 cents. Returns 201 projected task, state DRAFT/version1. Validated scope and acceptance criteria are persisted exactly and remain unchanged by workflow transitions. The internal `workflowScope:"STAGE_PREPARE"`, environment and createdBy are server-owned; input profile/actor/other arbitrary keys are discarded. Key max120, title max255, department max50. Proposal key is unique; sequential or concurrent duplicate keys return 409, without updating the original task or appending another proposal event. Fetch/reconcile the existing task rather than retry blindly.
|
||||
- `PUT /tasks/:id/transition`: `{toState:"GATE1_PENDING"|"READY",expectedVersion,idempotencyKey,reason?}`. Only DRAFT->GATE1_PENDING and GATE1_APPROVED->READY. Owner or SUPER_ADMIN. Returns 200 projected task. READY atomically creates dispatch outbox. Requires unexpired Gate1.
|
||||
- `POST /tasks/:id/gates/1/decide`: existing GateDecisionDto `{decision:"APPROVED"|"REJECTED",expectedVersion,idempotencyKey,expiresAt,notes?}`. SUPER_ADMIN independent from proposer. Future expiry <=24h. Returns 201 **projected task**, not decision object.
|
||||
- Gate2 decision, budget writes/reconcile, command routes (including release), chat and kill-switch HTTP mutations remain 403. Runtime resource reads remain 503.
|
||||
|
||||
Task projection keeps existing scalar fields and safe nested runs/artifacts, decisions, events and ledger. Adds only `metadata:{scope,acceptanceCriteria,environment:"staging",commitSha}` for bounded proposals, otherwise `{}`. Never includes createdBy, workflowScope, payloads, URI, signatures, tokens or lease credentials. Pre-fix rows with the legacy scope sentinel remain readable/operable but have no recovered acceptance criteria: do not represent those as complete immutable scope review evidence.
|
||||
|
||||
`availableActions:string[]` is state/identity-derived: DRAFT owner -> `SUBMIT_GATE1`; GATE1_PENDING independent SUPER_ADMIN -> `APPROVE_GATE1`,`REJECT_GATE1`; GATE1_APPROVED owner/SUPER_ADMIN -> `QUEUE_PREPARE`; all worker/verification/Gate2 and legacy states -> `[]`. Actions indicate workflow eligibility, not a guarantee: current kill switch, approval expiry and concurrent versions are rechecked transactionally on mutation. Panel must not infer actions from state or send actor identity.
|
||||
|
||||
Common errors: 400 invalid DTO/binding; 403 wrong scope/identity/independence/kill/budget/expiry; 409 duplicate proposal key, stale version or idempotency payload mismatch. Transition/decision replay returns current persisted task and does not duplicate side effects.
|
||||
|
||||
## Internal service contracts (NOT browser endpoints)
|
||||
`PrepareWorkflowService` is exported by OrchestrationModule. Trusted backend composition must override `PREPARE_WORKFLOW_CONFIG` with explicit `workerIds` and `stageCiIds`, default both empty. Do not populate from request bodies or user-selected runtime profiles. Worker identities require role WORKFLOW_WORKER AND ID allowlist; Stage identities require STAGE_CI AND separate ID allowlist. Credential/session provisioning and transport remain the integrating runtime owner's responsibility.
|
||||
|
||||
- `claim(identity, taskId?) -> null | {taskId,runId,attempt,leaseToken,leaseUntil,commitSha,environment:"staging",maxCostCents:20}`. Transactional task row lock with SKIP LOCKED, immutable run binding, 30s lease, max3 attempts. Each attempt reserves **a fixed hard upper bound of 20 cents**, not an estimated or measured departmental cost. Unknown cost on expired attempts remains reserved conservatively.
|
||||
- `heartbeat(identity,{taskId,runId,leaseToken}) -> {leaseUntil}`. Same owner/token, unexpired active lease required.
|
||||
- `complete(identity,{taskId,runId,leaseToken,sha256,uri,actualCents}) -> task`. QA success ingestion only; lowercase sha256, HTTPS URI, actual cost integer0..20 from trusted worker. Atomically records immutable QA artifact/attestation, SUCCEEDED run, reservation reconciliation, DONE outbox and VERIFYING task. Expired/stale/foreign token cannot finish. Not replayable after success; caller must read persisted task/run after response loss.
|
||||
- `stageEvidence(identity,{taskId,runId,sha256,commitSha,evidenceSha256,uri}) -> task`. Independently allowlisted Stage CI identity, successful exact staging commit/run, registered QA subject, independent QA signer. Atomically records Stage artifact/attestation and reaches GATE2_PENDING. No release or Gate2 approval. Does not perform or claim deployment itself. Caller supplies actual trusted CI evidence, not browser assertions.
|
||||
|
||||
## Verification and explicit gaps
|
||||
Real isolated PostgreSQL16, no published ports/no live volumes. Added migrations applied with ON_ERROR_STOP. TDD RED logs cover missing workflow behavior, heartbeat/QA behavior and HTTP enablement. Full backend regression excluding existing legacy release-containing orchestration.integration.spec: 106 suites /851 tests including seven real-DB workflow cases, build exit0. No RELEASE_AUTHORIZE service operation executed.
|
||||
|
||||
DB regressions: concurrent claim winner, queue race/replay mismatch, hard budget rollback, stale lease fencing, max3 attempts, conservative expired costs, kill switch, self approval, trusted IDs, QA reconciliation and Stage evidence to Gate2 pending. Test fixture artifact URIs/hashes are not real deployment evidence.
|
||||
|
||||
Remaining: actual authenticated runtime worker transport/executor and pinned CI evidence transport; shared department/company budget model and locking (NOT implemented or represented by fake totals); retry backoff/dead-letter UX/cancellation; hard total execution wall-time; durable request replay of completion/evidence; kill-switch concurrent insertion/version locking; migration rehearsal with full legacy schema; real Stage CI artifact/deployment verification. No push/deploy/live DB/runtime edits. Production release remains denied at HTTP boundary; unsafe legacy core release method is deliberately untouched and must never be called by this slice.
|
||||
|
||||
## Contract repair verification
|
||||
|
||||
Actual frontend inspection confirmed proposal submission sent human scope and acceptanceCriteria but omitted required commitSha. Frontend repair is separately assigned; no frontend file changed here. New proposals now require all three fields. Internal Stage restriction uses a separate server-owned workflowScope marker rather than overwriting user scope. No metadata update route exists in this bounded workflow.
|
||||
|
||||
TDD RED→GREEN against isolated PostgreSQL16 reproduced dropped scope/criteria, nine invalid metadata acceptances (including array-to-string commit coercion), and duplicate-key P2002 leakage. Final database suite: 24 cases, including authenticated HTTP create/list/detail persistence, 400 validation, and 409 duplicate responses. Full allowed backend regression: 106 suites / 868 tests passed; legacy orchestration.integration.spec.ts excluded to avoid executing RELEASE_AUTHORIZE. Build exit0. Scoped lint remains failing on inherited require-await/no-unnecessary-type-assertion errors (13 errors); no lint suppression or config change applied. Disposable DB only; no live edits/push/deploy.
|
||||
@@ -92,7 +92,7 @@ describe('integration read boundary', () => {
|
||||
[method]('/admin/orchestration/' + path)
|
||||
.set('Authorization', 'Admin')
|
||||
.send({})
|
||||
.expect(403);
|
||||
.expect(path === 'tasks' || path.endsWith('/transition') ? 400 : 403);
|
||||
}
|
||||
for (const name of [
|
||||
'createTask',
|
||||
|
||||
@@ -1,5 +1,9 @@
|
||||
import {
|
||||
Controller,
|
||||
Body,
|
||||
Req,
|
||||
Optional,
|
||||
ValidationPipe,
|
||||
Get,
|
||||
Param,
|
||||
Post,
|
||||
@@ -13,6 +17,20 @@ import { JwtAuthGuard } from '../auth/jwt-auth.guard';
|
||||
import { RolesGuard } from '../common/guards/roles.guard';
|
||||
import { Roles } from '../common/decorators/roles.decorator';
|
||||
import { OrchestrationService } from './orchestration.service';
|
||||
import {
|
||||
PrepareWorkflowService,
|
||||
WorkflowIdentity,
|
||||
} from './prepare-workflow.service';
|
||||
import {
|
||||
CreateOrchestrationTaskDto,
|
||||
TransitionTaskDto,
|
||||
GateDecisionDto,
|
||||
} from './dto/orchestration.dto';
|
||||
const workflowValidation = new ValidationPipe({
|
||||
transform: true,
|
||||
whitelist: true,
|
||||
forbidNonWhitelisted: true,
|
||||
});
|
||||
|
||||
// Explicit response allowlists: never serialize arbitrary JSON, signatures, URIs or BigInt.
|
||||
function pick(row: any, keys: string[]) {
|
||||
@@ -73,12 +91,37 @@ function project(row: any) {
|
||||
@Roles('Admin', 'SUPER_ADMIN')
|
||||
@Controller('admin/orchestration')
|
||||
export class OrchestrationController {
|
||||
constructor(private readonly service: OrchestrationService) {}
|
||||
@Get('tasks') async all() {
|
||||
return (await this.service.findAll()).map(project);
|
||||
constructor(
|
||||
private readonly service: OrchestrationService,
|
||||
@Optional() private readonly workflow?: PrepareWorkflowService,
|
||||
) {}
|
||||
private actor(req: any): WorkflowIdentity {
|
||||
const id = req?.user?.id ?? req?.user?.sub;
|
||||
if (
|
||||
typeof id !== 'string' ||
|
||||
!id ||
|
||||
!['Admin', 'SUPER_ADMIN'].includes(req?.user?.role)
|
||||
)
|
||||
throw new ForbiddenException('Authenticated workflow identity required');
|
||||
return { id, role: req.user.role };
|
||||
}
|
||||
@Get('tasks/:id') async one(@Param('id') id: string) {
|
||||
return project(await this.service.findOne(id));
|
||||
private safe(row: any, req: any) {
|
||||
const view = project(row);
|
||||
if (!this.workflow) return view;
|
||||
const w = this.workflow.project(row, this.actor(req));
|
||||
return {
|
||||
...view,
|
||||
metadata: w.metadata,
|
||||
availableActions: w.availableActions,
|
||||
};
|
||||
}
|
||||
@Get('tasks') async all(@Req() req: any) {
|
||||
return (await this.service.findAll()).map((row: any) =>
|
||||
this.safe(row, req),
|
||||
);
|
||||
}
|
||||
@Get('tasks/:id') async one(@Param('id') id: string, @Req() req: any) {
|
||||
return this.safe(await this.service.findOne(id), req);
|
||||
}
|
||||
@Get('overview') async overview() {
|
||||
const tasks = await this.service.findAll();
|
||||
@@ -87,8 +130,17 @@ export class OrchestrationController {
|
||||
{ id: 'tasks', title: 'Persisted task inventory', count: tasks.length },
|
||||
],
|
||||
measuredAt: new Date().toISOString(),
|
||||
capabilities: { mutations: false, chat: false, runtime: false },
|
||||
reason: 'security_hardening_pending',
|
||||
capabilities: {
|
||||
mutations: !!this.workflow,
|
||||
prepare: !!this.workflow,
|
||||
release: false,
|
||||
gate2Decisions: false,
|
||||
chat: false,
|
||||
runtime: false,
|
||||
},
|
||||
reason: this.workflow
|
||||
? 'bounded_stage_prepare_only'
|
||||
: 'security_hardening_pending',
|
||||
};
|
||||
}
|
||||
@Get('releases') async releases() {
|
||||
@@ -120,15 +172,40 @@ export class OrchestrationController {
|
||||
'No persisted model policy read contract available',
|
||||
);
|
||||
}
|
||||
// The committed legacy core is not safe to expose. No env switch or role may bypass this boundary.
|
||||
@Post('tasks') create() {
|
||||
return this.deny();
|
||||
// Only the separate bounded PREPARE service is exposed; legacy commands remain denied.
|
||||
@Post('tasks') async create(
|
||||
@Body(workflowValidation) dto: CreateOrchestrationTaskDto,
|
||||
@Req() req: any,
|
||||
) {
|
||||
if (!this.workflow) return this.deny();
|
||||
return this.safe(await this.workflow.create(dto, this.actor(req)), req);
|
||||
}
|
||||
@Put('tasks/:id/transition') transition() {
|
||||
return this.deny();
|
||||
@Put('tasks/:id/transition') async transition(
|
||||
@Param('id') id: string,
|
||||
@Body(workflowValidation) dto: TransitionTaskDto,
|
||||
@Req() req: any,
|
||||
) {
|
||||
if (!this.workflow) return this.deny();
|
||||
return this.safe(
|
||||
await this.workflow.transition(id, dto, this.actor(req)),
|
||||
req,
|
||||
);
|
||||
}
|
||||
@Post('tasks/:id/gates/:gate/decide') decide() {
|
||||
return this.deny();
|
||||
@Post('tasks/:id/gates/:gate/decide') async decide(
|
||||
@Param('id') id: string,
|
||||
@Param('gate') gate: string,
|
||||
@Body() body: unknown,
|
||||
@Req() req: any,
|
||||
) {
|
||||
if (!this.workflow || gate !== '1') return this.deny();
|
||||
const dto = await workflowValidation.transform(body, {
|
||||
type: 'body',
|
||||
metatype: GateDecisionDto,
|
||||
});
|
||||
return this.safe(
|
||||
await this.workflow.decide(id, 1, dto, this.actor(req)),
|
||||
req,
|
||||
);
|
||||
}
|
||||
@Post('tasks/:id/budget') budget() {
|
||||
return this.deny();
|
||||
|
||||
@@ -1,3 +1,7 @@
|
||||
import {
|
||||
PrepareWorkflowService,
|
||||
PREPARE_WORKFLOW_CONFIG,
|
||||
} from './prepare-workflow.service';
|
||||
import { Module } from '@nestjs/common';
|
||||
import { PrismaModule } from '../prisma/prisma.module';
|
||||
import { KillSwitchService } from './kill-switch.service';
|
||||
@@ -7,7 +11,21 @@ import { OrchestrationService } from './orchestration.service';
|
||||
@Module({
|
||||
imports: [PrismaModule],
|
||||
controllers: [OrchestrationController],
|
||||
providers: [OrchestrationService, KillSwitchService, ModelTierPolicyService],
|
||||
exports: [OrchestrationService, KillSwitchService, ModelTierPolicyService],
|
||||
providers: [
|
||||
OrchestrationService,
|
||||
KillSwitchService,
|
||||
ModelTierPolicyService,
|
||||
PrepareWorkflowService,
|
||||
{
|
||||
provide: PREPARE_WORKFLOW_CONFIG,
|
||||
useValue: { workerIds: [], stageCiIds: [] },
|
||||
},
|
||||
],
|
||||
exports: [
|
||||
OrchestrationService,
|
||||
KillSwitchService,
|
||||
ModelTierPolicyService,
|
||||
PrepareWorkflowService,
|
||||
],
|
||||
})
|
||||
export class OrchestrationModule {}
|
||||
@@ -0,0 +1,137 @@
|
||||
import { Test } from '@nestjs/testing';
|
||||
import { INestApplication } from '@nestjs/common';
|
||||
import request from 'supertest';
|
||||
import { OrchestrationController } from './orchestration.controller';
|
||||
import { OrchestrationService } from './orchestration.service';
|
||||
import { PrepareWorkflowService } from './prepare-workflow.service';
|
||||
import { JwtAuthGuard } from '../auth/jwt-auth.guard';
|
||||
import { RolesGuard } from '../common/guards/roles.guard';
|
||||
|
||||
describe('bounded workflow HTTP contracts', () => {
|
||||
let app: INestApplication;
|
||||
const t = {
|
||||
id: 't',
|
||||
key: 'k',
|
||||
state: 'DRAFT',
|
||||
version: 1,
|
||||
metadata: {
|
||||
scope: 'STAGE_PREPARE',
|
||||
environment: 'staging',
|
||||
commitSha: 'a'.repeat(40),
|
||||
token: 'secret',
|
||||
},
|
||||
};
|
||||
const workflow = {
|
||||
create: jest.fn(async () => t),
|
||||
transition: jest.fn(async () => t),
|
||||
decide: jest.fn(async () => t),
|
||||
project: jest.fn((row: any) => ({
|
||||
...row,
|
||||
metadata: {
|
||||
scope: 'STAGE_PREPARE',
|
||||
environment: 'staging',
|
||||
commitSha: 'a'.repeat(40),
|
||||
},
|
||||
availableActions: ['SUBMIT_GATE1'],
|
||||
})),
|
||||
};
|
||||
const core = {
|
||||
findAll: async () => [t],
|
||||
findOne: async () => t,
|
||||
command: jest.fn(),
|
||||
};
|
||||
beforeAll(async () => {
|
||||
const m = await Test.createTestingModule({
|
||||
controllers: [OrchestrationController],
|
||||
providers: [
|
||||
{ provide: OrchestrationService, useValue: core },
|
||||
{ provide: PrepareWorkflowService, useValue: workflow },
|
||||
RolesGuard,
|
||||
],
|
||||
})
|
||||
.overrideGuard(JwtAuthGuard)
|
||||
.useValue({
|
||||
canActivate(c: any) {
|
||||
const req = c.switchToHttp().getRequest();
|
||||
if (!req.headers.authorization) return false;
|
||||
req.user = {
|
||||
sub: 'verified-subject',
|
||||
role: req.headers.authorization,
|
||||
};
|
||||
return true;
|
||||
},
|
||||
})
|
||||
.compile();
|
||||
app = m.createNestApplication();
|
||||
await app.init();
|
||||
});
|
||||
afterAll(async () => {
|
||||
await app.close();
|
||||
});
|
||||
it('accepts existing proposal DTO, derives actor from JWT sub and returns projected task', async () => {
|
||||
const dto = {
|
||||
key: 'k',
|
||||
title: 'T',
|
||||
department: 'ENGINEERING',
|
||||
operationMode: 'PREPARE',
|
||||
budgetLimitCents: 20,
|
||||
metadata: {
|
||||
commitSha: 'a'.repeat(40),
|
||||
scope: 'Staging audit only',
|
||||
acceptanceCriteria: ['QA passes'],
|
||||
},
|
||||
};
|
||||
const r = await request(app.getHttpServer())
|
||||
.post('/admin/orchestration/tasks')
|
||||
.set('Authorization', 'Admin')
|
||||
.send(dto)
|
||||
.expect(201);
|
||||
expect(workflow.create).toHaveBeenCalledWith(dto, {
|
||||
id: 'verified-subject',
|
||||
role: 'Admin',
|
||||
});
|
||||
expect(r.body.availableActions).toEqual(['SUBMIT_GATE1']);
|
||||
expect(JSON.stringify(r.body)).not.toMatch(/token|secret/);
|
||||
await request(app.getHttpServer())
|
||||
.post('/admin/orchestration/tasks')
|
||||
.set('Authorization', 'Admin')
|
||||
.send({ ...dto, actorId: 'browser' })
|
||||
.expect(400);
|
||||
});
|
||||
it('routes fixed transition/gate contracts and rejects invalid gates plus release without invoking core', async () => {
|
||||
await request(app.getHttpServer())
|
||||
.put('/admin/orchestration/tasks/t/transition')
|
||||
.set('Authorization', 'Admin')
|
||||
.send({
|
||||
toState: 'GATE1_PENDING',
|
||||
expectedVersion: 1,
|
||||
idempotencyKey: 'submit',
|
||||
})
|
||||
.expect(200);
|
||||
await request(app.getHttpServer())
|
||||
.post('/admin/orchestration/tasks/t/gates/1/decide')
|
||||
.set('Authorization', 'SUPER_ADMIN')
|
||||
.send({
|
||||
decision: 'APPROVED',
|
||||
expectedVersion: 2,
|
||||
idempotencyKey: 'approval',
|
||||
expiresAt: new Date(Date.now() + 60000).toISOString(),
|
||||
})
|
||||
.expect(201);
|
||||
expect(workflow.decide).toHaveBeenCalledWith('t', 1, expect.any(Object), {
|
||||
id: 'verified-subject',
|
||||
role: 'SUPER_ADMIN',
|
||||
});
|
||||
await request(app.getHttpServer())
|
||||
.post('/admin/orchestration/tasks/t/gates/2/decide')
|
||||
.set('Authorization', 'SUPER_ADMIN')
|
||||
.send({})
|
||||
.expect(403);
|
||||
await request(app.getHttpServer())
|
||||
.post('/admin/orchestration/tasks/t/commands/RELEASE_AUTHORIZE')
|
||||
.set('Authorization', 'SUPER_ADMIN')
|
||||
.send({})
|
||||
.expect(403);
|
||||
expect(core.command).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,559 @@
|
||||
import {
|
||||
Injectable,
|
||||
Inject,
|
||||
ForbiddenException,
|
||||
BadRequestException,
|
||||
ConflictException,
|
||||
NotFoundException,
|
||||
} from '@nestjs/common';
|
||||
import { randomUUID } from 'crypto';
|
||||
import { PrismaService } from '../prisma/prisma.service';
|
||||
export const PREPARE_WORKFLOW_CONFIG = 'PREPARE_WORKFLOW_CONFIG';
|
||||
export interface WorkflowIdentity {
|
||||
id: string;
|
||||
role: string;
|
||||
}
|
||||
export interface WorkflowConfig {
|
||||
workerIds: string[];
|
||||
stageCiIds: string[];
|
||||
}
|
||||
|
||||
/** Bounded Stage preparation only. No release executor, production scope or browser worker identities. */
|
||||
@Injectable()
|
||||
export class PrepareWorkflowService {
|
||||
constructor(
|
||||
@Inject(PrismaService) private readonly db: any,
|
||||
@Inject(PREPARE_WORKFLOW_CONFIG) private readonly config: WorkflowConfig,
|
||||
) {}
|
||||
private identity(a: WorkflowIdentity, roles: string[]) {
|
||||
if (
|
||||
!a?.id ||
|
||||
typeof a.id !== 'string' ||
|
||||
a.id.length > 100 ||
|
||||
!roles.includes(a.role)
|
||||
)
|
||||
throw new ForbiddenException('Authenticated workflow identity required');
|
||||
}
|
||||
private worker(a: WorkflowIdentity) {
|
||||
this.identity(a, ['WORKFLOW_WORKER']);
|
||||
if (!this.config.workerIds.includes(a.id))
|
||||
throw new ForbiddenException('Worker identity not provisioned');
|
||||
}
|
||||
private async allowed(tx: any, claiming = false) {
|
||||
const rows =
|
||||
await tx.$queryRaw`SELECT level, active FROM orchestration_kill_switches WHERE scope = 'GLOBAL' FOR SHARE`;
|
||||
if (rows[0]?.active && !(rows[0].level === 'FREEZE_NEW_WORK' && !claiming))
|
||||
throw new ForbiddenException('Kill switch blocks workflow');
|
||||
}
|
||||
private async task(tx: any, id: string) {
|
||||
await tx.$queryRaw`SELECT id FROM orchestration_tasks WHERE id = ${id}::uuid FOR UPDATE`;
|
||||
const t = await tx.orchestrationTask.findUnique({ where: { id } });
|
||||
if (!t) throw new NotFoundException('Task not found');
|
||||
if (
|
||||
t.operationMode !== 'PREPARE' ||
|
||||
(t.metadata?.workflowScope ?? t.metadata?.scope) !== 'STAGE_PREPARE'
|
||||
)
|
||||
throw new ForbiddenException('Not a bounded Stage proposal');
|
||||
return t;
|
||||
}
|
||||
private async event(
|
||||
tx: any,
|
||||
t: any,
|
||||
state: string,
|
||||
a: WorkflowIdentity,
|
||||
key: string,
|
||||
type: string,
|
||||
fingerprint: string,
|
||||
) {
|
||||
const result = await tx.orchestrationTask.update({
|
||||
where: { id: t.id },
|
||||
data: { state, version: { increment: 1 } },
|
||||
});
|
||||
await tx.orchestrationEvent.create({
|
||||
data: {
|
||||
taskId: t.id,
|
||||
sequence: result.version,
|
||||
actorId: a.id,
|
||||
eventType: type,
|
||||
fromState: t.state,
|
||||
toState: state,
|
||||
idempotencyKey: key,
|
||||
payload: { fingerprint },
|
||||
},
|
||||
});
|
||||
return result;
|
||||
}
|
||||
private async replay(tx: any, key: string, fingerprint: string) {
|
||||
const e = await tx.orchestrationEvent.findUnique({
|
||||
where: { idempotencyKey: key },
|
||||
});
|
||||
if (e && e.payload?.fingerprint !== fingerprint)
|
||||
throw new ConflictException('Idempotency payload mismatch');
|
||||
return e;
|
||||
}
|
||||
private version(t: any, dto: any) {
|
||||
if (
|
||||
!Number.isSafeInteger(dto.expectedVersion) ||
|
||||
dto.expectedVersion !== t.version
|
||||
)
|
||||
throw new ConflictException('Stale task version');
|
||||
if (
|
||||
typeof dto.idempotencyKey !== 'string' ||
|
||||
!dto.idempotencyKey.length ||
|
||||
dto.idempotencyKey.length > 120
|
||||
)
|
||||
throw new BadRequestException('Invalid idempotency key');
|
||||
}
|
||||
async create(dto: any, a: WorkflowIdentity) {
|
||||
this.identity(a, ['Admin', 'SUPER_ADMIN']);
|
||||
const metadata = dto?.metadata;
|
||||
const boundedText = (value: unknown, max: number) =>
|
||||
typeof value === 'string' &&
|
||||
value.trim().length > 0 &&
|
||||
value.length <= max;
|
||||
if (
|
||||
!metadata ||
|
||||
typeof metadata !== 'object' ||
|
||||
Array.isArray(metadata) ||
|
||||
typeof metadata.commitSha !== 'string' ||
|
||||
!boundedText(metadata.scope, 2000) ||
|
||||
!Array.isArray(metadata.acceptanceCriteria) ||
|
||||
metadata.acceptanceCriteria.length < 1 ||
|
||||
metadata.acceptanceCriteria.length > 20 ||
|
||||
!metadata.acceptanceCriteria.every((criterion: unknown) =>
|
||||
boundedText(criterion, 500),
|
||||
)
|
||||
)
|
||||
throw new BadRequestException(
|
||||
'PREPARE requires commitSha, scope 1..2000 characters and 1..20 acceptance criteria of 1..500 characters',
|
||||
);
|
||||
if (
|
||||
dto.operationMode !== 'PREPARE' ||
|
||||
!/^[a-f0-9]{40}$/.test(dto.metadata?.commitSha ?? '') ||
|
||||
!Number.isSafeInteger(dto.budgetLimitCents) ||
|
||||
dto.budgetLimitCents < 20 ||
|
||||
dto.budgetLimitCents > 10000
|
||||
)
|
||||
throw new BadRequestException(
|
||||
'PREPARE requires immutable 40hex commit and budget 20..10000 cents',
|
||||
);
|
||||
if (
|
||||
!dto.key ||
|
||||
dto.key.length > 120 ||
|
||||
!dto.title ||
|
||||
dto.title.length > 255 ||
|
||||
!dto.department ||
|
||||
dto.department.length > 50
|
||||
)
|
||||
throw new BadRequestException('Invalid proposal fields');
|
||||
return this.db
|
||||
.$transaction(async (tx: any) => {
|
||||
await this.allowed(tx, true);
|
||||
const t = await tx.orchestrationTask.create({
|
||||
data: {
|
||||
key: dto.key,
|
||||
title: dto.title,
|
||||
department: dto.department,
|
||||
operationMode: 'PREPARE',
|
||||
budgetLimitCents: dto.budgetLimitCents,
|
||||
metadata: {
|
||||
workflowScope: 'STAGE_PREPARE',
|
||||
scope: dto.metadata.scope,
|
||||
acceptanceCriteria: [...dto.metadata.acceptanceCriteria],
|
||||
environment: 'staging',
|
||||
commitSha: dto.metadata.commitSha,
|
||||
createdBy: a.id,
|
||||
},
|
||||
state: 'DRAFT',
|
||||
},
|
||||
});
|
||||
await tx.orchestrationEvent.create({
|
||||
data: {
|
||||
taskId: t.id,
|
||||
sequence: 1,
|
||||
eventType: 'PREPARE_PROPOSED',
|
||||
actorId: a.id,
|
||||
toState: 'DRAFT',
|
||||
},
|
||||
});
|
||||
return t;
|
||||
})
|
||||
.catch((error: any) => {
|
||||
if (
|
||||
error?.code === 'P2002' &&
|
||||
Array.isArray(error.meta?.target) &&
|
||||
error.meta.target.includes('key')
|
||||
)
|
||||
throw new ConflictException(
|
||||
'Proposal key already exists; fetch and reconcile the existing task',
|
||||
);
|
||||
throw error;
|
||||
});
|
||||
}
|
||||
async transition(id: string, dto: any, a: WorkflowIdentity) {
|
||||
this.identity(a, ['Admin', 'SUPER_ADMIN']);
|
||||
return this.db.$transaction(async (tx: any) => {
|
||||
await this.allowed(tx, true);
|
||||
const t = await this.task(tx, id);
|
||||
const fingerprint = JSON.stringify({ id, dto, a });
|
||||
if (await this.replay(tx, dto.idempotencyKey, fingerprint)) return t;
|
||||
this.version(t, dto);
|
||||
if (t.metadata.createdBy !== a.id && a.role !== 'SUPER_ADMIN')
|
||||
throw new ForbiddenException('Proposal owner required');
|
||||
if (
|
||||
!(
|
||||
(t.state === 'DRAFT' && dto.toState === 'GATE1_PENDING') ||
|
||||
(t.state === 'GATE1_APPROVED' && dto.toState === 'READY')
|
||||
)
|
||||
)
|
||||
throw new ForbiddenException(
|
||||
'Only submission and approved queueing are available',
|
||||
);
|
||||
if (dto.toState === 'READY') {
|
||||
const approval = await tx.orchestrationApprovalDecision.findFirst({
|
||||
where: { taskId: id, gate: 1, decision: 'APPROVED' },
|
||||
orderBy: { createdAt: 'desc' },
|
||||
});
|
||||
if (!approval || approval.expiresAt <= new Date())
|
||||
throw new ForbiddenException('Gate1 expired');
|
||||
await tx.prepareWorkflowOutbox.create({ data: { taskId: id } });
|
||||
}
|
||||
return this.event(
|
||||
tx,
|
||||
t,
|
||||
dto.toState,
|
||||
a,
|
||||
dto.idempotencyKey,
|
||||
'PREPARE_TRANSITION',
|
||||
fingerprint,
|
||||
);
|
||||
});
|
||||
}
|
||||
async decide(id: string, gate: number, dto: any, a: WorkflowIdentity) {
|
||||
this.identity(a, ['SUPER_ADMIN']);
|
||||
if (gate !== 1)
|
||||
throw new ForbiddenException(
|
||||
'Gate2 decisions and release remain disabled',
|
||||
);
|
||||
if (
|
||||
!['APPROVED', 'REJECTED'].includes(dto.decision) ||
|
||||
!Number.isFinite(Date.parse(dto.expiresAt)) ||
|
||||
Date.parse(dto.expiresAt) <= Date.now() ||
|
||||
Date.parse(dto.expiresAt) > Date.now() + 86400000
|
||||
)
|
||||
throw new BadRequestException(
|
||||
'Decision requires future expiry within 24 hours',
|
||||
);
|
||||
return this.db.$transaction(async (tx: any) => {
|
||||
await this.allowed(tx);
|
||||
const t = await this.task(tx, id);
|
||||
const fingerprint = JSON.stringify({ id, gate, dto, a });
|
||||
if (await this.replay(tx, dto.idempotencyKey, fingerprint)) return t;
|
||||
this.version(t, dto);
|
||||
if (t.metadata.createdBy === a.id)
|
||||
throw new ForbiddenException('Independent reviewer required');
|
||||
if (t.state !== 'GATE1_PENDING')
|
||||
throw new ConflictException('Gate1 pending required');
|
||||
await tx.orchestrationApprovalDecision.create({
|
||||
data: {
|
||||
taskId: id,
|
||||
gate: 1,
|
||||
decision: dto.decision,
|
||||
actorId: a.id,
|
||||
expiresAt: new Date(dto.expiresAt),
|
||||
idempotencyKey: `decision:${dto.idempotencyKey}`,
|
||||
},
|
||||
});
|
||||
return this.event(
|
||||
tx,
|
||||
t,
|
||||
dto.decision === 'APPROVED' ? 'GATE1_APPROVED' : 'REJECTED',
|
||||
a,
|
||||
dto.idempotencyKey,
|
||||
'PREPARE_GATE1',
|
||||
fingerprint,
|
||||
);
|
||||
});
|
||||
}
|
||||
async claim(a: WorkflowIdentity, id?: string): Promise<any> {
|
||||
this.worker(a);
|
||||
return this.db.$transaction(async (tx: any) => {
|
||||
await this.allowed(tx, true);
|
||||
// Lock order is task then outbox for every worker operation. SKIP LOCKED permits competing workers.
|
||||
const rows: any[] = id
|
||||
? await tx.$queryRaw`SELECT t.id FROM orchestration_tasks t JOIN prepare_workflow_outbox o ON o.task_id=t.id WHERE t.id=${id}::uuid AND o.status='PENDING' AND (o.lease_until IS NULL OR o.lease_until < now()) ORDER BY o.created_at FOR UPDATE OF t SKIP LOCKED LIMIT 1`
|
||||
: await tx.$queryRaw`SELECT t.id FROM orchestration_tasks t JOIN prepare_workflow_outbox o ON o.task_id=t.id WHERE o.status='PENDING' AND (o.lease_until IS NULL OR o.lease_until < now()) ORDER BY o.created_at FOR UPDATE OF t SKIP LOCKED LIMIT 1`;
|
||||
if (!rows.length) return null;
|
||||
const t = await this.task(tx, rows[0].id);
|
||||
const o = await tx.prepareWorkflowOutbox.findUnique({
|
||||
where: { taskId: t.id },
|
||||
});
|
||||
const approval = await tx.orchestrationApprovalDecision.findFirst({
|
||||
where: { taskId: t.id, gate: 1, decision: 'APPROVED' },
|
||||
orderBy: { createdAt: 'desc' },
|
||||
});
|
||||
if (!approval || approval.expiresAt <= new Date())
|
||||
throw new ForbiddenException('Gate1 expired');
|
||||
if (o.attempts >= 3) throw new ForbiddenException('Attempts exhausted');
|
||||
const costs = await tx.orchestrationBudgetLedger.aggregate({
|
||||
where: { taskId: t.id },
|
||||
_sum: { amountCents: true },
|
||||
});
|
||||
if ((costs._sum.amountCents ?? 0) + 20 > t.budgetLimitCents)
|
||||
throw new ForbiddenException('Hard task budget exceeded');
|
||||
if (o.runId)
|
||||
await tx.orchestrationRun.update({
|
||||
where: { id: o.runId },
|
||||
data: {
|
||||
status: 'FAILED',
|
||||
retryClass: 'TRANSIENT',
|
||||
finishedAt: new Date(),
|
||||
},
|
||||
});
|
||||
const token = randomUUID(),
|
||||
attempt = o.attempts + 1;
|
||||
const run = await tx.orchestrationRun.create({
|
||||
data: {
|
||||
taskId: t.id,
|
||||
attempt,
|
||||
status: 'RUNNING',
|
||||
environment: 'staging',
|
||||
commitSha: t.metadata.commitSha,
|
||||
idempotencyKey: `prepare-run:${o.id}:${attempt}`,
|
||||
startedAt: new Date(),
|
||||
},
|
||||
});
|
||||
const reservation = await tx.orchestrationBudgetLedger.create({
|
||||
data: {
|
||||
taskId: t.id,
|
||||
amountCents: 20,
|
||||
kind: 'RESERVE',
|
||||
actorId: a.id,
|
||||
idempotencyKey: `prepare-reserve:${run.id}`,
|
||||
},
|
||||
});
|
||||
const leaseUntil = new Date(Date.now() + 30000);
|
||||
await tx.prepareWorkflowOutbox.update({
|
||||
where: { id: o.id },
|
||||
data: {
|
||||
attempts: attempt,
|
||||
leaseOwner: a.id,
|
||||
leaseToken: token,
|
||||
leaseUntil,
|
||||
heartbeatAt: new Date(),
|
||||
runId: run.id,
|
||||
reservationId: reservation.id,
|
||||
},
|
||||
});
|
||||
await this.event(
|
||||
tx,
|
||||
t,
|
||||
'RUNNING',
|
||||
a,
|
||||
`prepare-claim:${run.id}`,
|
||||
'PREPARE_CLAIMED',
|
||||
token,
|
||||
);
|
||||
return {
|
||||
taskId: t.id,
|
||||
runId: run.id,
|
||||
attempt,
|
||||
leaseToken: token,
|
||||
leaseUntil,
|
||||
commitSha: run.commitSha,
|
||||
environment: 'staging',
|
||||
maxCostCents: 20,
|
||||
};
|
||||
});
|
||||
}
|
||||
private async lease(tx: any, a: WorkflowIdentity, dto: any) {
|
||||
const t = await this.task(tx, dto.taskId);
|
||||
const o = await tx.prepareWorkflowOutbox.findUnique({
|
||||
where: { taskId: t.id },
|
||||
});
|
||||
if (
|
||||
!o ||
|
||||
o.status !== 'PENDING' ||
|
||||
o.leaseOwner !== a.id ||
|
||||
o.leaseToken !== dto.leaseToken ||
|
||||
o.runId !== dto.runId ||
|
||||
!o.leaseUntil ||
|
||||
o.leaseUntil <= new Date() ||
|
||||
t.state !== 'RUNNING'
|
||||
)
|
||||
throw new ForbiddenException('Active owned lease required');
|
||||
return { t, o };
|
||||
}
|
||||
async heartbeat(a: WorkflowIdentity, dto: any) {
|
||||
this.worker(a);
|
||||
return this.db.$transaction(async (tx: any) => {
|
||||
await this.allowed(tx);
|
||||
const { o } = await this.lease(tx, a, dto);
|
||||
return tx.prepareWorkflowOutbox.update({
|
||||
where: { id: o.id },
|
||||
data: {
|
||||
heartbeatAt: new Date(),
|
||||
leaseUntil: new Date(Date.now() + 30000),
|
||||
},
|
||||
select: { leaseUntil: true },
|
||||
});
|
||||
});
|
||||
}
|
||||
async complete(a: WorkflowIdentity, dto: any) {
|
||||
this.worker(a);
|
||||
if (
|
||||
!/^[a-f0-9]{64}$/.test(dto.sha256 ?? '') ||
|
||||
!/^https:\/\//.test(dto.uri ?? '') ||
|
||||
!Number.isSafeInteger(dto.actualCents) ||
|
||||
dto.actualCents < 0 ||
|
||||
dto.actualCents > 20
|
||||
)
|
||||
throw new BadRequestException(
|
||||
'QA hash, HTTPS URI and cost 0..20 required',
|
||||
);
|
||||
return this.db.$transaction(async (tx: any) => {
|
||||
await this.allowed(tx);
|
||||
const { t, o } = await this.lease(tx, a, dto);
|
||||
await tx.orchestrationArtifact.create({
|
||||
data: { runId: o.runId, kind: 'QA', sha256: dto.sha256, uri: dto.uri },
|
||||
});
|
||||
await tx.orchestrationAttestation.create({
|
||||
data: {
|
||||
runId: o.runId,
|
||||
kind: 'QA',
|
||||
subjectHash: dto.sha256,
|
||||
signer: a.id,
|
||||
predicate: { passed: true, scope: 'STAGE_PREPARE' },
|
||||
},
|
||||
});
|
||||
await tx.orchestrationRun.update({
|
||||
where: { id: o.runId },
|
||||
data: { status: 'SUCCEEDED', finishedAt: new Date() },
|
||||
});
|
||||
await tx.orchestrationBudgetLedger.create({
|
||||
data: {
|
||||
taskId: t.id,
|
||||
kind: 'RECONCILE',
|
||||
amountCents: dto.actualCents - 20,
|
||||
actorId: a.id,
|
||||
idempotencyKey: `prepare-cost:${o.runId}`,
|
||||
metadata: {
|
||||
reservationId: o.reservationId,
|
||||
actualCents: dto.actualCents,
|
||||
},
|
||||
},
|
||||
});
|
||||
await tx.prepareWorkflowOutbox.update({
|
||||
where: { id: o.id },
|
||||
data: { status: 'DONE', leaseToken: null, leaseUntil: null },
|
||||
});
|
||||
return this.event(
|
||||
tx,
|
||||
t,
|
||||
'VERIFYING',
|
||||
a,
|
||||
`prepare-qa:${o.runId}`,
|
||||
'PREPARE_QA_SUCCEEDED',
|
||||
dto.sha256,
|
||||
);
|
||||
});
|
||||
}
|
||||
async stageEvidence(a: WorkflowIdentity, dto: any) {
|
||||
this.identity(a, ['STAGE_CI']);
|
||||
if (!this.config.stageCiIds.includes(a.id))
|
||||
throw new ForbiddenException('Stage CI identity not provisioned');
|
||||
if (
|
||||
!/^[a-f0-9]{64}$/.test(dto.evidenceSha256 ?? '') ||
|
||||
!/^https:\/\//.test(dto.uri ?? '')
|
||||
)
|
||||
throw new BadRequestException(
|
||||
'Stage evidence hash and HTTPS URI required',
|
||||
);
|
||||
return this.db.$transaction(async (tx: any) => {
|
||||
await this.allowed(tx);
|
||||
const t = await this.task(tx, dto.taskId);
|
||||
const run = await tx.orchestrationRun.findUnique({
|
||||
where: { id: dto.runId },
|
||||
});
|
||||
if (
|
||||
t.state !== 'VERIFYING' ||
|
||||
!run ||
|
||||
run.taskId !== t.id ||
|
||||
run.environment !== 'staging' ||
|
||||
run.status !== 'SUCCEEDED' ||
|
||||
run.commitSha !== dto.commitSha ||
|
||||
t.metadata.commitSha !== dto.commitSha
|
||||
)
|
||||
throw new ForbiddenException(
|
||||
'Immutable successful Stage run binding required',
|
||||
);
|
||||
const qa = await tx.orchestrationAttestation.findFirst({
|
||||
where: { runId: run.id, kind: 'QA', subjectHash: dto.sha256 },
|
||||
});
|
||||
if (!qa || qa.signer === a.id)
|
||||
throw new ForbiddenException('Independent QA evidence required');
|
||||
await tx.orchestrationArtifact.create({
|
||||
data: {
|
||||
runId: run.id,
|
||||
kind: 'STAGE_EVIDENCE',
|
||||
sha256: dto.evidenceSha256,
|
||||
uri: dto.uri,
|
||||
},
|
||||
});
|
||||
await tx.orchestrationAttestation.create({
|
||||
data: {
|
||||
runId: run.id,
|
||||
kind: 'STAGE',
|
||||
subjectHash: dto.sha256,
|
||||
signer: a.id,
|
||||
predicate: {
|
||||
evidenceSha256: dto.evidenceSha256,
|
||||
commitSha: dto.commitSha,
|
||||
environment: 'staging',
|
||||
},
|
||||
},
|
||||
});
|
||||
return this.event(
|
||||
tx,
|
||||
t,
|
||||
'GATE2_PENDING',
|
||||
a,
|
||||
`prepare-stage:${run.id}`,
|
||||
'STAGE_EVIDENCE_RECORDED',
|
||||
dto.evidenceSha256,
|
||||
);
|
||||
});
|
||||
}
|
||||
project(t: any, a: WorkflowIdentity) {
|
||||
const safe =
|
||||
t.operationMode === 'PREPARE' &&
|
||||
(t.metadata?.workflowScope ?? t.metadata?.scope) === 'STAGE_PREPARE';
|
||||
const owner =
|
||||
a?.id && (t.metadata?.createdBy === a.id || a.role === 'SUPER_ADMIN');
|
||||
const actions: string[] = [];
|
||||
if (safe && owner && t.state === 'DRAFT') actions.push('SUBMIT_GATE1');
|
||||
if (safe && owner && t.state === 'GATE1_APPROVED')
|
||||
actions.push('QUEUE_PREPARE');
|
||||
if (
|
||||
safe &&
|
||||
a?.role === 'SUPER_ADMIN' &&
|
||||
t.metadata?.createdBy !== a.id &&
|
||||
t.state === 'GATE1_PENDING'
|
||||
)
|
||||
actions.push('APPROVE_GATE1', 'REJECT_GATE1');
|
||||
return {
|
||||
...t,
|
||||
metadata: safe
|
||||
? {
|
||||
scope: t.metadata.scope,
|
||||
...(t.metadata.workflowScope === 'STAGE_PREPARE'
|
||||
? { acceptanceCriteria: t.metadata.acceptanceCriteria }
|
||||
: {}),
|
||||
environment: 'staging',
|
||||
commitSha: t.metadata.commitSha,
|
||||
}
|
||||
: {},
|
||||
availableActions: actions,
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,513 @@
|
||||
import { Test } from '@nestjs/testing';
|
||||
import request from 'supertest';
|
||||
import { OrchestrationController } from './orchestration.controller';
|
||||
import { OrchestrationService } from './orchestration.service';
|
||||
import { JwtAuthGuard } from '../auth/jwt-auth.guard';
|
||||
import { RolesGuard } from '../common/guards/roles.guard';
|
||||
import { PrismaClient } from '@prisma/client';
|
||||
import { PrepareWorkflowService } from './prepare-workflow.service';
|
||||
import { OperationMode, WorkflowState } from './orchestration.domain';
|
||||
|
||||
const enabled = process.env.WORKFLOW_DB === '1';
|
||||
(enabled ? describe : describe.skip)('isolated prepare workflow', () => {
|
||||
const db = new PrismaClient();
|
||||
const admin = { id: 'proposer', role: 'Admin' };
|
||||
const reviewer = { id: 'reviewer', role: 'SUPER_ADMIN' };
|
||||
const worker = { id: 'worker-1', role: 'WORKFLOW_WORKER' };
|
||||
const ci = { id: 'isolated-ci', role: 'STAGE_CI' };
|
||||
const service = new PrepareWorkflowService(db as never, {
|
||||
workerIds: ['worker-1', 'worker-2'],
|
||||
stageCiIds: ['isolated-ci'],
|
||||
});
|
||||
let seq = 0;
|
||||
const dto = () => ({
|
||||
key: `wf-${Date.now()}-${seq++}`,
|
||||
title: 'Bounded isolated QA tracer',
|
||||
department: 'ENGINEERING',
|
||||
operationMode: OperationMode.PREPARE,
|
||||
budgetLimitCents: 20,
|
||||
metadata: {
|
||||
commitSha: 'a'.repeat(40),
|
||||
token: 'must-not-persist',
|
||||
scope: 'Staging schema audit only',
|
||||
acceptanceCriteria: ['Report all schema failures'],
|
||||
actorId: 'spoof',
|
||||
},
|
||||
});
|
||||
async function ready() {
|
||||
let t = await service.create(dto(), admin);
|
||||
t = await service.transition(
|
||||
t.id,
|
||||
{
|
||||
toState: WorkflowState.GATE1_PENDING,
|
||||
expectedVersion: t.version,
|
||||
idempotencyKey: `submit-${t.id}`,
|
||||
},
|
||||
admin,
|
||||
);
|
||||
t = await service.decide(
|
||||
t.id,
|
||||
1,
|
||||
{
|
||||
decision: 'APPROVED',
|
||||
expectedVersion: t.version,
|
||||
idempotencyKey: `approve-${t.id}`,
|
||||
expiresAt: new Date(Date.now() + 60000).toISOString(),
|
||||
},
|
||||
reviewer,
|
||||
);
|
||||
return service.transition(
|
||||
t.id,
|
||||
{
|
||||
toState: WorkflowState.READY,
|
||||
expectedVersion: t.version,
|
||||
idempotencyKey: `ready-${t.id}`,
|
||||
},
|
||||
admin,
|
||||
);
|
||||
}
|
||||
afterAll(async () => {
|
||||
await db.$disconnect();
|
||||
});
|
||||
it('persists and projects immutable bounded proposal scope without arbitrary metadata', async () => {
|
||||
const input = dto();
|
||||
const t = await service.create(input, admin);
|
||||
expect(t.metadata).toMatchObject({
|
||||
workflowScope: 'STAGE_PREPARE',
|
||||
scope: input.metadata.scope,
|
||||
acceptanceCriteria: input.metadata.acceptanceCriteria,
|
||||
commitSha: input.metadata.commitSha,
|
||||
});
|
||||
expect(t.metadata).not.toHaveProperty('token');
|
||||
expect(t.metadata).not.toHaveProperty('actorId');
|
||||
const submitted = await service.transition(
|
||||
t.id,
|
||||
{
|
||||
toState: 'GATE1_PENDING',
|
||||
expectedVersion: t.version,
|
||||
idempotencyKey: `immutable-${t.id}`,
|
||||
},
|
||||
admin,
|
||||
);
|
||||
expect(submitted.metadata).toEqual(t.metadata);
|
||||
expect(service.project(submitted, reviewer).metadata).toEqual({
|
||||
scope: input.metadata.scope,
|
||||
acceptanceCriteria: input.metadata.acceptanceCriteria,
|
||||
environment: 'staging',
|
||||
commitSha: input.metadata.commitSha,
|
||||
});
|
||||
});
|
||||
it.each([
|
||||
null,
|
||||
[],
|
||||
{},
|
||||
{ commitSha: 'a'.repeat(40), scope: '', acceptanceCriteria: ['ok'] },
|
||||
{
|
||||
commitSha: 'a'.repeat(40),
|
||||
scope: ' '.repeat(10),
|
||||
acceptanceCriteria: ['ok'],
|
||||
},
|
||||
{
|
||||
commitSha: 'a'.repeat(40),
|
||||
scope: 'x'.repeat(2001),
|
||||
acceptanceCriteria: ['ok'],
|
||||
},
|
||||
{ commitSha: 'a'.repeat(40), scope: 'ok', acceptanceCriteria: [] },
|
||||
{
|
||||
commitSha: 'a'.repeat(40),
|
||||
scope: 'ok',
|
||||
acceptanceCriteria: Array(21).fill('ok'),
|
||||
},
|
||||
{
|
||||
commitSha: 'a'.repeat(40),
|
||||
scope: 'ok',
|
||||
acceptanceCriteria: ['x'.repeat(501)],
|
||||
},
|
||||
{ commitSha: 'a'.repeat(40), scope: 'ok', acceptanceCriteria: [' '] },
|
||||
{ commitSha: 'a'.repeat(40), scope: 'ok', acceptanceCriteria: [123] },
|
||||
{ commitSha: ['a'.repeat(40)], scope: 'ok', acceptanceCriteria: ['ok'] },
|
||||
{ scope: 'ok', acceptanceCriteria: ['ok'] },
|
||||
])(
|
||||
'rejects unsafe or unbounded metadata %# without persisting',
|
||||
async (metadata) => {
|
||||
const input = { ...dto(), metadata };
|
||||
await expect(service.create(input, admin)).rejects.toMatchObject({
|
||||
status: 400,
|
||||
});
|
||||
expect(
|
||||
await db.orchestrationTask.count({ where: { key: input.key } }),
|
||||
).toBe(0);
|
||||
},
|
||||
);
|
||||
it('accepts exact metadata bounds', async () => {
|
||||
const metadata = {
|
||||
commitSha: 'a'.repeat(40),
|
||||
scope: 'x'.repeat(2000),
|
||||
acceptanceCriteria: Array(20).fill('x'.repeat(500)),
|
||||
};
|
||||
expect(
|
||||
(await service.create({ ...dto(), metadata }, admin)).metadata,
|
||||
).toMatchObject(metadata);
|
||||
});
|
||||
it('normalizes racing duplicate proposal keys to conflict without duplicate events', async () => {
|
||||
const input = dto();
|
||||
const results = await Promise.allSettled([
|
||||
service.create(input, admin),
|
||||
service.create({ ...input, title: 'Different intent' }, reviewer),
|
||||
]);
|
||||
expect(results.filter((r) => r.status === 'fulfilled')).toHaveLength(1);
|
||||
const rejected = results.find(
|
||||
(r) => r.status === 'rejected',
|
||||
) as PromiseRejectedResult;
|
||||
expect(rejected.reason).toMatchObject({ status: 409 });
|
||||
const persisted = await db.orchestrationTask.findUniqueOrThrow({
|
||||
where: { key: input.key },
|
||||
});
|
||||
expect(
|
||||
await db.orchestrationEvent.count({ where: { taskId: persisted.id } }),
|
||||
).toBe(1);
|
||||
await expect(service.create(input, admin)).rejects.toMatchObject({
|
||||
status: 409,
|
||||
});
|
||||
});
|
||||
it('enforces create and read contracts over HTTP with real PostgreSQL persistence', async () => {
|
||||
const module = await Test.createTestingModule({
|
||||
controllers: [OrchestrationController],
|
||||
providers: [
|
||||
{ provide: PrepareWorkflowService, useValue: service },
|
||||
{
|
||||
provide: OrchestrationService,
|
||||
useValue: {
|
||||
findAll: () => db.orchestrationTask.findMany(),
|
||||
findOne: (id: string) =>
|
||||
db.orchestrationTask.findUniqueOrThrow({ where: { id } }),
|
||||
},
|
||||
},
|
||||
],
|
||||
})
|
||||
.overrideGuard(JwtAuthGuard)
|
||||
.useValue({
|
||||
canActivate(context: any) {
|
||||
context.switchToHttp().getRequest().user = {
|
||||
sub: admin.id,
|
||||
role: admin.role,
|
||||
};
|
||||
return true;
|
||||
},
|
||||
})
|
||||
.overrideGuard(RolesGuard)
|
||||
.useValue({ canActivate: () => true })
|
||||
.compile();
|
||||
const app = module.createNestApplication();
|
||||
await app.init();
|
||||
try {
|
||||
const input = dto();
|
||||
const response = await request(app.getHttpServer())
|
||||
.post('/admin/orchestration/tasks')
|
||||
.send(input)
|
||||
.expect(201);
|
||||
expect(response.body.metadata).toEqual({
|
||||
scope: input.metadata.scope,
|
||||
acceptanceCriteria: input.metadata.acceptanceCriteria,
|
||||
commitSha: input.metadata.commitSha,
|
||||
environment: 'staging',
|
||||
});
|
||||
expect(response.body.availableActions).toEqual(['SUBMIT_GATE1']);
|
||||
const detail = await request(app.getHttpServer())
|
||||
.get(`/admin/orchestration/tasks/${response.body.id}`)
|
||||
.expect(200);
|
||||
expect(detail.body.metadata).toEqual(response.body.metadata);
|
||||
const list = await request(app.getHttpServer())
|
||||
.get('/admin/orchestration/tasks')
|
||||
.expect(200);
|
||||
expect(
|
||||
list.body.find((row: any) => row.id === response.body.id).metadata,
|
||||
).toEqual(response.body.metadata);
|
||||
expect(JSON.stringify(response.body)).not.toMatch(
|
||||
/createdBy|must-not-persist|actorId|workflowScope/,
|
||||
);
|
||||
await request(app.getHttpServer())
|
||||
.post('/admin/orchestration/tasks')
|
||||
.send(input)
|
||||
.expect(409);
|
||||
await request(app.getHttpServer())
|
||||
.post('/admin/orchestration/tasks')
|
||||
.send({
|
||||
...input,
|
||||
key: `${input.key}-invalid`,
|
||||
metadata: { scope: 'only stage', acceptanceCriteria: ['QA passes'] },
|
||||
})
|
||||
.expect(400);
|
||||
await request(app.getHttpServer())
|
||||
.post('/admin/orchestration/tasks')
|
||||
.send({
|
||||
...input,
|
||||
key: `${input.key}-invalid`,
|
||||
metadata: { ...input.metadata, acceptanceCriteria: [' '] },
|
||||
})
|
||||
.expect(400);
|
||||
} finally {
|
||||
await app.close();
|
||||
}
|
||||
});
|
||||
it('fences heartbeat and completes QA plus independent Stage evidence only to Gate2 pending', async () => {
|
||||
const t = await ready();
|
||||
const c = await service.claim(worker, t.id);
|
||||
await expect(
|
||||
service.heartbeat({ ...worker, id: 'worker-2' }, c),
|
||||
).rejects.toThrow(/lease/i);
|
||||
await service.heartbeat(worker, c);
|
||||
const qa = {
|
||||
...c,
|
||||
sha256: 'b'.repeat(64),
|
||||
uri: 'https://ci.example/qa',
|
||||
actualCents: 8,
|
||||
};
|
||||
await service.complete(worker, qa);
|
||||
await expect(
|
||||
service.stageEvidence(
|
||||
{ id: 'proposer', role: 'Admin' },
|
||||
{
|
||||
taskId: t.id,
|
||||
runId: c.runId,
|
||||
sha256: qa.sha256,
|
||||
commitSha: c.commitSha,
|
||||
evidenceSha256: 'c'.repeat(64),
|
||||
uri: 'https://ci.example/stage',
|
||||
},
|
||||
),
|
||||
).rejects.toThrow(/identity/i);
|
||||
await service.stageEvidence(ci, {
|
||||
taskId: t.id,
|
||||
runId: c.runId,
|
||||
sha256: qa.sha256,
|
||||
commitSha: c.commitSha,
|
||||
evidenceSha256: 'c'.repeat(64),
|
||||
uri: 'https://ci.example/stage',
|
||||
});
|
||||
expect(
|
||||
await db.orchestrationTask.findUnique({ where: { id: t.id } }),
|
||||
).toMatchObject({ state: 'GATE2_PENDING' });
|
||||
expect(
|
||||
await db.orchestrationBudgetLedger.aggregate({
|
||||
where: { taskId: t.id },
|
||||
_sum: { amountCents: true },
|
||||
}),
|
||||
).toMatchObject({ _sum: { amountCents: 8 } });
|
||||
await expect(service.complete(worker, qa)).rejects.toThrow(/lease|active/i);
|
||||
});
|
||||
it('bounds expired-lease retries and rejects stale completion without releasing unknown costs', async () => {
|
||||
const t = await service.create({ ...dto(), budgetLimitCents: 60 }, admin);
|
||||
let current = await service.transition(
|
||||
t.id,
|
||||
{
|
||||
toState: 'GATE1_PENDING',
|
||||
expectedVersion: 1,
|
||||
idempotencyKey: `s-${t.id}`,
|
||||
},
|
||||
admin,
|
||||
);
|
||||
current = await service.decide(
|
||||
t.id,
|
||||
1,
|
||||
{
|
||||
decision: 'APPROVED',
|
||||
expectedVersion: current.version,
|
||||
idempotencyKey: `a-${t.id}`,
|
||||
expiresAt: new Date(Date.now() + 60000).toISOString(),
|
||||
},
|
||||
reviewer,
|
||||
);
|
||||
await service.transition(
|
||||
t.id,
|
||||
{
|
||||
toState: 'READY',
|
||||
expectedVersion: current.version,
|
||||
idempotencyKey: `q-${t.id}`,
|
||||
},
|
||||
admin,
|
||||
);
|
||||
const first = await service.claim(worker, t.id);
|
||||
for (let i = 0; i < 2; i++) {
|
||||
await db.prepareWorkflowOutbox.update({
|
||||
where: { taskId: t.id },
|
||||
data: { leaseUntil: new Date(0) },
|
||||
});
|
||||
const next = await service.claim(worker, t.id);
|
||||
expect(next.attempt).toBe(i + 2);
|
||||
}
|
||||
await expect(
|
||||
service.complete(worker, {
|
||||
...first,
|
||||
sha256: 'b'.repeat(64),
|
||||
uri: 'https://ci.example/qa',
|
||||
actualCents: 0,
|
||||
}),
|
||||
).rejects.toThrow(/lease/i);
|
||||
await db.prepareWorkflowOutbox.update({
|
||||
where: { taskId: t.id },
|
||||
data: { leaseUntil: new Date(0) },
|
||||
});
|
||||
await expect(service.claim(worker, t.id)).rejects.toThrow(/exhausted/i);
|
||||
expect(await db.orchestrationRun.count({ where: { taskId: t.id } })).toBe(
|
||||
3,
|
||||
);
|
||||
expect(
|
||||
await db.orchestrationBudgetLedger.aggregate({
|
||||
where: { taskId: t.id },
|
||||
_sum: { amountCents: true },
|
||||
}),
|
||||
).toMatchObject({ _sum: { amountCents: 60 } });
|
||||
});
|
||||
it('rejects self approval, non-PREPARE and unprovisioned workers', async () => {
|
||||
await expect(
|
||||
service.create(
|
||||
{ ...dto(), operationMode: OperationMode.MUTATE_PRODUCTION },
|
||||
admin,
|
||||
),
|
||||
).rejects.toThrow(/PREPARE/);
|
||||
let t = await service.create(dto(), reviewer);
|
||||
t = await service.transition(
|
||||
t.id,
|
||||
{
|
||||
toState: 'GATE1_PENDING',
|
||||
expectedVersion: 1,
|
||||
idempotencyKey: `s-${t.id}`,
|
||||
},
|
||||
reviewer,
|
||||
);
|
||||
await expect(
|
||||
service.decide(
|
||||
t.id,
|
||||
1,
|
||||
{
|
||||
decision: 'APPROVED',
|
||||
expectedVersion: t.version,
|
||||
idempotencyKey: `a-${t.id}`,
|
||||
expiresAt: new Date(Date.now() + 60000).toISOString(),
|
||||
},
|
||||
reviewer,
|
||||
),
|
||||
).rejects.toThrow(/independent/i);
|
||||
await expect(
|
||||
service.claim({ id: 'spoof', role: 'WORKFLOW_WORKER' }),
|
||||
).rejects.toThrow(/provisioned/i);
|
||||
await expect(service.decide(t.id, 2, {}, reviewer)).rejects.toThrow(
|
||||
/disabled/i,
|
||||
);
|
||||
});
|
||||
it('serializes duplicate queue submission and denies claims beyond remaining task budget', async () => {
|
||||
const t = await ready();
|
||||
await db.orchestrationBudgetLedger.create({
|
||||
data: {
|
||||
taskId: t.id,
|
||||
amountCents: 1,
|
||||
kind: 'SPEND',
|
||||
actorId: 'fixture',
|
||||
idempotencyKey: `spend-${t.id}`,
|
||||
},
|
||||
});
|
||||
await expect(service.claim(worker, t.id)).rejects.toThrow(/budget/i);
|
||||
expect(await db.orchestrationRun.count({ where: { taskId: t.id } })).toBe(
|
||||
0,
|
||||
);
|
||||
expect(
|
||||
await db.prepareWorkflowOutbox.count({ where: { taskId: t.id } }),
|
||||
).toBe(1);
|
||||
});
|
||||
it('replays submission safely and serializes racing queue requests', async () => {
|
||||
let t = await service.create(dto(), admin);
|
||||
const submit = {
|
||||
toState: 'GATE1_PENDING',
|
||||
expectedVersion: t.version,
|
||||
idempotencyKey: `replay-${t.id}`,
|
||||
};
|
||||
t = await service.transition(t.id, submit, admin);
|
||||
expect((await service.transition(t.id, submit, admin)).version).toBe(
|
||||
t.version,
|
||||
);
|
||||
await expect(
|
||||
service.transition(t.id, { ...submit, toState: 'READY' }, admin),
|
||||
).rejects.toThrow(/mismatch/i);
|
||||
t = await service.decide(
|
||||
t.id,
|
||||
1,
|
||||
{
|
||||
decision: 'APPROVED',
|
||||
expectedVersion: t.version,
|
||||
idempotencyKey: `r-approve-${t.id}`,
|
||||
expiresAt: new Date(Date.now() + 60000).toISOString(),
|
||||
},
|
||||
reviewer,
|
||||
);
|
||||
const results = await Promise.allSettled(
|
||||
['one', 'two'].map((k) =>
|
||||
service.transition(
|
||||
t.id,
|
||||
{
|
||||
toState: 'READY',
|
||||
expectedVersion: t.version,
|
||||
idempotencyKey: `${k}-${t.id}`,
|
||||
},
|
||||
admin,
|
||||
),
|
||||
),
|
||||
);
|
||||
expect(results.filter((r) => r.status === 'fulfilled')).toHaveLength(1);
|
||||
expect(
|
||||
await db.prepareWorkflowOutbox.count({ where: { taskId: t.id } }),
|
||||
).toBe(1);
|
||||
});
|
||||
it('kill switch freezes claims without creating run or reservation', async () => {
|
||||
const t = await ready();
|
||||
await db.orchestrationKillSwitch.create({
|
||||
data: {
|
||||
scope: 'GLOBAL',
|
||||
level: 'FREEZE_NEW_WORK',
|
||||
reason: 'isolated test',
|
||||
actorId: 'fixture',
|
||||
},
|
||||
});
|
||||
try {
|
||||
await expect(service.claim(worker, t.id)).rejects.toThrow(/kill switch/i);
|
||||
} finally {
|
||||
await db.orchestrationKillSwitch.delete({ where: { scope: 'GLOBAL' } });
|
||||
}
|
||||
expect(await db.orchestrationRun.count({ where: { taskId: t.id } })).toBe(
|
||||
0,
|
||||
);
|
||||
});
|
||||
it('persists an atomic outbox and exactly one worker wins concurrent claims', async () => {
|
||||
const t = await ready();
|
||||
const claims = await Promise.all([
|
||||
service.claim(worker, t.id),
|
||||
service.claim({ id: 'worker-2', role: 'WORKFLOW_WORKER' }, t.id),
|
||||
]);
|
||||
expect(claims.filter(Boolean)).toHaveLength(1);
|
||||
const claim = claims.find(Boolean)!;
|
||||
expect(claim.taskId).toBe(t.id);
|
||||
expect(claim.attempt).toBe(1);
|
||||
expect(await db.orchestrationRun.count({ where: { taskId: t.id } })).toBe(
|
||||
1,
|
||||
);
|
||||
expect(
|
||||
await db.orchestrationBudgetLedger.aggregate({
|
||||
where: { taskId: t.id },
|
||||
_sum: { amountCents: true },
|
||||
}),
|
||||
).toMatchObject({ _sum: { amountCents: 20 } });
|
||||
expect(
|
||||
await service.project(
|
||||
await db.orchestrationTask.findUniqueOrThrow({ where: { id: t.id } }),
|
||||
admin,
|
||||
),
|
||||
).toMatchObject({
|
||||
metadata: {
|
||||
scope: 'Staging schema audit only',
|
||||
acceptanceCriteria: ['Report all schema failures'],
|
||||
environment: 'staging',
|
||||
commitSha: 'a'.repeat(40),
|
||||
},
|
||||
});
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user