536 lines
18 KiB
TypeScript
536 lines
18 KiB
TypeScript
import { Test } from '@nestjs/testing';
|
|
import request from 'supertest';
|
|
import { OrchestrationController } from './orchestration.controller';
|
|
import { OrchestrationService } from './orchestration.service';
|
|
import { JwtAuthGuard } from '../auth/jwt-auth.guard';
|
|
import { RolesGuard } from '../common/guards/roles.guard';
|
|
import { PrismaClient } from '@prisma/client';
|
|
import { PrepareWorkflowService } from './prepare-workflow.service';
|
|
import { OperationMode, WorkflowState } from './orchestration.domain';
|
|
|
|
const enabled = process.env.WORKFLOW_DB === '1';
|
|
(enabled ? describe : describe.skip)('isolated prepare workflow', () => {
|
|
const db = new PrismaClient();
|
|
const admin = { id: 'proposer', role: 'Admin' };
|
|
const reviewer = { id: 'reviewer', role: 'SUPER_ADMIN' };
|
|
const worker = { id: 'worker-1', role: 'WORKFLOW_WORKER' };
|
|
const ci = { id: 'isolated-ci', role: 'STAGE_CI' };
|
|
const service = new PrepareWorkflowService(db as never, {
|
|
workerIds: ['worker-1', 'worker-2'],
|
|
stageCiIds: ['isolated-ci'],
|
|
});
|
|
let seq = 0;
|
|
const dto = () => ({
|
|
key: `wf-${Date.now()}-${seq++}`,
|
|
title: 'Bounded isolated QA tracer',
|
|
department: 'ENGINEERING',
|
|
operationMode: OperationMode.PREPARE,
|
|
budgetLimitCents: 20,
|
|
metadata: {
|
|
commitSha: 'a'.repeat(40),
|
|
token: 'must-not-persist',
|
|
scope: 'Staging schema audit only',
|
|
acceptanceCriteria: ['Report all schema failures'],
|
|
actorId: 'spoof',
|
|
},
|
|
});
|
|
async function ready() {
|
|
let t = await service.create(dto(), admin);
|
|
t = await service.transition(
|
|
t.id,
|
|
{
|
|
toState: WorkflowState.GATE1_PENDING,
|
|
expectedVersion: t.version,
|
|
idempotencyKey: `submit-${t.id}`,
|
|
},
|
|
admin,
|
|
);
|
|
t = await service.decide(
|
|
t.id,
|
|
1,
|
|
{
|
|
decision: 'APPROVED',
|
|
expectedVersion: t.version,
|
|
idempotencyKey: `approve-${t.id}`,
|
|
expiresAt: new Date(Date.now() + 60000).toISOString(),
|
|
},
|
|
reviewer,
|
|
);
|
|
return service.transition(
|
|
t.id,
|
|
{
|
|
toState: WorkflowState.READY,
|
|
expectedVersion: t.version,
|
|
idempotencyKey: `ready-${t.id}`,
|
|
},
|
|
admin,
|
|
);
|
|
}
|
|
afterAll(async () => {
|
|
await db.$disconnect();
|
|
});
|
|
it('serializes competing owner holds and preserves audited reason, then decides held task', async () => {
|
|
let t = await service.create(dto(), admin);
|
|
t = await service.transition(t.id, {toState:'GATE1_PENDING',expectedVersion:t.version,idempotencyKey:`submit-hold-${t.id}`}, admin);
|
|
const command = {toState:'GATE1_HELD',expectedVersion:t.version,idempotencyKey:`hold-${t.id}`,reason:'Need source review'};
|
|
const results = await Promise.allSettled([
|
|
service.transition(t.id, command, reviewer),
|
|
service.transition(t.id, {...command,idempotencyKey:`competing-${t.id}`}, reviewer),
|
|
]);
|
|
expect(results.filter(r=>r.status==='fulfilled')).toHaveLength(1);
|
|
const held = await db.orchestrationTask.findUniqueOrThrow({where:{id:t.id}});
|
|
expect(held.state).toBe('GATE1_HELD');
|
|
expect(held.version).toBe(t.version+1);
|
|
const events = await db.orchestrationEvent.findMany({where:{taskId:t.id,eventType:'PREPARE_GATE1_HELD'}});
|
|
expect(events).toHaveLength(1);
|
|
expect(events[0].actorId).toBe(reviewer.id);
|
|
expect(JSON.stringify(events[0].payload)).toContain('Need source review');
|
|
expect(await db.orchestrationApprovalDecision.count({where:{taskId:t.id}})).toBe(0);
|
|
expect(await db.prepareWorkflowOutbox.count({where:{taskId:t.id}})).toBe(0);
|
|
await expect(service.transition(t.id, {...command,expectedVersion:held.version,reason:'changed'},reviewer)).rejects.toMatchObject({status:409});
|
|
const decision = await service.decide(t.id,1,{decision:'REJECTED',expectedVersion:held.version,idempotencyKey:`decline-held-${t.id}`,expiresAt:new Date(Date.now()+60000).toISOString()},reviewer);
|
|
expect(decision.state).toBe('REJECTED');
|
|
});
|
|
it('persists and projects immutable bounded proposal scope without arbitrary metadata', async () => {
|
|
const input = dto();
|
|
const t = await service.create(input, admin);
|
|
expect(t.metadata).toMatchObject({
|
|
workflowScope: 'STAGE_PREPARE',
|
|
scope: input.metadata.scope,
|
|
acceptanceCriteria: input.metadata.acceptanceCriteria,
|
|
commitSha: input.metadata.commitSha,
|
|
});
|
|
expect(t.metadata).not.toHaveProperty('token');
|
|
expect(t.metadata).not.toHaveProperty('actorId');
|
|
const submitted = await service.transition(
|
|
t.id,
|
|
{
|
|
toState: 'GATE1_PENDING',
|
|
expectedVersion: t.version,
|
|
idempotencyKey: `immutable-${t.id}`,
|
|
},
|
|
admin,
|
|
);
|
|
expect(submitted.metadata).toEqual(t.metadata);
|
|
expect(service.project(submitted, reviewer).metadata).toEqual({
|
|
scope: input.metadata.scope,
|
|
acceptanceCriteria: input.metadata.acceptanceCriteria,
|
|
environment: 'staging',
|
|
commitSha: input.metadata.commitSha,
|
|
});
|
|
});
|
|
it.each([
|
|
null,
|
|
[],
|
|
{},
|
|
{ commitSha: 'a'.repeat(40), scope: '', acceptanceCriteria: ['ok'] },
|
|
{
|
|
commitSha: 'a'.repeat(40),
|
|
scope: ' '.repeat(10),
|
|
acceptanceCriteria: ['ok'],
|
|
},
|
|
{
|
|
commitSha: 'a'.repeat(40),
|
|
scope: 'x'.repeat(2001),
|
|
acceptanceCriteria: ['ok'],
|
|
},
|
|
{ commitSha: 'a'.repeat(40), scope: 'ok', acceptanceCriteria: [] },
|
|
{
|
|
commitSha: 'a'.repeat(40),
|
|
scope: 'ok',
|
|
acceptanceCriteria: Array(21).fill('ok'),
|
|
},
|
|
{
|
|
commitSha: 'a'.repeat(40),
|
|
scope: 'ok',
|
|
acceptanceCriteria: ['x'.repeat(501)],
|
|
},
|
|
{ commitSha: 'a'.repeat(40), scope: 'ok', acceptanceCriteria: [' '] },
|
|
{ commitSha: 'a'.repeat(40), scope: 'ok', acceptanceCriteria: [123] },
|
|
{ commitSha: ['a'.repeat(40)], scope: 'ok', acceptanceCriteria: ['ok'] },
|
|
{ scope: 'ok', acceptanceCriteria: ['ok'] },
|
|
])(
|
|
'rejects unsafe or unbounded metadata %# without persisting',
|
|
async (metadata) => {
|
|
const input = { ...dto(), metadata };
|
|
await expect(service.create(input, admin)).rejects.toMatchObject({
|
|
status: 400,
|
|
});
|
|
expect(
|
|
await db.orchestrationTask.count({ where: { key: input.key } }),
|
|
).toBe(0);
|
|
},
|
|
);
|
|
it('accepts exact metadata bounds', async () => {
|
|
const metadata = {
|
|
commitSha: 'a'.repeat(40),
|
|
scope: 'x'.repeat(2000),
|
|
acceptanceCriteria: Array(20).fill('x'.repeat(500)),
|
|
};
|
|
expect(
|
|
(await service.create({ ...dto(), metadata }, admin)).metadata,
|
|
).toMatchObject(metadata);
|
|
});
|
|
it('normalizes racing duplicate proposal keys to conflict without duplicate events', async () => {
|
|
const input = dto();
|
|
const results = await Promise.allSettled([
|
|
service.create(input, admin),
|
|
service.create({ ...input, title: 'Different intent' }, reviewer),
|
|
]);
|
|
expect(results.filter((r) => r.status === 'fulfilled')).toHaveLength(1);
|
|
const rejected = results.find(
|
|
(r) => r.status === 'rejected',
|
|
) as PromiseRejectedResult;
|
|
expect(rejected.reason).toMatchObject({ status: 409 });
|
|
const persisted = await db.orchestrationTask.findUniqueOrThrow({
|
|
where: { key: input.key },
|
|
});
|
|
expect(
|
|
await db.orchestrationEvent.count({ where: { taskId: persisted.id } }),
|
|
).toBe(1);
|
|
await expect(service.create(input, admin)).rejects.toMatchObject({
|
|
status: 409,
|
|
});
|
|
});
|
|
it('enforces create and read contracts over HTTP with real PostgreSQL persistence', async () => {
|
|
const module = await Test.createTestingModule({
|
|
controllers: [OrchestrationController],
|
|
providers: [
|
|
{ provide: PrepareWorkflowService, useValue: service },
|
|
{
|
|
provide: OrchestrationService,
|
|
useValue: {
|
|
findAll: () => db.orchestrationTask.findMany(),
|
|
findOne: (id: string) =>
|
|
db.orchestrationTask.findUniqueOrThrow({ where: { id } }),
|
|
},
|
|
},
|
|
],
|
|
})
|
|
.overrideGuard(JwtAuthGuard)
|
|
.useValue({
|
|
canActivate(context: any) {
|
|
context.switchToHttp().getRequest().user = {
|
|
sub: admin.id,
|
|
role: admin.role,
|
|
};
|
|
return true;
|
|
},
|
|
})
|
|
.overrideGuard(RolesGuard)
|
|
.useValue({ canActivate: () => true })
|
|
.compile();
|
|
const app = module.createNestApplication();
|
|
await app.init();
|
|
try {
|
|
const input = dto();
|
|
const response = await request(app.getHttpServer())
|
|
.post('/admin/orchestration/tasks')
|
|
.send(input)
|
|
.expect(201);
|
|
expect(response.body.metadata).toEqual({
|
|
scope: input.metadata.scope,
|
|
acceptanceCriteria: input.metadata.acceptanceCriteria,
|
|
commitSha: input.metadata.commitSha,
|
|
environment: 'staging',
|
|
});
|
|
expect(response.body.availableActions).toEqual(['SUBMIT_GATE1']);
|
|
const detail = await request(app.getHttpServer())
|
|
.get(`/admin/orchestration/tasks/${response.body.id}`)
|
|
.expect(200);
|
|
expect(detail.body.metadata).toEqual(response.body.metadata);
|
|
const list = await request(app.getHttpServer())
|
|
.get('/admin/orchestration/tasks')
|
|
.expect(200);
|
|
expect(
|
|
list.body.find((row: any) => row.id === response.body.id).metadata,
|
|
).toEqual(response.body.metadata);
|
|
expect(JSON.stringify(response.body)).not.toMatch(
|
|
/createdBy|must-not-persist|actorId|workflowScope/,
|
|
);
|
|
await request(app.getHttpServer())
|
|
.post('/admin/orchestration/tasks')
|
|
.send(input)
|
|
.expect(409);
|
|
await request(app.getHttpServer())
|
|
.post('/admin/orchestration/tasks')
|
|
.send({
|
|
...input,
|
|
key: `${input.key}-invalid`,
|
|
metadata: { scope: 'only stage', acceptanceCriteria: ['QA passes'] },
|
|
})
|
|
.expect(400);
|
|
await request(app.getHttpServer())
|
|
.post('/admin/orchestration/tasks')
|
|
.send({
|
|
...input,
|
|
key: `${input.key}-invalid`,
|
|
metadata: { ...input.metadata, acceptanceCriteria: [' '] },
|
|
})
|
|
.expect(400);
|
|
} finally {
|
|
await app.close();
|
|
}
|
|
});
|
|
it('fences heartbeat and completes QA plus independent Stage evidence only to Gate2 pending', async () => {
|
|
const t = await ready();
|
|
const c = await service.claim(worker, t.id);
|
|
await expect(
|
|
service.heartbeat({ ...worker, id: 'worker-2' }, c),
|
|
).rejects.toThrow(/lease/i);
|
|
await service.heartbeat(worker, c);
|
|
const qa = {
|
|
...c,
|
|
sha256: 'b'.repeat(64),
|
|
uri: 'https://ci.example/qa',
|
|
actualCents: 8,
|
|
};
|
|
await service.complete(worker, qa);
|
|
await expect(
|
|
service.stageEvidence(
|
|
{ id: 'proposer', role: 'Admin' },
|
|
{
|
|
taskId: t.id,
|
|
runId: c.runId,
|
|
sha256: qa.sha256,
|
|
commitSha: c.commitSha,
|
|
evidenceSha256: 'c'.repeat(64),
|
|
uri: 'https://ci.example/stage',
|
|
},
|
|
),
|
|
).rejects.toThrow(/identity/i);
|
|
await service.stageEvidence(ci, {
|
|
taskId: t.id,
|
|
runId: c.runId,
|
|
sha256: qa.sha256,
|
|
commitSha: c.commitSha,
|
|
evidenceSha256: 'c'.repeat(64),
|
|
uri: 'https://ci.example/stage',
|
|
});
|
|
expect(
|
|
await db.orchestrationTask.findUnique({ where: { id: t.id } }),
|
|
).toMatchObject({ state: 'GATE2_PENDING' });
|
|
expect(
|
|
await db.orchestrationBudgetLedger.aggregate({
|
|
where: { taskId: t.id },
|
|
_sum: { amountCents: true },
|
|
}),
|
|
).toMatchObject({ _sum: { amountCents: 8 } });
|
|
await expect(service.complete(worker, qa)).rejects.toThrow(/lease|active/i);
|
|
});
|
|
it('bounds expired-lease retries and rejects stale completion without releasing unknown costs', async () => {
|
|
const t = await service.create({ ...dto(), budgetLimitCents: 60 }, admin);
|
|
let current = await service.transition(
|
|
t.id,
|
|
{
|
|
toState: 'GATE1_PENDING',
|
|
expectedVersion: 1,
|
|
idempotencyKey: `s-${t.id}`,
|
|
},
|
|
admin,
|
|
);
|
|
current = await service.decide(
|
|
t.id,
|
|
1,
|
|
{
|
|
decision: 'APPROVED',
|
|
expectedVersion: current.version,
|
|
idempotencyKey: `a-${t.id}`,
|
|
expiresAt: new Date(Date.now() + 60000).toISOString(),
|
|
},
|
|
reviewer,
|
|
);
|
|
await service.transition(
|
|
t.id,
|
|
{
|
|
toState: 'READY',
|
|
expectedVersion: current.version,
|
|
idempotencyKey: `q-${t.id}`,
|
|
},
|
|
admin,
|
|
);
|
|
const first = await service.claim(worker, t.id);
|
|
for (let i = 0; i < 2; i++) {
|
|
await db.prepareWorkflowOutbox.update({
|
|
where: { taskId: t.id },
|
|
data: { leaseUntil: new Date(0) },
|
|
});
|
|
const next = await service.claim(worker, t.id);
|
|
expect(next.attempt).toBe(i + 2);
|
|
}
|
|
await expect(
|
|
service.complete(worker, {
|
|
...first,
|
|
sha256: 'b'.repeat(64),
|
|
uri: 'https://ci.example/qa',
|
|
actualCents: 0,
|
|
}),
|
|
).rejects.toThrow(/lease/i);
|
|
await db.prepareWorkflowOutbox.update({
|
|
where: { taskId: t.id },
|
|
data: { leaseUntil: new Date(0) },
|
|
});
|
|
await expect(service.claim(worker, t.id)).rejects.toThrow(/exhausted/i);
|
|
expect(await db.orchestrationRun.count({ where: { taskId: t.id } })).toBe(
|
|
3,
|
|
);
|
|
expect(
|
|
await db.orchestrationBudgetLedger.aggregate({
|
|
where: { taskId: t.id },
|
|
_sum: { amountCents: true },
|
|
}),
|
|
).toMatchObject({ _sum: { amountCents: 60 } });
|
|
});
|
|
it('rejects self approval, non-PREPARE and unprovisioned workers', async () => {
|
|
await expect(
|
|
service.create(
|
|
{ ...dto(), operationMode: OperationMode.MUTATE_PRODUCTION },
|
|
admin,
|
|
),
|
|
).rejects.toThrow(/PREPARE/);
|
|
let t = await service.create(dto(), reviewer);
|
|
t = await service.transition(
|
|
t.id,
|
|
{
|
|
toState: 'GATE1_PENDING',
|
|
expectedVersion: 1,
|
|
idempotencyKey: `s-${t.id}`,
|
|
},
|
|
reviewer,
|
|
);
|
|
await expect(
|
|
service.decide(
|
|
t.id,
|
|
1,
|
|
{
|
|
decision: 'APPROVED',
|
|
expectedVersion: t.version,
|
|
idempotencyKey: `a-${t.id}`,
|
|
expiresAt: new Date(Date.now() + 60000).toISOString(),
|
|
},
|
|
reviewer,
|
|
),
|
|
).rejects.toThrow(/independent/i);
|
|
await expect(
|
|
service.claim({ id: 'spoof', role: 'WORKFLOW_WORKER' }),
|
|
).rejects.toThrow(/provisioned/i);
|
|
await expect(service.decide(t.id, 2, {}, reviewer)).rejects.toThrow(
|
|
/disabled/i,
|
|
);
|
|
});
|
|
it('serializes duplicate queue submission and denies claims beyond remaining task budget', async () => {
|
|
const t = await ready();
|
|
await db.orchestrationBudgetLedger.create({
|
|
data: {
|
|
taskId: t.id,
|
|
amountCents: 1,
|
|
kind: 'SPEND',
|
|
actorId: 'fixture',
|
|
idempotencyKey: `spend-${t.id}`,
|
|
},
|
|
});
|
|
await expect(service.claim(worker, t.id)).rejects.toThrow(/budget/i);
|
|
expect(await db.orchestrationRun.count({ where: { taskId: t.id } })).toBe(
|
|
0,
|
|
);
|
|
expect(
|
|
await db.prepareWorkflowOutbox.count({ where: { taskId: t.id } }),
|
|
).toBe(1);
|
|
});
|
|
it('replays submission safely and serializes racing queue requests', async () => {
|
|
let t = await service.create(dto(), admin);
|
|
const submit = {
|
|
toState: 'GATE1_PENDING',
|
|
expectedVersion: t.version,
|
|
idempotencyKey: `replay-${t.id}`,
|
|
};
|
|
t = await service.transition(t.id, submit, admin);
|
|
expect((await service.transition(t.id, submit, admin)).version).toBe(
|
|
t.version,
|
|
);
|
|
await expect(
|
|
service.transition(t.id, { ...submit, toState: 'READY' }, admin),
|
|
).rejects.toThrow(/mismatch/i);
|
|
t = await service.decide(
|
|
t.id,
|
|
1,
|
|
{
|
|
decision: 'APPROVED',
|
|
expectedVersion: t.version,
|
|
idempotencyKey: `r-approve-${t.id}`,
|
|
expiresAt: new Date(Date.now() + 60000).toISOString(),
|
|
},
|
|
reviewer,
|
|
);
|
|
const results = await Promise.allSettled(
|
|
['one', 'two'].map((k) =>
|
|
service.transition(
|
|
t.id,
|
|
{
|
|
toState: 'READY',
|
|
expectedVersion: t.version,
|
|
idempotencyKey: `${k}-${t.id}`,
|
|
},
|
|
admin,
|
|
),
|
|
),
|
|
);
|
|
expect(results.filter((r) => r.status === 'fulfilled')).toHaveLength(1);
|
|
expect(
|
|
await db.prepareWorkflowOutbox.count({ where: { taskId: t.id } }),
|
|
).toBe(1);
|
|
});
|
|
it('kill switch freezes claims without creating run or reservation', async () => {
|
|
const t = await ready();
|
|
await db.orchestrationKillSwitch.create({
|
|
data: {
|
|
scope: 'GLOBAL',
|
|
level: 'FREEZE_NEW_WORK',
|
|
reason: 'isolated test',
|
|
actorId: 'fixture',
|
|
},
|
|
});
|
|
try {
|
|
await expect(service.claim(worker, t.id)).rejects.toThrow(/kill switch/i);
|
|
} finally {
|
|
await db.orchestrationKillSwitch.delete({ where: { scope: 'GLOBAL' } });
|
|
}
|
|
expect(await db.orchestrationRun.count({ where: { taskId: t.id } })).toBe(
|
|
0,
|
|
);
|
|
});
|
|
it('persists an atomic outbox and exactly one worker wins concurrent claims', async () => {
|
|
const t = await ready();
|
|
const claims = await Promise.all([
|
|
service.claim(worker, t.id),
|
|
service.claim({ id: 'worker-2', role: 'WORKFLOW_WORKER' }, t.id),
|
|
]);
|
|
expect(claims.filter(Boolean)).toHaveLength(1);
|
|
const claim = claims.find(Boolean)!;
|
|
expect(claim.taskId).toBe(t.id);
|
|
expect(claim.attempt).toBe(1);
|
|
expect(await db.orchestrationRun.count({ where: { taskId: t.id } })).toBe(
|
|
1,
|
|
);
|
|
expect(
|
|
await db.orchestrationBudgetLedger.aggregate({
|
|
where: { taskId: t.id },
|
|
_sum: { amountCents: true },
|
|
}),
|
|
).toMatchObject({ _sum: { amountCents: 20 } });
|
|
expect(
|
|
await service.project(
|
|
await db.orchestrationTask.findUniqueOrThrow({ where: { id: t.id } }),
|
|
admin,
|
|
),
|
|
).toMatchObject({
|
|
metadata: {
|
|
scope: 'Staging schema audit only',
|
|
acceptanceCriteria: ['Report all schema failures'],
|
|
environment: 'staging',
|
|
commitSha: 'a'.repeat(40),
|
|
},
|
|
});
|
|
});
|
|
});
|