325 lines
9.6 KiB
TypeScript
325 lines
9.6 KiB
TypeScript
import { PrismaClient } from '@prisma/client';
|
|
import { PrepareWorkflowService } from './prepare-workflow.service';
|
|
import { OperationMode, WorkflowState } from './orchestration.domain';
|
|
|
|
const enabled = process.env.WORKFLOW_DB === '1';
|
|
(enabled ? describe : describe.skip)('isolated prepare workflow', () => {
|
|
const db = new PrismaClient();
|
|
const admin = { id: 'proposer', role: 'Admin' };
|
|
const reviewer = { id: 'reviewer', role: 'SUPER_ADMIN' };
|
|
const worker = { id: 'worker-1', role: 'WORKFLOW_WORKER' };
|
|
const ci = { id: 'isolated-ci', role: 'STAGE_CI' };
|
|
const service = new PrepareWorkflowService(db as never, {
|
|
workerIds: ['worker-1', 'worker-2'],
|
|
stageCiIds: ['isolated-ci'],
|
|
});
|
|
let seq = 0;
|
|
const dto = () => ({
|
|
key: `wf-${Date.now()}-${seq++}`,
|
|
title: 'Bounded isolated QA tracer',
|
|
department: 'ENGINEERING',
|
|
operationMode: OperationMode.PREPARE,
|
|
budgetLimitCents: 20,
|
|
metadata: {
|
|
commitSha: 'a'.repeat(40),
|
|
token: 'must-not-persist',
|
|
scope: 'production',
|
|
actorId: 'spoof',
|
|
},
|
|
});
|
|
async function ready() {
|
|
let t = await service.create(dto(), admin);
|
|
t = await service.transition(
|
|
t.id,
|
|
{
|
|
toState: WorkflowState.GATE1_PENDING,
|
|
expectedVersion: t.version,
|
|
idempotencyKey: `submit-${t.id}`,
|
|
},
|
|
admin,
|
|
);
|
|
t = await service.decide(
|
|
t.id,
|
|
1,
|
|
{
|
|
decision: 'APPROVED',
|
|
expectedVersion: t.version,
|
|
idempotencyKey: `approve-${t.id}`,
|
|
expiresAt: new Date(Date.now() + 60000).toISOString(),
|
|
},
|
|
reviewer,
|
|
);
|
|
return service.transition(
|
|
t.id,
|
|
{
|
|
toState: WorkflowState.READY,
|
|
expectedVersion: t.version,
|
|
idempotencyKey: `ready-${t.id}`,
|
|
},
|
|
admin,
|
|
);
|
|
}
|
|
afterAll(async () => {
|
|
await db.$disconnect();
|
|
});
|
|
it('fences heartbeat and completes QA plus independent Stage evidence only to Gate2 pending', async () => {
|
|
const t = await ready();
|
|
const c = await service.claim(worker, t.id);
|
|
await expect(
|
|
service.heartbeat({ ...worker, id: 'worker-2' }, c),
|
|
).rejects.toThrow(/lease/i);
|
|
await service.heartbeat(worker, c);
|
|
const qa = {
|
|
...c,
|
|
sha256: 'b'.repeat(64),
|
|
uri: 'https://ci.example/qa',
|
|
actualCents: 8,
|
|
};
|
|
await service.complete(worker, qa);
|
|
await expect(
|
|
service.stageEvidence(
|
|
{ id: 'proposer', role: 'Admin' },
|
|
{
|
|
taskId: t.id,
|
|
runId: c.runId,
|
|
sha256: qa.sha256,
|
|
commitSha: c.commitSha,
|
|
evidenceSha256: 'c'.repeat(64),
|
|
uri: 'https://ci.example/stage',
|
|
},
|
|
),
|
|
).rejects.toThrow(/identity/i);
|
|
await service.stageEvidence(ci, {
|
|
taskId: t.id,
|
|
runId: c.runId,
|
|
sha256: qa.sha256,
|
|
commitSha: c.commitSha,
|
|
evidenceSha256: 'c'.repeat(64),
|
|
uri: 'https://ci.example/stage',
|
|
});
|
|
expect(
|
|
await db.orchestrationTask.findUnique({ where: { id: t.id } }),
|
|
).toMatchObject({ state: 'GATE2_PENDING' });
|
|
expect(
|
|
await db.orchestrationBudgetLedger.aggregate({
|
|
where: { taskId: t.id },
|
|
_sum: { amountCents: true },
|
|
}),
|
|
).toMatchObject({ _sum: { amountCents: 8 } });
|
|
await expect(service.complete(worker, qa)).rejects.toThrow(/lease|active/i);
|
|
});
|
|
it('bounds expired-lease retries and rejects stale completion without releasing unknown costs', async () => {
|
|
const t = await service.create({ ...dto(), budgetLimitCents: 60 }, admin);
|
|
let current = await service.transition(
|
|
t.id,
|
|
{
|
|
toState: 'GATE1_PENDING',
|
|
expectedVersion: 1,
|
|
idempotencyKey: `s-${t.id}`,
|
|
},
|
|
admin,
|
|
);
|
|
current = await service.decide(
|
|
t.id,
|
|
1,
|
|
{
|
|
decision: 'APPROVED',
|
|
expectedVersion: current.version,
|
|
idempotencyKey: `a-${t.id}`,
|
|
expiresAt: new Date(Date.now() + 60000).toISOString(),
|
|
},
|
|
reviewer,
|
|
);
|
|
await service.transition(
|
|
t.id,
|
|
{
|
|
toState: 'READY',
|
|
expectedVersion: current.version,
|
|
idempotencyKey: `q-${t.id}`,
|
|
},
|
|
admin,
|
|
);
|
|
const first = await service.claim(worker, t.id);
|
|
for (let i = 0; i < 2; i++) {
|
|
await db.prepareWorkflowOutbox.update({
|
|
where: { taskId: t.id },
|
|
data: { leaseUntil: new Date(0) },
|
|
});
|
|
const next = await service.claim(worker, t.id);
|
|
expect(next.attempt).toBe(i + 2);
|
|
}
|
|
await expect(
|
|
service.complete(worker, {
|
|
...first,
|
|
sha256: 'b'.repeat(64),
|
|
uri: 'https://ci.example/qa',
|
|
actualCents: 0,
|
|
}),
|
|
).rejects.toThrow(/lease/i);
|
|
await db.prepareWorkflowOutbox.update({
|
|
where: { taskId: t.id },
|
|
data: { leaseUntil: new Date(0) },
|
|
});
|
|
await expect(service.claim(worker, t.id)).rejects.toThrow(/exhausted/i);
|
|
expect(await db.orchestrationRun.count({ where: { taskId: t.id } })).toBe(
|
|
3,
|
|
);
|
|
expect(
|
|
await db.orchestrationBudgetLedger.aggregate({
|
|
where: { taskId: t.id },
|
|
_sum: { amountCents: true },
|
|
}),
|
|
).toMatchObject({ _sum: { amountCents: 60 } });
|
|
});
|
|
it('rejects self approval, non-PREPARE and unprovisioned workers', async () => {
|
|
await expect(
|
|
service.create(
|
|
{ ...dto(), operationMode: OperationMode.MUTATE_PRODUCTION },
|
|
admin,
|
|
),
|
|
).rejects.toThrow(/PREPARE/);
|
|
let t = await service.create(dto(), reviewer);
|
|
t = await service.transition(
|
|
t.id,
|
|
{
|
|
toState: 'GATE1_PENDING',
|
|
expectedVersion: 1,
|
|
idempotencyKey: `s-${t.id}`,
|
|
},
|
|
reviewer,
|
|
);
|
|
await expect(
|
|
service.decide(
|
|
t.id,
|
|
1,
|
|
{
|
|
decision: 'APPROVED',
|
|
expectedVersion: t.version,
|
|
idempotencyKey: `a-${t.id}`,
|
|
expiresAt: new Date(Date.now() + 60000).toISOString(),
|
|
},
|
|
reviewer,
|
|
),
|
|
).rejects.toThrow(/independent/i);
|
|
await expect(
|
|
service.claim({ id: 'spoof', role: 'WORKFLOW_WORKER' }),
|
|
).rejects.toThrow(/provisioned/i);
|
|
await expect(service.decide(t.id, 2, {}, reviewer)).rejects.toThrow(
|
|
/disabled/i,
|
|
);
|
|
});
|
|
it('serializes duplicate queue submission and denies claims beyond remaining task budget', async () => {
|
|
const t = await ready();
|
|
await db.orchestrationBudgetLedger.create({
|
|
data: {
|
|
taskId: t.id,
|
|
amountCents: 1,
|
|
kind: 'SPEND',
|
|
actorId: 'fixture',
|
|
idempotencyKey: `spend-${t.id}`,
|
|
},
|
|
});
|
|
await expect(service.claim(worker, t.id)).rejects.toThrow(/budget/i);
|
|
expect(await db.orchestrationRun.count({ where: { taskId: t.id } })).toBe(
|
|
0,
|
|
);
|
|
expect(
|
|
await db.prepareWorkflowOutbox.count({ where: { taskId: t.id } }),
|
|
).toBe(1);
|
|
});
|
|
it('replays submission safely and serializes racing queue requests', async () => {
|
|
let t = await service.create(dto(), admin);
|
|
const submit = {
|
|
toState: 'GATE1_PENDING',
|
|
expectedVersion: t.version,
|
|
idempotencyKey: `replay-${t.id}`,
|
|
};
|
|
t = await service.transition(t.id, submit, admin);
|
|
expect((await service.transition(t.id, submit, admin)).version).toBe(
|
|
t.version,
|
|
);
|
|
await expect(
|
|
service.transition(t.id, { ...submit, toState: 'READY' }, admin),
|
|
).rejects.toThrow(/mismatch/i);
|
|
t = await service.decide(
|
|
t.id,
|
|
1,
|
|
{
|
|
decision: 'APPROVED',
|
|
expectedVersion: t.version,
|
|
idempotencyKey: `r-approve-${t.id}`,
|
|
expiresAt: new Date(Date.now() + 60000).toISOString(),
|
|
},
|
|
reviewer,
|
|
);
|
|
const results = await Promise.allSettled(
|
|
['one', 'two'].map((k) =>
|
|
service.transition(
|
|
t.id,
|
|
{
|
|
toState: 'READY',
|
|
expectedVersion: t.version,
|
|
idempotencyKey: `${k}-${t.id}`,
|
|
},
|
|
admin,
|
|
),
|
|
),
|
|
);
|
|
expect(results.filter((r) => r.status === 'fulfilled')).toHaveLength(1);
|
|
expect(
|
|
await db.prepareWorkflowOutbox.count({ where: { taskId: t.id } }),
|
|
).toBe(1);
|
|
});
|
|
it('kill switch freezes claims without creating run or reservation', async () => {
|
|
const t = await ready();
|
|
await db.orchestrationKillSwitch.create({
|
|
data: {
|
|
scope: 'GLOBAL',
|
|
level: 'FREEZE_NEW_WORK',
|
|
reason: 'isolated test',
|
|
actorId: 'fixture',
|
|
},
|
|
});
|
|
try {
|
|
await expect(service.claim(worker, t.id)).rejects.toThrow(/kill switch/i);
|
|
} finally {
|
|
await db.orchestrationKillSwitch.delete({ where: { scope: 'GLOBAL' } });
|
|
}
|
|
expect(await db.orchestrationRun.count({ where: { taskId: t.id } })).toBe(
|
|
0,
|
|
);
|
|
});
|
|
it('persists an atomic outbox and exactly one worker wins concurrent claims', async () => {
|
|
const t = await ready();
|
|
const claims = await Promise.all([
|
|
service.claim(worker, t.id),
|
|
service.claim({ id: 'worker-2', role: 'WORKFLOW_WORKER' }, t.id),
|
|
]);
|
|
expect(claims.filter(Boolean)).toHaveLength(1);
|
|
const claim = claims.find(Boolean)!;
|
|
expect(claim.taskId).toBe(t.id);
|
|
expect(claim.attempt).toBe(1);
|
|
expect(await db.orchestrationRun.count({ where: { taskId: t.id } })).toBe(
|
|
1,
|
|
);
|
|
expect(
|
|
await db.orchestrationBudgetLedger.aggregate({
|
|
where: { taskId: t.id },
|
|
_sum: { amountCents: true },
|
|
}),
|
|
).toMatchObject({ _sum: { amountCents: 20 } });
|
|
expect(
|
|
await service.project(
|
|
await db.orchestrationTask.findUniqueOrThrow({ where: { id: t.id } }),
|
|
admin,
|
|
),
|
|
).toMatchObject({
|
|
metadata: {
|
|
scope: 'STAGE_PREPARE',
|
|
environment: 'staging',
|
|
commitSha: 'a'.repeat(40),
|
|
},
|
|
});
|
|
});
|
|
});
|