Files
canina/backend/src/orchestration/workflow-db.spec.ts
T

536 lines
18 KiB
TypeScript

import { Test } from '@nestjs/testing';
import request from 'supertest';
import { OrchestrationController } from './orchestration.controller';
import { OrchestrationService } from './orchestration.service';
import { JwtAuthGuard } from '../auth/jwt-auth.guard';
import { RolesGuard } from '../common/guards/roles.guard';
import { PrismaClient } from '@prisma/client';
import { PrepareWorkflowService } from './prepare-workflow.service';
import { OperationMode, WorkflowState } from './orchestration.domain';
const enabled = process.env.WORKFLOW_DB === '1';
(enabled ? describe : describe.skip)('isolated prepare workflow', () => {
const db = new PrismaClient();
const admin = { id: 'proposer', role: 'Admin' };
const reviewer = { id: 'reviewer', role: 'SUPER_ADMIN' };
const worker = { id: 'worker-1', role: 'WORKFLOW_WORKER' };
const ci = { id: 'isolated-ci', role: 'STAGE_CI' };
const service = new PrepareWorkflowService(db as never, {
workerIds: ['worker-1', 'worker-2'],
stageCiIds: ['isolated-ci'],
});
let seq = 0;
const dto = () => ({
key: `wf-${Date.now()}-${seq++}`,
title: 'Bounded isolated QA tracer',
department: 'ENGINEERING',
operationMode: OperationMode.PREPARE,
budgetLimitCents: 20,
metadata: {
commitSha: 'a'.repeat(40),
token: 'must-not-persist',
scope: 'Staging schema audit only',
acceptanceCriteria: ['Report all schema failures'],
actorId: 'spoof',
},
});
async function ready() {
let t = await service.create(dto(), admin);
t = await service.transition(
t.id,
{
toState: WorkflowState.GATE1_PENDING,
expectedVersion: t.version,
idempotencyKey: `submit-${t.id}`,
},
admin,
);
t = await service.decide(
t.id,
1,
{
decision: 'APPROVED',
expectedVersion: t.version,
idempotencyKey: `approve-${t.id}`,
expiresAt: new Date(Date.now() + 60000).toISOString(),
},
reviewer,
);
return service.transition(
t.id,
{
toState: WorkflowState.READY,
expectedVersion: t.version,
idempotencyKey: `ready-${t.id}`,
},
admin,
);
}
afterAll(async () => {
await db.$disconnect();
});
it('serializes competing owner holds and preserves audited reason, then decides held task', async () => {
let t = await service.create(dto(), admin);
t = await service.transition(t.id, {toState:'GATE1_PENDING',expectedVersion:t.version,idempotencyKey:`submit-hold-${t.id}`}, admin);
const command = {toState:'GATE1_HELD',expectedVersion:t.version,idempotencyKey:`hold-${t.id}`,reason:'Need source review'};
const results = await Promise.allSettled([
service.transition(t.id, command, reviewer),
service.transition(t.id, {...command,idempotencyKey:`competing-${t.id}`}, reviewer),
]);
expect(results.filter(r=>r.status==='fulfilled')).toHaveLength(1);
const held = await db.orchestrationTask.findUniqueOrThrow({where:{id:t.id}});
expect(held.state).toBe('GATE1_HELD');
expect(held.version).toBe(t.version+1);
const events = await db.orchestrationEvent.findMany({where:{taskId:t.id,eventType:'PREPARE_GATE1_HELD'}});
expect(events).toHaveLength(1);
expect(events[0].actorId).toBe(reviewer.id);
expect(JSON.stringify(events[0].payload)).toContain('Need source review');
expect(await db.orchestrationApprovalDecision.count({where:{taskId:t.id}})).toBe(0);
expect(await db.prepareWorkflowOutbox.count({where:{taskId:t.id}})).toBe(0);
await expect(service.transition(t.id, {...command,expectedVersion:held.version,reason:'changed'},reviewer)).rejects.toMatchObject({status:409});
const decision = await service.decide(t.id,1,{decision:'REJECTED',expectedVersion:held.version,idempotencyKey:`decline-held-${t.id}`,expiresAt:new Date(Date.now()+60000).toISOString()},reviewer);
expect(decision.state).toBe('REJECTED');
});
it('persists and projects immutable bounded proposal scope without arbitrary metadata', async () => {
const input = dto();
const t = await service.create(input, admin);
expect(t.metadata).toMatchObject({
workflowScope: 'STAGE_PREPARE',
scope: input.metadata.scope,
acceptanceCriteria: input.metadata.acceptanceCriteria,
commitSha: input.metadata.commitSha,
});
expect(t.metadata).not.toHaveProperty('token');
expect(t.metadata).not.toHaveProperty('actorId');
const submitted = await service.transition(
t.id,
{
toState: 'GATE1_PENDING',
expectedVersion: t.version,
idempotencyKey: `immutable-${t.id}`,
},
admin,
);
expect(submitted.metadata).toEqual(t.metadata);
expect(service.project(submitted, reviewer).metadata).toEqual({
scope: input.metadata.scope,
acceptanceCriteria: input.metadata.acceptanceCriteria,
environment: 'staging',
commitSha: input.metadata.commitSha,
});
});
it.each([
null,
[],
{},
{ commitSha: 'a'.repeat(40), scope: '', acceptanceCriteria: ['ok'] },
{
commitSha: 'a'.repeat(40),
scope: ' '.repeat(10),
acceptanceCriteria: ['ok'],
},
{
commitSha: 'a'.repeat(40),
scope: 'x'.repeat(2001),
acceptanceCriteria: ['ok'],
},
{ commitSha: 'a'.repeat(40), scope: 'ok', acceptanceCriteria: [] },
{
commitSha: 'a'.repeat(40),
scope: 'ok',
acceptanceCriteria: Array(21).fill('ok'),
},
{
commitSha: 'a'.repeat(40),
scope: 'ok',
acceptanceCriteria: ['x'.repeat(501)],
},
{ commitSha: 'a'.repeat(40), scope: 'ok', acceptanceCriteria: [' '] },
{ commitSha: 'a'.repeat(40), scope: 'ok', acceptanceCriteria: [123] },
{ commitSha: ['a'.repeat(40)], scope: 'ok', acceptanceCriteria: ['ok'] },
{ scope: 'ok', acceptanceCriteria: ['ok'] },
])(
'rejects unsafe or unbounded metadata %# without persisting',
async (metadata) => {
const input = { ...dto(), metadata };
await expect(service.create(input, admin)).rejects.toMatchObject({
status: 400,
});
expect(
await db.orchestrationTask.count({ where: { key: input.key } }),
).toBe(0);
},
);
it('accepts exact metadata bounds', async () => {
const metadata = {
commitSha: 'a'.repeat(40),
scope: 'x'.repeat(2000),
acceptanceCriteria: Array(20).fill('x'.repeat(500)),
};
expect(
(await service.create({ ...dto(), metadata }, admin)).metadata,
).toMatchObject(metadata);
});
it('normalizes racing duplicate proposal keys to conflict without duplicate events', async () => {
const input = dto();
const results = await Promise.allSettled([
service.create(input, admin),
service.create({ ...input, title: 'Different intent' }, reviewer),
]);
expect(results.filter((r) => r.status === 'fulfilled')).toHaveLength(1);
const rejected = results.find(
(r) => r.status === 'rejected',
) as PromiseRejectedResult;
expect(rejected.reason).toMatchObject({ status: 409 });
const persisted = await db.orchestrationTask.findUniqueOrThrow({
where: { key: input.key },
});
expect(
await db.orchestrationEvent.count({ where: { taskId: persisted.id } }),
).toBe(1);
await expect(service.create(input, admin)).rejects.toMatchObject({
status: 409,
});
});
it('enforces create and read contracts over HTTP with real PostgreSQL persistence', async () => {
const module = await Test.createTestingModule({
controllers: [OrchestrationController],
providers: [
{ provide: PrepareWorkflowService, useValue: service },
{
provide: OrchestrationService,
useValue: {
findAll: () => db.orchestrationTask.findMany(),
findOne: (id: string) =>
db.orchestrationTask.findUniqueOrThrow({ where: { id } }),
},
},
],
})
.overrideGuard(JwtAuthGuard)
.useValue({
canActivate(context: any) {
context.switchToHttp().getRequest().user = {
sub: admin.id,
role: admin.role,
};
return true;
},
})
.overrideGuard(RolesGuard)
.useValue({ canActivate: () => true })
.compile();
const app = module.createNestApplication();
await app.init();
try {
const input = dto();
const response = await request(app.getHttpServer())
.post('/admin/orchestration/tasks')
.send(input)
.expect(201);
expect(response.body.metadata).toEqual({
scope: input.metadata.scope,
acceptanceCriteria: input.metadata.acceptanceCriteria,
commitSha: input.metadata.commitSha,
environment: 'staging',
});
expect(response.body.availableActions).toEqual(['SUBMIT_GATE1']);
const detail = await request(app.getHttpServer())
.get(`/admin/orchestration/tasks/${response.body.id}`)
.expect(200);
expect(detail.body.metadata).toEqual(response.body.metadata);
const list = await request(app.getHttpServer())
.get('/admin/orchestration/tasks')
.expect(200);
expect(
list.body.find((row: any) => row.id === response.body.id).metadata,
).toEqual(response.body.metadata);
expect(JSON.stringify(response.body)).not.toMatch(
/createdBy|must-not-persist|actorId|workflowScope/,
);
await request(app.getHttpServer())
.post('/admin/orchestration/tasks')
.send(input)
.expect(409);
await request(app.getHttpServer())
.post('/admin/orchestration/tasks')
.send({
...input,
key: `${input.key}-invalid`,
metadata: { scope: 'only stage', acceptanceCriteria: ['QA passes'] },
})
.expect(400);
await request(app.getHttpServer())
.post('/admin/orchestration/tasks')
.send({
...input,
key: `${input.key}-invalid`,
metadata: { ...input.metadata, acceptanceCriteria: [' '] },
})
.expect(400);
} finally {
await app.close();
}
});
it('fences heartbeat and completes QA plus independent Stage evidence only to Gate2 pending', async () => {
const t = await ready();
const c = await service.claim(worker, t.id);
await expect(
service.heartbeat({ ...worker, id: 'worker-2' }, c),
).rejects.toThrow(/lease/i);
await service.heartbeat(worker, c);
const qa = {
...c,
sha256: 'b'.repeat(64),
uri: 'https://ci.example/qa',
actualCents: 8,
};
await service.complete(worker, qa);
await expect(
service.stageEvidence(
{ id: 'proposer', role: 'Admin' },
{
taskId: t.id,
runId: c.runId,
sha256: qa.sha256,
commitSha: c.commitSha,
evidenceSha256: 'c'.repeat(64),
uri: 'https://ci.example/stage',
},
),
).rejects.toThrow(/identity/i);
await service.stageEvidence(ci, {
taskId: t.id,
runId: c.runId,
sha256: qa.sha256,
commitSha: c.commitSha,
evidenceSha256: 'c'.repeat(64),
uri: 'https://ci.example/stage',
});
expect(
await db.orchestrationTask.findUnique({ where: { id: t.id } }),
).toMatchObject({ state: 'GATE2_PENDING' });
expect(
await db.orchestrationBudgetLedger.aggregate({
where: { taskId: t.id },
_sum: { amountCents: true },
}),
).toMatchObject({ _sum: { amountCents: 8 } });
await expect(service.complete(worker, qa)).rejects.toThrow(/lease|active/i);
});
it('bounds expired-lease retries and rejects stale completion without releasing unknown costs', async () => {
const t = await service.create({ ...dto(), budgetLimitCents: 60 }, admin);
let current = await service.transition(
t.id,
{
toState: 'GATE1_PENDING',
expectedVersion: 1,
idempotencyKey: `s-${t.id}`,
},
admin,
);
current = await service.decide(
t.id,
1,
{
decision: 'APPROVED',
expectedVersion: current.version,
idempotencyKey: `a-${t.id}`,
expiresAt: new Date(Date.now() + 60000).toISOString(),
},
reviewer,
);
await service.transition(
t.id,
{
toState: 'READY',
expectedVersion: current.version,
idempotencyKey: `q-${t.id}`,
},
admin,
);
const first = await service.claim(worker, t.id);
for (let i = 0; i < 2; i++) {
await db.prepareWorkflowOutbox.update({
where: { taskId: t.id },
data: { leaseUntil: new Date(0) },
});
const next = await service.claim(worker, t.id);
expect(next.attempt).toBe(i + 2);
}
await expect(
service.complete(worker, {
...first,
sha256: 'b'.repeat(64),
uri: 'https://ci.example/qa',
actualCents: 0,
}),
).rejects.toThrow(/lease/i);
await db.prepareWorkflowOutbox.update({
where: { taskId: t.id },
data: { leaseUntil: new Date(0) },
});
await expect(service.claim(worker, t.id)).rejects.toThrow(/exhausted/i);
expect(await db.orchestrationRun.count({ where: { taskId: t.id } })).toBe(
3,
);
expect(
await db.orchestrationBudgetLedger.aggregate({
where: { taskId: t.id },
_sum: { amountCents: true },
}),
).toMatchObject({ _sum: { amountCents: 60 } });
});
it('rejects self approval, non-PREPARE and unprovisioned workers', async () => {
await expect(
service.create(
{ ...dto(), operationMode: OperationMode.MUTATE_PRODUCTION },
admin,
),
).rejects.toThrow(/PREPARE/);
let t = await service.create(dto(), reviewer);
t = await service.transition(
t.id,
{
toState: 'GATE1_PENDING',
expectedVersion: 1,
idempotencyKey: `s-${t.id}`,
},
reviewer,
);
await expect(
service.decide(
t.id,
1,
{
decision: 'APPROVED',
expectedVersion: t.version,
idempotencyKey: `a-${t.id}`,
expiresAt: new Date(Date.now() + 60000).toISOString(),
},
reviewer,
),
).rejects.toThrow(/independent/i);
await expect(
service.claim({ id: 'spoof', role: 'WORKFLOW_WORKER' }),
).rejects.toThrow(/provisioned/i);
await expect(service.decide(t.id, 2, {}, reviewer)).rejects.toThrow(
/disabled/i,
);
});
it('serializes duplicate queue submission and denies claims beyond remaining task budget', async () => {
const t = await ready();
await db.orchestrationBudgetLedger.create({
data: {
taskId: t.id,
amountCents: 1,
kind: 'SPEND',
actorId: 'fixture',
idempotencyKey: `spend-${t.id}`,
},
});
await expect(service.claim(worker, t.id)).rejects.toThrow(/budget/i);
expect(await db.orchestrationRun.count({ where: { taskId: t.id } })).toBe(
0,
);
expect(
await db.prepareWorkflowOutbox.count({ where: { taskId: t.id } }),
).toBe(1);
});
it('replays submission safely and serializes racing queue requests', async () => {
let t = await service.create(dto(), admin);
const submit = {
toState: 'GATE1_PENDING',
expectedVersion: t.version,
idempotencyKey: `replay-${t.id}`,
};
t = await service.transition(t.id, submit, admin);
expect((await service.transition(t.id, submit, admin)).version).toBe(
t.version,
);
await expect(
service.transition(t.id, { ...submit, toState: 'READY' }, admin),
).rejects.toThrow(/mismatch/i);
t = await service.decide(
t.id,
1,
{
decision: 'APPROVED',
expectedVersion: t.version,
idempotencyKey: `r-approve-${t.id}`,
expiresAt: new Date(Date.now() + 60000).toISOString(),
},
reviewer,
);
const results = await Promise.allSettled(
['one', 'two'].map((k) =>
service.transition(
t.id,
{
toState: 'READY',
expectedVersion: t.version,
idempotencyKey: `${k}-${t.id}`,
},
admin,
),
),
);
expect(results.filter((r) => r.status === 'fulfilled')).toHaveLength(1);
expect(
await db.prepareWorkflowOutbox.count({ where: { taskId: t.id } }),
).toBe(1);
});
it('kill switch freezes claims without creating run or reservation', async () => {
const t = await ready();
await db.orchestrationKillSwitch.create({
data: {
scope: 'GLOBAL',
level: 'FREEZE_NEW_WORK',
reason: 'isolated test',
actorId: 'fixture',
},
});
try {
await expect(service.claim(worker, t.id)).rejects.toThrow(/kill switch/i);
} finally {
await db.orchestrationKillSwitch.delete({ where: { scope: 'GLOBAL' } });
}
expect(await db.orchestrationRun.count({ where: { taskId: t.id } })).toBe(
0,
);
});
it('persists an atomic outbox and exactly one worker wins concurrent claims', async () => {
const t = await ready();
const claims = await Promise.all([
service.claim(worker, t.id),
service.claim({ id: 'worker-2', role: 'WORKFLOW_WORKER' }, t.id),
]);
expect(claims.filter(Boolean)).toHaveLength(1);
const claim = claims.find(Boolean)!;
expect(claim.taskId).toBe(t.id);
expect(claim.attempt).toBe(1);
expect(await db.orchestrationRun.count({ where: { taskId: t.id } })).toBe(
1,
);
expect(
await db.orchestrationBudgetLedger.aggregate({
where: { taskId: t.id },
_sum: { amountCents: true },
}),
).toMatchObject({ _sum: { amountCents: 20 } });
expect(
await service.project(
await db.orchestrationTask.findUniqueOrThrow({ where: { id: t.id } }),
admin,
),
).toMatchObject({
metadata: {
scope: 'Staging schema audit only',
acceptanceCriteria: ['Report all schema failures'],
environment: 'staging',
commitSha: 'a'.repeat(40),
},
});
});
});