fix(orchestration): atomically queue approvals with optional not-before schedule
Deploy Canina / deploy (push) Successful in 4m56s
Deploy Canina / deploy (push) Successful in 4m56s
This commit is contained in:
1 parent
ab2f330eb8
commit
50d0df822c
15 files changed
+216
-65
No files matched your search
@@ -0,0 +1,3 @@
|
||||
-- Scheduling is distinct from immutable approval validity. Existing queues run immediately.
|
||||
ALTER TABLE "prepare_workflow_outbox" ADD COLUMN "not_before" TIMESTAMPTZ;
|
||||
CREATE INDEX "prepare_workflow_outbox_due_idx" ON "prepare_workflow_outbox" ("status", "not_before", "created_at");
|
||||
@@ -1239,6 +1239,7 @@ model PrepareWorkflowOutbox {
|
||||
taskId String @unique @map("task_id") @db.Uuid
|
||||
status String @default("PENDING") @db.VarChar(20)
|
||||
attempts Int @default(0)
|
||||
notBefore DateTime? @map("not_before") @db.Timestamptz()
|
||||
leaseOwner String? @map("lease_owner") @db.VarChar(100)
|
||||
leaseToken String? @map("lease_token") @db.Uuid
|
||||
leaseUntil DateTime? @map("lease_until") @db.Timestamptz()
|
||||
@@ -1247,5 +1248,6 @@ model PrepareWorkflowOutbox {
|
||||
reservationId String? @map("reservation_id") @db.Uuid
|
||||
createdAt DateTime @default(now()) @map("created_at") @db.Timestamptz()
|
||||
@@index([status, leaseUntil])
|
||||
@@index([status, notBefore, createdAt], map: "prepare_workflow_outbox_due_idx")
|
||||
@@map("prepare_workflow_outbox")
|
||||
}
|
||||
@@ -4,7 +4,7 @@ import {
|
||||
Injectable,
|
||||
NotFoundException,
|
||||
} from '@nestjs/common';
|
||||
import { IsObject, IsUUID } from 'class-validator';
|
||||
import { IsInt, Min, IsNotEmpty, IsString, IsObject, IsUUID } from 'class-validator';
|
||||
import { createHash } from 'crypto';
|
||||
import {
|
||||
PrepareWorkflowService,
|
||||
@@ -16,6 +16,10 @@ import { EDITORIAL_PILOT_KIND } from './editorial-bridge';
|
||||
export class EditorialClaimDto {
|
||||
@IsUUID() taskId!: string;
|
||||
}
|
||||
export class EditorialQueueDto extends EditorialClaimDto {
|
||||
@IsInt() @Min(1) expectedVersion!: number;
|
||||
@IsString() @IsNotEmpty() idempotencyKey!: string;
|
||||
}
|
||||
export class EditorialResultDto extends EditorialClaimDto {
|
||||
@IsUUID() runId!: string;
|
||||
@IsUUID() leaseToken!: string;
|
||||
@@ -266,6 +270,18 @@ export class EditorialExecutionService {
|
||||
};
|
||||
});
|
||||
}
|
||||
async queue(a: WorkflowIdentity, dto: EditorialQueueDto) {
|
||||
this.pmo(a);
|
||||
return this.db.$transaction(async (tx:any)=>{
|
||||
await this.lock(tx,dto.taskId,true);
|
||||
const core=new PrepareWorkflowService(new Proxy(tx, {get(target,key){
|
||||
if(key==='$transaction') return (fn:any)=>fn(tx);
|
||||
const value=target[key]; return typeof value==='function'?value.bind(target):value;
|
||||
}}),{workerIds:['pmo'],stageCiIds:[]});
|
||||
const t=await core.enqueueApproved(a,dto);
|
||||
return {taskId:t.id,state:t.state,version:t.version};
|
||||
});
|
||||
}
|
||||
async claim(a: WorkflowIdentity, dto: EditorialClaimDto) {
|
||||
this.pmo(a);
|
||||
return this.db.$transaction(async (tx: any) => {
|
||||
|
||||
@@ -33,7 +33,7 @@ import {
|
||||
} from "../orchestration/prepare-workflow.service";
|
||||
|
||||
import { EditorialProposalBridge, EditorialProposalDto, isEditorialProposal } from "./editorial-bridge";
|
||||
import { EditorialExecutionService, EditorialClaimDto, EditorialResultDto } from './editorial-execution';
|
||||
import { EditorialExecutionService, EditorialClaimDto, EditorialQueueDto, EditorialResultDto } from './editorial-execution';
|
||||
import { admittedTechTask, TECH_PILOT_KIND, techArtifact, validateTechSource } from "./tech-pilot-validator";
|
||||
|
||||
// This is reviewed static configuration, not a URL fetched by the transport.
|
||||
@@ -324,6 +324,9 @@ class WorkerTransportController {
|
||||
@Get('editorial-proposals/:taskId') editorialStatus(@Req() req: any, @Param('taskId') taskId: string) {
|
||||
return this.execution.status(req.workerIdentity, taskId);
|
||||
}
|
||||
@Post('editorial-queue') editorialQueue(@Req() req: any, @Body() dto: EditorialQueueDto) {
|
||||
return this.execution.queue(req.workerIdentity, dto);
|
||||
}
|
||||
@Post('editorial-claim') editorialClaim(@Req() req: any, @Body() dto: EditorialClaimDto) {
|
||||
return this.execution.claim(req.workerIdentity, dto);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,59 @@
|
||||
import { PrismaClient } from '@prisma/client';
|
||||
import { PrepareWorkflowService } from './prepare-workflow.service';
|
||||
const enabled=process.env.WORKFLOW_DB==='1';
|
||||
(enabled?describe:describe.skip)('approval queue PostgreSQL atomicity',()=>{
|
||||
const db=new PrismaClient();
|
||||
const service=new PrepareWorkflowService(db,{workerIds:['pmo','worker'],stageCiIds:[]});
|
||||
const owner={id:'pmo',role:'Admin'}, reviewer={id:'reviewer',role:'SUPER_ADMIN'},worker={id:'worker',role:'WORKFLOW_WORKER'};
|
||||
async function pending(){const t=await service.create({key:`queue-${crypto.randomUUID()}`,title:'QA',department:'CONTENT',operationMode:'PREPARE',budgetLimitCents:20,metadata:{scope:'Isolated QA',acceptanceCriteria:['Test queue'],commitSha:'a'.repeat(40)}},owner);return service.transition(t.id,{toState:'GATE1_PENDING',expectedVersion:t.version,idempotencyKey:`submit-${t.id}`},owner);}
|
||||
afterAll(()=>db.$disconnect());
|
||||
it('racing approvals serialize once, replay once, and queue can immediately claim',async()=>{
|
||||
const t=await pending();const dto={decision:'APPROVED',expectedVersion:t.version,idempotencyKey:`approve-${t.id}`};
|
||||
const results=await Promise.all([service.decide(t.id,1,dto,reviewer),service.decide(t.id,1,dto,reviewer)]);
|
||||
expect(results.every(t=>t.state==='READY')).toBe(true);
|
||||
expect(await db.orchestrationApprovalDecision.count({where:{taskId:t.id}})).toBe(1);
|
||||
expect(await db.prepareWorkflowOutbox.count({where:{taskId:t.id}})).toBe(1);
|
||||
const claims=await Promise.all([service.claim(worker,t.id),service.claim(worker,t.id)]);
|
||||
expect(claims.filter(Boolean)).toHaveLength(1);
|
||||
});
|
||||
it('scheduled work cannot claim early via targeted or polling claim; becomes claimable at due time',async()=>{
|
||||
const t=await pending();const notBefore=new Date(Date.now()+3600000).toISOString();
|
||||
await service.decide(t.id,1,{decision:'APPROVED',expectedVersion:t.version,idempotencyKey:`schedule-${t.id}`,notBefore},reviewer);
|
||||
expect(await service.claim(worker,t.id)).toBeNull();
|
||||
expect(await service.claim(worker)).toBeNull();
|
||||
const o=await db.prepareWorkflowOutbox.findUniqueOrThrow({where:{taskId:t.id}});
|
||||
expect(o.notBefore).toEqual(new Date(notBefore));
|
||||
const approval=await db.orchestrationApprovalDecision.findFirstOrThrow({where:{taskId:t.id}});
|
||||
expect(approval.expiresAt.getTime()).toBeGreaterThan(Date.parse(notBefore));
|
||||
await db.prepareWorkflowOutbox.update({where:{taskId:t.id},data:{notBefore:new Date(0)}});
|
||||
expect(await service.claim(worker,t.id)).toMatchObject({taskId:t.id,attempt:1});
|
||||
});
|
||||
it('hold and rejection never queue',async()=>{
|
||||
const t=await pending();const held=await service.transition(t.id,{toState:'GATE1_HELD',expectedVersion:t.version,idempotencyKey:`hold-${t.id}`,reason:'Revise scope'},reviewer);
|
||||
expect(await db.prepareWorkflowOutbox.count({where:{taskId:t.id}})).toBe(0);
|
||||
await service.decide(t.id,1,{decision:'REJECTED',expectedVersion:held.version,idempotencyKey:`reject-${t.id}`},reviewer);
|
||||
expect(await db.prepareWorkflowOutbox.count({where:{taskId:t.id}})).toBe(0);
|
||||
});
|
||||
it('expired approval cannot claim and immutable decision is not edited',async()=>{
|
||||
const t=await pending();await service.decide(t.id,1,{decision:'APPROVED',expectedVersion:t.version,idempotencyKey:`expire-${t.id}`,expiresAt:new Date(Date.now()+300).toISOString()},reviewer);
|
||||
await new Promise(r=>setTimeout(r,350));
|
||||
await expect(service.claim(worker,t.id)).rejects.toThrow(/expired/i);
|
||||
expect(await db.orchestrationRun.count({where:{taskId:t.id}})).toBe(0);
|
||||
});
|
||||
it('PMO recovery queues stored v3 approval without creating a decision, no impersonation',async()=>{
|
||||
const t=await pending();
|
||||
await db.orchestrationTask.update({where:{id:t.id},data:{state:'GATE1_APPROVED',version:3}});
|
||||
await db.orchestrationApprovalDecision.create({data:{taskId:t.id,gate:1,decision:'APPROVED',actorId:'independent',expiresAt:new Date(Date.now()+60000),idempotencyKey:`legacy-${t.id}`}});
|
||||
const dto={taskId:t.id,expectedVersion:3,idempotencyKey:`recover-${t.id}`};
|
||||
const result=await Promise.all([service.enqueueApproved({id:'pmo',role:'WORKFLOW_WORKER'},dto),service.enqueueApproved({id:'pmo',role:'WORKFLOW_WORKER'},dto)]);
|
||||
expect(result.every(t=>t.state==='READY'&&t.version===4)).toBe(true);
|
||||
expect(await db.orchestrationApprovalDecision.count({where:{taskId:t.id}})).toBe(1);
|
||||
expect(await db.prepareWorkflowOutbox.count({where:{taskId:t.id}})).toBe(1);
|
||||
});
|
||||
it('outbox failure rolls back immutable approval and READY together',async()=>{
|
||||
const t=await pending();await db.prepareWorkflowOutbox.create({data:{taskId:t.id}});
|
||||
await expect(service.decide(t.id,1,{decision:'APPROVED',expectedVersion:t.version,idempotencyKey:`fail-${t.id}`},reviewer)).rejects.toThrow();
|
||||
expect((await db.orchestrationTask.findUniqueOrThrow({where:{id:t.id}})).state).toBe('GATE1_PENDING');
|
||||
expect(await db.orchestrationApprovalDecision.count({where:{taskId:t.id}})).toBe(0);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,47 @@
|
||||
import { PrepareWorkflowService } from './prepare-workflow.service';
|
||||
|
||||
describe('atomic approval queue contract', () => {
|
||||
let t:any, tx:any, service:PrepareWorkflowService;
|
||||
const actor={id:'reviewer',role:'SUPER_ADMIN'};
|
||||
const command={decision:'APPROVED',expectedVersion:2,idempotencyKey:'approve'};
|
||||
beforeEach(()=>{
|
||||
t={id:'t',state:'GATE1_PENDING',version:2,operationMode:'PREPARE',metadata:{workflowScope:'STAGE_PREPARE',createdBy:'pmo'}};
|
||||
tx={$queryRaw:jest.fn().mockResolvedValue([]),orchestrationTask:{findUnique:jest.fn(async()=>t),update:jest.fn(async({data}:any)=>(t={...t,...data,version:t.version+1}))},orchestrationEvent:{findUnique:jest.fn(),create:jest.fn()},orchestrationApprovalDecision:{create:jest.fn()},prepareWorkflowOutbox:{create:jest.fn()}};
|
||||
service=new PrepareWorkflowService({$transaction:async(fn:any)=>fn(tx)},{workerIds:['pmo'],stageCiIds:[]});
|
||||
});
|
||||
it('approval with no date enters READY and creates exactly one outbox atomically',async()=>{
|
||||
await expect(service.decide('t',1,command,actor)).resolves.toMatchObject({state:'READY',version:3});
|
||||
expect(tx.prepareWorkflowOutbox.create).toHaveBeenCalledTimes(1);
|
||||
expect(tx.orchestrationApprovalDecision.create).toHaveBeenCalledWith({data:expect.objectContaining({actorId:'reviewer',expiresAt:expect.any(Date),notes:expect.any(String)})});
|
||||
tx.orchestrationEvent.findUnique.mockResolvedValue(tx.orchestrationEvent.create.mock.calls[0][0].data);
|
||||
await service.decide('t',1,command,actor);
|
||||
expect(tx.prepareWorkflowOutbox.create).toHaveBeenCalledTimes(1);
|
||||
expect(tx.orchestrationApprovalDecision.create).toHaveBeenCalledTimes(1);
|
||||
await expect(service.decide('t',1,{...command,notes:'changed'},actor)).rejects.toMatchObject({status:409});
|
||||
});
|
||||
it('future schedule is durable and server expiry extends beyond it',async()=>{
|
||||
const notBefore=new Date(Date.now()+7*86400000).toISOString();
|
||||
await service.decide('t',1,{...command,notBefore},actor);
|
||||
const data=tx.prepareWorkflowOutbox.create.mock.calls[0][0].data;
|
||||
expect(data.notBefore).toEqual(new Date(notBefore));
|
||||
expect(tx.orchestrationApprovalDecision.create.mock.calls[0][0].data.expiresAt.getTime()).toBeGreaterThan(Date.parse(notBefore));
|
||||
});
|
||||
it('rejected decision never queues',async()=>{
|
||||
await expect(service.decide('t',1,{...command,decision:'REJECTED'},actor)).resolves.toMatchObject({state:'REJECTED'});
|
||||
expect(tx.prepareWorkflowOutbox.create).not.toHaveBeenCalled();
|
||||
});
|
||||
it.each([new Date(0).toISOString(),new Date(Date.now()+31*86400000).toISOString(),'bad'])('denies invalid schedule %s',async notBefore=>{
|
||||
await expect(service.decide('t',1,{...command,notBefore},actor)).rejects.toMatchObject({status:400});
|
||||
expect(tx.orchestrationApprovalDecision.create).not.toHaveBeenCalled();
|
||||
});
|
||||
it('legacy explicit expiry remains an expiry, never a schedule',async()=>{
|
||||
const expiresAt=new Date(Date.now()+60000).toISOString();
|
||||
await service.decide('t',1,{...command,expiresAt},actor);
|
||||
expect(tx.orchestrationApprovalDecision.create.mock.calls[0][0].data.expiresAt).toEqual(new Date(expiresAt));
|
||||
expect(tx.prepareWorkflowOutbox.create.mock.calls[0][0].data.notBefore).toBeNull();
|
||||
});
|
||||
it('expired approval is denied with no writes',async()=>{
|
||||
await expect(service.decide('t',1,{...command,expiresAt:new Date(0).toISOString()},actor)).rejects.toMatchObject({status:400});
|
||||
expect(tx.prepareWorkflowOutbox.create).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -30,7 +30,9 @@ export class GateDecisionDto {
|
||||
@IsIn(['APPROVED', 'REJECTED']) decision!: 'APPROVED' | 'REJECTED';
|
||||
@Type(() => Number) @IsInt() @Min(1) expectedVersion!: number;
|
||||
@IsString() @IsNotEmpty() idempotencyKey!: string;
|
||||
@IsISO8601() expiresAt!: string;
|
||||
// Legacy explicit validity cap; scheduling uses notBefore instead.
|
||||
@IsOptional() @IsISO8601() expiresAt?: string;
|
||||
@IsOptional() @IsISO8601() notBefore?: string;
|
||||
@IsOptional() @IsString() notes?: string;
|
||||
@IsOptional() @IsString() runId?: string;
|
||||
@IsOptional() @IsHash('sha256') artifactHash?: string;
|
||||
|
||||
@@ -200,6 +200,7 @@ export class OrchestrationService {
|
||||
return prior;
|
||||
}
|
||||
if (
|
||||
typeof dto.expiresAt !== 'string' ||
|
||||
!Number.isFinite(Date.parse(dto.expiresAt)) ||
|
||||
new Date(dto.expiresAt) <= new Date()
|
||||
)
|
||||
|
||||
@@ -5,14 +5,14 @@ describe('owner Gate1 hold operation', () => {
|
||||
let task:any, tx:any, service:PrepareWorkflowService;
|
||||
beforeEach(()=>{
|
||||
task={id:'t',state:'GATE1_PENDING',version:2,operationMode:'PREPARE',metadata:{workflowScope:'STAGE_PREPARE',createdBy:'proposer'}};
|
||||
tx={$queryRaw:jest.fn().mockResolvedValue([]),orchestrationTask:{findUnique:jest.fn(async()=>task),update:jest.fn(async({data}:any)=>(task={...task,state:data.state,version:task.version+1}))},orchestrationEvent:{findUnique:jest.fn(),create:jest.fn()},orchestrationApprovalDecision:{create:jest.fn()}};
|
||||
tx={prepareWorkflowOutbox:{create:jest.fn()},$queryRaw:jest.fn().mockResolvedValue([]),orchestrationTask:{findUnique:jest.fn(async()=>task),update:jest.fn(async({data}:any)=>(task={...task,state:data.state,version:task.version+1}))},orchestrationEvent:{findUnique:jest.fn(),create:jest.fn()},orchestrationApprovalDecision:{create:jest.fn()}};
|
||||
service=new PrepareWorkflowService({$transaction:async(cb:any)=>cb(tx)}, {workerIds:[],stageCiIds:[],ownerReviewerIds:['12345678-1234-1234-1234-123456789012']} as any);
|
||||
});
|
||||
it('advertises and permits the pinned canonical Admin independent reviewer only',async()=>{
|
||||
const owner={id:'12345678-1234-1234-1234-123456789012',role:'Admin'};
|
||||
expect(service.project(task,owner).availableActions).toEqual(['APPROVE_GATE1','REJECT_GATE1','HOLD_GATE1']);
|
||||
await expect(service.transition('t',hold,owner)).resolves.toMatchObject({state:'GATE1_HELD'});
|
||||
await expect(service.decide('t',1,{decision:'APPROVED',expectedVersion:3,idempotencyKey:'owner-approve',expiresAt:new Date(Date.now()+60000).toISOString()},owner)).resolves.toMatchObject({state:'GATE1_APPROVED'});
|
||||
await expect(service.decide('t',1,{decision:'APPROVED',expectedVersion:3,idempotencyKey:'owner-approve',expiresAt:new Date(Date.now()+60000).toISOString()},owner)).resolves.toMatchObject({state:'READY'});
|
||||
expect(tx.orchestrationApprovalDecision.create).toHaveBeenCalledWith({data:expect.objectContaining({actorId:owner.id})});
|
||||
});
|
||||
it.each(['Admin','SUPER_ADMIN'])('never permits proposer self-review even if allowlisted: %s',async role=>{
|
||||
@@ -62,6 +62,6 @@ describe('owner Gate1 hold operation', () => {
|
||||
});
|
||||
it.each(['APPROVED','REJECTED'])('decides held proposal via existing Gate1 command: %s',async decision=>{
|
||||
task.state='GATE1_HELD';
|
||||
await expect(service.decide('t',1,{decision,expectedVersion:2,idempotencyKey:'decision',expiresAt:new Date(Date.now()+60000).toISOString()},reviewer)).resolves.toMatchObject({state:decision==='APPROVED'?'GATE1_APPROVED':'REJECTED'});
|
||||
await expect(service.decide('t',1,{decision,expectedVersion:2,idempotencyKey:'decision',expiresAt:new Date(Date.now()+60000).toISOString()},reviewer)).resolves.toMatchObject({state:decision==='APPROVED'?'READY':'REJECTED'});
|
||||
});
|
||||
});
|
||||
@@ -10,7 +10,7 @@ describe('configured owner real workflow HTTP authorization',()=>{
|
||||
const owner='12345678-1234-1234-1234-123456789012';
|
||||
let app:INestApplication,task:any,tx:any;
|
||||
beforeAll(async()=>{
|
||||
tx={$queryRaw:jest.fn(async()=>[]),orchestrationTask:{findUnique:async()=>task,update:jest.fn(async({data}:any)=>(task={...task,state:data.state,version:task.version+1}))},orchestrationEvent:{findUnique:jest.fn(),create:jest.fn()},orchestrationApprovalDecision:{create:jest.fn()}};
|
||||
tx={prepareWorkflowOutbox:{create:jest.fn()},$queryRaw:jest.fn(async()=>[]),orchestrationTask:{findUnique:async()=>task,update:jest.fn(async({data}:any)=>(task={...task,state:data.state,version:task.version+1}))},orchestrationEvent:{findUnique:jest.fn(),create:jest.fn()},orchestrationApprovalDecision:{create:jest.fn()}};
|
||||
const workflow=new PrepareWorkflowService({$transaction:async(cb:any)=>cb(tx)},{workerIds:[],stageCiIds:[],ownerReviewerIds:[owner]});
|
||||
const m=await Test.createTestingModule({controllers:[OrchestrationController],providers:[{provide:OrchestrationService,useValue:{findAll:async()=>[task],findOne:async()=>task}},{provide:PrepareWorkflowService,useValue:workflow},RolesGuard]})
|
||||
.overrideGuard(JwtAuthGuard).useValue({canActivate(c:any){const r=c.switchToHttp().getRequest();if(!r.headers.authorization)return false;r.user={sub:r.headers.authorization,role:'ADMIN'};return true;}}).compile();
|
||||
|
||||
@@ -249,15 +249,10 @@ export class PrepareWorkflowService {
|
||||
throw new ForbiddenException(
|
||||
'Gate2 decisions and release remain disabled',
|
||||
);
|
||||
if (
|
||||
!['APPROVED', 'REJECTED'].includes(dto.decision) ||
|
||||
!Number.isFinite(Date.parse(dto.expiresAt)) ||
|
||||
Date.parse(dto.expiresAt) <= Date.now() ||
|
||||
Date.parse(dto.expiresAt) > Date.now() + 86400000
|
||||
)
|
||||
throw new BadRequestException(
|
||||
'Decision requires future expiry within 24 hours',
|
||||
);
|
||||
if (!['APPROVED', 'REJECTED'].includes(dto.decision))
|
||||
throw new BadRequestException('Invalid decision');
|
||||
// Time-sensitive validation happens after replay, so a successful command
|
||||
// remains replayable after its original schedule or validity window passes.
|
||||
return this.db.$transaction(async (tx: any) => {
|
||||
await this.allowed(tx);
|
||||
const t = await this.task(tx, id);
|
||||
@@ -268,20 +263,37 @@ export class PrepareWorkflowService {
|
||||
throw new ForbiddenException('Independent reviewer required');
|
||||
if (!['GATE1_PENDING', 'GATE1_HELD'].includes(t.state))
|
||||
throw new ConflictException('Gate1 pending required');
|
||||
const now = Date.now();
|
||||
const scheduled = dto.notBefore !== undefined;
|
||||
const notBefore = scheduled ? new Date(dto.notBefore) : null;
|
||||
if (scheduled && (dto.decision !== 'APPROVED' || typeof dto.notBefore !== 'string' || !Number.isFinite(notBefore!.getTime()) || notBefore!.getTime() <= now || notBefore!.getTime() > now + 30 * 86400000))
|
||||
throw new BadRequestException('notBefore must be a future approval schedule within 30 days');
|
||||
const policyExpiry = (notBefore?.getTime() ?? now) + 86400000;
|
||||
// Legacy expiresAt is an explicit validity cap, never a scheduled start.
|
||||
const expiresAt = dto.expiresAt === undefined ? new Date(policyExpiry) : new Date(dto.expiresAt);
|
||||
if (!Number.isFinite(expiresAt.getTime()) || expiresAt.getTime() <= (notBefore?.getTime() ?? now) || expiresAt.getTime() > policyExpiry)
|
||||
throw new BadRequestException('Approval expiry must cover schedule and remain within backend validity policy');
|
||||
if (dto.notes !== undefined && (typeof dto.notes !== 'string' || dto.notes.length > 2000))
|
||||
throw new BadRequestException('Audit notes must be at most 2000 characters');
|
||||
if (dto.decision === 'APPROVED') {
|
||||
await this.allowed(tx, true);
|
||||
await tx.prepareWorkflowOutbox.create({data:{taskId:id,notBefore}});
|
||||
}
|
||||
await tx.orchestrationApprovalDecision.create({
|
||||
data: {
|
||||
taskId: id,
|
||||
gate: 1,
|
||||
decision: dto.decision,
|
||||
actorId: a.id,
|
||||
expiresAt: new Date(dto.expiresAt),
|
||||
expiresAt,
|
||||
notes: dto.notes?.trim() || `Owner ${dto.decision.toLowerCase()} Gate1`,
|
||||
idempotencyKey: `decision:${dto.idempotencyKey}`,
|
||||
},
|
||||
});
|
||||
return this.event(
|
||||
tx,
|
||||
t,
|
||||
dto.decision === 'APPROVED' ? 'GATE1_APPROVED' : 'REJECTED',
|
||||
dto.decision === 'APPROVED' ? 'READY' : 'REJECTED',
|
||||
a,
|
||||
dto.idempotencyKey,
|
||||
'PREPARE_GATE1',
|
||||
@@ -289,14 +301,32 @@ export class PrepareWorkflowService {
|
||||
);
|
||||
});
|
||||
}
|
||||
/** Recovery of a stored approval by its provisioned proposal worker, not a new decision. */
|
||||
async enqueueApproved(a: WorkflowIdentity, dto: {taskId:string;expectedVersion:number;idempotencyKey:string}) {
|
||||
this.worker(a);
|
||||
return this.db.$transaction(async (tx:any)=>{
|
||||
await this.allowed(tx,true);
|
||||
const t=await this.task(tx,dto.taskId);
|
||||
if(t.metadata.createdBy!==a.id) throw new ForbiddenException('Proposal worker required');
|
||||
const fingerprint=JSON.stringify({dto,a});
|
||||
if(await this.replay(tx,dto.idempotencyKey,fingerprint)) return t;
|
||||
this.version(t,dto);
|
||||
if(t.state!=='GATE1_APPROVED') throw new ConflictException('Stored Gate1 approval required');
|
||||
const approval=await tx.orchestrationApprovalDecision.findFirst({where:{taskId:t.id,gate:1},orderBy:{createdAt:'desc'}});
|
||||
if(!approval || approval.decision!=='APPROVED' || approval.actorId===a.id || approval.expiresAt<=new Date())
|
||||
throw new ForbiddenException('Valid independent Gate1 required');
|
||||
await tx.prepareWorkflowOutbox.create({data:{taskId:t.id}});
|
||||
return this.event(tx,t,'READY',a,dto.idempotencyKey,'PREPARE_APPROVED_QUEUED',fingerprint);
|
||||
});
|
||||
}
|
||||
async claim(a: WorkflowIdentity, id?: string): Promise<any> {
|
||||
this.worker(a);
|
||||
return this.db.$transaction(async (tx: any) => {
|
||||
await this.allowed(tx, true);
|
||||
// Lock order is task then outbox for every worker operation. SKIP LOCKED permits competing workers.
|
||||
const rows: any[] = id
|
||||
? await tx.$queryRaw`SELECT t.id FROM orchestration_tasks t JOIN prepare_workflow_outbox o ON o.task_id=t.id WHERE t.id=${id}::uuid AND o.status='PENDING' AND (o.lease_until IS NULL OR o.lease_until < now()) ORDER BY o.created_at FOR UPDATE OF t SKIP LOCKED LIMIT 1`
|
||||
: await tx.$queryRaw`SELECT t.id FROM orchestration_tasks t JOIN prepare_workflow_outbox o ON o.task_id=t.id WHERE o.status='PENDING' AND (o.lease_until IS NULL OR o.lease_until < now()) ORDER BY o.created_at FOR UPDATE OF t SKIP LOCKED LIMIT 1`;
|
||||
? await tx.$queryRaw`SELECT t.id FROM orchestration_tasks t JOIN prepare_workflow_outbox o ON o.task_id=t.id WHERE t.id=${id}::uuid AND t.state IN ('READY','RUNNING') AND o.status='PENDING' AND (o.not_before IS NULL OR o.not_before <= now()) AND (o.lease_until IS NULL OR o.lease_until < now()) ORDER BY o.created_at FOR UPDATE OF t SKIP LOCKED LIMIT 1`
|
||||
: await tx.$queryRaw`SELECT t.id FROM orchestration_tasks t JOIN prepare_workflow_outbox o ON o.task_id=t.id WHERE t.state IN ('READY','RUNNING') AND o.status='PENDING' AND (o.not_before IS NULL OR o.not_before <= now()) AND (o.lease_until IS NULL OR o.lease_until < now()) ORDER BY o.created_at FOR UPDATE OF t SKIP LOCKED LIMIT 1`;
|
||||
if (!rows.length) return null;
|
||||
const t = await this.task(tx, rows[0].id);
|
||||
const o = await tx.prepareWorkflowOutbox.findUnique({
|
||||
|
||||
@@ -56,15 +56,7 @@ const enabled = process.env.WORKFLOW_DB === '1';
|
||||
},
|
||||
reviewer,
|
||||
);
|
||||
return service.transition(
|
||||
t.id,
|
||||
{
|
||||
toState: WorkflowState.READY,
|
||||
expectedVersion: t.version,
|
||||
idempotencyKey: `ready-${t.id}`,
|
||||
},
|
||||
admin,
|
||||
);
|
||||
return t;
|
||||
}
|
||||
afterAll(async () => {
|
||||
await db.$disconnect();
|
||||
@@ -87,7 +79,7 @@ const enabled = process.env.WORKFLOW_DB === '1';
|
||||
expect(JSON.stringify(events[0].payload)).toContain('Need source review');
|
||||
expect(await db.orchestrationApprovalDecision.count({where:{taskId:t.id}})).toBe(0);
|
||||
expect(await db.prepareWorkflowOutbox.count({where:{taskId:t.id}})).toBe(0);
|
||||
await expect(service.transition(t.id, {...command,expectedVersion:held.version,reason:'changed'},reviewer)).rejects.toMatchObject({status:409});
|
||||
await expect(service.transition(t.id, {...command,idempotencyKey:events[0].idempotencyKey,expectedVersion:held.version,reason:'changed'},reviewer)).rejects.toMatchObject({status:409});
|
||||
const decision = await service.decide(t.id,1,{decision:'REJECTED',expectedVersion:held.version,idempotencyKey:`decline-held-${t.id}`,expiresAt:new Date(Date.now()+60000).toISOString()},reviewer);
|
||||
expect(decision.state).toBe('REJECTED');
|
||||
});
|
||||
@@ -340,15 +332,7 @@ const enabled = process.env.WORKFLOW_DB === '1';
|
||||
},
|
||||
reviewer,
|
||||
);
|
||||
await service.transition(
|
||||
t.id,
|
||||
{
|
||||
toState: 'READY',
|
||||
expectedVersion: current.version,
|
||||
idempotencyKey: `q-${t.id}`,
|
||||
},
|
||||
admin,
|
||||
);
|
||||
expect(current.state).toBe('READY');
|
||||
const first = await service.claim(worker, t.id);
|
||||
for (let i = 0; i < 2; i++) {
|
||||
await db.prepareWorkflowOutbox.update({
|
||||
@@ -462,20 +446,7 @@ const enabled = process.env.WORKFLOW_DB === '1';
|
||||
},
|
||||
reviewer,
|
||||
);
|
||||
const results = await Promise.allSettled(
|
||||
['one', 'two'].map((k) =>
|
||||
service.transition(
|
||||
t.id,
|
||||
{
|
||||
toState: 'READY',
|
||||
expectedVersion: t.version,
|
||||
idempotencyKey: `${k}-${t.id}`,
|
||||
},
|
||||
admin,
|
||||
),
|
||||
),
|
||||
);
|
||||
expect(results.filter((r) => r.status === 'fulfilled')).toHaveLength(1);
|
||||
expect(t.state).toBe('READY');
|
||||
expect(
|
||||
await db.prepareWorkflowOutbox.count({ where: { taskId: t.id } }),
|
||||
).toBe(1);
|
||||
|
||||
@@ -30,20 +30,35 @@ describe('command center', () => {
|
||||
expect(await screen.findByText('محدوده و معیارهای پذیرش در دسترس نیست؛ سرور این اطلاعات را در نمای امن ارسال نکرده است.')).toBeInTheDocument();
|
||||
expect(screen.queryByText('Immutable scope / acceptance criteria')).not.toBeInTheDocument();
|
||||
});
|
||||
it('requires explicit confirmation and expiry; retains idempotency across failed retry', async () => {
|
||||
it('approves immediately without date or reason after explicit confirmation',async()=>{
|
||||
vi.mocked(api.post).mockResolvedValue({data:{...task,state:'READY'}});
|
||||
render(<OrchestrationCommandCenter/>);
|
||||
fireEvent.click(await screen.findByRole('button',{name:'Bounded SEO audit'}));
|
||||
fireEvent.click(await screen.findByRole('button',{name:'تأیید در Gate1'}));
|
||||
const dialog=screen.getByRole('dialog');
|
||||
expect(within(dialog).getByLabelText('زمان اجرای برنامهریزیشده')).toHaveValue('');
|
||||
expect(within(dialog).queryByLabelText('زمان انقضا')).not.toBeInTheDocument();
|
||||
fireEvent.click(within(dialog).getByLabelText('محدوده و اتصال تغییرناپذیر را بررسی کردم'));
|
||||
fireEvent.click(within(dialog).getByRole('button',{name:'تأیید فرمان'}));
|
||||
await waitFor(()=>expect(api.post).toHaveBeenCalledTimes(1));
|
||||
const payload=vi.mocked(api.post).mock.calls[0][1] as Record<string,unknown>;
|
||||
expect(payload.decision).toBe('APPROVED');
|
||||
expect(payload.expiresAt).toBeUndefined();expect(payload.notBefore).toBeUndefined();
|
||||
});
|
||||
it('requires explicit confirmation; retains idempotency across failed retry', async () => {
|
||||
vi.mocked(api.post).mockRejectedValue(new Error('network failure'));
|
||||
render(<OrchestrationCommandCenter />);
|
||||
fireEvent.click(await screen.findByRole('button',{name:'Bounded SEO audit'}));
|
||||
fireEvent.click(await screen.findByRole('button',{name:'تأیید در Gate1'}));
|
||||
const dialog=screen.getByRole('dialog');
|
||||
expect(within(dialog).getByRole('button',{name:'تأیید فرمان'})).toBeDisabled();
|
||||
fireEvent.change(within(dialog).getByLabelText('زمان انقضا'),{target:{value:'2099-01-01T12:00'}});
|
||||
fireEvent.change(within(dialog).getByLabelText('زمان اجرای برنامهریزیشده'),{target:{value:new Date(Date.now()+3600000).toISOString().slice(0,16)}});
|
||||
fireEvent.change(within(dialog).getByLabelText('دلیل'),{target:{value:'محدوده checked'}});
|
||||
fireEvent.click(within(dialog).getByLabelText('محدوده و اتصال تغییرناپذیر را بررسی کردم'));
|
||||
fireEvent.click(within(dialog).getByRole('button',{name:'تأیید فرمان'}));
|
||||
expect(await within(dialog).findByRole('alert')).toHaveTextContent('network failure');
|
||||
expect(within(dialog).getByLabelText('دلیل')).toBeDisabled();
|
||||
expect(within(dialog).getByLabelText('زمان انقضا')).toBeDisabled();
|
||||
expect(within(dialog).getByLabelText('زمان اجرای برنامهریزیشده')).toBeDisabled();
|
||||
fireEvent.click(within(dialog).getByRole('button',{name:'تأیید فرمان'}));
|
||||
await within(dialog).findByRole('alert');
|
||||
expect(api.post).toHaveBeenCalledTimes(2);
|
||||
|
||||
@@ -66,7 +66,7 @@ export default function OrchestrationCommandCenter() {
|
||||
if(!pending||busy||!confirmed||blocked||!capabilities.mutations)return;
|
||||
setAttempted(true);setBusy(true);setCommandError('');
|
||||
try { const [runId,artifactHash]=binding.split('|');
|
||||
await orchestration.command(pending.task,pending.action,{idempotencyKey:pending.key,reason,expiresAt:expires?new Date(expires).toISOString():undefined,runId,artifactHash});
|
||||
await orchestration.command(pending.task,pending.action,{idempotencyKey:pending.key,reason,notBefore:expires?new Date(expires).toISOString():undefined,runId,artifactHash});
|
||||
setPending(null);setSelected(null);setSuccess('فرمان توسط سرور پذیرفته شد؛ وضعیت مرجع بازخوانی شد.');await loadTasks();
|
||||
} catch(e) {const failure=commandFailure(e);setBlocked(!failure.retryable);setCommandError(failure.message)} finally {setBusy(false)}
|
||||
};
|
||||
@@ -105,11 +105,12 @@ export default function OrchestrationCommandCenter() {
|
||||
{section==='گفتوگو'&&!capabilities.chat&&<p>گفتوگو در دسترس نیست؛ پل مورد اعتماد اجرا فعال نیست.</p>}
|
||||
{section==='گفتوگو'&&capabilities.chat&&<section className="bg-white border rounded-2xl p-5 space-y-3"><h2>گفتوگوی متنی از مسیر امن سرور</h2><p>فقط متن مشورتی؛ بدون ترمینال، PTY، توکن اجرا یا اتصال مستقیم به Hermes. تصمیمها از کنترل تأیید مستقل عبور میکنند.</p><label>شناسه نشست<input value={session} onChange={e=>{setSession(e.target.value);setMessages([])}} className="border p-2"/></label><div role="log">{messages.map(m=><p key={m.id}>{m.role}: {m.content}</p>)}</div><label>پیام<textarea value={message} onChange={e=>{setMessage(e.target.value);setChatKey(crypto.randomUUID())}} className="border p-2 block w-full"/></label><Button disabled={busy||!session.trim()||!message.trim()} onClick={async()=>{setBusy(true);setError('');try{const reply=await orchestration.chat(session,message,chatKey);setMessages(prev=>[...prev,{id:chatKey,role:'user',content:message},reply]);setMessage('');setChatKey(crypto.randomUUID())}catch(e){setError(errorText(e))}finally{setBusy(false)}}}>ارسال پیام مشورتی</Button></section>}
|
||||
{pending&&<div role="dialog" aria-modal="true" aria-label="تأیید فرمان تغییرناپذیر" className="fixed inset-0 z-50 bg-black/50 overflow-y-auto p-8"><div className="bg-white rounded-2xl p-6 max-w-2xl mx-auto space-y-4"><h2>{actionLabel(pending.action)}</h2><Json value={{taskId:pending.task.id,expectedVersion:pending.task.version,idempotencyKey:pending.key,scope:pending.task.metadata,budgetLimitCents:pending.task.budgetLimitCents,operationMode:pending.task.operationMode}}/>
|
||||
{pending.action.gate&&<label className="block">زمان انقضا<input aria-label="زمان انقضا" type="datetime-local" value={expires} onChange={e=>setExpires(e.target.value)} disabled={busy||attempted}/></label>}
|
||||
{pending.action.gate===1&&pending.action.decision==='APPROVED'&&<label className="block">زمان اجرای برنامهریزیشده (اختیاری؛ پیشفرض فوری)<input aria-label="زمان اجرای برنامهریزیشده" type="datetime-local" value={expires} onChange={e=>setExpires(e.target.value)} disabled={busy||attempted}/></label>}
|
||||
{pending.action.gate===2&&pending.action.decision==='APPROVED'&&<label>اتصال اجرای ثبتشده به خروجی<select aria-label="اتصال اجرای ثبتشده به خروجی" value={binding} onChange={e=>{setBinding(e.target.value);setConfirmed(false)}} disabled={busy||attempted}><option value="">انتخاب شواهد تغییرناپذیر</option>{pending.task.runs?.filter(r=>r.status==='SUCCEEDED').flatMap(r=>r.artifacts.map(a=><option key={r.id+'|'+a.sha256} value={r.id+'|'+a.sha256}>{r.id} · {r.commitSha} · {r.environment} · {a.sha256}</option>))}</select></label>}
|
||||
<label className="block">دلیل<input aria-label="دلیل" value={reason} onChange={e=>setReason(e.target.value)} disabled={busy||attempted} className="border p-2"/></label>
|
||||
<p>تأیید، وظیفه را در اولویت صف آماده قرار میدهد؛ زمان اختیاری فقط زودترین زمان شروع است و اجرای دقیق در آن لحظه تضمین نمیشود. حداکثر زمانبندی ۳۰ روز است؛ اعتبار تأیید را سرور مدیریت میکند.</p>
|
||||
<label className="block">دلیل (برای تأیید اختیاری)<input aria-label="دلیل" value={reason} onChange={e=>setReason(e.target.value)} disabled={busy||attempted} className="border p-2"/></label>
|
||||
<label className="block"><input type="checkbox" checked={confirmed} onChange={e=>setConfirmed(e.target.checked)} disabled={busy}/>محدوده و اتصال تغییرناپذیر را بررسی کردم</label>
|
||||
{commandError&&<p role="alert">{commandError}</p>}<div className="flex gap-3"><Button disabled={busy||blocked||!capabilities.mutations||!confirmed||!reason.trim()|| (!!pending.action.gate&&!(Date.parse(expires)>now)) || (['APPROVE_GATE1','REJECT_GATE1'].includes(pending.action.id)&&Date.parse(expires)>now+86400000) || (pending.action.gate===2&&pending.action.decision==='APPROVED'&&!binding)} onClick={()=>void execute()}>تأیید فرمان</Button><Button variant="outline" disabled={busy} onClick={()=>setPending(null)}>انصراف</Button></div>
|
||||
{commandError&&<p role="alert">{commandError}</p>}<div className="flex gap-3"><Button disabled={busy||blocked||!capabilities.mutations||!confirmed||(pending.action.toState==='GATE1_HELD'&&!reason.trim())|| (!!expires&&!(Date.parse(expires)>now)) || (['APPROVE_GATE1','REJECT_GATE1'].includes(pending.action.id)&&Date.parse(expires)>now+30*86400000) || (pending.action.gate===2&&pending.action.decision==='APPROVED'&&!binding)} onClick={()=>void execute()}>تأیید فرمان</Button><Button variant="outline" disabled={busy} onClick={()=>setPending(null)}>انصراف</Button></div>
|
||||
</div></div>}
|
||||
</div>;
|
||||
}
|
||||
@@ -35,9 +35,9 @@ export function normalizeActions(value:unknown):ServerAction[] {
|
||||
export interface Artifact { id: string; sha256: string; kind?: string; uri?: string }
|
||||
export interface Run { id: string; commitSha: string; environment: string; status: string; artifacts: Artifact[] }
|
||||
export interface Task { id: string; key: string; title: string; department: string; state: string; version: number; operationMode: string; budgetLimitCents: number; metadata?: Record<string, unknown>; availableActions: ServerAction[]; runs?: Run[]; decisions?: Record<string, unknown>[]; events?: Record<string, unknown>[]; budgetLedger?: Record<string, unknown>[] }
|
||||
export interface CommandInput { idempotencyKey: string; reason?: string; expiresAt?: string; runId?: string; artifactHash?: string }
|
||||
export interface CommandInput { idempotencyKey: string; reason?: string; expiresAt?: string; notBefore?: string; runId?: string; artifactHash?: string }
|
||||
export interface TransitionDto {toState:string;expectedVersion:number;idempotencyKey:string;reason?:string}
|
||||
export interface GateDecisionDto {decision:'APPROVED'|'REJECTED';expectedVersion:number;idempotencyKey:string;expiresAt:string;notes?:string;runId?:string;artifactHash?:string;commitSha?:string;environment?:string}
|
||||
export interface GateDecisionDto {decision:'APPROVED'|'REJECTED';expectedVersion:number;idempotencyKey:string;expiresAt?:string;notBefore?:string;notes?:string;runId?:string;artifactHash?:string;commitSha?:string;environment?:string}
|
||||
export function actionBlockReason(task:Task,action:ServerAction):string|undefined {
|
||||
if(action.gate===2)return 'Gate2 و انتشار در این پنل غیرفعال هستند';
|
||||
if(action.disabledReason)return 'اقدام توسط سرور غیرفعال شده است: '+action.disabledReason;
|
||||
@@ -102,8 +102,9 @@ export const orchestration = {
|
||||
if (!input.idempotencyKey || !Number.isInteger(task.version)) throw new Error('نسخه و کلید جلوگیری از تکرار الزامی است');
|
||||
const base = '/admin/orchestration/tasks/' + encodeURIComponent(task.id);
|
||||
if (action.gate && action.decision) {
|
||||
if (!input.expiresAt || !(Date.parse(input.expiresAt) > Date.now())) throw new Error('زمان انقضا باید در آینده باشد');
|
||||
if(['APPROVE_GATE1','REJECT_GATE1'].includes(action.id)&&Date.parse(input.expiresAt)>Date.now()+86400000)throw new Error('انقضای Gate1 باید در ۲۴ ساعت آینده باشد');
|
||||
if (input.notBefore && (action.decision!=='APPROVED' || !(Date.parse(input.notBefore)>Date.now()) || Date.parse(input.notBefore)>Date.now()+30*86400000)) throw new Error('زمان اجرا باید در آینده و حداکثر ۳۰ روز باشد');
|
||||
if (input.expiresAt && !(Date.parse(input.expiresAt)>Date.now())) throw new Error('زمان انقضا باید در آینده باشد');
|
||||
if(input.expiresAt && Date.parse(input.expiresAt)>(input.notBefore?Date.parse(input.notBefore):Date.now())+86400000) throw new Error('انقضای Gate1 باید در ۲۴ ساعت پس از زمان اجرا باشد');
|
||||
let binding = {};
|
||||
if (action.gate === 2 && action.decision === 'APPROVED') {
|
||||
const run = task.runs?.find(r => r.id === input.runId);
|
||||
@@ -111,7 +112,7 @@ export const orchestration = {
|
||||
if (!run?.commitSha || !run.environment || !run.artifacts.some(a => a.sha256 === input.artifactHash)) throw new Error('Registered immutable Gate2 binding required');
|
||||
binding = {runId:run.id, artifactHash:input.artifactHash, commitSha:run.commitSha, environment:run.environment};
|
||||
}
|
||||
return (await api.post(base + `/gates/${action.gate}/decide`, {expectedVersion:task.version,idempotencyKey:input.idempotencyKey,decision:action.decision,expiresAt:input.expiresAt,notes:input.reason,...binding})).data;
|
||||
return (await api.post(base + `/gates/${action.gate}/decide`, {expectedVersion:task.version,idempotencyKey:input.idempotencyKey,decision:action.decision,...(input.expiresAt?{expiresAt:input.expiresAt}:{}),...(input.notBefore?{notBefore:input.notBefore}:{}),notes:input.reason,...binding})).data;
|
||||
}
|
||||
if (!action.toState) throw new Error('اقدام سرور پشتیبانی نمیشود');
|
||||
return (await api.put(base + '/transition', {toState:action.toState,expectedVersion:task.version,idempotencyKey:input.idempotencyKey,reason:input.reason})).data;
|
||||
|
||||
Reference in new issue
Block a user