fix(editorial): migrate exact CI ownership and fence failed research runs
Deploy Canina / deploy (push) Successful in 5m1s
Deploy Canina / deploy (push) Successful in 5m1s
This commit is contained in:
1 parent
01b6f1d3d7
commit
f4add07780
10 files changed
+311
-11
No files matched your search
@@ -8,6 +8,8 @@ import {
|
||||
IsNotEmpty,
|
||||
IsOptional,
|
||||
IsString,
|
||||
IsUUID,
|
||||
MaxLength,
|
||||
Max,
|
||||
Min,
|
||||
} from 'class-validator';
|
||||
@@ -39,6 +41,12 @@ export class GateDecisionDto {
|
||||
@IsOptional() @IsString() commitSha?: string;
|
||||
@IsOptional() @IsString() environment?: string;
|
||||
}
|
||||
export class EditorialFailureDto {
|
||||
@IsUUID() runId!: string;
|
||||
@IsInt() @Min(1) expectedVersion!: number;
|
||||
@IsString() @IsNotEmpty() @MaxLength(120) idempotencyKey!: string;
|
||||
@IsString() @IsNotEmpty() @MaxLength(500) reason!: string;
|
||||
}
|
||||
export class RecordBudgetDto {
|
||||
@Type(() => Number) @IsInt() @Min(1) amountCents!: number;
|
||||
@IsString() @IsNotEmpty() idempotencyKey!: string;
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
import { PrepareWorkflowService } from './prepare-workflow.service';
|
||||
import { randomUUID } from 'crypto';
|
||||
|
||||
describe('editorial failure recovery fencing',()=>{
|
||||
const actor={id:'owner',role:'SUPER_ADMIN'},worker={id:'pmo',role:'WORKFLOW_WORKER'};
|
||||
let db:any,core:PrepareWorkflowService,t:any,o:any,r:any,events:any[];
|
||||
beforeEach(()=>{
|
||||
t={id:randomUUID(),key:'editorial-v1:'+'a'.repeat(64),title:'Editorial research: fixture',department:'CONTENT',operationMode:'PREPARE',budgetLimitCents:20,state:'RUNNING',version:6,metadata:{createdBy:'pmo',workflowScope:'STAGE_PREPARE',environment:'staging',commitSha:'a'.repeat(40),scope:'editorial-research-to-brief-v1\nTopic: fixture\nResearch and evidence-backed brief only. No publication, release, production changes or execution authorization.',acceptanceCriteria:['evidence']}};
|
||||
r={id:randomUUID(),taskId:t.id,status:'RUNNING',commitSha:t.metadata.commitSha,environment:'staging'};
|
||||
o={id:randomUUID(),taskId:t.id,status:'PENDING',runId:r.id,leaseOwner:'pmo',leaseToken:randomUUID(),leaseUntil:new Date(Date.now()-1000),reservationId:'reserve'};events=[];
|
||||
const tx:any={$queryRaw:async()=>[],orchestrationTask:{findUnique:async()=>t,update:async({data}:any)=>(Object.assign(t,{state:data.state,version:t.version+1}),{...t})},prepareWorkflowOutbox:{findUnique:async()=>o,update:async({data}:any)=>Object.assign(o,data)},orchestrationRun:{findUnique:async()=>r,update:async({data}:any)=>Object.assign(r,data)},orchestrationEvent:{findUnique:async({where}:any)=>events.find(e=>e.idempotencyKey===where.idempotencyKey),findFirst:async()=>({}),create:async({data}:any)=>events.push(data)},orchestrationBudgetLedger:{findUnique:async()=>({id:'reserve',taskId:t.id,kind:'RESERVE',actorId:'pmo',amountCents:20,idempotencyKey:'prepare-reserve:'+r.id}),aggregate:async()=>({_sum:{amountCents:20}}),findFirst:async()=>null},orchestrationArtifact:{count:async()=>0}};
|
||||
let tail=Promise.resolve();db={$transaction:(fn:any)=>{const p=tail.then(()=>fn(tx));tail=p.catch(()=>{});return p;}};core=new PrepareWorkflowService(db,{workerIds:['pmo'],stageCiIds:[]});
|
||||
});
|
||||
const dto=()=>({runId:r.id,expectedVersion:6,idempotencyKey:'recover:'+r.id,reason:'local process ended; cost unknown'});
|
||||
it('terminalizes expired run without QA/cost writes and replays concurrently',async()=>{
|
||||
const d=dto();const results=await Promise.all([core.editorialFailure(t.id,d,actor),core.editorialFailure(t.id,d,actor)]);
|
||||
expect(results.map(x=>x.state)).toEqual(['FAILED','FAILED']);expect(t.version).toBe(7);expect(r.status).toBe('FAILED');expect(o.status).toBe('DONE');expect(o.leaseToken).toBeNull();expect(o.reservationId).toBe('reserve');expect(events).toHaveLength(1);expect(events[0].payload.retainedReservationCents).toBe(20);
|
||||
});
|
||||
it('rejects wrong run, stale version, unprivileged actor, active recovery and wrong held token',async()=>{
|
||||
for(const [d,a] of [[{...dto(),runId:randomUUID()},actor],[{...dto(),expectedVersion:5},actor],[dto(),{id:'other',role:'Admin'}],[dto(),worker]] as any)await expect(core.editorialFailure(t.id,d,a)).rejects.toThrow();
|
||||
o.leaseUntil=new Date(Date.now()+30000);
|
||||
await expect(core.editorialFailure(t.id,dto(),actor)).rejects.toThrow();
|
||||
await expect(core.editorialFailure(t.id,{...dto(),leaseToken:randomUUID()},worker)).rejects.toThrow();
|
||||
await core.editorialFailure(t.id,{...dto(),leaseToken:o.leaseToken},worker);expect(t.state).toBe('FAILED');
|
||||
});
|
||||
it('rejects cross-task run binding and changed idempotency payload',async()=>{
|
||||
r.taskId=randomUUID();await expect(core.editorialFailure(t.id,dto(),actor)).rejects.toThrow();r.taskId=t.id;
|
||||
const d=dto();await core.editorialFailure(t.id,d,actor);await expect(core.editorialFailure(t.id,{...d,reason:'changed'},actor)).rejects.toThrow();
|
||||
});
|
||||
});
|
||||
@@ -25,6 +25,7 @@ import {
|
||||
CreateOrchestrationTaskDto,
|
||||
TransitionTaskDto,
|
||||
GateDecisionDto,
|
||||
EditorialFailureDto,
|
||||
} from './dto/orchestration.dto';
|
||||
const workflowValidation = new ValidationPipe({
|
||||
transform: true,
|
||||
@@ -215,6 +216,14 @@ export class OrchestrationController {
|
||||
req,
|
||||
);
|
||||
}
|
||||
@Post('tasks/:id/editorial-failure') async editorialFailure(
|
||||
@Param('id') id: string,
|
||||
@Body(workflowValidation) dto: EditorialFailureDto,
|
||||
@Req() req: any,
|
||||
) {
|
||||
if (!this.workflow) return this.deny();
|
||||
return this.safe(await this.workflow.editorialFailure(id, dto, this.actor(req)), req);
|
||||
}
|
||||
@Post('tasks/:id/budget') budget() {
|
||||
return this.deny();
|
||||
}
|
||||
|
||||
@@ -517,6 +517,52 @@ export class PrepareWorkflowService {
|
||||
);
|
||||
});
|
||||
}
|
||||
/** Stop only; never turns unknown spend into free retry capacity. */
|
||||
async editorialFailure(id: string, dto: any, a: WorkflowIdentity) {
|
||||
const held = dto.leaseToken !== undefined;
|
||||
if (held) {
|
||||
this.worker(a);
|
||||
if (a.id !== 'pmo') throw new ForbiddenException('Canonical PMO required');
|
||||
} else if (!this.reviewer(a)) throw new ForbiddenException('Independent recovery reviewer required');
|
||||
if (typeof dto.reason !== 'string' || !dto.reason.trim() || dto.reason.length > 500)
|
||||
throw new BadRequestException('Bounded failure reason required');
|
||||
return this.db.$transaction(async (tx: any) => {
|
||||
await this.allowed(tx);
|
||||
const t = await this.task(tx, id);
|
||||
const { admittedEditorialTask } = require('../orchestration-worker/editorial-execution');
|
||||
if (!admittedEditorialTask(t)) throw new ForbiddenException('Exact editorial proposal required');
|
||||
const proposed = await tx.orchestrationEvent.findFirst({ where: { taskId: id, sequence: 1, eventType: 'PREPARE_PROPOSED', actorId: 'pmo', toState: 'DRAFT' } });
|
||||
const submitted = await tx.orchestrationEvent.findFirst({ where: { taskId: id, sequence: 2, eventType: 'PREPARE_TRANSITION', actorId: 'pmo', fromState: 'DRAFT', toState: 'GATE1_PENDING', idempotencyKey: t.key + ':submit' } });
|
||||
if (!proposed || !submitted) throw new ForbiddenException('Editorial provenance required');
|
||||
if (!held && t.metadata.createdBy === a.id) throw new ForbiddenException('Independent recovery reviewer required');
|
||||
const fingerprint = JSON.stringify({ id, dto, a });
|
||||
if (await this.replay(tx, dto.idempotencyKey, fingerprint)) return t;
|
||||
this.version(t, dto);
|
||||
const o = await tx.prepareWorkflowOutbox.findUnique({ where: { taskId: id } });
|
||||
const r = await tx.orchestrationRun.findUnique({ where: { id: dto.runId } });
|
||||
if (t.state !== 'RUNNING' || !o || o.status !== 'PENDING' || o.runId !== dto.runId ||
|
||||
o.leaseOwner !== 'pmo' || !o.leaseUntil || !r || r.taskId !== id || r.status !== 'RUNNING' ||
|
||||
r.environment !== 'staging' || r.commitSha !== t.metadata.commitSha ||
|
||||
(held ? (o.leaseOwner !== a.id || o.leaseToken !== dto.leaseToken || o.leaseUntil <= new Date()) : o.leaseUntil > new Date()))
|
||||
throw new ForbiddenException('Matching held lease or expired run required');
|
||||
const reserve = await tx.orchestrationBudgetLedger.findUnique({ where: { id: o.reservationId } });
|
||||
const ledger = await tx.orchestrationBudgetLedger.aggregate({ where: { taskId: id }, _sum: { amountCents: true } });
|
||||
if (!reserve || reserve.taskId !== id || reserve.actorId !== 'pmo' || reserve.kind !== 'RESERVE' || reserve.amountCents !== 20 ||
|
||||
reserve.idempotencyKey !== `prepare-reserve:${r.id}` || ledger._sum.amountCents !== 20 ||
|
||||
await tx.orchestrationBudgetLedger.findFirst({ where: { taskId: id, kind: 'RECONCILE' } }) ||
|
||||
await tx.orchestrationArtifact.count({ where: { runId: r.id } }))
|
||||
throw new ForbiddenException('Unreconciled artifact-free editorial reservation required');
|
||||
await tx.orchestrationRun.update({ where: { id: r.id }, data: { status: 'FAILED', retryClass: 'DEPENDENCY', finishedAt: new Date() } });
|
||||
await tx.prepareWorkflowOutbox.update({ where: { id: o.id }, data: { status: 'DONE', leaseToken: null, leaseUntil: null } });
|
||||
const updated = await tx.orchestrationTask.update({ where: { id }, data: { state: 'FAILED', version: { increment: 1 } } });
|
||||
await tx.orchestrationEvent.create({ data: {
|
||||
taskId: id, sequence: updated.version, actorId: a.id, eventType: 'EDITORIAL_RUN_STOPPED',
|
||||
fromState: 'RUNNING', toState: 'FAILED', idempotencyKey: dto.idempotencyKey,
|
||||
payload: { fingerprint, runId: r.id, reason: dto.reason, recoveryMode: held ? 'HELD_LEASE' : 'EXPIRED_RUN', costStatus: 'unknown-upper-bound-reserved', retainedReservationCents: 20 },
|
||||
} });
|
||||
return updated;
|
||||
});
|
||||
}
|
||||
async stageEvidence(a: WorkflowIdentity, dto: any) {
|
||||
this.identity(a, ['STAGE_CI']);
|
||||
if (!this.config.stageCiIds.includes(a.id))
|
||||
|
||||
Reference in new issue
Block a user