fix(editorial): migrate exact CI ownership and fence failed research runs
Deploy Canina / deploy (push) Successful in 5m1s

This commit is contained in:
devops authored and devops committed 2026-10-24 10:28:30 +00:00
1 parent 01b6f1d3d7
commit f4add07780
10 files changed
+311 -11

No files matched your search

@@ -8,6 +8,8 @@ import {
IsNotEmpty,
IsOptional,
IsString,
IsUUID,
MaxLength,
Max,
Min,
} from 'class-validator';
@@ -39,6 +41,12 @@ export class GateDecisionDto {
@IsOptional() @IsString() commitSha?: string;
@IsOptional() @IsString() environment?: string;
}
export class EditorialFailureDto {
@IsUUID() runId!: string;
@IsInt() @Min(1) expectedVersion!: number;
@IsString() @IsNotEmpty() @MaxLength(120) idempotencyKey!: string;
@IsString() @IsNotEmpty() @MaxLength(500) reason!: string;
}
export class RecordBudgetDto {
@Type(() => Number) @IsInt() @Min(1) amountCents!: number;
@IsString() @IsNotEmpty() idempotencyKey!: string;
@@ -0,0 +1,30 @@
import { PrepareWorkflowService } from './prepare-workflow.service';
import { randomUUID } from 'crypto';
describe('editorial failure recovery fencing',()=>{
const actor={id:'owner',role:'SUPER_ADMIN'},worker={id:'pmo',role:'WORKFLOW_WORKER'};
let db:any,core:PrepareWorkflowService,t:any,o:any,r:any,events:any[];
beforeEach(()=>{
t={id:randomUUID(),key:'editorial-v1:'+'a'.repeat(64),title:'Editorial research: fixture',department:'CONTENT',operationMode:'PREPARE',budgetLimitCents:20,state:'RUNNING',version:6,metadata:{createdBy:'pmo',workflowScope:'STAGE_PREPARE',environment:'staging',commitSha:'a'.repeat(40),scope:'editorial-research-to-brief-v1\nTopic: fixture\nResearch and evidence-backed brief only. No publication, release, production changes or execution authorization.',acceptanceCriteria:['evidence']}};
r={id:randomUUID(),taskId:t.id,status:'RUNNING',commitSha:t.metadata.commitSha,environment:'staging'};
o={id:randomUUID(),taskId:t.id,status:'PENDING',runId:r.id,leaseOwner:'pmo',leaseToken:randomUUID(),leaseUntil:new Date(Date.now()-1000),reservationId:'reserve'};events=[];
const tx:any={$queryRaw:async()=>[],orchestrationTask:{findUnique:async()=>t,update:async({data}:any)=>(Object.assign(t,{state:data.state,version:t.version+1}),{...t})},prepareWorkflowOutbox:{findUnique:async()=>o,update:async({data}:any)=>Object.assign(o,data)},orchestrationRun:{findUnique:async()=>r,update:async({data}:any)=>Object.assign(r,data)},orchestrationEvent:{findUnique:async({where}:any)=>events.find(e=>e.idempotencyKey===where.idempotencyKey),findFirst:async()=>({}),create:async({data}:any)=>events.push(data)},orchestrationBudgetLedger:{findUnique:async()=>({id:'reserve',taskId:t.id,kind:'RESERVE',actorId:'pmo',amountCents:20,idempotencyKey:'prepare-reserve:'+r.id}),aggregate:async()=>({_sum:{amountCents:20}}),findFirst:async()=>null},orchestrationArtifact:{count:async()=>0}};
let tail=Promise.resolve();db={$transaction:(fn:any)=>{const p=tail.then(()=>fn(tx));tail=p.catch(()=>{});return p;}};core=new PrepareWorkflowService(db,{workerIds:['pmo'],stageCiIds:[]});
});
const dto=()=>({runId:r.id,expectedVersion:6,idempotencyKey:'recover:'+r.id,reason:'local process ended; cost unknown'});
it('terminalizes expired run without QA/cost writes and replays concurrently',async()=>{
const d=dto();const results=await Promise.all([core.editorialFailure(t.id,d,actor),core.editorialFailure(t.id,d,actor)]);
expect(results.map(x=>x.state)).toEqual(['FAILED','FAILED']);expect(t.version).toBe(7);expect(r.status).toBe('FAILED');expect(o.status).toBe('DONE');expect(o.leaseToken).toBeNull();expect(o.reservationId).toBe('reserve');expect(events).toHaveLength(1);expect(events[0].payload.retainedReservationCents).toBe(20);
});
it('rejects wrong run, stale version, unprivileged actor, active recovery and wrong held token',async()=>{
for(const [d,a] of [[{...dto(),runId:randomUUID()},actor],[{...dto(),expectedVersion:5},actor],[dto(),{id:'other',role:'Admin'}],[dto(),worker]] as any)await expect(core.editorialFailure(t.id,d,a)).rejects.toThrow();
o.leaseUntil=new Date(Date.now()+30000);
await expect(core.editorialFailure(t.id,dto(),actor)).rejects.toThrow();
await expect(core.editorialFailure(t.id,{...dto(),leaseToken:randomUUID()},worker)).rejects.toThrow();
await core.editorialFailure(t.id,{...dto(),leaseToken:o.leaseToken},worker);expect(t.state).toBe('FAILED');
});
it('rejects cross-task run binding and changed idempotency payload',async()=>{
r.taskId=randomUUID();await expect(core.editorialFailure(t.id,dto(),actor)).rejects.toThrow();r.taskId=t.id;
const d=dto();await core.editorialFailure(t.id,d,actor);await expect(core.editorialFailure(t.id,{...d,reason:'changed'},actor)).rejects.toThrow();
});
});
@@ -25,6 +25,7 @@ import {
CreateOrchestrationTaskDto,
TransitionTaskDto,
GateDecisionDto,
EditorialFailureDto,
} from './dto/orchestration.dto';
const workflowValidation = new ValidationPipe({
transform: true,
@@ -215,6 +216,14 @@ export class OrchestrationController {
req,
);
}
@Post('tasks/:id/editorial-failure') async editorialFailure(
@Param('id') id: string,
@Body(workflowValidation) dto: EditorialFailureDto,
@Req() req: any,
) {
if (!this.workflow) return this.deny();
return this.safe(await this.workflow.editorialFailure(id, dto, this.actor(req)), req);
}
@Post('tasks/:id/budget') budget() {
return this.deny();
}
@@ -517,6 +517,52 @@ export class PrepareWorkflowService {
);
});
}
/** Stop only; never turns unknown spend into free retry capacity. */
async editorialFailure(id: string, dto: any, a: WorkflowIdentity) {
const held = dto.leaseToken !== undefined;
if (held) {
this.worker(a);
if (a.id !== 'pmo') throw new ForbiddenException('Canonical PMO required');
} else if (!this.reviewer(a)) throw new ForbiddenException('Independent recovery reviewer required');
if (typeof dto.reason !== 'string' || !dto.reason.trim() || dto.reason.length > 500)
throw new BadRequestException('Bounded failure reason required');
return this.db.$transaction(async (tx: any) => {
await this.allowed(tx);
const t = await this.task(tx, id);
const { admittedEditorialTask } = require('../orchestration-worker/editorial-execution');
if (!admittedEditorialTask(t)) throw new ForbiddenException('Exact editorial proposal required');
const proposed = await tx.orchestrationEvent.findFirst({ where: { taskId: id, sequence: 1, eventType: 'PREPARE_PROPOSED', actorId: 'pmo', toState: 'DRAFT' } });
const submitted = await tx.orchestrationEvent.findFirst({ where: { taskId: id, sequence: 2, eventType: 'PREPARE_TRANSITION', actorId: 'pmo', fromState: 'DRAFT', toState: 'GATE1_PENDING', idempotencyKey: t.key + ':submit' } });
if (!proposed || !submitted) throw new ForbiddenException('Editorial provenance required');
if (!held && t.metadata.createdBy === a.id) throw new ForbiddenException('Independent recovery reviewer required');
const fingerprint = JSON.stringify({ id, dto, a });
if (await this.replay(tx, dto.idempotencyKey, fingerprint)) return t;
this.version(t, dto);
const o = await tx.prepareWorkflowOutbox.findUnique({ where: { taskId: id } });
const r = await tx.orchestrationRun.findUnique({ where: { id: dto.runId } });
if (t.state !== 'RUNNING' || !o || o.status !== 'PENDING' || o.runId !== dto.runId ||
o.leaseOwner !== 'pmo' || !o.leaseUntil || !r || r.taskId !== id || r.status !== 'RUNNING' ||
r.environment !== 'staging' || r.commitSha !== t.metadata.commitSha ||
(held ? (o.leaseOwner !== a.id || o.leaseToken !== dto.leaseToken || o.leaseUntil <= new Date()) : o.leaseUntil > new Date()))
throw new ForbiddenException('Matching held lease or expired run required');
const reserve = await tx.orchestrationBudgetLedger.findUnique({ where: { id: o.reservationId } });
const ledger = await tx.orchestrationBudgetLedger.aggregate({ where: { taskId: id }, _sum: { amountCents: true } });
if (!reserve || reserve.taskId !== id || reserve.actorId !== 'pmo' || reserve.kind !== 'RESERVE' || reserve.amountCents !== 20 ||
reserve.idempotencyKey !== `prepare-reserve:${r.id}` || ledger._sum.amountCents !== 20 ||
await tx.orchestrationBudgetLedger.findFirst({ where: { taskId: id, kind: 'RECONCILE' } }) ||
await tx.orchestrationArtifact.count({ where: { runId: r.id } }))
throw new ForbiddenException('Unreconciled artifact-free editorial reservation required');
await tx.orchestrationRun.update({ where: { id: r.id }, data: { status: 'FAILED', retryClass: 'DEPENDENCY', finishedAt: new Date() } });
await tx.prepareWorkflowOutbox.update({ where: { id: o.id }, data: { status: 'DONE', leaseToken: null, leaseUntil: null } });
const updated = await tx.orchestrationTask.update({ where: { id }, data: { state: 'FAILED', version: { increment: 1 } } });
await tx.orchestrationEvent.create({ data: {
taskId: id, sequence: updated.version, actorId: a.id, eventType: 'EDITORIAL_RUN_STOPPED',
fromState: 'RUNNING', toState: 'FAILED', idempotencyKey: dto.idempotencyKey,
payload: { fingerprint, runId: r.id, reason: dto.reason, recoveryMode: held ? 'HELD_LEASE' : 'EXPIRED_RUN', costStatus: 'unknown-upper-bound-reserved', retainedReservationCents: 20 },
} });
return updated;
});
}
async stageEvidence(a: WorkflowIdentity, dto: any) {
this.identity(a, ['STAGE_CI']);
if (!this.config.stageCiIds.includes(a.id))