feat(worker): add authenticated bounded PREPARE HTTP executor tracer

This commit is contained in:
devops committed 2026-10-23 13:49:38 +00:00
1 parent 58d3794050
commit aa99216c49
5 files changed
+742

No files matched your search

@@ -0,0 +1,273 @@
import {
BadRequestException,
Body,
CanActivate,
ConflictException,
Controller,
DynamicModule,
ForbiddenException,
Get,
Inject,
Injectable,
Module,
NotFoundException,
Param,
Post,
Req,
UnauthorizedException,
UseGuards,
UsePipes,
ValidationPipe,
} from "@nestjs/common";
import { IsString, IsUUID, Matches, MaxLength } from "class-validator";
import { createHash, timingSafeEqual } from "crypto";
import { mkdir, readFile, writeFile } from "fs/promises";
import { join, isAbsolute } from "path";
import { PrismaModule } from "../prisma/prisma.module";
import { PrismaService } from "../prisma/prisma.service";
import {
PREPARE_WORKFLOW_CONFIG,
PrepareWorkflowService,
WorkflowIdentity,
} from "../orchestration/prepare-workflow.service";
const CONFIG = Symbol("WORKER_TRANSPORT_CONFIG");
export interface WorkerTransportConfig {
enabled: boolean;
grants: {
id: string;
role: "WORKFLOW_WORKER" | "STAGE_CI";
credentialSha256: string;
}[];
workerIds: string[];
stageCiIds: string[];
artifactDirectory: string;
artifactBaseUrl: string;
}
class ClaimDto {
@IsUUID() taskId!: string;
}
class LeaseDto extends ClaimDto {
@IsUUID() runId!: string;
@IsUUID() leaseToken!: string;
}
class CompleteDto extends LeaseDto {
@IsString() @MaxLength(2048) report!: string;
}
class StageEvidenceDto extends ClaimDto {
@IsUUID() runId!: string;
@Matches(/^[a-f0-9]{40}$/) commitSha!: string;
@Matches(/^[a-f0-9]{64}$/) sha256!: string;
@Matches(/^[a-f0-9]{64}$/) evidenceSha256!: string;
@Matches(/^https:\/\/[^\s]+$/) @MaxLength(2000) uri!: string;
}
export function tracerReport(lease: {
taskId: string;
runId: string;
commitSha: string;
}) {
return JSON.stringify({
kind: "DETERMINISTIC_PREPARE_TRACER",
schemaVersion: 1,
taskId: lease.taskId,
runId: lease.runId,
commitSha: lease.commitSha,
environment: "staging",
llmPilot: false,
browserExecuted: false,
shellExecuted: false,
deploymentExecuted: false,
checks: ["lease-bound identity", "deterministic JSON serialization"],
actualCents: 0,
});
}
@Injectable()
class WorkerIdentityGuard implements CanActivate {
constructor(@Inject(CONFIG) private readonly config: WorkerTransportConfig) {}
canActivate(context: any) {
if (!this.config.enabled)
throw new ForbiddenException("Worker transport disabled");
const req = context.switchToHttp().getRequest();
const auth = req.headers.authorization;
if (typeof auth !== "string" || !/^Bearer [\x21-\x7e]{32,512}$/.test(auth))
throw new UnauthorizedException();
const digest = createHash("sha256").update(auth.slice(7)).digest();
const grant = this.config.grants.find((g) =>
timingSafeEqual(Buffer.from(g.credentialSha256, "hex"), digest),
);
if (!grant) throw new UnauthorizedException();
req.workerIdentity = { id: grant.id, role: grant.role };
return true;
}
}
@Injectable()
class WorkerTransportService {
constructor(
private readonly workflow: PrepareWorkflowService,
private readonly db: PrismaService,
@Inject(CONFIG) private readonly config: WorkerTransportConfig,
) {}
private worker(a: WorkflowIdentity) {
if (a.role !== "WORKFLOW_WORKER" || !this.config.workerIds.includes(a.id))
throw new ForbiddenException("Worker allowlist required");
}
async claim(a: WorkflowIdentity, dto: ClaimDto) {
this.worker(a);
const t = await this.db.orchestrationTask.findUnique({
where: { id: dto.taskId },
});
// Explicit per-task pilot scope; never opportunistically consume arbitrary queued work.
if (!t || !["READY", "RUNNING"].includes(t.state)) return { lease: null };
if (t.budgetLimitCents !== 20)
throw new ForbiddenException("Pilot hard task cap is 20 cents");
return { lease: await this.workflow.claim(a, dto.taskId) };
}
heartbeat(a: WorkflowIdentity, dto: LeaseDto) {
this.worker(a);
return this.workflow.heartbeat(a, dto);
}
async complete(a: WorkflowIdentity, dto: CompleteDto) {
this.worker(a);
// Recheck stop/interrupt and fencing before artifact creation; core rechecks atomically at commit.
await this.workflow.heartbeat(a, dto);
const run = await this.db.orchestrationRun.findUniqueOrThrow({
where: { id: dto.runId },
});
const expected = tracerReport({ ...dto, commitSha: run.commitSha });
if (dto.report !== expected)
throw new BadRequestException(
"Only deterministic harmless tracer report accepted",
);
const sha256 = createHash("sha256").update(expected).digest("hex");
await mkdir(this.config.artifactDirectory, {
recursive: true,
mode: 0o700,
});
await writeFile(
join(this.config.artifactDirectory, sha256 + ".json"),
expected,
{ flag: "w", mode: 0o600 },
);
const result = await this.workflow.complete(a, {
...dto,
sha256,
uri: this.config.artifactBaseUrl + "/" + sha256,
actualCents: 0,
});
return {
state: result.state,
taskId: result.id,
runId: dto.runId,
sha256,
actualCents: 0,
};
}
async artifact(hash: string) {
if (!/^[a-f0-9]{64}$/.test(hash))
throw new BadRequestException("Invalid artifact hash");
const registered = await this.db.orchestrationArtifact.findFirst({
where: { sha256: hash, kind: "QA" },
});
if (!registered) throw new NotFoundException("Artifact not registered");
let bytes: string;
try {
bytes = await readFile(
join(this.config.artifactDirectory, hash + ".json"),
"utf8",
);
} catch {
throw new NotFoundException("Artifact unavailable");
}
if (createHash("sha256").update(bytes).digest("hex") !== hash)
throw new ConflictException("Artifact integrity mismatch");
return JSON.parse(bytes);
}
stage(a: WorkflowIdentity, dto: StageEvidenceDto) {
return this.workflow.stageEvidence(a, dto);
}
}
@Controller("internal/prepare-worker")
@UseGuards(WorkerIdentityGuard)
@UsePipes(
new ValidationPipe({
whitelist: true,
forbidNonWhitelisted: true,
transform: true,
}),
)
class WorkerTransportController {
constructor(private readonly service: WorkerTransportService) {}
@Post("claim") claim(@Req() req: any, @Body() dto: ClaimDto) {
return this.service.claim(req.workerIdentity, dto);
}
@Post("heartbeat") heartbeat(@Req() req: any, @Body() dto: LeaseDto) {
return this.service.heartbeat(req.workerIdentity, dto);
}
@Post("complete") complete(@Req() req: any, @Body() dto: CompleteDto) {
return this.service.complete(req.workerIdentity, dto);
}
@Post("stage-evidence") stage(
@Req() req: any,
@Body() dto: StageEvidenceDto,
) {
return this.service.stage(req.workerIdentity, dto);
}
@Get("artifacts/:hash") artifact(@Param("hash") hash: string) {
return this.service.artifact(hash);
}
}
@Module({})
export class WorkerTransportModule {
static register(config: WorkerTransportConfig): DynamicModule {
const ids = new Set<string>(),
digests = new Set<string>();
if (
!isAbsolute(config.artifactDirectory) ||
!/^https:\/\/[^\s?#]+$/.test(config.artifactBaseUrl)
)
throw new Error(
"Absolute artifact directory and fixed HTTPS artifact base required",
);
for (const g of config.grants) {
const allowed =
g.role === "WORKFLOW_WORKER"
? config.workerIds
: g.role === "STAGE_CI"
? config.stageCiIds
: [];
if (
!/^[a-zA-Z0-9_-]{1,100}$/.test(g.id) ||
!allowed.includes(g.id) ||
!/^[a-f0-9]{64}$/.test(g.credentialSha256) ||
ids.has(g.id) ||
digests.has(g.credentialSha256) ||
(config.workerIds.includes(g.id) && config.stageCiIds.includes(g.id))
)
throw new Error("Invalid or overlapping worker/CI credential grants");
ids.add(g.id);
digests.add(g.credentialSha256);
}
const snapshot = JSON.parse(
JSON.stringify(config),
) as WorkerTransportConfig;
return {
module: WorkerTransportModule,
imports: [PrismaModule],
controllers: [WorkerTransportController],
providers: [
PrepareWorkflowService,
WorkerIdentityGuard,
WorkerTransportService,
{ provide: CONFIG, useValue: snapshot },
{
provide: PREPARE_WORKFLOW_CONFIG,
useValue: {
workerIds: snapshot.workerIds,
stageCiIds: snapshot.stageCiIds,
},
},
],
exports: [PrepareWorkflowService],
};
}
}
@@ -0,0 +1,381 @@
import { Test } from "@nestjs/testing";
import { INestApplication } from "@nestjs/common";
import { PrismaClient } from "@prisma/client";
import { PrismaService } from "../prisma/prisma.service";
import { PrepareWorkflowService } from "../orchestration/prepare-workflow.service";
import { WorkerTransportModule } from "./worker-transport.module";
import { createHash, randomUUID } from "crypto";
import { mkdtemp, readFile, rm, writeFile } from "fs/promises";
import { tmpdir } from "os";
import { join } from "path";
import { execFile } from "child_process";
import { promisify } from "util";
const enabled = process.env.WORKER_TRANSPORT_DB === "1";
(enabled ? describe : describe.skip)(
"isolated PostgreSQL HTTP executor tracer",
() => {
const db = new PrismaClient();
const token = "test-only-worker-secret-".repeat(3);
const ciToken = "test-only-ci-secret-".repeat(3);
const digest = (s: string) => createHash("sha256").update(s).digest("hex");
let app: INestApplication,
workflow: PrepareWorkflowService,
base: string,
directory: string;
let pilot: any;
beforeAll(async () => {
directory = await mkdtemp(join(tmpdir(), "worker-artifacts-"));
const mod = await Test.createTestingModule({
imports: [
WorkerTransportModule.register({
enabled: true,
grants: [
{
id: "pilot",
role: "WORKFLOW_WORKER",
credentialSha256: digest(token),
},
{ id: "ci", role: "STAGE_CI", credentialSha256: digest(ciToken) },
],
workerIds: ["pilot"],
stageCiIds: ["ci"],
artifactDirectory: directory,
artifactBaseUrl:
"https://isolated.invalid/internal/prepare-worker/artifacts",
}),
],
})
.overrideProvider(PrismaService)
.useValue(db)
.compile();
workflow = mod.get(PrepareWorkflowService);
app = mod.createNestApplication();
await app.listen(0, "127.0.0.1");
base = await app.getUrl();
pilot = {
runOnce: async (options: any) => {
const program =
'import {runOnce} from "../infra/hermes-worker/pilot.mjs"; console.log(JSON.stringify(await runOnce(JSON.parse(process.env.TEST_PILOT_OPTIONS))));';
const result = await promisify(execFile)(
process.execPath,
["--input-type=module", "-e", program],
{
env: {
...process.env,
TEST_PILOT_OPTIONS: JSON.stringify(options),
},
timeout: 15000,
},
);
return JSON.parse(result.stdout);
},
};
});
afterAll(async () => {
await app?.close();
await db.$disconnect();
if (directory) await rm(directory, { recursive: true, force: true });
});
const owner = { id: "author", role: "Admin" },
reviewer = { id: "reviewer", role: "SUPER_ADMIN" };
async function proposal(queue = true, budget = 20) {
let t = await workflow.create(
{
key: randomUUID(),
title: "deterministic tracer",
department: "TEST",
operationMode: "PREPARE",
budgetLimitCents: budget,
metadata: {
commitSha: "a".repeat(40),
scope: "Harmless tracer only",
acceptanceCriteria: ["writes deterministic JSON; no release"],
},
},
owner,
);
if (queue) {
t = await workflow.transition(
t.id,
{
toState: "GATE1_PENDING",
expectedVersion: t.version,
idempotencyKey: randomUUID(),
},
owner,
);
t = await workflow.decide(
t.id,
1,
{
decision: "APPROVED",
expiresAt: new Date(Date.now() + 60000).toISOString(),
expectedVersion: t.version,
idempotencyKey: randomUUID(),
},
reviewer,
);
t = await workflow.transition(
t.id,
{
toState: "READY",
expectedVersion: t.version,
idempotencyKey: randomUUID(),
},
owner,
);
}
return t;
}
async function post(route: string, body: any = {}, secret = token) {
const r = await fetch(base + "/internal/prepare-worker/" + route, {
method: "POST",
headers: {
authorization: "Bearer " + secret,
"content-type": "application/json",
},
body: JSON.stringify(body),
});
return { status: r.status, body: await r.json() };
}
it("rejects overlapping worker and CI grants and accepts no credentials when disabled", async () => {
expect(() =>
WorkerTransportModule.register({
enabled: true,
grants: [
{
id: "both",
role: "WORKFLOW_WORKER",
credentialSha256: digest(token),
},
],
workerIds: ["both"],
stageCiIds: ["both"],
artifactDirectory: directory,
artifactBaseUrl: "https://isolated.invalid/artifacts",
}),
).toThrow(/overlapping/);
const mod = await Test.createTestingModule({
imports: [
WorkerTransportModule.register({
enabled: false,
grants: [],
workerIds: [],
stageCiIds: [],
artifactDirectory: directory,
artifactBaseUrl: "https://isolated.invalid/artifacts",
}),
],
})
.overrideProvider(PrismaService)
.useValue(db)
.compile();
const disabled = mod.createNestApplication();
await disabled.listen(0, "127.0.0.1");
try {
const r = await fetch(
(await disabled.getUrl()) + "/internal/prepare-worker/claim",
{
method: "POST",
headers: {
authorization: "Bearer " + token,
"content-type": "application/json",
},
body: JSON.stringify({ taskId: randomUUID() }),
},
);
expect(r.status).toBe(403);
} finally {
await disabled.close();
}
});
it("denies untrusted identities and body impersonation; dispatches nothing before Gate1", async () => {
const t = await proposal(false);
expect((await post("claim", {}, "wrong")).status).toBe(401);
expect((await post("claim", { taskId: t.id }, ciToken)).status).toBe(403);
expect(
(await post("claim", { taskId: t.id, actorId: "pilot" })).status,
).toBe(400);
expect((await post("claim", { taskId: t.id })).body).toEqual({
lease: null,
});
expect(await db.orchestrationRun.count({ where: { taskId: t.id } })).toBe(
0,
);
});
it("races claims, fences expired leases and reserves unknown cost against the 20 cent cap", async () => {
const t = await proposal();
const claims = await Promise.all([
post("claim", { taskId: t.id }),
post("claim", { taskId: t.id }),
]);
const leases = claims.map((r) => r.body.lease).filter(Boolean);
expect(leases).toHaveLength(1);
const lease = leases[0],
binding = {
taskId: t.id,
runId: lease.runId,
leaseToken: lease.leaseToken,
};
expect(
(await post("heartbeat", { ...binding, leaseToken: randomUUID() }))
.status,
).toBe(403);
expect(
(
await post("complete", {
...binding,
report: "arbitrary shell or prompt",
})
).status,
).toBe(400);
expect(
(await post("complete", { ...binding, report: "{}", actualCents: 21 }))
.status,
).toBe(400);
await db.prepareWorkflowOutbox.update({
where: { taskId: t.id },
data: { leaseUntil: new Date(Date.now() - 1000) },
});
expect((await post("heartbeat", binding)).status).toBe(403);
expect(
(await post("complete", { ...binding, report: "{}" })).status,
).toBe(403);
expect((await post("claim", { taskId: t.id })).status).toBe(403);
expect(await db.orchestrationRun.count({ where: { taskId: t.id } })).toBe(
1,
);
expect(
(
await db.orchestrationBudgetLedger.aggregate({
where: { taskId: t.id },
_sum: { amountCents: true },
})
)._sum.amountCents,
).toBe(20);
const larger = await proposal(true, 40);
expect((await post("claim", { taskId: larger.id })).status).toBe(403);
});
it("rechecks kill switch and task interrupt on heartbeat and completion; separates CI authority", async () => {
const t = await proposal(),
lease = (await post("claim", { taskId: t.id })).body.lease;
const binding = {
taskId: t.id,
runId: lease.runId,
leaseToken: lease.leaseToken,
};
await db.orchestrationKillSwitch.upsert({
where: { scope: "GLOBAL" },
create: {
scope: "GLOBAL",
level: "STOP_AGENTS",
active: true,
reason: "isolated test",
actorId: "test",
},
update: { level: "STOP_AGENTS", active: true },
});
try {
expect((await post("heartbeat", binding)).status).toBe(403);
expect(
(await post("complete", { ...binding, report: "{}" })).status,
).toBe(403);
expect((await post("claim", { taskId: t.id })).status).toBe(403);
} finally {
await db.orchestrationKillSwitch.deleteMany();
}
await db.orchestrationTask.update({
where: { id: t.id },
data: { state: "CANCELLED" },
});
expect((await post("heartbeat", binding)).status).toBe(403);
expect(
(await post("complete", { ...binding, report: "{}" })).status,
).toBe(403);
expect(
await db.orchestrationArtifact.count({ where: { runId: lease.runId } }),
).toBe(0);
const evidence = {
taskId: t.id,
runId: lease.runId,
commitSha: "a".repeat(40),
sha256: "b".repeat(64),
evidenceSha256: "c".repeat(64),
uri: "https://isolated.invalid/not-real-ci",
};
expect((await post("stage-evidence", evidence)).status).toBe(403);
expect((await post("stage-evidence", evidence, ciToken)).status).toBe(
403,
);
});
it("runs actual HTTP deterministic executor to VERIFYING with retrievable content-addressed artifact", async () => {
const t = await proposal();
const result = await pilot.runOnce({
baseUrl: base,
credential: token,
taskId: t.id,
allowLoopbackHttp: true,
});
expect(result.state).toBe("VERIFYING");
const row = await db.orchestrationTask.findUniqueOrThrow({
where: { id: t.id },
});
expect(row.state).toBe("VERIFYING");
const artifact = await db.orchestrationArtifact.findFirstOrThrow({
where: { run: { taskId: t.id } },
});
const bytes = await readFile(join(directory, artifact.sha256 + ".json"));
expect(digest(bytes.toString())).toBe(artifact.sha256);
expect(JSON.parse(bytes.toString())).toMatchObject({
kind: "DETERMINISTIC_PREPARE_TRACER",
commitSha: "a".repeat(40),
environment: "staging",
llmPilot: false,
});
const get = await fetch(
base + "/internal/prepare-worker/artifacts/" + artifact.sha256,
{ headers: { authorization: "Bearer " + token } },
);
expect(get.status).toBe(200);
expect(await get.text()).toBe(bytes.toString());
const cost = await db.orchestrationBudgetLedger.aggregate({
where: { taskId: t.id },
_sum: { amountCents: true },
});
expect(cost._sum.amountCents).toBe(0);
expect(
await db.orchestrationAttestation.findFirst({
where: { run: { taskId: t.id }, kind: "STAGE" },
}),
).toBeNull();
const evidence = {
taskId: t.id,
state: row.state,
sha256: artifact.sha256,
artifact: JSON.parse(bytes.toString()),
httpGet: get.status,
costCents: cost._sum.amountCents,
uriExternalReachability: "NOT_TESTED_ISOLATED_FIXTURE_HOST",
};
console.log("TRACER_EVIDENCE", JSON.stringify(evidence));
if (process.env.TRACER_EVIDENCE_PATH)
await writeFile(
process.env.TRACER_EVIDENCE_PATH,
JSON.stringify(evidence, null, 2),
);
await writeFile(
join(directory, artifact.sha256 + ".json"),
'{"tampered":true}',
);
expect(
(
await fetch(
base + "/internal/prepare-worker/artifacts/" + artifact.sha256,
{ headers: { authorization: "Bearer " + token } },
)
).status,
).toBe(409);
});
},
);