feat(worker): add authenticated bounded PREPARE HTTP executor tracer
This commit is contained in:
1 parent
58d3794050
commit
f7048ab832
5 files changed
+742
No files matched your search
@@ -0,0 +1,273 @@
|
||||
import {
|
||||
BadRequestException,
|
||||
Body,
|
||||
CanActivate,
|
||||
ConflictException,
|
||||
Controller,
|
||||
DynamicModule,
|
||||
ForbiddenException,
|
||||
Get,
|
||||
Inject,
|
||||
Injectable,
|
||||
Module,
|
||||
NotFoundException,
|
||||
Param,
|
||||
Post,
|
||||
Req,
|
||||
UnauthorizedException,
|
||||
UseGuards,
|
||||
UsePipes,
|
||||
ValidationPipe,
|
||||
} from "@nestjs/common";
|
||||
import { IsString, IsUUID, Matches, MaxLength } from "class-validator";
|
||||
import { createHash, timingSafeEqual } from "crypto";
|
||||
import { mkdir, readFile, writeFile } from "fs/promises";
|
||||
import { join, isAbsolute } from "path";
|
||||
import { PrismaModule } from "../prisma/prisma.module";
|
||||
import { PrismaService } from "../prisma/prisma.service";
|
||||
import {
|
||||
PREPARE_WORKFLOW_CONFIG,
|
||||
PrepareWorkflowService,
|
||||
WorkflowIdentity,
|
||||
} from "../orchestration/prepare-workflow.service";
|
||||
|
||||
const CONFIG = Symbol("WORKER_TRANSPORT_CONFIG");
|
||||
export interface WorkerTransportConfig {
|
||||
enabled: boolean;
|
||||
grants: {
|
||||
id: string;
|
||||
role: "WORKFLOW_WORKER" | "STAGE_CI";
|
||||
credentialSha256: string;
|
||||
}[];
|
||||
workerIds: string[];
|
||||
stageCiIds: string[];
|
||||
artifactDirectory: string;
|
||||
artifactBaseUrl: string;
|
||||
}
|
||||
class ClaimDto {
|
||||
@IsUUID() taskId!: string;
|
||||
}
|
||||
class LeaseDto extends ClaimDto {
|
||||
@IsUUID() runId!: string;
|
||||
@IsUUID() leaseToken!: string;
|
||||
}
|
||||
class CompleteDto extends LeaseDto {
|
||||
@IsString() @MaxLength(2048) report!: string;
|
||||
}
|
||||
class StageEvidenceDto extends ClaimDto {
|
||||
@IsUUID() runId!: string;
|
||||
@Matches(/^[a-f0-9]{40}$/) commitSha!: string;
|
||||
@Matches(/^[a-f0-9]{64}$/) sha256!: string;
|
||||
@Matches(/^[a-f0-9]{64}$/) evidenceSha256!: string;
|
||||
@Matches(/^https:\/\/[^\s]+$/) @MaxLength(2000) uri!: string;
|
||||
}
|
||||
export function tracerReport(lease: {
|
||||
taskId: string;
|
||||
runId: string;
|
||||
commitSha: string;
|
||||
}) {
|
||||
return JSON.stringify({
|
||||
kind: "DETERMINISTIC_PREPARE_TRACER",
|
||||
schemaVersion: 1,
|
||||
taskId: lease.taskId,
|
||||
runId: lease.runId,
|
||||
commitSha: lease.commitSha,
|
||||
environment: "staging",
|
||||
llmPilot: false,
|
||||
browserExecuted: false,
|
||||
shellExecuted: false,
|
||||
deploymentExecuted: false,
|
||||
checks: ["lease-bound identity", "deterministic JSON serialization"],
|
||||
actualCents: 0,
|
||||
});
|
||||
}
|
||||
@Injectable()
|
||||
class WorkerIdentityGuard implements CanActivate {
|
||||
constructor(@Inject(CONFIG) private readonly config: WorkerTransportConfig) {}
|
||||
canActivate(context: any) {
|
||||
if (!this.config.enabled)
|
||||
throw new ForbiddenException("Worker transport disabled");
|
||||
const req = context.switchToHttp().getRequest();
|
||||
const auth = req.headers.authorization;
|
||||
if (typeof auth !== "string" || !/^Bearer [\x21-\x7e]{32,512}$/.test(auth))
|
||||
throw new UnauthorizedException();
|
||||
const digest = createHash("sha256").update(auth.slice(7)).digest();
|
||||
const grant = this.config.grants.find((g) =>
|
||||
timingSafeEqual(Buffer.from(g.credentialSha256, "hex"), digest),
|
||||
);
|
||||
if (!grant) throw new UnauthorizedException();
|
||||
req.workerIdentity = { id: grant.id, role: grant.role };
|
||||
return true;
|
||||
}
|
||||
}
|
||||
@Injectable()
|
||||
class WorkerTransportService {
|
||||
constructor(
|
||||
private readonly workflow: PrepareWorkflowService,
|
||||
private readonly db: PrismaService,
|
||||
@Inject(CONFIG) private readonly config: WorkerTransportConfig,
|
||||
) {}
|
||||
private worker(a: WorkflowIdentity) {
|
||||
if (a.role !== "WORKFLOW_WORKER" || !this.config.workerIds.includes(a.id))
|
||||
throw new ForbiddenException("Worker allowlist required");
|
||||
}
|
||||
async claim(a: WorkflowIdentity, dto: ClaimDto) {
|
||||
this.worker(a);
|
||||
const t = await this.db.orchestrationTask.findUnique({
|
||||
where: { id: dto.taskId },
|
||||
});
|
||||
// Explicit per-task pilot scope; never opportunistically consume arbitrary queued work.
|
||||
if (!t || !["READY", "RUNNING"].includes(t.state)) return { lease: null };
|
||||
if (t.budgetLimitCents !== 20)
|
||||
throw new ForbiddenException("Pilot hard task cap is 20 cents");
|
||||
return { lease: await this.workflow.claim(a, dto.taskId) };
|
||||
}
|
||||
heartbeat(a: WorkflowIdentity, dto: LeaseDto) {
|
||||
this.worker(a);
|
||||
return this.workflow.heartbeat(a, dto);
|
||||
}
|
||||
async complete(a: WorkflowIdentity, dto: CompleteDto) {
|
||||
this.worker(a);
|
||||
// Recheck stop/interrupt and fencing before artifact creation; core rechecks atomically at commit.
|
||||
await this.workflow.heartbeat(a, dto);
|
||||
const run = await this.db.orchestrationRun.findUniqueOrThrow({
|
||||
where: { id: dto.runId },
|
||||
});
|
||||
const expected = tracerReport({ ...dto, commitSha: run.commitSha });
|
||||
if (dto.report !== expected)
|
||||
throw new BadRequestException(
|
||||
"Only deterministic harmless tracer report accepted",
|
||||
);
|
||||
const sha256 = createHash("sha256").update(expected).digest("hex");
|
||||
await mkdir(this.config.artifactDirectory, {
|
||||
recursive: true,
|
||||
mode: 0o700,
|
||||
});
|
||||
await writeFile(
|
||||
join(this.config.artifactDirectory, sha256 + ".json"),
|
||||
expected,
|
||||
{ flag: "w", mode: 0o600 },
|
||||
);
|
||||
const result = await this.workflow.complete(a, {
|
||||
...dto,
|
||||
sha256,
|
||||
uri: this.config.artifactBaseUrl + "/" + sha256,
|
||||
actualCents: 0,
|
||||
});
|
||||
return {
|
||||
state: result.state,
|
||||
taskId: result.id,
|
||||
runId: dto.runId,
|
||||
sha256,
|
||||
actualCents: 0,
|
||||
};
|
||||
}
|
||||
async artifact(hash: string) {
|
||||
if (!/^[a-f0-9]{64}$/.test(hash))
|
||||
throw new BadRequestException("Invalid artifact hash");
|
||||
const registered = await this.db.orchestrationArtifact.findFirst({
|
||||
where: { sha256: hash, kind: "QA" },
|
||||
});
|
||||
if (!registered) throw new NotFoundException("Artifact not registered");
|
||||
let bytes: string;
|
||||
try {
|
||||
bytes = await readFile(
|
||||
join(this.config.artifactDirectory, hash + ".json"),
|
||||
"utf8",
|
||||
);
|
||||
} catch {
|
||||
throw new NotFoundException("Artifact unavailable");
|
||||
}
|
||||
if (createHash("sha256").update(bytes).digest("hex") !== hash)
|
||||
throw new ConflictException("Artifact integrity mismatch");
|
||||
return JSON.parse(bytes);
|
||||
}
|
||||
stage(a: WorkflowIdentity, dto: StageEvidenceDto) {
|
||||
return this.workflow.stageEvidence(a, dto);
|
||||
}
|
||||
}
|
||||
@Controller("internal/prepare-worker")
|
||||
@UseGuards(WorkerIdentityGuard)
|
||||
@UsePipes(
|
||||
new ValidationPipe({
|
||||
whitelist: true,
|
||||
forbidNonWhitelisted: true,
|
||||
transform: true,
|
||||
}),
|
||||
)
|
||||
class WorkerTransportController {
|
||||
constructor(private readonly service: WorkerTransportService) {}
|
||||
@Post("claim") claim(@Req() req: any, @Body() dto: ClaimDto) {
|
||||
return this.service.claim(req.workerIdentity, dto);
|
||||
}
|
||||
@Post("heartbeat") heartbeat(@Req() req: any, @Body() dto: LeaseDto) {
|
||||
return this.service.heartbeat(req.workerIdentity, dto);
|
||||
}
|
||||
@Post("complete") complete(@Req() req: any, @Body() dto: CompleteDto) {
|
||||
return this.service.complete(req.workerIdentity, dto);
|
||||
}
|
||||
@Post("stage-evidence") stage(
|
||||
@Req() req: any,
|
||||
@Body() dto: StageEvidenceDto,
|
||||
) {
|
||||
return this.service.stage(req.workerIdentity, dto);
|
||||
}
|
||||
@Get("artifacts/:hash") artifact(@Param("hash") hash: string) {
|
||||
return this.service.artifact(hash);
|
||||
}
|
||||
}
|
||||
@Module({})
|
||||
export class WorkerTransportModule {
|
||||
static register(config: WorkerTransportConfig): DynamicModule {
|
||||
const ids = new Set<string>(),
|
||||
digests = new Set<string>();
|
||||
if (
|
||||
!isAbsolute(config.artifactDirectory) ||
|
||||
!/^https:\/\/[^\s?#]+$/.test(config.artifactBaseUrl)
|
||||
)
|
||||
throw new Error(
|
||||
"Absolute artifact directory and fixed HTTPS artifact base required",
|
||||
);
|
||||
for (const g of config.grants) {
|
||||
const allowed =
|
||||
g.role === "WORKFLOW_WORKER"
|
||||
? config.workerIds
|
||||
: g.role === "STAGE_CI"
|
||||
? config.stageCiIds
|
||||
: [];
|
||||
if (
|
||||
!/^[a-zA-Z0-9_-]{1,100}$/.test(g.id) ||
|
||||
!allowed.includes(g.id) ||
|
||||
!/^[a-f0-9]{64}$/.test(g.credentialSha256) ||
|
||||
ids.has(g.id) ||
|
||||
digests.has(g.credentialSha256) ||
|
||||
(config.workerIds.includes(g.id) && config.stageCiIds.includes(g.id))
|
||||
)
|
||||
throw new Error("Invalid or overlapping worker/CI credential grants");
|
||||
ids.add(g.id);
|
||||
digests.add(g.credentialSha256);
|
||||
}
|
||||
const snapshot = JSON.parse(
|
||||
JSON.stringify(config),
|
||||
) as WorkerTransportConfig;
|
||||
return {
|
||||
module: WorkerTransportModule,
|
||||
imports: [PrismaModule],
|
||||
controllers: [WorkerTransportController],
|
||||
providers: [
|
||||
PrepareWorkflowService,
|
||||
WorkerIdentityGuard,
|
||||
WorkerTransportService,
|
||||
{ provide: CONFIG, useValue: snapshot },
|
||||
{
|
||||
provide: PREPARE_WORKFLOW_CONFIG,
|
||||
useValue: {
|
||||
workerIds: snapshot.workerIds,
|
||||
stageCiIds: snapshot.stageCiIds,
|
||||
},
|
||||
},
|
||||
],
|
||||
exports: [PrepareWorkflowService],
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,381 @@
|
||||
import { Test } from "@nestjs/testing";
|
||||
import { INestApplication } from "@nestjs/common";
|
||||
import { PrismaClient } from "@prisma/client";
|
||||
import { PrismaService } from "../prisma/prisma.service";
|
||||
import { PrepareWorkflowService } from "../orchestration/prepare-workflow.service";
|
||||
import { WorkerTransportModule } from "./worker-transport.module";
|
||||
import { createHash, randomUUID } from "crypto";
|
||||
import { mkdtemp, readFile, rm, writeFile } from "fs/promises";
|
||||
import { tmpdir } from "os";
|
||||
import { join } from "path";
|
||||
import { execFile } from "child_process";
|
||||
import { promisify } from "util";
|
||||
|
||||
const enabled = process.env.WORKER_TRANSPORT_DB === "1";
|
||||
(enabled ? describe : describe.skip)(
|
||||
"isolated PostgreSQL HTTP executor tracer",
|
||||
() => {
|
||||
const db = new PrismaClient();
|
||||
const token = "test-only-worker-secret-".repeat(3);
|
||||
const ciToken = "test-only-ci-secret-".repeat(3);
|
||||
const digest = (s: string) => createHash("sha256").update(s).digest("hex");
|
||||
let app: INestApplication,
|
||||
workflow: PrepareWorkflowService,
|
||||
base: string,
|
||||
directory: string;
|
||||
let pilot: any;
|
||||
beforeAll(async () => {
|
||||
directory = await mkdtemp(join(tmpdir(), "worker-artifacts-"));
|
||||
const mod = await Test.createTestingModule({
|
||||
imports: [
|
||||
WorkerTransportModule.register({
|
||||
enabled: true,
|
||||
grants: [
|
||||
{
|
||||
id: "pilot",
|
||||
role: "WORKFLOW_WORKER",
|
||||
credentialSha256: digest(token),
|
||||
},
|
||||
{ id: "ci", role: "STAGE_CI", credentialSha256: digest(ciToken) },
|
||||
],
|
||||
workerIds: ["pilot"],
|
||||
stageCiIds: ["ci"],
|
||||
artifactDirectory: directory,
|
||||
artifactBaseUrl:
|
||||
"https://isolated.invalid/internal/prepare-worker/artifacts",
|
||||
}),
|
||||
],
|
||||
})
|
||||
.overrideProvider(PrismaService)
|
||||
.useValue(db)
|
||||
.compile();
|
||||
workflow = mod.get(PrepareWorkflowService);
|
||||
app = mod.createNestApplication();
|
||||
await app.listen(0, "127.0.0.1");
|
||||
base = await app.getUrl();
|
||||
pilot = {
|
||||
runOnce: async (options: any) => {
|
||||
const program =
|
||||
'import {runOnce} from "../infra/hermes-worker/pilot.mjs"; console.log(JSON.stringify(await runOnce(JSON.parse(process.env.TEST_PILOT_OPTIONS))));';
|
||||
const result = await promisify(execFile)(
|
||||
process.execPath,
|
||||
["--input-type=module", "-e", program],
|
||||
{
|
||||
env: {
|
||||
...process.env,
|
||||
TEST_PILOT_OPTIONS: JSON.stringify(options),
|
||||
},
|
||||
timeout: 15000,
|
||||
},
|
||||
);
|
||||
return JSON.parse(result.stdout);
|
||||
},
|
||||
};
|
||||
});
|
||||
afterAll(async () => {
|
||||
await app?.close();
|
||||
await db.$disconnect();
|
||||
if (directory) await rm(directory, { recursive: true, force: true });
|
||||
});
|
||||
const owner = { id: "author", role: "Admin" },
|
||||
reviewer = { id: "reviewer", role: "SUPER_ADMIN" };
|
||||
async function proposal(queue = true, budget = 20) {
|
||||
let t = await workflow.create(
|
||||
{
|
||||
key: randomUUID(),
|
||||
title: "deterministic tracer",
|
||||
department: "TEST",
|
||||
operationMode: "PREPARE",
|
||||
budgetLimitCents: budget,
|
||||
metadata: {
|
||||
commitSha: "a".repeat(40),
|
||||
scope: "Harmless tracer only",
|
||||
acceptanceCriteria: ["writes deterministic JSON; no release"],
|
||||
},
|
||||
},
|
||||
owner,
|
||||
);
|
||||
if (queue) {
|
||||
t = await workflow.transition(
|
||||
t.id,
|
||||
{
|
||||
toState: "GATE1_PENDING",
|
||||
expectedVersion: t.version,
|
||||
idempotencyKey: randomUUID(),
|
||||
},
|
||||
owner,
|
||||
);
|
||||
t = await workflow.decide(
|
||||
t.id,
|
||||
1,
|
||||
{
|
||||
decision: "APPROVED",
|
||||
expiresAt: new Date(Date.now() + 60000).toISOString(),
|
||||
expectedVersion: t.version,
|
||||
idempotencyKey: randomUUID(),
|
||||
},
|
||||
reviewer,
|
||||
);
|
||||
t = await workflow.transition(
|
||||
t.id,
|
||||
{
|
||||
toState: "READY",
|
||||
expectedVersion: t.version,
|
||||
idempotencyKey: randomUUID(),
|
||||
},
|
||||
owner,
|
||||
);
|
||||
}
|
||||
return t;
|
||||
}
|
||||
async function post(route: string, body: any = {}, secret = token) {
|
||||
const r = await fetch(base + "/internal/prepare-worker/" + route, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
authorization: "Bearer " + secret,
|
||||
"content-type": "application/json",
|
||||
},
|
||||
body: JSON.stringify(body),
|
||||
});
|
||||
return { status: r.status, body: await r.json() };
|
||||
}
|
||||
it("rejects overlapping worker and CI grants and accepts no credentials when disabled", async () => {
|
||||
expect(() =>
|
||||
WorkerTransportModule.register({
|
||||
enabled: true,
|
||||
grants: [
|
||||
{
|
||||
id: "both",
|
||||
role: "WORKFLOW_WORKER",
|
||||
credentialSha256: digest(token),
|
||||
},
|
||||
],
|
||||
workerIds: ["both"],
|
||||
stageCiIds: ["both"],
|
||||
artifactDirectory: directory,
|
||||
artifactBaseUrl: "https://isolated.invalid/artifacts",
|
||||
}),
|
||||
).toThrow(/overlapping/);
|
||||
const mod = await Test.createTestingModule({
|
||||
imports: [
|
||||
WorkerTransportModule.register({
|
||||
enabled: false,
|
||||
grants: [],
|
||||
workerIds: [],
|
||||
stageCiIds: [],
|
||||
artifactDirectory: directory,
|
||||
artifactBaseUrl: "https://isolated.invalid/artifacts",
|
||||
}),
|
||||
],
|
||||
})
|
||||
.overrideProvider(PrismaService)
|
||||
.useValue(db)
|
||||
.compile();
|
||||
const disabled = mod.createNestApplication();
|
||||
await disabled.listen(0, "127.0.0.1");
|
||||
try {
|
||||
const r = await fetch(
|
||||
(await disabled.getUrl()) + "/internal/prepare-worker/claim",
|
||||
{
|
||||
method: "POST",
|
||||
headers: {
|
||||
authorization: "Bearer " + token,
|
||||
"content-type": "application/json",
|
||||
},
|
||||
body: JSON.stringify({ taskId: randomUUID() }),
|
||||
},
|
||||
);
|
||||
expect(r.status).toBe(403);
|
||||
} finally {
|
||||
await disabled.close();
|
||||
}
|
||||
});
|
||||
it("denies untrusted identities and body impersonation; dispatches nothing before Gate1", async () => {
|
||||
const t = await proposal(false);
|
||||
expect((await post("claim", {}, "wrong")).status).toBe(401);
|
||||
expect((await post("claim", { taskId: t.id }, ciToken)).status).toBe(403);
|
||||
expect(
|
||||
(await post("claim", { taskId: t.id, actorId: "pilot" })).status,
|
||||
).toBe(400);
|
||||
expect((await post("claim", { taskId: t.id })).body).toEqual({
|
||||
lease: null,
|
||||
});
|
||||
expect(await db.orchestrationRun.count({ where: { taskId: t.id } })).toBe(
|
||||
0,
|
||||
);
|
||||
});
|
||||
it("races claims, fences expired leases and reserves unknown cost against the 20 cent cap", async () => {
|
||||
const t = await proposal();
|
||||
const claims = await Promise.all([
|
||||
post("claim", { taskId: t.id }),
|
||||
post("claim", { taskId: t.id }),
|
||||
]);
|
||||
const leases = claims.map((r) => r.body.lease).filter(Boolean);
|
||||
expect(leases).toHaveLength(1);
|
||||
const lease = leases[0],
|
||||
binding = {
|
||||
taskId: t.id,
|
||||
runId: lease.runId,
|
||||
leaseToken: lease.leaseToken,
|
||||
};
|
||||
expect(
|
||||
(await post("heartbeat", { ...binding, leaseToken: randomUUID() }))
|
||||
.status,
|
||||
).toBe(403);
|
||||
expect(
|
||||
(
|
||||
await post("complete", {
|
||||
...binding,
|
||||
report: "arbitrary shell or prompt",
|
||||
})
|
||||
).status,
|
||||
).toBe(400);
|
||||
expect(
|
||||
(await post("complete", { ...binding, report: "{}", actualCents: 21 }))
|
||||
.status,
|
||||
).toBe(400);
|
||||
await db.prepareWorkflowOutbox.update({
|
||||
where: { taskId: t.id },
|
||||
data: { leaseUntil: new Date(Date.now() - 1000) },
|
||||
});
|
||||
expect((await post("heartbeat", binding)).status).toBe(403);
|
||||
expect(
|
||||
(await post("complete", { ...binding, report: "{}" })).status,
|
||||
).toBe(403);
|
||||
expect((await post("claim", { taskId: t.id })).status).toBe(403);
|
||||
expect(await db.orchestrationRun.count({ where: { taskId: t.id } })).toBe(
|
||||
1,
|
||||
);
|
||||
expect(
|
||||
(
|
||||
await db.orchestrationBudgetLedger.aggregate({
|
||||
where: { taskId: t.id },
|
||||
_sum: { amountCents: true },
|
||||
})
|
||||
)._sum.amountCents,
|
||||
).toBe(20);
|
||||
const larger = await proposal(true, 40);
|
||||
expect((await post("claim", { taskId: larger.id })).status).toBe(403);
|
||||
});
|
||||
it("rechecks kill switch and task interrupt on heartbeat and completion; separates CI authority", async () => {
|
||||
const t = await proposal(),
|
||||
lease = (await post("claim", { taskId: t.id })).body.lease;
|
||||
const binding = {
|
||||
taskId: t.id,
|
||||
runId: lease.runId,
|
||||
leaseToken: lease.leaseToken,
|
||||
};
|
||||
await db.orchestrationKillSwitch.upsert({
|
||||
where: { scope: "GLOBAL" },
|
||||
create: {
|
||||
scope: "GLOBAL",
|
||||
level: "STOP_AGENTS",
|
||||
active: true,
|
||||
reason: "isolated test",
|
||||
actorId: "test",
|
||||
},
|
||||
update: { level: "STOP_AGENTS", active: true },
|
||||
});
|
||||
try {
|
||||
expect((await post("heartbeat", binding)).status).toBe(403);
|
||||
expect(
|
||||
(await post("complete", { ...binding, report: "{}" })).status,
|
||||
).toBe(403);
|
||||
expect((await post("claim", { taskId: t.id })).status).toBe(403);
|
||||
} finally {
|
||||
await db.orchestrationKillSwitch.deleteMany();
|
||||
}
|
||||
await db.orchestrationTask.update({
|
||||
where: { id: t.id },
|
||||
data: { state: "CANCELLED" },
|
||||
});
|
||||
expect((await post("heartbeat", binding)).status).toBe(403);
|
||||
expect(
|
||||
(await post("complete", { ...binding, report: "{}" })).status,
|
||||
).toBe(403);
|
||||
expect(
|
||||
await db.orchestrationArtifact.count({ where: { runId: lease.runId } }),
|
||||
).toBe(0);
|
||||
const evidence = {
|
||||
taskId: t.id,
|
||||
runId: lease.runId,
|
||||
commitSha: "a".repeat(40),
|
||||
sha256: "b".repeat(64),
|
||||
evidenceSha256: "c".repeat(64),
|
||||
uri: "https://isolated.invalid/not-real-ci",
|
||||
};
|
||||
expect((await post("stage-evidence", evidence)).status).toBe(403);
|
||||
expect((await post("stage-evidence", evidence, ciToken)).status).toBe(
|
||||
403,
|
||||
);
|
||||
});
|
||||
it("runs actual HTTP deterministic executor to VERIFYING with retrievable content-addressed artifact", async () => {
|
||||
const t = await proposal();
|
||||
const result = await pilot.runOnce({
|
||||
baseUrl: base,
|
||||
credential: token,
|
||||
taskId: t.id,
|
||||
allowLoopbackHttp: true,
|
||||
});
|
||||
expect(result.state).toBe("VERIFYING");
|
||||
const row = await db.orchestrationTask.findUniqueOrThrow({
|
||||
where: { id: t.id },
|
||||
});
|
||||
expect(row.state).toBe("VERIFYING");
|
||||
const artifact = await db.orchestrationArtifact.findFirstOrThrow({
|
||||
where: { run: { taskId: t.id } },
|
||||
});
|
||||
const bytes = await readFile(join(directory, artifact.sha256 + ".json"));
|
||||
expect(digest(bytes.toString())).toBe(artifact.sha256);
|
||||
expect(JSON.parse(bytes.toString())).toMatchObject({
|
||||
kind: "DETERMINISTIC_PREPARE_TRACER",
|
||||
commitSha: "a".repeat(40),
|
||||
environment: "staging",
|
||||
llmPilot: false,
|
||||
});
|
||||
const get = await fetch(
|
||||
base + "/internal/prepare-worker/artifacts/" + artifact.sha256,
|
||||
{ headers: { authorization: "Bearer " + token } },
|
||||
);
|
||||
expect(get.status).toBe(200);
|
||||
expect(await get.text()).toBe(bytes.toString());
|
||||
const cost = await db.orchestrationBudgetLedger.aggregate({
|
||||
where: { taskId: t.id },
|
||||
_sum: { amountCents: true },
|
||||
});
|
||||
expect(cost._sum.amountCents).toBe(0);
|
||||
expect(
|
||||
await db.orchestrationAttestation.findFirst({
|
||||
where: { run: { taskId: t.id }, kind: "STAGE" },
|
||||
}),
|
||||
).toBeNull();
|
||||
const evidence = {
|
||||
taskId: t.id,
|
||||
state: row.state,
|
||||
sha256: artifact.sha256,
|
||||
artifact: JSON.parse(bytes.toString()),
|
||||
httpGet: get.status,
|
||||
costCents: cost._sum.amountCents,
|
||||
uriExternalReachability: "NOT_TESTED_ISOLATED_FIXTURE_HOST",
|
||||
};
|
||||
console.log("TRACER_EVIDENCE", JSON.stringify(evidence));
|
||||
if (process.env.TRACER_EVIDENCE_PATH)
|
||||
await writeFile(
|
||||
process.env.TRACER_EVIDENCE_PATH,
|
||||
JSON.stringify(evidence, null, 2),
|
||||
);
|
||||
await writeFile(
|
||||
join(directory, artifact.sha256 + ".json"),
|
||||
'{"tampered":true}',
|
||||
);
|
||||
expect(
|
||||
(
|
||||
await fetch(
|
||||
base + "/internal/prepare-worker/artifacts/" + artifact.sha256,
|
||||
{ headers: { authorization: "Bearer " + token } },
|
||||
)
|
||||
).status,
|
||||
).toBe(409);
|
||||
});
|
||||
},
|
||||
);
|
||||
@@ -0,0 +1,28 @@
|
||||
# Bounded PREPARE worker transport (not an LLM pilot)
|
||||
|
||||
This is an opt-in Nest dynamic module and one-shot deterministic Node executor. Nothing imports it into AppModule. No live service, credential, dispatch schedule or deployment is created.
|
||||
|
||||
## Required parent wiring
|
||||
|
||||
- Import `WorkerTransportModule.register(config)` from `backend/src/orchestration-worker/worker-transport.module.ts` in the parent application only after review. Default disabled; provide empty grants until secret provisioning. The module owns its own PrepareWorkflowService/config; do not broaden existing orchestration providers.
|
||||
- Provision >=32 random-byte service credential through a mounted secret `/run/secrets/prepare_worker_credential` to the pilot, and provision only its SHA256 in server grants. Never accept identity/role from request bodies, JWT browser users, headers other than this credential, or a snapshot inventory.
|
||||
- `grants` is an explicit digest -> `{id,role}` mapping with independent `workerIds`/`stageCiIds`. Worker and CI identities/digests cannot overlap. Use a separate CI-only secret for `STAGE_CI`; do not give it to this executor.
|
||||
- Fixed config: absolute private writable `artifactDirectory`, authenticated HTTPS `artifactBaseUrl` pointing to `/internal/prepare-worker/artifacts`, separate persistent artifact volume. The fixture host `isolated.invalid` is NOT a real uploaded artifact URI. TLS routing/URI reachability must be verified by parent Stage CI.
|
||||
- Restrict routes to service network, TLS, ingress body limit (8KiB recommended), rate limit, no browser CORS credentials, no public user JWT access. The module validates strict DTO fields, but production ingress/body limits/rate controls remain parent's wiring.
|
||||
- Run `node infra/hermes-worker/pilot.mjs` with explicit `PREPARE_TASK_ID` UUID and `PREPARE_WORKER_BASE_URL` HTTPS origin. It consumes exactly one task/lease, has 5s per HTTP timeout, does not retry, and exits on SIGINT/SIGTERM. No polling or global dispatch. Disable all egress except backend; do not mount shell/Docker/browser credentials or repo write access. Loopback HTTP is available only via the exported test API option, not CLI.
|
||||
|
||||
## Contract and bounds
|
||||
|
||||
POST `/internal/prepare-worker/claim`: `{taskId}` -> `{lease:null|{...}}`; heartbeat: `{taskId,runId,leaseToken}`; complete: same binding plus `report` (max 2048 chars). Server accepts only byte-exact deterministic tracer JSON derived from the immutable run; it computes SHA256, writes real content, and chooses artifact URI/cost (0 cents), not client-supplied costs/URIs/pass flags. It does **not** inspect repository contents or validate application acceptance criteria. QA here means this harmless tracer succeeded, not application QA.
|
||||
|
||||
Core service still owns transactional Gate1, scope, kill checks, task locks, lease fencing, reservations and VERIFYING event. Transport requires exactly 20-cent task budgets and explicit task IDs. One expired unknown-cost reservation exhausts this pilot budget; do not release it to permit retries. Interrupt means cancellation/non-RUNNING state or STOP_AGENTS; heartbeat before compute, before complete, and atomic completion recheck. File writes may leave unregistered orphan content if the final transaction is denied; retrieval requires a registered QA artifact and verifies its bytes against the requested digest.
|
||||
|
||||
POST `/internal/prepare-worker/stage-evidence` has a separate strict CI DTO `{taskId,runId,commitSha,sha256,evidenceSha256,uri}` and calls independently authenticated core Stage evidence ingestion. This tracer never submits it. Valid CI evidence retrieval/provenance remains CI's responsibility. No Gate2 approval/release transport exists.
|
||||
|
||||
Shared department budgets are not implemented by core and are not claimed here. Do not expand beyond a single bounded pilot until aggregate budgeting and live provisioning are reviewed.
|
||||
|
||||
## Real isolated verification
|
||||
|
||||
Run `bash infra/hermes-worker/test-isolated.sh green`. It creates no-network disposable PostgreSQL16, applies only the three repository v4 migrations, copies readonly source and Node22 dependency donor into private writable storage, regenerates Prisma, runs actual Nest TCP HTTP and a child Node executor, then full backend compilation. Docker DB/anonymous volume removed on exit. No live tables/containers/volumes, RELEASE_AUTHORIZE, push or deploy touched.
|
||||
|
||||
Evidence: `/opt/services/canina/validation/v4-worker-transport/`: `red.log` (missing module initial RED), `integrity-red.log` (real 200 vs expected409 corruption regression), `green.log`, `build.log`, `generate.log`, `tracer-evidence.json` (actual artifact content/hash). Final tests cover credential/body identity denial, disabled/overlap config, Gate1 no-dispatch, competing claims, expired/token fencing, retained20-cent budget, interrupt/kill rechecks, CI separation, real executor to VERIFYING, authenticated artifact GET and tamper rejection. Fixture commit is 40 `a` characters; this is not a repository QA claim or live Stage deployment.
|
||||
@@ -0,0 +1,37 @@
|
||||
import { readFile } from 'node:fs/promises';
|
||||
import { pathToFileURL } from 'node:url';
|
||||
|
||||
/** One explicit task, one attempt, fixed computation; no shell, prompt, browser or LLM. */
|
||||
export async function runOnce({baseUrl, credential, taskId, allowLoopbackHttp=false, signal}) {
|
||||
const url = new URL(baseUrl);
|
||||
if (url.username || url.password || url.search || url.hash || (url.protocol !== 'https:' && !(allowLoopbackHttp && url.protocol === 'http:' && ['127.0.0.1','[::1]'].includes(url.hostname)))) throw new Error('HTTPS service origin required');
|
||||
if (!credential || credential.length < 32) throw new Error('Service credential required');
|
||||
async function post(route, body) {
|
||||
signal?.throwIfAborted();
|
||||
const timeout=AbortSignal.timeout(5000);
|
||||
const r=await fetch(new URL('/internal/prepare-worker/'+route,url), {method:'POST',redirect:'error',signal:signal ? AbortSignal.any([signal,timeout]) : timeout,headers:{authorization:'Bearer '+credential,'content-type':'application/json'},body:JSON.stringify(body)});
|
||||
const text=await r.text();
|
||||
if(text.length>8192) throw new Error('Oversized service response');
|
||||
if(!r.ok) throw new Error('Worker request denied: '+r.status);
|
||||
return JSON.parse(text);
|
||||
}
|
||||
const {lease}=await post('claim',{taskId});
|
||||
if(!lease) return {state:'IDLE'};
|
||||
if(lease.taskId!==taskId || lease.environment!=='staging' || lease.maxCostCents!==20 || !/^[a-f0-9]{40}$/.test(lease.commitSha)) throw new Error('Unbounded lease rejected');
|
||||
const binding={taskId,runId:lease.runId,leaseToken:lease.leaseToken};
|
||||
await post('heartbeat',binding); // stop/interrupt recheck before any computation
|
||||
signal?.throwIfAborted();
|
||||
const report=JSON.stringify({kind:'DETERMINISTIC_PREPARE_TRACER',schemaVersion:1,taskId,runId:lease.runId,commitSha:lease.commitSha,environment:'staging',llmPilot:false,browserExecuted:false,shellExecuted:false,deploymentExecuted:false,checks:['lease-bound identity','deterministic JSON serialization'],actualCents:0});
|
||||
await post('heartbeat',binding); // stop/interrupt recheck immediately before completion
|
||||
return post('complete',{...binding,report});
|
||||
}
|
||||
if (process.argv[1] && import.meta.url===pathToFileURL(process.argv[1]).href) {
|
||||
const controller=new AbortController();
|
||||
process.once('SIGINT',()=>controller.abort());
|
||||
process.once('SIGTERM',()=>controller.abort());
|
||||
try {
|
||||
const credential=(await readFile('/run/secrets/prepare_worker_credential','utf8')).trim();
|
||||
const result=await runOnce({baseUrl:process.env.PREPARE_WORKER_BASE_URL,credential,taskId:process.env.PREPARE_TASK_ID,signal:controller.signal});
|
||||
console.log(JSON.stringify(result));
|
||||
} catch { console.error('Bounded PREPARE tracer failed; no completion assumed'); process.exitCode=1; }
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
||||
E="${WORKER_EVIDENCE_DIR:-/opt/services/canina/validation/v4-worker-transport}"
|
||||
mkdir -p "$E"
|
||||
DB="v4-worker-test-${RANDOM}"
|
||||
trap 'docker rm -fv "$DB" >/dev/null 2>&1 || true' EXIT
|
||||
docker run -d --name "$DB" --network none -e POSTGRES_PASSWORD=disposable -e POSTGRES_DB=worker postgres:16-alpine >/dev/null
|
||||
for n in $(seq 1 60); do if docker exec "$DB" pg_isready -U postgres >/dev/null 2>&1; then break; fi; sleep 1; done
|
||||
for migration in 20261022000000_add_orchestration_v4_core 20261022190000_harden_orchestration_ledger 20261023170000_prepare_workflow_outbox; do
|
||||
docker exec -i "$DB" psql -v ON_ERROR_STOP=1 -U postgres -d worker < "$ROOT/backend/prisma/migrations/$migration/migration.sql" > "$E/$migration.log"
|
||||
done
|
||||
docker run --rm --userns=host --network "container:$DB" -e DATABASE_URL=postgresql://postgres:disposable@127.0.0.1:5432/worker -e WORKER_TRANSPORT_DB=1 -e TRACER_EVIDENCE_PATH=/evidence/tracer-evidence.json -e TMPDIR=/work/tmp -v "$ROOT:/source:ro" -v /opt/services/canina/worktrees/orchestration-v4-runtime/backend/node_modules:/deps:ro -v "$E:/evidence" node:22-bookworm bash -c '
|
||||
set -euo pipefail
|
||||
mkdir -p /work/backend /work/infra /work/tmp
|
||||
cp -a /source/backend/. /work/backend/
|
||||
cp -a /source/infra/. /work/infra/
|
||||
cp -a /deps /work/backend/node_modules
|
||||
cd /work/backend
|
||||
./node_modules/.bin/prisma generate > /evidence/generate.log 2>&1
|
||||
npm test -- --runInBand orchestration-worker > /evidence/'"${1:-green}"'.log 2>&1
|
||||
npm run build > /evidence/build.log 2>&1
|
||||
'
|
||||
Reference in new issue
Block a user