fix(worker): scope artifact access and harden immutable transport storage

This commit is contained in:
devops committed 2026-10-23 14:07:01 +00:00
1 parent aa99216c49
commit 9d93707848
3 files changed
+193 -32

No files matched your search

@@ -21,8 +21,9 @@ import {
} from "@nestjs/common";
import { IsString, IsUUID, Matches, MaxLength } from "class-validator";
import { createHash, timingSafeEqual } from "crypto";
import { mkdir, readFile, writeFile } from "fs/promises";
import { join, isAbsolute } from "path";
import { mkdir, open, FileHandle } from "fs/promises";
import { constants } from "fs";
import { isAbsolute } from "path";
import { PrismaModule } from "../prisma/prisma.module";
import { PrismaService } from "../prisma/prisma.service";
import {
@@ -31,6 +32,20 @@ import {
WorkflowIdentity,
} from "../orchestration/prepare-workflow.service";
// This is reviewed static configuration, not a URL fetched by the transport.
function validArtifactBase(value: string): boolean {
try {
const u = new URL(value);
// DNS name only: reject all IP literals, localhost/private single-label names,
// noncanonical paths/escapes, credentials, query, fragment and alternate ports.
return u.protocol === "https:" && !u.username && !u.password && !u.port &&
/^[a-z0-9](?:[a-z0-9.-]*[a-z0-9])?$/.test(u.hostname) &&
u.hostname.includes(".") && !/^[0-9.]+$/.test(u.hostname) &&
!/(^|\.)(localhost|local|internal|lan|home)$/.test(u.hostname) &&
u.pathname === "/internal/prepare-worker/artifacts" &&
value === u.origin + u.pathname;
} catch { return false; }
}
const CONFIG = Symbol("WORKER_TRANSPORT_CONFIG");
export interface WorkerTransportConfig {
enabled: boolean;
@@ -107,6 +122,25 @@ class WorkerTransportService {
private readonly db: PrismaService,
@Inject(CONFIG) private readonly config: WorkerTransportConfig,
) {}
// Linux descriptor-relative traversal pins each directory across renames. Never
// realpath-check then reopen by name: that reintroduces a symlink race.
private async artifactDir(): Promise<FileHandle> {
let dir = await open("/", constants.O_RDONLY | constants.O_DIRECTORY);
try {
for (const part of this.config.artifactDirectory.split("/").filter(Boolean)) {
const path = `/proc/self/fd/${dir.fd}/${part}`;
try { await mkdir(path, { mode: 0o700 }); }
catch (e: any) { if (e.code !== "EEXIST") throw e; }
const next = await open(path, constants.O_RDONLY | constants.O_DIRECTORY | constants.O_NOFOLLOW);
await dir.close();
dir = next;
}
const stat = await dir.stat();
if (stat.uid !== process.getuid!() || (stat.mode & 0o077) !== 0)
throw new Error("Private owned artifact directory required");
return dir;
} catch (e) { await dir.close(); throw e; }
}
private worker(a: WorkflowIdentity) {
if (a.role !== "WORKFLOW_WORKER" || !this.config.workerIds.includes(a.id))
throw new ForbiddenException("Worker allowlist required");
@@ -139,15 +173,15 @@ class WorkerTransportService {
"Only deterministic harmless tracer report accepted",
);
const sha256 = createHash("sha256").update(expected).digest("hex");
await mkdir(this.config.artifactDirectory, {
recursive: true,
mode: 0o700,
});
await writeFile(
join(this.config.artifactDirectory, sha256 + ".json"),
expected,
{ flag: "w", mode: 0o600 },
);
let dir: FileHandle | undefined, file: FileHandle | undefined;
try {
dir = await this.artifactDir();
file = await open(`/proc/self/fd/${dir.fd}/${sha256}.json`,
constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, 0o600);
await file.writeFile(expected, "utf8");
await file.sync();
} catch { throw new ConflictException("Artifact creation refused"); }
finally { await file?.close(); await dir?.close(); }
const result = await this.workflow.complete(a, {
...dto,
sha256,
@@ -162,24 +196,49 @@ class WorkerTransportService {
actualCents: 0,
};
}
async artifact(hash: string) {
async artifact(a: WorkflowIdentity, hash: string) {
if (!/^[a-f0-9]{64}$/.test(hash))
throw new BadRequestException("Invalid artifact hash");
const registered = await this.db.orchestrationArtifact.findFirst({
where: { sha256: hash, kind: "QA" },
where: {
sha256: hash, kind: "QA", uri: this.config.artifactBaseUrl + "/" + hash,
run: { environment: "staging", status: "SUCCEEDED", task: {
operationMode: "PREPARE", budgetLimitCents: 20,
} },
},
});
if (!registered) throw new NotFoundException("Artifact not registered");
const run = await this.db.orchestrationRun.findUniqueOrThrow({ where: { id: registered.runId } });
const task = await this.db.orchestrationTask.findUniqueOrThrow({ where: { id: run.taskId } });
const outbox = await this.db.prepareWorkflowOutbox.findUnique({ where: { taskId: run.taskId } });
if (!outbox || outbox.status !== "DONE" || outbox.runId !== run.id ||
(a.role === "WORKFLOW_WORKER" && outbox.leaseOwner !== a.id))
throw new NotFoundException("Artifact not registered");
const metadata = task.metadata as any;
if (metadata?.workflowScope !== "STAGE_PREPARE" || metadata?.commitSha !== run.commitSha)
throw new NotFoundException("Artifact not registered");
let bytes: string;
let dir: FileHandle | undefined, file: FileHandle | undefined;
try {
bytes = await readFile(
join(this.config.artifactDirectory, hash + ".json"),
"utf8",
);
dir = await this.artifactDir();
file = await open(`/proc/self/fd/${dir.fd}/${hash}.json`,
constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK);
const stat = await file.stat();
if (!stat.isFile() || stat.nlink !== 1 || stat.uid !== process.getuid!() ||
(stat.mode & 0o077) !== 0 || stat.size > 2048)
throw new Error("Unsafe artifact file");
// Fixed-size descriptor read: concurrent growth cannot allocate unbounded memory.
const buffer = Buffer.alloc(2049);
const { bytesRead } = await file.read(buffer, 0, buffer.length, 0);
if (bytesRead > 2048) throw new Error("Oversized artifact");
bytes = buffer.subarray(0, bytesRead).toString("utf8");
} catch {
throw new NotFoundException("Artifact unavailable");
}
} finally { await file?.close(); await dir?.close(); }
if (createHash("sha256").update(bytes).digest("hex") !== hash)
throw new ConflictException("Artifact integrity mismatch");
if (bytes !== tracerReport({ taskId: run.taskId, runId: run.id, commitSha: run.commitSha }))
throw new ConflictException("Artifact binding mismatch");
return JSON.parse(bytes);
}
stage(a: WorkflowIdentity, dto: StageEvidenceDto) {
@@ -212,18 +271,26 @@ class WorkerTransportController {
) {
return this.service.stage(req.workerIdentity, dto);
}
@Get("artifacts/:hash") artifact(@Param("hash") hash: string) {
return this.service.artifact(hash);
@Get("artifacts/:hash") artifact(@Req() req: any, @Param("hash") hash: string) {
return this.service.artifact(req.workerIdentity, hash);
}
}
@Module({})
export class WorkerTransportModule {
static register(config: WorkerTransportConfig): DynamicModule {
if (typeof config.enabled !== "boolean" || !Array.isArray(config.grants) ||
!Array.isArray(config.workerIds) || !Array.isArray(config.stageCiIds))
throw new Error("Explicit boolean enable and credential allowlists required");
const allIds = [...config.workerIds, ...config.stageCiIds];
if (allIds.some(id => typeof id !== "string" || !/^[a-zA-Z0-9_-]{1,100}$/.test(id)) ||
new Set(allIds).size !== allIds.length)
throw new Error("Invalid or overlapping worker/CI allowlists");
const ids = new Set<string>(),
digests = new Set<string>();
if (
!isAbsolute(config.artifactDirectory) ||
!/^https:\/\/[^\s?#]+$/.test(config.artifactBaseUrl)
!isAbsolute(config.artifactDirectory) || config.artifactDirectory === "/" ||
config.artifactDirectory.split("/").some(p => p === "." || p === "..") ||
!validArtifactBase(config.artifactBaseUrl)
)
throw new Error(
"Absolute artifact directory and fixed HTTPS artifact base required",
@@ -247,9 +314,12 @@ export class WorkerTransportModule {
ids.add(g.id);
digests.add(g.credentialSha256);
}
const snapshot = JSON.parse(
JSON.stringify(config),
) as WorkerTransportConfig;
const snapshot = JSON.parse(JSON.stringify(config)) as WorkerTransportConfig;
snapshot.grants.forEach(Object.freeze);
Object.freeze(snapshot.grants);
Object.freeze(snapshot.workerIds);
Object.freeze(snapshot.stageCiIds);
Object.freeze(snapshot);
return {
module: WorkerTransportModule,
imports: [PrismaModule],
@@ -261,10 +331,10 @@ export class WorkerTransportModule {
{ provide: CONFIG, useValue: snapshot },
{
provide: PREPARE_WORKFLOW_CONFIG,
useValue: {
useValue: Object.freeze({
workerIds: snapshot.workerIds,
stageCiIds: snapshot.stageCiIds,
},
}),
},
],
exports: [PrepareWorkflowService],
@@ -3,9 +3,9 @@ import { INestApplication } from "@nestjs/common";
import { PrismaClient } from "@prisma/client";
import { PrismaService } from "../prisma/prisma.service";
import { PrepareWorkflowService } from "../orchestration/prepare-workflow.service";
import { WorkerTransportModule } from "./worker-transport.module";
import { WorkerTransportModule, tracerReport } from "./worker-transport.module";
import { createHash, randomUUID } from "crypto";
import { mkdtemp, readFile, rm, writeFile } from "fs/promises";
import { mkdtemp, readFile, rm, writeFile, symlink, link, rename } from "fs/promises";
import { tmpdir } from "os";
import { join } from "path";
import { execFile } from "child_process";
@@ -37,8 +37,9 @@ const enabled = process.env.WORKER_TRANSPORT_DB === "1";
credentialSha256: digest(token),
},
{ id: "ci", role: "STAGE_CI", credentialSha256: digest(ciToken) },
{ id: "other", role: "WORKFLOW_WORKER", credentialSha256: digest("other-worker-secret-".repeat(3)) },
],
workerIds: ["pilot"],
workerIds: ["pilot", "other"],
stageCiIds: ["ci"],
artifactDirectory: directory,
artifactBaseUrl:
@@ -153,7 +154,7 @@ const enabled = process.env.WORKER_TRANSPORT_DB === "1";
workerIds: ["both"],
stageCiIds: ["both"],
artifactDirectory: directory,
artifactBaseUrl: "https://isolated.invalid/artifacts",
artifactBaseUrl: "https://isolated.invalid/internal/prepare-worker/artifacts",
}),
).toThrow(/overlapping/);
const mod = await Test.createTestingModule({
@@ -164,7 +165,7 @@ const enabled = process.env.WORKER_TRANSPORT_DB === "1";
workerIds: [],
stageCiIds: [],
artifactDirectory: directory,
artifactBaseUrl: "https://isolated.invalid/artifacts",
artifactBaseUrl: "https://isolated.invalid/internal/prepare-worker/artifacts",
}),
],
})
@@ -190,6 +191,21 @@ const enabled = process.env.WORKER_TRANSPORT_DB === "1";
await disabled.close();
}
});
it("rejects unsafe artifact bases and freezes validated configuration", () => {
const config = { enabled: true, grants: [], workerIds: [], stageCiIds: [], artifactDirectory: directory,
artifactBaseUrl: "https://isolated.invalid/internal/prepare-worker/artifacts" };
for (const artifactBaseUrl of ["https://user:pass@isolated.invalid/internal/prepare-worker/artifacts", "https://127.0.0.1/internal/prepare-worker/artifacts", "https://10.1.2.3/internal/prepare-worker/artifacts", "https://localhost/internal/prepare-worker/artifacts", "https://isolated.invalid/arbitrary", "https://isolated.invalid/x/../internal/prepare-worker/artifacts", "https://[::1]/internal/prepare-worker/artifacts"]) {
expect(() => WorkerTransportModule.register({ ...config, artifactBaseUrl })).toThrow();
}
const registered = WorkerTransportModule.register(config);
const snapshot = (registered.providers as any[]).find(p => p.useValue?.artifactBaseUrl)?.useValue;
expect(Object.isFrozen(snapshot)).toBe(true);
expect(Object.isFrozen(snapshot.workerIds)).toBe(true);
expect(Object.isFrozen(snapshot.grants)).toBe(true);
const workflowConfig = (registered.providers as any[]).find(p => p.provide === "PREPARE_WORKFLOW_CONFIG").useValue;
expect(Object.isFrozen(workflowConfig)).toBe(true);
expect(() => WorkerTransportModule.register({ ...config, enabled: "true" as any })).toThrow();
});
it("denies untrusted identities and body impersonation; dispatches nothing before Gate1", async () => {
const t = await proposal(false);
expect((await post("claim", {}, "wrong")).status).toBe(401);
@@ -309,6 +325,23 @@ const enabled = process.env.WORKER_TRANSPORT_DB === "1";
403,
);
});
it("never follows or overwrites preexisting artifact links on completion", async () => {
const t = await proposal();
const lease = (await post("claim", { taskId: t.id })).body.lease;
const report = tracerReport(lease), hash = digest(report);
const target = join(directory, "sentinel");
await writeFile(target, "do not overwrite");
const file = join(directory, hash + ".json");
await symlink(target, file);
const result = await post("complete", { taskId: t.id, runId: lease.runId, leaseToken: lease.leaseToken, report });
expect(await readFile(target, "utf8")).toBe("do not overwrite");
expect(result.status).toBe(409);
expect(await db.orchestrationArtifact.count({ where: { runId: lease.runId } })).toBe(0);
await rm(file);
await link(target, file);
expect((await post("complete", { taskId: t.id, runId: lease.runId, leaseToken: lease.leaseToken, report })).status).toBe(409);
expect(await readFile(target, "utf8")).toBe("do not overwrite");
});
it("runs actual HTTP deterministic executor to VERIFYING with retrievable content-addressed artifact", async () => {
const t = await proposal();
const result = await pilot.runOnce({
@@ -339,6 +372,30 @@ const enabled = process.env.WORKER_TRANSPORT_DB === "1";
);
expect(get.status).toBe(200);
expect(await get.text()).toBe(bytes.toString());
const artifactGet = (secret: string) => fetch(base + "/internal/prepare-worker/artifacts/" + artifact.sha256,
{ headers: { authorization: "Bearer " + secret } });
expect((await artifactGet("other-worker-secret-".repeat(3))).status).toBe(404);
expect((await artifactGet(ciToken)).status).toBe(200);
const foreignHash = digest("{}");
await writeFile(join(directory, foreignHash + ".json"), "{}");
await db.orchestrationArtifact.create({ data: { runId: artifact.runId, kind: "QA", sha256: foreignHash, uri: "https://foreign.invalid/" + foreignHash } });
expect((await fetch(base + "/internal/prepare-worker/artifacts/" + foreignHash, { headers: { authorization: "Bearer " + token } })).status).toBe(404);
const moved = directory + "-moved";
await rename(directory, moved);
try {
await symlink(moved, directory);
expect((await artifactGet(token)).status).toBe(404);
} finally { await rm(directory, { force: true }); await rename(moved, directory); }
const file = join(directory, artifact.sha256 + ".json"), saved = file + ".saved";
await rename(file, saved);
try {
await symlink(saved, file);
expect((await artifactGet(token)).status).toBe(404);
await rm(file);
await link(saved, file);
expect((await artifactGet(token)).status).toBe(404);
await rm(file);
} finally { await rm(file, { force: true }); await rename(saved, file); }
const cost = await db.orchestrationBudgetLedger.aggregate({
where: { taskId: t.id },
_sum: { amountCents: true },