fix(worker): scope artifact access and harden immutable transport storage

This commit is contained in:
devops committed 2026-10-23 14:05:53 +00:00
1 parent f7048ab832
commit faa17650d3
3 files changed
+193 -32

No files matched your search

@@ -21,8 +21,9 @@ import {
} from "@nestjs/common";
import { IsString, IsUUID, Matches, MaxLength } from "class-validator";
import { createHash, timingSafeEqual } from "crypto";
import { mkdir, readFile, writeFile } from "fs/promises";
import { join, isAbsolute } from "path";
import { mkdir, open, FileHandle } from "fs/promises";
import { constants } from "fs";
import { isAbsolute } from "path";
import { PrismaModule } from "../prisma/prisma.module";
import { PrismaService } from "../prisma/prisma.service";
import {
@@ -31,6 +32,20 @@ import {
WorkflowIdentity,
} from "../orchestration/prepare-workflow.service";
// This is reviewed static configuration, not a URL fetched by the transport.
function validArtifactBase(value: string): boolean {
try {
const u = new URL(value);
// DNS name only: reject all IP literals, localhost/private single-label names,
// noncanonical paths/escapes, credentials, query, fragment and alternate ports.
return u.protocol === "https:" && !u.username && !u.password && !u.port &&
/^[a-z0-9](?:[a-z0-9.-]*[a-z0-9])?$/.test(u.hostname) &&
u.hostname.includes(".") && !/^[0-9.]+$/.test(u.hostname) &&
!/(^|\.)(localhost|local|internal|lan|home)$/.test(u.hostname) &&
u.pathname === "/internal/prepare-worker/artifacts" &&
value === u.origin + u.pathname;
} catch { return false; }
}
const CONFIG = Symbol("WORKER_TRANSPORT_CONFIG");
export interface WorkerTransportConfig {
enabled: boolean;
@@ -107,6 +122,25 @@ class WorkerTransportService {
private readonly db: PrismaService,
@Inject(CONFIG) private readonly config: WorkerTransportConfig,
) {}
// Linux descriptor-relative traversal pins each directory across renames. Never
// realpath-check then reopen by name: that reintroduces a symlink race.
private async artifactDir(): Promise<FileHandle> {
let dir = await open("/", constants.O_RDONLY | constants.O_DIRECTORY);
try {
for (const part of this.config.artifactDirectory.split("/").filter(Boolean)) {
const path = `/proc/self/fd/${dir.fd}/${part}`;
try { await mkdir(path, { mode: 0o700 }); }
catch (e: any) { if (e.code !== "EEXIST") throw e; }
const next = await open(path, constants.O_RDONLY | constants.O_DIRECTORY | constants.O_NOFOLLOW);
await dir.close();
dir = next;
}
const stat = await dir.stat();
if (stat.uid !== process.getuid!() || (stat.mode & 0o077) !== 0)
throw new Error("Private owned artifact directory required");
return dir;
} catch (e) { await dir.close(); throw e; }
}
private worker(a: WorkflowIdentity) {
if (a.role !== "WORKFLOW_WORKER" || !this.config.workerIds.includes(a.id))
throw new ForbiddenException("Worker allowlist required");
@@ -139,15 +173,15 @@ class WorkerTransportService {
"Only deterministic harmless tracer report accepted",
);
const sha256 = createHash("sha256").update(expected).digest("hex");
await mkdir(this.config.artifactDirectory, {
recursive: true,
mode: 0o700,
});
await writeFile(
join(this.config.artifactDirectory, sha256 + ".json"),
expected,
{ flag: "w", mode: 0o600 },
);
let dir: FileHandle | undefined, file: FileHandle | undefined;
try {
dir = await this.artifactDir();
file = await open(`/proc/self/fd/${dir.fd}/${sha256}.json`,
constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, 0o600);
await file.writeFile(expected, "utf8");
await file.sync();
} catch { throw new ConflictException("Artifact creation refused"); }
finally { await file?.close(); await dir?.close(); }
const result = await this.workflow.complete(a, {
...dto,
sha256,
@@ -162,24 +196,49 @@ class WorkerTransportService {
actualCents: 0,
};
}
async artifact(hash: string) {
async artifact(a: WorkflowIdentity, hash: string) {
if (!/^[a-f0-9]{64}$/.test(hash))
throw new BadRequestException("Invalid artifact hash");
const registered = await this.db.orchestrationArtifact.findFirst({
where: { sha256: hash, kind: "QA" },
where: {
sha256: hash, kind: "QA", uri: this.config.artifactBaseUrl + "/" + hash,
run: { environment: "staging", status: "SUCCEEDED", task: {
operationMode: "PREPARE", budgetLimitCents: 20,
} },
},
});
if (!registered) throw new NotFoundException("Artifact not registered");
const run = await this.db.orchestrationRun.findUniqueOrThrow({ where: { id: registered.runId } });
const task = await this.db.orchestrationTask.findUniqueOrThrow({ where: { id: run.taskId } });
const outbox = await this.db.prepareWorkflowOutbox.findUnique({ where: { taskId: run.taskId } });
if (!outbox || outbox.status !== "DONE" || outbox.runId !== run.id ||
(a.role === "WORKFLOW_WORKER" && outbox.leaseOwner !== a.id))
throw new NotFoundException("Artifact not registered");
const metadata = task.metadata as any;
if (metadata?.workflowScope !== "STAGE_PREPARE" || metadata?.commitSha !== run.commitSha)
throw new NotFoundException("Artifact not registered");
let bytes: string;
let dir: FileHandle | undefined, file: FileHandle | undefined;
try {
bytes = await readFile(
join(this.config.artifactDirectory, hash + ".json"),
"utf8",
);
dir = await this.artifactDir();
file = await open(`/proc/self/fd/${dir.fd}/${hash}.json`,
constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK);
const stat = await file.stat();
if (!stat.isFile() || stat.nlink !== 1 || stat.uid !== process.getuid!() ||
(stat.mode & 0o077) !== 0 || stat.size > 2048)
throw new Error("Unsafe artifact file");
// Fixed-size descriptor read: concurrent growth cannot allocate unbounded memory.
const buffer = Buffer.alloc(2049);
const { bytesRead } = await file.read(buffer, 0, buffer.length, 0);
if (bytesRead > 2048) throw new Error("Oversized artifact");
bytes = buffer.subarray(0, bytesRead).toString("utf8");
} catch {
throw new NotFoundException("Artifact unavailable");
}
} finally { await file?.close(); await dir?.close(); }
if (createHash("sha256").update(bytes).digest("hex") !== hash)
throw new ConflictException("Artifact integrity mismatch");
if (bytes !== tracerReport({ taskId: run.taskId, runId: run.id, commitSha: run.commitSha }))
throw new ConflictException("Artifact binding mismatch");
return JSON.parse(bytes);
}
stage(a: WorkflowIdentity, dto: StageEvidenceDto) {
@@ -212,18 +271,26 @@ class WorkerTransportController {
) {
return this.service.stage(req.workerIdentity, dto);
}
@Get("artifacts/:hash") artifact(@Param("hash") hash: string) {
return this.service.artifact(hash);
@Get("artifacts/:hash") artifact(@Req() req: any, @Param("hash") hash: string) {
return this.service.artifact(req.workerIdentity, hash);
}
}
@Module({})
export class WorkerTransportModule {
static register(config: WorkerTransportConfig): DynamicModule {
if (typeof config.enabled !== "boolean" || !Array.isArray(config.grants) ||
!Array.isArray(config.workerIds) || !Array.isArray(config.stageCiIds))
throw new Error("Explicit boolean enable and credential allowlists required");
const allIds = [...config.workerIds, ...config.stageCiIds];
if (allIds.some(id => typeof id !== "string" || !/^[a-zA-Z0-9_-]{1,100}$/.test(id)) ||
new Set(allIds).size !== allIds.length)
throw new Error("Invalid or overlapping worker/CI allowlists");
const ids = new Set<string>(),
digests = new Set<string>();
if (
!isAbsolute(config.artifactDirectory) ||
!/^https:\/\/[^\s?#]+$/.test(config.artifactBaseUrl)
!isAbsolute(config.artifactDirectory) || config.artifactDirectory === "/" ||
config.artifactDirectory.split("/").some(p => p === "." || p === "..") ||
!validArtifactBase(config.artifactBaseUrl)
)
throw new Error(
"Absolute artifact directory and fixed HTTPS artifact base required",
@@ -247,9 +314,12 @@ export class WorkerTransportModule {
ids.add(g.id);
digests.add(g.credentialSha256);
}
const snapshot = JSON.parse(
JSON.stringify(config),
) as WorkerTransportConfig;
const snapshot = JSON.parse(JSON.stringify(config)) as WorkerTransportConfig;
snapshot.grants.forEach(Object.freeze);
Object.freeze(snapshot.grants);
Object.freeze(snapshot.workerIds);
Object.freeze(snapshot.stageCiIds);
Object.freeze(snapshot);
return {
module: WorkerTransportModule,
imports: [PrismaModule],
@@ -261,10 +331,10 @@ export class WorkerTransportModule {
{ provide: CONFIG, useValue: snapshot },
{
provide: PREPARE_WORKFLOW_CONFIG,
useValue: {
useValue: Object.freeze({
workerIds: snapshot.workerIds,
stageCiIds: snapshot.stageCiIds,
},
}),
},
],
exports: [PrepareWorkflowService],
@@ -3,9 +3,9 @@ import { INestApplication } from "@nestjs/common";
import { PrismaClient } from "@prisma/client";
import { PrismaService } from "../prisma/prisma.service";
import { PrepareWorkflowService } from "../orchestration/prepare-workflow.service";
import { WorkerTransportModule } from "./worker-transport.module";
import { WorkerTransportModule, tracerReport } from "./worker-transport.module";
import { createHash, randomUUID } from "crypto";
import { mkdtemp, readFile, rm, writeFile } from "fs/promises";
import { mkdtemp, readFile, rm, writeFile, symlink, link, rename } from "fs/promises";
import { tmpdir } from "os";
import { join } from "path";
import { execFile } from "child_process";
@@ -37,8 +37,9 @@ const enabled = process.env.WORKER_TRANSPORT_DB === "1";
credentialSha256: digest(token),
},
{ id: "ci", role: "STAGE_CI", credentialSha256: digest(ciToken) },
{ id: "other", role: "WORKFLOW_WORKER", credentialSha256: digest("other-worker-secret-".repeat(3)) },
],
workerIds: ["pilot"],
workerIds: ["pilot", "other"],
stageCiIds: ["ci"],
artifactDirectory: directory,
artifactBaseUrl:
@@ -153,7 +154,7 @@ const enabled = process.env.WORKER_TRANSPORT_DB === "1";
workerIds: ["both"],
stageCiIds: ["both"],
artifactDirectory: directory,
artifactBaseUrl: "https://isolated.invalid/artifacts",
artifactBaseUrl: "https://isolated.invalid/internal/prepare-worker/artifacts",
}),
).toThrow(/overlapping/);
const mod = await Test.createTestingModule({
@@ -164,7 +165,7 @@ const enabled = process.env.WORKER_TRANSPORT_DB === "1";
workerIds: [],
stageCiIds: [],
artifactDirectory: directory,
artifactBaseUrl: "https://isolated.invalid/artifacts",
artifactBaseUrl: "https://isolated.invalid/internal/prepare-worker/artifacts",
}),
],
})
@@ -190,6 +191,21 @@ const enabled = process.env.WORKER_TRANSPORT_DB === "1";
await disabled.close();
}
});
it("rejects unsafe artifact bases and freezes validated configuration", () => {
const config = { enabled: true, grants: [], workerIds: [], stageCiIds: [], artifactDirectory: directory,
artifactBaseUrl: "https://isolated.invalid/internal/prepare-worker/artifacts" };
for (const artifactBaseUrl of ["https://user:pass@isolated.invalid/internal/prepare-worker/artifacts", "https://127.0.0.1/internal/prepare-worker/artifacts", "https://10.1.2.3/internal/prepare-worker/artifacts", "https://localhost/internal/prepare-worker/artifacts", "https://isolated.invalid/arbitrary", "https://isolated.invalid/x/../internal/prepare-worker/artifacts", "https://[::1]/internal/prepare-worker/artifacts"]) {
expect(() => WorkerTransportModule.register({ ...config, artifactBaseUrl })).toThrow();
}
const registered = WorkerTransportModule.register(config);
const snapshot = (registered.providers as any[]).find(p => p.useValue?.artifactBaseUrl)?.useValue;
expect(Object.isFrozen(snapshot)).toBe(true);
expect(Object.isFrozen(snapshot.workerIds)).toBe(true);
expect(Object.isFrozen(snapshot.grants)).toBe(true);
const workflowConfig = (registered.providers as any[]).find(p => p.provide === "PREPARE_WORKFLOW_CONFIG").useValue;
expect(Object.isFrozen(workflowConfig)).toBe(true);
expect(() => WorkerTransportModule.register({ ...config, enabled: "true" as any })).toThrow();
});
it("denies untrusted identities and body impersonation; dispatches nothing before Gate1", async () => {
const t = await proposal(false);
expect((await post("claim", {}, "wrong")).status).toBe(401);
@@ -309,6 +325,23 @@ const enabled = process.env.WORKER_TRANSPORT_DB === "1";
403,
);
});
it("never follows or overwrites preexisting artifact links on completion", async () => {
const t = await proposal();
const lease = (await post("claim", { taskId: t.id })).body.lease;
const report = tracerReport(lease), hash = digest(report);
const target = join(directory, "sentinel");
await writeFile(target, "do not overwrite");
const file = join(directory, hash + ".json");
await symlink(target, file);
const result = await post("complete", { taskId: t.id, runId: lease.runId, leaseToken: lease.leaseToken, report });
expect(await readFile(target, "utf8")).toBe("do not overwrite");
expect(result.status).toBe(409);
expect(await db.orchestrationArtifact.count({ where: { runId: lease.runId } })).toBe(0);
await rm(file);
await link(target, file);
expect((await post("complete", { taskId: t.id, runId: lease.runId, leaseToken: lease.leaseToken, report })).status).toBe(409);
expect(await readFile(target, "utf8")).toBe("do not overwrite");
});
it("runs actual HTTP deterministic executor to VERIFYING with retrievable content-addressed artifact", async () => {
const t = await proposal();
const result = await pilot.runOnce({
@@ -339,6 +372,30 @@ const enabled = process.env.WORKER_TRANSPORT_DB === "1";
);
expect(get.status).toBe(200);
expect(await get.text()).toBe(bytes.toString());
const artifactGet = (secret: string) => fetch(base + "/internal/prepare-worker/artifacts/" + artifact.sha256,
{ headers: { authorization: "Bearer " + secret } });
expect((await artifactGet("other-worker-secret-".repeat(3))).status).toBe(404);
expect((await artifactGet(ciToken)).status).toBe(200);
const foreignHash = digest("{}");
await writeFile(join(directory, foreignHash + ".json"), "{}");
await db.orchestrationArtifact.create({ data: { runId: artifact.runId, kind: "QA", sha256: foreignHash, uri: "https://foreign.invalid/" + foreignHash } });
expect((await fetch(base + "/internal/prepare-worker/artifacts/" + foreignHash, { headers: { authorization: "Bearer " + token } })).status).toBe(404);
const moved = directory + "-moved";
await rename(directory, moved);
try {
await symlink(moved, directory);
expect((await artifactGet(token)).status).toBe(404);
} finally { await rm(directory, { force: true }); await rename(moved, directory); }
const file = join(directory, artifact.sha256 + ".json"), saved = file + ".saved";
await rename(file, saved);
try {
await symlink(saved, file);
expect((await artifactGet(token)).status).toBe(404);
await rm(file);
await link(saved, file);
expect((await artifactGet(token)).status).toBe(404);
await rm(file);
} finally { await rm(file, { force: true }); await rename(saved, file); }
const cost = await db.orchestrationBudgetLedger.aggregate({
where: { taskId: t.id },
_sum: { amountCents: true },
+34
View File
@@ -0,0 +1,34 @@
# Worker transport security review
Scope: f7048ab (parent equivalent aa99216). Narrow transport-only hardening; no orchestration/core/AppModule changes, release authority, live provisioning or deployment.
## Fixed
- Authenticated artifact retrieval now requires a local fixed-base QA registration, successful staging PREPARE run, matching immutable commit, 20-cent task, and DONE outbox bound to that exact run. Workers can read only their own outbox/run; separately provisioned Stage CI can read bounded completed tracer artifacts for independent verification. Foreign QA registrations and arbitrary authenticated workers do not suffice. Returned bytes must match both content hash and deterministic run-bound tracer report.
- Artifact base is canonical HTTPS with the exact `/internal/prepare-worker/artifacts` path: no userinfo, query, fragment, alternate port, IP literals, localhost/private-name suffixes, escaped or normalized arbitrary paths. This is static reviewed configuration, not a fetched URL. DNS resolution/reachability is deliberately not attempted; the operator must review the fixed origin and DNS/TLS routing.
- Configuration requires an actual boolean enable flag, non-overlapping valid allowlists and valid unique credential grants. Both transport snapshot and workflow config are frozen, including arrays and grant objects; caller mutation cannot change the snapshot.
- Linux descriptor-relative directory traversal with O_DIRECTORY/O_NOFOLLOW prevents symlinked path components and pins directories across renames. Final directory must be private and owned by process UID. File creation uses exclusive O_CREAT/O_EXCL/O_NOFOLLOW, never truncating existing symlinks, hardlinks or regular files. Reads use a pinned O_NOFOLLOW/O_NONBLOCK descriptor, require a private owned regular single-link file, and are bounded to 2049 bytes before digest/run binding verification.
## Preserved and verified
Bearer service digest identity, strict DTO/body impersonation rejection, independent CI role, explicit task-only claim (no queue polling), exact 20-cent cap, Gate1 gating, lease owner/token/task/run fences, unknown-cost reservation exhaustion, STOP_AGENTS and cancellation checks, atomic core completion recheck. No network fetch of artifact URLs was introduced.
The executor remains a deterministic JSON tracer: no LLM, shell, browser, deployment, repository inspection or real application QA. Successful tracer QA is not acceptance-criteria verification.
## Evidence
Actual disposable PostgreSQL16/no-network Docker fixture, Nest TCP HTTP listener, child Node22 executor and full backend `tsc -b` build: `/opt/services/canina/validation/v4-worker-security/`.
- `auth-red.log`: unrelated registered worker received 200 instead of expected 404.
- `config-red.log`: unsafe credential-bearing base accepted.
- `file-red.log`: symlink target sentinel overwritten by deterministic report.
- `freeze-red.log`: workflow config remained mutable.
- `security-green.log`: all 7 tests pass, including own-worker/CI retrieval, foreign URI denial, directory/file symlink and hardlink rejection, tamper integrity rejection, kill/lease/task fencing and actual child executor to VERIFYING.
- `build.log`: full backend `tsc -b` exit 0.
- `tracer-evidence.json`: actual fixture task/run/hash/content, GET200, cost0. External URI reachability explicitly NOT_TESTED_ISOLATED_FIXTURE_HOST.
## Limits / residual responsibilities
Linux `/proc/self/fd` is required; unsupported platforms fail closed. Protect the artifact mount and its ancestry from same-UID/root attackers; they remain outside this process boundary. File and database cannot commit atomically: a killed/denied/competing completion can leave an unregistered orphan, deliberately unreadable and never overwritten automatically. No retry/cleanup redesign was added. Descriptor pinning prevents path redirection but does not promise availability during hostile renames. Symlink/hardlink swaps are tested; exhaustive adversarial scheduling is not claimed.
Parent still owns service-network restriction, reviewed origin/DNS/TLS reachability, body/rate limits, secret provisioning, Stage CI evidence provenance, aggregate department budgets and integration tests. All workers remain unwired/default-disabled; no live secrets or RELEASE_AUTHORIZE used.