411 lines
19 KiB
TypeScript
411 lines
19 KiB
TypeScript
import {
|
|
BadRequestException,
|
|
Body,
|
|
CanActivate,
|
|
ConflictException,
|
|
Controller,
|
|
DynamicModule,
|
|
ForbiddenException,
|
|
Get,
|
|
Inject,
|
|
Injectable,
|
|
Module,
|
|
NotFoundException,
|
|
Param,
|
|
Post,
|
|
Req,
|
|
UnauthorizedException,
|
|
UseGuards,
|
|
UsePipes,
|
|
ValidationPipe,
|
|
} from "@nestjs/common";
|
|
import { IsString, IsUUID, Matches, MaxLength } from "class-validator";
|
|
import { createHash, timingSafeEqual } from "crypto";
|
|
import { mkdir, open, FileHandle } from "fs/promises";
|
|
import { constants } from "fs";
|
|
import { isAbsolute } from "path";
|
|
import { PrismaModule } from "../prisma/prisma.module";
|
|
import { PrismaService } from "../prisma/prisma.service";
|
|
import {
|
|
PREPARE_WORKFLOW_CONFIG,
|
|
PrepareWorkflowService,
|
|
WorkflowIdentity,
|
|
} from "../orchestration/prepare-workflow.service";
|
|
|
|
import { admittedTechTask, TECH_PILOT_KIND, techArtifact, validateTechSource } from "./tech-pilot-validator";
|
|
|
|
// This is reviewed static configuration, not a URL fetched by the transport.
|
|
function validArtifactBase(value: string): boolean {
|
|
try {
|
|
const u = new URL(value);
|
|
// DNS name only: reject all IP literals, localhost/private single-label names,
|
|
// noncanonical paths/escapes, credentials, query, fragment and alternate ports.
|
|
return u.protocol === "https:" && !u.username && !u.password && !u.port &&
|
|
/^[a-z0-9](?:[a-z0-9.-]*[a-z0-9])?$/.test(u.hostname) &&
|
|
u.hostname.includes(".") && !/^[0-9.]+$/.test(u.hostname) &&
|
|
!/(^|\.)(localhost|local|internal|lan|home)$/.test(u.hostname) &&
|
|
["/internal/prepare-worker/artifacts", "/api/internal/prepare-worker/artifacts"].includes(u.pathname) &&
|
|
value === u.origin + u.pathname;
|
|
} catch { return false; }
|
|
}
|
|
const CONFIG = Symbol("WORKER_TRANSPORT_CONFIG");
|
|
export interface WorkerTransportConfig {
|
|
enabled: boolean;
|
|
grants: {
|
|
id: string;
|
|
role: "WORKFLOW_WORKER" | "STAGE_CI";
|
|
credentialSha256: string;
|
|
}[];
|
|
workerIds: string[];
|
|
stageCiIds: string[];
|
|
artifactDirectory: string;
|
|
artifactBaseUrl: string;
|
|
}
|
|
class ClaimDto {
|
|
@IsUUID() taskId!: string;
|
|
}
|
|
class LeaseDto extends ClaimDto {
|
|
@IsUUID() runId!: string;
|
|
@IsUUID() leaseToken!: string;
|
|
}
|
|
class CompleteDto extends LeaseDto {
|
|
@IsString() @MaxLength(2048) report!: string;
|
|
}
|
|
class TechCompleteDto extends LeaseDto {
|
|
@IsString() @MaxLength(2048) source!: string;
|
|
}
|
|
class StageEvidenceDto extends ClaimDto {
|
|
@IsUUID() runId!: string;
|
|
@Matches(/^[a-f0-9]{40}$/) commitSha!: string;
|
|
@Matches(/^[a-f0-9]{64}$/) sha256!: string;
|
|
@Matches(/^[a-f0-9]{64}$/) evidenceSha256!: string;
|
|
@Matches(/^https:\/\/[^\s]+$/) @MaxLength(2000) uri!: string;
|
|
}
|
|
export function tracerReport(lease: {
|
|
taskId: string;
|
|
runId: string;
|
|
commitSha: string;
|
|
}) {
|
|
return JSON.stringify({
|
|
kind: "DETERMINISTIC_PREPARE_TRACER",
|
|
schemaVersion: 1,
|
|
taskId: lease.taskId,
|
|
runId: lease.runId,
|
|
commitSha: lease.commitSha,
|
|
environment: "staging",
|
|
llmPilot: false,
|
|
browserExecuted: false,
|
|
shellExecuted: false,
|
|
deploymentExecuted: false,
|
|
checks: ["lease-bound identity", "deterministic JSON serialization"],
|
|
actualCents: 0,
|
|
});
|
|
}
|
|
@Injectable()
|
|
class WorkerIdentityGuard implements CanActivate {
|
|
constructor(@Inject(CONFIG) private readonly config: WorkerTransportConfig) {}
|
|
canActivate(context: any) {
|
|
if (!this.config.enabled)
|
|
throw new ForbiddenException("Worker transport disabled");
|
|
const req = context.switchToHttp().getRequest();
|
|
const auth = req.headers.authorization;
|
|
if (typeof auth !== "string" || !/^Bearer [\x21-\x7e]{32,512}$/.test(auth))
|
|
throw new UnauthorizedException();
|
|
const digest = createHash("sha256").update(auth.slice(7)).digest();
|
|
const grant = this.config.grants.find((g) =>
|
|
timingSafeEqual(Buffer.from(g.credentialSha256, "hex"), digest),
|
|
);
|
|
if (!grant) throw new UnauthorizedException();
|
|
req.workerIdentity = { id: grant.id, role: grant.role };
|
|
return true;
|
|
}
|
|
}
|
|
@Injectable()
|
|
class WorkerTransportService {
|
|
constructor(
|
|
private readonly workflow: PrepareWorkflowService,
|
|
private readonly db: PrismaService,
|
|
@Inject(CONFIG) private readonly config: WorkerTransportConfig,
|
|
) {}
|
|
// Linux descriptor-relative traversal pins each directory across renames. Never
|
|
// realpath-check then reopen by name: that reintroduces a symlink race.
|
|
private async artifactDir(): Promise<FileHandle> {
|
|
let dir = await open("/", constants.O_RDONLY | constants.O_DIRECTORY);
|
|
try {
|
|
for (const part of this.config.artifactDirectory.split("/").filter(Boolean)) {
|
|
const path = `/proc/self/fd/${dir.fd}/${part}`;
|
|
try { await mkdir(path, { mode: 0o700 }); }
|
|
catch (e: any) { if (e.code !== "EEXIST") throw e; }
|
|
const next = await open(path, constants.O_RDONLY | constants.O_DIRECTORY | constants.O_NOFOLLOW);
|
|
await dir.close();
|
|
dir = next;
|
|
}
|
|
const stat = await dir.stat();
|
|
if (stat.uid !== process.getuid!() || (stat.mode & 0o077) !== 0)
|
|
throw new Error("Private owned artifact directory required");
|
|
return dir;
|
|
} catch (e) { await dir.close(); throw e; }
|
|
}
|
|
private worker(a: WorkflowIdentity) {
|
|
if (a.role !== "WORKFLOW_WORKER" || !this.config.workerIds.includes(a.id))
|
|
throw new ForbiddenException("Worker allowlist required");
|
|
}
|
|
async claim(a: WorkflowIdentity, dto: ClaimDto) {
|
|
this.worker(a);
|
|
const t = await this.db.orchestrationTask.findUnique({
|
|
where: { id: dto.taskId },
|
|
});
|
|
// Explicit per-task pilot scope; never opportunistically consume arbitrary queued work.
|
|
if (!t || !["READY", "RUNNING"].includes(t.state)) return { lease: null };
|
|
if (t.budgetLimitCents !== 20)
|
|
throw new ForbiddenException("Pilot hard task cap is 20 cents");
|
|
return { lease: await this.workflow.claim(a, dto.taskId) };
|
|
}
|
|
heartbeat(a: WorkflowIdentity, dto: LeaseDto) {
|
|
this.worker(a);
|
|
return this.workflow.heartbeat(a, dto);
|
|
}
|
|
async complete(a: WorkflowIdentity, dto: CompleteDto) {
|
|
this.worker(a);
|
|
const task = await this.db.orchestrationTask.findUnique({ where: { id: dto.taskId } });
|
|
if (task?.key === TECH_PILOT_KIND || (task?.metadata as any)?.scope === TECH_PILOT_KIND)
|
|
throw new ForbiddenException("Tech pilot requires separate source completion");
|
|
// Recheck stop/interrupt and fencing before artifact creation; core rechecks atomically at commit.
|
|
await this.workflow.heartbeat(a, dto);
|
|
const run = await this.db.orchestrationRun.findUniqueOrThrow({
|
|
where: { id: dto.runId },
|
|
});
|
|
const expected = tracerReport({ ...dto, commitSha: run.commitSha });
|
|
if (dto.report !== expected)
|
|
throw new BadRequestException(
|
|
"Only deterministic harmless tracer report accepted",
|
|
);
|
|
const sha256 = createHash("sha256").update(expected).digest("hex");
|
|
let dir: FileHandle | undefined, file: FileHandle | undefined;
|
|
try {
|
|
dir = await this.artifactDir();
|
|
file = await open(`/proc/self/fd/${dir.fd}/${sha256}.json`,
|
|
constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, 0o600);
|
|
await file.writeFile(expected, "utf8");
|
|
await file.sync();
|
|
} catch { throw new ConflictException("Artifact creation refused"); }
|
|
finally { await file?.close(); await dir?.close(); }
|
|
const result = await this.workflow.complete(a, {
|
|
...dto,
|
|
sha256,
|
|
uri: this.config.artifactBaseUrl + "/" + sha256,
|
|
actualCents: 0,
|
|
});
|
|
return {
|
|
state: result.state,
|
|
taskId: result.id,
|
|
runId: dto.runId,
|
|
sha256,
|
|
actualCents: 0,
|
|
};
|
|
}
|
|
async techClaim(a: WorkflowIdentity, dto: ClaimDto) {
|
|
this.worker(a);
|
|
const t = await this.db.orchestrationTask.findUnique({ where: { id: dto.taskId } });
|
|
if (!admittedTechTask(t)) throw new ForbiddenException("Reviewed Tech pilot task required");
|
|
const { lease } = await this.claim(a, dto);
|
|
return { lease: lease ? { ...lease, pilotKind: TECH_PILOT_KIND } : null };
|
|
}
|
|
async techComplete(a: WorkflowIdentity, dto: TechCompleteDto) {
|
|
this.worker(a);
|
|
await this.workflow.heartbeat(a, dto);
|
|
const task = await this.db.orchestrationTask.findUnique({ where: { id: dto.taskId } });
|
|
const run = await this.db.orchestrationRun.findUnique({ where: { id: dto.runId } });
|
|
if (!admittedTechTask(task) || !run || run.taskId !== dto.taskId || run.commitSha !== (task!.metadata as any).commitSha || run.environment !== 'staging')
|
|
throw new ForbiddenException('Reviewed Tech pilot task required');
|
|
const qa = validateTechSource(dto.source);
|
|
const bytes = techArtifact({ taskId: dto.taskId, runId: dto.runId, commitSha: run.commitSha }, dto.source);
|
|
const hash = createHash('sha256').update(bytes).digest('hex');
|
|
let dir: FileHandle | undefined, file: FileHandle | undefined;
|
|
try {
|
|
dir = await this.artifactDir();
|
|
file = await open(`/proc/self/fd/${dir.fd}/${hash}.json`, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, 0o600);
|
|
await file.writeFile(bytes, 'utf8'); await file.sync();
|
|
} catch { throw new ConflictException('Artifact creation refused'); }
|
|
finally { await file?.close(); await dir?.close(); }
|
|
// This fixture path deliberately does not call core complete: its QA attestation and
|
|
// measured-cost reconciliation would misrepresent generated toy-source evidence.
|
|
return this.db.$transaction(async (tx: any) => {
|
|
const switches = await tx.$queryRaw`SELECT level, active FROM orchestration_kill_switches WHERE scope = 'GLOBAL' FOR SHARE`;
|
|
if (switches[0]?.active && switches[0].level !== 'FREEZE_NEW_WORK') throw new ForbiddenException('Kill switch blocks workflow');
|
|
await tx.$queryRaw`SELECT id FROM orchestration_tasks WHERE id = ${dto.taskId}::uuid FOR UPDATE`;
|
|
const t = await tx.orchestrationTask.findUnique({ where: { id: dto.taskId } });
|
|
const o = await tx.prepareWorkflowOutbox.findUnique({ where: { taskId: dto.taskId } });
|
|
const r = await tx.orchestrationRun.findUnique({ where: { id: dto.runId } });
|
|
if (!admittedTechTask(t) || t.state !== 'RUNNING' || !o || o.status !== 'PENDING' || o.leaseOwner !== a.id || o.leaseToken !== dto.leaseToken || o.runId !== dto.runId || !o.leaseUntil || o.leaseUntil <= new Date() || !r || r.taskId !== t.id || r.status !== 'RUNNING' || r.commitSha !== t.metadata.commitSha || r.commitSha !== run.commitSha || r.environment !== 'staging')
|
|
throw new ForbiddenException('Active owned Tech lease required');
|
|
const reservation = await tx.orchestrationBudgetLedger.findUnique({ where: { id: o.reservationId } });
|
|
if (!reservation || reservation.taskId !== t.id || reservation.kind !== 'RESERVE' || reservation.amountCents !== 20 || await tx.orchestrationBudgetLedger.findFirst({ where: { taskId: t.id, kind: 'RECONCILE', metadata: { path: ['reservationId'], equals: reservation.id } } }))
|
|
throw new ForbiddenException('Unreconciled 20 cent reservation required');
|
|
await tx.orchestrationArtifact.create({ data: { runId: r.id, kind: 'TECH_PILOT_SOURCE', sha256: hash, uri: this.config.artifactBaseUrl + '/' + hash } });
|
|
await tx.orchestrationAttestation.create({ data: { runId: r.id, kind: 'TECH_PILOT_FIXTURE_QA', subjectHash: hash, signer: 'server-bounded-minmax-v1', predicate: { ...qa, applicationQa: false, costStatus: 'upper-bound-reserved' } } });
|
|
await tx.orchestrationRun.update({ where: { id: r.id }, data: { status: 'SUCCEEDED', finishedAt: new Date() } });
|
|
await tx.prepareWorkflowOutbox.update({ where: { id: o.id }, data: { status: 'DONE', leaseToken: null, leaseUntil: null } });
|
|
const updated = await tx.orchestrationTask.update({ where: { id: t.id }, data: { state: 'VERIFYING', version: { increment: 1 } } });
|
|
await tx.orchestrationEvent.create({ data: { taskId: t.id, sequence: updated.version, actorId: a.id, eventType: 'TECH_PILOT_FIXTURE_VALIDATED', fromState: t.state, toState: 'VERIFYING', idempotencyKey: `tech-pilot:${r.id}`, payload: { artifactSha256: hash, ...qa, applicationQa: false, costStatus: 'upper-bound-reserved', retainedReservationCents: 20 } } });
|
|
return { taskId: t.id, runId: r.id, state: updated.state, artifactSha256: hash, costStatus: 'upper-bound-reserved' };
|
|
});
|
|
}
|
|
async artifact(a: WorkflowIdentity, hash: string) {
|
|
if (!/^[a-f0-9]{64}$/.test(hash))
|
|
throw new BadRequestException("Invalid artifact hash");
|
|
const registered = await this.db.orchestrationArtifact.findFirst({
|
|
where: {
|
|
sha256: hash, kind: { in: ["QA", "TECH_PILOT_SOURCE"] }, uri: this.config.artifactBaseUrl + "/" + hash,
|
|
run: { environment: "staging", status: "SUCCEEDED", task: {
|
|
operationMode: "PREPARE", budgetLimitCents: 20,
|
|
} },
|
|
},
|
|
});
|
|
if (!registered) throw new NotFoundException("Artifact not registered");
|
|
const run = await this.db.orchestrationRun.findUniqueOrThrow({ where: { id: registered.runId } });
|
|
const task = await this.db.orchestrationTask.findUniqueOrThrow({ where: { id: run.taskId } });
|
|
const outbox = await this.db.prepareWorkflowOutbox.findUnique({ where: { taskId: run.taskId } });
|
|
if (!outbox || outbox.status !== "DONE" || outbox.runId !== run.id ||
|
|
(a.role === "WORKFLOW_WORKER" && outbox.leaseOwner !== a.id))
|
|
throw new NotFoundException("Artifact not registered");
|
|
const metadata = task.metadata as any;
|
|
if (metadata?.workflowScope !== "STAGE_PREPARE" || metadata?.commitSha !== run.commitSha)
|
|
throw new NotFoundException("Artifact not registered");
|
|
const tech = registered.kind === "TECH_PILOT_SOURCE";
|
|
if (tech && !admittedTechTask(task)) throw new NotFoundException("Artifact not registered");
|
|
const limit = tech ? 8192 : 2048;
|
|
let bytes: string;
|
|
let dir: FileHandle | undefined, file: FileHandle | undefined;
|
|
try {
|
|
dir = await this.artifactDir();
|
|
file = await open(`/proc/self/fd/${dir.fd}/${hash}.json`,
|
|
constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK);
|
|
const stat = await file.stat();
|
|
if (!stat.isFile() || stat.nlink !== 1 || stat.uid !== process.getuid!() ||
|
|
(stat.mode & 0o077) !== 0 || stat.size > limit)
|
|
throw new Error("Unsafe artifact file");
|
|
// Fixed-size descriptor read: concurrent growth cannot allocate unbounded memory.
|
|
const buffer = Buffer.alloc(limit + 1);
|
|
const { bytesRead } = await file.read(buffer, 0, buffer.length, 0);
|
|
if (bytesRead > limit) throw new Error("Oversized artifact");
|
|
bytes = buffer.subarray(0, bytesRead).toString("utf8");
|
|
} catch {
|
|
throw new NotFoundException("Artifact unavailable");
|
|
} finally { await file?.close(); await dir?.close(); }
|
|
if (createHash("sha256").update(bytes).digest("hex") !== hash)
|
|
throw new ConflictException("Artifact integrity mismatch");
|
|
let expected: string;
|
|
try { expected = tech ? techArtifact({ taskId: run.taskId, runId: run.id, commitSha: run.commitSha }, JSON.parse(bytes).source) : tracerReport({ taskId: run.taskId, runId: run.id, commitSha: run.commitSha }); }
|
|
catch { throw new ConflictException("Artifact binding mismatch"); }
|
|
if (bytes !== expected)
|
|
throw new ConflictException("Artifact binding mismatch");
|
|
return JSON.parse(bytes);
|
|
}
|
|
stage(a: WorkflowIdentity, dto: StageEvidenceDto) {
|
|
return this.workflow.stageEvidence(a, dto);
|
|
}
|
|
}
|
|
@Controller("internal/prepare-worker")
|
|
@UseGuards(WorkerIdentityGuard)
|
|
@UsePipes(
|
|
new ValidationPipe({
|
|
whitelist: true,
|
|
forbidNonWhitelisted: true,
|
|
transform: true,
|
|
}),
|
|
)
|
|
class WorkerTransportController {
|
|
constructor(private readonly service: WorkerTransportService) {}
|
|
@Post("claim") claim(@Req() req: any, @Body() dto: ClaimDto) {
|
|
return this.service.claim(req.workerIdentity, dto);
|
|
}
|
|
@Post("tech-claim") techClaim(@Req() req: any, @Body() dto: ClaimDto) {
|
|
return this.service.techClaim(req.workerIdentity, dto);
|
|
}
|
|
@Post("tech-complete") techComplete(@Req() req: any, @Body() dto: TechCompleteDto) {
|
|
return this.service.techComplete(req.workerIdentity, dto);
|
|
}
|
|
@Post("heartbeat") heartbeat(@Req() req: any, @Body() dto: LeaseDto) {
|
|
return this.service.heartbeat(req.workerIdentity, dto);
|
|
}
|
|
@Post("complete") complete(@Req() req: any, @Body() dto: CompleteDto) {
|
|
return this.service.complete(req.workerIdentity, dto);
|
|
}
|
|
@Post("stage-evidence") stage(
|
|
@Req() req: any,
|
|
@Body() dto: StageEvidenceDto,
|
|
) {
|
|
return this.service.stage(req.workerIdentity, dto);
|
|
}
|
|
@Get("artifacts/:hash") artifact(@Req() req: any, @Param("hash") hash: string) {
|
|
return this.service.artifact(req.workerIdentity, hash);
|
|
}
|
|
}
|
|
@Module({})
|
|
export class WorkerTransportModule {
|
|
static register(config: WorkerTransportConfig): DynamicModule {
|
|
if (typeof config.enabled !== "boolean" || !Array.isArray(config.grants) ||
|
|
!Array.isArray(config.workerIds) || !Array.isArray(config.stageCiIds))
|
|
throw new Error("Explicit boolean enable and credential allowlists required");
|
|
const allIds = [...config.workerIds, ...config.stageCiIds];
|
|
if (allIds.some(id => typeof id !== "string" || !/^[a-zA-Z0-9_-]{1,100}$/.test(id)) ||
|
|
new Set(allIds).size !== allIds.length)
|
|
throw new Error("Invalid or overlapping worker/CI allowlists");
|
|
const ids = new Set<string>(),
|
|
digests = new Set<string>();
|
|
if (
|
|
!isAbsolute(config.artifactDirectory) || config.artifactDirectory === "/" ||
|
|
config.artifactDirectory.split("/").some(p => p === "." || p === "..") ||
|
|
!validArtifactBase(config.artifactBaseUrl)
|
|
)
|
|
throw new Error(
|
|
"Absolute artifact directory and fixed HTTPS artifact base required",
|
|
);
|
|
for (const g of config.grants) {
|
|
const allowed =
|
|
g.role === "WORKFLOW_WORKER"
|
|
? config.workerIds
|
|
: g.role === "STAGE_CI"
|
|
? config.stageCiIds
|
|
: [];
|
|
if (
|
|
!/^[a-zA-Z0-9_-]{1,100}$/.test(g.id) ||
|
|
!allowed.includes(g.id) ||
|
|
!/^[a-f0-9]{64}$/.test(g.credentialSha256) ||
|
|
ids.has(g.id) ||
|
|
digests.has(g.credentialSha256) ||
|
|
(config.workerIds.includes(g.id) && config.stageCiIds.includes(g.id))
|
|
)
|
|
throw new Error("Invalid or overlapping worker/CI credential grants");
|
|
ids.add(g.id);
|
|
digests.add(g.credentialSha256);
|
|
}
|
|
const snapshot = JSON.parse(JSON.stringify(config)) as WorkerTransportConfig;
|
|
snapshot.grants.forEach(Object.freeze);
|
|
Object.freeze(snapshot.grants);
|
|
Object.freeze(snapshot.workerIds);
|
|
Object.freeze(snapshot.stageCiIds);
|
|
Object.freeze(snapshot);
|
|
return {
|
|
module: WorkerTransportModule,
|
|
imports: [PrismaModule],
|
|
controllers: [WorkerTransportController],
|
|
providers: [
|
|
PrepareWorkflowService,
|
|
WorkerIdentityGuard,
|
|
WorkerTransportService,
|
|
{ provide: CONFIG, useValue: snapshot },
|
|
{
|
|
provide: PREPARE_WORKFLOW_CONFIG,
|
|
useValue: Object.freeze({
|
|
workerIds: snapshot.workerIds,
|
|
stageCiIds: snapshot.stageCiIds,
|
|
}),
|
|
},
|
|
],
|
|
exports: [PrepareWorkflowService],
|
|
};
|
|
}
|
|
}
|