439 lines
17 KiB
TypeScript
439 lines
17 KiB
TypeScript
import { Test } from "@nestjs/testing";
|
|
import { INestApplication } from "@nestjs/common";
|
|
import { PrismaClient } from "@prisma/client";
|
|
import { PrismaService } from "../prisma/prisma.service";
|
|
import { PrepareWorkflowService } from "../orchestration/prepare-workflow.service";
|
|
import { WorkerTransportModule, tracerReport } from "./worker-transport.module";
|
|
import { createHash, randomUUID } from "crypto";
|
|
import { mkdtemp, readFile, rm, writeFile, symlink, link, rename } from "fs/promises";
|
|
import { tmpdir } from "os";
|
|
import { join } from "path";
|
|
import { execFile } from "child_process";
|
|
import { promisify } from "util";
|
|
|
|
const enabled = process.env.WORKER_TRANSPORT_DB === "1";
|
|
(enabled ? describe : describe.skip)(
|
|
"isolated PostgreSQL HTTP executor tracer",
|
|
() => {
|
|
const db = new PrismaClient();
|
|
const token = "test-only-worker-secret-".repeat(3);
|
|
const ciToken = "test-only-ci-secret-".repeat(3);
|
|
const digest = (s: string) => createHash("sha256").update(s).digest("hex");
|
|
let app: INestApplication,
|
|
workflow: PrepareWorkflowService,
|
|
base: string,
|
|
directory: string;
|
|
let pilot: any;
|
|
beforeAll(async () => {
|
|
directory = await mkdtemp(join(tmpdir(), "worker-artifacts-"));
|
|
const mod = await Test.createTestingModule({
|
|
imports: [
|
|
WorkerTransportModule.register({
|
|
enabled: true,
|
|
grants: [
|
|
{
|
|
id: "pilot",
|
|
role: "WORKFLOW_WORKER",
|
|
credentialSha256: digest(token),
|
|
},
|
|
{ id: "ci", role: "STAGE_CI", credentialSha256: digest(ciToken) },
|
|
{ id: "other", role: "WORKFLOW_WORKER", credentialSha256: digest("other-worker-secret-".repeat(3)) },
|
|
],
|
|
workerIds: ["pilot", "other"],
|
|
stageCiIds: ["ci"],
|
|
artifactDirectory: directory,
|
|
artifactBaseUrl:
|
|
"https://isolated.invalid/internal/prepare-worker/artifacts",
|
|
}),
|
|
],
|
|
})
|
|
.overrideProvider(PrismaService)
|
|
.useValue(db)
|
|
.compile();
|
|
workflow = mod.get(PrepareWorkflowService);
|
|
app = mod.createNestApplication();
|
|
await app.listen(0, "127.0.0.1");
|
|
base = await app.getUrl();
|
|
pilot = {
|
|
runOnce: async (options: any) => {
|
|
const program =
|
|
'import {runOnce} from "../infra/hermes-worker/pilot.mjs"; console.log(JSON.stringify(await runOnce(JSON.parse(process.env.TEST_PILOT_OPTIONS))));';
|
|
const result = await promisify(execFile)(
|
|
process.execPath,
|
|
["--input-type=module", "-e", program],
|
|
{
|
|
env: {
|
|
...process.env,
|
|
TEST_PILOT_OPTIONS: JSON.stringify(options),
|
|
},
|
|
timeout: 15000,
|
|
},
|
|
);
|
|
return JSON.parse(result.stdout);
|
|
},
|
|
};
|
|
});
|
|
afterAll(async () => {
|
|
await app?.close();
|
|
await db.$disconnect();
|
|
if (directory) await rm(directory, { recursive: true, force: true });
|
|
});
|
|
const owner = { id: "author", role: "Admin" },
|
|
reviewer = { id: "reviewer", role: "SUPER_ADMIN" };
|
|
async function proposal(queue = true, budget = 20) {
|
|
let t = await workflow.create(
|
|
{
|
|
key: randomUUID(),
|
|
title: "deterministic tracer",
|
|
department: "TEST",
|
|
operationMode: "PREPARE",
|
|
budgetLimitCents: budget,
|
|
metadata: {
|
|
commitSha: "a".repeat(40),
|
|
scope: "Harmless tracer only",
|
|
acceptanceCriteria: ["writes deterministic JSON; no release"],
|
|
},
|
|
},
|
|
owner,
|
|
);
|
|
if (queue) {
|
|
t = await workflow.transition(
|
|
t.id,
|
|
{
|
|
toState: "GATE1_PENDING",
|
|
expectedVersion: t.version,
|
|
idempotencyKey: randomUUID(),
|
|
},
|
|
owner,
|
|
);
|
|
t = await workflow.decide(
|
|
t.id,
|
|
1,
|
|
{
|
|
decision: "APPROVED",
|
|
expiresAt: new Date(Date.now() + 60000).toISOString(),
|
|
expectedVersion: t.version,
|
|
idempotencyKey: randomUUID(),
|
|
},
|
|
reviewer,
|
|
);
|
|
t = await workflow.transition(
|
|
t.id,
|
|
{
|
|
toState: "READY",
|
|
expectedVersion: t.version,
|
|
idempotencyKey: randomUUID(),
|
|
},
|
|
owner,
|
|
);
|
|
}
|
|
return t;
|
|
}
|
|
async function post(route: string, body: any = {}, secret = token) {
|
|
const r = await fetch(base + "/internal/prepare-worker/" + route, {
|
|
method: "POST",
|
|
headers: {
|
|
authorization: "Bearer " + secret,
|
|
"content-type": "application/json",
|
|
},
|
|
body: JSON.stringify(body),
|
|
});
|
|
return { status: r.status, body: await r.json() };
|
|
}
|
|
it("rejects overlapping worker and CI grants and accepts no credentials when disabled", async () => {
|
|
expect(() =>
|
|
WorkerTransportModule.register({
|
|
enabled: true,
|
|
grants: [
|
|
{
|
|
id: "both",
|
|
role: "WORKFLOW_WORKER",
|
|
credentialSha256: digest(token),
|
|
},
|
|
],
|
|
workerIds: ["both"],
|
|
stageCiIds: ["both"],
|
|
artifactDirectory: directory,
|
|
artifactBaseUrl: "https://isolated.invalid/internal/prepare-worker/artifacts",
|
|
}),
|
|
).toThrow(/overlapping/);
|
|
const mod = await Test.createTestingModule({
|
|
imports: [
|
|
WorkerTransportModule.register({
|
|
enabled: false,
|
|
grants: [],
|
|
workerIds: [],
|
|
stageCiIds: [],
|
|
artifactDirectory: directory,
|
|
artifactBaseUrl: "https://isolated.invalid/internal/prepare-worker/artifacts",
|
|
}),
|
|
],
|
|
})
|
|
.overrideProvider(PrismaService)
|
|
.useValue(db)
|
|
.compile();
|
|
const disabled = mod.createNestApplication();
|
|
await disabled.listen(0, "127.0.0.1");
|
|
try {
|
|
const r = await fetch(
|
|
(await disabled.getUrl()) + "/internal/prepare-worker/claim",
|
|
{
|
|
method: "POST",
|
|
headers: {
|
|
authorization: "Bearer " + token,
|
|
"content-type": "application/json",
|
|
},
|
|
body: JSON.stringify({ taskId: randomUUID() }),
|
|
},
|
|
);
|
|
expect(r.status).toBe(403);
|
|
} finally {
|
|
await disabled.close();
|
|
}
|
|
});
|
|
it("rejects unsafe artifact bases and freezes validated configuration", () => {
|
|
const config = { enabled: true, grants: [], workerIds: [], stageCiIds: [], artifactDirectory: directory,
|
|
artifactBaseUrl: "https://isolated.invalid/internal/prepare-worker/artifacts" };
|
|
for (const artifactBaseUrl of ["https://user:pass@isolated.invalid/internal/prepare-worker/artifacts", "https://127.0.0.1/internal/prepare-worker/artifacts", "https://10.1.2.3/internal/prepare-worker/artifacts", "https://localhost/internal/prepare-worker/artifacts", "https://isolated.invalid/arbitrary", "https://isolated.invalid/x/../internal/prepare-worker/artifacts", "https://[::1]/internal/prepare-worker/artifacts"]) {
|
|
expect(() => WorkerTransportModule.register({ ...config, artifactBaseUrl })).toThrow();
|
|
}
|
|
const registered = WorkerTransportModule.register(config);
|
|
const snapshot = (registered.providers as any[]).find(p => p.useValue?.artifactBaseUrl)?.useValue;
|
|
expect(Object.isFrozen(snapshot)).toBe(true);
|
|
expect(Object.isFrozen(snapshot.workerIds)).toBe(true);
|
|
expect(Object.isFrozen(snapshot.grants)).toBe(true);
|
|
const workflowConfig = (registered.providers as any[]).find(p => p.provide === "PREPARE_WORKFLOW_CONFIG").useValue;
|
|
expect(Object.isFrozen(workflowConfig)).toBe(true);
|
|
expect(() => WorkerTransportModule.register({ ...config, enabled: "true" as any })).toThrow();
|
|
});
|
|
it("denies untrusted identities and body impersonation; dispatches nothing before Gate1", async () => {
|
|
const t = await proposal(false);
|
|
expect((await post("claim", {}, "wrong")).status).toBe(401);
|
|
expect((await post("claim", { taskId: t.id }, ciToken)).status).toBe(403);
|
|
expect(
|
|
(await post("claim", { taskId: t.id, actorId: "pilot" })).status,
|
|
).toBe(400);
|
|
expect((await post("claim", { taskId: t.id })).body).toEqual({
|
|
lease: null,
|
|
});
|
|
expect(await db.orchestrationRun.count({ where: { taskId: t.id } })).toBe(
|
|
0,
|
|
);
|
|
});
|
|
it("races claims, fences expired leases and reserves unknown cost against the 20 cent cap", async () => {
|
|
const t = await proposal();
|
|
const claims = await Promise.all([
|
|
post("claim", { taskId: t.id }),
|
|
post("claim", { taskId: t.id }),
|
|
]);
|
|
const leases = claims.map((r) => r.body.lease).filter(Boolean);
|
|
expect(leases).toHaveLength(1);
|
|
const lease = leases[0],
|
|
binding = {
|
|
taskId: t.id,
|
|
runId: lease.runId,
|
|
leaseToken: lease.leaseToken,
|
|
};
|
|
expect(
|
|
(await post("heartbeat", { ...binding, leaseToken: randomUUID() }))
|
|
.status,
|
|
).toBe(403);
|
|
expect(
|
|
(
|
|
await post("complete", {
|
|
...binding,
|
|
report: "arbitrary shell or prompt",
|
|
})
|
|
).status,
|
|
).toBe(400);
|
|
expect(
|
|
(await post("complete", { ...binding, report: "{}", actualCents: 21 }))
|
|
.status,
|
|
).toBe(400);
|
|
await db.prepareWorkflowOutbox.update({
|
|
where: { taskId: t.id },
|
|
data: { leaseUntil: new Date(Date.now() - 1000) },
|
|
});
|
|
expect((await post("heartbeat", binding)).status).toBe(403);
|
|
expect(
|
|
(await post("complete", { ...binding, report: "{}" })).status,
|
|
).toBe(403);
|
|
expect((await post("claim", { taskId: t.id })).status).toBe(403);
|
|
expect(await db.orchestrationRun.count({ where: { taskId: t.id } })).toBe(
|
|
1,
|
|
);
|
|
expect(
|
|
(
|
|
await db.orchestrationBudgetLedger.aggregate({
|
|
where: { taskId: t.id },
|
|
_sum: { amountCents: true },
|
|
})
|
|
)._sum.amountCents,
|
|
).toBe(20);
|
|
const larger = await proposal(true, 40);
|
|
expect((await post("claim", { taskId: larger.id })).status).toBe(403);
|
|
});
|
|
it("rechecks kill switch and task interrupt on heartbeat and completion; separates CI authority", async () => {
|
|
const t = await proposal(),
|
|
lease = (await post("claim", { taskId: t.id })).body.lease;
|
|
const binding = {
|
|
taskId: t.id,
|
|
runId: lease.runId,
|
|
leaseToken: lease.leaseToken,
|
|
};
|
|
await db.orchestrationKillSwitch.upsert({
|
|
where: { scope: "GLOBAL" },
|
|
create: {
|
|
scope: "GLOBAL",
|
|
level: "STOP_AGENTS",
|
|
active: true,
|
|
reason: "isolated test",
|
|
actorId: "test",
|
|
},
|
|
update: { level: "STOP_AGENTS", active: true },
|
|
});
|
|
try {
|
|
expect((await post("heartbeat", binding)).status).toBe(403);
|
|
expect(
|
|
(await post("complete", { ...binding, report: "{}" })).status,
|
|
).toBe(403);
|
|
expect((await post("claim", { taskId: t.id })).status).toBe(403);
|
|
} finally {
|
|
await db.orchestrationKillSwitch.deleteMany();
|
|
}
|
|
await db.orchestrationTask.update({
|
|
where: { id: t.id },
|
|
data: { state: "CANCELLED" },
|
|
});
|
|
expect((await post("heartbeat", binding)).status).toBe(403);
|
|
expect(
|
|
(await post("complete", { ...binding, report: "{}" })).status,
|
|
).toBe(403);
|
|
expect(
|
|
await db.orchestrationArtifact.count({ where: { runId: lease.runId } }),
|
|
).toBe(0);
|
|
const evidence = {
|
|
taskId: t.id,
|
|
runId: lease.runId,
|
|
commitSha: "a".repeat(40),
|
|
sha256: "b".repeat(64),
|
|
evidenceSha256: "c".repeat(64),
|
|
uri: "https://isolated.invalid/not-real-ci",
|
|
};
|
|
expect((await post("stage-evidence", evidence)).status).toBe(403);
|
|
expect((await post("stage-evidence", evidence, ciToken)).status).toBe(
|
|
403,
|
|
);
|
|
});
|
|
it("never follows or overwrites preexisting artifact links on completion", async () => {
|
|
const t = await proposal();
|
|
const lease = (await post("claim", { taskId: t.id })).body.lease;
|
|
const report = tracerReport(lease), hash = digest(report);
|
|
const target = join(directory, "sentinel");
|
|
await writeFile(target, "do not overwrite");
|
|
const file = join(directory, hash + ".json");
|
|
await symlink(target, file);
|
|
const result = await post("complete", { taskId: t.id, runId: lease.runId, leaseToken: lease.leaseToken, report });
|
|
expect(await readFile(target, "utf8")).toBe("do not overwrite");
|
|
expect(result.status).toBe(409);
|
|
expect(await db.orchestrationArtifact.count({ where: { runId: lease.runId } })).toBe(0);
|
|
await rm(file);
|
|
await link(target, file);
|
|
expect((await post("complete", { taskId: t.id, runId: lease.runId, leaseToken: lease.leaseToken, report })).status).toBe(409);
|
|
expect(await readFile(target, "utf8")).toBe("do not overwrite");
|
|
});
|
|
it("runs actual HTTP deterministic executor to VERIFYING with retrievable content-addressed artifact", async () => {
|
|
const t = await proposal();
|
|
const result = await pilot.runOnce({
|
|
baseUrl: base,
|
|
credential: token,
|
|
taskId: t.id,
|
|
allowLoopbackHttp: true,
|
|
});
|
|
expect(result.state).toBe("VERIFYING");
|
|
const row = await db.orchestrationTask.findUniqueOrThrow({
|
|
where: { id: t.id },
|
|
});
|
|
expect(row.state).toBe("VERIFYING");
|
|
const artifact = await db.orchestrationArtifact.findFirstOrThrow({
|
|
where: { run: { taskId: t.id } },
|
|
});
|
|
const bytes = await readFile(join(directory, artifact.sha256 + ".json"));
|
|
expect(digest(bytes.toString())).toBe(artifact.sha256);
|
|
expect(JSON.parse(bytes.toString())).toMatchObject({
|
|
kind: "DETERMINISTIC_PREPARE_TRACER",
|
|
commitSha: "a".repeat(40),
|
|
environment: "staging",
|
|
llmPilot: false,
|
|
});
|
|
const get = await fetch(
|
|
base + "/internal/prepare-worker/artifacts/" + artifact.sha256,
|
|
{ headers: { authorization: "Bearer " + token } },
|
|
);
|
|
expect(get.status).toBe(200);
|
|
expect(await get.text()).toBe(bytes.toString());
|
|
const artifactGet = (secret: string) => fetch(base + "/internal/prepare-worker/artifacts/" + artifact.sha256,
|
|
{ headers: { authorization: "Bearer " + secret } });
|
|
expect((await artifactGet("other-worker-secret-".repeat(3))).status).toBe(404);
|
|
expect((await artifactGet(ciToken)).status).toBe(200);
|
|
const foreignHash = digest("{}");
|
|
await writeFile(join(directory, foreignHash + ".json"), "{}");
|
|
await db.orchestrationArtifact.create({ data: { runId: artifact.runId, kind: "QA", sha256: foreignHash, uri: "https://foreign.invalid/" + foreignHash } });
|
|
expect((await fetch(base + "/internal/prepare-worker/artifacts/" + foreignHash, { headers: { authorization: "Bearer " + token } })).status).toBe(404);
|
|
const moved = directory + "-moved";
|
|
await rename(directory, moved);
|
|
try {
|
|
await symlink(moved, directory);
|
|
expect((await artifactGet(token)).status).toBe(404);
|
|
} finally { await rm(directory, { force: true }); await rename(moved, directory); }
|
|
const file = join(directory, artifact.sha256 + ".json"), saved = file + ".saved";
|
|
await rename(file, saved);
|
|
try {
|
|
await symlink(saved, file);
|
|
expect((await artifactGet(token)).status).toBe(404);
|
|
await rm(file);
|
|
await link(saved, file);
|
|
expect((await artifactGet(token)).status).toBe(404);
|
|
await rm(file);
|
|
} finally { await rm(file, { force: true }); await rename(saved, file); }
|
|
const cost = await db.orchestrationBudgetLedger.aggregate({
|
|
where: { taskId: t.id },
|
|
_sum: { amountCents: true },
|
|
});
|
|
expect(cost._sum.amountCents).toBe(0);
|
|
expect(
|
|
await db.orchestrationAttestation.findFirst({
|
|
where: { run: { taskId: t.id }, kind: "STAGE" },
|
|
}),
|
|
).toBeNull();
|
|
const evidence = {
|
|
taskId: t.id,
|
|
state: row.state,
|
|
sha256: artifact.sha256,
|
|
artifact: JSON.parse(bytes.toString()),
|
|
httpGet: get.status,
|
|
costCents: cost._sum.amountCents,
|
|
uriExternalReachability: "NOT_TESTED_ISOLATED_FIXTURE_HOST",
|
|
};
|
|
console.log("TRACER_EVIDENCE", JSON.stringify(evidence));
|
|
if (process.env.TRACER_EVIDENCE_PATH)
|
|
await writeFile(
|
|
process.env.TRACER_EVIDENCE_PATH,
|
|
JSON.stringify(evidence, null, 2),
|
|
);
|
|
await writeFile(
|
|
join(directory, artifact.sha256 + ".json"),
|
|
'{"tampered":true}',
|
|
);
|
|
expect(
|
|
(
|
|
await fetch(
|
|
base + "/internal/prepare-worker/artifacts/" + artifact.sha256,
|
|
{ headers: { authorization: "Bearer " + token } },
|
|
)
|
|
).status,
|
|
).toBe(409);
|
|
});
|
|
},
|
|
);
|