Compare commits

..
Author SHA1 Message Date
devops 8cfd44cd09 Bound pilot inference with ongoing heartbeat and process-group cancellation 2026-10-23 16:57:33 +00:00
devops c66bc58f73 Add fail-closed real Hermes Tech pilot and independent artifact QA proof 2026-10-23 16:45:03 +00:00
devops 4b147b0091 Route CI Stage deployment through authenticated pinned SSH host boundary
Deploy Canina / deploy (push) Successful in 4m35s
2026-10-23 15:06:24 +00:00
devops 6950ff0d79 Isolate commit-pinned Stage releases from dirty operational checkout 2026-10-23 14:50:32 +00:00
devops cc9ec25388 Wire explicitly enabled worker startup and exact legacy rehearsal comparison 2026-10-23 14:41:48 +00:00
devops be2ff0719a feat(worker): add fail-closed reviewed startup config registration 2026-10-23 14:38:54 +00:00
devops 9d93707848 fix(worker): scope artifact access and harden immutable transport storage 2026-10-23 14:07:01 +00:00
devops aa99216c49 feat(worker): add authenticated bounded PREPARE HTTP executor tracer 2026-10-23 13:49:38 +00:00
devops 58d3794050 fix(admin): distinguish snapshot inventory from live runtime telemetry 2026-10-23 13:34:53 +00:00
devops 16f9610027 Wire runtime read routes with explicit file-granted startup opt-in 2026-10-23 13:26:29 +00:00
devops 2023dd4e36 Make snapshot inventory provenance explicit and deny false live readiness 2026-10-23 13:22:21 +00:00
devops 0bed6e0313 fix(admin): restore immutable PREPARE scope contract
Match d43aa02 metadata validation and exact DTO; distinguish missing safe-projection workflow scope from human review evidence. Preserve action normalization and explicit kill-switch capability. TDD regression RED verified; disposable admin 61 files / 381 tests, app+node TypeScript and build pass.
2026-10-23 13:13:41 +00:00
devops f29ba4ab8b fix(admin): reconcile bounded PREPARE backend contracts 2026-10-23 13:04:06 +00:00
devops 56b6696ca3 Preserve bounded PREPARE scope and normalize proposal conflicts 2026-10-23 13:04:06 +00:00
devops 0d0c67704c Complete safe admin workflow and authenticated runtime read UI 2026-10-23 12:53:06 +00:00
devops 09e8a890e0 Add bounded Stage PREPARE workflow with transactional leased outbox 2026-10-23 12:53:06 +00:00
devops 4b17a6fcc3 Add admin-scoped Hermes runtime reads and isolated pinned HTTP inventory 2026-10-23 12:53:06 +00:00
59 changed files with 4358 additions and 183 deletions

No files matched your search

+16 -16
View File
@@ -10,25 +10,25 @@ jobs:
deploy:
runs-on: canina
steps:
- name: Run deploy.sh from within the scripts folder
- name: Authenticated host Stage deployment (Production blocked)
shell: bash
env:
BRANCH: ${{ github.ref_name }}
DEPLOY_SHA: ${{ github.sha }}
SUPPORT_CHAT_KEY: ${{ secrets.SUPPORT_CHAT_KEY }}
DEPLOY_KEY: ${{ secrets.DEPLOY_KEY }}
DEPLOY_KNOWN_HOSTS: ${{ secrets.DEPLOY_KNOWN_HOSTS }}
run: |
set -euo pipefail
# 1. رفع مشکل DNS و غیرفعال کردن پروکسی برای دامنه محلی
echo "87.248.133.138 git.parsaaghayi.ir" >> /etc/hosts
export http_proxy=""
export https_proxy=""
export HTTP_PROXY=""
export HTTPS_PROXY=""
set +x
[[ "${BRANCH}" == develop ]] || { echo 'Production blocked' >&2; exit 2; }
[[ "${DEPLOY_SHA}" =~ ^[0-9a-f]{40}$ ]] || exit 2
# Provision DEPLOY_KNOWN_HOSTS from an independently verified host key.
# Never use baked-in /opt/services, runner SSH keys or docker.sock.
[[ -n "${DEPLOY_KEY}" && -n "${DEPLOY_KNOWN_HOSTS}" ]] || { echo 'Host boundary requires DEPLOY_KEY and verified DEPLOY_KNOWN_HOSTS' >&2; exit 2; }
export http_proxy="" https_proxy="" HTTP_PROXY="" HTTPS_PROXY=""
export no_proxy="git.parsaaghayi.ir,localhost,127.0.0.1,87.248.133.138"
# 2. دانلود و اجرای اسکریپت deploy.sh بر اساس برنچ جاری
mkdir -p scripts
wget --no-proxy -qO scripts/deploy.sh https://git.parsaaghayi.ir/parsa/canina/raw/commit/${DEPLOY_SHA}/scripts/deploy.sh
chmod +x scripts/deploy.sh
cd scripts
./deploy.sh "${BRANCH}"
umask 077
wrapper=$(mktemp)
trap 'rm -f "$wrapper"' EXIT
wget --no-proxy -qO "$wrapper" "https://git.parsaaghayi.ir/parsa/canina/raw/commit/${DEPLOY_SHA}/scripts/ci-host-deploy.sh"
bash "$wrapper"
@@ -0,0 +1,10 @@
CREATE TABLE prepare_workflow_outbox (
id uuid PRIMARY KEY, task_id uuid NOT NULL UNIQUE REFERENCES orchestration_tasks(id) ON DELETE RESTRICT,
status varchar(20) NOT NULL DEFAULT 'PENDING' CHECK (status IN ('PENDING','DONE')),
attempts integer NOT NULL DEFAULT 0 CHECK (attempts BETWEEN 0 AND 3),
lease_owner varchar(100), lease_token uuid, lease_until timestamptz, heartbeat_at timestamptz,
run_id uuid REFERENCES orchestration_runs(id) ON DELETE RESTRICT,
reservation_id uuid REFERENCES orchestration_budget_ledger(id) ON DELETE RESTRICT,
created_at timestamptz NOT NULL DEFAULT now()
);
CREATE INDEX prepare_workflow_outbox_status_lease_until_idx ON prepare_workflow_outbox(status,lease_until);
+17
View File
@@ -1232,3 +1232,20 @@ model OrchestrationKillSwitch {
createdAt DateTime @default(now()) @map("created_at") @db.Timestamptz()
@@map("orchestration_kill_switches")
}
// Transactional dispatch intent. Lease tokens are server-only fencing credentials.
model PrepareWorkflowOutbox {
id String @id @default(uuid()) @db.Uuid
taskId String @unique @map("task_id") @db.Uuid
status String @default("PENDING") @db.VarChar(20)
attempts Int @default(0)
leaseOwner String? @map("lease_owner") @db.VarChar(100)
leaseToken String? @map("lease_token") @db.Uuid
leaseUntil DateTime? @map("lease_until") @db.Timestamptz()
heartbeatAt DateTime? @map("heartbeat_at") @db.Timestamptz()
runId String? @map("run_id") @db.Uuid
reservationId String? @map("reservation_id") @db.Uuid
createdAt DateTime @default(now()) @map("created_at") @db.Timestamptz()
@@index([status, leaseUntil])
@@map("prepare_workflow_outbox")
}
+4 -2
View File
@@ -1,5 +1,6 @@
import { registerPrepareWorker } from './orchestration-worker/worker-registration';
import { RouterUsageModule } from './router-usage/router-usage.module';
import { HermesManagementModule } from './hermes-management/hermes-management.module';
import { runtimeReadRegistration } from './hermes-management/runtime-read.registration';
import { TasksModule } from './common/services/tasks.module';
import { Module, NestModule, MiddlewareConsumer } from "@nestjs/common";
import { PrismaModule } from "./prisma/prisma.module";
@@ -45,9 +46,10 @@ import { Request, Response, NextFunction } from "express";
@Module({
imports: [
...registerPrepareWorker(),
RouterUsageModule,
// Deny-all registration: no request identity or runtime credentials are accepted yet.
HermesManagementModule.register({origin:'http://canina-hermes:9119',token:async()=>''}),
runtimeReadRegistration(),
OrchestrationModule,
SupportChatModule,
RevalidationModule,
@@ -12,7 +12,7 @@ export class HermesManagementModule {
policy: RuntimeAuthorizer = {authorize: async () => false},
): DynamicModule {
const broker = new HermesSessionBroker(credentials);
return this.register({origin:'http://canina-hermes:9119',token:()=>broker.token()},policy);
return this.register({origin:'http://canina-hermes-inventory:9119',token:()=>broker.token()},policy);
}
static register(
@@ -34,7 +34,7 @@ describe('server-only Hermes session broker',()=>{
expect(await broker.token()).toBe('verified-test-token');
expect(fetcher).toHaveBeenCalledTimes(1);
const [url,options]=fetcher.mock.calls[0];
expect(url).toBe('http://canina-hermes:9119/auth/password-login');
expect(url).toBe('http://canina-hermes-inventory:9119/auth/password-login');
expect(JSON.parse(options.body)).toEqual({provider:'basic',username:'backend',password:'test-only'});
expect(options.redirect).toBe('error');
});
@@ -15,7 +15,7 @@ export class HermesSessionBroker {
try{
const creds=await Promise.race([this.credentials(),new Promise<never>((_,reject)=>controller.signal.addEventListener('abort',()=>reject(new HermesError('UNAVAILABLE')),{once:true}))]);
if(!creds.username || !creds.password) throw new HermesError('AUTH_REQUIRED');
const response=await this.fetcher('http://canina-hermes:9119/auth/password-login',{method:'POST',redirect:'error',signal:controller.signal,headers:{'Content-Type':'application/json'},body:JSON.stringify({provider:'basic',...creds})});
const response=await this.fetcher('http://canina-hermes-inventory:9119/auth/password-login',{method:'POST',redirect:'error',signal:controller.signal,headers:{'Content-Type':'application/json'},body:JSON.stringify({provider:'basic',...creds})});
if(!response.ok) throw new HermesError('AUTH_REQUIRED');
const reader=response.body?.getReader();
if(!reader) throw new HermesError('AUTH_REQUIRED');
+94 -21
View File
@@ -28,6 +28,7 @@ export interface Health {
version: string;
commit: string;
auth_required: boolean;
provenance?: Row;
}
function object(value: unknown): Record<string, unknown> {
if (!value || typeof value !== 'object' || Array.isArray(value))
@@ -57,7 +58,12 @@ export class HermesAdapter {
private readonly config: HermesConfig,
private readonly fetcher: typeof fetch = fetch,
) {
if (config.origin !== 'http://canina-hermes:9119')
if (
![
'http://canina-hermes:9119',
'http://canina-hermes-inventory:9119',
].includes(config.origin)
)
throw new HermesError('CONFIGURATION');
this.origin = config.origin;
}
@@ -122,7 +128,12 @@ export class HermesAdapter {
} finally {
await reader.cancel().catch(() => undefined);
}
return JSON.parse(Buffer.concat(chunks).toString('utf8')) as unknown;
const data: unknown = JSON.parse(Buffer.concat(chunks).toString('utf8'));
if (this.origin === 'http://canina-hermes-inventory:9119') {
const rows = Array.isArray(data) ? data : [data];
for (const row of rows) provenance(row, true);
}
return data;
} catch (error) {
if (error instanceof HermesError) throw error;
throw new HermesError('UNAVAILABLE');
@@ -132,12 +143,16 @@ export class HermesAdapter {
}
}
async health(): Promise<Health> {
const health = fields(await this.request('/api/health'), {
ok: 'boolean',
version: 'string',
commit: 'string',
auth_required: 'boolean',
}) as unknown as Health;
const raw = await this.request('/api/health');
const health = {
...fields(raw, {
ok: 'boolean',
version: 'string',
commit: 'string',
auth_required: 'boolean',
}),
...provenance(raw),
} as unknown as Health;
if (!health.auth_required) throw new HermesError('AUTH_REQUIRED');
return health;
}
@@ -151,23 +166,47 @@ export class HermesAdapter {
});
if (raw.gateway_state !== null && typeof raw.gateway_state !== 'string')
throw new HermesError('CONTRACT');
return { ...row, gateway_state: raw.gateway_state as Json };
return {
...row,
gateway_state: raw.gateway_state as Json,
...provenance(raw),
...(raw.provenance
? {
gateway_running: false,
active_agents: 0,
gateway_state: null,
overall: 'unknown',
liveGatewayState: 'unknown',
liveActiveAgents: null,
}
: {}),
};
}
async profiles(): Promise<Row[]> {
return list(object(await this.request('/api/profiles')).profiles).map((r) =>
fields(r, {
name: 'string',
is_default: 'boolean',
gateway_running: 'boolean',
return list(object(await this.request('/api/profiles')).profiles).map(
(r) => ({
...fields(r, {
name: 'string',
is_default: 'boolean',
gateway_running: 'boolean',
}),
...provenance(r),
...(object(r).provenance
? { gateway_running: false, liveGatewayState: 'unknown' }
: {}),
}),
);
}
async modelInfo(profile: string): Promise<Row> {
return fields(await this.request('/api/model/info', profile), {
model: 'string',
provider: 'string',
effective_context_length: 'number',
});
const raw = await this.request('/api/model/info', profile);
return {
...fields(raw, {
model: 'string',
provider: 'string',
effective_context_length: 'number',
}),
...provenance(raw),
};
}
async modelOptions(profile: string): Promise<Row[]> {
return list(
@@ -189,6 +228,7 @@ export class HermesAdapter {
);
return {
...fields(raw, { period_days: 'number' }),
...provenance(raw),
totals: fields(raw.totals, {
total_input: 'number',
total_output: 'number',
@@ -218,7 +258,11 @@ export class HermesAdapter {
is_active: 'boolean',
}),
);
return { ...fields(raw, { total: 'number' }), sessions: rows };
return {
...fields(raw, { total: 'number' }),
sessions: rows,
...provenance(raw),
};
}
async crons(profile: string): Promise<Row[]> {
return list(await this.request('/api/cron/jobs', profile)).map((r) =>
@@ -265,5 +309,34 @@ function scoped(
): Row {
const row = fields(value, schema);
if (row.profile !== profile) throw new HermesError('FORBIDDEN');
return row;
return { ...row, ...provenance(value) };
}
/** Snapshot provenance is mandatory on inventory origin; never synthesize a capture time. */
function provenance(value: unknown, required = false): Row {
const raw = object(value);
if (!raw.provenance && !required) return {};
const p = object(raw.provenance);
if (
p.source !== 'snapshot' ||
p.liveProcess !== 'unknown' ||
typeof p.capturedAt !== 'string' ||
!p.capturedAt.endsWith('Z') ||
!Number.isFinite(Date.parse(p.capturedAt)) ||
Date.parse(p.capturedAt) > Date.now() ||
typeof p.ageSeconds !== 'number' ||
!Number.isInteger(p.ageSeconds) ||
p.ageSeconds < 0 ||
!['fresh', 'stale'].includes(String(p.freshness))
)
throw new HermesError('CONTRACT');
return {
provenance: {
source: 'snapshot',
capturedAt: p.capturedAt,
ageSeconds: p.ageSeconds,
freshness: p.freshness as string,
liveProcess: 'unknown',
},
};
}
@@ -0,0 +1,31 @@
import { loadRuntimeGrants } from './runtime-read.module';
import { mkdtemp, writeFile, rm } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
describe('reviewed runtime grants file', () => {
it('loads an exact reviewed file and fails closed for malformed grants', async () => {
const dir = await mkdtemp(join(tmpdir(), 'runtime-grants-'));
const file = join(dir, 'grants.json');
try {
await writeFile(file, JSON.stringify({ 'subject-1': ['default'] }));
expect(await loadRuntimeGrants(file, dir)).toEqual({
'subject-1': ['default'],
});
for (const invalid of [
null,
[],
{ 'subject-1': [] },
{ 'subject-1': ['default', 'default'] },
{ 'subject-1': ['../x'] },
{ 'subject-1': [3] },
{ 'subject-1': ['default'], extra: true },
]) {
await writeFile(file, JSON.stringify(invalid));
await expect(loadRuntimeGrants(file, dir)).rejects.toThrow();
}
await expect(loadRuntimeGrants('/etc/passwd')).rejects.toThrow();
} finally {
await rm(dir, { recursive: true });
}
});
});
@@ -0,0 +1,83 @@
import { Test } from '@nestjs/testing';
import { INestApplication } from '@nestjs/common';
import { PassportModule } from '@nestjs/passport';
import { Strategy, ExtractJwt } from 'passport-jwt';
import passport from 'passport';
import { sign } from 'jsonwebtoken';
import request from 'supertest';
import { RuntimeReadModule } from './runtime-read.module';
import { HermesIntegrationService } from './hermes-integration.service';
describe('runtime HTTP authentication and read contract', () => {
let app: INestApplication;
const health = jest.fn().mockResolvedValue({ ok: true, auth_required: true });
beforeAll(async () => {
passport.use(
'jwt',
new Strategy(
{
jwtFromRequest: ExtractJwt.fromAuthHeaderAsBearerToken(),
secretOrKey: 'test-only-http-key',
},
(payload, done) => done(null, { id: payload.sub, role: payload.role }),
),
);
const module = await Test.createTestingModule({
imports: [
PassportModule,
RuntimeReadModule.register({ 'admin-1': ['default'] }),
],
})
.overrideProvider(HermesIntegrationService)
.useValue({ health })
.compile();
app = module.createNestApplication();
await app.init();
});
afterAll(async () => {
await app.close();
passport.unuse('jwt');
});
it('rejects missing and forged credentials before upstream', async () => {
health.mockClear();
await request(app.getHttpServer())
.get('/admin/orchestration/runtime/health')
.expect(401);
await request(app.getHttpServer())
.get('/admin/orchestration/runtime/health')
.set(
'Authorization',
'Bearer ' + sign({ sub: 'admin-1', role: 'Admin' }, 'forged'),
)
.expect(401);
expect(health).not.toHaveBeenCalled();
});
it('accepts verified allowlisted admin but rejects signed ungranted subject', async () => {
await request(app.getHttpServer())
.get('/admin/orchestration/runtime/health')
.set(
'Authorization',
'Bearer ' + sign({ sub: 'other', role: 'Admin' }, 'test-only-http-key'),
)
.expect(403);
const r = await request(app.getHttpServer())
.get('/admin/orchestration/runtime/health')
.set(
'Authorization',
'Bearer ' +
sign({ sub: 'admin-1', role: 'Admin' }, 'test-only-http-key'),
)
.expect(200);
expect(r.body).toEqual({ ok: true, auth_required: true });
expect(r.headers['set-cookie']).toBeUndefined();
});
it('does not register mutation handlers', async () => {
await request(app.getHttpServer())
.post('/admin/orchestration/runtime/schedules')
.set(
'Authorization',
'Bearer ' +
sign({ sub: 'admin-1', role: 'Admin' }, 'test-only-http-key'),
)
.expect(404);
});
});
@@ -0,0 +1,183 @@
import {
Controller,
Get,
Req,
Query,
UseGuards,
Injectable,
ForbiddenException,
ServiceUnavailableException,
HttpException,
} from '@nestjs/common';
import { AuthGuard } from '@nestjs/passport';
import {
HermesIntegrationService,
RuntimePrincipal,
} from './hermes-integration.service';
import { HermesError } from './hermes.adapter';
export interface VerifiedRuntimeRequest {
user?: {
id?: string;
role?: string;
isApiKey?: boolean;
profiles?: string[];
};
}
@Injectable()
export class RuntimeReadAccess {
private readonly grants: Readonly<Record<string, readonly string[]>>;
constructor(grants: Readonly<Record<string, readonly string[]>>) {
if (
!grants ||
typeof grants !== 'object' ||
Array.isArray(grants) ||
Object.keys(grants).length > 1000
)
throw new Error('Invalid runtime access configuration');
const copy: Record<string, readonly string[]> = Object.create(null);
for (const [subject, profiles] of Object.entries(grants)) {
if (
!/^[a-zA-Z0-9][a-zA-Z0-9@._:-]{0,127}$/.test(subject) ||
['__proto__', 'constructor', 'prototype'].includes(subject) ||
!Array.isArray(profiles) ||
profiles.length === 0 ||
profiles.length > 100 ||
new Set(profiles).size !== profiles.length ||
profiles.some(
(p) =>
typeof p !== 'string' ||
!/^[a-zA-Z0-9][a-zA-Z0-9_-]{0,63}$/.test(p),
)
)
throw new Error('Invalid runtime access configuration');
copy[subject] = Object.freeze([...profiles]);
}
this.grants = Object.freeze(copy);
}
principal(req: VerifiedRuntimeRequest): RuntimePrincipal {
const u = req.user;
if (
!u?.id ||
u.isApiKey ||
!['admin', 'superadmin'].includes(
(u.role ?? '').toLowerCase().replace(/[_-]/g, ''),
) ||
!Object.hasOwn(this.grants, u.id)
)
throw new ForbiddenException('Runtime access denied');
return { subject: u.id, profiles: this.grants[u.id] };
}
authorize(
p: RuntimePrincipal,
permission: string,
profile?: string,
): Promise<boolean> {
return Promise.resolve(
permission === 'runtime.read' &&
Object.hasOwn(this.grants, p.subject) &&
(!profile || this.grants[p.subject].includes(profile)),
);
}
}
@Controller('admin/orchestration/runtime')
@UseGuards(AuthGuard('jwt'))
export class RuntimeReadController {
constructor(
private readonly runtime: HermesIntegrationService,
private readonly access: RuntimeReadAccess,
) {}
private async read(
req: VerifiedRuntimeRequest,
call: (p: RuntimePrincipal) => Promise<unknown>,
) {
const p = this.access.principal(req);
try {
return await call(p);
} catch (e) {
if (e instanceof HermesError) {
if (e.code === 'FORBIDDEN')
throw new ForbiddenException('Runtime access denied');
if (e.code === 'LIMIT')
throw new HttpException('Runtime request limit', 429);
}
throw new ServiceUnavailableException('Runtime unavailable');
}
}
private profile(
q: Record<string, unknown>,
p: RuntimePrincipal,
extras: string[] = [],
): string {
if (
Object.keys(q).some((k) => !['profile', ...extras].includes(k)) ||
typeof q.profile !== 'string' ||
!p.profiles.includes(q.profile)
)
throw new HermesError('FORBIDDEN');
return q.profile;
}
private number(
q: Record<string, unknown>,
key: string,
fallback: number,
min: number,
max: number,
): number {
if (q[key] === undefined) return fallback;
if (typeof q[key] !== 'string' || !/^\d+$/.test(q[key]))
throw new HermesError('LIMIT');
const n = Number(q[key]);
if (!Number.isSafeInteger(n) || n < min || n > max)
throw new HermesError('LIMIT');
return n;
}
@Get('health') health(@Req() req: VerifiedRuntimeRequest) {
return this.read(req, (p) => this.runtime.health(p));
}
@Get('profiles') profiles(@Req() req: VerifiedRuntimeRequest) {
return this.read(req, (p) => this.runtime.profiles(p));
}
@Get('status') status(
@Req() req: VerifiedRuntimeRequest,
@Query() q: Record<string, unknown>,
) {
return this.read(req, (p) => this.runtime.status(p, this.profile(q, p)));
}
@Get('usage') usage(
@Req() req: VerifiedRuntimeRequest,
@Query() q: Record<string, unknown>,
) {
return this.read(req, (p) =>
this.runtime.usage(
p,
this.profile(q, p, ['days']),
this.number(q, 'days', 30, 1, 365),
),
);
}
@Get('sessions') sessions(
@Req() req: VerifiedRuntimeRequest,
@Query() q: Record<string, unknown>,
) {
return this.read(req, (p) =>
this.runtime.sessions(
p,
this.profile(q, p, ['limit', 'offset']),
this.number(q, 'limit', 20, 1, 100),
this.number(q, 'offset', 0, 0, 10000),
),
);
}
@Get('schedules') schedules(
@Req() req: VerifiedRuntimeRequest,
@Query() q: Record<string, unknown>,
) {
return this.read(req, (p) => this.runtime.crons(p, this.profile(q, p)));
}
@Get('model-info') modelInfo(
@Req() req: VerifiedRuntimeRequest,
@Query() q: Record<string, unknown>,
) {
return this.read(req, (p) => this.runtime.modelInfo(p, this.profile(q, p)));
}
}
@@ -0,0 +1,88 @@
import { DynamicModule, Module } from '@nestjs/common';
import { open, lstat } from 'node:fs/promises';
import { constants } from 'node:fs';
import { dirname, resolve } from 'node:path';
import { HermesManagementModule } from './hermes-management.module';
import {
RuntimeReadAccess,
RuntimeReadController,
} from './runtime-read.controller';
import { HermesError } from './hermes.adapter';
export async function runtimeSecret(path: string): Promise<string> {
if (!/^\/run\/secrets\/[a-zA-Z0-9_-]+$/.test(path))
throw new HermesError('CONFIGURATION');
try {
const file = await open(path, 'r');
try {
const stat = await file.stat();
if (!stat.isFile() || stat.size > 4096) throw new Error();
const value = (await file.readFile('utf8')).trim();
if (!value || /[\r\n]/.test(value)) throw new Error();
return value;
} finally {
await file.close();
}
} catch {
throw new HermesError('AUTH_REQUIRED');
}
}
/** Exact fixed mount roots only; startup fails closed, never browser/JWT grants. */
export async function loadRuntimeGrants(
path = '/run/secrets/hermes_runtime_grants',
reviewedRoot?: string,
): Promise<Readonly<Record<string, readonly string[]>>> {
const roots = reviewedRoot
? [resolve(reviewedRoot)]
: ['/run/secrets', '/etc/canina'];
if (!roots.includes(dirname(path)) || resolve(path) !== path)
throw new HermesError('CONFIGURATION');
try {
const stat = await lstat(path);
if (
!stat.isFile() ||
stat.isSymbolicLink() ||
stat.size > 65536 ||
stat.mode & 0o022
)
throw new Error();
const file = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW);
try {
const data: unknown = JSON.parse(await file.readFile('utf8'));
if (!data || typeof data !== 'object' || Array.isArray(data))
throw new Error();
new RuntimeReadAccess(data as Record<string, readonly string[]>);
return data as Record<string, readonly string[]>;
} finally {
await file.close();
}
} catch {
throw new HermesError('CONFIGURATION');
}
}
@Module({})
export class RuntimeReadModule {
static async registerFromFile(
path = '/run/secrets/hermes_runtime_grants',
): Promise<DynamicModule> {
return this.register(await loadRuntimeGrants(path));
}
static register(
grants: Readonly<Record<string, readonly string[]>>,
): DynamicModule {
const access = new RuntimeReadAccess(grants);
return {
module: RuntimeReadModule,
imports: [
HermesManagementModule.registerAuthenticated(
async () => ({
username: await runtimeSecret('/run/secrets/hermes_username'),
password: await runtimeSecret('/run/secrets/hermes_password'),
}),
access,
),
],
controllers: [RuntimeReadController],
providers: [{ provide: RuntimeReadAccess, useValue: access }],
};
}
}
@@ -0,0 +1,16 @@
import { runtimeReadRegistration } from './runtime-read.registration';
import { RuntimeReadModule } from './runtime-read.module';
describe('application runtime registration',()=>{
afterEach(()=>jest.restoreAllMocks());
it('registers denied routes without reading files when not explicitly enabled',async()=>{
const file=jest.spyOn(RuntimeReadModule,'registerFromFile');
const register=jest.spyOn(RuntimeReadModule,'register');
await runtimeReadRegistration({});
expect(file).not.toHaveBeenCalled();expect(register).toHaveBeenCalledWith({});
});
it('uses only the fixed reviewed grants mount when enabled',async()=>{
const file=jest.spyOn(RuntimeReadModule,'registerFromFile').mockResolvedValue({module:RuntimeReadModule});
await runtimeReadRegistration({CANINA_RUNTIME_READ_ENABLED:'true'});
expect(file).toHaveBeenCalledWith('/run/secrets/hermes_runtime_grants');
});
});
@@ -0,0 +1,6 @@
import { RuntimeReadModule } from './runtime-read.module';
export function runtimeReadRegistration(env:Record<string,string|undefined>=process.env) {
return env.CANINA_RUNTIME_READ_ENABLED === 'true'
? RuntimeReadModule.registerFromFile('/run/secrets/hermes_runtime_grants')
: RuntimeReadModule.register({});
}
@@ -0,0 +1,75 @@
import { Test } from '@nestjs/testing';
import {
RuntimeReadController,
RuntimeReadAccess,
} from './runtime-read.controller';
import { HermesIntegrationService } from './hermes-integration.service';
import { HermesError } from './hermes.adapter';
import {
ForbiddenException,
ServiceUnavailableException,
} from '@nestjs/common';
describe('runtime read boundary', () => {
const health = jest.fn();
const profiles = jest.fn();
const status = jest.fn();
const service = {
health,
profiles,
status,
} as unknown as HermesIntegrationService;
const access = new RuntimeReadAccess({ 'admin-1': ['default'] });
const controller = new RuntimeReadController(service, access);
beforeEach(() => jest.clearAllMocks());
it('denies unconfigured identity without IO', async () => {
await expect(
controller.health({ user: { id: 'other', role: 'Admin' } }),
).rejects.toBeInstanceOf(ForbiddenException);
expect(health).not.toHaveBeenCalled();
});
it('uses server scope not body or user supplied profiles', async () => {
status.mockResolvedValue({ overall: 'ok' });
await expect(
controller.status(
{ user: { id: 'admin-1', role: 'Admin', profiles: ['other'] } },
{ profile: 'other' },
),
).rejects.toBeInstanceOf(ForbiddenException);
expect(status).not.toHaveBeenCalled();
await controller.status(
{ user: { id: 'admin-1', role: 'Admin' } },
{ profile: 'default' },
);
expect(status).toHaveBeenCalledWith(
{ subject: 'admin-1', profiles: ['default'] },
'default',
);
});
it('rejects duplicate/missing/unknown query parameters before IO', async () => {
for (const q of [
{},
{ profile: ['default', 'other'] },
{ profile: 'default', token: 'leak' },
])
await expect(
controller.status({ user: { id: 'admin-1', role: 'Admin' } }, q),
).rejects.toBeInstanceOf(ForbiddenException);
expect(status).not.toHaveBeenCalled();
});
it('maps credential failure without leaking upstream errors', async () => {
health.mockRejectedValue(new HermesError('AUTH_REQUIRED'));
await expect(
controller.health({ user: { id: 'admin-1', role: 'Admin' } }),
).rejects.toThrow('Runtime unavailable');
});
it('rejects API keys and non-admins even when subject is configured', async () => {
for (const user of [
{ id: 'admin-1', role: 'User' },
{ id: 'admin-1', role: 'Admin', isApiKey: true },
])
await expect(controller.health({ user })).rejects.toBeInstanceOf(
ForbiddenException,
);
expect(health).not.toHaveBeenCalled();
});
});
@@ -0,0 +1,68 @@
import { HermesAdapter } from './hermes.adapter';
const provenance = {
source: 'snapshot',
capturedAt: '2020-01-01T00:00:00Z',
ageSeconds: 100,
freshness: 'stale',
liveProcess: 'unknown',
};
function adapter(data: unknown) {
return new HermesAdapter(
{
origin: 'http://canina-hermes-inventory:9119',
token: () => Promise.resolve('test'),
},
jest.fn(() =>
Promise.resolve(new Response(JSON.stringify(data), { status: 200 })),
) as typeof fetch,
);
}
describe('snapshot provenance boundary', () => {
it('rejects missing provenance from dedicated inventory origin', async () => {
await expect(
adapter({
ok: true,
version: '1',
commit: 'x',
auth_required: true,
}).health(),
).rejects.toMatchObject({ code: 'CONTRACT' });
});
it('cannot present copied gateway scalars as live telemetry', async () => {
const row = await adapter({
version: '1',
gateway_running: true,
active_agents: 12,
gateway_state: 'running',
overall: 'ok',
provenance,
}).status('default');
expect(row).toMatchObject({
gateway_running: false,
active_agents: 0,
gateway_state: null,
overall: 'unknown',
liveGatewayState: 'unknown',
liveActiveAgents: null,
provenance,
});
});
it('profile readiness cannot be inferred from copied PID state', async () => {
const rows = await adapter({
profiles: [
{
name: 'default',
is_default: true,
gateway_running: true,
provenance,
},
],
provenance,
}).profiles();
expect(rows[0]).toMatchObject({
gateway_running: false,
liveGatewayState: 'unknown',
provenance,
});
});
});
@@ -0,0 +1,19 @@
# Opt-in fragment ONLY: merge into the reviewed Stage backend service definition.
# Not a standalone compose file; does not enable the worker automatically.
# Host files/dirs must already exist, with reviewed ownership and modes.
x-prepare-worker-backend:
environment:
PREPARE_WORKER_ENABLED: "${PREPARE_WORKER_ENABLED:-false}"
secrets:
- source: prepare_worker_grants
target: prepare_worker_grants
mode: 0400
volumes:
- type: bind
source: ${PREPARE_WORKER_ARTIFACT_HOST_DIRECTORY:?review private artifact directory}
target: /var/lib/canina/prepare-artifacts
bind:
create_host_path: false
secrets:
prepare_worker_grants:
file: ${PREPARE_WORKER_GRANTS_FILE:?review mounted grants JSON file}
@@ -0,0 +1,48 @@
import { constants, openSync, closeSync, fstatSync, readSync } from 'fs';
import { WorkerTransportConfig, WorkerTransportModule } from './worker-transport.module';
export const PREPARE_WORKER_CONFIG_PATH = '/run/secrets/prepare_worker_grants';
export const PREPARE_WORKER_ARTIFACT_DIRECTORY = '/var/lib/canina/prepare-artifacts';
const MAX_BYTES = 16384;
const fail = () => { throw new Error('Prepare worker startup configuration rejected'); };
function exact(value: any, keys: string[]): boolean {
return value !== null && typeof value === 'object' && !Array.isArray(value) &&
Object.keys(value).length === keys.length && keys.every(k => Object.hasOwn(value, k));
}
/** Fixed-path, descriptor-pinned, bounded startup read; never logs file contents. */
export function loadPrepareWorkerConfig(): WorkerTransportConfig {
let dir: number | undefined, file: number | undefined;
try {
dir = openSync('/', constants.O_RDONLY | constants.O_DIRECTORY);
for (const part of ['run', 'secrets']) {
const next = openSync(`/proc/self/fd/${dir}/${part}`, constants.O_RDONLY | constants.O_DIRECTORY | constants.O_NOFOLLOW);
closeSync(dir); dir = next;
if ((fstatSync(dir).mode & 0o022) !== 0) fail();
}
file = openSync(`/proc/self/fd/${dir}/prepare_worker_grants`, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK);
const stat = fstatSync(file);
if (!stat.isFile() || stat.nlink !== 1 || (stat.mode & 0o022) !== 0 || stat.size > MAX_BYTES ||
(stat.uid !== 0 && stat.uid !== process.getuid!())) fail();
const bytes = Buffer.alloc(MAX_BYTES + 1);
let length = 0, n: number;
do { n = readSync(file, bytes, length, bytes.length - length, length); length += n; }
while (n > 0 && length < bytes.length);
if (length > MAX_BYTES) fail();
const input = JSON.parse(new TextDecoder('utf-8', { fatal: true }).decode(bytes.subarray(0, length)));
if (!exact(input, ['grants', 'workerIds', 'stageCiIds', 'artifactBaseUrl']) ||
!Array.isArray(input.grants) || input.grants.length < 1 || input.grants.length > 32 ||
!Array.isArray(input.workerIds) || !Array.isArray(input.stageCiIds) ||
input.workerIds.length < 1 || input.stageCiIds.length < 1 ||
input.workerIds.length + input.stageCiIds.length !== input.grants.length ||
typeof input.artifactBaseUrl !== 'string' || input.artifactBaseUrl.length > 2048 ||
input.grants.some((g: any) => !exact(g, ['id', 'role', 'credentialSha256']))) fail();
// Transport registration performs shared role, SHA256, ID, overlap and HTTPS validation.
const config: WorkerTransportConfig = { ...input, enabled: true, artifactDirectory: PREPARE_WORKER_ARTIFACT_DIRECTORY };
WorkerTransportModule.register(config); // Reuse the real transport's reviewed validation, no IO.
config.grants.forEach(Object.freeze);
Object.freeze(config.grants); Object.freeze(config.workerIds); Object.freeze(config.stageCiIds);
return Object.freeze(config);
} catch { return fail(); }
finally { if (file !== undefined) closeSync(file); if (dir !== undefined) closeSync(dir); }
}
@@ -0,0 +1,21 @@
# Prepare worker startup boundary
Parent integration API (AppModule remains untouched here):
```ts
import { registerPrepareWorker } from './orchestration-worker/worker-registration';
// In AppModule imports:
...registerPrepareWorker(),
```
Only literal `PREPARE_WORKER_ENABLED=true` registers the transport. Missing or other values register nothing (404 routes), do not read configuration, and do not instantiate Prisma. Enabled malformed/missing configuration throws synchronously before Nest starts, with one generic error and no secret logging.
The fixed mount `/run/secrets/prepare_worker_grants` contains JSON with exactly `grants`, `workerIds`, `stageCiIds`, `artifactBaseUrl`. Each grant contains exactly `id`, `role` (`WORKFLOW_WORKER` or `STAGE_CI`), `credentialSha256` (64 lowercase hexadecimal characters; never the bearer credential). Require at least one worker and CI, distinct role-bound IDs/digests, exact allowlist coverage, and at most 32 grants. No enabled flag, directory, token, user, or request-derived fields accepted in JSON.
`artifactBaseUrl` must be explicitly reviewed by the operator: canonical public-DNS HTTPS origin plus `/internal/prepare-worker/artifacts`, no credentials, IP, alternate port, query or fragment. Shared transport validation is authoritative. This loader does not fetch the URL or perform DNS resolution. Directory is always `/var/lib/canina/prepare-artifacts`.
Descriptor-relative Linux `/proc/self/fd` traversal refuses symlink directories/files, nonregular files, hardlinks, files over 16 KiB, group/world writable files/directories, and foreign-owned files (root/runtime UID only). Reads are bounded even on concurrent growth. Configuration is frozen recursively and transport takes its own frozen snapshot. Trusted operator owns mounted configuration; edits require restart and review.
`compose.prepare-worker.fragment.yml` is an extension fragment, not a deployable compose file. Parent must merge its `x-prepare-worker-backend` into the correct reviewed Stage backend service and merge `secrets`. Compose bind-backed secrets may ignore mode/UID declarations: provision actual host grants file as root/runtime-owned mode 0400 and artifact directory runtime-owned mode 0700 first. No mounts or live configuration are created by this change.
Tests deliberately use the fixed `/run/secrets` path. Run ONLY in a fresh disposable container with source and donor dependencies read-only, private dependency copy, and private `prisma generate`; never on a live service. Tests need a disposable Linux root filesystem and do not require DB credentials. Registration test uses a real Nest HTTP listener to prove disabled routes are absent. Existing DB-backed transport tracer remains separately gated by `WORKER_TRANSPORT_DB=1`.
@@ -0,0 +1,84 @@
import { Test } from '@nestjs/testing';
import { writeFileSync, chmodSync, symlinkSync, mkdirSync, rmSync } from 'fs';
import { execFileSync } from 'child_process';
import { registerPrepareWorker } from './worker-registration';
import { loadPrepareWorkerConfig } from './worker-config';
import { linkSync } from 'fs';
describe('reviewed prepare-worker startup registration', () => {
const path = '/run/secrets/prepare_worker_grants';
const valid = () => ({ grants: [{ id: 'worker', role: 'WORKFLOW_WORKER', credentialSha256: 'a'.repeat(64) }, { id: 'ci', role: 'STAGE_CI', credentialSha256: 'b'.repeat(64) }], workerIds: ['worker'], stageCiIds: ['ci'], artifactBaseUrl: 'https://stage.example.com/internal/prepare-worker/artifacts' });
it('loader itself rejects malformed SHA grants', () => {
mkdirSync('/run/secrets', { recursive: true });
const c = valid(); c.grants[0].credentialSha256 = 'secret-not-a-sha';
writeFileSync(path, JSON.stringify(c), { mode: 0o400 });
try { expect(() => loadPrepareWorkerConfig()).toThrow('Prepare worker startup configuration rejected'); }
finally { rmSync(path, { force: true }); }
});
it.each(['1', 'TRUE', 'yes', ' true', 'false', ''])('does not opt in for %s', value => {
expect(registerPrepareWorker({ PREPARE_WORKER_ENABLED: value })).toEqual([]);
});
it.each(['missing', 'symlink', 'directory', 'fifo', 'oversize', 'group-write', 'world-write', 'hardlink'])('fails startup on unsafe file: %s', kind => {
mkdirSync('/run/secrets', { recursive: true });
const other = '/run/secrets/test-other';
try {
if (kind === 'fifo') execFileSync('mkfifo', [path]);
else if (kind === 'directory') mkdirSync(path);
else if (kind === 'symlink') symlinkSync(other, path);
else if (kind !== 'missing') {
writeFileSync(path, kind === 'oversize' ? ' '.repeat(16385) : JSON.stringify(valid()), { mode: 0o400 });
if (kind === 'group-write') chmodSync(path, 0o420);
if (kind === 'world-write') chmodSync(path, 0o402);
if (kind === 'hardlink') linkSync(path, other);
}
expect(() => registerPrepareWorker({ PREPARE_WORKER_ENABLED: 'true' })).toThrow('Prepare worker startup configuration rejected');
} finally { rmSync(path, { recursive: true, force: true }); rmSync(other, { force: true }); }
});
it.each(['extra', 'raw-token', 'wrong-role', 'overlap', 'duplicate-sha', 'missing-grant', 'http', 'credentials', 'localhost', 'query', 'bad-json', 'null'])('rejects unreviewed JSON: %s without leaking contents or logging', kind => {
mkdirSync('/run/secrets', { recursive: true });
const c: any = valid();
if (kind === 'extra') c.artifactDirectory = '/user-selected';
if (kind === 'raw-token') c.grants[0].token = 'DO_NOT_LOG_THIS_SECRET';
if (kind === 'wrong-role') c.grants[0].role = 'Admin';
if (kind === 'overlap') c.stageCiIds = ['worker'];
if (kind === 'duplicate-sha') c.grants[1].credentialSha256 = c.grants[0].credentialSha256;
if (kind === 'missing-grant') c.workerIds.push('unbound');
if (kind === 'http') c.artifactBaseUrl = 'http://stage.example.com/internal/prepare-worker/artifacts';
if (kind === 'credentials') c.artifactBaseUrl = 'https://DO_NOT_LOG_THIS_SECRET@stage.example.com/internal/prepare-worker/artifacts';
if (kind === 'localhost') c.artifactBaseUrl = 'https://127.0.0.1/internal/prepare-worker/artifacts';
if (kind === 'query') c.artifactBaseUrl += '?secret=DO_NOT_LOG_THIS_SECRET';
writeFileSync(path, kind === 'bad-json' ? 'DO_NOT_LOG_THIS_SECRET' : kind === 'null' ? 'null' : JSON.stringify(c), { mode: 0o400 });
const logs = [jest.spyOn(console, 'log'), jest.spyOn(console, 'warn'), jest.spyOn(console, 'error')];
try {
expect(() => registerPrepareWorker({ PREPARE_WORKER_ENABLED: 'true' })).toThrow(/^Prepare worker startup configuration rejected$/);
logs.forEach(log => expect(log).not.toHaveBeenCalled());
} finally { logs.forEach(log => log.mockRestore()); rmSync(path, { force: true }); }
});
it('defaults to no module/no config IO and exposes no worker routes', async () => {
expect(registerPrepareWorker({})).toEqual([]);
expect(registerPrepareWorker({ PREPARE_WORKER_ENABLED: 'false' })).toEqual([]);
const module = await Test.createTestingModule({ imports: registerPrepareWorker({}) }).compile();
const app = module.createNestApplication();
await app.listen(0, '127.0.0.1');
try {
const response = await fetch((await app.getUrl()) + '/internal/prepare-worker/claim', { method: 'POST' });
expect(response.status).toBe(404);
} finally { await app.close(); }
});
it('loads immutable reviewed JSON into the real transport registration', async () => {
mkdirSync('/run/secrets', { recursive: true });
const file = '/run/secrets/prepare_worker_grants';
const config = { grants: [{ id: 'worker', role: 'WORKFLOW_WORKER', credentialSha256: 'a'.repeat(64) }, { id: 'ci', role: 'STAGE_CI', credentialSha256: 'b'.repeat(64) }], workerIds: ['worker'], stageCiIds: ['ci'], artifactBaseUrl: 'https://stage.example.com/internal/prepare-worker/artifacts' };
writeFileSync(file, JSON.stringify(config), { mode: 0o400 });
try {
const [registered] = registerPrepareWorker({ PREPARE_WORKER_ENABLED: 'true' });
expect(registered.module.name).toBe('WorkerTransportModule');
expect(registered.controllers?.length).toBe(1);
const snapshot = (registered.providers as any[]).find(p => p.useValue?.artifactDirectory)?.useValue;
expect(snapshot).toEqual({ ...config, enabled: true, artifactDirectory: '/var/lib/canina/prepare-artifacts' });
expect(Object.isFrozen(snapshot)).toBe(true);
expect(Object.isFrozen(snapshot.grants[0])).toBe(true);
} finally { rmSync(file, { force: true }); }
});
});
@@ -0,0 +1,10 @@
import { DynamicModule } from '@nestjs/common';
import { loadPrepareWorkerConfig } from './worker-config';
import { WorkerTransportModule } from './worker-transport.module';
/** AppModule imports: ...registerPrepareWorker(); disabled means no module or IO. */
export function registerPrepareWorker(env: NodeJS.ProcessEnv = process.env): DynamicModule[] {
if (env.PREPARE_WORKER_ENABLED !== 'true') return [];
try { return [WorkerTransportModule.register(loadPrepareWorkerConfig())]; }
catch { throw new Error('Prepare worker startup configuration rejected'); }
}
@@ -0,0 +1,343 @@
import {
BadRequestException,
Body,
CanActivate,
ConflictException,
Controller,
DynamicModule,
ForbiddenException,
Get,
Inject,
Injectable,
Module,
NotFoundException,
Param,
Post,
Req,
UnauthorizedException,
UseGuards,
UsePipes,
ValidationPipe,
} from "@nestjs/common";
import { IsString, IsUUID, Matches, MaxLength } from "class-validator";
import { createHash, timingSafeEqual } from "crypto";
import { mkdir, open, FileHandle } from "fs/promises";
import { constants } from "fs";
import { isAbsolute } from "path";
import { PrismaModule } from "../prisma/prisma.module";
import { PrismaService } from "../prisma/prisma.service";
import {
PREPARE_WORKFLOW_CONFIG,
PrepareWorkflowService,
WorkflowIdentity,
} from "../orchestration/prepare-workflow.service";
// This is reviewed static configuration, not a URL fetched by the transport.
function validArtifactBase(value: string): boolean {
try {
const u = new URL(value);
// DNS name only: reject all IP literals, localhost/private single-label names,
// noncanonical paths/escapes, credentials, query, fragment and alternate ports.
return u.protocol === "https:" && !u.username && !u.password && !u.port &&
/^[a-z0-9](?:[a-z0-9.-]*[a-z0-9])?$/.test(u.hostname) &&
u.hostname.includes(".") && !/^[0-9.]+$/.test(u.hostname) &&
!/(^|\.)(localhost|local|internal|lan|home)$/.test(u.hostname) &&
u.pathname === "/internal/prepare-worker/artifacts" &&
value === u.origin + u.pathname;
} catch { return false; }
}
const CONFIG = Symbol("WORKER_TRANSPORT_CONFIG");
export interface WorkerTransportConfig {
enabled: boolean;
grants: {
id: string;
role: "WORKFLOW_WORKER" | "STAGE_CI";
credentialSha256: string;
}[];
workerIds: string[];
stageCiIds: string[];
artifactDirectory: string;
artifactBaseUrl: string;
}
class ClaimDto {
@IsUUID() taskId!: string;
}
class LeaseDto extends ClaimDto {
@IsUUID() runId!: string;
@IsUUID() leaseToken!: string;
}
class CompleteDto extends LeaseDto {
@IsString() @MaxLength(2048) report!: string;
}
class StageEvidenceDto extends ClaimDto {
@IsUUID() runId!: string;
@Matches(/^[a-f0-9]{40}$/) commitSha!: string;
@Matches(/^[a-f0-9]{64}$/) sha256!: string;
@Matches(/^[a-f0-9]{64}$/) evidenceSha256!: string;
@Matches(/^https:\/\/[^\s]+$/) @MaxLength(2000) uri!: string;
}
export function tracerReport(lease: {
taskId: string;
runId: string;
commitSha: string;
}) {
return JSON.stringify({
kind: "DETERMINISTIC_PREPARE_TRACER",
schemaVersion: 1,
taskId: lease.taskId,
runId: lease.runId,
commitSha: lease.commitSha,
environment: "staging",
llmPilot: false,
browserExecuted: false,
shellExecuted: false,
deploymentExecuted: false,
checks: ["lease-bound identity", "deterministic JSON serialization"],
actualCents: 0,
});
}
@Injectable()
class WorkerIdentityGuard implements CanActivate {
constructor(@Inject(CONFIG) private readonly config: WorkerTransportConfig) {}
canActivate(context: any) {
if (!this.config.enabled)
throw new ForbiddenException("Worker transport disabled");
const req = context.switchToHttp().getRequest();
const auth = req.headers.authorization;
if (typeof auth !== "string" || !/^Bearer [\x21-\x7e]{32,512}$/.test(auth))
throw new UnauthorizedException();
const digest = createHash("sha256").update(auth.slice(7)).digest();
const grant = this.config.grants.find((g) =>
timingSafeEqual(Buffer.from(g.credentialSha256, "hex"), digest),
);
if (!grant) throw new UnauthorizedException();
req.workerIdentity = { id: grant.id, role: grant.role };
return true;
}
}
@Injectable()
class WorkerTransportService {
constructor(
private readonly workflow: PrepareWorkflowService,
private readonly db: PrismaService,
@Inject(CONFIG) private readonly config: WorkerTransportConfig,
) {}
// Linux descriptor-relative traversal pins each directory across renames. Never
// realpath-check then reopen by name: that reintroduces a symlink race.
private async artifactDir(): Promise<FileHandle> {
let dir = await open("/", constants.O_RDONLY | constants.O_DIRECTORY);
try {
for (const part of this.config.artifactDirectory.split("/").filter(Boolean)) {
const path = `/proc/self/fd/${dir.fd}/${part}`;
try { await mkdir(path, { mode: 0o700 }); }
catch (e: any) { if (e.code !== "EEXIST") throw e; }
const next = await open(path, constants.O_RDONLY | constants.O_DIRECTORY | constants.O_NOFOLLOW);
await dir.close();
dir = next;
}
const stat = await dir.stat();
if (stat.uid !== process.getuid!() || (stat.mode & 0o077) !== 0)
throw new Error("Private owned artifact directory required");
return dir;
} catch (e) { await dir.close(); throw e; }
}
private worker(a: WorkflowIdentity) {
if (a.role !== "WORKFLOW_WORKER" || !this.config.workerIds.includes(a.id))
throw new ForbiddenException("Worker allowlist required");
}
async claim(a: WorkflowIdentity, dto: ClaimDto) {
this.worker(a);
const t = await this.db.orchestrationTask.findUnique({
where: { id: dto.taskId },
});
// Explicit per-task pilot scope; never opportunistically consume arbitrary queued work.
if (!t || !["READY", "RUNNING"].includes(t.state)) return { lease: null };
if (t.budgetLimitCents !== 20)
throw new ForbiddenException("Pilot hard task cap is 20 cents");
return { lease: await this.workflow.claim(a, dto.taskId) };
}
heartbeat(a: WorkflowIdentity, dto: LeaseDto) {
this.worker(a);
return this.workflow.heartbeat(a, dto);
}
async complete(a: WorkflowIdentity, dto: CompleteDto) {
this.worker(a);
// Recheck stop/interrupt and fencing before artifact creation; core rechecks atomically at commit.
await this.workflow.heartbeat(a, dto);
const run = await this.db.orchestrationRun.findUniqueOrThrow({
where: { id: dto.runId },
});
const expected = tracerReport({ ...dto, commitSha: run.commitSha });
if (dto.report !== expected)
throw new BadRequestException(
"Only deterministic harmless tracer report accepted",
);
const sha256 = createHash("sha256").update(expected).digest("hex");
let dir: FileHandle | undefined, file: FileHandle | undefined;
try {
dir = await this.artifactDir();
file = await open(`/proc/self/fd/${dir.fd}/${sha256}.json`,
constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, 0o600);
await file.writeFile(expected, "utf8");
await file.sync();
} catch { throw new ConflictException("Artifact creation refused"); }
finally { await file?.close(); await dir?.close(); }
const result = await this.workflow.complete(a, {
...dto,
sha256,
uri: this.config.artifactBaseUrl + "/" + sha256,
actualCents: 0,
});
return {
state: result.state,
taskId: result.id,
runId: dto.runId,
sha256,
actualCents: 0,
};
}
async artifact(a: WorkflowIdentity, hash: string) {
if (!/^[a-f0-9]{64}$/.test(hash))
throw new BadRequestException("Invalid artifact hash");
const registered = await this.db.orchestrationArtifact.findFirst({
where: {
sha256: hash, kind: "QA", uri: this.config.artifactBaseUrl + "/" + hash,
run: { environment: "staging", status: "SUCCEEDED", task: {
operationMode: "PREPARE", budgetLimitCents: 20,
} },
},
});
if (!registered) throw new NotFoundException("Artifact not registered");
const run = await this.db.orchestrationRun.findUniqueOrThrow({ where: { id: registered.runId } });
const task = await this.db.orchestrationTask.findUniqueOrThrow({ where: { id: run.taskId } });
const outbox = await this.db.prepareWorkflowOutbox.findUnique({ where: { taskId: run.taskId } });
if (!outbox || outbox.status !== "DONE" || outbox.runId !== run.id ||
(a.role === "WORKFLOW_WORKER" && outbox.leaseOwner !== a.id))
throw new NotFoundException("Artifact not registered");
const metadata = task.metadata as any;
if (metadata?.workflowScope !== "STAGE_PREPARE" || metadata?.commitSha !== run.commitSha)
throw new NotFoundException("Artifact not registered");
let bytes: string;
let dir: FileHandle | undefined, file: FileHandle | undefined;
try {
dir = await this.artifactDir();
file = await open(`/proc/self/fd/${dir.fd}/${hash}.json`,
constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK);
const stat = await file.stat();
if (!stat.isFile() || stat.nlink !== 1 || stat.uid !== process.getuid!() ||
(stat.mode & 0o077) !== 0 || stat.size > 2048)
throw new Error("Unsafe artifact file");
// Fixed-size descriptor read: concurrent growth cannot allocate unbounded memory.
const buffer = Buffer.alloc(2049);
const { bytesRead } = await file.read(buffer, 0, buffer.length, 0);
if (bytesRead > 2048) throw new Error("Oversized artifact");
bytes = buffer.subarray(0, bytesRead).toString("utf8");
} catch {
throw new NotFoundException("Artifact unavailable");
} finally { await file?.close(); await dir?.close(); }
if (createHash("sha256").update(bytes).digest("hex") !== hash)
throw new ConflictException("Artifact integrity mismatch");
if (bytes !== tracerReport({ taskId: run.taskId, runId: run.id, commitSha: run.commitSha }))
throw new ConflictException("Artifact binding mismatch");
return JSON.parse(bytes);
}
stage(a: WorkflowIdentity, dto: StageEvidenceDto) {
return this.workflow.stageEvidence(a, dto);
}
}
@Controller("internal/prepare-worker")
@UseGuards(WorkerIdentityGuard)
@UsePipes(
new ValidationPipe({
whitelist: true,
forbidNonWhitelisted: true,
transform: true,
}),
)
class WorkerTransportController {
constructor(private readonly service: WorkerTransportService) {}
@Post("claim") claim(@Req() req: any, @Body() dto: ClaimDto) {
return this.service.claim(req.workerIdentity, dto);
}
@Post("heartbeat") heartbeat(@Req() req: any, @Body() dto: LeaseDto) {
return this.service.heartbeat(req.workerIdentity, dto);
}
@Post("complete") complete(@Req() req: any, @Body() dto: CompleteDto) {
return this.service.complete(req.workerIdentity, dto);
}
@Post("stage-evidence") stage(
@Req() req: any,
@Body() dto: StageEvidenceDto,
) {
return this.service.stage(req.workerIdentity, dto);
}
@Get("artifacts/:hash") artifact(@Req() req: any, @Param("hash") hash: string) {
return this.service.artifact(req.workerIdentity, hash);
}
}
@Module({})
export class WorkerTransportModule {
static register(config: WorkerTransportConfig): DynamicModule {
if (typeof config.enabled !== "boolean" || !Array.isArray(config.grants) ||
!Array.isArray(config.workerIds) || !Array.isArray(config.stageCiIds))
throw new Error("Explicit boolean enable and credential allowlists required");
const allIds = [...config.workerIds, ...config.stageCiIds];
if (allIds.some(id => typeof id !== "string" || !/^[a-zA-Z0-9_-]{1,100}$/.test(id)) ||
new Set(allIds).size !== allIds.length)
throw new Error("Invalid or overlapping worker/CI allowlists");
const ids = new Set<string>(),
digests = new Set<string>();
if (
!isAbsolute(config.artifactDirectory) || config.artifactDirectory === "/" ||
config.artifactDirectory.split("/").some(p => p === "." || p === "..") ||
!validArtifactBase(config.artifactBaseUrl)
)
throw new Error(
"Absolute artifact directory and fixed HTTPS artifact base required",
);
for (const g of config.grants) {
const allowed =
g.role === "WORKFLOW_WORKER"
? config.workerIds
: g.role === "STAGE_CI"
? config.stageCiIds
: [];
if (
!/^[a-zA-Z0-9_-]{1,100}$/.test(g.id) ||
!allowed.includes(g.id) ||
!/^[a-f0-9]{64}$/.test(g.credentialSha256) ||
ids.has(g.id) ||
digests.has(g.credentialSha256) ||
(config.workerIds.includes(g.id) && config.stageCiIds.includes(g.id))
)
throw new Error("Invalid or overlapping worker/CI credential grants");
ids.add(g.id);
digests.add(g.credentialSha256);
}
const snapshot = JSON.parse(JSON.stringify(config)) as WorkerTransportConfig;
snapshot.grants.forEach(Object.freeze);
Object.freeze(snapshot.grants);
Object.freeze(snapshot.workerIds);
Object.freeze(snapshot.stageCiIds);
Object.freeze(snapshot);
return {
module: WorkerTransportModule,
imports: [PrismaModule],
controllers: [WorkerTransportController],
providers: [
PrepareWorkflowService,
WorkerIdentityGuard,
WorkerTransportService,
{ provide: CONFIG, useValue: snapshot },
{
provide: PREPARE_WORKFLOW_CONFIG,
useValue: Object.freeze({
workerIds: snapshot.workerIds,
stageCiIds: snapshot.stageCiIds,
}),
},
],
exports: [PrepareWorkflowService],
};
}
}
@@ -0,0 +1,438 @@
import { Test } from "@nestjs/testing";
import { INestApplication } from "@nestjs/common";
import { PrismaClient } from "@prisma/client";
import { PrismaService } from "../prisma/prisma.service";
import { PrepareWorkflowService } from "../orchestration/prepare-workflow.service";
import { WorkerTransportModule, tracerReport } from "./worker-transport.module";
import { createHash, randomUUID } from "crypto";
import { mkdtemp, readFile, rm, writeFile, symlink, link, rename } from "fs/promises";
import { tmpdir } from "os";
import { join } from "path";
import { execFile } from "child_process";
import { promisify } from "util";
const enabled = process.env.WORKER_TRANSPORT_DB === "1";
(enabled ? describe : describe.skip)(
"isolated PostgreSQL HTTP executor tracer",
() => {
const db = new PrismaClient();
const token = "test-only-worker-secret-".repeat(3);
const ciToken = "test-only-ci-secret-".repeat(3);
const digest = (s: string) => createHash("sha256").update(s).digest("hex");
let app: INestApplication,
workflow: PrepareWorkflowService,
base: string,
directory: string;
let pilot: any;
beforeAll(async () => {
directory = await mkdtemp(join(tmpdir(), "worker-artifacts-"));
const mod = await Test.createTestingModule({
imports: [
WorkerTransportModule.register({
enabled: true,
grants: [
{
id: "pilot",
role: "WORKFLOW_WORKER",
credentialSha256: digest(token),
},
{ id: "ci", role: "STAGE_CI", credentialSha256: digest(ciToken) },
{ id: "other", role: "WORKFLOW_WORKER", credentialSha256: digest("other-worker-secret-".repeat(3)) },
],
workerIds: ["pilot", "other"],
stageCiIds: ["ci"],
artifactDirectory: directory,
artifactBaseUrl:
"https://isolated.invalid/internal/prepare-worker/artifacts",
}),
],
})
.overrideProvider(PrismaService)
.useValue(db)
.compile();
workflow = mod.get(PrepareWorkflowService);
app = mod.createNestApplication();
await app.listen(0, "127.0.0.1");
base = await app.getUrl();
pilot = {
runOnce: async (options: any) => {
const program =
'import {runOnce} from "../infra/hermes-worker/pilot.mjs"; console.log(JSON.stringify(await runOnce(JSON.parse(process.env.TEST_PILOT_OPTIONS))));';
const result = await promisify(execFile)(
process.execPath,
["--input-type=module", "-e", program],
{
env: {
...process.env,
TEST_PILOT_OPTIONS: JSON.stringify(options),
},
timeout: 15000,
},
);
return JSON.parse(result.stdout);
},
};
});
afterAll(async () => {
await app?.close();
await db.$disconnect();
if (directory) await rm(directory, { recursive: true, force: true });
});
const owner = { id: "author", role: "Admin" },
reviewer = { id: "reviewer", role: "SUPER_ADMIN" };
async function proposal(queue = true, budget = 20) {
let t = await workflow.create(
{
key: randomUUID(),
title: "deterministic tracer",
department: "TEST",
operationMode: "PREPARE",
budgetLimitCents: budget,
metadata: {
commitSha: "a".repeat(40),
scope: "Harmless tracer only",
acceptanceCriteria: ["writes deterministic JSON; no release"],
},
},
owner,
);
if (queue) {
t = await workflow.transition(
t.id,
{
toState: "GATE1_PENDING",
expectedVersion: t.version,
idempotencyKey: randomUUID(),
},
owner,
);
t = await workflow.decide(
t.id,
1,
{
decision: "APPROVED",
expiresAt: new Date(Date.now() + 60000).toISOString(),
expectedVersion: t.version,
idempotencyKey: randomUUID(),
},
reviewer,
);
t = await workflow.transition(
t.id,
{
toState: "READY",
expectedVersion: t.version,
idempotencyKey: randomUUID(),
},
owner,
);
}
return t;
}
async function post(route: string, body: any = {}, secret = token) {
const r = await fetch(base + "/internal/prepare-worker/" + route, {
method: "POST",
headers: {
authorization: "Bearer " + secret,
"content-type": "application/json",
},
body: JSON.stringify(body),
});
return { status: r.status, body: await r.json() };
}
it("rejects overlapping worker and CI grants and accepts no credentials when disabled", async () => {
expect(() =>
WorkerTransportModule.register({
enabled: true,
grants: [
{
id: "both",
role: "WORKFLOW_WORKER",
credentialSha256: digest(token),
},
],
workerIds: ["both"],
stageCiIds: ["both"],
artifactDirectory: directory,
artifactBaseUrl: "https://isolated.invalid/internal/prepare-worker/artifacts",
}),
).toThrow(/overlapping/);
const mod = await Test.createTestingModule({
imports: [
WorkerTransportModule.register({
enabled: false,
grants: [],
workerIds: [],
stageCiIds: [],
artifactDirectory: directory,
artifactBaseUrl: "https://isolated.invalid/internal/prepare-worker/artifacts",
}),
],
})
.overrideProvider(PrismaService)
.useValue(db)
.compile();
const disabled = mod.createNestApplication();
await disabled.listen(0, "127.0.0.1");
try {
const r = await fetch(
(await disabled.getUrl()) + "/internal/prepare-worker/claim",
{
method: "POST",
headers: {
authorization: "Bearer " + token,
"content-type": "application/json",
},
body: JSON.stringify({ taskId: randomUUID() }),
},
);
expect(r.status).toBe(403);
} finally {
await disabled.close();
}
});
it("rejects unsafe artifact bases and freezes validated configuration", () => {
const config = { enabled: true, grants: [], workerIds: [], stageCiIds: [], artifactDirectory: directory,
artifactBaseUrl: "https://isolated.invalid/internal/prepare-worker/artifacts" };
for (const artifactBaseUrl of ["https://user:pass@isolated.invalid/internal/prepare-worker/artifacts", "https://127.0.0.1/internal/prepare-worker/artifacts", "https://10.1.2.3/internal/prepare-worker/artifacts", "https://localhost/internal/prepare-worker/artifacts", "https://isolated.invalid/arbitrary", "https://isolated.invalid/x/../internal/prepare-worker/artifacts", "https://[::1]/internal/prepare-worker/artifacts"]) {
expect(() => WorkerTransportModule.register({ ...config, artifactBaseUrl })).toThrow();
}
const registered = WorkerTransportModule.register(config);
const snapshot = (registered.providers as any[]).find(p => p.useValue?.artifactBaseUrl)?.useValue;
expect(Object.isFrozen(snapshot)).toBe(true);
expect(Object.isFrozen(snapshot.workerIds)).toBe(true);
expect(Object.isFrozen(snapshot.grants)).toBe(true);
const workflowConfig = (registered.providers as any[]).find(p => p.provide === "PREPARE_WORKFLOW_CONFIG").useValue;
expect(Object.isFrozen(workflowConfig)).toBe(true);
expect(() => WorkerTransportModule.register({ ...config, enabled: "true" as any })).toThrow();
});
it("denies untrusted identities and body impersonation; dispatches nothing before Gate1", async () => {
const t = await proposal(false);
expect((await post("claim", {}, "wrong")).status).toBe(401);
expect((await post("claim", { taskId: t.id }, ciToken)).status).toBe(403);
expect(
(await post("claim", { taskId: t.id, actorId: "pilot" })).status,
).toBe(400);
expect((await post("claim", { taskId: t.id })).body).toEqual({
lease: null,
});
expect(await db.orchestrationRun.count({ where: { taskId: t.id } })).toBe(
0,
);
});
it("races claims, fences expired leases and reserves unknown cost against the 20 cent cap", async () => {
const t = await proposal();
const claims = await Promise.all([
post("claim", { taskId: t.id }),
post("claim", { taskId: t.id }),
]);
const leases = claims.map((r) => r.body.lease).filter(Boolean);
expect(leases).toHaveLength(1);
const lease = leases[0],
binding = {
taskId: t.id,
runId: lease.runId,
leaseToken: lease.leaseToken,
};
expect(
(await post("heartbeat", { ...binding, leaseToken: randomUUID() }))
.status,
).toBe(403);
expect(
(
await post("complete", {
...binding,
report: "arbitrary shell or prompt",
})
).status,
).toBe(400);
expect(
(await post("complete", { ...binding, report: "{}", actualCents: 21 }))
.status,
).toBe(400);
await db.prepareWorkflowOutbox.update({
where: { taskId: t.id },
data: { leaseUntil: new Date(Date.now() - 1000) },
});
expect((await post("heartbeat", binding)).status).toBe(403);
expect(
(await post("complete", { ...binding, report: "{}" })).status,
).toBe(403);
expect((await post("claim", { taskId: t.id })).status).toBe(403);
expect(await db.orchestrationRun.count({ where: { taskId: t.id } })).toBe(
1,
);
expect(
(
await db.orchestrationBudgetLedger.aggregate({
where: { taskId: t.id },
_sum: { amountCents: true },
})
)._sum.amountCents,
).toBe(20);
const larger = await proposal(true, 40);
expect((await post("claim", { taskId: larger.id })).status).toBe(403);
});
it("rechecks kill switch and task interrupt on heartbeat and completion; separates CI authority", async () => {
const t = await proposal(),
lease = (await post("claim", { taskId: t.id })).body.lease;
const binding = {
taskId: t.id,
runId: lease.runId,
leaseToken: lease.leaseToken,
};
await db.orchestrationKillSwitch.upsert({
where: { scope: "GLOBAL" },
create: {
scope: "GLOBAL",
level: "STOP_AGENTS",
active: true,
reason: "isolated test",
actorId: "test",
},
update: { level: "STOP_AGENTS", active: true },
});
try {
expect((await post("heartbeat", binding)).status).toBe(403);
expect(
(await post("complete", { ...binding, report: "{}" })).status,
).toBe(403);
expect((await post("claim", { taskId: t.id })).status).toBe(403);
} finally {
await db.orchestrationKillSwitch.deleteMany();
}
await db.orchestrationTask.update({
where: { id: t.id },
data: { state: "CANCELLED" },
});
expect((await post("heartbeat", binding)).status).toBe(403);
expect(
(await post("complete", { ...binding, report: "{}" })).status,
).toBe(403);
expect(
await db.orchestrationArtifact.count({ where: { runId: lease.runId } }),
).toBe(0);
const evidence = {
taskId: t.id,
runId: lease.runId,
commitSha: "a".repeat(40),
sha256: "b".repeat(64),
evidenceSha256: "c".repeat(64),
uri: "https://isolated.invalid/not-real-ci",
};
expect((await post("stage-evidence", evidence)).status).toBe(403);
expect((await post("stage-evidence", evidence, ciToken)).status).toBe(
403,
);
});
it("never follows or overwrites preexisting artifact links on completion", async () => {
const t = await proposal();
const lease = (await post("claim", { taskId: t.id })).body.lease;
const report = tracerReport(lease), hash = digest(report);
const target = join(directory, "sentinel");
await writeFile(target, "do not overwrite");
const file = join(directory, hash + ".json");
await symlink(target, file);
const result = await post("complete", { taskId: t.id, runId: lease.runId, leaseToken: lease.leaseToken, report });
expect(await readFile(target, "utf8")).toBe("do not overwrite");
expect(result.status).toBe(409);
expect(await db.orchestrationArtifact.count({ where: { runId: lease.runId } })).toBe(0);
await rm(file);
await link(target, file);
expect((await post("complete", { taskId: t.id, runId: lease.runId, leaseToken: lease.leaseToken, report })).status).toBe(409);
expect(await readFile(target, "utf8")).toBe("do not overwrite");
});
it("runs actual HTTP deterministic executor to VERIFYING with retrievable content-addressed artifact", async () => {
const t = await proposal();
const result = await pilot.runOnce({
baseUrl: base,
credential: token,
taskId: t.id,
allowLoopbackHttp: true,
});
expect(result.state).toBe("VERIFYING");
const row = await db.orchestrationTask.findUniqueOrThrow({
where: { id: t.id },
});
expect(row.state).toBe("VERIFYING");
const artifact = await db.orchestrationArtifact.findFirstOrThrow({
where: { run: { taskId: t.id } },
});
const bytes = await readFile(join(directory, artifact.sha256 + ".json"));
expect(digest(bytes.toString())).toBe(artifact.sha256);
expect(JSON.parse(bytes.toString())).toMatchObject({
kind: "DETERMINISTIC_PREPARE_TRACER",
commitSha: "a".repeat(40),
environment: "staging",
llmPilot: false,
});
const get = await fetch(
base + "/internal/prepare-worker/artifacts/" + artifact.sha256,
{ headers: { authorization: "Bearer " + token } },
);
expect(get.status).toBe(200);
expect(await get.text()).toBe(bytes.toString());
const artifactGet = (secret: string) => fetch(base + "/internal/prepare-worker/artifacts/" + artifact.sha256,
{ headers: { authorization: "Bearer " + secret } });
expect((await artifactGet("other-worker-secret-".repeat(3))).status).toBe(404);
expect((await artifactGet(ciToken)).status).toBe(200);
const foreignHash = digest("{}");
await writeFile(join(directory, foreignHash + ".json"), "{}");
await db.orchestrationArtifact.create({ data: { runId: artifact.runId, kind: "QA", sha256: foreignHash, uri: "https://foreign.invalid/" + foreignHash } });
expect((await fetch(base + "/internal/prepare-worker/artifacts/" + foreignHash, { headers: { authorization: "Bearer " + token } })).status).toBe(404);
const moved = directory + "-moved";
await rename(directory, moved);
try {
await symlink(moved, directory);
expect((await artifactGet(token)).status).toBe(404);
} finally { await rm(directory, { force: true }); await rename(moved, directory); }
const file = join(directory, artifact.sha256 + ".json"), saved = file + ".saved";
await rename(file, saved);
try {
await symlink(saved, file);
expect((await artifactGet(token)).status).toBe(404);
await rm(file);
await link(saved, file);
expect((await artifactGet(token)).status).toBe(404);
await rm(file);
} finally { await rm(file, { force: true }); await rename(saved, file); }
const cost = await db.orchestrationBudgetLedger.aggregate({
where: { taskId: t.id },
_sum: { amountCents: true },
});
expect(cost._sum.amountCents).toBe(0);
expect(
await db.orchestrationAttestation.findFirst({
where: { run: { taskId: t.id }, kind: "STAGE" },
}),
).toBeNull();
const evidence = {
taskId: t.id,
state: row.state,
sha256: artifact.sha256,
artifact: JSON.parse(bytes.toString()),
httpGet: get.status,
costCents: cost._sum.amountCents,
uriExternalReachability: "NOT_TESTED_ISOLATED_FIXTURE_HOST",
};
console.log("TRACER_EVIDENCE", JSON.stringify(evidence));
if (process.env.TRACER_EVIDENCE_PATH)
await writeFile(
process.env.TRACER_EVIDENCE_PATH,
JSON.stringify(evidence, null, 2),
);
await writeFile(
join(directory, artifact.sha256 + ".json"),
'{"tampered":true}',
);
expect(
(
await fetch(
base + "/internal/prepare-worker/artifacts/" + artifact.sha256,
{ headers: { authorization: "Bearer " + token } },
)
).status,
).toBe(409);
});
},
);
@@ -0,0 +1,43 @@
# Bounded Stage PREPARE workflow foundation
This slice is NOT a runtime executor, deployment mechanism, or full v4 completion. It enables authenticated proposal/Gate1/queue APIs and provides internal worker/CI service methods. Default worker/CI identity allowlists are empty: no browser or production execution route exists.
## Panel HTTP contract
All routes require existing JWT authentication with `Admin` or `SUPER_ADMIN`. Actor is exclusively `req.user.id ?? req.user.sub`, role exclusively `req.user.role`. Top-level unknown mutation fields return 400; client actor/profile fields are never trusted.
Base `/admin/orchestration`:
- `GET /tasks`: task array.
- `GET /tasks/:id`: task object.
- `GET /overview`: existing inventory items plus capabilities `{mutations:true, prepare:true, release:false, gate2Decisions:false, chat:false, runtime:false}`, reason `bounded_stage_prepare_only`.
- `POST /tasks`: existing CreateOrchestrationTaskDto `{key,title,department,operationMode:"PREPARE",budgetLimitCents,metadata:{commitSha,scope,acceptanceCriteria}}`; commitSha is required lowercase 40hex; scope is a non-blank string of at most 2000 characters; acceptanceCriteria is 1..20 non-blank strings of at most 500 characters each; budget is integer 20..10000 cents. Returns 201 projected task, state DRAFT/version1. Validated scope and acceptance criteria are persisted exactly and remain unchanged by workflow transitions. The internal `workflowScope:"STAGE_PREPARE"`, environment and createdBy are server-owned; input profile/actor/other arbitrary keys are discarded. Key max120, title max255, department max50. Proposal key is unique; sequential or concurrent duplicate keys return 409, without updating the original task or appending another proposal event. Fetch/reconcile the existing task rather than retry blindly.
- `PUT /tasks/:id/transition`: `{toState:"GATE1_PENDING"|"READY",expectedVersion,idempotencyKey,reason?}`. Only DRAFT->GATE1_PENDING and GATE1_APPROVED->READY. Owner or SUPER_ADMIN. Returns 200 projected task. READY atomically creates dispatch outbox. Requires unexpired Gate1.
- `POST /tasks/:id/gates/1/decide`: existing GateDecisionDto `{decision:"APPROVED"|"REJECTED",expectedVersion,idempotencyKey,expiresAt,notes?}`. SUPER_ADMIN independent from proposer. Future expiry <=24h. Returns 201 **projected task**, not decision object.
- Gate2 decision, budget writes/reconcile, command routes (including release), chat and kill-switch HTTP mutations remain 403. Runtime resource reads remain 503.
Task projection keeps existing scalar fields and safe nested runs/artifacts, decisions, events and ledger. Adds only `metadata:{scope,acceptanceCriteria,environment:"staging",commitSha}` for bounded proposals, otherwise `{}`. Never includes createdBy, workflowScope, payloads, URI, signatures, tokens or lease credentials. Pre-fix rows with the legacy scope sentinel remain readable/operable but have no recovered acceptance criteria: do not represent those as complete immutable scope review evidence.
`availableActions:string[]` is state/identity-derived: DRAFT owner -> `SUBMIT_GATE1`; GATE1_PENDING independent SUPER_ADMIN -> `APPROVE_GATE1`,`REJECT_GATE1`; GATE1_APPROVED owner/SUPER_ADMIN -> `QUEUE_PREPARE`; all worker/verification/Gate2 and legacy states -> `[]`. Actions indicate workflow eligibility, not a guarantee: current kill switch, approval expiry and concurrent versions are rechecked transactionally on mutation. Panel must not infer actions from state or send actor identity.
Common errors: 400 invalid DTO/binding; 403 wrong scope/identity/independence/kill/budget/expiry; 409 duplicate proposal key, stale version or idempotency payload mismatch. Transition/decision replay returns current persisted task and does not duplicate side effects.
## Internal service contracts (NOT browser endpoints)
`PrepareWorkflowService` is exported by OrchestrationModule. Trusted backend composition must override `PREPARE_WORKFLOW_CONFIG` with explicit `workerIds` and `stageCiIds`, default both empty. Do not populate from request bodies or user-selected runtime profiles. Worker identities require role WORKFLOW_WORKER AND ID allowlist; Stage identities require STAGE_CI AND separate ID allowlist. Credential/session provisioning and transport remain the integrating runtime owner's responsibility.
- `claim(identity, taskId?) -> null | {taskId,runId,attempt,leaseToken,leaseUntil,commitSha,environment:"staging",maxCostCents:20}`. Transactional task row lock with SKIP LOCKED, immutable run binding, 30s lease, max3 attempts. Each attempt reserves **a fixed hard upper bound of 20 cents**, not an estimated or measured departmental cost. Unknown cost on expired attempts remains reserved conservatively.
- `heartbeat(identity,{taskId,runId,leaseToken}) -> {leaseUntil}`. Same owner/token, unexpired active lease required.
- `complete(identity,{taskId,runId,leaseToken,sha256,uri,actualCents}) -> task`. QA success ingestion only; lowercase sha256, HTTPS URI, actual cost integer0..20 from trusted worker. Atomically records immutable QA artifact/attestation, SUCCEEDED run, reservation reconciliation, DONE outbox and VERIFYING task. Expired/stale/foreign token cannot finish. Not replayable after success; caller must read persisted task/run after response loss.
- `stageEvidence(identity,{taskId,runId,sha256,commitSha,evidenceSha256,uri}) -> task`. Independently allowlisted Stage CI identity, successful exact staging commit/run, registered QA subject, independent QA signer. Atomically records Stage artifact/attestation and reaches GATE2_PENDING. No release or Gate2 approval. Does not perform or claim deployment itself. Caller supplies actual trusted CI evidence, not browser assertions.
## Verification and explicit gaps
Real isolated PostgreSQL16, no published ports/no live volumes. Added migrations applied with ON_ERROR_STOP. TDD RED logs cover missing workflow behavior, heartbeat/QA behavior and HTTP enablement. Full backend regression excluding existing legacy release-containing orchestration.integration.spec: 106 suites /851 tests including seven real-DB workflow cases, build exit0. No RELEASE_AUTHORIZE service operation executed.
DB regressions: concurrent claim winner, queue race/replay mismatch, hard budget rollback, stale lease fencing, max3 attempts, conservative expired costs, kill switch, self approval, trusted IDs, QA reconciliation and Stage evidence to Gate2 pending. Test fixture artifact URIs/hashes are not real deployment evidence.
Remaining: actual authenticated runtime worker transport/executor and pinned CI evidence transport; shared department/company budget model and locking (NOT implemented or represented by fake totals); retry backoff/dead-letter UX/cancellation; hard total execution wall-time; durable request replay of completion/evidence; kill-switch concurrent insertion/version locking; migration rehearsal with full legacy schema; real Stage CI artifact/deployment verification. No push/deploy/live DB/runtime edits. Production release remains denied at HTTP boundary; unsafe legacy core release method is deliberately untouched and must never be called by this slice.
## Contract repair verification
Actual frontend inspection confirmed proposal submission sent human scope and acceptanceCriteria but omitted required commitSha. Frontend repair is separately assigned; no frontend file changed here. New proposals now require all three fields. Internal Stage restriction uses a separate server-owned workflowScope marker rather than overwriting user scope. No metadata update route exists in this bounded workflow.
TDD RED→GREEN against isolated PostgreSQL16 reproduced dropped scope/criteria, nine invalid metadata acceptances (including array-to-string commit coercion), and duplicate-key P2002 leakage. Final database suite: 24 cases, including authenticated HTTP create/list/detail persistence, 400 validation, and 409 duplicate responses. Full allowed backend regression: 106 suites / 868 tests passed; legacy orchestration.integration.spec.ts excluded to avoid executing RELEASE_AUTHORIZE. Build exit0. Scoped lint remains failing on inherited require-await/no-unnecessary-type-assertion errors (13 errors); no lint suppression or config change applied. Disposable DB only; no live edits/push/deploy.
@@ -92,7 +92,7 @@ describe('integration read boundary', () => {
[method]('/admin/orchestration/' + path)
.set('Authorization', 'Admin')
.send({})
.expect(403);
.expect(path === 'tasks' || path.endsWith('/transition') ? 400 : 403);
}
for (const name of [
'createTask',
@@ -1,5 +1,9 @@
import {
Controller,
Body,
Req,
Optional,
ValidationPipe,
Get,
Param,
Post,
@@ -13,6 +17,20 @@ import { JwtAuthGuard } from '../auth/jwt-auth.guard';
import { RolesGuard } from '../common/guards/roles.guard';
import { Roles } from '../common/decorators/roles.decorator';
import { OrchestrationService } from './orchestration.service';
import {
PrepareWorkflowService,
WorkflowIdentity,
} from './prepare-workflow.service';
import {
CreateOrchestrationTaskDto,
TransitionTaskDto,
GateDecisionDto,
} from './dto/orchestration.dto';
const workflowValidation = new ValidationPipe({
transform: true,
whitelist: true,
forbidNonWhitelisted: true,
});
// Explicit response allowlists: never serialize arbitrary JSON, signatures, URIs or BigInt.
function pick(row: any, keys: string[]) {
@@ -73,12 +91,37 @@ function project(row: any) {
@Roles('Admin', 'SUPER_ADMIN')
@Controller('admin/orchestration')
export class OrchestrationController {
constructor(private readonly service: OrchestrationService) {}
@Get('tasks') async all() {
return (await this.service.findAll()).map(project);
constructor(
private readonly service: OrchestrationService,
@Optional() private readonly workflow?: PrepareWorkflowService,
) {}
private actor(req: any): WorkflowIdentity {
const id = req?.user?.id ?? req?.user?.sub;
if (
typeof id !== 'string' ||
!id ||
!['Admin', 'SUPER_ADMIN'].includes(req?.user?.role)
)
throw new ForbiddenException('Authenticated workflow identity required');
return { id, role: req.user.role };
}
@Get('tasks/:id') async one(@Param('id') id: string) {
return project(await this.service.findOne(id));
private safe(row: any, req: any) {
const view = project(row);
if (!this.workflow) return view;
const w = this.workflow.project(row, this.actor(req));
return {
...view,
metadata: w.metadata,
availableActions: w.availableActions,
};
}
@Get('tasks') async all(@Req() req: any) {
return (await this.service.findAll()).map((row: any) =>
this.safe(row, req),
);
}
@Get('tasks/:id') async one(@Param('id') id: string, @Req() req: any) {
return this.safe(await this.service.findOne(id), req);
}
@Get('overview') async overview() {
const tasks = await this.service.findAll();
@@ -87,8 +130,17 @@ export class OrchestrationController {
{ id: 'tasks', title: 'Persisted task inventory', count: tasks.length },
],
measuredAt: new Date().toISOString(),
capabilities: { mutations: false, chat: false, runtime: false },
reason: 'security_hardening_pending',
capabilities: {
mutations: !!this.workflow,
prepare: !!this.workflow,
release: false,
gate2Decisions: false,
chat: false,
runtime: false,
},
reason: this.workflow
? 'bounded_stage_prepare_only'
: 'security_hardening_pending',
};
}
@Get('releases') async releases() {
@@ -120,15 +172,40 @@ export class OrchestrationController {
'No persisted model policy read contract available',
);
}
// The committed legacy core is not safe to expose. No env switch or role may bypass this boundary.
@Post('tasks') create() {
return this.deny();
// Only the separate bounded PREPARE service is exposed; legacy commands remain denied.
@Post('tasks') async create(
@Body(workflowValidation) dto: CreateOrchestrationTaskDto,
@Req() req: any,
) {
if (!this.workflow) return this.deny();
return this.safe(await this.workflow.create(dto, this.actor(req)), req);
}
@Put('tasks/:id/transition') transition() {
return this.deny();
@Put('tasks/:id/transition') async transition(
@Param('id') id: string,
@Body(workflowValidation) dto: TransitionTaskDto,
@Req() req: any,
) {
if (!this.workflow) return this.deny();
return this.safe(
await this.workflow.transition(id, dto, this.actor(req)),
req,
);
}
@Post('tasks/:id/gates/:gate/decide') decide() {
return this.deny();
@Post('tasks/:id/gates/:gate/decide') async decide(
@Param('id') id: string,
@Param('gate') gate: string,
@Body() body: unknown,
@Req() req: any,
) {
if (!this.workflow || gate !== '1') return this.deny();
const dto = await workflowValidation.transform(body, {
type: 'body',
metatype: GateDecisionDto,
});
return this.safe(
await this.workflow.decide(id, 1, dto, this.actor(req)),
req,
);
}
@Post('tasks/:id/budget') budget() {
return this.deny();
@@ -1,3 +1,7 @@
import {
PrepareWorkflowService,
PREPARE_WORKFLOW_CONFIG,
} from './prepare-workflow.service';
import { Module } from '@nestjs/common';
import { PrismaModule } from '../prisma/prisma.module';
import { KillSwitchService } from './kill-switch.service';
@@ -7,7 +11,21 @@ import { OrchestrationService } from './orchestration.service';
@Module({
imports: [PrismaModule],
controllers: [OrchestrationController],
providers: [OrchestrationService, KillSwitchService, ModelTierPolicyService],
exports: [OrchestrationService, KillSwitchService, ModelTierPolicyService],
providers: [
OrchestrationService,
KillSwitchService,
ModelTierPolicyService,
PrepareWorkflowService,
{
provide: PREPARE_WORKFLOW_CONFIG,
useValue: { workerIds: [], stageCiIds: [] },
},
],
exports: [
OrchestrationService,
KillSwitchService,
ModelTierPolicyService,
PrepareWorkflowService,
],
})
export class OrchestrationModule {}
@@ -0,0 +1,137 @@
import { Test } from '@nestjs/testing';
import { INestApplication } from '@nestjs/common';
import request from 'supertest';
import { OrchestrationController } from './orchestration.controller';
import { OrchestrationService } from './orchestration.service';
import { PrepareWorkflowService } from './prepare-workflow.service';
import { JwtAuthGuard } from '../auth/jwt-auth.guard';
import { RolesGuard } from '../common/guards/roles.guard';
describe('bounded workflow HTTP contracts', () => {
let app: INestApplication;
const t = {
id: 't',
key: 'k',
state: 'DRAFT',
version: 1,
metadata: {
scope: 'STAGE_PREPARE',
environment: 'staging',
commitSha: 'a'.repeat(40),
token: 'secret',
},
};
const workflow = {
create: jest.fn(async () => t),
transition: jest.fn(async () => t),
decide: jest.fn(async () => t),
project: jest.fn((row: any) => ({
...row,
metadata: {
scope: 'STAGE_PREPARE',
environment: 'staging',
commitSha: 'a'.repeat(40),
},
availableActions: ['SUBMIT_GATE1'],
})),
};
const core = {
findAll: async () => [t],
findOne: async () => t,
command: jest.fn(),
};
beforeAll(async () => {
const m = await Test.createTestingModule({
controllers: [OrchestrationController],
providers: [
{ provide: OrchestrationService, useValue: core },
{ provide: PrepareWorkflowService, useValue: workflow },
RolesGuard,
],
})
.overrideGuard(JwtAuthGuard)
.useValue({
canActivate(c: any) {
const req = c.switchToHttp().getRequest();
if (!req.headers.authorization) return false;
req.user = {
sub: 'verified-subject',
role: req.headers.authorization,
};
return true;
},
})
.compile();
app = m.createNestApplication();
await app.init();
});
afterAll(async () => {
await app.close();
});
it('accepts existing proposal DTO, derives actor from JWT sub and returns projected task', async () => {
const dto = {
key: 'k',
title: 'T',
department: 'ENGINEERING',
operationMode: 'PREPARE',
budgetLimitCents: 20,
metadata: {
commitSha: 'a'.repeat(40),
scope: 'Staging audit only',
acceptanceCriteria: ['QA passes'],
},
};
const r = await request(app.getHttpServer())
.post('/admin/orchestration/tasks')
.set('Authorization', 'Admin')
.send(dto)
.expect(201);
expect(workflow.create).toHaveBeenCalledWith(dto, {
id: 'verified-subject',
role: 'Admin',
});
expect(r.body.availableActions).toEqual(['SUBMIT_GATE1']);
expect(JSON.stringify(r.body)).not.toMatch(/token|secret/);
await request(app.getHttpServer())
.post('/admin/orchestration/tasks')
.set('Authorization', 'Admin')
.send({ ...dto, actorId: 'browser' })
.expect(400);
});
it('routes fixed transition/gate contracts and rejects invalid gates plus release without invoking core', async () => {
await request(app.getHttpServer())
.put('/admin/orchestration/tasks/t/transition')
.set('Authorization', 'Admin')
.send({
toState: 'GATE1_PENDING',
expectedVersion: 1,
idempotencyKey: 'submit',
})
.expect(200);
await request(app.getHttpServer())
.post('/admin/orchestration/tasks/t/gates/1/decide')
.set('Authorization', 'SUPER_ADMIN')
.send({
decision: 'APPROVED',
expectedVersion: 2,
idempotencyKey: 'approval',
expiresAt: new Date(Date.now() + 60000).toISOString(),
})
.expect(201);
expect(workflow.decide).toHaveBeenCalledWith('t', 1, expect.any(Object), {
id: 'verified-subject',
role: 'SUPER_ADMIN',
});
await request(app.getHttpServer())
.post('/admin/orchestration/tasks/t/gates/2/decide')
.set('Authorization', 'SUPER_ADMIN')
.send({})
.expect(403);
await request(app.getHttpServer())
.post('/admin/orchestration/tasks/t/commands/RELEASE_AUTHORIZE')
.set('Authorization', 'SUPER_ADMIN')
.send({})
.expect(403);
expect(core.command).not.toHaveBeenCalled();
});
});
@@ -0,0 +1,559 @@
import {
Injectable,
Inject,
ForbiddenException,
BadRequestException,
ConflictException,
NotFoundException,
} from '@nestjs/common';
import { randomUUID } from 'crypto';
import { PrismaService } from '../prisma/prisma.service';
export const PREPARE_WORKFLOW_CONFIG = 'PREPARE_WORKFLOW_CONFIG';
export interface WorkflowIdentity {
id: string;
role: string;
}
export interface WorkflowConfig {
workerIds: string[];
stageCiIds: string[];
}
/** Bounded Stage preparation only. No release executor, production scope or browser worker identities. */
@Injectable()
export class PrepareWorkflowService {
constructor(
@Inject(PrismaService) private readonly db: any,
@Inject(PREPARE_WORKFLOW_CONFIG) private readonly config: WorkflowConfig,
) {}
private identity(a: WorkflowIdentity, roles: string[]) {
if (
!a?.id ||
typeof a.id !== 'string' ||
a.id.length > 100 ||
!roles.includes(a.role)
)
throw new ForbiddenException('Authenticated workflow identity required');
}
private worker(a: WorkflowIdentity) {
this.identity(a, ['WORKFLOW_WORKER']);
if (!this.config.workerIds.includes(a.id))
throw new ForbiddenException('Worker identity not provisioned');
}
private async allowed(tx: any, claiming = false) {
const rows =
await tx.$queryRaw`SELECT level, active FROM orchestration_kill_switches WHERE scope = 'GLOBAL' FOR SHARE`;
if (rows[0]?.active && !(rows[0].level === 'FREEZE_NEW_WORK' && !claiming))
throw new ForbiddenException('Kill switch blocks workflow');
}
private async task(tx: any, id: string) {
await tx.$queryRaw`SELECT id FROM orchestration_tasks WHERE id = ${id}::uuid FOR UPDATE`;
const t = await tx.orchestrationTask.findUnique({ where: { id } });
if (!t) throw new NotFoundException('Task not found');
if (
t.operationMode !== 'PREPARE' ||
(t.metadata?.workflowScope ?? t.metadata?.scope) !== 'STAGE_PREPARE'
)
throw new ForbiddenException('Not a bounded Stage proposal');
return t;
}
private async event(
tx: any,
t: any,
state: string,
a: WorkflowIdentity,
key: string,
type: string,
fingerprint: string,
) {
const result = await tx.orchestrationTask.update({
where: { id: t.id },
data: { state, version: { increment: 1 } },
});
await tx.orchestrationEvent.create({
data: {
taskId: t.id,
sequence: result.version,
actorId: a.id,
eventType: type,
fromState: t.state,
toState: state,
idempotencyKey: key,
payload: { fingerprint },
},
});
return result;
}
private async replay(tx: any, key: string, fingerprint: string) {
const e = await tx.orchestrationEvent.findUnique({
where: { idempotencyKey: key },
});
if (e && e.payload?.fingerprint !== fingerprint)
throw new ConflictException('Idempotency payload mismatch');
return e;
}
private version(t: any, dto: any) {
if (
!Number.isSafeInteger(dto.expectedVersion) ||
dto.expectedVersion !== t.version
)
throw new ConflictException('Stale task version');
if (
typeof dto.idempotencyKey !== 'string' ||
!dto.idempotencyKey.length ||
dto.idempotencyKey.length > 120
)
throw new BadRequestException('Invalid idempotency key');
}
async create(dto: any, a: WorkflowIdentity) {
this.identity(a, ['Admin', 'SUPER_ADMIN']);
const metadata = dto?.metadata;
const boundedText = (value: unknown, max: number) =>
typeof value === 'string' &&
value.trim().length > 0 &&
value.length <= max;
if (
!metadata ||
typeof metadata !== 'object' ||
Array.isArray(metadata) ||
typeof metadata.commitSha !== 'string' ||
!boundedText(metadata.scope, 2000) ||
!Array.isArray(metadata.acceptanceCriteria) ||
metadata.acceptanceCriteria.length < 1 ||
metadata.acceptanceCriteria.length > 20 ||
!metadata.acceptanceCriteria.every((criterion: unknown) =>
boundedText(criterion, 500),
)
)
throw new BadRequestException(
'PREPARE requires commitSha, scope 1..2000 characters and 1..20 acceptance criteria of 1..500 characters',
);
if (
dto.operationMode !== 'PREPARE' ||
!/^[a-f0-9]{40}$/.test(dto.metadata?.commitSha ?? '') ||
!Number.isSafeInteger(dto.budgetLimitCents) ||
dto.budgetLimitCents < 20 ||
dto.budgetLimitCents > 10000
)
throw new BadRequestException(
'PREPARE requires immutable 40hex commit and budget 20..10000 cents',
);
if (
!dto.key ||
dto.key.length > 120 ||
!dto.title ||
dto.title.length > 255 ||
!dto.department ||
dto.department.length > 50
)
throw new BadRequestException('Invalid proposal fields');
return this.db
.$transaction(async (tx: any) => {
await this.allowed(tx, true);
const t = await tx.orchestrationTask.create({
data: {
key: dto.key,
title: dto.title,
department: dto.department,
operationMode: 'PREPARE',
budgetLimitCents: dto.budgetLimitCents,
metadata: {
workflowScope: 'STAGE_PREPARE',
scope: dto.metadata.scope,
acceptanceCriteria: [...dto.metadata.acceptanceCriteria],
environment: 'staging',
commitSha: dto.metadata.commitSha,
createdBy: a.id,
},
state: 'DRAFT',
},
});
await tx.orchestrationEvent.create({
data: {
taskId: t.id,
sequence: 1,
eventType: 'PREPARE_PROPOSED',
actorId: a.id,
toState: 'DRAFT',
},
});
return t;
})
.catch((error: any) => {
if (
error?.code === 'P2002' &&
Array.isArray(error.meta?.target) &&
error.meta.target.includes('key')
)
throw new ConflictException(
'Proposal key already exists; fetch and reconcile the existing task',
);
throw error;
});
}
async transition(id: string, dto: any, a: WorkflowIdentity) {
this.identity(a, ['Admin', 'SUPER_ADMIN']);
return this.db.$transaction(async (tx: any) => {
await this.allowed(tx, true);
const t = await this.task(tx, id);
const fingerprint = JSON.stringify({ id, dto, a });
if (await this.replay(tx, dto.idempotencyKey, fingerprint)) return t;
this.version(t, dto);
if (t.metadata.createdBy !== a.id && a.role !== 'SUPER_ADMIN')
throw new ForbiddenException('Proposal owner required');
if (
!(
(t.state === 'DRAFT' && dto.toState === 'GATE1_PENDING') ||
(t.state === 'GATE1_APPROVED' && dto.toState === 'READY')
)
)
throw new ForbiddenException(
'Only submission and approved queueing are available',
);
if (dto.toState === 'READY') {
const approval = await tx.orchestrationApprovalDecision.findFirst({
where: { taskId: id, gate: 1, decision: 'APPROVED' },
orderBy: { createdAt: 'desc' },
});
if (!approval || approval.expiresAt <= new Date())
throw new ForbiddenException('Gate1 expired');
await tx.prepareWorkflowOutbox.create({ data: { taskId: id } });
}
return this.event(
tx,
t,
dto.toState,
a,
dto.idempotencyKey,
'PREPARE_TRANSITION',
fingerprint,
);
});
}
async decide(id: string, gate: number, dto: any, a: WorkflowIdentity) {
this.identity(a, ['SUPER_ADMIN']);
if (gate !== 1)
throw new ForbiddenException(
'Gate2 decisions and release remain disabled',
);
if (
!['APPROVED', 'REJECTED'].includes(dto.decision) ||
!Number.isFinite(Date.parse(dto.expiresAt)) ||
Date.parse(dto.expiresAt) <= Date.now() ||
Date.parse(dto.expiresAt) > Date.now() + 86400000
)
throw new BadRequestException(
'Decision requires future expiry within 24 hours',
);
return this.db.$transaction(async (tx: any) => {
await this.allowed(tx);
const t = await this.task(tx, id);
const fingerprint = JSON.stringify({ id, gate, dto, a });
if (await this.replay(tx, dto.idempotencyKey, fingerprint)) return t;
this.version(t, dto);
if (t.metadata.createdBy === a.id)
throw new ForbiddenException('Independent reviewer required');
if (t.state !== 'GATE1_PENDING')
throw new ConflictException('Gate1 pending required');
await tx.orchestrationApprovalDecision.create({
data: {
taskId: id,
gate: 1,
decision: dto.decision,
actorId: a.id,
expiresAt: new Date(dto.expiresAt),
idempotencyKey: `decision:${dto.idempotencyKey}`,
},
});
return this.event(
tx,
t,
dto.decision === 'APPROVED' ? 'GATE1_APPROVED' : 'REJECTED',
a,
dto.idempotencyKey,
'PREPARE_GATE1',
fingerprint,
);
});
}
async claim(a: WorkflowIdentity, id?: string): Promise<any> {
this.worker(a);
return this.db.$transaction(async (tx: any) => {
await this.allowed(tx, true);
// Lock order is task then outbox for every worker operation. SKIP LOCKED permits competing workers.
const rows: any[] = id
? await tx.$queryRaw`SELECT t.id FROM orchestration_tasks t JOIN prepare_workflow_outbox o ON o.task_id=t.id WHERE t.id=${id}::uuid AND o.status='PENDING' AND (o.lease_until IS NULL OR o.lease_until < now()) ORDER BY o.created_at FOR UPDATE OF t SKIP LOCKED LIMIT 1`
: await tx.$queryRaw`SELECT t.id FROM orchestration_tasks t JOIN prepare_workflow_outbox o ON o.task_id=t.id WHERE o.status='PENDING' AND (o.lease_until IS NULL OR o.lease_until < now()) ORDER BY o.created_at FOR UPDATE OF t SKIP LOCKED LIMIT 1`;
if (!rows.length) return null;
const t = await this.task(tx, rows[0].id);
const o = await tx.prepareWorkflowOutbox.findUnique({
where: { taskId: t.id },
});
const approval = await tx.orchestrationApprovalDecision.findFirst({
where: { taskId: t.id, gate: 1, decision: 'APPROVED' },
orderBy: { createdAt: 'desc' },
});
if (!approval || approval.expiresAt <= new Date())
throw new ForbiddenException('Gate1 expired');
if (o.attempts >= 3) throw new ForbiddenException('Attempts exhausted');
const costs = await tx.orchestrationBudgetLedger.aggregate({
where: { taskId: t.id },
_sum: { amountCents: true },
});
if ((costs._sum.amountCents ?? 0) + 20 > t.budgetLimitCents)
throw new ForbiddenException('Hard task budget exceeded');
if (o.runId)
await tx.orchestrationRun.update({
where: { id: o.runId },
data: {
status: 'FAILED',
retryClass: 'TRANSIENT',
finishedAt: new Date(),
},
});
const token = randomUUID(),
attempt = o.attempts + 1;
const run = await tx.orchestrationRun.create({
data: {
taskId: t.id,
attempt,
status: 'RUNNING',
environment: 'staging',
commitSha: t.metadata.commitSha,
idempotencyKey: `prepare-run:${o.id}:${attempt}`,
startedAt: new Date(),
},
});
const reservation = await tx.orchestrationBudgetLedger.create({
data: {
taskId: t.id,
amountCents: 20,
kind: 'RESERVE',
actorId: a.id,
idempotencyKey: `prepare-reserve:${run.id}`,
},
});
const leaseUntil = new Date(Date.now() + 30000);
await tx.prepareWorkflowOutbox.update({
where: { id: o.id },
data: {
attempts: attempt,
leaseOwner: a.id,
leaseToken: token,
leaseUntil,
heartbeatAt: new Date(),
runId: run.id,
reservationId: reservation.id,
},
});
await this.event(
tx,
t,
'RUNNING',
a,
`prepare-claim:${run.id}`,
'PREPARE_CLAIMED',
token,
);
return {
taskId: t.id,
runId: run.id,
attempt,
leaseToken: token,
leaseUntil,
commitSha: run.commitSha,
environment: 'staging',
maxCostCents: 20,
};
});
}
private async lease(tx: any, a: WorkflowIdentity, dto: any) {
const t = await this.task(tx, dto.taskId);
const o = await tx.prepareWorkflowOutbox.findUnique({
where: { taskId: t.id },
});
if (
!o ||
o.status !== 'PENDING' ||
o.leaseOwner !== a.id ||
o.leaseToken !== dto.leaseToken ||
o.runId !== dto.runId ||
!o.leaseUntil ||
o.leaseUntil <= new Date() ||
t.state !== 'RUNNING'
)
throw new ForbiddenException('Active owned lease required');
return { t, o };
}
async heartbeat(a: WorkflowIdentity, dto: any) {
this.worker(a);
return this.db.$transaction(async (tx: any) => {
await this.allowed(tx);
const { o } = await this.lease(tx, a, dto);
return tx.prepareWorkflowOutbox.update({
where: { id: o.id },
data: {
heartbeatAt: new Date(),
leaseUntil: new Date(Date.now() + 30000),
},
select: { leaseUntil: true },
});
});
}
async complete(a: WorkflowIdentity, dto: any) {
this.worker(a);
if (
!/^[a-f0-9]{64}$/.test(dto.sha256 ?? '') ||
!/^https:\/\//.test(dto.uri ?? '') ||
!Number.isSafeInteger(dto.actualCents) ||
dto.actualCents < 0 ||
dto.actualCents > 20
)
throw new BadRequestException(
'QA hash, HTTPS URI and cost 0..20 required',
);
return this.db.$transaction(async (tx: any) => {
await this.allowed(tx);
const { t, o } = await this.lease(tx, a, dto);
await tx.orchestrationArtifact.create({
data: { runId: o.runId, kind: 'QA', sha256: dto.sha256, uri: dto.uri },
});
await tx.orchestrationAttestation.create({
data: {
runId: o.runId,
kind: 'QA',
subjectHash: dto.sha256,
signer: a.id,
predicate: { passed: true, scope: 'STAGE_PREPARE' },
},
});
await tx.orchestrationRun.update({
where: { id: o.runId },
data: { status: 'SUCCEEDED', finishedAt: new Date() },
});
await tx.orchestrationBudgetLedger.create({
data: {
taskId: t.id,
kind: 'RECONCILE',
amountCents: dto.actualCents - 20,
actorId: a.id,
idempotencyKey: `prepare-cost:${o.runId}`,
metadata: {
reservationId: o.reservationId,
actualCents: dto.actualCents,
},
},
});
await tx.prepareWorkflowOutbox.update({
where: { id: o.id },
data: { status: 'DONE', leaseToken: null, leaseUntil: null },
});
return this.event(
tx,
t,
'VERIFYING',
a,
`prepare-qa:${o.runId}`,
'PREPARE_QA_SUCCEEDED',
dto.sha256,
);
});
}
async stageEvidence(a: WorkflowIdentity, dto: any) {
this.identity(a, ['STAGE_CI']);
if (!this.config.stageCiIds.includes(a.id))
throw new ForbiddenException('Stage CI identity not provisioned');
if (
!/^[a-f0-9]{64}$/.test(dto.evidenceSha256 ?? '') ||
!/^https:\/\//.test(dto.uri ?? '')
)
throw new BadRequestException(
'Stage evidence hash and HTTPS URI required',
);
return this.db.$transaction(async (tx: any) => {
await this.allowed(tx);
const t = await this.task(tx, dto.taskId);
const run = await tx.orchestrationRun.findUnique({
where: { id: dto.runId },
});
if (
t.state !== 'VERIFYING' ||
!run ||
run.taskId !== t.id ||
run.environment !== 'staging' ||
run.status !== 'SUCCEEDED' ||
run.commitSha !== dto.commitSha ||
t.metadata.commitSha !== dto.commitSha
)
throw new ForbiddenException(
'Immutable successful Stage run binding required',
);
const qa = await tx.orchestrationAttestation.findFirst({
where: { runId: run.id, kind: 'QA', subjectHash: dto.sha256 },
});
if (!qa || qa.signer === a.id)
throw new ForbiddenException('Independent QA evidence required');
await tx.orchestrationArtifact.create({
data: {
runId: run.id,
kind: 'STAGE_EVIDENCE',
sha256: dto.evidenceSha256,
uri: dto.uri,
},
});
await tx.orchestrationAttestation.create({
data: {
runId: run.id,
kind: 'STAGE',
subjectHash: dto.sha256,
signer: a.id,
predicate: {
evidenceSha256: dto.evidenceSha256,
commitSha: dto.commitSha,
environment: 'staging',
},
},
});
return this.event(
tx,
t,
'GATE2_PENDING',
a,
`prepare-stage:${run.id}`,
'STAGE_EVIDENCE_RECORDED',
dto.evidenceSha256,
);
});
}
project(t: any, a: WorkflowIdentity) {
const safe =
t.operationMode === 'PREPARE' &&
(t.metadata?.workflowScope ?? t.metadata?.scope) === 'STAGE_PREPARE';
const owner =
a?.id && (t.metadata?.createdBy === a.id || a.role === 'SUPER_ADMIN');
const actions: string[] = [];
if (safe && owner && t.state === 'DRAFT') actions.push('SUBMIT_GATE1');
if (safe && owner && t.state === 'GATE1_APPROVED')
actions.push('QUEUE_PREPARE');
if (
safe &&
a?.role === 'SUPER_ADMIN' &&
t.metadata?.createdBy !== a.id &&
t.state === 'GATE1_PENDING'
)
actions.push('APPROVE_GATE1', 'REJECT_GATE1');
return {
...t,
metadata: safe
? {
scope: t.metadata.scope,
...(t.metadata.workflowScope === 'STAGE_PREPARE'
? { acceptanceCriteria: t.metadata.acceptanceCriteria }
: {}),
environment: 'staging',
commitSha: t.metadata.commitSha,
}
: {},
availableActions: actions,
};
}
}
@@ -0,0 +1,513 @@
import { Test } from '@nestjs/testing';
import request from 'supertest';
import { OrchestrationController } from './orchestration.controller';
import { OrchestrationService } from './orchestration.service';
import { JwtAuthGuard } from '../auth/jwt-auth.guard';
import { RolesGuard } from '../common/guards/roles.guard';
import { PrismaClient } from '@prisma/client';
import { PrepareWorkflowService } from './prepare-workflow.service';
import { OperationMode, WorkflowState } from './orchestration.domain';
const enabled = process.env.WORKFLOW_DB === '1';
(enabled ? describe : describe.skip)('isolated prepare workflow', () => {
const db = new PrismaClient();
const admin = { id: 'proposer', role: 'Admin' };
const reviewer = { id: 'reviewer', role: 'SUPER_ADMIN' };
const worker = { id: 'worker-1', role: 'WORKFLOW_WORKER' };
const ci = { id: 'isolated-ci', role: 'STAGE_CI' };
const service = new PrepareWorkflowService(db as never, {
workerIds: ['worker-1', 'worker-2'],
stageCiIds: ['isolated-ci'],
});
let seq = 0;
const dto = () => ({
key: `wf-${Date.now()}-${seq++}`,
title: 'Bounded isolated QA tracer',
department: 'ENGINEERING',
operationMode: OperationMode.PREPARE,
budgetLimitCents: 20,
metadata: {
commitSha: 'a'.repeat(40),
token: 'must-not-persist',
scope: 'Staging schema audit only',
acceptanceCriteria: ['Report all schema failures'],
actorId: 'spoof',
},
});
async function ready() {
let t = await service.create(dto(), admin);
t = await service.transition(
t.id,
{
toState: WorkflowState.GATE1_PENDING,
expectedVersion: t.version,
idempotencyKey: `submit-${t.id}`,
},
admin,
);
t = await service.decide(
t.id,
1,
{
decision: 'APPROVED',
expectedVersion: t.version,
idempotencyKey: `approve-${t.id}`,
expiresAt: new Date(Date.now() + 60000).toISOString(),
},
reviewer,
);
return service.transition(
t.id,
{
toState: WorkflowState.READY,
expectedVersion: t.version,
idempotencyKey: `ready-${t.id}`,
},
admin,
);
}
afterAll(async () => {
await db.$disconnect();
});
it('persists and projects immutable bounded proposal scope without arbitrary metadata', async () => {
const input = dto();
const t = await service.create(input, admin);
expect(t.metadata).toMatchObject({
workflowScope: 'STAGE_PREPARE',
scope: input.metadata.scope,
acceptanceCriteria: input.metadata.acceptanceCriteria,
commitSha: input.metadata.commitSha,
});
expect(t.metadata).not.toHaveProperty('token');
expect(t.metadata).not.toHaveProperty('actorId');
const submitted = await service.transition(
t.id,
{
toState: 'GATE1_PENDING',
expectedVersion: t.version,
idempotencyKey: `immutable-${t.id}`,
},
admin,
);
expect(submitted.metadata).toEqual(t.metadata);
expect(service.project(submitted, reviewer).metadata).toEqual({
scope: input.metadata.scope,
acceptanceCriteria: input.metadata.acceptanceCriteria,
environment: 'staging',
commitSha: input.metadata.commitSha,
});
});
it.each([
null,
[],
{},
{ commitSha: 'a'.repeat(40), scope: '', acceptanceCriteria: ['ok'] },
{
commitSha: 'a'.repeat(40),
scope: ' '.repeat(10),
acceptanceCriteria: ['ok'],
},
{
commitSha: 'a'.repeat(40),
scope: 'x'.repeat(2001),
acceptanceCriteria: ['ok'],
},
{ commitSha: 'a'.repeat(40), scope: 'ok', acceptanceCriteria: [] },
{
commitSha: 'a'.repeat(40),
scope: 'ok',
acceptanceCriteria: Array(21).fill('ok'),
},
{
commitSha: 'a'.repeat(40),
scope: 'ok',
acceptanceCriteria: ['x'.repeat(501)],
},
{ commitSha: 'a'.repeat(40), scope: 'ok', acceptanceCriteria: [' '] },
{ commitSha: 'a'.repeat(40), scope: 'ok', acceptanceCriteria: [123] },
{ commitSha: ['a'.repeat(40)], scope: 'ok', acceptanceCriteria: ['ok'] },
{ scope: 'ok', acceptanceCriteria: ['ok'] },
])(
'rejects unsafe or unbounded metadata %# without persisting',
async (metadata) => {
const input = { ...dto(), metadata };
await expect(service.create(input, admin)).rejects.toMatchObject({
status: 400,
});
expect(
await db.orchestrationTask.count({ where: { key: input.key } }),
).toBe(0);
},
);
it('accepts exact metadata bounds', async () => {
const metadata = {
commitSha: 'a'.repeat(40),
scope: 'x'.repeat(2000),
acceptanceCriteria: Array(20).fill('x'.repeat(500)),
};
expect(
(await service.create({ ...dto(), metadata }, admin)).metadata,
).toMatchObject(metadata);
});
it('normalizes racing duplicate proposal keys to conflict without duplicate events', async () => {
const input = dto();
const results = await Promise.allSettled([
service.create(input, admin),
service.create({ ...input, title: 'Different intent' }, reviewer),
]);
expect(results.filter((r) => r.status === 'fulfilled')).toHaveLength(1);
const rejected = results.find(
(r) => r.status === 'rejected',
) as PromiseRejectedResult;
expect(rejected.reason).toMatchObject({ status: 409 });
const persisted = await db.orchestrationTask.findUniqueOrThrow({
where: { key: input.key },
});
expect(
await db.orchestrationEvent.count({ where: { taskId: persisted.id } }),
).toBe(1);
await expect(service.create(input, admin)).rejects.toMatchObject({
status: 409,
});
});
it('enforces create and read contracts over HTTP with real PostgreSQL persistence', async () => {
const module = await Test.createTestingModule({
controllers: [OrchestrationController],
providers: [
{ provide: PrepareWorkflowService, useValue: service },
{
provide: OrchestrationService,
useValue: {
findAll: () => db.orchestrationTask.findMany(),
findOne: (id: string) =>
db.orchestrationTask.findUniqueOrThrow({ where: { id } }),
},
},
],
})
.overrideGuard(JwtAuthGuard)
.useValue({
canActivate(context: any) {
context.switchToHttp().getRequest().user = {
sub: admin.id,
role: admin.role,
};
return true;
},
})
.overrideGuard(RolesGuard)
.useValue({ canActivate: () => true })
.compile();
const app = module.createNestApplication();
await app.init();
try {
const input = dto();
const response = await request(app.getHttpServer())
.post('/admin/orchestration/tasks')
.send(input)
.expect(201);
expect(response.body.metadata).toEqual({
scope: input.metadata.scope,
acceptanceCriteria: input.metadata.acceptanceCriteria,
commitSha: input.metadata.commitSha,
environment: 'staging',
});
expect(response.body.availableActions).toEqual(['SUBMIT_GATE1']);
const detail = await request(app.getHttpServer())
.get(`/admin/orchestration/tasks/${response.body.id}`)
.expect(200);
expect(detail.body.metadata).toEqual(response.body.metadata);
const list = await request(app.getHttpServer())
.get('/admin/orchestration/tasks')
.expect(200);
expect(
list.body.find((row: any) => row.id === response.body.id).metadata,
).toEqual(response.body.metadata);
expect(JSON.stringify(response.body)).not.toMatch(
/createdBy|must-not-persist|actorId|workflowScope/,
);
await request(app.getHttpServer())
.post('/admin/orchestration/tasks')
.send(input)
.expect(409);
await request(app.getHttpServer())
.post('/admin/orchestration/tasks')
.send({
...input,
key: `${input.key}-invalid`,
metadata: { scope: 'only stage', acceptanceCriteria: ['QA passes'] },
})
.expect(400);
await request(app.getHttpServer())
.post('/admin/orchestration/tasks')
.send({
...input,
key: `${input.key}-invalid`,
metadata: { ...input.metadata, acceptanceCriteria: [' '] },
})
.expect(400);
} finally {
await app.close();
}
});
it('fences heartbeat and completes QA plus independent Stage evidence only to Gate2 pending', async () => {
const t = await ready();
const c = await service.claim(worker, t.id);
await expect(
service.heartbeat({ ...worker, id: 'worker-2' }, c),
).rejects.toThrow(/lease/i);
await service.heartbeat(worker, c);
const qa = {
...c,
sha256: 'b'.repeat(64),
uri: 'https://ci.example/qa',
actualCents: 8,
};
await service.complete(worker, qa);
await expect(
service.stageEvidence(
{ id: 'proposer', role: 'Admin' },
{
taskId: t.id,
runId: c.runId,
sha256: qa.sha256,
commitSha: c.commitSha,
evidenceSha256: 'c'.repeat(64),
uri: 'https://ci.example/stage',
},
),
).rejects.toThrow(/identity/i);
await service.stageEvidence(ci, {
taskId: t.id,
runId: c.runId,
sha256: qa.sha256,
commitSha: c.commitSha,
evidenceSha256: 'c'.repeat(64),
uri: 'https://ci.example/stage',
});
expect(
await db.orchestrationTask.findUnique({ where: { id: t.id } }),
).toMatchObject({ state: 'GATE2_PENDING' });
expect(
await db.orchestrationBudgetLedger.aggregate({
where: { taskId: t.id },
_sum: { amountCents: true },
}),
).toMatchObject({ _sum: { amountCents: 8 } });
await expect(service.complete(worker, qa)).rejects.toThrow(/lease|active/i);
});
it('bounds expired-lease retries and rejects stale completion without releasing unknown costs', async () => {
const t = await service.create({ ...dto(), budgetLimitCents: 60 }, admin);
let current = await service.transition(
t.id,
{
toState: 'GATE1_PENDING',
expectedVersion: 1,
idempotencyKey: `s-${t.id}`,
},
admin,
);
current = await service.decide(
t.id,
1,
{
decision: 'APPROVED',
expectedVersion: current.version,
idempotencyKey: `a-${t.id}`,
expiresAt: new Date(Date.now() + 60000).toISOString(),
},
reviewer,
);
await service.transition(
t.id,
{
toState: 'READY',
expectedVersion: current.version,
idempotencyKey: `q-${t.id}`,
},
admin,
);
const first = await service.claim(worker, t.id);
for (let i = 0; i < 2; i++) {
await db.prepareWorkflowOutbox.update({
where: { taskId: t.id },
data: { leaseUntil: new Date(0) },
});
const next = await service.claim(worker, t.id);
expect(next.attempt).toBe(i + 2);
}
await expect(
service.complete(worker, {
...first,
sha256: 'b'.repeat(64),
uri: 'https://ci.example/qa',
actualCents: 0,
}),
).rejects.toThrow(/lease/i);
await db.prepareWorkflowOutbox.update({
where: { taskId: t.id },
data: { leaseUntil: new Date(0) },
});
await expect(service.claim(worker, t.id)).rejects.toThrow(/exhausted/i);
expect(await db.orchestrationRun.count({ where: { taskId: t.id } })).toBe(
3,
);
expect(
await db.orchestrationBudgetLedger.aggregate({
where: { taskId: t.id },
_sum: { amountCents: true },
}),
).toMatchObject({ _sum: { amountCents: 60 } });
});
it('rejects self approval, non-PREPARE and unprovisioned workers', async () => {
await expect(
service.create(
{ ...dto(), operationMode: OperationMode.MUTATE_PRODUCTION },
admin,
),
).rejects.toThrow(/PREPARE/);
let t = await service.create(dto(), reviewer);
t = await service.transition(
t.id,
{
toState: 'GATE1_PENDING',
expectedVersion: 1,
idempotencyKey: `s-${t.id}`,
},
reviewer,
);
await expect(
service.decide(
t.id,
1,
{
decision: 'APPROVED',
expectedVersion: t.version,
idempotencyKey: `a-${t.id}`,
expiresAt: new Date(Date.now() + 60000).toISOString(),
},
reviewer,
),
).rejects.toThrow(/independent/i);
await expect(
service.claim({ id: 'spoof', role: 'WORKFLOW_WORKER' }),
).rejects.toThrow(/provisioned/i);
await expect(service.decide(t.id, 2, {}, reviewer)).rejects.toThrow(
/disabled/i,
);
});
it('serializes duplicate queue submission and denies claims beyond remaining task budget', async () => {
const t = await ready();
await db.orchestrationBudgetLedger.create({
data: {
taskId: t.id,
amountCents: 1,
kind: 'SPEND',
actorId: 'fixture',
idempotencyKey: `spend-${t.id}`,
},
});
await expect(service.claim(worker, t.id)).rejects.toThrow(/budget/i);
expect(await db.orchestrationRun.count({ where: { taskId: t.id } })).toBe(
0,
);
expect(
await db.prepareWorkflowOutbox.count({ where: { taskId: t.id } }),
).toBe(1);
});
it('replays submission safely and serializes racing queue requests', async () => {
let t = await service.create(dto(), admin);
const submit = {
toState: 'GATE1_PENDING',
expectedVersion: t.version,
idempotencyKey: `replay-${t.id}`,
};
t = await service.transition(t.id, submit, admin);
expect((await service.transition(t.id, submit, admin)).version).toBe(
t.version,
);
await expect(
service.transition(t.id, { ...submit, toState: 'READY' }, admin),
).rejects.toThrow(/mismatch/i);
t = await service.decide(
t.id,
1,
{
decision: 'APPROVED',
expectedVersion: t.version,
idempotencyKey: `r-approve-${t.id}`,
expiresAt: new Date(Date.now() + 60000).toISOString(),
},
reviewer,
);
const results = await Promise.allSettled(
['one', 'two'].map((k) =>
service.transition(
t.id,
{
toState: 'READY',
expectedVersion: t.version,
idempotencyKey: `${k}-${t.id}`,
},
admin,
),
),
);
expect(results.filter((r) => r.status === 'fulfilled')).toHaveLength(1);
expect(
await db.prepareWorkflowOutbox.count({ where: { taskId: t.id } }),
).toBe(1);
});
it('kill switch freezes claims without creating run or reservation', async () => {
const t = await ready();
await db.orchestrationKillSwitch.create({
data: {
scope: 'GLOBAL',
level: 'FREEZE_NEW_WORK',
reason: 'isolated test',
actorId: 'fixture',
},
});
try {
await expect(service.claim(worker, t.id)).rejects.toThrow(/kill switch/i);
} finally {
await db.orchestrationKillSwitch.delete({ where: { scope: 'GLOBAL' } });
}
expect(await db.orchestrationRun.count({ where: { taskId: t.id } })).toBe(
0,
);
});
it('persists an atomic outbox and exactly one worker wins concurrent claims', async () => {
const t = await ready();
const claims = await Promise.all([
service.claim(worker, t.id),
service.claim({ id: 'worker-2', role: 'WORKFLOW_WORKER' }, t.id),
]);
expect(claims.filter(Boolean)).toHaveLength(1);
const claim = claims.find(Boolean)!;
expect(claim.taskId).toBe(t.id);
expect(claim.attempt).toBe(1);
expect(await db.orchestrationRun.count({ where: { taskId: t.id } })).toBe(
1,
);
expect(
await db.orchestrationBudgetLedger.aggregate({
where: { taskId: t.id },
_sum: { amountCents: true },
}),
).toMatchObject({ _sum: { amountCents: 20 } });
expect(
await service.project(
await db.orchestrationTask.findUniqueOrThrow({ where: { id: t.id } }),
admin,
),
).toMatchObject({
metadata: {
scope: 'Staging schema audit only',
acceptanceCriteria: ['Report all schema failures'],
environment: 'staging',
commitSha: 'a'.repeat(40),
},
});
});
});
+95
View File
@@ -0,0 +1,95 @@
**Superseded wiring/provenance: see [snapshot repair](hermes-snapshot-provenance.md). Do not use literal grants or the old DNS alias below.**
# Runtime read vertical slice (not deployed)
Branch feature/v4-runtime-completion, base 5507fbc. No AppModule, live gateway,
production database, cron, dispatch, or existing compose was changed.
## Parent registration
Import `RuntimeReadModule.register({ '<verified-user-id>': ['default', '<department>'] })`
from `backend/src/hermes-management/runtime-read.module` into AppModule.
This module uses the existing Passport `jwt` strategy (DB-resolved user identity),
not the API-key fallback. Parent must retain AuthModule/JwtStrategy initialization.
Missing grants deny. Never construct grants from browser body, JWT profile claims,
or unreviewed environment input. Allowlist is copied/frozen. Admin/SUPER_ADMIN
role AND exact subject grant required; API-key identities forbidden.
GET `/admin/orchestration/runtime/{health,profiles,status,usage,sessions,schedules,model-info}`.
All except health/profiles require exactly one allowed `profile` query string.
Usage accepts days 1..365 (30); sessions limit 1..100 (20), offset 0..10000 (0).
Unknown parameters on profile routes rejected. Profiles is filtered to server grants.
Responses use existing adapter projections, no prompts/transcripts/config/tokens.
Missing/forged JWT ->401; missing grant/role/profile ->403; runtime auth/network/
contract failure ->503 `Runtime unavailable`; limits ->429. No browser Hermes
cookie, credential, socket ticket or mutation handler is provided.
Mount backend `/run/secrets/hermes_username` and `hermes_password` as files.
Only these fixed secret paths are read, after authorization. Broker login/cache
stays server-side. Backend and isolated inventory runtime must receive matching
credentials. Runtime additionally mounts `hermes_signing_key` (>=32 random chars).
Do not put credentials in YAML/env, commands, logs or Git. Rotate key to invalidate
sessions; rotate password and restart inventory plus backend to clear cached access.
Stateless old sessions remain valid until TTL/key rotation.
## Deliberately separate snapshot topology
Installed Hermes `serve` lifespan starts reconciliation and hosted-room recovery;
GET sessions auto-archives. Sharing its writable home with existing gateway is
therefore NOT a safe read-only sidecar. `serve.py` runs the pinned installed app
with lifespan OFF and an outer exact GET allowlist, password-login only and all
WebSockets denied1008. It does NOT start scheduler/gateway/MCP/rooms/provider chat.
The existing gateway and external volume `canina-hermes_canina_hermes_data` remain
unchanged, solely owned by existing gateway. Inventory mounts an independently
provisioned snapshot volume `canina_hermes_inventory_snapshot`; it NEVER mounts
original home. Its incidental session auto-archive writes affect only snapshot.
This is snapshot inventory, NOT live runtime freshness/status acceptance. A copy
of gateway PID state is not evidence of live gateway health. UI must not claim live
readiness from this deployment. Parent should expose snapshot provenance/freshness
before enabling it for business use.
Provisioning must use an approved consistent SQLite backup/checkpoint/export and
redacted profile config inventory, not raw `cp` of changing SQLite/WAL or credentials.
No automatic snapshot job is installed here. No secrets need be copied from live
home. Review which configs the model/status APIs need and omit auth.json/.env.
Missing store/schema -> unavailable, not fabricated empty healthy telemetry.
Dedicated internal network must contain only backend and inventory; do NOT connect
existing gateway with same `canina-hermes` DNS alias. Backend name resolves inventory.
Inventory network must have no external egress. Compose refers to external network;
operator must create it `--internal`. No host ports/socket are published.
## CI ownership / apply
`compose.readonly.yml` is deployable ONLY after external snapshot, internal network,
secret files and backend mounts/registration are provisioned. `ci-apply.sh` requires
CI=true, exact reviewed HEAD SHA, clean checkout, snapshot-reviewed assertion,
existing volume/network/files, then compose config/up. Never run alongside any
other inventory instance writing same snapshot. CI job fragment is tracked under
infra ownership; parent owns real workflow wiring and approval environment. No
active deployment trigger added. Existing review-only compose template is unchanged.
## Executed evidence
Remote `/opt/services/canina/validation/v4-runtime-completion/`:
- read-red.log missing controller regression; read-green.log 6 suites/27 tests.
- read-http-green.log 7 suites/30 tests: real Nest HTTP with signed JWT strategy,
bad signature/no auth401, ungranted admin403, allowed admin200, no cookie,
mutation404. This uses a mocked runtime service, not live Canina credentials.
- pinned-http.log exact installed image FastAPI login/protected inventory:
wrong password401, real login200/HttpOnly provider-verifiable token, profiles200,
tampered token401. No lifespan/live home.
- serve-http.log actual loopback TCP9119 in network-none disposable pinned image:
unauth inventory401, wrong password401, login200, profiles200, health auth true,
mutation/config/ws HTTP404. Generated test-only credentials. No live volume.
- ws-red.log existing socket rejection code differed; ws-green.log explicit
outer boundary denies authenticated sockets1008 independently of upstream tickets.
- build.log private Prisma generation and full backend tsc build exit0.
Pinned local exact image ID sha256:373ddd5e6649f525b17993e1c885cdf1352127f8e98fd329140a73237125e52f;
tracked registry digest sha256:9774f4f39a9bb8c2f68ce728ed5e99ddbad282163be56764afacf88ed952b784.
Node22 disposable runner copied read-only source/dependency donor privately;
no DB migration. broker-pinned-e2e.log proves actual compiled Node broker -> TCP pinned Hermes login -> adapter -> policy-filtered inventory (and deny zero secret reads). Nest HTTP authentication tested separately; these tests do not establish full Canina application E2E, snapshot
refresh, CI deployment, live gateway truth or business workflow acceptance.
Those remain explicit integration/provisioning gates, not claimed completion.
Final full non-DB regression: 106 suites / 850 tests pass; expected legacy negative SSL/payment logs. Scoped ESLint has zero errors, six type-safety warnings; full build passes. No RELEASE_AUTHORIZE integration suite run.
+86
View File
@@ -0,0 +1,86 @@
# Snapshot provenance repair — integration contract
This is independent snapshot inventory, **not a live Hermes executor**. No Production,
deployment, live gateway edits, or RELEASE_AUTHORIZE execution is included.
## Parent-owned application wiring
Keep AuthModule/JwtStrategy. Replace literal user grants in parent-owned app.module.ts with:
```ts
RuntimeReadModule.registerFromFile('/run/secrets/hermes_runtime_grants')
```
Nest supports the resulting Promise<DynamicModule> in imports. Alternatively pass the
reviewed nonsecret file `/etc/canina/runtime-grants.json`. No env/JWT/browser grant
input is accepted by this registration. File is JSON object mapping exact verified DB
user IDs to nonempty unique profile arrays; `{}` explicitly denies everyone. Missing,
malformed, symlinked, group/world-writable, oversized files fail startup closed. Mount
read-only and provision via reviewed configuration/secrets management; do not commit
real user grants. Existing explicit register(grants) remains for tests/legacy consumers.
Backend also mounts `/run/secrets/hermes_username` and `hermes_password` read-only.
No app.module.ts modification belongs to this commit.
## Required coordinated origin change
Authenticated management module and broker now exclusively use
`http://canina-hermes-inventory:9119`. compose.readonly.yml uses the distinct alias
`canina-hermes-inventory`, NEVER `canina-hermes`. Generic adapter allowlist retains
both reviewed exact origins for compatibility with separate existing consumers.
No arbitrary origin/env URL accepted. Parent must cherry-pick broker/module/adapter
and inventory compose together; do not add an alias to the existing gateway.
No deployment performed. External network should still be internal and only contain
backend/inventory; the distinct alias cannot shadow the live canina-hermes hostname.
## Snapshot exporter/operator gate
Every independent inventory root must include `snapshot-manifest.json`:
```json
{"schemaVersion":1,"source":"snapshot","capturedAt":"2026-10-23T12:00:00Z"}
```
The example is a schema example, NOT an actual capture attestation. Exporter must
write the real capture instant from the approved consistent export. Never copy the
example, use startup time, filesystem mtime, or claim this change captures/refreshes
snapshots. Missing/invalid/future manifest prevents startup. It is revalidated per
successful GET; invalid after startup returns503. Manifest is exporter metadata,
not a cryptographic proof of consistency. Immutable reviewed export provisioning
remains required. Fresh means age <=3600 seconds, otherwise stale; even fresh is
NOT live readiness. No snapshot scheduler installed.
## Additive public response fields / panel handoff
All object read DTOs (health/status/usage/sessions/model-info) add:
`provenance:{source:'snapshot',capturedAt:string,ageSeconds:number,freshness:'fresh'|'stale',liveProcess:'unknown'}`.
Profile and schedule array shapes are preserved: provenance is on each row. Empty
arrays have no rows to annotate; use the health response to display snapshot metadata.
Profile rows add `liveGatewayState:'unknown'`; status adds the same plus
`liveActiveAgents:null`. Existing required scalars remain typed but are conservative
placeholders: `gateway_running:false`, `active_agents:0`, `gateway_state:null`,
`overall:'unknown'`. False/zero MUST NOT render as measured offline/idle: when
provenance.source is snapshot use the explicit unknown fields, label snapshot and
show capturedAt/freshness. Health.ok only means inventory HTTP app responsiveness,
never live gateway readiness. Profile existence is inventory availability, NOT
process readiness. Adapter enforces provenance on dedicated snapshot origin and
conservatively overwrites copied PID-derived status/profile scalars again.
Installed status handler was unsafe here: it probes gateway HTTP/PIDs/platforms and
can stall despite this being an independent snapshot. Snapshot server replaces that
route with measured package version and explicit unknown process fields; no status
probe is run. Password auth/mutation denial/WebSocket denial remain intact.
## Executed acceptance
Remote disposable network-none exact pinned Hermes image, no original volumes:
manifest-missing regression failed first; TCP login/profile/health/status provenance
and stale capture assertions now pass; authenticated sockets denied1008. Disposable
Node22 copy of source/dependencies: 9 hermes-management suites /34 tests pass and
full backend tsc build passes. Adapter copied-live-state regression failed first;
grants-loader missing-function regression failed first. Compiled actual Node broker
-> TCP pinned server -> adapter -> policy-filtered profiles/status passes, deny reads
zero credentials. Test timestamps/credentials are fixtures, not live telemetry.
The first HTTP retry raced fixture startup and returned UNAVAILABLE; retry after
server startup passed. No claim of full app/user-flow or live runtime integration.
Full non-DB Jest regression: 108 suites/854 tests passed; 1 suite/18 tests skipped. No RELEASE_AUTHORIZE suite was enabled.
+78
View File
@@ -0,0 +1,78 @@
# Real bounded Tech/Dev Hermes pilot (experimental, NOT enabled)
Owned files only in `infra/hermes-pilot`; no existing core, module, tracer, profile or runtime config changed. No push/deploy/RELEASE_AUTHORIZE. Do not wire this into the existing tracer completion endpoint: that endpoint deliberately accepts only deterministic JSON.
## Fixed task and boundaries
Task `tech-dev-clamp-quantity-v1`: produce one <=2048-byte Python `clamp_quantity(quantity)` implementation, integer range 0..99. No input/browser prompt is accepted. The model has **zero tools**, no shell/browser/Git/deploy credentials. Trusted code alone writes `quantity.py`, runs separate AST-allowlisted QA (9 boundary cases), and commits an isolated, remote-free fixture repository. This is real generated-code QA, not application QA. Source commit is recorded as provenance, not represented as the fixture's base commit.
`infer.py` uses installed `run_agent.AIAgent`, verified from the installed signature and `agent/agent_init.py`; `enabled_toolsets=[]` is additionally enforced by asserting empty `agent.tools`. Existing approved `OPENAI_API_KEY` is read in memory only; no messaging/API-server credentials/auth pools copied. Separate temporary HERMES_HOME, no memories/context/background review/fallback. Provider/model pinned to current canina-hermes `custom`, `https://nr.parsaaghayi.ir/v1`, `Free`; no provider installation or account changes. Existing configs remain untouched. Model output is untrusted and never directly shell-executed; QA rejects all syntax except the single nested min/max return expression before isolated Python execution.
Installed image identity: `sha256:373ddd5e6649f525b17993e1c885cdf1352127f8e98fd329140a73237125e52f`, install stamp commit `818c13be1dc4fd28987e1e881a9408224afd4535`, v0.21.6. Discovery: `/opt/hermes/hermes chat --help` supports `--oneshot`, `--max-turns`, `--run-budget`; `/opt/hermes/.venv/bin/python` loads installed API. Official skills and https://hermes-agent.nousresearch.com/docs/llms.txt consulted. The initially guessed developer API docs URL returned404; installed source/signature is authoritative for this pinned build.
## PREPARE mapping — parent integration required
`executor.execute(lease, artifact_root, infer, heartbeat)` only accepts a fresh staging, first-attempt, 20-cent, 40hex-source lease and exact lease fields; heartbeat failure aborts before inference and before QA/commit. Trusted adapter must obtain this lease via existing authenticated claim AFTER independent Gate1/READY, verify the explicitly allowlisted task identity/scope/criteria server-side, and supply heartbeat connected to the authenticated backend. A caller-created dict is NOT an authorization. Never expose this Python interface to browser requests.
## Heartbeat runner and parent API (this revision)
After independent READY/Gate1, allowlist and provider-cap admission, parent constructs
`runner = executor.pinned_inference(executor.IMAGE)` and calls
`executor.execute(authenticated_lease, artifact_root, runner, authenticated_heartbeat)`.
Only trusted Python may construct runners. Callback receives an unchanged COPY of the
original lease binding (taskId/runId/leaseToken), must use authenticated server transport,
and must raise on any non-2xx, ownership/fencing/expiry denial or malformed response.
Explicit `False` also denies; `None` means success for existing raise-on-error adapters.
Callback MUST impose <=5s network timeout, no retries; a blocked callback thread cannot
be forcibly killed by Python. It is daemonized; its result cannot restart inference or
extend a local lease. Parent must serialize executions per server lease; the runner is
single-use and concurrent reuse of that runner is locked out. This is not a browser API.
Immediate initial heartbeat; dedicated ongoing thread waits 5s between acknowledged
calls. With required <=5s transport ceiling, heartbeat starts are <=10s apart. A stalled
heartbeat (10s without acknowledgment), exception or denial aborts actual subprocess
inference, not just its caller. No leaseUntil mutation/local extension occurs. Fresh
server-authorized heartbeats retain fencing; this code cannot create authorization.
Guard checks also fence QA/artifact commit/report return. A denial during QA prevents
report return; any partial local fixture is not accepted completion evidence.
Monotonic wall ceiling 90s with cancellation reserve: work aborts at85s, remote kill
command <=2s and local reap <=1s; bounded QA3s/Git1s. Pinned runner uses one model
call, max_iterations1/max_tokens160, run_budget80s, tools0, no retries. An independent
in-container timeout85s survives loss of the Docker client. Cancellation addresses
the actual in-container process session via a random PID file plus cancellation
marker (handles cancellation before PID publication); it kills that process group.
Cancellation transport failure raises and prevents success, but cannot prove immediate
remote death; independent inner timeout is the fallback. Small random marker files
in container /tmp are intentional fencing tombstones, not configuration changes.
`rehearse.py` remains synthetic, never a live lease. It uses this cancellable runner
but is NOT admission evidence. Do not run it until provider-cap approval. Existing
20-cent reservation remains UNKNOWN, never zero, and no completion/claim/release or
backend transport is implemented here. Existing tracer endpoint still rejects model
reports; parent must separately implement authenticated allowlisted artifact ingestion.
No changes to installed Hermes or runtime/profile configuration.
## Actual evidence and blockers
Evidence directory `/opt/services/canina/validation/v4-real-pilot`:
- `red.log`: missing executor assertion failed before implementation.
- `inference-private.log`: actual installed Hermes/provider inference exit0 under75s diagnostic ceiling; generated source saved, not fabricated. This ceiling was ONLY diagnostic, not accepted as a PREPARE lease run.
- `model-artifact/quantity.py`: actual model artifact; independent QA passes9 cases, SHA256 `0125246b551896c5f9ee98fabeb02982a6690a9974c0eb24ef3ae676d46380d5`; isolated Git commit `95cb3d27d636ae817d6a778e8053e2f87b50e57b`.
- `actual.log`: bounded full rehearsal failed with inner timeout exit137 at18s; no completion/QA success was assumed. Previous client-only timeout was corrected to inner process kill. **Installed model/startup cannot currently complete reliably inside30s PREPARE lease without authenticated ongoing heartbeat.** Do not extend lease locally or call the75s diagnostic a bounded lease success.
Contract tests: `python3 infra/hermes-pilot/test_contract.py`. Parent still owns backend integration, authenticated heartbeat orchestration, allowlist admission, billing caps, signed artifact transport and Stage CI. This commit is a real generation/independent-QA proof and fail-closed executor seam, not a live autonomous Tech department or released product.
## Verification of heartbeat revision
`python3 -m unittest discover -s infra/hermes-pilot -v`: 11 passing tests, including
real OS child/descendant cancellation, ongoing rejection, stalled heartbeat, wall
ceiling, concurrent runner reuse, immutable lease and unknown-cost preservation.
The artifact success test is explicitly a deterministic process fixture, NOT model
inference. TDD red observed for missing ongoing runner and explicit false denial;
additional cancellation/concurrency regression cases were added and executed.
Previously generated real model artifact was rechecked by current AST allowlist QA:
9 cases pass, SHA256 `0125246b551896c5f9ee98fabeb02982a6690a9974c0eb24ef3ae676d46380d5`.
No new real inference was run: README provider pricing/cap gate remains unresolved.
Docker cancellation path is implemented but not runtime-exercised here; OS process
group cancellation is exercised. No live backend/claim/release, push or deploy.
+64
View File
@@ -0,0 +1,64 @@
"""Fixed Tech/Dev pilot. Adapter must supply a fresh authenticated PREPARE lease."""
import datetime, hashlib, json, os, pathlib, re, subprocess, tempfile, time
TASK='tech-dev-clamp-quantity-v1'
IMAGE='sha256:373ddd5e6649f525b17993e1c885cdf1352127f8e98fd329140a73237125e52f'
def validate_lease(lease):
if (set(lease) != {'taskId','runId','leaseToken','leaseUntil','commitSha','environment','maxCostCents','attempt'}
or lease['environment']!='staging' or lease['maxCostCents']!=20 or lease['attempt']!=1
or not re.fullmatch('[a-f0-9]{40}',lease['commitSha'])):
raise ValueError('Bounded PREPARE lease required')
for k in ('taskId','runId','leaseToken'):
if not re.fullmatch('[a-f0-9-]{36}',lease[k]): raise ValueError('Invalid lease binding')
expiry=datetime.datetime.fromisoformat(lease['leaseUntil'].replace('Z','+00:00')).timestamp()
if not 5 < expiry-time.time() <= 31: raise ValueError('Fresh 30-second lease required')
return lease
def _execute(lease, artifact_root, infer, heartbeat):
"""No claim/approval generation. heartbeat must call trusted server; errors abort.
infer is the pinned runner, not a browser-supplied command or prompt.
Completion is deliberately NOT sent: existing transport accepts tracer JSON only.
"""
validate_lease(lease)
heartbeat(lease)
root=pathlib.Path(tempfile.mkdtemp(prefix='real-pilot-',dir=artifact_root))
def git(*args):
return subprocess.check_output(['git','-C',str(root),*args],stderr=subprocess.DEVNULL,text=True,timeout=1).strip()
git('init','-q'); git('config','user.email','pilot@invalid'); git('config','user.name','Bounded Hermes Pilot')
(root/'README').write_text('Disposable fixture; PREPARE source '+lease['commitSha']+'\n')
git('add','README'); git('commit','-qm','Seed isolated pilot fixture')
base=git('rev-parse','HEAD')
# One cancellable inference, no tools; ongoing authenticated guard.
source=infer()
heartbeat(lease)
if not isinstance(source,str) or len(source.encode())>2048: raise ValueError('Invalid generated artifact')
(root/'quantity.py').write_text(source)
qa=subprocess.check_output(['python3','-I',str(pathlib.Path(__file__).with_name('qa.py')),str(root)],timeout=3,text=True)
heartbeat(lease)
git('add','quantity.py'); git('commit','-qm','Hermes generated quantity clamp; independent QA passed')
patch=git('diff',base,'HEAD','--','quantity.py')+'\n'
(root/'artifact.patch').write_text(patch)
report={'kind':'REAL_HERMES_TECH_DEV_PILOT','task':TASK,'taskId':lease['taskId'],'runId':lease['runId'],
'sourceCommit':lease['commitSha'],'artifactCommit':git('rev-parse','HEAD'),'baseCommit':base,
'patchSha256':hashlib.sha256(patch.encode()).hexdigest(),'qa':json.loads(qa),
'costCents':None,'costStatus':'UNKNOWN_RETAIN_FULL_20_CENT_RESERVATION',
'gate1':'EXTERNAL_LEASE_REQUIRED','published':False,'deploymentExecuted':False}
(root/'evidence.json').write_text(json.dumps(report,indent=2)+'\n')
return root,report
from runtime import LeaseGuard, ProcessInference, pinned_inference
def execute(lease, artifact_root, infer, heartbeat):
validate_lease(lease)
with LeaseGuard(lease, heartbeat) as guard:
guard.check()
if not isinstance(infer, ProcessInference):
raise TypeError("Cancellable ProcessInference required")
def generate():
response = json.loads(infer.run(guard))
if not isinstance(response,dict) or response.get("tools") != 0 or response.get("model") != "Free":
raise ValueError("Pinned tool-less model required")
return response["source"]
result = _execute(lease, artifact_root, generate, lambda _: guard.check())
guard.check()
return result
+22
View File
@@ -0,0 +1,22 @@
"""Executed only inside pinned canina-hermes image by controlled runner."""
import contextlib, io, json, os, pathlib, tempfile
# Copy ONE existing inference credential in memory; no auth pools or messaging keys.
from dotenv import dotenv_values
key=dotenv_values('/opt/data/.env').get('OPENAI_API_KEY')
if not key: raise RuntimeError('Existing OPENAI_API_KEY unavailable')
home=tempfile.TemporaryDirectory(prefix='canina-real-pilot-')
os.environ.clear()
os.environ.update({'PATH':'/usr/bin:/bin','HOME':home.name,'HERMES_HOME':home.name,'OPENAI_API_KEY':key,'OPENAI_BASE_URL':'https://nr.parsaaghayi.ir/v1','HERMES_DISABLE_PLUGINS':'1'})
# Installed API verified by inspect.signature at pinned image commit 818c13be.
with contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(io.StringIO()):
from run_agent import AIAgent
agent=AIAgent(model='Free',provider='custom',base_url='https://nr.parsaaghayi.ir/v1',api_key=key,
enabled_toolsets=[],max_iterations=1,max_tokens=160,run_budget_seconds=80,
skip_context_files=True,skip_memory=True,skip_background_review=True,load_soul_identity=False,
quiet_mode=True,fallback_model={},cwd=home.name)
if agent.tools: raise RuntimeError('Tool-less invariant failed')
result=agent.run_conversation('Write only Python source, no markdown: define clamp_quantity(quantity) returning integer quantity clamped to inclusive 0..99. Use one return with nested min/max calls only. No annotations, comments, imports, decorators, docstrings or other statements.')
final=result.get('final_response')
if not isinstance(final,str) or not final.strip(): raise RuntimeError('No model artifact; result keys='+','.join(result.keys()))
print(json.dumps({'source':final.strip()+'\n','model':'Free','tools':0}))
home.cleanup()
+26
View File
@@ -0,0 +1,26 @@
"""Independent structural + behavior QA. Never executes arbitrary model Python."""
import ast, hashlib, json, pathlib, sys
def verify(root):
p=pathlib.Path(root)/'quantity.py'
if p.is_symlink() or not p.is_file() or p.stat().st_size>2048: raise ValueError('Invalid file')
raw=p.read_bytes(); tree=ast.parse(raw)
if len(tree.body)!=1 or not isinstance(tree.body[0],ast.FunctionDef): raise ValueError('Single function required')
fn=tree.body[0]
if fn.name!='clamp_quantity' or fn.decorator_list or fn.returns or fn.args.defaults or fn.args.kw_defaults or fn.args.kwonlyargs or fn.args.posonlyargs or fn.args.vararg or fn.args.kwarg or len(fn.args.args)!=1 or fn.args.args[0].arg!='quantity' or fn.args.args[0].annotation or len(fn.body)!=1 or not isinstance(fn.body[0],ast.Return): raise ValueError('Unexpected function structure')
def expr(n):
if isinstance(n,ast.Constant) and type(n.value) is int and 0<=n.value<=99: return
if isinstance(n,ast.Name) and n.id=='quantity': return
if isinstance(n,ast.Call) and isinstance(n.func,ast.Name) and n.func.id in ('min','max') and len(n.args)==2 and not n.keywords:
for a in n.args: expr(a)
return
raise ValueError('Nonallowlisted syntax')
expr(fn.body[0].value)
scope={'__builtins__':{},'min':min,'max':max}
exec(compile(tree,'generated-quantity','exec'),scope)
cases=[-1000,-1,0,1,50,98,99,100,10**30]
for q in cases:
actual=scope['clamp_quantity'](q)
if type(actual) is not int or actual!=max(0,min(q,99)): raise ValueError('Behavior QA failed')
return {'passed':True,'cases':len(cases),'sha256':hashlib.sha256(raw).hexdigest(),'scope':'quantity clamp fixture only'}
if __name__=='__main__': print(json.dumps(verify(sys.argv[1])))
+18
View File
@@ -0,0 +1,18 @@
"""Disposable model/artifact exercise, NOT a live Gate1 approval or service completion."""
import datetime, json, pathlib, subprocess, time, uuid
from executor import execute, IMAGE, pinned_inference
def main():
image=subprocess.check_output(['docker','inspect','--format','{{.Image}}','canina-hermes'],text=True).strip()
if image!=IMAGE: raise RuntimeError('Pinned installed image mismatch')
infer=pinned_inference(IMAGE)
deadline=time.monotonic()+90
lease={k:str(uuid.uuid4()) for k in ('taskId','runId','leaseToken')}
lease.update(leaseUntil=(datetime.datetime.now(datetime.timezone.utc)+datetime.timedelta(seconds=30)).isoformat(),commitSha=subprocess.check_output(['git','rev-parse','HEAD'],text=True).strip(),environment='staging',maxCostCents=20,attempt=1)
def fixture_heartbeat(binding):
if time.monotonic()>deadline: raise TimeoutError('Fixture hard deadline')
root,report=execute(lease,'/opt/services/canina/validation/v4-real-pilot',infer,fixture_heartbeat)
report['executionContext']='DISPOSABLE_FIXTURE_NOT_LIVE_PREPARE_LEASE'
(root/'evidence.json').write_text(json.dumps(report,indent=2)+'\n')
print(json.dumps({'artifactDirectory':str(root),'report':report}))
if __name__=='__main__': main()
+127
View File
@@ -0,0 +1,127 @@
"""Trusted subprocess boundary; no claim, lease extension or completion transport."""
import json
import os
import pathlib
import signal
import subprocess
import tempfile
import threading
import time
import uuid
HEARTBEAT_INTERVAL = 5.0
HEARTBEAT_TIMEOUT = 5.0
MAX_WALL_SECONDS = 90.0
class LeaseGuard:
def __init__(self, lease, heartbeat):
self.lease, self.heartbeat = lease, heartbeat
self.stop = threading.Event()
self.ready = threading.Event()
self.error = None
self.started = time.monotonic()
self.last = self.started
self.thread = threading.Thread(target=self._run, daemon=True)
def _run(self):
while not self.stop.is_set():
try:
if self.heartbeat(dict(self.lease)) is False:
raise PermissionError("Authenticated heartbeat denied")
self.last = time.monotonic()
self.ready.set()
except BaseException as exc:
self.error = exc
self.ready.set()
return
if self.stop.wait(HEARTBEAT_INTERVAL):
return
def check(self):
if self.error is not None:
raise self.error
now = time.monotonic()
if now - self.started >= MAX_WALL_SECONDS - 5:
raise TimeoutError('Pilot wall deadline (cancellation reserve included)')
if now - self.last >= HEARTBEAT_INTERVAL + HEARTBEAT_TIMEOUT:
raise TimeoutError('Authenticated heartbeat stalled')
def __enter__(self):
self.thread.start()
if not self.ready.wait(HEARTBEAT_TIMEOUT):
self.stop.set()
raise TimeoutError('Initial authenticated heartbeat stalled')
try:
self.check()
except BaseException:
self.stop.set()
raise
return self
def __exit__(self, *args):
self.stop.set()
self.thread.join(timeout=.05)
class ProcessInference:
"""Trusted argv only. cancel_remote MUST kill the in-container process group."""
def __init__(self, argv, input_text=None, cancel_remote=None):
self.argv, self.input_text, self.cancel_remote = argv, input_text, cancel_remote
self.used = False
self.lock = threading.Lock()
def run(self, guard):
with self.lock:
if self.used:
raise RuntimeError('Exactly one inference allowed; no retry')
self.used = True
guard.check()
# Files avoid pipe deadlocks and unbounded captured output in memory.
with tempfile.TemporaryFile(mode='w+') as incoming, tempfile.TemporaryFile(mode='w+') as out, tempfile.TemporaryFile(mode='w+') as err:
incoming.write(self.input_text or '')
incoming.seek(0)
child = subprocess.Popen(self.argv, stdin=incoming, stdout=out, stderr=err, start_new_session=True)
try:
while child.poll() is None:
guard.check()
time.sleep(.02)
guard.check()
if child.returncode:
raise RuntimeError('Inference failed (exit %d)' % child.returncode)
out.seek(0)
result = out.read(4097)
if len(result) > 4096:
raise ValueError('Oversized inference response')
return result
finally:
# Always cancel remote, even when docker exec itself failed.
try:
if self.cancel_remote:
self.cancel_remote()
finally:
try:
os.killpg(child.pid, signal.SIGKILL)
except ProcessLookupError:
pass
child.wait(timeout=1)
def pinned_inference(image):
actual = subprocess.check_output(['docker','inspect','--format','{{.Image}}','canina-hermes'], text=True, timeout=2).strip()
if actual != image:
raise RuntimeError('Pinned installed image mismatch')
token = uuid.uuid4().hex
pidfile = '/tmp/canina-pilot-' + token + '.pid'
script = pathlib.Path(__file__).with_name('infer.py').read_text()
# Session leader is created INSIDE the container. Inner kill is independent of client.
wrapper = ('import os,signal,subprocess,sys; '
'p=subprocess.Popen(["timeout","--signal=KILL","85","/opt/hermes/.venv/bin/python","-"],stdin=sys.stdin,start_new_session=True); '
'open(%r,"w").write(str(p.pid)); '
'os.path.exists(%r) and os.killpg(p.pid,signal.SIGKILL); sys.exit(p.wait())') % (pidfile, pidfile+'.cancel')
cancel = ('import os,signal; p=%r; '
'\nopen(p+".cancel","w").close()\ntry:\n pid=int(open(p).read()); os.killpg(pid,signal.SIGKILL)\n'
'except (FileNotFoundError,ProcessLookupError): pass\n'
'finally:\n'
' try: os.unlink(p)\n except FileNotFoundError: pass\n') % pidfile
def cancel_remote():
subprocess.run(['docker','exec','canina-hermes','/opt/hermes/.venv/bin/python','-c',cancel], check=True, timeout=2, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
return ProcessInference(['docker','exec','-i','canina-hermes','/opt/hermes/.venv/bin/python','-c',wrapper], script, cancel_remote)
+68
View File
@@ -0,0 +1,68 @@
import json, pathlib, sys, tempfile, threading, time, unittest
from unittest.mock import patch
sys.path.insert(0,str(pathlib.Path(__file__).parent))
import executor as m
import runtime
from test_contract import lease
class Cancellation(unittest.TestCase):
def test_wall_deadline_kills_descendant(self):
with tempfile.TemporaryDirectory() as root:
marker=pathlib.Path(root,'late')
child=f'import pathlib,time; time.sleep(.7); pathlib.Path({str(marker)!r}).touch()'
code=f'import subprocess,sys,time; subprocess.Popen([sys.executable,"-c",{child!r}]); time.sleep(5)'
with patch.object(runtime,'MAX_WALL_SECONDS',5.2):
with self.assertRaises(TimeoutError): m.execute(lease(),root,m.ProcessInference([sys.executable,'-c',code]),lambda _: None)
time.sleep(.8)
self.assertFalse(marker.exists())
def test_stalled_heartbeat_cancels_child(self):
with tempfile.TemporaryDirectory() as root:
marker=pathlib.Path(root,'late'); release=threading.Event(); calls=[]
def heartbeat(_):
calls.append(time.monotonic())
if len(calls)>1: release.wait(3)
code=f'import time,pathlib; time.sleep(.7); pathlib.Path({str(marker)!r}).touch()'
try:
with patch.object(runtime,'HEARTBEAT_INTERVAL',.05),patch.object(runtime,'HEARTBEAT_TIMEOUT',.1):
with self.assertRaises(TimeoutError): m.execute(lease(),root,m.ProcessInference([sys.executable,'-c',code]),heartbeat)
finally: release.set()
time.sleep(.8); self.assertFalse(marker.exists())
def test_fixture_artifact_qa_and_cost_reservation(self):
# Explicit deterministic PROCESS fixture, never reported as real model inference.
source='def clamp_quantity(quantity):\n return min(99, max(0, quantity))\n'
response=json.dumps(dict(source=source,model='Free',tools=0))
beats=[]; original=lease(); snapshot=dict(original)
def heartbeat(binding):
beats.append(time.monotonic()); binding['leaseUntil']='untrusted mutation'
with tempfile.TemporaryDirectory() as root:
infer=m.ProcessInference([sys.executable,'-c',f'import time; time.sleep(.25); print({response!r})'])
with patch.object(runtime,'HEARTBEAT_INTERVAL',.05):
_,report=m.execute(original,root,infer,heartbeat)
self.assertEqual(original,snapshot)
self.assertGreater(len(beats),2)
self.assertIsNone(report['costCents'])
self.assertEqual(report['costStatus'],'UNKNOWN_RETAIN_FULL_20_CENT_RESERVATION')
with self.assertRaises(RuntimeError):
with runtime.LeaseGuard(lease(),lambda _:None) as guard: infer.run(guard)
def test_concurrent_reuse_rejected(self):
infer=m.ProcessInference([sys.executable,"-c","import time; time.sleep(.3); print(1)"])
errors=[]
with runtime.LeaseGuard(lease(),lambda _:None) as guard:
t=threading.Thread(target=lambda:infer.run(guard)); t.start()
time.sleep(.05)
with self.assertRaises(RuntimeError): infer.run(guard)
t.join()
def test_explicit_false_heartbeat_denied(self):
with tempfile.TemporaryDirectory() as root:
with self.assertRaises(PermissionError):
m.execute(lease(),root,m.ProcessInference([sys.executable,"-c","print(1)"]),lambda _:False)
def test_callable_not_cancellable_rejected(self):
with tempfile.TemporaryDirectory() as root:
with self.assertRaises(TypeError): m.execute(lease(),root,lambda:'',lambda _:None)
if __name__=='__main__': unittest.main()
+23
View File
@@ -0,0 +1,23 @@
import datetime, importlib.util, pathlib, tempfile, unittest, uuid
p=pathlib.Path(__file__).with_name('executor.py')
spec=importlib.util.spec_from_file_location('pilot',p); m=importlib.util.module_from_spec(spec); spec.loader.exec_module(m)
qs=importlib.util.spec_from_file_location('qa',p.with_name('qa.py')); qa=importlib.util.module_from_spec(qs); qs.loader.exec_module(qa)
def lease():
d={k:str(uuid.uuid4()) for k in ('taskId','runId','leaseToken')}; d.update(environment='staging',maxCostCents=20,attempt=1,commitSha='a'*40,leaseUntil=(datetime.datetime.now(datetime.timezone.utc)+datetime.timedelta(seconds=30)).isoformat()); return d
class Contract(unittest.TestCase):
def test_missing_lease_rejected(self):
with self.assertRaises(ValueError): m.validate_lease({})
def test_bounds(self):
for k,v in [('environment','production'),('attempt',2),('maxCostCents',21),('commitSha','main'),('leaseUntil','2000-01-01T00:00:00+00:00')]:
d=lease(); d[k]=v
with self.assertRaises(ValueError): m.validate_lease(d)
def test_heartbeat_denial_prevents_inference(self):
with tempfile.TemporaryDirectory() as root:
def deny(_): raise PermissionError('killed')
with self.assertRaises(PermissionError): m.execute(lease(),root,lambda:self.fail('must not infer'),deny)
def test_qa_rejects_malicious_and_wrong_behavior(self):
for source in ['import os\n','def clamp_quantity(quantity):\n return quantity\n','def clamp_quantity(quantity):\n return __import__("os").system("true")\n']:
with tempfile.TemporaryDirectory() as root:
pathlib.Path(root,'quantity.py').write_text(source)
with self.assertRaises(ValueError): qa.verify(root)
if __name__=='__main__': unittest.main()
+24
View File
@@ -0,0 +1,24 @@
import datetime, pathlib, sys, tempfile, threading, time, unittest
from unittest.mock import patch
sys.path.insert(0, str(pathlib.Path(__file__).parent))
import executor as m
import runtime
from test_contract import lease
class Heartbeats(unittest.TestCase):
def test_ongoing_denial_kills_real_child(self):
with tempfile.TemporaryDirectory() as root:
marker=pathlib.Path(root,'late')
code=f'import time,pathlib; time.sleep(1); pathlib.Path({str(marker)!r}).touch()'
beats=[]
def beat(binding):
beats.append(time.monotonic())
if len(beats)>1: raise PermissionError('revoked')
with patch.object(runtime,'HEARTBEAT_INTERVAL',.1):
with self.assertRaises(PermissionError):
m.execute(lease(),root,m.ProcessInference([sys.executable,'-c',code]),beat)
time.sleep(1.1)
self.assertFalse(marker.exists())
self.assertGreaterEqual(len(beats),2)
if __name__=='__main__': unittest.main()
+17
View File
@@ -0,0 +1,17 @@
#!/usr/bin/env bash
set -euo pipefail
# Invoked ONLY by an approved CI job on the target host; never by application startup.
[[ "${CI:-}" == true && "${RUNTIME_READ_APPROVED_SHA:-}" =~ ^[0-9a-f]{40}$ ]] || exit 64
root=$(git rev-parse --show-toplevel)
[[ "$(git rev-parse HEAD)" == "$RUNTIME_READ_APPROVED_SHA" ]] || exit 65
[[ -z "$(git status --porcelain)" ]] || exit 65
[[ "${RUNTIME_SNAPSHOT_REVIEWED:-}" == true ]] || exit 66
# Gateway remains untouched. Its original writable data volume is NEVER mounted.
# The external snapshot must already exist; do not silently initialize empty data.
docker volume inspect canina_hermes_inventory_snapshot >/dev/null
docker network inspect canina_hermes_backend >/dev/null
for name in hermes_username hermes_password hermes_signing_key; do
test -s "/opt/services/canina/secrets/$name"
done
docker compose -f "$root/infra/hermes-runtime/compose.readonly.yml" config --quiet
docker compose -f "$root/infra/hermes-runtime/compose.readonly.yml" up -d --no-build
+16
View File
@@ -0,0 +1,16 @@
# Merge this job into the owning Stage workflow only after provisioning review.
# No production trigger or secret material belongs in this file.
runtime-read-acceptance:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Pinned Hermes HTTP acceptance
run: |
set -euo pipefail
image=nousresearch/hermes-agent@sha256:9774f4f39a9bb8c2f68ce728ed5e99ddbad282163be56764afacf88ed952b784
for test in test_http_acceptance.py test_serve.py test_ws_denial.py; do
docker run --rm --network none --entrypoint /opt/hermes/.venv/bin/python3 -v "$PWD/infra/hermes-runtime:/contract:ro" "$image" "/contract/$test"
done
# Apply job must require approved SHA, Stage environment approval, provisioned external
# snapshot/network/secrets, and call bash infra/hermes-runtime/ci-apply.sh on target.
# Not enabled here: parent owns actual workflow and Stage promotion.
+36
View File
@@ -0,0 +1,36 @@
name: canina-hermes-inventory
services:
runtime-inventory:
image: nousresearch/hermes-agent@sha256:9774f4f39a9bb8c2f68ce728ed5e99ddbad282163be56764afacf88ed952b784
entrypoint: [/opt/hermes/.venv/bin/python3, /runtime/serve.py]
restart: unless-stopped
environment:
HERMES_HOME: /inventory
HERMES_SERVE_HEADLESS: "1"
volumes:
- ./serve.py:/runtime/serve.py:ro
- inventory_snapshot:/inventory
secrets: [hermes_username, hermes_password, hermes_signing_key]
networks:
backend:
aliases: [canina-hermes-inventory]
security_opt: [no-new-privileges:true]
cap_drop: [ALL]
pids_limit: 128
mem_limit: 1g
# No ports, host socket, original home, or egress network.
volumes:
inventory_snapshot:
external: true
name: canina_hermes_inventory_snapshot
networks:
backend:
external: true
name: canina_hermes_backend
secrets:
hermes_username:
file: /opt/services/canina/secrets/hermes_username
hermes_password:
file: /opt/services/canina/secrets/hermes_password
hermes_signing_key:
file: /opt/services/canina/secrets/hermes_signing_key
+11
View File
@@ -0,0 +1,11 @@
# Disposable acceptance fixture only; not production entrypoint.
import os,tempfile,importlib.util
os.environ['HERMES_HOME']=tempfile.mkdtemp()
os.environ['HERMES_SERVE_HEADLESS']='1'
from pathlib import Path
import json
(Path(os.environ["HERMES_HOME"])/"snapshot-manifest.json").write_text(json.dumps({"schemaVersion":1,"source":"snapshot","capturedAt":"2020-01-01T00:00:00Z"}))
spec=importlib.util.spec_from_file_location('runtime_serve','/contract/serve.py');m=importlib.util.module_from_spec(spec);spec.loader.exec_module(m)
m.secret=lambda name:'isolated-test-only-'+name+'x'*32
import uvicorn
uvicorn.run(m.build_app(),host='0.0.0.0',port=9119,lifespan='off',access_log=False,log_level='warning')
+94
View File
@@ -0,0 +1,94 @@
"""Dedicated read-inventory process: never starts cron, gateway, rooms, MCP or PTY."""
from pathlib import Path
import os, json
from datetime import datetime, timezone
from fastapi import Request
from fastapi.responses import JSONResponse
from plugins.dashboard_auth.basic import BasicAuthProvider, hash_password
from hermes_cli.dashboard_auth.registry import register_global_provider
from hermes_cli import web_server
import uvicorn
def secret(name):
p=Path('/run/secrets')/name
if not p.is_file() or p.stat().st_size>4096: raise RuntimeError('Runtime credential unavailable')
value=p.read_text().strip()
if not value or '\n' in value or '\r' in value: raise RuntimeError('Runtime credential unavailable')
return value
def snapshot_provenance():
try:
path=Path(os.environ['HERMES_HOME'])/'snapshot-manifest.json'
if path.is_symlink() or not path.is_file() or path.stat().st_size>4096:
raise ValueError()
data=json.loads(path.read_text())
if set(data)!={'schemaVersion','source','capturedAt'} or data['schemaVersion']!=1 or data['source']!='snapshot':
raise ValueError()
captured=data['capturedAt']
if not isinstance(captured,str) or not captured.endswith('Z'):
raise ValueError()
at=datetime.fromisoformat(captured.replace('Z','+00:00'))
age=(datetime.now(timezone.utc)-at).total_seconds()
if age<0: raise ValueError()
return {'source':'snapshot','capturedAt':captured,'ageSeconds':int(age),
'freshness':'fresh' if age<=3600 else 'stale','liveProcess':'unknown'}
except (OSError, ValueError, TypeError, KeyError):
raise RuntimeError('Valid snapshot manifest required') from None
def build_app():
snapshot_provenance()
# Never execute installed status: it probes PIDs, gateway HTTP and platform health.
web_server.app.router.routes[:] = [route for route in web_server.app.router.routes
if getattr(route,'path',None)!='/api/status']
@web_server.app.get('/api/status')
async def snapshot_status():
from hermes_cli.web_server import get_version_info
return {'version':get_version_info().base_version,'gateway_running':False,
'gateway_state':None,'active_agents':0,'overall':'unknown',
'liveGatewayState':'unknown','liveActiveAgents':None}
# Place before installed catch-all UI route.
web_server.app.router.routes.insert(0, web_server.app.router.routes.pop())
register_global_provider(BasicAuthProvider(username=secret('hermes_username'),password_hash=hash_password(secret('hermes_password')),secret=secret('hermes_signing_key').encode(),ttl_seconds=300))
web_server.app.state.auth_required=True
web_server.app.state.bound_host='0.0.0.0'
# Enforce an outer exact route/method allowlist. Installed public health/status stay
# internal; protected inventory still uses installed provider middleware.
allowed={'/api/health','/api/profiles','/api/status','/api/analytics/usage','/api/sessions','/api/cron/jobs','/api/model/info'}
@web_server.app.middleware('http')
async def readonly_gate(request:Request,call_next):
path=request.url.path
if not ((request.method=='GET' and path in allowed) or (request.method=='POST' and path=='/auth/password-login')):
return JSONResponse({'detail':'Unavailable'},status_code=404)
response=await call_next(request)
if request.method!='GET' or response.status_code!=200: return response
try:
provenance=snapshot_provenance()
except RuntimeError:
return JSONResponse({'detail':'Snapshot unavailable'},status_code=503)
body=b''.join([chunk async for chunk in response.body_iterator])
data=json.loads(body)
def annotate(row):
return {**row,'provenance':provenance}
if isinstance(data,list):
data=[annotate(row) for row in data]
else:
data=annotate(data)
if path=='/api/profiles':
data['profiles']=[{**annotate(row),'gateway_running':False,'liveGatewayState':'unknown'} for row in data['profiles']]
if path=='/api/status':
# Legacy required scalars are conservative placeholders, NOT measurements.
data.update(gateway_running=False,active_agents=0,overall='unknown',gateway_state=None,
liveGatewayState='unknown',liveActiveAgents=None)
return JSONResponse(data,headers={'Cache-Control':'no-store'})
class NoSockets:
async def __call__(self, scope, receive, send):
if scope['type']=='websocket':
await send({'type':'websocket.close','code':1008})
return
await web_server.app(scope,receive,send)
return NoSockets()
if __name__=='__main__':
# No upstream lifespan: no eager schema reconciliation or hosted-room recovery.
# Data is an operator-prepared independent snapshot, NEVER the gateway's volume.
uvicorn.run(build_app(),host='0.0.0.0',port=9119,lifespan='off',access_log=False,log_level='warning')
+21
View File
@@ -0,0 +1,21 @@
// Execute inside disposable compiled backend runner sharing fixture network namespace.
const assert=require('node:assert/strict');
const {HermesSessionBroker}=require('/opt/backend/dist/hermes-management/hermes-session-broker');
const {HermesAdapter}=require('/opt/backend/dist/hermes-management/hermes.adapter');
const {HermesIntegrationService}=require('/opt/backend/dist/hermes-management/hermes-integration.service');
const fetcher=(input,options)=>{const url=new URL(String(input));url.hostname='127.0.0.1';return fetch(url,options);};
(async()=>{
let reads=0;
const broker=new HermesSessionBroker(async()=>{reads++;return {username:'isolated-test-only-hermes_username'+'x'.repeat(32),password:'isolated-test-only-hermes_password'+'x'.repeat(32)};},fetcher);
const adapter=new HermesAdapter({origin:'http://canina-hermes-inventory:9119',token:()=>broker.token()},fetcher);
const runtime=new HermesIntegrationService(adapter,{authorize:async(p,action,profile)=>p.subject==='test-admin'&&action==='runtime.read'&&(!profile||profile==='default')});
await assert.rejects(()=>runtime.health({subject:'denied',profiles:[]}),{code:'FORBIDDEN'});assert.equal(reads,0);
const principal={subject:'test-admin',profiles:['default']};
assert.equal((await runtime.health(principal)).auth_required,true);
const profiles=await runtime.profiles(principal);assert(profiles.some(p=>p.name==='default'));assert(profiles.every(p=>p.name==='default'));
await assert.rejects(()=>runtime.status(principal,'outside'),{code:'FORBIDDEN'});
const status=await runtime.status(principal,'default');assert.equal(status.overall,'unknown');assert.equal(status.liveActiveAgents,null);assert.equal(status.provenance.source,'snapshot');
assert.equal(profiles[0].provenance.source,'snapshot');assert.equal(profiles[0].liveGatewayState,'unknown');
assert.equal(reads,1);
console.log('PASS real Node broker -> pinned TCP Hermes login -> adapter -> policy-filtered profiles; deny performs zero secret reads; cache coalesced');
})().catch(e=>{console.error(e.code||'contract failure');process.exitCode=1;});
@@ -0,0 +1,30 @@
"""Pinned image HTTP contract, no live volumes/network or provider calls."""
import os, secrets, tempfile
from pathlib import Path
os.environ['HERMES_HOME']=tempfile.mkdtemp(prefix='runtime-contract-')
os.environ['HERMES_SERVE_HEADLESS']='1'
from fastapi.testclient import TestClient
from plugins.dashboard_auth.basic import BasicAuthProvider, hash_password
from hermes_cli.dashboard_auth.registry import register_global_provider
from hermes_cli import web_server
password=secrets.token_urlsafe(32)
provider=BasicAuthProvider(username='contract',password_hash=hash_password(password),secret=secrets.token_bytes(32))
register_global_provider(provider)
web_server.app.state.auth_required=True
web_server.app.state.bound_host='0.0.0.0'
# No lifespan: avoids hosted-room recovery, cron/MCP and schema reconciliation threads.
client=TestClient(web_server.app,base_url='http://localhost')
def test_protected_inventory():
assert client.get('/api/profiles').status_code==401
assert client.post('/auth/password-login',json={'provider':'basic','username':'contract','password':'wrong'}).status_code==401
response=client.post('/auth/password-login',json={'provider':'basic','username':'contract','password':password})
assert response.status_code==200 and response.json()['ok'] is True
token=client.cookies.get('hermes_session_at')
assert token and provider.verify_session(access_token=token)
assert 'HttpOnly' in response.headers['set-cookie']
response=client.get('/api/profiles',headers={'Authorization':'Bearer '+token})
assert response.status_code==200 and any(p['name']=='default' for p in response.json()['profiles'])
client.cookies.clear()
assert client.get('/api/profiles',headers={'Authorization':'Bearer tampered'}).status_code==401
print('PASS exact-image HTTP wrong-login=401 login=200 HttpOnly verified-token protected-profiles=200 tampered=401')
if __name__=='__main__':test_protected_inventory()
+65
View File
@@ -0,0 +1,65 @@
import importlib.util, os, secrets, tempfile, threading, socket, time
from pathlib import Path
from fastapi.testclient import TestClient
os.environ['HERMES_HOME']=tempfile.mkdtemp(prefix='inventory-snapshot-')
os.environ['HERMES_SERVE_HEADLESS']='1'
spec=importlib.util.spec_from_file_location('runtime_serve','/contract/serve.py')
module=importlib.util.module_from_spec(spec);spec.loader.exec_module(module)
values={'hermes_username':'contract','hermes_password':secrets.token_urlsafe(32),'hermes_signing_key':secrets.token_urlsafe(32)}
module.secret=lambda name:values[name]
# A capture timestamp is supplied by a reviewed exporter, never inferred at boot.
import json
manifest=Path(os.environ['HERMES_HOME'])/'snapshot-manifest.json'
for invalid in [None, {}, {'schemaVersion':1,'source':'snapshot','capturedAt':'not-a-date'}, {'schemaVersion':1,'source':'snapshot','capturedAt':'2999-01-01T00:00:00Z'}]:
if invalid is None:
manifest.unlink(missing_ok=True)
else:
manifest.write_text(json.dumps(invalid))
try:
module.build_app()
except (RuntimeError, ValueError):
pass
else:
raise AssertionError('invalid or missing manifest accepted')
manifest.write_text(json.dumps({'schemaVersion':1,'source':'snapshot','capturedAt':'2020-01-01T00:00:00Z'}))
app=module.build_app()
import uvicorn
server=uvicorn.Server(uvicorn.Config(app,host='0.0.0.0',port=9119,lifespan='off',access_log=False,log_level='warning'))
thread=threading.Thread(target=server.run,daemon=True);thread.start()
for _ in range(200):
if server.started:break
time.sleep(.05)
assert server.started
import httpx
with httpx.Client(base_url='http://127.0.0.1:9119') as c:
assert c.get('/api/profiles').status_code==401
assert c.post('/auth/password-login',json={'provider':'basic','username':values['hermes_username'],'password':'wrong'}).status_code==401
r=c.post('/auth/password-login',json={'provider':'basic','username':values['hermes_username'],'password':values['hermes_password']})
assert r.status_code==200 and r.json()['ok'] is True
assert c.get('/api/profiles').status_code==200
health=c.get('/api/health').json()
assert health['auth_required'] is True
assert health['provenance']['source']=='snapshot'
assert health['provenance']['capturedAt']=='2020-01-01T00:00:00Z'
assert health['provenance']['freshness']=='stale'
assert health['provenance']['liveProcess']=='unknown'
profiles=c.get('/api/profiles').json()['profiles']
assert profiles and all(p['liveGatewayState']=='unknown' and p['gateway_running'] is False and p['provenance']['source']=='snapshot' for p in profiles)
status=c.get('/api/status',params={'profile':'default'}).json()
assert status['overall']=='unknown' and status['gateway_state'] is None
assert status['gateway_running'] is False and status['active_agents']==0
assert status['liveActiveAgents'] is None and status['liveGatewayState']=='unknown'
for path in ['/api/model/info','/api/analytics/usage','/api/sessions','/api/cron/jobs']:
response=c.get(path,params={'profile':'default'})
if response.status_code==200:
data=response.json()
if isinstance(data,list):
assert all(r['provenance']['source']=='snapshot' for r in data)
else:
assert data['provenance']['source']=='snapshot'
assert c.post('/api/cron/jobs/example/resume').status_code==404
assert c.get('/api/config').status_code==404
assert c.get('/api/ws').status_code==404
print('PASS real TCP pinned serve: unauth401 wrong-password401 login200 inventory200 health.auth_required=true mutations/config/ws404')
server.should_exit=True;thread.join(10)
assert not thread.is_alive()
+21
View File
@@ -0,0 +1,21 @@
import os,tempfile,importlib.util
os.environ['HERMES_HOME']=tempfile.mkdtemp()
from pathlib import Path
import json
(Path(os.environ['HERMES_HOME'])/'snapshot-manifest.json').write_text(json.dumps({'schemaVersion':1,'source':'snapshot','capturedAt':'2020-01-01T00:00:00Z'}))
from fastapi.testclient import TestClient
from starlette.websockets import WebSocketDisconnect
spec=importlib.util.spec_from_file_location('runtime_serve','/contract/serve.py');m=importlib.util.module_from_spec(spec);spec.loader.exec_module(m)
m.secret=lambda name:'test-only-'+'x'*32
app=m.build_app()
# A permitted session must NOT unlock powerful WebSocket routes.
client=TestClient(app)
r=client.post('/auth/password-login',json={'provider':'basic','username':m.secret('hermes_username'),'password':m.secret('hermes_password')})
assert r.status_code==200
for path in ['/api/ws','/api/pty','/api/events']:
try:
with client.websocket_connect(path):
raise AssertionError('Read-only boundary admitted socket '+path)
except WebSocketDisconnect as e:
assert e.code==1008, (path,e.code)
print('PASS authenticated sockets denied1008')
+28
View File
@@ -0,0 +1,28 @@
# Bounded PREPARE worker transport (not an LLM pilot)
This is an opt-in Nest dynamic module and one-shot deterministic Node executor. Nothing imports it into AppModule. No live service, credential, dispatch schedule or deployment is created.
## Required parent wiring
- Import `WorkerTransportModule.register(config)` from `backend/src/orchestration-worker/worker-transport.module.ts` in the parent application only after review. Default disabled; provide empty grants until secret provisioning. The module owns its own PrepareWorkflowService/config; do not broaden existing orchestration providers.
- Provision >=32 random-byte service credential through a mounted secret `/run/secrets/prepare_worker_credential` to the pilot, and provision only its SHA256 in server grants. Never accept identity/role from request bodies, JWT browser users, headers other than this credential, or a snapshot inventory.
- `grants` is an explicit digest -> `{id,role}` mapping with independent `workerIds`/`stageCiIds`. Worker and CI identities/digests cannot overlap. Use a separate CI-only secret for `STAGE_CI`; do not give it to this executor.
- Fixed config: absolute private writable `artifactDirectory`, authenticated HTTPS `artifactBaseUrl` pointing to `/internal/prepare-worker/artifacts`, separate persistent artifact volume. The fixture host `isolated.invalid` is NOT a real uploaded artifact URI. TLS routing/URI reachability must be verified by parent Stage CI.
- Restrict routes to service network, TLS, ingress body limit (8KiB recommended), rate limit, no browser CORS credentials, no public user JWT access. The module validates strict DTO fields, but production ingress/body limits/rate controls remain parent's wiring.
- Run `node infra/hermes-worker/pilot.mjs` with explicit `PREPARE_TASK_ID` UUID and `PREPARE_WORKER_BASE_URL` HTTPS origin. It consumes exactly one task/lease, has 5s per HTTP timeout, does not retry, and exits on SIGINT/SIGTERM. No polling or global dispatch. Disable all egress except backend; do not mount shell/Docker/browser credentials or repo write access. Loopback HTTP is available only via the exported test API option, not CLI.
## Contract and bounds
POST `/internal/prepare-worker/claim`: `{taskId}` -> `{lease:null|{...}}`; heartbeat: `{taskId,runId,leaseToken}`; complete: same binding plus `report` (max 2048 chars). Server accepts only byte-exact deterministic tracer JSON derived from the immutable run; it computes SHA256, writes real content, and chooses artifact URI/cost (0 cents), not client-supplied costs/URIs/pass flags. It does **not** inspect repository contents or validate application acceptance criteria. QA here means this harmless tracer succeeded, not application QA.
Core service still owns transactional Gate1, scope, kill checks, task locks, lease fencing, reservations and VERIFYING event. Transport requires exactly 20-cent task budgets and explicit task IDs. One expired unknown-cost reservation exhausts this pilot budget; do not release it to permit retries. Interrupt means cancellation/non-RUNNING state or STOP_AGENTS; heartbeat before compute, before complete, and atomic completion recheck. File writes may leave unregistered orphan content if the final transaction is denied; retrieval requires a registered QA artifact and verifies its bytes against the requested digest.
POST `/internal/prepare-worker/stage-evidence` has a separate strict CI DTO `{taskId,runId,commitSha,sha256,evidenceSha256,uri}` and calls independently authenticated core Stage evidence ingestion. This tracer never submits it. Valid CI evidence retrieval/provenance remains CI's responsibility. No Gate2 approval/release transport exists.
Shared department budgets are not implemented by core and are not claimed here. Do not expand beyond a single bounded pilot until aggregate budgeting and live provisioning are reviewed.
## Real isolated verification
Run `bash infra/hermes-worker/test-isolated.sh green`. It creates no-network disposable PostgreSQL16, applies only the three repository v4 migrations, copies readonly source and Node22 dependency donor into private writable storage, regenerates Prisma, runs actual Nest TCP HTTP and a child Node executor, then full backend compilation. Docker DB/anonymous volume removed on exit. No live tables/containers/volumes, RELEASE_AUTHORIZE, push or deploy touched.
Evidence: `/opt/services/canina/validation/v4-worker-transport/`: `red.log` (missing module initial RED), `integrity-red.log` (real 200 vs expected409 corruption regression), `green.log`, `build.log`, `generate.log`, `tracer-evidence.json` (actual artifact content/hash). Final tests cover credential/body identity denial, disabled/overlap config, Gate1 no-dispatch, competing claims, expired/token fencing, retained20-cent budget, interrupt/kill rechecks, CI separation, real executor to VERIFYING, authenticated artifact GET and tamper rejection. Fixture commit is 40 `a` characters; this is not a repository QA claim or live Stage deployment.
+34
View File
@@ -0,0 +1,34 @@
# Worker transport security review
Scope: f7048ab (parent equivalent aa99216). Narrow transport-only hardening; no orchestration/core/AppModule changes, release authority, live provisioning or deployment.
## Fixed
- Authenticated artifact retrieval now requires a local fixed-base QA registration, successful staging PREPARE run, matching immutable commit, 20-cent task, and DONE outbox bound to that exact run. Workers can read only their own outbox/run; separately provisioned Stage CI can read bounded completed tracer artifacts for independent verification. Foreign QA registrations and arbitrary authenticated workers do not suffice. Returned bytes must match both content hash and deterministic run-bound tracer report.
- Artifact base is canonical HTTPS with the exact `/internal/prepare-worker/artifacts` path: no userinfo, query, fragment, alternate port, IP literals, localhost/private-name suffixes, escaped or normalized arbitrary paths. This is static reviewed configuration, not a fetched URL. DNS resolution/reachability is deliberately not attempted; the operator must review the fixed origin and DNS/TLS routing.
- Configuration requires an actual boolean enable flag, non-overlapping valid allowlists and valid unique credential grants. Both transport snapshot and workflow config are frozen, including arrays and grant objects; caller mutation cannot change the snapshot.
- Linux descriptor-relative directory traversal with O_DIRECTORY/O_NOFOLLOW prevents symlinked path components and pins directories across renames. Final directory must be private and owned by process UID. File creation uses exclusive O_CREAT/O_EXCL/O_NOFOLLOW, never truncating existing symlinks, hardlinks or regular files. Reads use a pinned O_NOFOLLOW/O_NONBLOCK descriptor, require a private owned regular single-link file, and are bounded to 2049 bytes before digest/run binding verification.
## Preserved and verified
Bearer service digest identity, strict DTO/body impersonation rejection, independent CI role, explicit task-only claim (no queue polling), exact 20-cent cap, Gate1 gating, lease owner/token/task/run fences, unknown-cost reservation exhaustion, STOP_AGENTS and cancellation checks, atomic core completion recheck. No network fetch of artifact URLs was introduced.
The executor remains a deterministic JSON tracer: no LLM, shell, browser, deployment, repository inspection or real application QA. Successful tracer QA is not acceptance-criteria verification.
## Evidence
Actual disposable PostgreSQL16/no-network Docker fixture, Nest TCP HTTP listener, child Node22 executor and full backend `tsc -b` build: `/opt/services/canina/validation/v4-worker-security/`.
- `auth-red.log`: unrelated registered worker received 200 instead of expected 404.
- `config-red.log`: unsafe credential-bearing base accepted.
- `file-red.log`: symlink target sentinel overwritten by deterministic report.
- `freeze-red.log`: workflow config remained mutable.
- `security-green.log`: all 7 tests pass, including own-worker/CI retrieval, foreign URI denial, directory/file symlink and hardlink rejection, tamper integrity rejection, kill/lease/task fencing and actual child executor to VERIFYING.
- `build.log`: full backend `tsc -b` exit 0.
- `tracer-evidence.json`: actual fixture task/run/hash/content, GET200, cost0. External URI reachability explicitly NOT_TESTED_ISOLATED_FIXTURE_HOST.
## Limits / residual responsibilities
Linux `/proc/self/fd` is required; unsupported platforms fail closed. Protect the artifact mount and its ancestry from same-UID/root attackers; they remain outside this process boundary. File and database cannot commit atomically: a killed/denied/competing completion can leave an unregistered orphan, deliberately unreadable and never overwritten automatically. No retry/cleanup redesign was added. Descriptor pinning prevents path redirection but does not promise availability during hostile renames. Symlink/hardlink swaps are tested; exhaustive adversarial scheduling is not claimed.
Parent still owns service-network restriction, reviewed origin/DNS/TLS reachability, body/rate limits, secret provisioning, Stage CI evidence provenance, aggregate department budgets and integration tests. All workers remain unwired/default-disabled; no live secrets or RELEASE_AUTHORIZE used.
+37
View File
@@ -0,0 +1,37 @@
import { readFile } from 'node:fs/promises';
import { pathToFileURL } from 'node:url';
/** One explicit task, one attempt, fixed computation; no shell, prompt, browser or LLM. */
export async function runOnce({baseUrl, credential, taskId, allowLoopbackHttp=false, signal}) {
const url = new URL(baseUrl);
if (url.username || url.password || url.search || url.hash || (url.protocol !== 'https:' && !(allowLoopbackHttp && url.protocol === 'http:' && ['127.0.0.1','[::1]'].includes(url.hostname)))) throw new Error('HTTPS service origin required');
if (!credential || credential.length < 32) throw new Error('Service credential required');
async function post(route, body) {
signal?.throwIfAborted();
const timeout=AbortSignal.timeout(5000);
const r=await fetch(new URL('/internal/prepare-worker/'+route,url), {method:'POST',redirect:'error',signal:signal ? AbortSignal.any([signal,timeout]) : timeout,headers:{authorization:'Bearer '+credential,'content-type':'application/json'},body:JSON.stringify(body)});
const text=await r.text();
if(text.length>8192) throw new Error('Oversized service response');
if(!r.ok) throw new Error('Worker request denied: '+r.status);
return JSON.parse(text);
}
const {lease}=await post('claim',{taskId});
if(!lease) return {state:'IDLE'};
if(lease.taskId!==taskId || lease.environment!=='staging' || lease.maxCostCents!==20 || !/^[a-f0-9]{40}$/.test(lease.commitSha)) throw new Error('Unbounded lease rejected');
const binding={taskId,runId:lease.runId,leaseToken:lease.leaseToken};
await post('heartbeat',binding); // stop/interrupt recheck before any computation
signal?.throwIfAborted();
const report=JSON.stringify({kind:'DETERMINISTIC_PREPARE_TRACER',schemaVersion:1,taskId,runId:lease.runId,commitSha:lease.commitSha,environment:'staging',llmPilot:false,browserExecuted:false,shellExecuted:false,deploymentExecuted:false,checks:['lease-bound identity','deterministic JSON serialization'],actualCents:0});
await post('heartbeat',binding); // stop/interrupt recheck immediately before completion
return post('complete',{...binding,report});
}
if (process.argv[1] && import.meta.url===pathToFileURL(process.argv[1]).href) {
const controller=new AbortController();
process.once('SIGINT',()=>controller.abort());
process.once('SIGTERM',()=>controller.abort());
try {
const credential=(await readFile('/run/secrets/prepare_worker_credential','utf8')).trim();
const result=await runOnce({baseUrl:process.env.PREPARE_WORKER_BASE_URL,credential,taskId:process.env.PREPARE_TASK_ID,signal:controller.signal});
console.log(JSON.stringify(result));
} catch { console.error('Bounded PREPARE tracer failed; no completion assumed'); process.exitCode=1; }
}
+23
View File
@@ -0,0 +1,23 @@
#!/usr/bin/env bash
set -euo pipefail
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
E="${WORKER_EVIDENCE_DIR:-/opt/services/canina/validation/v4-worker-transport}"
mkdir -p "$E"
DB="v4-worker-test-${RANDOM}"
trap 'docker rm -fv "$DB" >/dev/null 2>&1 || true' EXIT
docker run -d --name "$DB" --network none -e POSTGRES_PASSWORD=disposable -e POSTGRES_DB=worker postgres:16-alpine >/dev/null
for n in $(seq 1 60); do if docker exec "$DB" pg_isready -U postgres >/dev/null 2>&1; then break; fi; sleep 1; done
for migration in 20261022000000_add_orchestration_v4_core 20261022190000_harden_orchestration_ledger 20261023170000_prepare_workflow_outbox; do
docker exec -i "$DB" psql -v ON_ERROR_STOP=1 -U postgres -d worker < "$ROOT/backend/prisma/migrations/$migration/migration.sql" > "$E/$migration.log"
done
docker run --rm --userns=host --network "container:$DB" -e DATABASE_URL=postgresql://postgres:disposable@127.0.0.1:5432/worker -e WORKER_TRANSPORT_DB=1 -e TRACER_EVIDENCE_PATH=/evidence/tracer-evidence.json -e TMPDIR=/work/tmp -v "$ROOT:/source:ro" -v /opt/services/canina/worktrees/orchestration-v4-runtime/backend/node_modules:/deps:ro -v "$E:/evidence" node:22-bookworm bash -c '
set -euo pipefail
mkdir -p /work/backend /work/infra /work/tmp
cp -a /source/backend/. /work/backend/
cp -a /source/infra/. /work/infra/
cp -a /deps /work/backend/node_modules
cd /work/backend
./node_modules/.bin/prisma generate > /evidence/generate.log 2>&1
npm test -- --runInBand orchestration-worker > /evidence/'"${1:-green}"'.log 2>&1
npm run build > /evidence/build.log 2>&1
'
+45
View File
@@ -0,0 +1,45 @@
#!/usr/bin/env bash
# CI job side only. Never execute host deploy logic in the runner filesystem.
set -euo pipefail
set +x
fail() { printf 'ERROR: %s\n' "$1" >&2; exit 2; }
[[ ${GITHUB_ACTIONS:-${GITEA_ACTIONS:-}} == true && ${BRANCH:-} == develop && $# == 0 ]] || fail 'CI-only Stage deployment; Production is blocked'
[[ ${DEPLOY_SHA:-} =~ ^[0-9a-f]{40}$ ]] || fail 'immutable DEPLOY_SHA required'
# Repository secret DEPLOY_KEY is the historical approved devops SSH credential.
# Never fall back to baked-in runner keys, agent forwarding or docker.sock.
[[ -n ${DEPLOY_KNOWN_HOSTS:-} ]] || fail 'provision DEPLOY_KNOWN_HOSTS: independently verified [87.248.133.138]:2234 host key; never use ssh-keyscan trust-on-first-use'
[[ -n ${DEPLOY_KEY:-} ]] || fail 'provision existing approved DEPLOY_KEY for devops@87.248.133.138:2234'
for tool in ssh ssh-keygen mktemp; do command -v "$tool" >/dev/null || fail "job context missing $tool"; done
umask 077
scratch=$(mktemp -d)
trap 'rm -rf "$scratch"' EXIT
printf '%s\n' "$DEPLOY_KNOWN_HOSTS" > "$scratch/known_hosts"
printf '%s\n' "$DEPLOY_KEY" > "$scratch/key"
unset DEPLOY_KEY DEPLOY_KNOWN_HOSTS
ssh-keygen -F '[87.248.133.138]:2234' -f "$scratch/known_hosts" >/dev/null || fail 'DEPLOY_KNOWN_HOSTS has no pinned deployment endpoint'
# Only the validated SHA enters the remote command; no user-controlled shell text.
ssh -F /dev/null -T -p 2234 -i "$scratch/key" \
-o BatchMode=yes -o IdentitiesOnly=yes -o IdentityAgent=none \
-o StrictHostKeyChecking=yes -o UserKnownHostsFile="$scratch/known_hosts" \
-o GlobalKnownHostsFile=/dev/null -o ConnectTimeout=15 \
devops@87.248.133.138 "sudo -n bash -s -- $DEPLOY_SHA" <<'HOST'
set -euo pipefail
set +x
umask 077
fail() { printf 'ERROR: host context: %s\n' "$1" >&2; exit 2; }
[[ $# == 1 && $1 =~ ^[0-9a-f]{40}$ ]] || fail 'immutable SHA required'
[[ ! -e /.dockerenv && ! -e /run/.containerenv ]] || fail 'refusing container filesystem'
[[ $(id -u) == 0 ]] || fail 'noninteractive approved sudo required'
for tool in bash python3 docker git flock realpath install wget; do command -v "$tool" >/dev/null || fail "missing $tool"; done
[[ -d /opt/services/canina/stage/.git && -d /opt/services/proxy/certs/canina ]] || fail 'Stage operational checkout/certificates unavailable'
[[ -s /opt/services/canina/stage/.support-chat.env && ! -L /opt/services/canina/stage/.support-chat.env ]] || fail 'Stage runtime env unavailable'
docker compose version >/dev/null
docker volume inspect stage_canina_stage_db stage_canina_stage_uploads >/dev/null
docker network inspect traefik_public >/dev/null
printf 'Host context validated; Stage commit=%s; runtime env present; Production blocked\n' "$1"
script=$(mktemp)
trap 'rm -f "$script"' EXIT
wget --no-proxy -qO "$script" "https://git.parsaaghayi.ir/parsa/canina/raw/commit/${1}/scripts/deploy.sh"
# CI marker exists only across this authenticated, fixed Stage boundary.
GITHUB_ACTIONS=true DEPLOY_SHA="$1" bash "$script" develop
HOST
+85 -119
View File
@@ -1,137 +1,103 @@
#!/bin/bash
set -euo pipefail
set +x
# CI/CD-only Stage rollout; Production awaits separate reviewed authorization.
if [ "${GITHUB_ACTIONS:-${GITEA_ACTIONS:-}}" != true ] || [ "${1:-}" != develop ] || [ "$#" != 1 ]; then
echo 'ERROR: refusing deployment outside CI/CD Stage rollout' >&2
exit 2
fi
if [[ ! "${DEPLOY_SHA:-}" =~ ^[0-9a-f]{40}$ ]]; then
echo "ERROR: refusing deployment without an immutable commit SHA" >&2
echo 'ERROR: refusing deployment without an immutable commit SHA' >&2
exit 2
fi
# END ROLLOUT GUARD
exec 9>/opt/services/canina/stage-backend-deploy.lock
flock -n 9 || { echo 'Stage deployment already in progress' >&2; exit 1; }
export DOCKER_BUILDKIT=1
export COMPOSE_DOCKER_CLI_BUILD=1
# Ensure nexus is connected to default bridge network so docker build can reach it
docker network connect bridge nexus 2>/dev/null || true
# Nexus npm registry
NEXUS_IP=$(docker inspect -f '{{.NetworkSettings.Networks.bridge.IPAddress}}' nexus 2>/dev/null || true)
if [ -z "$NEXUS_IP" ]; then
NEXUS_IP="172.17.0.3"
fi
NPM_REGISTRY="http://${NEXUS_IP}:8081/repository/bitchpedar/"
BRANCH="${1:-develop}"
echo "Starting Deployment for branch: ${BRANCH}"
if [ "$BRANCH" == "develop" ]; then
COMPOSE_FILE="compose.stage.yml"
TARGET_DIR="/opt/services/canina/stage"
BACKEND_HOST="backend_stage"
APP_HOST="stage.canina.ir"
ADMIN_HOST="stageadmin.canina.ir"
API_URL="https://stageapi.canina.ir"
CONTAINER_PREFIX="canino_backend_stage"
else
COMPOSE_FILE="compose.prod.yml"
TARGET_DIR="/opt/services/canina/prod"
BACKEND_HOST="backend_prod"
APP_HOST="canina.ir"
ADMIN_HOST="admin.canina.ir"
API_URL="https://api.canina.ir"
CONTAINER_PREFIX="canino_backend_prod"
fi
mkdir -p "$TARGET_DIR"
if [ ! -d "$TARGET_DIR/.git" ]; then
echo "Cloning repository into $TARGET_DIR..."
git clone -b "$BRANCH" https://git.parsaaghayi.ir/parsa/canina.git "$TARGET_DIR"
git -C "$TARGET_DIR" cat-file -e "${DEPLOY_SHA}^{commit}"
git -C "$TARGET_DIR" merge-base --is-ancestor "$DEPLOY_SHA" "origin/$BRANCH"
git -C "$TARGET_DIR" reset --hard "$DEPLOY_SHA"
else
echo "Pulling latest changes in $TARGET_DIR..."
cd "$TARGET_DIR"
git fetch origin "$BRANCH"
git cat-file -e "${DEPLOY_SHA}^{commit}"
git merge-base --is-ancestor "$DEPLOY_SHA" "origin/$BRANCH"
git reset --hard "$DEPLOY_SHA"
cd -
fi
umask 077
export DOCKER_BUILDKIT=1 COMPOSE_DOCKER_CLI_BUILD=1
# Stage is an operational reference, NEVER a checkout/build target.
STAGE_DIR=/opt/services/canina/stage
RELEASE_ROOT=/opt/services/canina/releases
TARGET_DIR="$RELEASE_ROOT/$DEPLOY_SHA"
mkdir -p "$RELEASE_ROOT"
[ ! -L "$RELEASE_ROOT" ] && [ "$(realpath "$RELEASE_ROOT")" = "$RELEASE_ROOT" ]
# Do not overwrite a prior release or retry with stale generated configuration.
mkdir "$TARGET_DIR"
git clone --no-checkout --single-branch --branch develop https://git.parsaaghayi.ir/parsa/canina.git "$TARGET_DIR"
cd "$TARGET_DIR"
# Capture the bot key from Docker before recreating the bot; runner mounts may not include the host env file.
if docker inspect canina-bot >/dev/null 2>&1; then
LIVE_BOT_KEY=$(docker inspect -f '{{range .Config.Env}}{{println .}}{{end}}' canina-bot | awk -F= '$1 == "SUPPORT_INTERNAL_KEY" {print substr($0, index($0, "=") + 1); exit}')
if [ -n "$LIVE_BOT_KEY" ]; then
grep -v '^SUPPORT_CHAT_KEY=' .support-chat.env > .support-chat.env.tmp 2>/dev/null || true
printf 'SUPPORT_CHAT_KEY=%s
' "$LIVE_BOT_KEY" >> .support-chat.env.tmp
mv .support-chat.env.tmp .support-chat.env
chmod 600 .support-chat.env
git cat-file -e "${DEPLOY_SHA}^{commit}"
git merge-base --is-ancestor "$DEPLOY_SHA" origin/develop
git checkout --detach "$DEPLOY_SHA"
[ "$(git rev-parse HEAD)" = "$DEPLOY_SHA" ]
# Keep operational files out of status/accidental commits in this disposable clone.
printf '\n.support-chat.env\n.revalidation.env\n.stage-base.json\n.stage-compose.json\n' >> .git/info/exclude
# Runtime secrets/config must never enter the frontend Docker build context.
printf '\n.support-chat.env\n.revalidation.env\n.stage-*.json\n' >> .dockerignore
# Only these backend/frontend runtime env files are required. Do not copy the
# dirty compose, .env, uploads, bot state, certificates, or arbitrary Stage files.
for name in .support-chat.env .revalidation.env; do
if [ -e "$STAGE_DIR/$name" ] || [ -L "$STAGE_DIR/$name" ]; then
[ -f "$STAGE_DIR/$name" ] && [ ! -L "$STAGE_DIR/$name" ]
[ -z "$(git -C "$STAGE_DIR" ls-files -- "$name")" ]
[ -z "$(git ls-files -- "$name")" ]
install -m 600 "$STAGE_DIR/$name" "$TARGET_DIR/$name"
fi
unset LIVE_BOT_KEY
fi
# Inject support-chat runtime configuration from CI/CD without committing secrets.
# The bot is the runtime owner of the internal key; synchronize only the
# matching value into the colocated backend env file, never into Git or logs.
BOT_ENV="/opt/services/ai/canina-bot/support.env"
if [ -r "$BOT_ENV" ]; then
BOT_KEY=$(awk -F= '$1 == "SUPPORT_INTERNAL_KEY" {print substr($0, index($0, "=") + 1); exit}' "$BOT_ENV")
if [ -n "$BOT_KEY" ]; then
touch .support-chat.env
grep -v '^SUPPORT_CHAT_KEY=' .support-chat.env > .support-chat.env.tmp || true
printf "SUPPORT_CHAT_KEY=%s\n" "$BOT_KEY" >> .support-chat.env.tmp
mv .support-chat.env.tmp .support-chat.env
chmod 600 .support-chat.env
done
[ -s .support-chat.env ] || { echo 'ERROR: missing Stage support runtime env' >&2; exit 1; }
# Never inspect/recreate/mutate the shared support bot or infer keys from it.
# Certs remain the explicit operational absolute bind from tracked compose.
python3 - <<'PY'
from pathlib import Path
p = Path('nginx.conf')
s = p.read_text()
for key, value in {'__BACKEND_HOST__': 'backend_stage', '__APP_HOST__': 'stage.canina.ir', '__ADMIN_HOST__': 'stageadmin.canina.ir'}.items():
s = s.replace(key, value)
p.write_text(s)
PY
# Wrapper also covers every compose call in the sourced rollout helper. Explicit
# project/files/env-file prevent release-SHA projects and ambient .env loading.
docker() {
if [ "${1:-}" = compose ]; then
shift
command docker compose -p stage --env-file /dev/null --project-directory "$TARGET_DIR" -f "$TARGET_DIR/.stage-compose.json" "$@"
else
command docker "$@"
fi
unset BOT_KEY
fi
if [ -n "${SUPPORT_CHAT_KEY:-}" ]; then
touch .support-chat.env
grep -v '^SUPPORT_CHAT_KEY=' .support-chat.env > .support-chat.env.tmp || true
printf 'SUPPORT_CHAT_KEY=%s\n' "$SUPPORT_CHAT_KEY" >> .support-chat.env.tmp
mv .support-chat.env.tmp .support-chat.env
chmod 600 .support-chat.env
fi
if [ ! -s .support-chat.env ]; then
echo "ERROR: missing .support-chat.env / SUPPORT_CHAT_KEY; refusing deployment" >&2
exit 1
fi
# ✅ فایل‌های کانفیگ را از پوشه‌ی scripts مخزن به TARGET_DIR کپی می‌کنیم
echo "Copying configuration files in $TARGET_DIR..."
cp "scripts/$COMPOSE_FILE" "docker-compose.yml"
sed -i 's/- REDIS_PORT: 6379/- REDIS_PORT=6379/g' "docker-compose.yml" 2>/dev/null || true
echo "Patching nginx.conf for environment..."
sed -i "s/__BACKEND_HOST__/${BACKEND_HOST}/g" "nginx.conf"
sed -i "s/__APP_HOST__/${APP_HOST}/g" "nginx.conf"
sed -i "s/__ADMIN_HOST__/${ADMIN_HOST}/g" "nginx.conf"
echo "Using Nexus npm proxy: $NPM_REGISTRY"
NPM_REGISTRY_ARG="--build-arg NPM_REGISTRY=${NPM_REGISTRY}"
echo "Building Docker images..."
docker compose --progress plain build \
--build-arg NEXT_PUBLIC_API_URL="${API_URL}/api" \
--build-arg VITE_API_URL="${API_URL}" \
${NPM_REGISTRY_ARG}
# Never recreate the shared Production support bot during a Stage rollout.
}
# Render privately: config contains env secrets, so never emit it to CI logs.
command docker compose -p stage --env-file /dev/null --project-directory "$TARGET_DIR" -f scripts/compose.stage.yml config --format json > .stage-base.json
python3 - <<'PY'
import json
from pathlib import Path
config = json.loads(Path('.stage-base.json').read_text())
expected = {'canina_stage_db': 'stage_canina_stage_db', 'canina_stage_uploads': 'stage_canina_stage_uploads'}
if set(config['services']) != {'db_stage', 'redis_stage', 'backend_stage', 'frontend_stage'}:
raise SystemExit('ERROR: unexpected Stage service set')
for key, name in expected.items():
if config['volumes'][key].get('name') != name:
raise SystemExit('ERROR: Stage volume identity mismatch')
config['volumes'][key] = {'name': name, 'external': True}
if config['networks']['traefik_public'].get('name') != 'traefik_public' or not config['networks']['traefik_public'].get('external'):
raise SystemExit('ERROR: Stage network identity mismatch')
backend = config['services']['backend_stage']
if backend['container_name'] != 'canino_backend_stage' or config['services']['frontend_stage']['container_name'] != 'canino_frontend_stage':
raise SystemExit('ERROR: Stage container identity mismatch')
mounts = backend.get('volumes', [])
if not any(m.get('type') == 'volume' and m.get('source') == 'canina_stage_uploads' and m.get('target') == '/app/uploads' for m in mounts):
raise SystemExit('ERROR: missing Stage uploads volume')
for service in config['services'].values():
for mount in service.get('volumes', []):
if mount.get('type') == 'bind' and (mount.get('source'), mount.get('target')) != ('/opt/services/proxy/certs/canina', '/app/ssl_certs'):
raise SystemExit('ERROR: unexpected Stage bind mount')
Path('.stage-compose.json').write_text(json.dumps(config))
Path('.stage-base.json').unlink()
PY
# Read-only checks: missing external storage must fail, never provision blank DB.
docker volume inspect stage_canina_stage_db stage_canina_stage_uploads >/dev/null
docker network inspect traefik_public >/dev/null
[ -d /opt/services/proxy/certs/canina ]
# Builds only app images; DB/Redis and support-bot remain untouched.
docker compose --progress plain build backend_stage frontend_stage
source scripts/rollout-stage.sh
stage_rollout
# Auto-cleanup old docker builder cache to keep disk healthy (keeps latest 6GB)
echo "Cleaning up old docker build cache..."
docker builder prune -af --keep-storage 6GB 2>/dev/null || true
docker image prune -f 2>/dev/null || true
echo "Deployment for $BRANCH completed successfully!"
echo 'Deployment for develop completed successfully!'
+2
View File
@@ -27,6 +27,8 @@ with tempfile.TemporaryDirectory(prefix='canina-rollout-') as d:
query = "const {PrismaClient}=require('@prisma/client');const p=new PrismaClient();(async()=>{await p.product.findMany({take:1,include:{ingredientList:true,symptoms:true}});await p.supportConversation.findMany({take:1});await p.orchestrationTask.findMany({take:1,include:{runs:true,events:true,budgetLedger:true}});console.log('PASS: actual Prisma legacy catalog/support and v4 relation queries');})().catch(()=>{console.error('Prisma compatibility failed');process.exitCode=1}).finally(()=>p.$disconnect());"
run(cmd[:-3]+['node','-e',query])
after=sql("SELECT table_name,column_name,data_type,is_nullable,column_default FROM information_schema.columns WHERE table_schema='public' AND table_name <> '_prisma_migrations' AND table_name NOT LIKE 'orchestration_%' ORDER BY 1,2;")
# Compare only the exact pre-existing table set; new additive tables are not legacy.
after='\n'.join(line for line in after.splitlines() if line.split('|',1)[0] in tables)+'\n'
assert before==after,'Legacy column definitions changed'
for table in tables: sql(f'SELECT * FROM "{table}" LIMIT 0;')
v4=sql("SELECT tablename FROM pg_tables WHERE schemaname='public' AND tablename LIKE 'orchestration_%' ORDER BY 1;").splitlines()
+50
View File
@@ -0,0 +1,50 @@
import os, pathlib, subprocess, tempfile, unittest
ROOT = pathlib.Path(__file__).resolve().parents[2]
class BoundaryTests(unittest.TestCase):
def run_wrapper(self, **updates):
env = dict(os.environ, GITHUB_ACTIONS='true', BRANCH='develop', DEPLOY_SHA='a'*40)
for name in ('DEPLOY_KEY','DEPLOY_KNOWN_HOSTS'): env.pop(name, None)
env.update(updates)
return subprocess.run(['bash', str(ROOT/'scripts/ci-host-deploy.sh')], env=env, capture_output=True, text=True)
def test_missing_pinned_host_identity_fails_before_network(self):
result = self.run_wrapper(DEPLOY_KEY='sentinel-secret')
self.assertNotEqual(result.returncode, 0)
self.assertIn('DEPLOY_KNOWN_HOSTS', result.stderr)
self.assertNotIn('sentinel-secret', result.stdout+result.stderr)
def test_guards_refuse_non_ci_production_and_injection(self):
for updates in ({'GITHUB_ACTIONS':'false'}, {'BRANCH':'main'}, {'DEPLOY_SHA':'a'*40+';touch bad'}, {'DEPLOY_SHA':''}):
result = self.run_wrapper(**updates)
self.assertEqual(result.returncode, 2)
self.assertNotIn('provision', result.stderr)
def test_authentication_is_noninteractive_and_sha_is_only_remote_argument(self):
with tempfile.TemporaryDirectory() as d:
tools = pathlib.Path(d)
# Deliberate transport double: validates invocation, never a live deploy.
(tools/'ssh').write_text('#!/bin/bash\nprintf "%s\\n" "$@" > "$TRACE"\n/bin/cat > "$PAYLOAD"\nexit "${SSH_RESULT:-0}"\n')
(tools/'ssh').chmod(0o755)
known = '[87.248.133.138]:2234 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA'
for code in ('0','255'):
result = self.run_wrapper(DEPLOY_KEY='sentinel-secret', DEPLOY_KNOWN_HOSTS=known, PATH=d+':'+os.environ['PATH'], TRACE=d+'/trace', PAYLOAD=d+'/payload', SSH_RESULT=code)
self.assertEqual(result.returncode, int(code), result.stderr)
trace = (tools/'trace').read_text()
for option in ('StrictHostKeyChecking=yes','BatchMode=yes','IdentitiesOnly=yes','IdentityAgent=none','devops@87.248.133.138','sudo -n bash -s -- '+'a'*40): self.assertIn(option,trace)
self.assertNotIn('sentinel-secret', trace+result.stdout+result.stderr)
lines = trace.splitlines()
key = pathlib.Path(lines[lines.index('-i')+1])
self.assertFalse(key.exists(), 'temporary credential must be cleaned even on SSH failure')
payload = (tools/'payload').read_text()
self.assertIn('/raw/commit/${1}/scripts/deploy.sh', payload)
self.assertIn('python3',payload)
self.assertIn('refusing container filesystem',payload)
self.assertNotIn('docker run', payload)
self.assertNotIn('git reset', payload)
def test_workflow_uses_only_authenticated_wrapper(self):
workflow = (ROOT/'.gitea/workflows/deploy.yml').read_text()
self.assertIn('scripts/ci-host-deploy.sh',workflow)
self.assertIn('DEPLOY_KNOWN_HOSTS: ${{ secrets.DEPLOY_KNOWN_HOSTS }}',workflow)
self.assertIn('DEPLOY_KEY: ${{ secrets.DEPLOY_KEY }}',workflow)
self.assertNotIn('cd scripts',workflow)
self.assertNotIn('./deploy.sh',workflow)
if __name__ == '__main__': unittest.main()
+91 -5
View File
@@ -13,9 +13,12 @@ class RolloutTests(unittest.TestCase):
script=(ROOT/'scripts/deploy.sh').read_text()
workflow=(ROOT/'.gitea/workflows/deploy.yml').read_text()
self.assertIn('DEPLOY_SHA: ${{ github.sha }}',workflow)
self.assertIn('/raw/commit/${DEPLOY_SHA}/scripts/deploy.sh',workflow)
self.assertIn('git reset --hard "$DEPLOY_SHA"',script)
self.assertNotIn('git reset --hard "origin/$BRANCH"',script)
self.assertIn('/raw/commit/${DEPLOY_SHA}/scripts/ci-host-deploy.sh',workflow)
wrapper=(ROOT/'scripts/ci-host-deploy.sh').read_text()
self.assertIn('/raw/commit/${1}/scripts/deploy.sh',wrapper)
self.assertNotIn('git reset',script)
self.assertIn('git checkout --detach "$DEPLOY_SHA"',script)
self.assertIn('/opt/services/canina/releases',script)
def test_commit_guard_rejects_missing_and_malformed_sha_before_side_effects(self):
guard=(ROOT/'scripts/deploy.sh').read_text().split('# END ROLLOUT GUARD')[0]
@@ -33,7 +36,91 @@ class RolloutTests(unittest.TestCase):
workflow=(ROOT/'.gitea/workflows/deploy.yml').read_text()
run=workflow.split(' run: |',1)[1]
self.assertEqual(run.strip().splitlines()[0].strip(),'set -euo pipefail')
self.assertIn('./deploy.sh "${BRANCH}"',run)
self.assertIn('bash "$wrapper"',run)
import textwrap
with tempfile.TemporaryDirectory() as d:
tools=pathlib.Path(d)
(tools/'wget').write_text('#!/bin/sh\nexit 42\n')
(tools/'wget').chmod(0o755)
# Execute the real job block; failed download must not launch wrapper.
result=subprocess.run(['bash','-c',textwrap.dedent(run)], env=dict(os.environ, PATH=d+':'+os.environ['PATH'], BRANCH='develop', DEPLOY_SHA='a'*40, DEPLOY_KEY='test-not-a-key', DEPLOY_KNOWN_HOSTS='test-pin'), capture_output=True,text=True)
self.assertEqual(result.returncode,42,result.stderr)
def test_isolated_deploy_preserves_dirty_stage_and_stable_storage(self):
import json, shutil
with tempfile.TemporaryDirectory() as directory:
base = pathlib.Path(directory)
source, stage, releases, tools = [base / n for n in ('source', 'stage', 'releases', 'tools')]
for p in (source, stage, tools): p.mkdir()
def git(*args, cwd=source):
return subprocess.check_output(['git', *args], cwd=cwd, text=True).strip()
git('init', '-b', 'develop')
git('config', 'user.email', 'test@example.invalid')
git('config', 'user.name', 'Test')
shutil.copytree(ROOT/'scripts', source/'scripts')
shutil.copyfile(ROOT/'.dockerignore', source/'.dockerignore')
(source/'nginx.conf').write_text('__BACKEND_HOST__')
git('add', '.')
git('commit', '-qm', 'fixture')
sha = git('rev-parse', 'HEAD')
git('init', '-b', 'develop', cwd=stage)
git('config', 'user.email', 'test@example.invalid', cwd=stage)
git('config', 'user.name', 'Test', cwd=stage)
(stage/'dirty').write_text('original')
git('add', '.', cwd=stage)
git('commit', '-qm', 'stage', cwd=stage)
(stage/'dirty').write_text('staged')
git('add', '.', cwd=stage)
(stage/'dirty').write_text('unstaged')
for n in ('.support-chat.env', '.revalidation.env'):
(stage/n).write_text('TEST_SECRET=never-log-this\n')
before = git('status', '--porcelain', cwd=stage), git('diff', '--binary', 'HEAD', cwd=stage), (stage/'.git/index').read_bytes()
config = {'services': {
'db_stage': {'volumes': [{'type': 'volume', 'source': 'canina_stage_db', 'target': '/var/lib/postgresql/data'}]},
'redis_stage': {},
'backend_stage': {'container_name': 'canino_backend_stage', 'volumes': [
{'type': 'volume', 'source': 'canina_stage_uploads', 'target': '/app/uploads'},
{'type': 'bind', 'source': '/opt/services/proxy/certs/canina', 'target': '/app/ssl_certs'}]},
'frontend_stage': {'container_name': 'canino_frontend_stage'}},
'volumes': {n: {'name': 'stage_'+n} for n in ('canina_stage_db','canina_stage_uploads')},
'networks': {'traefik_public': {'name': 'traefik_public', 'external': True}}}
(base/'config.json').write_text(json.dumps(config))
(tools/'docker').write_text('#!/bin/bash\nprintf "%s\\n" "$*" >> "$TRACE"\nif [[ "$*" == *"config --format json"* ]]; then /bin/cat "$FIXTURE"; fi\nif [[ "$*" == *"run --rm"* && "${FAIL_MIGRATION:-}" == 1 ]]; then exit 42; fi\n')
(tools/'docker').chmod(0o755)
script = (ROOT/'scripts/deploy.sh').read_text()
script = script.replace('/opt/services/canina/stage-backend-deploy.lock', str(base/'lock')).replace('/opt/services/canina/stage', str(stage)).replace('/opt/services/canina/releases', str(releases)).replace('https://git.parsaaghayi.ir/parsa/canina.git', str(source)).replace('[ -d /opt/services/proxy/certs/canina ]', 'true')
deploy = base/'deploy.sh'
deploy.write_text(script)
env = dict(os.environ, GITHUB_ACTIONS='true', DEPLOY_SHA=sha, PATH=str(tools)+':'+os.environ['PATH'], TRACE=str(base/'trace'), FIXTURE=str(base/'config.json'))
for fail in ('1', '0'):
if releases.exists(): shutil.rmtree(releases)
(base/'trace').write_text('')
result = subprocess.run(['bash', str(deploy), 'develop'], env=dict(env, FAIL_MIGRATION=fail), capture_output=True, text=True)
self.assertEqual(result.returncode, 42 if fail == '1' else 0, result.stderr)
trace = (base/'trace').read_text()
compose = [line for line in trace.splitlines() if line.startswith('compose ')]
self.assertTrue(compose)
self.assertTrue(all(line.startswith('compose -p stage --env-file /dev/null --project-directory '+str(releases/sha)) for line in compose))
self.assertNotIn('canina-bot', trace)
self.assertNotIn('remove-orphans', trace)
self.assertNotIn('up -d --no-deps db_stage', trace)
self.assertNotIn('never-log-this', result.stdout+result.stderr)
if fail == '1': self.assertNotIn('up -d', trace)
else:
self.assertLess(trace.index('migrate deploy'), trace.index('up -d --no-deps backend_stage'))
self.assertLess(trace.index('exec -T backend_stage'), trace.index('up -d --no-deps frontend_stage'))
release = releases/sha
self.assertEqual(git('rev-parse', 'HEAD', cwd=release), sha)
self.assertEqual((release/'.support-chat.env').stat().st_mode & 0o777, 0o600)
self.assertIn('.stage-*.json', (release/'.dockerignore').read_text())
self.assertIn('.support-chat.env', (release/'.dockerignore').read_text())
self.assertIn('.revalidation.env', (release/'.dockerignore').read_text())
self.assertEqual((release/'.stage-compose.json').stat().st_mode & 0o777, 0o600)
effective = json.loads((release/'.stage-compose.json').read_text())
self.assertEqual(effective['volumes'], {n: {'name': 'stage_'+n, 'external': True} for n in ('canina_stage_db','canina_stage_uploads')})
self.assertEqual(before, (git('status', '--porcelain', cwd=stage), git('diff', '--binary', 'HEAD', cwd=stage), (stage/'.git/index').read_bytes()))
retry = subprocess.run(['bash', str(deploy), 'develop'], env=env, capture_output=True, text=True)
self.assertNotEqual(retry.returncode, 0)
def test_migration_failure_never_starts_apps(self):
with tempfile.TemporaryDirectory() as d:
@@ -60,4 +147,3 @@ class RolloutTests(unittest.TestCase):
self.assertIn('exec -T backend_stage node',trace)
self.assertNotIn('up -d --no-deps frontend_stage',trace)
self.assertNotIn('SUCCESS',result.stdout)
if __name__=='__main__': unittest.main()