Compare commits
17
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
8cfd44cd09 | ||
|
|
c66bc58f73 | ||
|
|
4b147b0091 | ||
|
|
6950ff0d79 | ||
|
|
cc9ec25388 | ||
|
|
be2ff0719a | ||
|
|
9d93707848 | ||
|
|
aa99216c49 | ||
|
|
58d3794050 | ||
|
|
16f9610027 | ||
|
|
2023dd4e36 | ||
|
|
0bed6e0313 | ||
|
|
f29ba4ab8b | ||
|
|
56b6696ca3 | ||
|
|
0d0c67704c | ||
|
|
09e8a890e0 | ||
|
|
4b17a6fcc3 |
No files matched your search
+16
-16
@@ -10,25 +10,25 @@ jobs:
|
||||
deploy:
|
||||
runs-on: canina
|
||||
steps:
|
||||
- name: Run deploy.sh from within the scripts folder
|
||||
- name: Authenticated host Stage deployment (Production blocked)
|
||||
shell: bash
|
||||
env:
|
||||
BRANCH: ${{ github.ref_name }}
|
||||
DEPLOY_SHA: ${{ github.sha }}
|
||||
SUPPORT_CHAT_KEY: ${{ secrets.SUPPORT_CHAT_KEY }}
|
||||
DEPLOY_KEY: ${{ secrets.DEPLOY_KEY }}
|
||||
DEPLOY_KNOWN_HOSTS: ${{ secrets.DEPLOY_KNOWN_HOSTS }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
# 1. رفع مشکل DNS و غیرفعال کردن پروکسی برای دامنه محلی
|
||||
echo "87.248.133.138 git.parsaaghayi.ir" >> /etc/hosts
|
||||
export http_proxy=""
|
||||
export https_proxy=""
|
||||
export HTTP_PROXY=""
|
||||
export HTTPS_PROXY=""
|
||||
set +x
|
||||
[[ "${BRANCH}" == develop ]] || { echo 'Production blocked' >&2; exit 2; }
|
||||
[[ "${DEPLOY_SHA}" =~ ^[0-9a-f]{40}$ ]] || exit 2
|
||||
# Provision DEPLOY_KNOWN_HOSTS from an independently verified host key.
|
||||
# Never use baked-in /opt/services, runner SSH keys or docker.sock.
|
||||
[[ -n "${DEPLOY_KEY}" && -n "${DEPLOY_KNOWN_HOSTS}" ]] || { echo 'Host boundary requires DEPLOY_KEY and verified DEPLOY_KNOWN_HOSTS' >&2; exit 2; }
|
||||
export http_proxy="" https_proxy="" HTTP_PROXY="" HTTPS_PROXY=""
|
||||
export no_proxy="git.parsaaghayi.ir,localhost,127.0.0.1,87.248.133.138"
|
||||
|
||||
# 2. دانلود و اجرای اسکریپت deploy.sh بر اساس برنچ جاری
|
||||
mkdir -p scripts
|
||||
wget --no-proxy -qO scripts/deploy.sh https://git.parsaaghayi.ir/parsa/canina/raw/commit/${DEPLOY_SHA}/scripts/deploy.sh
|
||||
|
||||
chmod +x scripts/deploy.sh
|
||||
cd scripts
|
||||
./deploy.sh "${BRANCH}"
|
||||
umask 077
|
||||
wrapper=$(mktemp)
|
||||
trap 'rm -f "$wrapper"' EXIT
|
||||
wget --no-proxy -qO "$wrapper" "https://git.parsaaghayi.ir/parsa/canina/raw/commit/${DEPLOY_SHA}/scripts/ci-host-deploy.sh"
|
||||
bash "$wrapper"
|
||||
@@ -0,0 +1,10 @@
|
||||
CREATE TABLE prepare_workflow_outbox (
|
||||
id uuid PRIMARY KEY, task_id uuid NOT NULL UNIQUE REFERENCES orchestration_tasks(id) ON DELETE RESTRICT,
|
||||
status varchar(20) NOT NULL DEFAULT 'PENDING' CHECK (status IN ('PENDING','DONE')),
|
||||
attempts integer NOT NULL DEFAULT 0 CHECK (attempts BETWEEN 0 AND 3),
|
||||
lease_owner varchar(100), lease_token uuid, lease_until timestamptz, heartbeat_at timestamptz,
|
||||
run_id uuid REFERENCES orchestration_runs(id) ON DELETE RESTRICT,
|
||||
reservation_id uuid REFERENCES orchestration_budget_ledger(id) ON DELETE RESTRICT,
|
||||
created_at timestamptz NOT NULL DEFAULT now()
|
||||
);
|
||||
CREATE INDEX prepare_workflow_outbox_status_lease_until_idx ON prepare_workflow_outbox(status,lease_until);
|
||||
@@ -1232,3 +1232,20 @@ model OrchestrationKillSwitch {
|
||||
createdAt DateTime @default(now()) @map("created_at") @db.Timestamptz()
|
||||
@@map("orchestration_kill_switches")
|
||||
}
|
||||
|
||||
// Transactional dispatch intent. Lease tokens are server-only fencing credentials.
|
||||
model PrepareWorkflowOutbox {
|
||||
id String @id @default(uuid()) @db.Uuid
|
||||
taskId String @unique @map("task_id") @db.Uuid
|
||||
status String @default("PENDING") @db.VarChar(20)
|
||||
attempts Int @default(0)
|
||||
leaseOwner String? @map("lease_owner") @db.VarChar(100)
|
||||
leaseToken String? @map("lease_token") @db.Uuid
|
||||
leaseUntil DateTime? @map("lease_until") @db.Timestamptz()
|
||||
heartbeatAt DateTime? @map("heartbeat_at") @db.Timestamptz()
|
||||
runId String? @map("run_id") @db.Uuid
|
||||
reservationId String? @map("reservation_id") @db.Uuid
|
||||
createdAt DateTime @default(now()) @map("created_at") @db.Timestamptz()
|
||||
@@index([status, leaseUntil])
|
||||
@@map("prepare_workflow_outbox")
|
||||
}
|
||||
@@ -1,5 +1,6 @@
|
||||
import { registerPrepareWorker } from './orchestration-worker/worker-registration';
|
||||
import { RouterUsageModule } from './router-usage/router-usage.module';
|
||||
import { HermesManagementModule } from './hermes-management/hermes-management.module';
|
||||
import { runtimeReadRegistration } from './hermes-management/runtime-read.registration';
|
||||
import { TasksModule } from './common/services/tasks.module';
|
||||
import { Module, NestModule, MiddlewareConsumer } from "@nestjs/common";
|
||||
import { PrismaModule } from "./prisma/prisma.module";
|
||||
@@ -45,9 +46,10 @@ import { Request, Response, NextFunction } from "express";
|
||||
|
||||
@Module({
|
||||
imports: [
|
||||
...registerPrepareWorker(),
|
||||
RouterUsageModule,
|
||||
// Deny-all registration: no request identity or runtime credentials are accepted yet.
|
||||
HermesManagementModule.register({origin:'http://canina-hermes:9119',token:async()=>''}),
|
||||
runtimeReadRegistration(),
|
||||
OrchestrationModule,
|
||||
SupportChatModule,
|
||||
RevalidationModule,
|
||||
|
||||
@@ -12,7 +12,7 @@ export class HermesManagementModule {
|
||||
policy: RuntimeAuthorizer = {authorize: async () => false},
|
||||
): DynamicModule {
|
||||
const broker = new HermesSessionBroker(credentials);
|
||||
return this.register({origin:'http://canina-hermes:9119',token:()=>broker.token()},policy);
|
||||
return this.register({origin:'http://canina-hermes-inventory:9119',token:()=>broker.token()},policy);
|
||||
}
|
||||
|
||||
static register(
|
||||
|
||||
@@ -34,7 +34,7 @@ describe('server-only Hermes session broker',()=>{
|
||||
expect(await broker.token()).toBe('verified-test-token');
|
||||
expect(fetcher).toHaveBeenCalledTimes(1);
|
||||
const [url,options]=fetcher.mock.calls[0];
|
||||
expect(url).toBe('http://canina-hermes:9119/auth/password-login');
|
||||
expect(url).toBe('http://canina-hermes-inventory:9119/auth/password-login');
|
||||
expect(JSON.parse(options.body)).toEqual({provider:'basic',username:'backend',password:'test-only'});
|
||||
expect(options.redirect).toBe('error');
|
||||
});
|
||||
|
||||
@@ -15,7 +15,7 @@ export class HermesSessionBroker {
|
||||
try{
|
||||
const creds=await Promise.race([this.credentials(),new Promise<never>((_,reject)=>controller.signal.addEventListener('abort',()=>reject(new HermesError('UNAVAILABLE')),{once:true}))]);
|
||||
if(!creds.username || !creds.password) throw new HermesError('AUTH_REQUIRED');
|
||||
const response=await this.fetcher('http://canina-hermes:9119/auth/password-login',{method:'POST',redirect:'error',signal:controller.signal,headers:{'Content-Type':'application/json'},body:JSON.stringify({provider:'basic',...creds})});
|
||||
const response=await this.fetcher('http://canina-hermes-inventory:9119/auth/password-login',{method:'POST',redirect:'error',signal:controller.signal,headers:{'Content-Type':'application/json'},body:JSON.stringify({provider:'basic',...creds})});
|
||||
if(!response.ok) throw new HermesError('AUTH_REQUIRED');
|
||||
const reader=response.body?.getReader();
|
||||
if(!reader) throw new HermesError('AUTH_REQUIRED');
|
||||
|
||||
@@ -28,6 +28,7 @@ export interface Health {
|
||||
version: string;
|
||||
commit: string;
|
||||
auth_required: boolean;
|
||||
provenance?: Row;
|
||||
}
|
||||
function object(value: unknown): Record<string, unknown> {
|
||||
if (!value || typeof value !== 'object' || Array.isArray(value))
|
||||
@@ -57,7 +58,12 @@ export class HermesAdapter {
|
||||
private readonly config: HermesConfig,
|
||||
private readonly fetcher: typeof fetch = fetch,
|
||||
) {
|
||||
if (config.origin !== 'http://canina-hermes:9119')
|
||||
if (
|
||||
![
|
||||
'http://canina-hermes:9119',
|
||||
'http://canina-hermes-inventory:9119',
|
||||
].includes(config.origin)
|
||||
)
|
||||
throw new HermesError('CONFIGURATION');
|
||||
this.origin = config.origin;
|
||||
}
|
||||
@@ -122,7 +128,12 @@ export class HermesAdapter {
|
||||
} finally {
|
||||
await reader.cancel().catch(() => undefined);
|
||||
}
|
||||
return JSON.parse(Buffer.concat(chunks).toString('utf8')) as unknown;
|
||||
const data: unknown = JSON.parse(Buffer.concat(chunks).toString('utf8'));
|
||||
if (this.origin === 'http://canina-hermes-inventory:9119') {
|
||||
const rows = Array.isArray(data) ? data : [data];
|
||||
for (const row of rows) provenance(row, true);
|
||||
}
|
||||
return data;
|
||||
} catch (error) {
|
||||
if (error instanceof HermesError) throw error;
|
||||
throw new HermesError('UNAVAILABLE');
|
||||
@@ -132,12 +143,16 @@ export class HermesAdapter {
|
||||
}
|
||||
}
|
||||
async health(): Promise<Health> {
|
||||
const health = fields(await this.request('/api/health'), {
|
||||
ok: 'boolean',
|
||||
version: 'string',
|
||||
commit: 'string',
|
||||
auth_required: 'boolean',
|
||||
}) as unknown as Health;
|
||||
const raw = await this.request('/api/health');
|
||||
const health = {
|
||||
...fields(raw, {
|
||||
ok: 'boolean',
|
||||
version: 'string',
|
||||
commit: 'string',
|
||||
auth_required: 'boolean',
|
||||
}),
|
||||
...provenance(raw),
|
||||
} as unknown as Health;
|
||||
if (!health.auth_required) throw new HermesError('AUTH_REQUIRED');
|
||||
return health;
|
||||
}
|
||||
@@ -151,23 +166,47 @@ export class HermesAdapter {
|
||||
});
|
||||
if (raw.gateway_state !== null && typeof raw.gateway_state !== 'string')
|
||||
throw new HermesError('CONTRACT');
|
||||
return { ...row, gateway_state: raw.gateway_state as Json };
|
||||
return {
|
||||
...row,
|
||||
gateway_state: raw.gateway_state as Json,
|
||||
...provenance(raw),
|
||||
...(raw.provenance
|
||||
? {
|
||||
gateway_running: false,
|
||||
active_agents: 0,
|
||||
gateway_state: null,
|
||||
overall: 'unknown',
|
||||
liveGatewayState: 'unknown',
|
||||
liveActiveAgents: null,
|
||||
}
|
||||
: {}),
|
||||
};
|
||||
}
|
||||
async profiles(): Promise<Row[]> {
|
||||
return list(object(await this.request('/api/profiles')).profiles).map((r) =>
|
||||
fields(r, {
|
||||
name: 'string',
|
||||
is_default: 'boolean',
|
||||
gateway_running: 'boolean',
|
||||
return list(object(await this.request('/api/profiles')).profiles).map(
|
||||
(r) => ({
|
||||
...fields(r, {
|
||||
name: 'string',
|
||||
is_default: 'boolean',
|
||||
gateway_running: 'boolean',
|
||||
}),
|
||||
...provenance(r),
|
||||
...(object(r).provenance
|
||||
? { gateway_running: false, liveGatewayState: 'unknown' }
|
||||
: {}),
|
||||
}),
|
||||
);
|
||||
}
|
||||
async modelInfo(profile: string): Promise<Row> {
|
||||
return fields(await this.request('/api/model/info', profile), {
|
||||
model: 'string',
|
||||
provider: 'string',
|
||||
effective_context_length: 'number',
|
||||
});
|
||||
const raw = await this.request('/api/model/info', profile);
|
||||
return {
|
||||
...fields(raw, {
|
||||
model: 'string',
|
||||
provider: 'string',
|
||||
effective_context_length: 'number',
|
||||
}),
|
||||
...provenance(raw),
|
||||
};
|
||||
}
|
||||
async modelOptions(profile: string): Promise<Row[]> {
|
||||
return list(
|
||||
@@ -189,6 +228,7 @@ export class HermesAdapter {
|
||||
);
|
||||
return {
|
||||
...fields(raw, { period_days: 'number' }),
|
||||
...provenance(raw),
|
||||
totals: fields(raw.totals, {
|
||||
total_input: 'number',
|
||||
total_output: 'number',
|
||||
@@ -218,7 +258,11 @@ export class HermesAdapter {
|
||||
is_active: 'boolean',
|
||||
}),
|
||||
);
|
||||
return { ...fields(raw, { total: 'number' }), sessions: rows };
|
||||
return {
|
||||
...fields(raw, { total: 'number' }),
|
||||
sessions: rows,
|
||||
...provenance(raw),
|
||||
};
|
||||
}
|
||||
async crons(profile: string): Promise<Row[]> {
|
||||
return list(await this.request('/api/cron/jobs', profile)).map((r) =>
|
||||
@@ -265,5 +309,34 @@ function scoped(
|
||||
): Row {
|
||||
const row = fields(value, schema);
|
||||
if (row.profile !== profile) throw new HermesError('FORBIDDEN');
|
||||
return row;
|
||||
return { ...row, ...provenance(value) };
|
||||
}
|
||||
|
||||
/** Snapshot provenance is mandatory on inventory origin; never synthesize a capture time. */
|
||||
function provenance(value: unknown, required = false): Row {
|
||||
const raw = object(value);
|
||||
if (!raw.provenance && !required) return {};
|
||||
const p = object(raw.provenance);
|
||||
if (
|
||||
p.source !== 'snapshot' ||
|
||||
p.liveProcess !== 'unknown' ||
|
||||
typeof p.capturedAt !== 'string' ||
|
||||
!p.capturedAt.endsWith('Z') ||
|
||||
!Number.isFinite(Date.parse(p.capturedAt)) ||
|
||||
Date.parse(p.capturedAt) > Date.now() ||
|
||||
typeof p.ageSeconds !== 'number' ||
|
||||
!Number.isInteger(p.ageSeconds) ||
|
||||
p.ageSeconds < 0 ||
|
||||
!['fresh', 'stale'].includes(String(p.freshness))
|
||||
)
|
||||
throw new HermesError('CONTRACT');
|
||||
return {
|
||||
provenance: {
|
||||
source: 'snapshot',
|
||||
capturedAt: p.capturedAt,
|
||||
ageSeconds: p.ageSeconds,
|
||||
freshness: p.freshness as string,
|
||||
liveProcess: 'unknown',
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
import { loadRuntimeGrants } from './runtime-read.module';
|
||||
import { mkdtemp, writeFile, rm } from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
describe('reviewed runtime grants file', () => {
|
||||
it('loads an exact reviewed file and fails closed for malformed grants', async () => {
|
||||
const dir = await mkdtemp(join(tmpdir(), 'runtime-grants-'));
|
||||
const file = join(dir, 'grants.json');
|
||||
try {
|
||||
await writeFile(file, JSON.stringify({ 'subject-1': ['default'] }));
|
||||
expect(await loadRuntimeGrants(file, dir)).toEqual({
|
||||
'subject-1': ['default'],
|
||||
});
|
||||
for (const invalid of [
|
||||
null,
|
||||
[],
|
||||
{ 'subject-1': [] },
|
||||
{ 'subject-1': ['default', 'default'] },
|
||||
{ 'subject-1': ['../x'] },
|
||||
{ 'subject-1': [3] },
|
||||
{ 'subject-1': ['default'], extra: true },
|
||||
]) {
|
||||
await writeFile(file, JSON.stringify(invalid));
|
||||
await expect(loadRuntimeGrants(file, dir)).rejects.toThrow();
|
||||
}
|
||||
await expect(loadRuntimeGrants('/etc/passwd')).rejects.toThrow();
|
||||
} finally {
|
||||
await rm(dir, { recursive: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,83 @@
|
||||
import { Test } from '@nestjs/testing';
|
||||
import { INestApplication } from '@nestjs/common';
|
||||
import { PassportModule } from '@nestjs/passport';
|
||||
import { Strategy, ExtractJwt } from 'passport-jwt';
|
||||
import passport from 'passport';
|
||||
import { sign } from 'jsonwebtoken';
|
||||
import request from 'supertest';
|
||||
import { RuntimeReadModule } from './runtime-read.module';
|
||||
import { HermesIntegrationService } from './hermes-integration.service';
|
||||
describe('runtime HTTP authentication and read contract', () => {
|
||||
let app: INestApplication;
|
||||
const health = jest.fn().mockResolvedValue({ ok: true, auth_required: true });
|
||||
beforeAll(async () => {
|
||||
passport.use(
|
||||
'jwt',
|
||||
new Strategy(
|
||||
{
|
||||
jwtFromRequest: ExtractJwt.fromAuthHeaderAsBearerToken(),
|
||||
secretOrKey: 'test-only-http-key',
|
||||
},
|
||||
(payload, done) => done(null, { id: payload.sub, role: payload.role }),
|
||||
),
|
||||
);
|
||||
const module = await Test.createTestingModule({
|
||||
imports: [
|
||||
PassportModule,
|
||||
RuntimeReadModule.register({ 'admin-1': ['default'] }),
|
||||
],
|
||||
})
|
||||
.overrideProvider(HermesIntegrationService)
|
||||
.useValue({ health })
|
||||
.compile();
|
||||
app = module.createNestApplication();
|
||||
await app.init();
|
||||
});
|
||||
afterAll(async () => {
|
||||
await app.close();
|
||||
passport.unuse('jwt');
|
||||
});
|
||||
it('rejects missing and forged credentials before upstream', async () => {
|
||||
health.mockClear();
|
||||
await request(app.getHttpServer())
|
||||
.get('/admin/orchestration/runtime/health')
|
||||
.expect(401);
|
||||
await request(app.getHttpServer())
|
||||
.get('/admin/orchestration/runtime/health')
|
||||
.set(
|
||||
'Authorization',
|
||||
'Bearer ' + sign({ sub: 'admin-1', role: 'Admin' }, 'forged'),
|
||||
)
|
||||
.expect(401);
|
||||
expect(health).not.toHaveBeenCalled();
|
||||
});
|
||||
it('accepts verified allowlisted admin but rejects signed ungranted subject', async () => {
|
||||
await request(app.getHttpServer())
|
||||
.get('/admin/orchestration/runtime/health')
|
||||
.set(
|
||||
'Authorization',
|
||||
'Bearer ' + sign({ sub: 'other', role: 'Admin' }, 'test-only-http-key'),
|
||||
)
|
||||
.expect(403);
|
||||
const r = await request(app.getHttpServer())
|
||||
.get('/admin/orchestration/runtime/health')
|
||||
.set(
|
||||
'Authorization',
|
||||
'Bearer ' +
|
||||
sign({ sub: 'admin-1', role: 'Admin' }, 'test-only-http-key'),
|
||||
)
|
||||
.expect(200);
|
||||
expect(r.body).toEqual({ ok: true, auth_required: true });
|
||||
expect(r.headers['set-cookie']).toBeUndefined();
|
||||
});
|
||||
it('does not register mutation handlers', async () => {
|
||||
await request(app.getHttpServer())
|
||||
.post('/admin/orchestration/runtime/schedules')
|
||||
.set(
|
||||
'Authorization',
|
||||
'Bearer ' +
|
||||
sign({ sub: 'admin-1', role: 'Admin' }, 'test-only-http-key'),
|
||||
)
|
||||
.expect(404);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,183 @@
|
||||
import {
|
||||
Controller,
|
||||
Get,
|
||||
Req,
|
||||
Query,
|
||||
UseGuards,
|
||||
Injectable,
|
||||
ForbiddenException,
|
||||
ServiceUnavailableException,
|
||||
HttpException,
|
||||
} from '@nestjs/common';
|
||||
import { AuthGuard } from '@nestjs/passport';
|
||||
import {
|
||||
HermesIntegrationService,
|
||||
RuntimePrincipal,
|
||||
} from './hermes-integration.service';
|
||||
import { HermesError } from './hermes.adapter';
|
||||
export interface VerifiedRuntimeRequest {
|
||||
user?: {
|
||||
id?: string;
|
||||
role?: string;
|
||||
isApiKey?: boolean;
|
||||
profiles?: string[];
|
||||
};
|
||||
}
|
||||
@Injectable()
|
||||
export class RuntimeReadAccess {
|
||||
private readonly grants: Readonly<Record<string, readonly string[]>>;
|
||||
constructor(grants: Readonly<Record<string, readonly string[]>>) {
|
||||
if (
|
||||
!grants ||
|
||||
typeof grants !== 'object' ||
|
||||
Array.isArray(grants) ||
|
||||
Object.keys(grants).length > 1000
|
||||
)
|
||||
throw new Error('Invalid runtime access configuration');
|
||||
const copy: Record<string, readonly string[]> = Object.create(null);
|
||||
for (const [subject, profiles] of Object.entries(grants)) {
|
||||
if (
|
||||
!/^[a-zA-Z0-9][a-zA-Z0-9@._:-]{0,127}$/.test(subject) ||
|
||||
['__proto__', 'constructor', 'prototype'].includes(subject) ||
|
||||
!Array.isArray(profiles) ||
|
||||
profiles.length === 0 ||
|
||||
profiles.length > 100 ||
|
||||
new Set(profiles).size !== profiles.length ||
|
||||
profiles.some(
|
||||
(p) =>
|
||||
typeof p !== 'string' ||
|
||||
!/^[a-zA-Z0-9][a-zA-Z0-9_-]{0,63}$/.test(p),
|
||||
)
|
||||
)
|
||||
throw new Error('Invalid runtime access configuration');
|
||||
copy[subject] = Object.freeze([...profiles]);
|
||||
}
|
||||
this.grants = Object.freeze(copy);
|
||||
}
|
||||
principal(req: VerifiedRuntimeRequest): RuntimePrincipal {
|
||||
const u = req.user;
|
||||
if (
|
||||
!u?.id ||
|
||||
u.isApiKey ||
|
||||
!['admin', 'superadmin'].includes(
|
||||
(u.role ?? '').toLowerCase().replace(/[_-]/g, ''),
|
||||
) ||
|
||||
!Object.hasOwn(this.grants, u.id)
|
||||
)
|
||||
throw new ForbiddenException('Runtime access denied');
|
||||
return { subject: u.id, profiles: this.grants[u.id] };
|
||||
}
|
||||
authorize(
|
||||
p: RuntimePrincipal,
|
||||
permission: string,
|
||||
profile?: string,
|
||||
): Promise<boolean> {
|
||||
return Promise.resolve(
|
||||
permission === 'runtime.read' &&
|
||||
Object.hasOwn(this.grants, p.subject) &&
|
||||
(!profile || this.grants[p.subject].includes(profile)),
|
||||
);
|
||||
}
|
||||
}
|
||||
@Controller('admin/orchestration/runtime')
|
||||
@UseGuards(AuthGuard('jwt'))
|
||||
export class RuntimeReadController {
|
||||
constructor(
|
||||
private readonly runtime: HermesIntegrationService,
|
||||
private readonly access: RuntimeReadAccess,
|
||||
) {}
|
||||
private async read(
|
||||
req: VerifiedRuntimeRequest,
|
||||
call: (p: RuntimePrincipal) => Promise<unknown>,
|
||||
) {
|
||||
const p = this.access.principal(req);
|
||||
try {
|
||||
return await call(p);
|
||||
} catch (e) {
|
||||
if (e instanceof HermesError) {
|
||||
if (e.code === 'FORBIDDEN')
|
||||
throw new ForbiddenException('Runtime access denied');
|
||||
if (e.code === 'LIMIT')
|
||||
throw new HttpException('Runtime request limit', 429);
|
||||
}
|
||||
throw new ServiceUnavailableException('Runtime unavailable');
|
||||
}
|
||||
}
|
||||
private profile(
|
||||
q: Record<string, unknown>,
|
||||
p: RuntimePrincipal,
|
||||
extras: string[] = [],
|
||||
): string {
|
||||
if (
|
||||
Object.keys(q).some((k) => !['profile', ...extras].includes(k)) ||
|
||||
typeof q.profile !== 'string' ||
|
||||
!p.profiles.includes(q.profile)
|
||||
)
|
||||
throw new HermesError('FORBIDDEN');
|
||||
return q.profile;
|
||||
}
|
||||
private number(
|
||||
q: Record<string, unknown>,
|
||||
key: string,
|
||||
fallback: number,
|
||||
min: number,
|
||||
max: number,
|
||||
): number {
|
||||
if (q[key] === undefined) return fallback;
|
||||
if (typeof q[key] !== 'string' || !/^\d+$/.test(q[key]))
|
||||
throw new HermesError('LIMIT');
|
||||
const n = Number(q[key]);
|
||||
if (!Number.isSafeInteger(n) || n < min || n > max)
|
||||
throw new HermesError('LIMIT');
|
||||
return n;
|
||||
}
|
||||
@Get('health') health(@Req() req: VerifiedRuntimeRequest) {
|
||||
return this.read(req, (p) => this.runtime.health(p));
|
||||
}
|
||||
@Get('profiles') profiles(@Req() req: VerifiedRuntimeRequest) {
|
||||
return this.read(req, (p) => this.runtime.profiles(p));
|
||||
}
|
||||
@Get('status') status(
|
||||
@Req() req: VerifiedRuntimeRequest,
|
||||
@Query() q: Record<string, unknown>,
|
||||
) {
|
||||
return this.read(req, (p) => this.runtime.status(p, this.profile(q, p)));
|
||||
}
|
||||
@Get('usage') usage(
|
||||
@Req() req: VerifiedRuntimeRequest,
|
||||
@Query() q: Record<string, unknown>,
|
||||
) {
|
||||
return this.read(req, (p) =>
|
||||
this.runtime.usage(
|
||||
p,
|
||||
this.profile(q, p, ['days']),
|
||||
this.number(q, 'days', 30, 1, 365),
|
||||
),
|
||||
);
|
||||
}
|
||||
@Get('sessions') sessions(
|
||||
@Req() req: VerifiedRuntimeRequest,
|
||||
@Query() q: Record<string, unknown>,
|
||||
) {
|
||||
return this.read(req, (p) =>
|
||||
this.runtime.sessions(
|
||||
p,
|
||||
this.profile(q, p, ['limit', 'offset']),
|
||||
this.number(q, 'limit', 20, 1, 100),
|
||||
this.number(q, 'offset', 0, 0, 10000),
|
||||
),
|
||||
);
|
||||
}
|
||||
@Get('schedules') schedules(
|
||||
@Req() req: VerifiedRuntimeRequest,
|
||||
@Query() q: Record<string, unknown>,
|
||||
) {
|
||||
return this.read(req, (p) => this.runtime.crons(p, this.profile(q, p)));
|
||||
}
|
||||
@Get('model-info') modelInfo(
|
||||
@Req() req: VerifiedRuntimeRequest,
|
||||
@Query() q: Record<string, unknown>,
|
||||
) {
|
||||
return this.read(req, (p) => this.runtime.modelInfo(p, this.profile(q, p)));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,88 @@
|
||||
import { DynamicModule, Module } from '@nestjs/common';
|
||||
import { open, lstat } from 'node:fs/promises';
|
||||
import { constants } from 'node:fs';
|
||||
import { dirname, resolve } from 'node:path';
|
||||
import { HermesManagementModule } from './hermes-management.module';
|
||||
import {
|
||||
RuntimeReadAccess,
|
||||
RuntimeReadController,
|
||||
} from './runtime-read.controller';
|
||||
import { HermesError } from './hermes.adapter';
|
||||
export async function runtimeSecret(path: string): Promise<string> {
|
||||
if (!/^\/run\/secrets\/[a-zA-Z0-9_-]+$/.test(path))
|
||||
throw new HermesError('CONFIGURATION');
|
||||
try {
|
||||
const file = await open(path, 'r');
|
||||
try {
|
||||
const stat = await file.stat();
|
||||
if (!stat.isFile() || stat.size > 4096) throw new Error();
|
||||
const value = (await file.readFile('utf8')).trim();
|
||||
if (!value || /[\r\n]/.test(value)) throw new Error();
|
||||
return value;
|
||||
} finally {
|
||||
await file.close();
|
||||
}
|
||||
} catch {
|
||||
throw new HermesError('AUTH_REQUIRED');
|
||||
}
|
||||
}
|
||||
/** Exact fixed mount roots only; startup fails closed, never browser/JWT grants. */
|
||||
export async function loadRuntimeGrants(
|
||||
path = '/run/secrets/hermes_runtime_grants',
|
||||
reviewedRoot?: string,
|
||||
): Promise<Readonly<Record<string, readonly string[]>>> {
|
||||
const roots = reviewedRoot
|
||||
? [resolve(reviewedRoot)]
|
||||
: ['/run/secrets', '/etc/canina'];
|
||||
if (!roots.includes(dirname(path)) || resolve(path) !== path)
|
||||
throw new HermesError('CONFIGURATION');
|
||||
try {
|
||||
const stat = await lstat(path);
|
||||
if (
|
||||
!stat.isFile() ||
|
||||
stat.isSymbolicLink() ||
|
||||
stat.size > 65536 ||
|
||||
stat.mode & 0o022
|
||||
)
|
||||
throw new Error();
|
||||
const file = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW);
|
||||
try {
|
||||
const data: unknown = JSON.parse(await file.readFile('utf8'));
|
||||
if (!data || typeof data !== 'object' || Array.isArray(data))
|
||||
throw new Error();
|
||||
new RuntimeReadAccess(data as Record<string, readonly string[]>);
|
||||
return data as Record<string, readonly string[]>;
|
||||
} finally {
|
||||
await file.close();
|
||||
}
|
||||
} catch {
|
||||
throw new HermesError('CONFIGURATION');
|
||||
}
|
||||
}
|
||||
@Module({})
|
||||
export class RuntimeReadModule {
|
||||
static async registerFromFile(
|
||||
path = '/run/secrets/hermes_runtime_grants',
|
||||
): Promise<DynamicModule> {
|
||||
return this.register(await loadRuntimeGrants(path));
|
||||
}
|
||||
static register(
|
||||
grants: Readonly<Record<string, readonly string[]>>,
|
||||
): DynamicModule {
|
||||
const access = new RuntimeReadAccess(grants);
|
||||
return {
|
||||
module: RuntimeReadModule,
|
||||
imports: [
|
||||
HermesManagementModule.registerAuthenticated(
|
||||
async () => ({
|
||||
username: await runtimeSecret('/run/secrets/hermes_username'),
|
||||
password: await runtimeSecret('/run/secrets/hermes_password'),
|
||||
}),
|
||||
access,
|
||||
),
|
||||
],
|
||||
controllers: [RuntimeReadController],
|
||||
providers: [{ provide: RuntimeReadAccess, useValue: access }],
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
import { runtimeReadRegistration } from './runtime-read.registration';
|
||||
import { RuntimeReadModule } from './runtime-read.module';
|
||||
describe('application runtime registration',()=>{
|
||||
afterEach(()=>jest.restoreAllMocks());
|
||||
it('registers denied routes without reading files when not explicitly enabled',async()=>{
|
||||
const file=jest.spyOn(RuntimeReadModule,'registerFromFile');
|
||||
const register=jest.spyOn(RuntimeReadModule,'register');
|
||||
await runtimeReadRegistration({});
|
||||
expect(file).not.toHaveBeenCalled();expect(register).toHaveBeenCalledWith({});
|
||||
});
|
||||
it('uses only the fixed reviewed grants mount when enabled',async()=>{
|
||||
const file=jest.spyOn(RuntimeReadModule,'registerFromFile').mockResolvedValue({module:RuntimeReadModule});
|
||||
await runtimeReadRegistration({CANINA_RUNTIME_READ_ENABLED:'true'});
|
||||
expect(file).toHaveBeenCalledWith('/run/secrets/hermes_runtime_grants');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,6 @@
|
||||
import { RuntimeReadModule } from './runtime-read.module';
|
||||
export function runtimeReadRegistration(env:Record<string,string|undefined>=process.env) {
|
||||
return env.CANINA_RUNTIME_READ_ENABLED === 'true'
|
||||
? RuntimeReadModule.registerFromFile('/run/secrets/hermes_runtime_grants')
|
||||
: RuntimeReadModule.register({});
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
import { Test } from '@nestjs/testing';
|
||||
import {
|
||||
RuntimeReadController,
|
||||
RuntimeReadAccess,
|
||||
} from './runtime-read.controller';
|
||||
import { HermesIntegrationService } from './hermes-integration.service';
|
||||
import { HermesError } from './hermes.adapter';
|
||||
import {
|
||||
ForbiddenException,
|
||||
ServiceUnavailableException,
|
||||
} from '@nestjs/common';
|
||||
describe('runtime read boundary', () => {
|
||||
const health = jest.fn();
|
||||
const profiles = jest.fn();
|
||||
const status = jest.fn();
|
||||
const service = {
|
||||
health,
|
||||
profiles,
|
||||
status,
|
||||
} as unknown as HermesIntegrationService;
|
||||
const access = new RuntimeReadAccess({ 'admin-1': ['default'] });
|
||||
const controller = new RuntimeReadController(service, access);
|
||||
beforeEach(() => jest.clearAllMocks());
|
||||
it('denies unconfigured identity without IO', async () => {
|
||||
await expect(
|
||||
controller.health({ user: { id: 'other', role: 'Admin' } }),
|
||||
).rejects.toBeInstanceOf(ForbiddenException);
|
||||
expect(health).not.toHaveBeenCalled();
|
||||
});
|
||||
it('uses server scope not body or user supplied profiles', async () => {
|
||||
status.mockResolvedValue({ overall: 'ok' });
|
||||
await expect(
|
||||
controller.status(
|
||||
{ user: { id: 'admin-1', role: 'Admin', profiles: ['other'] } },
|
||||
{ profile: 'other' },
|
||||
),
|
||||
).rejects.toBeInstanceOf(ForbiddenException);
|
||||
expect(status).not.toHaveBeenCalled();
|
||||
await controller.status(
|
||||
{ user: { id: 'admin-1', role: 'Admin' } },
|
||||
{ profile: 'default' },
|
||||
);
|
||||
expect(status).toHaveBeenCalledWith(
|
||||
{ subject: 'admin-1', profiles: ['default'] },
|
||||
'default',
|
||||
);
|
||||
});
|
||||
it('rejects duplicate/missing/unknown query parameters before IO', async () => {
|
||||
for (const q of [
|
||||
{},
|
||||
{ profile: ['default', 'other'] },
|
||||
{ profile: 'default', token: 'leak' },
|
||||
])
|
||||
await expect(
|
||||
controller.status({ user: { id: 'admin-1', role: 'Admin' } }, q),
|
||||
).rejects.toBeInstanceOf(ForbiddenException);
|
||||
expect(status).not.toHaveBeenCalled();
|
||||
});
|
||||
it('maps credential failure without leaking upstream errors', async () => {
|
||||
health.mockRejectedValue(new HermesError('AUTH_REQUIRED'));
|
||||
await expect(
|
||||
controller.health({ user: { id: 'admin-1', role: 'Admin' } }),
|
||||
).rejects.toThrow('Runtime unavailable');
|
||||
});
|
||||
it('rejects API keys and non-admins even when subject is configured', async () => {
|
||||
for (const user of [
|
||||
{ id: 'admin-1', role: 'User' },
|
||||
{ id: 'admin-1', role: 'Admin', isApiKey: true },
|
||||
])
|
||||
await expect(controller.health({ user })).rejects.toBeInstanceOf(
|
||||
ForbiddenException,
|
||||
);
|
||||
expect(health).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,68 @@
|
||||
import { HermesAdapter } from './hermes.adapter';
|
||||
const provenance = {
|
||||
source: 'snapshot',
|
||||
capturedAt: '2020-01-01T00:00:00Z',
|
||||
ageSeconds: 100,
|
||||
freshness: 'stale',
|
||||
liveProcess: 'unknown',
|
||||
};
|
||||
function adapter(data: unknown) {
|
||||
return new HermesAdapter(
|
||||
{
|
||||
origin: 'http://canina-hermes-inventory:9119',
|
||||
token: () => Promise.resolve('test'),
|
||||
},
|
||||
jest.fn(() =>
|
||||
Promise.resolve(new Response(JSON.stringify(data), { status: 200 })),
|
||||
) as typeof fetch,
|
||||
);
|
||||
}
|
||||
describe('snapshot provenance boundary', () => {
|
||||
it('rejects missing provenance from dedicated inventory origin', async () => {
|
||||
await expect(
|
||||
adapter({
|
||||
ok: true,
|
||||
version: '1',
|
||||
commit: 'x',
|
||||
auth_required: true,
|
||||
}).health(),
|
||||
).rejects.toMatchObject({ code: 'CONTRACT' });
|
||||
});
|
||||
it('cannot present copied gateway scalars as live telemetry', async () => {
|
||||
const row = await adapter({
|
||||
version: '1',
|
||||
gateway_running: true,
|
||||
active_agents: 12,
|
||||
gateway_state: 'running',
|
||||
overall: 'ok',
|
||||
provenance,
|
||||
}).status('default');
|
||||
expect(row).toMatchObject({
|
||||
gateway_running: false,
|
||||
active_agents: 0,
|
||||
gateway_state: null,
|
||||
overall: 'unknown',
|
||||
liveGatewayState: 'unknown',
|
||||
liveActiveAgents: null,
|
||||
provenance,
|
||||
});
|
||||
});
|
||||
it('profile readiness cannot be inferred from copied PID state', async () => {
|
||||
const rows = await adapter({
|
||||
profiles: [
|
||||
{
|
||||
name: 'default',
|
||||
is_default: true,
|
||||
gateway_running: true,
|
||||
provenance,
|
||||
},
|
||||
],
|
||||
provenance,
|
||||
}).profiles();
|
||||
expect(rows[0]).toMatchObject({
|
||||
gateway_running: false,
|
||||
liveGatewayState: 'unknown',
|
||||
provenance,
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,19 @@
|
||||
# Opt-in fragment ONLY: merge into the reviewed Stage backend service definition.
|
||||
# Not a standalone compose file; does not enable the worker automatically.
|
||||
# Host files/dirs must already exist, with reviewed ownership and modes.
|
||||
x-prepare-worker-backend:
|
||||
environment:
|
||||
PREPARE_WORKER_ENABLED: "${PREPARE_WORKER_ENABLED:-false}"
|
||||
secrets:
|
||||
- source: prepare_worker_grants
|
||||
target: prepare_worker_grants
|
||||
mode: 0400
|
||||
volumes:
|
||||
- type: bind
|
||||
source: ${PREPARE_WORKER_ARTIFACT_HOST_DIRECTORY:?review private artifact directory}
|
||||
target: /var/lib/canina/prepare-artifacts
|
||||
bind:
|
||||
create_host_path: false
|
||||
secrets:
|
||||
prepare_worker_grants:
|
||||
file: ${PREPARE_WORKER_GRANTS_FILE:?review mounted grants JSON file}
|
||||
@@ -0,0 +1,48 @@
|
||||
import { constants, openSync, closeSync, fstatSync, readSync } from 'fs';
|
||||
import { WorkerTransportConfig, WorkerTransportModule } from './worker-transport.module';
|
||||
|
||||
export const PREPARE_WORKER_CONFIG_PATH = '/run/secrets/prepare_worker_grants';
|
||||
export const PREPARE_WORKER_ARTIFACT_DIRECTORY = '/var/lib/canina/prepare-artifacts';
|
||||
const MAX_BYTES = 16384;
|
||||
const fail = () => { throw new Error('Prepare worker startup configuration rejected'); };
|
||||
function exact(value: any, keys: string[]): boolean {
|
||||
return value !== null && typeof value === 'object' && !Array.isArray(value) &&
|
||||
Object.keys(value).length === keys.length && keys.every(k => Object.hasOwn(value, k));
|
||||
}
|
||||
|
||||
/** Fixed-path, descriptor-pinned, bounded startup read; never logs file contents. */
|
||||
export function loadPrepareWorkerConfig(): WorkerTransportConfig {
|
||||
let dir: number | undefined, file: number | undefined;
|
||||
try {
|
||||
dir = openSync('/', constants.O_RDONLY | constants.O_DIRECTORY);
|
||||
for (const part of ['run', 'secrets']) {
|
||||
const next = openSync(`/proc/self/fd/${dir}/${part}`, constants.O_RDONLY | constants.O_DIRECTORY | constants.O_NOFOLLOW);
|
||||
closeSync(dir); dir = next;
|
||||
if ((fstatSync(dir).mode & 0o022) !== 0) fail();
|
||||
}
|
||||
file = openSync(`/proc/self/fd/${dir}/prepare_worker_grants`, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK);
|
||||
const stat = fstatSync(file);
|
||||
if (!stat.isFile() || stat.nlink !== 1 || (stat.mode & 0o022) !== 0 || stat.size > MAX_BYTES ||
|
||||
(stat.uid !== 0 && stat.uid !== process.getuid!())) fail();
|
||||
const bytes = Buffer.alloc(MAX_BYTES + 1);
|
||||
let length = 0, n: number;
|
||||
do { n = readSync(file, bytes, length, bytes.length - length, length); length += n; }
|
||||
while (n > 0 && length < bytes.length);
|
||||
if (length > MAX_BYTES) fail();
|
||||
const input = JSON.parse(new TextDecoder('utf-8', { fatal: true }).decode(bytes.subarray(0, length)));
|
||||
if (!exact(input, ['grants', 'workerIds', 'stageCiIds', 'artifactBaseUrl']) ||
|
||||
!Array.isArray(input.grants) || input.grants.length < 1 || input.grants.length > 32 ||
|
||||
!Array.isArray(input.workerIds) || !Array.isArray(input.stageCiIds) ||
|
||||
input.workerIds.length < 1 || input.stageCiIds.length < 1 ||
|
||||
input.workerIds.length + input.stageCiIds.length !== input.grants.length ||
|
||||
typeof input.artifactBaseUrl !== 'string' || input.artifactBaseUrl.length > 2048 ||
|
||||
input.grants.some((g: any) => !exact(g, ['id', 'role', 'credentialSha256']))) fail();
|
||||
// Transport registration performs shared role, SHA256, ID, overlap and HTTPS validation.
|
||||
const config: WorkerTransportConfig = { ...input, enabled: true, artifactDirectory: PREPARE_WORKER_ARTIFACT_DIRECTORY };
|
||||
WorkerTransportModule.register(config); // Reuse the real transport's reviewed validation, no IO.
|
||||
config.grants.forEach(Object.freeze);
|
||||
Object.freeze(config.grants); Object.freeze(config.workerIds); Object.freeze(config.stageCiIds);
|
||||
return Object.freeze(config);
|
||||
} catch { return fail(); }
|
||||
finally { if (file !== undefined) closeSync(file); if (dir !== undefined) closeSync(dir); }
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
# Prepare worker startup boundary
|
||||
|
||||
Parent integration API (AppModule remains untouched here):
|
||||
|
||||
```ts
|
||||
import { registerPrepareWorker } from './orchestration-worker/worker-registration';
|
||||
// In AppModule imports:
|
||||
...registerPrepareWorker(),
|
||||
```
|
||||
|
||||
Only literal `PREPARE_WORKER_ENABLED=true` registers the transport. Missing or other values register nothing (404 routes), do not read configuration, and do not instantiate Prisma. Enabled malformed/missing configuration throws synchronously before Nest starts, with one generic error and no secret logging.
|
||||
|
||||
The fixed mount `/run/secrets/prepare_worker_grants` contains JSON with exactly `grants`, `workerIds`, `stageCiIds`, `artifactBaseUrl`. Each grant contains exactly `id`, `role` (`WORKFLOW_WORKER` or `STAGE_CI`), `credentialSha256` (64 lowercase hexadecimal characters; never the bearer credential). Require at least one worker and CI, distinct role-bound IDs/digests, exact allowlist coverage, and at most 32 grants. No enabled flag, directory, token, user, or request-derived fields accepted in JSON.
|
||||
|
||||
`artifactBaseUrl` must be explicitly reviewed by the operator: canonical public-DNS HTTPS origin plus `/internal/prepare-worker/artifacts`, no credentials, IP, alternate port, query or fragment. Shared transport validation is authoritative. This loader does not fetch the URL or perform DNS resolution. Directory is always `/var/lib/canina/prepare-artifacts`.
|
||||
|
||||
Descriptor-relative Linux `/proc/self/fd` traversal refuses symlink directories/files, nonregular files, hardlinks, files over 16 KiB, group/world writable files/directories, and foreign-owned files (root/runtime UID only). Reads are bounded even on concurrent growth. Configuration is frozen recursively and transport takes its own frozen snapshot. Trusted operator owns mounted configuration; edits require restart and review.
|
||||
|
||||
`compose.prepare-worker.fragment.yml` is an extension fragment, not a deployable compose file. Parent must merge its `x-prepare-worker-backend` into the correct reviewed Stage backend service and merge `secrets`. Compose bind-backed secrets may ignore mode/UID declarations: provision actual host grants file as root/runtime-owned mode 0400 and artifact directory runtime-owned mode 0700 first. No mounts or live configuration are created by this change.
|
||||
|
||||
Tests deliberately use the fixed `/run/secrets` path. Run ONLY in a fresh disposable container with source and donor dependencies read-only, private dependency copy, and private `prisma generate`; never on a live service. Tests need a disposable Linux root filesystem and do not require DB credentials. Registration test uses a real Nest HTTP listener to prove disabled routes are absent. Existing DB-backed transport tracer remains separately gated by `WORKER_TRANSPORT_DB=1`.
|
||||
@@ -0,0 +1,84 @@
|
||||
import { Test } from '@nestjs/testing';
|
||||
import { writeFileSync, chmodSync, symlinkSync, mkdirSync, rmSync } from 'fs';
|
||||
import { execFileSync } from 'child_process';
|
||||
import { registerPrepareWorker } from './worker-registration';
|
||||
import { loadPrepareWorkerConfig } from './worker-config';
|
||||
import { linkSync } from 'fs';
|
||||
|
||||
|
||||
describe('reviewed prepare-worker startup registration', () => {
|
||||
const path = '/run/secrets/prepare_worker_grants';
|
||||
const valid = () => ({ grants: [{ id: 'worker', role: 'WORKFLOW_WORKER', credentialSha256: 'a'.repeat(64) }, { id: 'ci', role: 'STAGE_CI', credentialSha256: 'b'.repeat(64) }], workerIds: ['worker'], stageCiIds: ['ci'], artifactBaseUrl: 'https://stage.example.com/internal/prepare-worker/artifacts' });
|
||||
it('loader itself rejects malformed SHA grants', () => {
|
||||
mkdirSync('/run/secrets', { recursive: true });
|
||||
const c = valid(); c.grants[0].credentialSha256 = 'secret-not-a-sha';
|
||||
writeFileSync(path, JSON.stringify(c), { mode: 0o400 });
|
||||
try { expect(() => loadPrepareWorkerConfig()).toThrow('Prepare worker startup configuration rejected'); }
|
||||
finally { rmSync(path, { force: true }); }
|
||||
});
|
||||
it.each(['1', 'TRUE', 'yes', ' true', 'false', ''])('does not opt in for %s', value => {
|
||||
expect(registerPrepareWorker({ PREPARE_WORKER_ENABLED: value })).toEqual([]);
|
||||
});
|
||||
it.each(['missing', 'symlink', 'directory', 'fifo', 'oversize', 'group-write', 'world-write', 'hardlink'])('fails startup on unsafe file: %s', kind => {
|
||||
mkdirSync('/run/secrets', { recursive: true });
|
||||
const other = '/run/secrets/test-other';
|
||||
try {
|
||||
if (kind === 'fifo') execFileSync('mkfifo', [path]);
|
||||
else if (kind === 'directory') mkdirSync(path);
|
||||
else if (kind === 'symlink') symlinkSync(other, path);
|
||||
else if (kind !== 'missing') {
|
||||
writeFileSync(path, kind === 'oversize' ? ' '.repeat(16385) : JSON.stringify(valid()), { mode: 0o400 });
|
||||
if (kind === 'group-write') chmodSync(path, 0o420);
|
||||
if (kind === 'world-write') chmodSync(path, 0o402);
|
||||
if (kind === 'hardlink') linkSync(path, other);
|
||||
}
|
||||
expect(() => registerPrepareWorker({ PREPARE_WORKER_ENABLED: 'true' })).toThrow('Prepare worker startup configuration rejected');
|
||||
} finally { rmSync(path, { recursive: true, force: true }); rmSync(other, { force: true }); }
|
||||
});
|
||||
it.each(['extra', 'raw-token', 'wrong-role', 'overlap', 'duplicate-sha', 'missing-grant', 'http', 'credentials', 'localhost', 'query', 'bad-json', 'null'])('rejects unreviewed JSON: %s without leaking contents or logging', kind => {
|
||||
mkdirSync('/run/secrets', { recursive: true });
|
||||
const c: any = valid();
|
||||
if (kind === 'extra') c.artifactDirectory = '/user-selected';
|
||||
if (kind === 'raw-token') c.grants[0].token = 'DO_NOT_LOG_THIS_SECRET';
|
||||
if (kind === 'wrong-role') c.grants[0].role = 'Admin';
|
||||
if (kind === 'overlap') c.stageCiIds = ['worker'];
|
||||
if (kind === 'duplicate-sha') c.grants[1].credentialSha256 = c.grants[0].credentialSha256;
|
||||
if (kind === 'missing-grant') c.workerIds.push('unbound');
|
||||
if (kind === 'http') c.artifactBaseUrl = 'http://stage.example.com/internal/prepare-worker/artifacts';
|
||||
if (kind === 'credentials') c.artifactBaseUrl = 'https://DO_NOT_LOG_THIS_SECRET@stage.example.com/internal/prepare-worker/artifacts';
|
||||
if (kind === 'localhost') c.artifactBaseUrl = 'https://127.0.0.1/internal/prepare-worker/artifacts';
|
||||
if (kind === 'query') c.artifactBaseUrl += '?secret=DO_NOT_LOG_THIS_SECRET';
|
||||
writeFileSync(path, kind === 'bad-json' ? 'DO_NOT_LOG_THIS_SECRET' : kind === 'null' ? 'null' : JSON.stringify(c), { mode: 0o400 });
|
||||
const logs = [jest.spyOn(console, 'log'), jest.spyOn(console, 'warn'), jest.spyOn(console, 'error')];
|
||||
try {
|
||||
expect(() => registerPrepareWorker({ PREPARE_WORKER_ENABLED: 'true' })).toThrow(/^Prepare worker startup configuration rejected$/);
|
||||
logs.forEach(log => expect(log).not.toHaveBeenCalled());
|
||||
} finally { logs.forEach(log => log.mockRestore()); rmSync(path, { force: true }); }
|
||||
});
|
||||
it('defaults to no module/no config IO and exposes no worker routes', async () => {
|
||||
expect(registerPrepareWorker({})).toEqual([]);
|
||||
expect(registerPrepareWorker({ PREPARE_WORKER_ENABLED: 'false' })).toEqual([]);
|
||||
const module = await Test.createTestingModule({ imports: registerPrepareWorker({}) }).compile();
|
||||
const app = module.createNestApplication();
|
||||
await app.listen(0, '127.0.0.1');
|
||||
try {
|
||||
const response = await fetch((await app.getUrl()) + '/internal/prepare-worker/claim', { method: 'POST' });
|
||||
expect(response.status).toBe(404);
|
||||
} finally { await app.close(); }
|
||||
});
|
||||
it('loads immutable reviewed JSON into the real transport registration', async () => {
|
||||
mkdirSync('/run/secrets', { recursive: true });
|
||||
const file = '/run/secrets/prepare_worker_grants';
|
||||
const config = { grants: [{ id: 'worker', role: 'WORKFLOW_WORKER', credentialSha256: 'a'.repeat(64) }, { id: 'ci', role: 'STAGE_CI', credentialSha256: 'b'.repeat(64) }], workerIds: ['worker'], stageCiIds: ['ci'], artifactBaseUrl: 'https://stage.example.com/internal/prepare-worker/artifacts' };
|
||||
writeFileSync(file, JSON.stringify(config), { mode: 0o400 });
|
||||
try {
|
||||
const [registered] = registerPrepareWorker({ PREPARE_WORKER_ENABLED: 'true' });
|
||||
expect(registered.module.name).toBe('WorkerTransportModule');
|
||||
expect(registered.controllers?.length).toBe(1);
|
||||
const snapshot = (registered.providers as any[]).find(p => p.useValue?.artifactDirectory)?.useValue;
|
||||
expect(snapshot).toEqual({ ...config, enabled: true, artifactDirectory: '/var/lib/canina/prepare-artifacts' });
|
||||
expect(Object.isFrozen(snapshot)).toBe(true);
|
||||
expect(Object.isFrozen(snapshot.grants[0])).toBe(true);
|
||||
} finally { rmSync(file, { force: true }); }
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,10 @@
|
||||
import { DynamicModule } from '@nestjs/common';
|
||||
import { loadPrepareWorkerConfig } from './worker-config';
|
||||
import { WorkerTransportModule } from './worker-transport.module';
|
||||
|
||||
/** AppModule imports: ...registerPrepareWorker(); disabled means no module or IO. */
|
||||
export function registerPrepareWorker(env: NodeJS.ProcessEnv = process.env): DynamicModule[] {
|
||||
if (env.PREPARE_WORKER_ENABLED !== 'true') return [];
|
||||
try { return [WorkerTransportModule.register(loadPrepareWorkerConfig())]; }
|
||||
catch { throw new Error('Prepare worker startup configuration rejected'); }
|
||||
}
|
||||
@@ -0,0 +1,343 @@
|
||||
import {
|
||||
BadRequestException,
|
||||
Body,
|
||||
CanActivate,
|
||||
ConflictException,
|
||||
Controller,
|
||||
DynamicModule,
|
||||
ForbiddenException,
|
||||
Get,
|
||||
Inject,
|
||||
Injectable,
|
||||
Module,
|
||||
NotFoundException,
|
||||
Param,
|
||||
Post,
|
||||
Req,
|
||||
UnauthorizedException,
|
||||
UseGuards,
|
||||
UsePipes,
|
||||
ValidationPipe,
|
||||
} from "@nestjs/common";
|
||||
import { IsString, IsUUID, Matches, MaxLength } from "class-validator";
|
||||
import { createHash, timingSafeEqual } from "crypto";
|
||||
import { mkdir, open, FileHandle } from "fs/promises";
|
||||
import { constants } from "fs";
|
||||
import { isAbsolute } from "path";
|
||||
import { PrismaModule } from "../prisma/prisma.module";
|
||||
import { PrismaService } from "../prisma/prisma.service";
|
||||
import {
|
||||
PREPARE_WORKFLOW_CONFIG,
|
||||
PrepareWorkflowService,
|
||||
WorkflowIdentity,
|
||||
} from "../orchestration/prepare-workflow.service";
|
||||
|
||||
// This is reviewed static configuration, not a URL fetched by the transport.
|
||||
function validArtifactBase(value: string): boolean {
|
||||
try {
|
||||
const u = new URL(value);
|
||||
// DNS name only: reject all IP literals, localhost/private single-label names,
|
||||
// noncanonical paths/escapes, credentials, query, fragment and alternate ports.
|
||||
return u.protocol === "https:" && !u.username && !u.password && !u.port &&
|
||||
/^[a-z0-9](?:[a-z0-9.-]*[a-z0-9])?$/.test(u.hostname) &&
|
||||
u.hostname.includes(".") && !/^[0-9.]+$/.test(u.hostname) &&
|
||||
!/(^|\.)(localhost|local|internal|lan|home)$/.test(u.hostname) &&
|
||||
u.pathname === "/internal/prepare-worker/artifacts" &&
|
||||
value === u.origin + u.pathname;
|
||||
} catch { return false; }
|
||||
}
|
||||
const CONFIG = Symbol("WORKER_TRANSPORT_CONFIG");
|
||||
export interface WorkerTransportConfig {
|
||||
enabled: boolean;
|
||||
grants: {
|
||||
id: string;
|
||||
role: "WORKFLOW_WORKER" | "STAGE_CI";
|
||||
credentialSha256: string;
|
||||
}[];
|
||||
workerIds: string[];
|
||||
stageCiIds: string[];
|
||||
artifactDirectory: string;
|
||||
artifactBaseUrl: string;
|
||||
}
|
||||
class ClaimDto {
|
||||
@IsUUID() taskId!: string;
|
||||
}
|
||||
class LeaseDto extends ClaimDto {
|
||||
@IsUUID() runId!: string;
|
||||
@IsUUID() leaseToken!: string;
|
||||
}
|
||||
class CompleteDto extends LeaseDto {
|
||||
@IsString() @MaxLength(2048) report!: string;
|
||||
}
|
||||
class StageEvidenceDto extends ClaimDto {
|
||||
@IsUUID() runId!: string;
|
||||
@Matches(/^[a-f0-9]{40}$/) commitSha!: string;
|
||||
@Matches(/^[a-f0-9]{64}$/) sha256!: string;
|
||||
@Matches(/^[a-f0-9]{64}$/) evidenceSha256!: string;
|
||||
@Matches(/^https:\/\/[^\s]+$/) @MaxLength(2000) uri!: string;
|
||||
}
|
||||
export function tracerReport(lease: {
|
||||
taskId: string;
|
||||
runId: string;
|
||||
commitSha: string;
|
||||
}) {
|
||||
return JSON.stringify({
|
||||
kind: "DETERMINISTIC_PREPARE_TRACER",
|
||||
schemaVersion: 1,
|
||||
taskId: lease.taskId,
|
||||
runId: lease.runId,
|
||||
commitSha: lease.commitSha,
|
||||
environment: "staging",
|
||||
llmPilot: false,
|
||||
browserExecuted: false,
|
||||
shellExecuted: false,
|
||||
deploymentExecuted: false,
|
||||
checks: ["lease-bound identity", "deterministic JSON serialization"],
|
||||
actualCents: 0,
|
||||
});
|
||||
}
|
||||
@Injectable()
|
||||
class WorkerIdentityGuard implements CanActivate {
|
||||
constructor(@Inject(CONFIG) private readonly config: WorkerTransportConfig) {}
|
||||
canActivate(context: any) {
|
||||
if (!this.config.enabled)
|
||||
throw new ForbiddenException("Worker transport disabled");
|
||||
const req = context.switchToHttp().getRequest();
|
||||
const auth = req.headers.authorization;
|
||||
if (typeof auth !== "string" || !/^Bearer [\x21-\x7e]{32,512}$/.test(auth))
|
||||
throw new UnauthorizedException();
|
||||
const digest = createHash("sha256").update(auth.slice(7)).digest();
|
||||
const grant = this.config.grants.find((g) =>
|
||||
timingSafeEqual(Buffer.from(g.credentialSha256, "hex"), digest),
|
||||
);
|
||||
if (!grant) throw new UnauthorizedException();
|
||||
req.workerIdentity = { id: grant.id, role: grant.role };
|
||||
return true;
|
||||
}
|
||||
}
|
||||
@Injectable()
|
||||
class WorkerTransportService {
|
||||
constructor(
|
||||
private readonly workflow: PrepareWorkflowService,
|
||||
private readonly db: PrismaService,
|
||||
@Inject(CONFIG) private readonly config: WorkerTransportConfig,
|
||||
) {}
|
||||
// Linux descriptor-relative traversal pins each directory across renames. Never
|
||||
// realpath-check then reopen by name: that reintroduces a symlink race.
|
||||
private async artifactDir(): Promise<FileHandle> {
|
||||
let dir = await open("/", constants.O_RDONLY | constants.O_DIRECTORY);
|
||||
try {
|
||||
for (const part of this.config.artifactDirectory.split("/").filter(Boolean)) {
|
||||
const path = `/proc/self/fd/${dir.fd}/${part}`;
|
||||
try { await mkdir(path, { mode: 0o700 }); }
|
||||
catch (e: any) { if (e.code !== "EEXIST") throw e; }
|
||||
const next = await open(path, constants.O_RDONLY | constants.O_DIRECTORY | constants.O_NOFOLLOW);
|
||||
await dir.close();
|
||||
dir = next;
|
||||
}
|
||||
const stat = await dir.stat();
|
||||
if (stat.uid !== process.getuid!() || (stat.mode & 0o077) !== 0)
|
||||
throw new Error("Private owned artifact directory required");
|
||||
return dir;
|
||||
} catch (e) { await dir.close(); throw e; }
|
||||
}
|
||||
private worker(a: WorkflowIdentity) {
|
||||
if (a.role !== "WORKFLOW_WORKER" || !this.config.workerIds.includes(a.id))
|
||||
throw new ForbiddenException("Worker allowlist required");
|
||||
}
|
||||
async claim(a: WorkflowIdentity, dto: ClaimDto) {
|
||||
this.worker(a);
|
||||
const t = await this.db.orchestrationTask.findUnique({
|
||||
where: { id: dto.taskId },
|
||||
});
|
||||
// Explicit per-task pilot scope; never opportunistically consume arbitrary queued work.
|
||||
if (!t || !["READY", "RUNNING"].includes(t.state)) return { lease: null };
|
||||
if (t.budgetLimitCents !== 20)
|
||||
throw new ForbiddenException("Pilot hard task cap is 20 cents");
|
||||
return { lease: await this.workflow.claim(a, dto.taskId) };
|
||||
}
|
||||
heartbeat(a: WorkflowIdentity, dto: LeaseDto) {
|
||||
this.worker(a);
|
||||
return this.workflow.heartbeat(a, dto);
|
||||
}
|
||||
async complete(a: WorkflowIdentity, dto: CompleteDto) {
|
||||
this.worker(a);
|
||||
// Recheck stop/interrupt and fencing before artifact creation; core rechecks atomically at commit.
|
||||
await this.workflow.heartbeat(a, dto);
|
||||
const run = await this.db.orchestrationRun.findUniqueOrThrow({
|
||||
where: { id: dto.runId },
|
||||
});
|
||||
const expected = tracerReport({ ...dto, commitSha: run.commitSha });
|
||||
if (dto.report !== expected)
|
||||
throw new BadRequestException(
|
||||
"Only deterministic harmless tracer report accepted",
|
||||
);
|
||||
const sha256 = createHash("sha256").update(expected).digest("hex");
|
||||
let dir: FileHandle | undefined, file: FileHandle | undefined;
|
||||
try {
|
||||
dir = await this.artifactDir();
|
||||
file = await open(`/proc/self/fd/${dir.fd}/${sha256}.json`,
|
||||
constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, 0o600);
|
||||
await file.writeFile(expected, "utf8");
|
||||
await file.sync();
|
||||
} catch { throw new ConflictException("Artifact creation refused"); }
|
||||
finally { await file?.close(); await dir?.close(); }
|
||||
const result = await this.workflow.complete(a, {
|
||||
...dto,
|
||||
sha256,
|
||||
uri: this.config.artifactBaseUrl + "/" + sha256,
|
||||
actualCents: 0,
|
||||
});
|
||||
return {
|
||||
state: result.state,
|
||||
taskId: result.id,
|
||||
runId: dto.runId,
|
||||
sha256,
|
||||
actualCents: 0,
|
||||
};
|
||||
}
|
||||
async artifact(a: WorkflowIdentity, hash: string) {
|
||||
if (!/^[a-f0-9]{64}$/.test(hash))
|
||||
throw new BadRequestException("Invalid artifact hash");
|
||||
const registered = await this.db.orchestrationArtifact.findFirst({
|
||||
where: {
|
||||
sha256: hash, kind: "QA", uri: this.config.artifactBaseUrl + "/" + hash,
|
||||
run: { environment: "staging", status: "SUCCEEDED", task: {
|
||||
operationMode: "PREPARE", budgetLimitCents: 20,
|
||||
} },
|
||||
},
|
||||
});
|
||||
if (!registered) throw new NotFoundException("Artifact not registered");
|
||||
const run = await this.db.orchestrationRun.findUniqueOrThrow({ where: { id: registered.runId } });
|
||||
const task = await this.db.orchestrationTask.findUniqueOrThrow({ where: { id: run.taskId } });
|
||||
const outbox = await this.db.prepareWorkflowOutbox.findUnique({ where: { taskId: run.taskId } });
|
||||
if (!outbox || outbox.status !== "DONE" || outbox.runId !== run.id ||
|
||||
(a.role === "WORKFLOW_WORKER" && outbox.leaseOwner !== a.id))
|
||||
throw new NotFoundException("Artifact not registered");
|
||||
const metadata = task.metadata as any;
|
||||
if (metadata?.workflowScope !== "STAGE_PREPARE" || metadata?.commitSha !== run.commitSha)
|
||||
throw new NotFoundException("Artifact not registered");
|
||||
let bytes: string;
|
||||
let dir: FileHandle | undefined, file: FileHandle | undefined;
|
||||
try {
|
||||
dir = await this.artifactDir();
|
||||
file = await open(`/proc/self/fd/${dir.fd}/${hash}.json`,
|
||||
constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK);
|
||||
const stat = await file.stat();
|
||||
if (!stat.isFile() || stat.nlink !== 1 || stat.uid !== process.getuid!() ||
|
||||
(stat.mode & 0o077) !== 0 || stat.size > 2048)
|
||||
throw new Error("Unsafe artifact file");
|
||||
// Fixed-size descriptor read: concurrent growth cannot allocate unbounded memory.
|
||||
const buffer = Buffer.alloc(2049);
|
||||
const { bytesRead } = await file.read(buffer, 0, buffer.length, 0);
|
||||
if (bytesRead > 2048) throw new Error("Oversized artifact");
|
||||
bytes = buffer.subarray(0, bytesRead).toString("utf8");
|
||||
} catch {
|
||||
throw new NotFoundException("Artifact unavailable");
|
||||
} finally { await file?.close(); await dir?.close(); }
|
||||
if (createHash("sha256").update(bytes).digest("hex") !== hash)
|
||||
throw new ConflictException("Artifact integrity mismatch");
|
||||
if (bytes !== tracerReport({ taskId: run.taskId, runId: run.id, commitSha: run.commitSha }))
|
||||
throw new ConflictException("Artifact binding mismatch");
|
||||
return JSON.parse(bytes);
|
||||
}
|
||||
stage(a: WorkflowIdentity, dto: StageEvidenceDto) {
|
||||
return this.workflow.stageEvidence(a, dto);
|
||||
}
|
||||
}
|
||||
@Controller("internal/prepare-worker")
|
||||
@UseGuards(WorkerIdentityGuard)
|
||||
@UsePipes(
|
||||
new ValidationPipe({
|
||||
whitelist: true,
|
||||
forbidNonWhitelisted: true,
|
||||
transform: true,
|
||||
}),
|
||||
)
|
||||
class WorkerTransportController {
|
||||
constructor(private readonly service: WorkerTransportService) {}
|
||||
@Post("claim") claim(@Req() req: any, @Body() dto: ClaimDto) {
|
||||
return this.service.claim(req.workerIdentity, dto);
|
||||
}
|
||||
@Post("heartbeat") heartbeat(@Req() req: any, @Body() dto: LeaseDto) {
|
||||
return this.service.heartbeat(req.workerIdentity, dto);
|
||||
}
|
||||
@Post("complete") complete(@Req() req: any, @Body() dto: CompleteDto) {
|
||||
return this.service.complete(req.workerIdentity, dto);
|
||||
}
|
||||
@Post("stage-evidence") stage(
|
||||
@Req() req: any,
|
||||
@Body() dto: StageEvidenceDto,
|
||||
) {
|
||||
return this.service.stage(req.workerIdentity, dto);
|
||||
}
|
||||
@Get("artifacts/:hash") artifact(@Req() req: any, @Param("hash") hash: string) {
|
||||
return this.service.artifact(req.workerIdentity, hash);
|
||||
}
|
||||
}
|
||||
@Module({})
|
||||
export class WorkerTransportModule {
|
||||
static register(config: WorkerTransportConfig): DynamicModule {
|
||||
if (typeof config.enabled !== "boolean" || !Array.isArray(config.grants) ||
|
||||
!Array.isArray(config.workerIds) || !Array.isArray(config.stageCiIds))
|
||||
throw new Error("Explicit boolean enable and credential allowlists required");
|
||||
const allIds = [...config.workerIds, ...config.stageCiIds];
|
||||
if (allIds.some(id => typeof id !== "string" || !/^[a-zA-Z0-9_-]{1,100}$/.test(id)) ||
|
||||
new Set(allIds).size !== allIds.length)
|
||||
throw new Error("Invalid or overlapping worker/CI allowlists");
|
||||
const ids = new Set<string>(),
|
||||
digests = new Set<string>();
|
||||
if (
|
||||
!isAbsolute(config.artifactDirectory) || config.artifactDirectory === "/" ||
|
||||
config.artifactDirectory.split("/").some(p => p === "." || p === "..") ||
|
||||
!validArtifactBase(config.artifactBaseUrl)
|
||||
)
|
||||
throw new Error(
|
||||
"Absolute artifact directory and fixed HTTPS artifact base required",
|
||||
);
|
||||
for (const g of config.grants) {
|
||||
const allowed =
|
||||
g.role === "WORKFLOW_WORKER"
|
||||
? config.workerIds
|
||||
: g.role === "STAGE_CI"
|
||||
? config.stageCiIds
|
||||
: [];
|
||||
if (
|
||||
!/^[a-zA-Z0-9_-]{1,100}$/.test(g.id) ||
|
||||
!allowed.includes(g.id) ||
|
||||
!/^[a-f0-9]{64}$/.test(g.credentialSha256) ||
|
||||
ids.has(g.id) ||
|
||||
digests.has(g.credentialSha256) ||
|
||||
(config.workerIds.includes(g.id) && config.stageCiIds.includes(g.id))
|
||||
)
|
||||
throw new Error("Invalid or overlapping worker/CI credential grants");
|
||||
ids.add(g.id);
|
||||
digests.add(g.credentialSha256);
|
||||
}
|
||||
const snapshot = JSON.parse(JSON.stringify(config)) as WorkerTransportConfig;
|
||||
snapshot.grants.forEach(Object.freeze);
|
||||
Object.freeze(snapshot.grants);
|
||||
Object.freeze(snapshot.workerIds);
|
||||
Object.freeze(snapshot.stageCiIds);
|
||||
Object.freeze(snapshot);
|
||||
return {
|
||||
module: WorkerTransportModule,
|
||||
imports: [PrismaModule],
|
||||
controllers: [WorkerTransportController],
|
||||
providers: [
|
||||
PrepareWorkflowService,
|
||||
WorkerIdentityGuard,
|
||||
WorkerTransportService,
|
||||
{ provide: CONFIG, useValue: snapshot },
|
||||
{
|
||||
provide: PREPARE_WORKFLOW_CONFIG,
|
||||
useValue: Object.freeze({
|
||||
workerIds: snapshot.workerIds,
|
||||
stageCiIds: snapshot.stageCiIds,
|
||||
}),
|
||||
},
|
||||
],
|
||||
exports: [PrepareWorkflowService],
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,438 @@
|
||||
import { Test } from "@nestjs/testing";
|
||||
import { INestApplication } from "@nestjs/common";
|
||||
import { PrismaClient } from "@prisma/client";
|
||||
import { PrismaService } from "../prisma/prisma.service";
|
||||
import { PrepareWorkflowService } from "../orchestration/prepare-workflow.service";
|
||||
import { WorkerTransportModule, tracerReport } from "./worker-transport.module";
|
||||
import { createHash, randomUUID } from "crypto";
|
||||
import { mkdtemp, readFile, rm, writeFile, symlink, link, rename } from "fs/promises";
|
||||
import { tmpdir } from "os";
|
||||
import { join } from "path";
|
||||
import { execFile } from "child_process";
|
||||
import { promisify } from "util";
|
||||
|
||||
const enabled = process.env.WORKER_TRANSPORT_DB === "1";
|
||||
(enabled ? describe : describe.skip)(
|
||||
"isolated PostgreSQL HTTP executor tracer",
|
||||
() => {
|
||||
const db = new PrismaClient();
|
||||
const token = "test-only-worker-secret-".repeat(3);
|
||||
const ciToken = "test-only-ci-secret-".repeat(3);
|
||||
const digest = (s: string) => createHash("sha256").update(s).digest("hex");
|
||||
let app: INestApplication,
|
||||
workflow: PrepareWorkflowService,
|
||||
base: string,
|
||||
directory: string;
|
||||
let pilot: any;
|
||||
beforeAll(async () => {
|
||||
directory = await mkdtemp(join(tmpdir(), "worker-artifacts-"));
|
||||
const mod = await Test.createTestingModule({
|
||||
imports: [
|
||||
WorkerTransportModule.register({
|
||||
enabled: true,
|
||||
grants: [
|
||||
{
|
||||
id: "pilot",
|
||||
role: "WORKFLOW_WORKER",
|
||||
credentialSha256: digest(token),
|
||||
},
|
||||
{ id: "ci", role: "STAGE_CI", credentialSha256: digest(ciToken) },
|
||||
{ id: "other", role: "WORKFLOW_WORKER", credentialSha256: digest("other-worker-secret-".repeat(3)) },
|
||||
],
|
||||
workerIds: ["pilot", "other"],
|
||||
stageCiIds: ["ci"],
|
||||
artifactDirectory: directory,
|
||||
artifactBaseUrl:
|
||||
"https://isolated.invalid/internal/prepare-worker/artifacts",
|
||||
}),
|
||||
],
|
||||
})
|
||||
.overrideProvider(PrismaService)
|
||||
.useValue(db)
|
||||
.compile();
|
||||
workflow = mod.get(PrepareWorkflowService);
|
||||
app = mod.createNestApplication();
|
||||
await app.listen(0, "127.0.0.1");
|
||||
base = await app.getUrl();
|
||||
pilot = {
|
||||
runOnce: async (options: any) => {
|
||||
const program =
|
||||
'import {runOnce} from "../infra/hermes-worker/pilot.mjs"; console.log(JSON.stringify(await runOnce(JSON.parse(process.env.TEST_PILOT_OPTIONS))));';
|
||||
const result = await promisify(execFile)(
|
||||
process.execPath,
|
||||
["--input-type=module", "-e", program],
|
||||
{
|
||||
env: {
|
||||
...process.env,
|
||||
TEST_PILOT_OPTIONS: JSON.stringify(options),
|
||||
},
|
||||
timeout: 15000,
|
||||
},
|
||||
);
|
||||
return JSON.parse(result.stdout);
|
||||
},
|
||||
};
|
||||
});
|
||||
afterAll(async () => {
|
||||
await app?.close();
|
||||
await db.$disconnect();
|
||||
if (directory) await rm(directory, { recursive: true, force: true });
|
||||
});
|
||||
const owner = { id: "author", role: "Admin" },
|
||||
reviewer = { id: "reviewer", role: "SUPER_ADMIN" };
|
||||
async function proposal(queue = true, budget = 20) {
|
||||
let t = await workflow.create(
|
||||
{
|
||||
key: randomUUID(),
|
||||
title: "deterministic tracer",
|
||||
department: "TEST",
|
||||
operationMode: "PREPARE",
|
||||
budgetLimitCents: budget,
|
||||
metadata: {
|
||||
commitSha: "a".repeat(40),
|
||||
scope: "Harmless tracer only",
|
||||
acceptanceCriteria: ["writes deterministic JSON; no release"],
|
||||
},
|
||||
},
|
||||
owner,
|
||||
);
|
||||
if (queue) {
|
||||
t = await workflow.transition(
|
||||
t.id,
|
||||
{
|
||||
toState: "GATE1_PENDING",
|
||||
expectedVersion: t.version,
|
||||
idempotencyKey: randomUUID(),
|
||||
},
|
||||
owner,
|
||||
);
|
||||
t = await workflow.decide(
|
||||
t.id,
|
||||
1,
|
||||
{
|
||||
decision: "APPROVED",
|
||||
expiresAt: new Date(Date.now() + 60000).toISOString(),
|
||||
expectedVersion: t.version,
|
||||
idempotencyKey: randomUUID(),
|
||||
},
|
||||
reviewer,
|
||||
);
|
||||
t = await workflow.transition(
|
||||
t.id,
|
||||
{
|
||||
toState: "READY",
|
||||
expectedVersion: t.version,
|
||||
idempotencyKey: randomUUID(),
|
||||
},
|
||||
owner,
|
||||
);
|
||||
}
|
||||
return t;
|
||||
}
|
||||
async function post(route: string, body: any = {}, secret = token) {
|
||||
const r = await fetch(base + "/internal/prepare-worker/" + route, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
authorization: "Bearer " + secret,
|
||||
"content-type": "application/json",
|
||||
},
|
||||
body: JSON.stringify(body),
|
||||
});
|
||||
return { status: r.status, body: await r.json() };
|
||||
}
|
||||
it("rejects overlapping worker and CI grants and accepts no credentials when disabled", async () => {
|
||||
expect(() =>
|
||||
WorkerTransportModule.register({
|
||||
enabled: true,
|
||||
grants: [
|
||||
{
|
||||
id: "both",
|
||||
role: "WORKFLOW_WORKER",
|
||||
credentialSha256: digest(token),
|
||||
},
|
||||
],
|
||||
workerIds: ["both"],
|
||||
stageCiIds: ["both"],
|
||||
artifactDirectory: directory,
|
||||
artifactBaseUrl: "https://isolated.invalid/internal/prepare-worker/artifacts",
|
||||
}),
|
||||
).toThrow(/overlapping/);
|
||||
const mod = await Test.createTestingModule({
|
||||
imports: [
|
||||
WorkerTransportModule.register({
|
||||
enabled: false,
|
||||
grants: [],
|
||||
workerIds: [],
|
||||
stageCiIds: [],
|
||||
artifactDirectory: directory,
|
||||
artifactBaseUrl: "https://isolated.invalid/internal/prepare-worker/artifacts",
|
||||
}),
|
||||
],
|
||||
})
|
||||
.overrideProvider(PrismaService)
|
||||
.useValue(db)
|
||||
.compile();
|
||||
const disabled = mod.createNestApplication();
|
||||
await disabled.listen(0, "127.0.0.1");
|
||||
try {
|
||||
const r = await fetch(
|
||||
(await disabled.getUrl()) + "/internal/prepare-worker/claim",
|
||||
{
|
||||
method: "POST",
|
||||
headers: {
|
||||
authorization: "Bearer " + token,
|
||||
"content-type": "application/json",
|
||||
},
|
||||
body: JSON.stringify({ taskId: randomUUID() }),
|
||||
},
|
||||
);
|
||||
expect(r.status).toBe(403);
|
||||
} finally {
|
||||
await disabled.close();
|
||||
}
|
||||
});
|
||||
it("rejects unsafe artifact bases and freezes validated configuration", () => {
|
||||
const config = { enabled: true, grants: [], workerIds: [], stageCiIds: [], artifactDirectory: directory,
|
||||
artifactBaseUrl: "https://isolated.invalid/internal/prepare-worker/artifacts" };
|
||||
for (const artifactBaseUrl of ["https://user:pass@isolated.invalid/internal/prepare-worker/artifacts", "https://127.0.0.1/internal/prepare-worker/artifacts", "https://10.1.2.3/internal/prepare-worker/artifacts", "https://localhost/internal/prepare-worker/artifacts", "https://isolated.invalid/arbitrary", "https://isolated.invalid/x/../internal/prepare-worker/artifacts", "https://[::1]/internal/prepare-worker/artifacts"]) {
|
||||
expect(() => WorkerTransportModule.register({ ...config, artifactBaseUrl })).toThrow();
|
||||
}
|
||||
const registered = WorkerTransportModule.register(config);
|
||||
const snapshot = (registered.providers as any[]).find(p => p.useValue?.artifactBaseUrl)?.useValue;
|
||||
expect(Object.isFrozen(snapshot)).toBe(true);
|
||||
expect(Object.isFrozen(snapshot.workerIds)).toBe(true);
|
||||
expect(Object.isFrozen(snapshot.grants)).toBe(true);
|
||||
const workflowConfig = (registered.providers as any[]).find(p => p.provide === "PREPARE_WORKFLOW_CONFIG").useValue;
|
||||
expect(Object.isFrozen(workflowConfig)).toBe(true);
|
||||
expect(() => WorkerTransportModule.register({ ...config, enabled: "true" as any })).toThrow();
|
||||
});
|
||||
it("denies untrusted identities and body impersonation; dispatches nothing before Gate1", async () => {
|
||||
const t = await proposal(false);
|
||||
expect((await post("claim", {}, "wrong")).status).toBe(401);
|
||||
expect((await post("claim", { taskId: t.id }, ciToken)).status).toBe(403);
|
||||
expect(
|
||||
(await post("claim", { taskId: t.id, actorId: "pilot" })).status,
|
||||
).toBe(400);
|
||||
expect((await post("claim", { taskId: t.id })).body).toEqual({
|
||||
lease: null,
|
||||
});
|
||||
expect(await db.orchestrationRun.count({ where: { taskId: t.id } })).toBe(
|
||||
0,
|
||||
);
|
||||
});
|
||||
it("races claims, fences expired leases and reserves unknown cost against the 20 cent cap", async () => {
|
||||
const t = await proposal();
|
||||
const claims = await Promise.all([
|
||||
post("claim", { taskId: t.id }),
|
||||
post("claim", { taskId: t.id }),
|
||||
]);
|
||||
const leases = claims.map((r) => r.body.lease).filter(Boolean);
|
||||
expect(leases).toHaveLength(1);
|
||||
const lease = leases[0],
|
||||
binding = {
|
||||
taskId: t.id,
|
||||
runId: lease.runId,
|
||||
leaseToken: lease.leaseToken,
|
||||
};
|
||||
expect(
|
||||
(await post("heartbeat", { ...binding, leaseToken: randomUUID() }))
|
||||
.status,
|
||||
).toBe(403);
|
||||
expect(
|
||||
(
|
||||
await post("complete", {
|
||||
...binding,
|
||||
report: "arbitrary shell or prompt",
|
||||
})
|
||||
).status,
|
||||
).toBe(400);
|
||||
expect(
|
||||
(await post("complete", { ...binding, report: "{}", actualCents: 21 }))
|
||||
.status,
|
||||
).toBe(400);
|
||||
await db.prepareWorkflowOutbox.update({
|
||||
where: { taskId: t.id },
|
||||
data: { leaseUntil: new Date(Date.now() - 1000) },
|
||||
});
|
||||
expect((await post("heartbeat", binding)).status).toBe(403);
|
||||
expect(
|
||||
(await post("complete", { ...binding, report: "{}" })).status,
|
||||
).toBe(403);
|
||||
expect((await post("claim", { taskId: t.id })).status).toBe(403);
|
||||
expect(await db.orchestrationRun.count({ where: { taskId: t.id } })).toBe(
|
||||
1,
|
||||
);
|
||||
expect(
|
||||
(
|
||||
await db.orchestrationBudgetLedger.aggregate({
|
||||
where: { taskId: t.id },
|
||||
_sum: { amountCents: true },
|
||||
})
|
||||
)._sum.amountCents,
|
||||
).toBe(20);
|
||||
const larger = await proposal(true, 40);
|
||||
expect((await post("claim", { taskId: larger.id })).status).toBe(403);
|
||||
});
|
||||
it("rechecks kill switch and task interrupt on heartbeat and completion; separates CI authority", async () => {
|
||||
const t = await proposal(),
|
||||
lease = (await post("claim", { taskId: t.id })).body.lease;
|
||||
const binding = {
|
||||
taskId: t.id,
|
||||
runId: lease.runId,
|
||||
leaseToken: lease.leaseToken,
|
||||
};
|
||||
await db.orchestrationKillSwitch.upsert({
|
||||
where: { scope: "GLOBAL" },
|
||||
create: {
|
||||
scope: "GLOBAL",
|
||||
level: "STOP_AGENTS",
|
||||
active: true,
|
||||
reason: "isolated test",
|
||||
actorId: "test",
|
||||
},
|
||||
update: { level: "STOP_AGENTS", active: true },
|
||||
});
|
||||
try {
|
||||
expect((await post("heartbeat", binding)).status).toBe(403);
|
||||
expect(
|
||||
(await post("complete", { ...binding, report: "{}" })).status,
|
||||
).toBe(403);
|
||||
expect((await post("claim", { taskId: t.id })).status).toBe(403);
|
||||
} finally {
|
||||
await db.orchestrationKillSwitch.deleteMany();
|
||||
}
|
||||
await db.orchestrationTask.update({
|
||||
where: { id: t.id },
|
||||
data: { state: "CANCELLED" },
|
||||
});
|
||||
expect((await post("heartbeat", binding)).status).toBe(403);
|
||||
expect(
|
||||
(await post("complete", { ...binding, report: "{}" })).status,
|
||||
).toBe(403);
|
||||
expect(
|
||||
await db.orchestrationArtifact.count({ where: { runId: lease.runId } }),
|
||||
).toBe(0);
|
||||
const evidence = {
|
||||
taskId: t.id,
|
||||
runId: lease.runId,
|
||||
commitSha: "a".repeat(40),
|
||||
sha256: "b".repeat(64),
|
||||
evidenceSha256: "c".repeat(64),
|
||||
uri: "https://isolated.invalid/not-real-ci",
|
||||
};
|
||||
expect((await post("stage-evidence", evidence)).status).toBe(403);
|
||||
expect((await post("stage-evidence", evidence, ciToken)).status).toBe(
|
||||
403,
|
||||
);
|
||||
});
|
||||
it("never follows or overwrites preexisting artifact links on completion", async () => {
|
||||
const t = await proposal();
|
||||
const lease = (await post("claim", { taskId: t.id })).body.lease;
|
||||
const report = tracerReport(lease), hash = digest(report);
|
||||
const target = join(directory, "sentinel");
|
||||
await writeFile(target, "do not overwrite");
|
||||
const file = join(directory, hash + ".json");
|
||||
await symlink(target, file);
|
||||
const result = await post("complete", { taskId: t.id, runId: lease.runId, leaseToken: lease.leaseToken, report });
|
||||
expect(await readFile(target, "utf8")).toBe("do not overwrite");
|
||||
expect(result.status).toBe(409);
|
||||
expect(await db.orchestrationArtifact.count({ where: { runId: lease.runId } })).toBe(0);
|
||||
await rm(file);
|
||||
await link(target, file);
|
||||
expect((await post("complete", { taskId: t.id, runId: lease.runId, leaseToken: lease.leaseToken, report })).status).toBe(409);
|
||||
expect(await readFile(target, "utf8")).toBe("do not overwrite");
|
||||
});
|
||||
it("runs actual HTTP deterministic executor to VERIFYING with retrievable content-addressed artifact", async () => {
|
||||
const t = await proposal();
|
||||
const result = await pilot.runOnce({
|
||||
baseUrl: base,
|
||||
credential: token,
|
||||
taskId: t.id,
|
||||
allowLoopbackHttp: true,
|
||||
});
|
||||
expect(result.state).toBe("VERIFYING");
|
||||
const row = await db.orchestrationTask.findUniqueOrThrow({
|
||||
where: { id: t.id },
|
||||
});
|
||||
expect(row.state).toBe("VERIFYING");
|
||||
const artifact = await db.orchestrationArtifact.findFirstOrThrow({
|
||||
where: { run: { taskId: t.id } },
|
||||
});
|
||||
const bytes = await readFile(join(directory, artifact.sha256 + ".json"));
|
||||
expect(digest(bytes.toString())).toBe(artifact.sha256);
|
||||
expect(JSON.parse(bytes.toString())).toMatchObject({
|
||||
kind: "DETERMINISTIC_PREPARE_TRACER",
|
||||
commitSha: "a".repeat(40),
|
||||
environment: "staging",
|
||||
llmPilot: false,
|
||||
});
|
||||
const get = await fetch(
|
||||
base + "/internal/prepare-worker/artifacts/" + artifact.sha256,
|
||||
{ headers: { authorization: "Bearer " + token } },
|
||||
);
|
||||
expect(get.status).toBe(200);
|
||||
expect(await get.text()).toBe(bytes.toString());
|
||||
const artifactGet = (secret: string) => fetch(base + "/internal/prepare-worker/artifacts/" + artifact.sha256,
|
||||
{ headers: { authorization: "Bearer " + secret } });
|
||||
expect((await artifactGet("other-worker-secret-".repeat(3))).status).toBe(404);
|
||||
expect((await artifactGet(ciToken)).status).toBe(200);
|
||||
const foreignHash = digest("{}");
|
||||
await writeFile(join(directory, foreignHash + ".json"), "{}");
|
||||
await db.orchestrationArtifact.create({ data: { runId: artifact.runId, kind: "QA", sha256: foreignHash, uri: "https://foreign.invalid/" + foreignHash } });
|
||||
expect((await fetch(base + "/internal/prepare-worker/artifacts/" + foreignHash, { headers: { authorization: "Bearer " + token } })).status).toBe(404);
|
||||
const moved = directory + "-moved";
|
||||
await rename(directory, moved);
|
||||
try {
|
||||
await symlink(moved, directory);
|
||||
expect((await artifactGet(token)).status).toBe(404);
|
||||
} finally { await rm(directory, { force: true }); await rename(moved, directory); }
|
||||
const file = join(directory, artifact.sha256 + ".json"), saved = file + ".saved";
|
||||
await rename(file, saved);
|
||||
try {
|
||||
await symlink(saved, file);
|
||||
expect((await artifactGet(token)).status).toBe(404);
|
||||
await rm(file);
|
||||
await link(saved, file);
|
||||
expect((await artifactGet(token)).status).toBe(404);
|
||||
await rm(file);
|
||||
} finally { await rm(file, { force: true }); await rename(saved, file); }
|
||||
const cost = await db.orchestrationBudgetLedger.aggregate({
|
||||
where: { taskId: t.id },
|
||||
_sum: { amountCents: true },
|
||||
});
|
||||
expect(cost._sum.amountCents).toBe(0);
|
||||
expect(
|
||||
await db.orchestrationAttestation.findFirst({
|
||||
where: { run: { taskId: t.id }, kind: "STAGE" },
|
||||
}),
|
||||
).toBeNull();
|
||||
const evidence = {
|
||||
taskId: t.id,
|
||||
state: row.state,
|
||||
sha256: artifact.sha256,
|
||||
artifact: JSON.parse(bytes.toString()),
|
||||
httpGet: get.status,
|
||||
costCents: cost._sum.amountCents,
|
||||
uriExternalReachability: "NOT_TESTED_ISOLATED_FIXTURE_HOST",
|
||||
};
|
||||
console.log("TRACER_EVIDENCE", JSON.stringify(evidence));
|
||||
if (process.env.TRACER_EVIDENCE_PATH)
|
||||
await writeFile(
|
||||
process.env.TRACER_EVIDENCE_PATH,
|
||||
JSON.stringify(evidence, null, 2),
|
||||
);
|
||||
await writeFile(
|
||||
join(directory, artifact.sha256 + ".json"),
|
||||
'{"tampered":true}',
|
||||
);
|
||||
expect(
|
||||
(
|
||||
await fetch(
|
||||
base + "/internal/prepare-worker/artifacts/" + artifact.sha256,
|
||||
{ headers: { authorization: "Bearer " + token } },
|
||||
)
|
||||
).status,
|
||||
).toBe(409);
|
||||
});
|
||||
},
|
||||
);
|
||||
@@ -0,0 +1,43 @@
|
||||
# Bounded Stage PREPARE workflow foundation
|
||||
|
||||
This slice is NOT a runtime executor, deployment mechanism, or full v4 completion. It enables authenticated proposal/Gate1/queue APIs and provides internal worker/CI service methods. Default worker/CI identity allowlists are empty: no browser or production execution route exists.
|
||||
|
||||
## Panel HTTP contract
|
||||
All routes require existing JWT authentication with `Admin` or `SUPER_ADMIN`. Actor is exclusively `req.user.id ?? req.user.sub`, role exclusively `req.user.role`. Top-level unknown mutation fields return 400; client actor/profile fields are never trusted.
|
||||
|
||||
Base `/admin/orchestration`:
|
||||
|
||||
- `GET /tasks`: task array.
|
||||
- `GET /tasks/:id`: task object.
|
||||
- `GET /overview`: existing inventory items plus capabilities `{mutations:true, prepare:true, release:false, gate2Decisions:false, chat:false, runtime:false}`, reason `bounded_stage_prepare_only`.
|
||||
- `POST /tasks`: existing CreateOrchestrationTaskDto `{key,title,department,operationMode:"PREPARE",budgetLimitCents,metadata:{commitSha,scope,acceptanceCriteria}}`; commitSha is required lowercase 40hex; scope is a non-blank string of at most 2000 characters; acceptanceCriteria is 1..20 non-blank strings of at most 500 characters each; budget is integer 20..10000 cents. Returns 201 projected task, state DRAFT/version1. Validated scope and acceptance criteria are persisted exactly and remain unchanged by workflow transitions. The internal `workflowScope:"STAGE_PREPARE"`, environment and createdBy are server-owned; input profile/actor/other arbitrary keys are discarded. Key max120, title max255, department max50. Proposal key is unique; sequential or concurrent duplicate keys return 409, without updating the original task or appending another proposal event. Fetch/reconcile the existing task rather than retry blindly.
|
||||
- `PUT /tasks/:id/transition`: `{toState:"GATE1_PENDING"|"READY",expectedVersion,idempotencyKey,reason?}`. Only DRAFT->GATE1_PENDING and GATE1_APPROVED->READY. Owner or SUPER_ADMIN. Returns 200 projected task. READY atomically creates dispatch outbox. Requires unexpired Gate1.
|
||||
- `POST /tasks/:id/gates/1/decide`: existing GateDecisionDto `{decision:"APPROVED"|"REJECTED",expectedVersion,idempotencyKey,expiresAt,notes?}`. SUPER_ADMIN independent from proposer. Future expiry <=24h. Returns 201 **projected task**, not decision object.
|
||||
- Gate2 decision, budget writes/reconcile, command routes (including release), chat and kill-switch HTTP mutations remain 403. Runtime resource reads remain 503.
|
||||
|
||||
Task projection keeps existing scalar fields and safe nested runs/artifacts, decisions, events and ledger. Adds only `metadata:{scope,acceptanceCriteria,environment:"staging",commitSha}` for bounded proposals, otherwise `{}`. Never includes createdBy, workflowScope, payloads, URI, signatures, tokens or lease credentials. Pre-fix rows with the legacy scope sentinel remain readable/operable but have no recovered acceptance criteria: do not represent those as complete immutable scope review evidence.
|
||||
|
||||
`availableActions:string[]` is state/identity-derived: DRAFT owner -> `SUBMIT_GATE1`; GATE1_PENDING independent SUPER_ADMIN -> `APPROVE_GATE1`,`REJECT_GATE1`; GATE1_APPROVED owner/SUPER_ADMIN -> `QUEUE_PREPARE`; all worker/verification/Gate2 and legacy states -> `[]`. Actions indicate workflow eligibility, not a guarantee: current kill switch, approval expiry and concurrent versions are rechecked transactionally on mutation. Panel must not infer actions from state or send actor identity.
|
||||
|
||||
Common errors: 400 invalid DTO/binding; 403 wrong scope/identity/independence/kill/budget/expiry; 409 duplicate proposal key, stale version or idempotency payload mismatch. Transition/decision replay returns current persisted task and does not duplicate side effects.
|
||||
|
||||
## Internal service contracts (NOT browser endpoints)
|
||||
`PrepareWorkflowService` is exported by OrchestrationModule. Trusted backend composition must override `PREPARE_WORKFLOW_CONFIG` with explicit `workerIds` and `stageCiIds`, default both empty. Do not populate from request bodies or user-selected runtime profiles. Worker identities require role WORKFLOW_WORKER AND ID allowlist; Stage identities require STAGE_CI AND separate ID allowlist. Credential/session provisioning and transport remain the integrating runtime owner's responsibility.
|
||||
|
||||
- `claim(identity, taskId?) -> null | {taskId,runId,attempt,leaseToken,leaseUntil,commitSha,environment:"staging",maxCostCents:20}`. Transactional task row lock with SKIP LOCKED, immutable run binding, 30s lease, max3 attempts. Each attempt reserves **a fixed hard upper bound of 20 cents**, not an estimated or measured departmental cost. Unknown cost on expired attempts remains reserved conservatively.
|
||||
- `heartbeat(identity,{taskId,runId,leaseToken}) -> {leaseUntil}`. Same owner/token, unexpired active lease required.
|
||||
- `complete(identity,{taskId,runId,leaseToken,sha256,uri,actualCents}) -> task`. QA success ingestion only; lowercase sha256, HTTPS URI, actual cost integer0..20 from trusted worker. Atomically records immutable QA artifact/attestation, SUCCEEDED run, reservation reconciliation, DONE outbox and VERIFYING task. Expired/stale/foreign token cannot finish. Not replayable after success; caller must read persisted task/run after response loss.
|
||||
- `stageEvidence(identity,{taskId,runId,sha256,commitSha,evidenceSha256,uri}) -> task`. Independently allowlisted Stage CI identity, successful exact staging commit/run, registered QA subject, independent QA signer. Atomically records Stage artifact/attestation and reaches GATE2_PENDING. No release or Gate2 approval. Does not perform or claim deployment itself. Caller supplies actual trusted CI evidence, not browser assertions.
|
||||
|
||||
## Verification and explicit gaps
|
||||
Real isolated PostgreSQL16, no published ports/no live volumes. Added migrations applied with ON_ERROR_STOP. TDD RED logs cover missing workflow behavior, heartbeat/QA behavior and HTTP enablement. Full backend regression excluding existing legacy release-containing orchestration.integration.spec: 106 suites /851 tests including seven real-DB workflow cases, build exit0. No RELEASE_AUTHORIZE service operation executed.
|
||||
|
||||
DB regressions: concurrent claim winner, queue race/replay mismatch, hard budget rollback, stale lease fencing, max3 attempts, conservative expired costs, kill switch, self approval, trusted IDs, QA reconciliation and Stage evidence to Gate2 pending. Test fixture artifact URIs/hashes are not real deployment evidence.
|
||||
|
||||
Remaining: actual authenticated runtime worker transport/executor and pinned CI evidence transport; shared department/company budget model and locking (NOT implemented or represented by fake totals); retry backoff/dead-letter UX/cancellation; hard total execution wall-time; durable request replay of completion/evidence; kill-switch concurrent insertion/version locking; migration rehearsal with full legacy schema; real Stage CI artifact/deployment verification. No push/deploy/live DB/runtime edits. Production release remains denied at HTTP boundary; unsafe legacy core release method is deliberately untouched and must never be called by this slice.
|
||||
|
||||
## Contract repair verification
|
||||
|
||||
Actual frontend inspection confirmed proposal submission sent human scope and acceptanceCriteria but omitted required commitSha. Frontend repair is separately assigned; no frontend file changed here. New proposals now require all three fields. Internal Stage restriction uses a separate server-owned workflowScope marker rather than overwriting user scope. No metadata update route exists in this bounded workflow.
|
||||
|
||||
TDD RED→GREEN against isolated PostgreSQL16 reproduced dropped scope/criteria, nine invalid metadata acceptances (including array-to-string commit coercion), and duplicate-key P2002 leakage. Final database suite: 24 cases, including authenticated HTTP create/list/detail persistence, 400 validation, and 409 duplicate responses. Full allowed backend regression: 106 suites / 868 tests passed; legacy orchestration.integration.spec.ts excluded to avoid executing RELEASE_AUTHORIZE. Build exit0. Scoped lint remains failing on inherited require-await/no-unnecessary-type-assertion errors (13 errors); no lint suppression or config change applied. Disposable DB only; no live edits/push/deploy.
|
||||
@@ -92,7 +92,7 @@ describe('integration read boundary', () => {
|
||||
[method]('/admin/orchestration/' + path)
|
||||
.set('Authorization', 'Admin')
|
||||
.send({})
|
||||
.expect(403);
|
||||
.expect(path === 'tasks' || path.endsWith('/transition') ? 400 : 403);
|
||||
}
|
||||
for (const name of [
|
||||
'createTask',
|
||||
|
||||
@@ -1,5 +1,9 @@
|
||||
import {
|
||||
Controller,
|
||||
Body,
|
||||
Req,
|
||||
Optional,
|
||||
ValidationPipe,
|
||||
Get,
|
||||
Param,
|
||||
Post,
|
||||
@@ -13,6 +17,20 @@ import { JwtAuthGuard } from '../auth/jwt-auth.guard';
|
||||
import { RolesGuard } from '../common/guards/roles.guard';
|
||||
import { Roles } from '../common/decorators/roles.decorator';
|
||||
import { OrchestrationService } from './orchestration.service';
|
||||
import {
|
||||
PrepareWorkflowService,
|
||||
WorkflowIdentity,
|
||||
} from './prepare-workflow.service';
|
||||
import {
|
||||
CreateOrchestrationTaskDto,
|
||||
TransitionTaskDto,
|
||||
GateDecisionDto,
|
||||
} from './dto/orchestration.dto';
|
||||
const workflowValidation = new ValidationPipe({
|
||||
transform: true,
|
||||
whitelist: true,
|
||||
forbidNonWhitelisted: true,
|
||||
});
|
||||
|
||||
// Explicit response allowlists: never serialize arbitrary JSON, signatures, URIs or BigInt.
|
||||
function pick(row: any, keys: string[]) {
|
||||
@@ -73,12 +91,37 @@ function project(row: any) {
|
||||
@Roles('Admin', 'SUPER_ADMIN')
|
||||
@Controller('admin/orchestration')
|
||||
export class OrchestrationController {
|
||||
constructor(private readonly service: OrchestrationService) {}
|
||||
@Get('tasks') async all() {
|
||||
return (await this.service.findAll()).map(project);
|
||||
constructor(
|
||||
private readonly service: OrchestrationService,
|
||||
@Optional() private readonly workflow?: PrepareWorkflowService,
|
||||
) {}
|
||||
private actor(req: any): WorkflowIdentity {
|
||||
const id = req?.user?.id ?? req?.user?.sub;
|
||||
if (
|
||||
typeof id !== 'string' ||
|
||||
!id ||
|
||||
!['Admin', 'SUPER_ADMIN'].includes(req?.user?.role)
|
||||
)
|
||||
throw new ForbiddenException('Authenticated workflow identity required');
|
||||
return { id, role: req.user.role };
|
||||
}
|
||||
@Get('tasks/:id') async one(@Param('id') id: string) {
|
||||
return project(await this.service.findOne(id));
|
||||
private safe(row: any, req: any) {
|
||||
const view = project(row);
|
||||
if (!this.workflow) return view;
|
||||
const w = this.workflow.project(row, this.actor(req));
|
||||
return {
|
||||
...view,
|
||||
metadata: w.metadata,
|
||||
availableActions: w.availableActions,
|
||||
};
|
||||
}
|
||||
@Get('tasks') async all(@Req() req: any) {
|
||||
return (await this.service.findAll()).map((row: any) =>
|
||||
this.safe(row, req),
|
||||
);
|
||||
}
|
||||
@Get('tasks/:id') async one(@Param('id') id: string, @Req() req: any) {
|
||||
return this.safe(await this.service.findOne(id), req);
|
||||
}
|
||||
@Get('overview') async overview() {
|
||||
const tasks = await this.service.findAll();
|
||||
@@ -87,8 +130,17 @@ export class OrchestrationController {
|
||||
{ id: 'tasks', title: 'Persisted task inventory', count: tasks.length },
|
||||
],
|
||||
measuredAt: new Date().toISOString(),
|
||||
capabilities: { mutations: false, chat: false, runtime: false },
|
||||
reason: 'security_hardening_pending',
|
||||
capabilities: {
|
||||
mutations: !!this.workflow,
|
||||
prepare: !!this.workflow,
|
||||
release: false,
|
||||
gate2Decisions: false,
|
||||
chat: false,
|
||||
runtime: false,
|
||||
},
|
||||
reason: this.workflow
|
||||
? 'bounded_stage_prepare_only'
|
||||
: 'security_hardening_pending',
|
||||
};
|
||||
}
|
||||
@Get('releases') async releases() {
|
||||
@@ -120,15 +172,40 @@ export class OrchestrationController {
|
||||
'No persisted model policy read contract available',
|
||||
);
|
||||
}
|
||||
// The committed legacy core is not safe to expose. No env switch or role may bypass this boundary.
|
||||
@Post('tasks') create() {
|
||||
return this.deny();
|
||||
// Only the separate bounded PREPARE service is exposed; legacy commands remain denied.
|
||||
@Post('tasks') async create(
|
||||
@Body(workflowValidation) dto: CreateOrchestrationTaskDto,
|
||||
@Req() req: any,
|
||||
) {
|
||||
if (!this.workflow) return this.deny();
|
||||
return this.safe(await this.workflow.create(dto, this.actor(req)), req);
|
||||
}
|
||||
@Put('tasks/:id/transition') transition() {
|
||||
return this.deny();
|
||||
@Put('tasks/:id/transition') async transition(
|
||||
@Param('id') id: string,
|
||||
@Body(workflowValidation) dto: TransitionTaskDto,
|
||||
@Req() req: any,
|
||||
) {
|
||||
if (!this.workflow) return this.deny();
|
||||
return this.safe(
|
||||
await this.workflow.transition(id, dto, this.actor(req)),
|
||||
req,
|
||||
);
|
||||
}
|
||||
@Post('tasks/:id/gates/:gate/decide') decide() {
|
||||
return this.deny();
|
||||
@Post('tasks/:id/gates/:gate/decide') async decide(
|
||||
@Param('id') id: string,
|
||||
@Param('gate') gate: string,
|
||||
@Body() body: unknown,
|
||||
@Req() req: any,
|
||||
) {
|
||||
if (!this.workflow || gate !== '1') return this.deny();
|
||||
const dto = await workflowValidation.transform(body, {
|
||||
type: 'body',
|
||||
metatype: GateDecisionDto,
|
||||
});
|
||||
return this.safe(
|
||||
await this.workflow.decide(id, 1, dto, this.actor(req)),
|
||||
req,
|
||||
);
|
||||
}
|
||||
@Post('tasks/:id/budget') budget() {
|
||||
return this.deny();
|
||||
|
||||
@@ -1,3 +1,7 @@
|
||||
import {
|
||||
PrepareWorkflowService,
|
||||
PREPARE_WORKFLOW_CONFIG,
|
||||
} from './prepare-workflow.service';
|
||||
import { Module } from '@nestjs/common';
|
||||
import { PrismaModule } from '../prisma/prisma.module';
|
||||
import { KillSwitchService } from './kill-switch.service';
|
||||
@@ -7,7 +11,21 @@ import { OrchestrationService } from './orchestration.service';
|
||||
@Module({
|
||||
imports: [PrismaModule],
|
||||
controllers: [OrchestrationController],
|
||||
providers: [OrchestrationService, KillSwitchService, ModelTierPolicyService],
|
||||
exports: [OrchestrationService, KillSwitchService, ModelTierPolicyService],
|
||||
providers: [
|
||||
OrchestrationService,
|
||||
KillSwitchService,
|
||||
ModelTierPolicyService,
|
||||
PrepareWorkflowService,
|
||||
{
|
||||
provide: PREPARE_WORKFLOW_CONFIG,
|
||||
useValue: { workerIds: [], stageCiIds: [] },
|
||||
},
|
||||
],
|
||||
exports: [
|
||||
OrchestrationService,
|
||||
KillSwitchService,
|
||||
ModelTierPolicyService,
|
||||
PrepareWorkflowService,
|
||||
],
|
||||
})
|
||||
export class OrchestrationModule {}
|
||||
@@ -0,0 +1,137 @@
|
||||
import { Test } from '@nestjs/testing';
|
||||
import { INestApplication } from '@nestjs/common';
|
||||
import request from 'supertest';
|
||||
import { OrchestrationController } from './orchestration.controller';
|
||||
import { OrchestrationService } from './orchestration.service';
|
||||
import { PrepareWorkflowService } from './prepare-workflow.service';
|
||||
import { JwtAuthGuard } from '../auth/jwt-auth.guard';
|
||||
import { RolesGuard } from '../common/guards/roles.guard';
|
||||
|
||||
describe('bounded workflow HTTP contracts', () => {
|
||||
let app: INestApplication;
|
||||
const t = {
|
||||
id: 't',
|
||||
key: 'k',
|
||||
state: 'DRAFT',
|
||||
version: 1,
|
||||
metadata: {
|
||||
scope: 'STAGE_PREPARE',
|
||||
environment: 'staging',
|
||||
commitSha: 'a'.repeat(40),
|
||||
token: 'secret',
|
||||
},
|
||||
};
|
||||
const workflow = {
|
||||
create: jest.fn(async () => t),
|
||||
transition: jest.fn(async () => t),
|
||||
decide: jest.fn(async () => t),
|
||||
project: jest.fn((row: any) => ({
|
||||
...row,
|
||||
metadata: {
|
||||
scope: 'STAGE_PREPARE',
|
||||
environment: 'staging',
|
||||
commitSha: 'a'.repeat(40),
|
||||
},
|
||||
availableActions: ['SUBMIT_GATE1'],
|
||||
})),
|
||||
};
|
||||
const core = {
|
||||
findAll: async () => [t],
|
||||
findOne: async () => t,
|
||||
command: jest.fn(),
|
||||
};
|
||||
beforeAll(async () => {
|
||||
const m = await Test.createTestingModule({
|
||||
controllers: [OrchestrationController],
|
||||
providers: [
|
||||
{ provide: OrchestrationService, useValue: core },
|
||||
{ provide: PrepareWorkflowService, useValue: workflow },
|
||||
RolesGuard,
|
||||
],
|
||||
})
|
||||
.overrideGuard(JwtAuthGuard)
|
||||
.useValue({
|
||||
canActivate(c: any) {
|
||||
const req = c.switchToHttp().getRequest();
|
||||
if (!req.headers.authorization) return false;
|
||||
req.user = {
|
||||
sub: 'verified-subject',
|
||||
role: req.headers.authorization,
|
||||
};
|
||||
return true;
|
||||
},
|
||||
})
|
||||
.compile();
|
||||
app = m.createNestApplication();
|
||||
await app.init();
|
||||
});
|
||||
afterAll(async () => {
|
||||
await app.close();
|
||||
});
|
||||
it('accepts existing proposal DTO, derives actor from JWT sub and returns projected task', async () => {
|
||||
const dto = {
|
||||
key: 'k',
|
||||
title: 'T',
|
||||
department: 'ENGINEERING',
|
||||
operationMode: 'PREPARE',
|
||||
budgetLimitCents: 20,
|
||||
metadata: {
|
||||
commitSha: 'a'.repeat(40),
|
||||
scope: 'Staging audit only',
|
||||
acceptanceCriteria: ['QA passes'],
|
||||
},
|
||||
};
|
||||
const r = await request(app.getHttpServer())
|
||||
.post('/admin/orchestration/tasks')
|
||||
.set('Authorization', 'Admin')
|
||||
.send(dto)
|
||||
.expect(201);
|
||||
expect(workflow.create).toHaveBeenCalledWith(dto, {
|
||||
id: 'verified-subject',
|
||||
role: 'Admin',
|
||||
});
|
||||
expect(r.body.availableActions).toEqual(['SUBMIT_GATE1']);
|
||||
expect(JSON.stringify(r.body)).not.toMatch(/token|secret/);
|
||||
await request(app.getHttpServer())
|
||||
.post('/admin/orchestration/tasks')
|
||||
.set('Authorization', 'Admin')
|
||||
.send({ ...dto, actorId: 'browser' })
|
||||
.expect(400);
|
||||
});
|
||||
it('routes fixed transition/gate contracts and rejects invalid gates plus release without invoking core', async () => {
|
||||
await request(app.getHttpServer())
|
||||
.put('/admin/orchestration/tasks/t/transition')
|
||||
.set('Authorization', 'Admin')
|
||||
.send({
|
||||
toState: 'GATE1_PENDING',
|
||||
expectedVersion: 1,
|
||||
idempotencyKey: 'submit',
|
||||
})
|
||||
.expect(200);
|
||||
await request(app.getHttpServer())
|
||||
.post('/admin/orchestration/tasks/t/gates/1/decide')
|
||||
.set('Authorization', 'SUPER_ADMIN')
|
||||
.send({
|
||||
decision: 'APPROVED',
|
||||
expectedVersion: 2,
|
||||
idempotencyKey: 'approval',
|
||||
expiresAt: new Date(Date.now() + 60000).toISOString(),
|
||||
})
|
||||
.expect(201);
|
||||
expect(workflow.decide).toHaveBeenCalledWith('t', 1, expect.any(Object), {
|
||||
id: 'verified-subject',
|
||||
role: 'SUPER_ADMIN',
|
||||
});
|
||||
await request(app.getHttpServer())
|
||||
.post('/admin/orchestration/tasks/t/gates/2/decide')
|
||||
.set('Authorization', 'SUPER_ADMIN')
|
||||
.send({})
|
||||
.expect(403);
|
||||
await request(app.getHttpServer())
|
||||
.post('/admin/orchestration/tasks/t/commands/RELEASE_AUTHORIZE')
|
||||
.set('Authorization', 'SUPER_ADMIN')
|
||||
.send({})
|
||||
.expect(403);
|
||||
expect(core.command).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,559 @@
|
||||
import {
|
||||
Injectable,
|
||||
Inject,
|
||||
ForbiddenException,
|
||||
BadRequestException,
|
||||
ConflictException,
|
||||
NotFoundException,
|
||||
} from '@nestjs/common';
|
||||
import { randomUUID } from 'crypto';
|
||||
import { PrismaService } from '../prisma/prisma.service';
|
||||
export const PREPARE_WORKFLOW_CONFIG = 'PREPARE_WORKFLOW_CONFIG';
|
||||
export interface WorkflowIdentity {
|
||||
id: string;
|
||||
role: string;
|
||||
}
|
||||
export interface WorkflowConfig {
|
||||
workerIds: string[];
|
||||
stageCiIds: string[];
|
||||
}
|
||||
|
||||
/** Bounded Stage preparation only. No release executor, production scope or browser worker identities. */
|
||||
@Injectable()
|
||||
export class PrepareWorkflowService {
|
||||
constructor(
|
||||
@Inject(PrismaService) private readonly db: any,
|
||||
@Inject(PREPARE_WORKFLOW_CONFIG) private readonly config: WorkflowConfig,
|
||||
) {}
|
||||
private identity(a: WorkflowIdentity, roles: string[]) {
|
||||
if (
|
||||
!a?.id ||
|
||||
typeof a.id !== 'string' ||
|
||||
a.id.length > 100 ||
|
||||
!roles.includes(a.role)
|
||||
)
|
||||
throw new ForbiddenException('Authenticated workflow identity required');
|
||||
}
|
||||
private worker(a: WorkflowIdentity) {
|
||||
this.identity(a, ['WORKFLOW_WORKER']);
|
||||
if (!this.config.workerIds.includes(a.id))
|
||||
throw new ForbiddenException('Worker identity not provisioned');
|
||||
}
|
||||
private async allowed(tx: any, claiming = false) {
|
||||
const rows =
|
||||
await tx.$queryRaw`SELECT level, active FROM orchestration_kill_switches WHERE scope = 'GLOBAL' FOR SHARE`;
|
||||
if (rows[0]?.active && !(rows[0].level === 'FREEZE_NEW_WORK' && !claiming))
|
||||
throw new ForbiddenException('Kill switch blocks workflow');
|
||||
}
|
||||
private async task(tx: any, id: string) {
|
||||
await tx.$queryRaw`SELECT id FROM orchestration_tasks WHERE id = ${id}::uuid FOR UPDATE`;
|
||||
const t = await tx.orchestrationTask.findUnique({ where: { id } });
|
||||
if (!t) throw new NotFoundException('Task not found');
|
||||
if (
|
||||
t.operationMode !== 'PREPARE' ||
|
||||
(t.metadata?.workflowScope ?? t.metadata?.scope) !== 'STAGE_PREPARE'
|
||||
)
|
||||
throw new ForbiddenException('Not a bounded Stage proposal');
|
||||
return t;
|
||||
}
|
||||
private async event(
|
||||
tx: any,
|
||||
t: any,
|
||||
state: string,
|
||||
a: WorkflowIdentity,
|
||||
key: string,
|
||||
type: string,
|
||||
fingerprint: string,
|
||||
) {
|
||||
const result = await tx.orchestrationTask.update({
|
||||
where: { id: t.id },
|
||||
data: { state, version: { increment: 1 } },
|
||||
});
|
||||
await tx.orchestrationEvent.create({
|
||||
data: {
|
||||
taskId: t.id,
|
||||
sequence: result.version,
|
||||
actorId: a.id,
|
||||
eventType: type,
|
||||
fromState: t.state,
|
||||
toState: state,
|
||||
idempotencyKey: key,
|
||||
payload: { fingerprint },
|
||||
},
|
||||
});
|
||||
return result;
|
||||
}
|
||||
private async replay(tx: any, key: string, fingerprint: string) {
|
||||
const e = await tx.orchestrationEvent.findUnique({
|
||||
where: { idempotencyKey: key },
|
||||
});
|
||||
if (e && e.payload?.fingerprint !== fingerprint)
|
||||
throw new ConflictException('Idempotency payload mismatch');
|
||||
return e;
|
||||
}
|
||||
private version(t: any, dto: any) {
|
||||
if (
|
||||
!Number.isSafeInteger(dto.expectedVersion) ||
|
||||
dto.expectedVersion !== t.version
|
||||
)
|
||||
throw new ConflictException('Stale task version');
|
||||
if (
|
||||
typeof dto.idempotencyKey !== 'string' ||
|
||||
!dto.idempotencyKey.length ||
|
||||
dto.idempotencyKey.length > 120
|
||||
)
|
||||
throw new BadRequestException('Invalid idempotency key');
|
||||
}
|
||||
async create(dto: any, a: WorkflowIdentity) {
|
||||
this.identity(a, ['Admin', 'SUPER_ADMIN']);
|
||||
const metadata = dto?.metadata;
|
||||
const boundedText = (value: unknown, max: number) =>
|
||||
typeof value === 'string' &&
|
||||
value.trim().length > 0 &&
|
||||
value.length <= max;
|
||||
if (
|
||||
!metadata ||
|
||||
typeof metadata !== 'object' ||
|
||||
Array.isArray(metadata) ||
|
||||
typeof metadata.commitSha !== 'string' ||
|
||||
!boundedText(metadata.scope, 2000) ||
|
||||
!Array.isArray(metadata.acceptanceCriteria) ||
|
||||
metadata.acceptanceCriteria.length < 1 ||
|
||||
metadata.acceptanceCriteria.length > 20 ||
|
||||
!metadata.acceptanceCriteria.every((criterion: unknown) =>
|
||||
boundedText(criterion, 500),
|
||||
)
|
||||
)
|
||||
throw new BadRequestException(
|
||||
'PREPARE requires commitSha, scope 1..2000 characters and 1..20 acceptance criteria of 1..500 characters',
|
||||
);
|
||||
if (
|
||||
dto.operationMode !== 'PREPARE' ||
|
||||
!/^[a-f0-9]{40}$/.test(dto.metadata?.commitSha ?? '') ||
|
||||
!Number.isSafeInteger(dto.budgetLimitCents) ||
|
||||
dto.budgetLimitCents < 20 ||
|
||||
dto.budgetLimitCents > 10000
|
||||
)
|
||||
throw new BadRequestException(
|
||||
'PREPARE requires immutable 40hex commit and budget 20..10000 cents',
|
||||
);
|
||||
if (
|
||||
!dto.key ||
|
||||
dto.key.length > 120 ||
|
||||
!dto.title ||
|
||||
dto.title.length > 255 ||
|
||||
!dto.department ||
|
||||
dto.department.length > 50
|
||||
)
|
||||
throw new BadRequestException('Invalid proposal fields');
|
||||
return this.db
|
||||
.$transaction(async (tx: any) => {
|
||||
await this.allowed(tx, true);
|
||||
const t = await tx.orchestrationTask.create({
|
||||
data: {
|
||||
key: dto.key,
|
||||
title: dto.title,
|
||||
department: dto.department,
|
||||
operationMode: 'PREPARE',
|
||||
budgetLimitCents: dto.budgetLimitCents,
|
||||
metadata: {
|
||||
workflowScope: 'STAGE_PREPARE',
|
||||
scope: dto.metadata.scope,
|
||||
acceptanceCriteria: [...dto.metadata.acceptanceCriteria],
|
||||
environment: 'staging',
|
||||
commitSha: dto.metadata.commitSha,
|
||||
createdBy: a.id,
|
||||
},
|
||||
state: 'DRAFT',
|
||||
},
|
||||
});
|
||||
await tx.orchestrationEvent.create({
|
||||
data: {
|
||||
taskId: t.id,
|
||||
sequence: 1,
|
||||
eventType: 'PREPARE_PROPOSED',
|
||||
actorId: a.id,
|
||||
toState: 'DRAFT',
|
||||
},
|
||||
});
|
||||
return t;
|
||||
})
|
||||
.catch((error: any) => {
|
||||
if (
|
||||
error?.code === 'P2002' &&
|
||||
Array.isArray(error.meta?.target) &&
|
||||
error.meta.target.includes('key')
|
||||
)
|
||||
throw new ConflictException(
|
||||
'Proposal key already exists; fetch and reconcile the existing task',
|
||||
);
|
||||
throw error;
|
||||
});
|
||||
}
|
||||
async transition(id: string, dto: any, a: WorkflowIdentity) {
|
||||
this.identity(a, ['Admin', 'SUPER_ADMIN']);
|
||||
return this.db.$transaction(async (tx: any) => {
|
||||
await this.allowed(tx, true);
|
||||
const t = await this.task(tx, id);
|
||||
const fingerprint = JSON.stringify({ id, dto, a });
|
||||
if (await this.replay(tx, dto.idempotencyKey, fingerprint)) return t;
|
||||
this.version(t, dto);
|
||||
if (t.metadata.createdBy !== a.id && a.role !== 'SUPER_ADMIN')
|
||||
throw new ForbiddenException('Proposal owner required');
|
||||
if (
|
||||
!(
|
||||
(t.state === 'DRAFT' && dto.toState === 'GATE1_PENDING') ||
|
||||
(t.state === 'GATE1_APPROVED' && dto.toState === 'READY')
|
||||
)
|
||||
)
|
||||
throw new ForbiddenException(
|
||||
'Only submission and approved queueing are available',
|
||||
);
|
||||
if (dto.toState === 'READY') {
|
||||
const approval = await tx.orchestrationApprovalDecision.findFirst({
|
||||
where: { taskId: id, gate: 1, decision: 'APPROVED' },
|
||||
orderBy: { createdAt: 'desc' },
|
||||
});
|
||||
if (!approval || approval.expiresAt <= new Date())
|
||||
throw new ForbiddenException('Gate1 expired');
|
||||
await tx.prepareWorkflowOutbox.create({ data: { taskId: id } });
|
||||
}
|
||||
return this.event(
|
||||
tx,
|
||||
t,
|
||||
dto.toState,
|
||||
a,
|
||||
dto.idempotencyKey,
|
||||
'PREPARE_TRANSITION',
|
||||
fingerprint,
|
||||
);
|
||||
});
|
||||
}
|
||||
async decide(id: string, gate: number, dto: any, a: WorkflowIdentity) {
|
||||
this.identity(a, ['SUPER_ADMIN']);
|
||||
if (gate !== 1)
|
||||
throw new ForbiddenException(
|
||||
'Gate2 decisions and release remain disabled',
|
||||
);
|
||||
if (
|
||||
!['APPROVED', 'REJECTED'].includes(dto.decision) ||
|
||||
!Number.isFinite(Date.parse(dto.expiresAt)) ||
|
||||
Date.parse(dto.expiresAt) <= Date.now() ||
|
||||
Date.parse(dto.expiresAt) > Date.now() + 86400000
|
||||
)
|
||||
throw new BadRequestException(
|
||||
'Decision requires future expiry within 24 hours',
|
||||
);
|
||||
return this.db.$transaction(async (tx: any) => {
|
||||
await this.allowed(tx);
|
||||
const t = await this.task(tx, id);
|
||||
const fingerprint = JSON.stringify({ id, gate, dto, a });
|
||||
if (await this.replay(tx, dto.idempotencyKey, fingerprint)) return t;
|
||||
this.version(t, dto);
|
||||
if (t.metadata.createdBy === a.id)
|
||||
throw new ForbiddenException('Independent reviewer required');
|
||||
if (t.state !== 'GATE1_PENDING')
|
||||
throw new ConflictException('Gate1 pending required');
|
||||
await tx.orchestrationApprovalDecision.create({
|
||||
data: {
|
||||
taskId: id,
|
||||
gate: 1,
|
||||
decision: dto.decision,
|
||||
actorId: a.id,
|
||||
expiresAt: new Date(dto.expiresAt),
|
||||
idempotencyKey: `decision:${dto.idempotencyKey}`,
|
||||
},
|
||||
});
|
||||
return this.event(
|
||||
tx,
|
||||
t,
|
||||
dto.decision === 'APPROVED' ? 'GATE1_APPROVED' : 'REJECTED',
|
||||
a,
|
||||
dto.idempotencyKey,
|
||||
'PREPARE_GATE1',
|
||||
fingerprint,
|
||||
);
|
||||
});
|
||||
}
|
||||
async claim(a: WorkflowIdentity, id?: string): Promise<any> {
|
||||
this.worker(a);
|
||||
return this.db.$transaction(async (tx: any) => {
|
||||
await this.allowed(tx, true);
|
||||
// Lock order is task then outbox for every worker operation. SKIP LOCKED permits competing workers.
|
||||
const rows: any[] = id
|
||||
? await tx.$queryRaw`SELECT t.id FROM orchestration_tasks t JOIN prepare_workflow_outbox o ON o.task_id=t.id WHERE t.id=${id}::uuid AND o.status='PENDING' AND (o.lease_until IS NULL OR o.lease_until < now()) ORDER BY o.created_at FOR UPDATE OF t SKIP LOCKED LIMIT 1`
|
||||
: await tx.$queryRaw`SELECT t.id FROM orchestration_tasks t JOIN prepare_workflow_outbox o ON o.task_id=t.id WHERE o.status='PENDING' AND (o.lease_until IS NULL OR o.lease_until < now()) ORDER BY o.created_at FOR UPDATE OF t SKIP LOCKED LIMIT 1`;
|
||||
if (!rows.length) return null;
|
||||
const t = await this.task(tx, rows[0].id);
|
||||
const o = await tx.prepareWorkflowOutbox.findUnique({
|
||||
where: { taskId: t.id },
|
||||
});
|
||||
const approval = await tx.orchestrationApprovalDecision.findFirst({
|
||||
where: { taskId: t.id, gate: 1, decision: 'APPROVED' },
|
||||
orderBy: { createdAt: 'desc' },
|
||||
});
|
||||
if (!approval || approval.expiresAt <= new Date())
|
||||
throw new ForbiddenException('Gate1 expired');
|
||||
if (o.attempts >= 3) throw new ForbiddenException('Attempts exhausted');
|
||||
const costs = await tx.orchestrationBudgetLedger.aggregate({
|
||||
where: { taskId: t.id },
|
||||
_sum: { amountCents: true },
|
||||
});
|
||||
if ((costs._sum.amountCents ?? 0) + 20 > t.budgetLimitCents)
|
||||
throw new ForbiddenException('Hard task budget exceeded');
|
||||
if (o.runId)
|
||||
await tx.orchestrationRun.update({
|
||||
where: { id: o.runId },
|
||||
data: {
|
||||
status: 'FAILED',
|
||||
retryClass: 'TRANSIENT',
|
||||
finishedAt: new Date(),
|
||||
},
|
||||
});
|
||||
const token = randomUUID(),
|
||||
attempt = o.attempts + 1;
|
||||
const run = await tx.orchestrationRun.create({
|
||||
data: {
|
||||
taskId: t.id,
|
||||
attempt,
|
||||
status: 'RUNNING',
|
||||
environment: 'staging',
|
||||
commitSha: t.metadata.commitSha,
|
||||
idempotencyKey: `prepare-run:${o.id}:${attempt}`,
|
||||
startedAt: new Date(),
|
||||
},
|
||||
});
|
||||
const reservation = await tx.orchestrationBudgetLedger.create({
|
||||
data: {
|
||||
taskId: t.id,
|
||||
amountCents: 20,
|
||||
kind: 'RESERVE',
|
||||
actorId: a.id,
|
||||
idempotencyKey: `prepare-reserve:${run.id}`,
|
||||
},
|
||||
});
|
||||
const leaseUntil = new Date(Date.now() + 30000);
|
||||
await tx.prepareWorkflowOutbox.update({
|
||||
where: { id: o.id },
|
||||
data: {
|
||||
attempts: attempt,
|
||||
leaseOwner: a.id,
|
||||
leaseToken: token,
|
||||
leaseUntil,
|
||||
heartbeatAt: new Date(),
|
||||
runId: run.id,
|
||||
reservationId: reservation.id,
|
||||
},
|
||||
});
|
||||
await this.event(
|
||||
tx,
|
||||
t,
|
||||
'RUNNING',
|
||||
a,
|
||||
`prepare-claim:${run.id}`,
|
||||
'PREPARE_CLAIMED',
|
||||
token,
|
||||
);
|
||||
return {
|
||||
taskId: t.id,
|
||||
runId: run.id,
|
||||
attempt,
|
||||
leaseToken: token,
|
||||
leaseUntil,
|
||||
commitSha: run.commitSha,
|
||||
environment: 'staging',
|
||||
maxCostCents: 20,
|
||||
};
|
||||
});
|
||||
}
|
||||
private async lease(tx: any, a: WorkflowIdentity, dto: any) {
|
||||
const t = await this.task(tx, dto.taskId);
|
||||
const o = await tx.prepareWorkflowOutbox.findUnique({
|
||||
where: { taskId: t.id },
|
||||
});
|
||||
if (
|
||||
!o ||
|
||||
o.status !== 'PENDING' ||
|
||||
o.leaseOwner !== a.id ||
|
||||
o.leaseToken !== dto.leaseToken ||
|
||||
o.runId !== dto.runId ||
|
||||
!o.leaseUntil ||
|
||||
o.leaseUntil <= new Date() ||
|
||||
t.state !== 'RUNNING'
|
||||
)
|
||||
throw new ForbiddenException('Active owned lease required');
|
||||
return { t, o };
|
||||
}
|
||||
async heartbeat(a: WorkflowIdentity, dto: any) {
|
||||
this.worker(a);
|
||||
return this.db.$transaction(async (tx: any) => {
|
||||
await this.allowed(tx);
|
||||
const { o } = await this.lease(tx, a, dto);
|
||||
return tx.prepareWorkflowOutbox.update({
|
||||
where: { id: o.id },
|
||||
data: {
|
||||
heartbeatAt: new Date(),
|
||||
leaseUntil: new Date(Date.now() + 30000),
|
||||
},
|
||||
select: { leaseUntil: true },
|
||||
});
|
||||
});
|
||||
}
|
||||
async complete(a: WorkflowIdentity, dto: any) {
|
||||
this.worker(a);
|
||||
if (
|
||||
!/^[a-f0-9]{64}$/.test(dto.sha256 ?? '') ||
|
||||
!/^https:\/\//.test(dto.uri ?? '') ||
|
||||
!Number.isSafeInteger(dto.actualCents) ||
|
||||
dto.actualCents < 0 ||
|
||||
dto.actualCents > 20
|
||||
)
|
||||
throw new BadRequestException(
|
||||
'QA hash, HTTPS URI and cost 0..20 required',
|
||||
);
|
||||
return this.db.$transaction(async (tx: any) => {
|
||||
await this.allowed(tx);
|
||||
const { t, o } = await this.lease(tx, a, dto);
|
||||
await tx.orchestrationArtifact.create({
|
||||
data: { runId: o.runId, kind: 'QA', sha256: dto.sha256, uri: dto.uri },
|
||||
});
|
||||
await tx.orchestrationAttestation.create({
|
||||
data: {
|
||||
runId: o.runId,
|
||||
kind: 'QA',
|
||||
subjectHash: dto.sha256,
|
||||
signer: a.id,
|
||||
predicate: { passed: true, scope: 'STAGE_PREPARE' },
|
||||
},
|
||||
});
|
||||
await tx.orchestrationRun.update({
|
||||
where: { id: o.runId },
|
||||
data: { status: 'SUCCEEDED', finishedAt: new Date() },
|
||||
});
|
||||
await tx.orchestrationBudgetLedger.create({
|
||||
data: {
|
||||
taskId: t.id,
|
||||
kind: 'RECONCILE',
|
||||
amountCents: dto.actualCents - 20,
|
||||
actorId: a.id,
|
||||
idempotencyKey: `prepare-cost:${o.runId}`,
|
||||
metadata: {
|
||||
reservationId: o.reservationId,
|
||||
actualCents: dto.actualCents,
|
||||
},
|
||||
},
|
||||
});
|
||||
await tx.prepareWorkflowOutbox.update({
|
||||
where: { id: o.id },
|
||||
data: { status: 'DONE', leaseToken: null, leaseUntil: null },
|
||||
});
|
||||
return this.event(
|
||||
tx,
|
||||
t,
|
||||
'VERIFYING',
|
||||
a,
|
||||
`prepare-qa:${o.runId}`,
|
||||
'PREPARE_QA_SUCCEEDED',
|
||||
dto.sha256,
|
||||
);
|
||||
});
|
||||
}
|
||||
async stageEvidence(a: WorkflowIdentity, dto: any) {
|
||||
this.identity(a, ['STAGE_CI']);
|
||||
if (!this.config.stageCiIds.includes(a.id))
|
||||
throw new ForbiddenException('Stage CI identity not provisioned');
|
||||
if (
|
||||
!/^[a-f0-9]{64}$/.test(dto.evidenceSha256 ?? '') ||
|
||||
!/^https:\/\//.test(dto.uri ?? '')
|
||||
)
|
||||
throw new BadRequestException(
|
||||
'Stage evidence hash and HTTPS URI required',
|
||||
);
|
||||
return this.db.$transaction(async (tx: any) => {
|
||||
await this.allowed(tx);
|
||||
const t = await this.task(tx, dto.taskId);
|
||||
const run = await tx.orchestrationRun.findUnique({
|
||||
where: { id: dto.runId },
|
||||
});
|
||||
if (
|
||||
t.state !== 'VERIFYING' ||
|
||||
!run ||
|
||||
run.taskId !== t.id ||
|
||||
run.environment !== 'staging' ||
|
||||
run.status !== 'SUCCEEDED' ||
|
||||
run.commitSha !== dto.commitSha ||
|
||||
t.metadata.commitSha !== dto.commitSha
|
||||
)
|
||||
throw new ForbiddenException(
|
||||
'Immutable successful Stage run binding required',
|
||||
);
|
||||
const qa = await tx.orchestrationAttestation.findFirst({
|
||||
where: { runId: run.id, kind: 'QA', subjectHash: dto.sha256 },
|
||||
});
|
||||
if (!qa || qa.signer === a.id)
|
||||
throw new ForbiddenException('Independent QA evidence required');
|
||||
await tx.orchestrationArtifact.create({
|
||||
data: {
|
||||
runId: run.id,
|
||||
kind: 'STAGE_EVIDENCE',
|
||||
sha256: dto.evidenceSha256,
|
||||
uri: dto.uri,
|
||||
},
|
||||
});
|
||||
await tx.orchestrationAttestation.create({
|
||||
data: {
|
||||
runId: run.id,
|
||||
kind: 'STAGE',
|
||||
subjectHash: dto.sha256,
|
||||
signer: a.id,
|
||||
predicate: {
|
||||
evidenceSha256: dto.evidenceSha256,
|
||||
commitSha: dto.commitSha,
|
||||
environment: 'staging',
|
||||
},
|
||||
},
|
||||
});
|
||||
return this.event(
|
||||
tx,
|
||||
t,
|
||||
'GATE2_PENDING',
|
||||
a,
|
||||
`prepare-stage:${run.id}`,
|
||||
'STAGE_EVIDENCE_RECORDED',
|
||||
dto.evidenceSha256,
|
||||
);
|
||||
});
|
||||
}
|
||||
project(t: any, a: WorkflowIdentity) {
|
||||
const safe =
|
||||
t.operationMode === 'PREPARE' &&
|
||||
(t.metadata?.workflowScope ?? t.metadata?.scope) === 'STAGE_PREPARE';
|
||||
const owner =
|
||||
a?.id && (t.metadata?.createdBy === a.id || a.role === 'SUPER_ADMIN');
|
||||
const actions: string[] = [];
|
||||
if (safe && owner && t.state === 'DRAFT') actions.push('SUBMIT_GATE1');
|
||||
if (safe && owner && t.state === 'GATE1_APPROVED')
|
||||
actions.push('QUEUE_PREPARE');
|
||||
if (
|
||||
safe &&
|
||||
a?.role === 'SUPER_ADMIN' &&
|
||||
t.metadata?.createdBy !== a.id &&
|
||||
t.state === 'GATE1_PENDING'
|
||||
)
|
||||
actions.push('APPROVE_GATE1', 'REJECT_GATE1');
|
||||
return {
|
||||
...t,
|
||||
metadata: safe
|
||||
? {
|
||||
scope: t.metadata.scope,
|
||||
...(t.metadata.workflowScope === 'STAGE_PREPARE'
|
||||
? { acceptanceCriteria: t.metadata.acceptanceCriteria }
|
||||
: {}),
|
||||
environment: 'staging',
|
||||
commitSha: t.metadata.commitSha,
|
||||
}
|
||||
: {},
|
||||
availableActions: actions,
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,513 @@
|
||||
import { Test } from '@nestjs/testing';
|
||||
import request from 'supertest';
|
||||
import { OrchestrationController } from './orchestration.controller';
|
||||
import { OrchestrationService } from './orchestration.service';
|
||||
import { JwtAuthGuard } from '../auth/jwt-auth.guard';
|
||||
import { RolesGuard } from '../common/guards/roles.guard';
|
||||
import { PrismaClient } from '@prisma/client';
|
||||
import { PrepareWorkflowService } from './prepare-workflow.service';
|
||||
import { OperationMode, WorkflowState } from './orchestration.domain';
|
||||
|
||||
const enabled = process.env.WORKFLOW_DB === '1';
|
||||
(enabled ? describe : describe.skip)('isolated prepare workflow', () => {
|
||||
const db = new PrismaClient();
|
||||
const admin = { id: 'proposer', role: 'Admin' };
|
||||
const reviewer = { id: 'reviewer', role: 'SUPER_ADMIN' };
|
||||
const worker = { id: 'worker-1', role: 'WORKFLOW_WORKER' };
|
||||
const ci = { id: 'isolated-ci', role: 'STAGE_CI' };
|
||||
const service = new PrepareWorkflowService(db as never, {
|
||||
workerIds: ['worker-1', 'worker-2'],
|
||||
stageCiIds: ['isolated-ci'],
|
||||
});
|
||||
let seq = 0;
|
||||
const dto = () => ({
|
||||
key: `wf-${Date.now()}-${seq++}`,
|
||||
title: 'Bounded isolated QA tracer',
|
||||
department: 'ENGINEERING',
|
||||
operationMode: OperationMode.PREPARE,
|
||||
budgetLimitCents: 20,
|
||||
metadata: {
|
||||
commitSha: 'a'.repeat(40),
|
||||
token: 'must-not-persist',
|
||||
scope: 'Staging schema audit only',
|
||||
acceptanceCriteria: ['Report all schema failures'],
|
||||
actorId: 'spoof',
|
||||
},
|
||||
});
|
||||
async function ready() {
|
||||
let t = await service.create(dto(), admin);
|
||||
t = await service.transition(
|
||||
t.id,
|
||||
{
|
||||
toState: WorkflowState.GATE1_PENDING,
|
||||
expectedVersion: t.version,
|
||||
idempotencyKey: `submit-${t.id}`,
|
||||
},
|
||||
admin,
|
||||
);
|
||||
t = await service.decide(
|
||||
t.id,
|
||||
1,
|
||||
{
|
||||
decision: 'APPROVED',
|
||||
expectedVersion: t.version,
|
||||
idempotencyKey: `approve-${t.id}`,
|
||||
expiresAt: new Date(Date.now() + 60000).toISOString(),
|
||||
},
|
||||
reviewer,
|
||||
);
|
||||
return service.transition(
|
||||
t.id,
|
||||
{
|
||||
toState: WorkflowState.READY,
|
||||
expectedVersion: t.version,
|
||||
idempotencyKey: `ready-${t.id}`,
|
||||
},
|
||||
admin,
|
||||
);
|
||||
}
|
||||
afterAll(async () => {
|
||||
await db.$disconnect();
|
||||
});
|
||||
it('persists and projects immutable bounded proposal scope without arbitrary metadata', async () => {
|
||||
const input = dto();
|
||||
const t = await service.create(input, admin);
|
||||
expect(t.metadata).toMatchObject({
|
||||
workflowScope: 'STAGE_PREPARE',
|
||||
scope: input.metadata.scope,
|
||||
acceptanceCriteria: input.metadata.acceptanceCriteria,
|
||||
commitSha: input.metadata.commitSha,
|
||||
});
|
||||
expect(t.metadata).not.toHaveProperty('token');
|
||||
expect(t.metadata).not.toHaveProperty('actorId');
|
||||
const submitted = await service.transition(
|
||||
t.id,
|
||||
{
|
||||
toState: 'GATE1_PENDING',
|
||||
expectedVersion: t.version,
|
||||
idempotencyKey: `immutable-${t.id}`,
|
||||
},
|
||||
admin,
|
||||
);
|
||||
expect(submitted.metadata).toEqual(t.metadata);
|
||||
expect(service.project(submitted, reviewer).metadata).toEqual({
|
||||
scope: input.metadata.scope,
|
||||
acceptanceCriteria: input.metadata.acceptanceCriteria,
|
||||
environment: 'staging',
|
||||
commitSha: input.metadata.commitSha,
|
||||
});
|
||||
});
|
||||
it.each([
|
||||
null,
|
||||
[],
|
||||
{},
|
||||
{ commitSha: 'a'.repeat(40), scope: '', acceptanceCriteria: ['ok'] },
|
||||
{
|
||||
commitSha: 'a'.repeat(40),
|
||||
scope: ' '.repeat(10),
|
||||
acceptanceCriteria: ['ok'],
|
||||
},
|
||||
{
|
||||
commitSha: 'a'.repeat(40),
|
||||
scope: 'x'.repeat(2001),
|
||||
acceptanceCriteria: ['ok'],
|
||||
},
|
||||
{ commitSha: 'a'.repeat(40), scope: 'ok', acceptanceCriteria: [] },
|
||||
{
|
||||
commitSha: 'a'.repeat(40),
|
||||
scope: 'ok',
|
||||
acceptanceCriteria: Array(21).fill('ok'),
|
||||
},
|
||||
{
|
||||
commitSha: 'a'.repeat(40),
|
||||
scope: 'ok',
|
||||
acceptanceCriteria: ['x'.repeat(501)],
|
||||
},
|
||||
{ commitSha: 'a'.repeat(40), scope: 'ok', acceptanceCriteria: [' '] },
|
||||
{ commitSha: 'a'.repeat(40), scope: 'ok', acceptanceCriteria: [123] },
|
||||
{ commitSha: ['a'.repeat(40)], scope: 'ok', acceptanceCriteria: ['ok'] },
|
||||
{ scope: 'ok', acceptanceCriteria: ['ok'] },
|
||||
])(
|
||||
'rejects unsafe or unbounded metadata %# without persisting',
|
||||
async (metadata) => {
|
||||
const input = { ...dto(), metadata };
|
||||
await expect(service.create(input, admin)).rejects.toMatchObject({
|
||||
status: 400,
|
||||
});
|
||||
expect(
|
||||
await db.orchestrationTask.count({ where: { key: input.key } }),
|
||||
).toBe(0);
|
||||
},
|
||||
);
|
||||
it('accepts exact metadata bounds', async () => {
|
||||
const metadata = {
|
||||
commitSha: 'a'.repeat(40),
|
||||
scope: 'x'.repeat(2000),
|
||||
acceptanceCriteria: Array(20).fill('x'.repeat(500)),
|
||||
};
|
||||
expect(
|
||||
(await service.create({ ...dto(), metadata }, admin)).metadata,
|
||||
).toMatchObject(metadata);
|
||||
});
|
||||
it('normalizes racing duplicate proposal keys to conflict without duplicate events', async () => {
|
||||
const input = dto();
|
||||
const results = await Promise.allSettled([
|
||||
service.create(input, admin),
|
||||
service.create({ ...input, title: 'Different intent' }, reviewer),
|
||||
]);
|
||||
expect(results.filter((r) => r.status === 'fulfilled')).toHaveLength(1);
|
||||
const rejected = results.find(
|
||||
(r) => r.status === 'rejected',
|
||||
) as PromiseRejectedResult;
|
||||
expect(rejected.reason).toMatchObject({ status: 409 });
|
||||
const persisted = await db.orchestrationTask.findUniqueOrThrow({
|
||||
where: { key: input.key },
|
||||
});
|
||||
expect(
|
||||
await db.orchestrationEvent.count({ where: { taskId: persisted.id } }),
|
||||
).toBe(1);
|
||||
await expect(service.create(input, admin)).rejects.toMatchObject({
|
||||
status: 409,
|
||||
});
|
||||
});
|
||||
it('enforces create and read contracts over HTTP with real PostgreSQL persistence', async () => {
|
||||
const module = await Test.createTestingModule({
|
||||
controllers: [OrchestrationController],
|
||||
providers: [
|
||||
{ provide: PrepareWorkflowService, useValue: service },
|
||||
{
|
||||
provide: OrchestrationService,
|
||||
useValue: {
|
||||
findAll: () => db.orchestrationTask.findMany(),
|
||||
findOne: (id: string) =>
|
||||
db.orchestrationTask.findUniqueOrThrow({ where: { id } }),
|
||||
},
|
||||
},
|
||||
],
|
||||
})
|
||||
.overrideGuard(JwtAuthGuard)
|
||||
.useValue({
|
||||
canActivate(context: any) {
|
||||
context.switchToHttp().getRequest().user = {
|
||||
sub: admin.id,
|
||||
role: admin.role,
|
||||
};
|
||||
return true;
|
||||
},
|
||||
})
|
||||
.overrideGuard(RolesGuard)
|
||||
.useValue({ canActivate: () => true })
|
||||
.compile();
|
||||
const app = module.createNestApplication();
|
||||
await app.init();
|
||||
try {
|
||||
const input = dto();
|
||||
const response = await request(app.getHttpServer())
|
||||
.post('/admin/orchestration/tasks')
|
||||
.send(input)
|
||||
.expect(201);
|
||||
expect(response.body.metadata).toEqual({
|
||||
scope: input.metadata.scope,
|
||||
acceptanceCriteria: input.metadata.acceptanceCriteria,
|
||||
commitSha: input.metadata.commitSha,
|
||||
environment: 'staging',
|
||||
});
|
||||
expect(response.body.availableActions).toEqual(['SUBMIT_GATE1']);
|
||||
const detail = await request(app.getHttpServer())
|
||||
.get(`/admin/orchestration/tasks/${response.body.id}`)
|
||||
.expect(200);
|
||||
expect(detail.body.metadata).toEqual(response.body.metadata);
|
||||
const list = await request(app.getHttpServer())
|
||||
.get('/admin/orchestration/tasks')
|
||||
.expect(200);
|
||||
expect(
|
||||
list.body.find((row: any) => row.id === response.body.id).metadata,
|
||||
).toEqual(response.body.metadata);
|
||||
expect(JSON.stringify(response.body)).not.toMatch(
|
||||
/createdBy|must-not-persist|actorId|workflowScope/,
|
||||
);
|
||||
await request(app.getHttpServer())
|
||||
.post('/admin/orchestration/tasks')
|
||||
.send(input)
|
||||
.expect(409);
|
||||
await request(app.getHttpServer())
|
||||
.post('/admin/orchestration/tasks')
|
||||
.send({
|
||||
...input,
|
||||
key: `${input.key}-invalid`,
|
||||
metadata: { scope: 'only stage', acceptanceCriteria: ['QA passes'] },
|
||||
})
|
||||
.expect(400);
|
||||
await request(app.getHttpServer())
|
||||
.post('/admin/orchestration/tasks')
|
||||
.send({
|
||||
...input,
|
||||
key: `${input.key}-invalid`,
|
||||
metadata: { ...input.metadata, acceptanceCriteria: [' '] },
|
||||
})
|
||||
.expect(400);
|
||||
} finally {
|
||||
await app.close();
|
||||
}
|
||||
});
|
||||
it('fences heartbeat and completes QA plus independent Stage evidence only to Gate2 pending', async () => {
|
||||
const t = await ready();
|
||||
const c = await service.claim(worker, t.id);
|
||||
await expect(
|
||||
service.heartbeat({ ...worker, id: 'worker-2' }, c),
|
||||
).rejects.toThrow(/lease/i);
|
||||
await service.heartbeat(worker, c);
|
||||
const qa = {
|
||||
...c,
|
||||
sha256: 'b'.repeat(64),
|
||||
uri: 'https://ci.example/qa',
|
||||
actualCents: 8,
|
||||
};
|
||||
await service.complete(worker, qa);
|
||||
await expect(
|
||||
service.stageEvidence(
|
||||
{ id: 'proposer', role: 'Admin' },
|
||||
{
|
||||
taskId: t.id,
|
||||
runId: c.runId,
|
||||
sha256: qa.sha256,
|
||||
commitSha: c.commitSha,
|
||||
evidenceSha256: 'c'.repeat(64),
|
||||
uri: 'https://ci.example/stage',
|
||||
},
|
||||
),
|
||||
).rejects.toThrow(/identity/i);
|
||||
await service.stageEvidence(ci, {
|
||||
taskId: t.id,
|
||||
runId: c.runId,
|
||||
sha256: qa.sha256,
|
||||
commitSha: c.commitSha,
|
||||
evidenceSha256: 'c'.repeat(64),
|
||||
uri: 'https://ci.example/stage',
|
||||
});
|
||||
expect(
|
||||
await db.orchestrationTask.findUnique({ where: { id: t.id } }),
|
||||
).toMatchObject({ state: 'GATE2_PENDING' });
|
||||
expect(
|
||||
await db.orchestrationBudgetLedger.aggregate({
|
||||
where: { taskId: t.id },
|
||||
_sum: { amountCents: true },
|
||||
}),
|
||||
).toMatchObject({ _sum: { amountCents: 8 } });
|
||||
await expect(service.complete(worker, qa)).rejects.toThrow(/lease|active/i);
|
||||
});
|
||||
it('bounds expired-lease retries and rejects stale completion without releasing unknown costs', async () => {
|
||||
const t = await service.create({ ...dto(), budgetLimitCents: 60 }, admin);
|
||||
let current = await service.transition(
|
||||
t.id,
|
||||
{
|
||||
toState: 'GATE1_PENDING',
|
||||
expectedVersion: 1,
|
||||
idempotencyKey: `s-${t.id}`,
|
||||
},
|
||||
admin,
|
||||
);
|
||||
current = await service.decide(
|
||||
t.id,
|
||||
1,
|
||||
{
|
||||
decision: 'APPROVED',
|
||||
expectedVersion: current.version,
|
||||
idempotencyKey: `a-${t.id}`,
|
||||
expiresAt: new Date(Date.now() + 60000).toISOString(),
|
||||
},
|
||||
reviewer,
|
||||
);
|
||||
await service.transition(
|
||||
t.id,
|
||||
{
|
||||
toState: 'READY',
|
||||
expectedVersion: current.version,
|
||||
idempotencyKey: `q-${t.id}`,
|
||||
},
|
||||
admin,
|
||||
);
|
||||
const first = await service.claim(worker, t.id);
|
||||
for (let i = 0; i < 2; i++) {
|
||||
await db.prepareWorkflowOutbox.update({
|
||||
where: { taskId: t.id },
|
||||
data: { leaseUntil: new Date(0) },
|
||||
});
|
||||
const next = await service.claim(worker, t.id);
|
||||
expect(next.attempt).toBe(i + 2);
|
||||
}
|
||||
await expect(
|
||||
service.complete(worker, {
|
||||
...first,
|
||||
sha256: 'b'.repeat(64),
|
||||
uri: 'https://ci.example/qa',
|
||||
actualCents: 0,
|
||||
}),
|
||||
).rejects.toThrow(/lease/i);
|
||||
await db.prepareWorkflowOutbox.update({
|
||||
where: { taskId: t.id },
|
||||
data: { leaseUntil: new Date(0) },
|
||||
});
|
||||
await expect(service.claim(worker, t.id)).rejects.toThrow(/exhausted/i);
|
||||
expect(await db.orchestrationRun.count({ where: { taskId: t.id } })).toBe(
|
||||
3,
|
||||
);
|
||||
expect(
|
||||
await db.orchestrationBudgetLedger.aggregate({
|
||||
where: { taskId: t.id },
|
||||
_sum: { amountCents: true },
|
||||
}),
|
||||
).toMatchObject({ _sum: { amountCents: 60 } });
|
||||
});
|
||||
it('rejects self approval, non-PREPARE and unprovisioned workers', async () => {
|
||||
await expect(
|
||||
service.create(
|
||||
{ ...dto(), operationMode: OperationMode.MUTATE_PRODUCTION },
|
||||
admin,
|
||||
),
|
||||
).rejects.toThrow(/PREPARE/);
|
||||
let t = await service.create(dto(), reviewer);
|
||||
t = await service.transition(
|
||||
t.id,
|
||||
{
|
||||
toState: 'GATE1_PENDING',
|
||||
expectedVersion: 1,
|
||||
idempotencyKey: `s-${t.id}`,
|
||||
},
|
||||
reviewer,
|
||||
);
|
||||
await expect(
|
||||
service.decide(
|
||||
t.id,
|
||||
1,
|
||||
{
|
||||
decision: 'APPROVED',
|
||||
expectedVersion: t.version,
|
||||
idempotencyKey: `a-${t.id}`,
|
||||
expiresAt: new Date(Date.now() + 60000).toISOString(),
|
||||
},
|
||||
reviewer,
|
||||
),
|
||||
).rejects.toThrow(/independent/i);
|
||||
await expect(
|
||||
service.claim({ id: 'spoof', role: 'WORKFLOW_WORKER' }),
|
||||
).rejects.toThrow(/provisioned/i);
|
||||
await expect(service.decide(t.id, 2, {}, reviewer)).rejects.toThrow(
|
||||
/disabled/i,
|
||||
);
|
||||
});
|
||||
it('serializes duplicate queue submission and denies claims beyond remaining task budget', async () => {
|
||||
const t = await ready();
|
||||
await db.orchestrationBudgetLedger.create({
|
||||
data: {
|
||||
taskId: t.id,
|
||||
amountCents: 1,
|
||||
kind: 'SPEND',
|
||||
actorId: 'fixture',
|
||||
idempotencyKey: `spend-${t.id}`,
|
||||
},
|
||||
});
|
||||
await expect(service.claim(worker, t.id)).rejects.toThrow(/budget/i);
|
||||
expect(await db.orchestrationRun.count({ where: { taskId: t.id } })).toBe(
|
||||
0,
|
||||
);
|
||||
expect(
|
||||
await db.prepareWorkflowOutbox.count({ where: { taskId: t.id } }),
|
||||
).toBe(1);
|
||||
});
|
||||
it('replays submission safely and serializes racing queue requests', async () => {
|
||||
let t = await service.create(dto(), admin);
|
||||
const submit = {
|
||||
toState: 'GATE1_PENDING',
|
||||
expectedVersion: t.version,
|
||||
idempotencyKey: `replay-${t.id}`,
|
||||
};
|
||||
t = await service.transition(t.id, submit, admin);
|
||||
expect((await service.transition(t.id, submit, admin)).version).toBe(
|
||||
t.version,
|
||||
);
|
||||
await expect(
|
||||
service.transition(t.id, { ...submit, toState: 'READY' }, admin),
|
||||
).rejects.toThrow(/mismatch/i);
|
||||
t = await service.decide(
|
||||
t.id,
|
||||
1,
|
||||
{
|
||||
decision: 'APPROVED',
|
||||
expectedVersion: t.version,
|
||||
idempotencyKey: `r-approve-${t.id}`,
|
||||
expiresAt: new Date(Date.now() + 60000).toISOString(),
|
||||
},
|
||||
reviewer,
|
||||
);
|
||||
const results = await Promise.allSettled(
|
||||
['one', 'two'].map((k) =>
|
||||
service.transition(
|
||||
t.id,
|
||||
{
|
||||
toState: 'READY',
|
||||
expectedVersion: t.version,
|
||||
idempotencyKey: `${k}-${t.id}`,
|
||||
},
|
||||
admin,
|
||||
),
|
||||
),
|
||||
);
|
||||
expect(results.filter((r) => r.status === 'fulfilled')).toHaveLength(1);
|
||||
expect(
|
||||
await db.prepareWorkflowOutbox.count({ where: { taskId: t.id } }),
|
||||
).toBe(1);
|
||||
});
|
||||
it('kill switch freezes claims without creating run or reservation', async () => {
|
||||
const t = await ready();
|
||||
await db.orchestrationKillSwitch.create({
|
||||
data: {
|
||||
scope: 'GLOBAL',
|
||||
level: 'FREEZE_NEW_WORK',
|
||||
reason: 'isolated test',
|
||||
actorId: 'fixture',
|
||||
},
|
||||
});
|
||||
try {
|
||||
await expect(service.claim(worker, t.id)).rejects.toThrow(/kill switch/i);
|
||||
} finally {
|
||||
await db.orchestrationKillSwitch.delete({ where: { scope: 'GLOBAL' } });
|
||||
}
|
||||
expect(await db.orchestrationRun.count({ where: { taskId: t.id } })).toBe(
|
||||
0,
|
||||
);
|
||||
});
|
||||
it('persists an atomic outbox and exactly one worker wins concurrent claims', async () => {
|
||||
const t = await ready();
|
||||
const claims = await Promise.all([
|
||||
service.claim(worker, t.id),
|
||||
service.claim({ id: 'worker-2', role: 'WORKFLOW_WORKER' }, t.id),
|
||||
]);
|
||||
expect(claims.filter(Boolean)).toHaveLength(1);
|
||||
const claim = claims.find(Boolean)!;
|
||||
expect(claim.taskId).toBe(t.id);
|
||||
expect(claim.attempt).toBe(1);
|
||||
expect(await db.orchestrationRun.count({ where: { taskId: t.id } })).toBe(
|
||||
1,
|
||||
);
|
||||
expect(
|
||||
await db.orchestrationBudgetLedger.aggregate({
|
||||
where: { taskId: t.id },
|
||||
_sum: { amountCents: true },
|
||||
}),
|
||||
).toMatchObject({ _sum: { amountCents: 20 } });
|
||||
expect(
|
||||
await service.project(
|
||||
await db.orchestrationTask.findUniqueOrThrow({ where: { id: t.id } }),
|
||||
admin,
|
||||
),
|
||||
).toMatchObject({
|
||||
metadata: {
|
||||
scope: 'Staging schema audit only',
|
||||
acceptanceCriteria: ['Report all schema failures'],
|
||||
environment: 'staging',
|
||||
commitSha: 'a'.repeat(40),
|
||||
},
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,95 @@
|
||||
**Superseded wiring/provenance: see [snapshot repair](hermes-snapshot-provenance.md). Do not use literal grants or the old DNS alias below.**
|
||||
|
||||
# Runtime read vertical slice (not deployed)
|
||||
|
||||
Branch feature/v4-runtime-completion, base 5507fbc. No AppModule, live gateway,
|
||||
production database, cron, dispatch, or existing compose was changed.
|
||||
|
||||
## Parent registration
|
||||
|
||||
Import `RuntimeReadModule.register({ '<verified-user-id>': ['default', '<department>'] })`
|
||||
from `backend/src/hermes-management/runtime-read.module` into AppModule.
|
||||
This module uses the existing Passport `jwt` strategy (DB-resolved user identity),
|
||||
not the API-key fallback. Parent must retain AuthModule/JwtStrategy initialization.
|
||||
Missing grants deny. Never construct grants from browser body, JWT profile claims,
|
||||
or unreviewed environment input. Allowlist is copied/frozen. Admin/SUPER_ADMIN
|
||||
role AND exact subject grant required; API-key identities forbidden.
|
||||
|
||||
GET `/admin/orchestration/runtime/{health,profiles,status,usage,sessions,schedules,model-info}`.
|
||||
All except health/profiles require exactly one allowed `profile` query string.
|
||||
Usage accepts days 1..365 (30); sessions limit 1..100 (20), offset 0..10000 (0).
|
||||
Unknown parameters on profile routes rejected. Profiles is filtered to server grants.
|
||||
Responses use existing adapter projections, no prompts/transcripts/config/tokens.
|
||||
Missing/forged JWT ->401; missing grant/role/profile ->403; runtime auth/network/
|
||||
contract failure ->503 `Runtime unavailable`; limits ->429. No browser Hermes
|
||||
cookie, credential, socket ticket or mutation handler is provided.
|
||||
|
||||
Mount backend `/run/secrets/hermes_username` and `hermes_password` as files.
|
||||
Only these fixed secret paths are read, after authorization. Broker login/cache
|
||||
stays server-side. Backend and isolated inventory runtime must receive matching
|
||||
credentials. Runtime additionally mounts `hermes_signing_key` (>=32 random chars).
|
||||
Do not put credentials in YAML/env, commands, logs or Git. Rotate key to invalidate
|
||||
sessions; rotate password and restart inventory plus backend to clear cached access.
|
||||
Stateless old sessions remain valid until TTL/key rotation.
|
||||
|
||||
## Deliberately separate snapshot topology
|
||||
|
||||
Installed Hermes `serve` lifespan starts reconciliation and hosted-room recovery;
|
||||
GET sessions auto-archives. Sharing its writable home with existing gateway is
|
||||
therefore NOT a safe read-only sidecar. `serve.py` runs the pinned installed app
|
||||
with lifespan OFF and an outer exact GET allowlist, password-login only and all
|
||||
WebSockets denied1008. It does NOT start scheduler/gateway/MCP/rooms/provider chat.
|
||||
The existing gateway and external volume `canina-hermes_canina_hermes_data` remain
|
||||
unchanged, solely owned by existing gateway. Inventory mounts an independently
|
||||
provisioned snapshot volume `canina_hermes_inventory_snapshot`; it NEVER mounts
|
||||
original home. Its incidental session auto-archive writes affect only snapshot.
|
||||
This is snapshot inventory, NOT live runtime freshness/status acceptance. A copy
|
||||
of gateway PID state is not evidence of live gateway health. UI must not claim live
|
||||
readiness from this deployment. Parent should expose snapshot provenance/freshness
|
||||
before enabling it for business use.
|
||||
|
||||
Provisioning must use an approved consistent SQLite backup/checkpoint/export and
|
||||
redacted profile config inventory, not raw `cp` of changing SQLite/WAL or credentials.
|
||||
No automatic snapshot job is installed here. No secrets need be copied from live
|
||||
home. Review which configs the model/status APIs need and omit auth.json/.env.
|
||||
Missing store/schema -> unavailable, not fabricated empty healthy telemetry.
|
||||
Dedicated internal network must contain only backend and inventory; do NOT connect
|
||||
existing gateway with same `canina-hermes` DNS alias. Backend name resolves inventory.
|
||||
Inventory network must have no external egress. Compose refers to external network;
|
||||
operator must create it `--internal`. No host ports/socket are published.
|
||||
|
||||
## CI ownership / apply
|
||||
|
||||
`compose.readonly.yml` is deployable ONLY after external snapshot, internal network,
|
||||
secret files and backend mounts/registration are provisioned. `ci-apply.sh` requires
|
||||
CI=true, exact reviewed HEAD SHA, clean checkout, snapshot-reviewed assertion,
|
||||
existing volume/network/files, then compose config/up. Never run alongside any
|
||||
other inventory instance writing same snapshot. CI job fragment is tracked under
|
||||
infra ownership; parent owns real workflow wiring and approval environment. No
|
||||
active deployment trigger added. Existing review-only compose template is unchanged.
|
||||
|
||||
## Executed evidence
|
||||
|
||||
Remote `/opt/services/canina/validation/v4-runtime-completion/`:
|
||||
- read-red.log missing controller regression; read-green.log 6 suites/27 tests.
|
||||
- read-http-green.log 7 suites/30 tests: real Nest HTTP with signed JWT strategy,
|
||||
bad signature/no auth401, ungranted admin403, allowed admin200, no cookie,
|
||||
mutation404. This uses a mocked runtime service, not live Canina credentials.
|
||||
- pinned-http.log exact installed image FastAPI login/protected inventory:
|
||||
wrong password401, real login200/HttpOnly provider-verifiable token, profiles200,
|
||||
tampered token401. No lifespan/live home.
|
||||
- serve-http.log actual loopback TCP9119 in network-none disposable pinned image:
|
||||
unauth inventory401, wrong password401, login200, profiles200, health auth true,
|
||||
mutation/config/ws HTTP404. Generated test-only credentials. No live volume.
|
||||
- ws-red.log existing socket rejection code differed; ws-green.log explicit
|
||||
outer boundary denies authenticated sockets1008 independently of upstream tickets.
|
||||
- build.log private Prisma generation and full backend tsc build exit0.
|
||||
|
||||
Pinned local exact image ID sha256:373ddd5e6649f525b17993e1c885cdf1352127f8e98fd329140a73237125e52f;
|
||||
tracked registry digest sha256:9774f4f39a9bb8c2f68ce728ed5e99ddbad282163be56764afacf88ed952b784.
|
||||
Node22 disposable runner copied read-only source/dependency donor privately;
|
||||
no DB migration. broker-pinned-e2e.log proves actual compiled Node broker -> TCP pinned Hermes login -> adapter -> policy-filtered inventory (and deny zero secret reads). Nest HTTP authentication tested separately; these tests do not establish full Canina application E2E, snapshot
|
||||
refresh, CI deployment, live gateway truth or business workflow acceptance.
|
||||
Those remain explicit integration/provisioning gates, not claimed completion.
|
||||
|
||||
Final full non-DB regression: 106 suites / 850 tests pass; expected legacy negative SSL/payment logs. Scoped ESLint has zero errors, six type-safety warnings; full build passes. No RELEASE_AUTHORIZE integration suite run.
|
||||
@@ -0,0 +1,86 @@
|
||||
# Snapshot provenance repair — integration contract
|
||||
|
||||
This is independent snapshot inventory, **not a live Hermes executor**. No Production,
|
||||
deployment, live gateway edits, or RELEASE_AUTHORIZE execution is included.
|
||||
|
||||
## Parent-owned application wiring
|
||||
|
||||
Keep AuthModule/JwtStrategy. Replace literal user grants in parent-owned app.module.ts with:
|
||||
|
||||
```ts
|
||||
RuntimeReadModule.registerFromFile('/run/secrets/hermes_runtime_grants')
|
||||
```
|
||||
|
||||
Nest supports the resulting Promise<DynamicModule> in imports. Alternatively pass the
|
||||
reviewed nonsecret file `/etc/canina/runtime-grants.json`. No env/JWT/browser grant
|
||||
input is accepted by this registration. File is JSON object mapping exact verified DB
|
||||
user IDs to nonempty unique profile arrays; `{}` explicitly denies everyone. Missing,
|
||||
malformed, symlinked, group/world-writable, oversized files fail startup closed. Mount
|
||||
read-only and provision via reviewed configuration/secrets management; do not commit
|
||||
real user grants. Existing explicit register(grants) remains for tests/legacy consumers.
|
||||
Backend also mounts `/run/secrets/hermes_username` and `hermes_password` read-only.
|
||||
No app.module.ts modification belongs to this commit.
|
||||
|
||||
## Required coordinated origin change
|
||||
|
||||
Authenticated management module and broker now exclusively use
|
||||
`http://canina-hermes-inventory:9119`. compose.readonly.yml uses the distinct alias
|
||||
`canina-hermes-inventory`, NEVER `canina-hermes`. Generic adapter allowlist retains
|
||||
both reviewed exact origins for compatibility with separate existing consumers.
|
||||
No arbitrary origin/env URL accepted. Parent must cherry-pick broker/module/adapter
|
||||
and inventory compose together; do not add an alias to the existing gateway.
|
||||
No deployment performed. External network should still be internal and only contain
|
||||
backend/inventory; the distinct alias cannot shadow the live canina-hermes hostname.
|
||||
|
||||
## Snapshot exporter/operator gate
|
||||
|
||||
Every independent inventory root must include `snapshot-manifest.json`:
|
||||
|
||||
```json
|
||||
{"schemaVersion":1,"source":"snapshot","capturedAt":"2026-10-23T12:00:00Z"}
|
||||
```
|
||||
|
||||
The example is a schema example, NOT an actual capture attestation. Exporter must
|
||||
write the real capture instant from the approved consistent export. Never copy the
|
||||
example, use startup time, filesystem mtime, or claim this change captures/refreshes
|
||||
snapshots. Missing/invalid/future manifest prevents startup. It is revalidated per
|
||||
successful GET; invalid after startup returns503. Manifest is exporter metadata,
|
||||
not a cryptographic proof of consistency. Immutable reviewed export provisioning
|
||||
remains required. Fresh means age <=3600 seconds, otherwise stale; even fresh is
|
||||
NOT live readiness. No snapshot scheduler installed.
|
||||
|
||||
## Additive public response fields / panel handoff
|
||||
|
||||
All object read DTOs (health/status/usage/sessions/model-info) add:
|
||||
`provenance:{source:'snapshot',capturedAt:string,ageSeconds:number,freshness:'fresh'|'stale',liveProcess:'unknown'}`.
|
||||
Profile and schedule array shapes are preserved: provenance is on each row. Empty
|
||||
arrays have no rows to annotate; use the health response to display snapshot metadata.
|
||||
Profile rows add `liveGatewayState:'unknown'`; status adds the same plus
|
||||
`liveActiveAgents:null`. Existing required scalars remain typed but are conservative
|
||||
placeholders: `gateway_running:false`, `active_agents:0`, `gateway_state:null`,
|
||||
`overall:'unknown'`. False/zero MUST NOT render as measured offline/idle: when
|
||||
provenance.source is snapshot use the explicit unknown fields, label snapshot and
|
||||
show capturedAt/freshness. Health.ok only means inventory HTTP app responsiveness,
|
||||
never live gateway readiness. Profile existence is inventory availability, NOT
|
||||
process readiness. Adapter enforces provenance on dedicated snapshot origin and
|
||||
conservatively overwrites copied PID-derived status/profile scalars again.
|
||||
|
||||
Installed status handler was unsafe here: it probes gateway HTTP/PIDs/platforms and
|
||||
can stall despite this being an independent snapshot. Snapshot server replaces that
|
||||
route with measured package version and explicit unknown process fields; no status
|
||||
probe is run. Password auth/mutation denial/WebSocket denial remain intact.
|
||||
|
||||
## Executed acceptance
|
||||
|
||||
Remote disposable network-none exact pinned Hermes image, no original volumes:
|
||||
manifest-missing regression failed first; TCP login/profile/health/status provenance
|
||||
and stale capture assertions now pass; authenticated sockets denied1008. Disposable
|
||||
Node22 copy of source/dependencies: 9 hermes-management suites /34 tests pass and
|
||||
full backend tsc build passes. Adapter copied-live-state regression failed first;
|
||||
grants-loader missing-function regression failed first. Compiled actual Node broker
|
||||
-> TCP pinned server -> adapter -> policy-filtered profiles/status passes, deny reads
|
||||
zero credentials. Test timestamps/credentials are fixtures, not live telemetry.
|
||||
The first HTTP retry raced fixture startup and returned UNAVAILABLE; retry after
|
||||
server startup passed. No claim of full app/user-flow or live runtime integration.
|
||||
|
||||
Full non-DB Jest regression: 108 suites/854 tests passed; 1 suite/18 tests skipped. No RELEASE_AUTHORIZE suite was enabled.
|
||||
@@ -0,0 +1,33 @@
|
||||
# Admin v4 completion integration contract
|
||||
|
||||
Frontend-only slice based on 5507fbc. No deployment or runtime writes. The historical read-only integration document does not authorize enabling routes. Backend policy remains the authority; UI adds deny-only safety restrictions, never permissions.
|
||||
|
||||
## Workflow (existing routes retained)
|
||||
|
||||
- GET `/admin/orchestration/overview`: ResourceSnapshot with `items`, ISO `measuredAt`, `capabilities:{mutations:boolean,chat:boolean,runtime:boolean}`. Missing/failed capability read defaults false.
|
||||
- GET `/admin/orchestration/tasks`: Task[] (or `{success:true,data:Task[]}`).
|
||||
- GET `/admin/orchestration/tasks/:id`: Task. Required identity/version/state/mode/budget and server-computed `availableActions`; optional scope/acceptance metadata, runs/artifacts, immutable decisions, events and budgetLedger. Missing actions means no permission. Include structured redacted scope metadata for a useful Gate1 review; no secrets or arbitrary event payloads.
|
||||
- POST `/admin/orchestration/tasks`: existing CreateTaskInput `{key,title,department,operationMode,budgetLimitCents,metadata:{scope,acceptanceCriteria:string[]}}`. UI allows OBSERVE/PREPARE only, not Production. Key is unique business identity; after an ambiguous create, reconcile key before resubmission. Success locks the form to prevent accidental duplicate submission.
|
||||
- PUT `/admin/orchestration/tasks/:id/transition`: TransitionDto `{toState,expectedVersion,idempotencyKey,reason}`.
|
||||
- POST `/admin/orchestration/tasks/:id/gates/:gate/decide`: GateDecisionDto `{decision:'APPROVED'|'REJECTED',expectedVersion,idempotencyKey,expiresAt,notes}`; Gate2 approval additionally binds `{runId,artifactHash,commitSha,environment}` from a registered SUCCEEDED run, never editable commit/environment. Gate1 is bounded work, not release permission.
|
||||
- ServerAction `{id,label,toState? ,gate?:1|2,decision?:'APPROVED'|'REJECTED',disabledReason?}`. Exactly one of transition or gate decision. All role/policy/kill/version checks remain backend-side. Browser identity fields are never posted; actor must derive from verified JWT. HOLD/request changes only appear if advertised as supported transitions. EDIT_AND_APPROVE is unavailable without a separately reviewed DTO; no guessed mutation endpoint.
|
||||
- HTTP409 stale version/idempotency mismatch, 410 expired, 401/403 denied and other 4xx permanently lock that dialog; refresh and review afresh. Network/5xx ambiguous failures allow only identical payload/key. Inputs and navigation are locked after first attempt; authoritative refresh invalidates selection and pending detail reads. No optimistic status update.
|
||||
- Production task actions and release transitions remain disabled even if advertised. Gate2 evidence inspection is not deployment authorization. Server still enforces TTL; browser time is a usability check, not trusted authorization.
|
||||
|
||||
## Runtime read API (aligned by read-only inspection with runtime sibling)
|
||||
|
||||
Uses existing JWT Axios client, never upstream Hermes paths or tokens. No automatic runtime mutations. Tab is explicitly selected; failed auth/config/routes show unavailable.
|
||||
|
||||
- GET `/admin/orchestration/runtime/health` -> `{ok:boolean,version:string,commit:string,auth_required:true}`. Any false/missing authentication flag is rejected.
|
||||
- GET `/admin/orchestration/runtime/profiles` -> `{name:string,is_default:boolean,gateway_running:boolean}[]`, already filtered to principal grants.
|
||||
- GET `/admin/orchestration/runtime/status?profile=<authorized>` -> `{version,gateway_running,overall,active_agents,gateway_state:string|null}`.
|
||||
- GET `/admin/orchestration/runtime/sessions?profile=<authorized>&limit=20&offset=0` -> `{total:number,sessions:{id,profile,source,is_active}[]}`. UI paginates; profile mismatch is rejected.
|
||||
- GET `/admin/orchestration/runtime/usage?profile=<authorized>&days=30` -> `{period_days:number,totals:{total_input,total_output,total_sessions,total_estimated_cost}}`.
|
||||
|
||||
Runtime client explicitly projects known fields; unknown secret fields are discarded. Numeric zero survives. Cost is labelled **upstream estimate**, unspecified currency, not settled billing and never added to overlapping 9router totals. Client observation time is not server heartbeat/freshness. Profile gateway flags and active agent count are source observations, not invented readiness/department counts. Generic Resources for existing agents/model-policy/schedules/releases remain unchanged; unavailable contracts stay visibly unavailable. Runtime schedules/model-info may exist backend-side but are not guessed into generic Resources.
|
||||
|
||||
## Verification
|
||||
|
||||
Runner: disposable node:22-bookworm, --userns=host --network none; source and dependency donor read-only, private copies in /work. No donor/source build writes. Required final gates: `tsc --noEmit -p tsconfig.app.json`, `npm run build`, `npm test -- --maxWorkers=2`. Logs under `/opt/services/canina/validation/v4-panel-completion/` (outside discovery). Earlier RED proves 7 workflow failures and 2 selection/idempotency failures; runtime RED is missing new component. Existing test fixtures migrated from invented VERIFIED run status to backend SUCCEEDED status. No tests skipped or permissions inferred.
|
||||
|
||||
Not claimed: live backend browser acceptance, real authenticated runtime availability, Stage publication, CI evidence authority or Production execution. Parent owns integration and publication.
|
||||
@@ -3,12 +3,14 @@ import {it,expect,vi} from 'vitest';
|
||||
import api from '../services/api';
|
||||
import CreateOrchestrationTask from './CreateOrchestrationTask';
|
||||
vi.mock('../services/api',()=>({default:{post:vi.fn()}}));
|
||||
it('submits explicit bounded scope and acceptance criteria, only after valid input',async()=>{
|
||||
it('submits exact bounded PREPARE contract only after valid commit and budget',async()=>{
|
||||
vi.mocked(api.post).mockResolvedValue({data:{id:'new'}});const done=vi.fn();render(<CreateOrchestrationTask onCreated={done}/>);
|
||||
expect(screen.getByRole('button',{name:'Create bounded task'})).toBeDisabled();
|
||||
for(const [label,value] of Object.entries({'Task key':'audit-1','Title':'Audit schema','Department':'SEO','Bounded scope':'Staging schema audit only','Acceptance criteria (one per line)':'Report all failures','Budget cap in cents':'500'}))fireEvent.change(screen.getByLabelText(label),{target:{value}});
|
||||
for(const [label,value] of Object.entries({'Task key':'audit-1','Title':'Audit schema','Department':'SEO','Commit SHA (40 lowercase hex)':'a'.repeat(40),'Budget cap in cents':'500','Scope':'Audit staging only','Acceptance criteria (one per line)':'Schema checked\nEvidence attached'}))fireEvent.change(screen.getByLabelText(label),{target:{value}});
|
||||
for(const value of ['19','10001','20.5']){fireEvent.change(screen.getByLabelText('Budget cap in cents'),{target:{value}});expect(screen.getByRole('button',{name:'Create bounded task'})).toBeDisabled();}
|
||||
fireEvent.change(screen.getByLabelText('Budget cap in cents'),{target:{value:'500'}});
|
||||
fireEvent.click(screen.getByRole('button',{name:'Create bounded task'}));
|
||||
await screen.findByText('Task created by backend.');
|
||||
expect(api.post).toHaveBeenCalledWith('/admin/orchestration/tasks',expect.objectContaining({operationMode:'PREPARE',budgetLimitCents:500,metadata:{scope:'Staging schema audit only',acceptanceCriteria:['Report all failures']}}));
|
||||
expect(api.post).toHaveBeenCalledWith('/admin/orchestration/tasks',{key:'audit-1',title:'Audit schema',department:'SEO',operationMode:'PREPARE',budgetLimitCents:500,metadata:{commitSha:'a'.repeat(40),scope:'Audit staging only',acceptanceCriteria:['Schema checked','Evidence attached']}});
|
||||
expect(done).toHaveBeenCalledOnce();
|
||||
});
|
||||
@@ -1,14 +1,15 @@
|
||||
import {useState} from 'react';
|
||||
import Button from './ui/Button';
|
||||
import {orchestration} from '../services/orchestration';
|
||||
import {orchestration,proposalError} from '../services/orchestration';
|
||||
import type {CreateTaskInput} from '../services/orchestration';
|
||||
export default function CreateOrchestrationTask({onCreated}:{onCreated:()=>void}){
|
||||
const [fields,setFields]=useState({key:'',title:'',department:'',scope:'',criteria:'',budget:''});
|
||||
const [mode,setMode]=useState<CreateTaskInput['operationMode']>('PREPARE');const [busy,setBusy]=useState(false);const [error,setError]=useState('');const [success,setSuccess]=useState('');
|
||||
const valid=Object.values(fields).every(value=>value.trim())&&Number.isSafeInteger(Number(fields.budget))&&Number(fields.budget)>=0;
|
||||
return <details className="bg-white border rounded-2xl p-5"><summary className="font-bold cursor-pointer">Create bounded work</summary><form className="space-y-3 mt-4" onSubmit={async e=>{e.preventDefault();if(!valid||busy)return;setBusy(true);setError('');setSuccess('');try{await orchestration.create({key:fields.key,title:fields.title,department:fields.department,operationMode:mode,budgetLimitCents:Number(fields.budget),metadata:{scope:fields.scope,acceptanceCriteria:fields.criteria.split('\n').map(s=>s.trim()).filter(Boolean)}});setSuccess('Task created by backend.');onCreated()}catch(error){setError(error instanceof Error?error.message:'Create failed; reconcile task key before retry')}finally{setBusy(false)}}}>
|
||||
{([['key','Task key'],['title','Title'],['department','Department'],['scope','Bounded scope'],['criteria','Acceptance criteria (one per line)'],['budget','Budget cap in cents']] as const).map(([key,label])=><label key={key} className="block">{label}<textarea aria-label={label} disabled={busy} required value={fields[key]} onChange={e=>setFields(prev=>({...prev,[key]:e.target.value}))} className="block border p-2 w-full rounded-lg"/></label>)}
|
||||
<label>Operation mode<select aria-label="Operation mode" value={mode} disabled={busy} onChange={e=>setMode(e.target.value as CreateTaskInput['operationMode'])}>{['OBSERVE','PREPARE','MUTATE_PRODUCTION'].map(m=><option key={m}>{m}</option>)}</select></label><p>Creation does not authorize work. Gate1 is a separate immutable decision.</p>
|
||||
{error&&<p role="alert">{error}</p>}{success&&<p role="status">{success}</p>}<Button type="submit" disabled={!valid||busy}>Create bounded task</Button>
|
||||
const [fields,setFields]=useState({key:'',title:'',department:'',commitSha:'',budget:'',scope:'',criteria:''});
|
||||
const [busy,setBusy]=useState(false);const [error,setError]=useState('');const [success,setSuccess]=useState('');
|
||||
const input:CreateTaskInput={key:fields.key,title:fields.title,department:fields.department,operationMode:'PREPARE',budgetLimitCents:Number(fields.budget),metadata:{commitSha:fields.commitSha,scope:fields.scope,acceptanceCriteria:fields.criteria.split('\n')}};
|
||||
const valid=Object.values(fields).every(value=>value.trim())&&!proposalError(input);
|
||||
return <details className="bg-white border rounded-2xl p-5"><summary className="font-bold cursor-pointer">Create bounded work</summary><form className="space-y-3 mt-4" onSubmit={async e=>{e.preventDefault();if(!valid||busy||success)return;setBusy(true);setError('');setSuccess('');try{await orchestration.create(input);setSuccess('Task created by backend.');onCreated()}catch(error){setError(error instanceof Error?error.message+' — Refresh and reconcile the task key before resubmitting.':'Create failed; reconcile task key before retry')}finally{setBusy(false)}}}>
|
||||
{([['key','Task key'],['title','Title'],['department','Department'],['commitSha','Commit SHA (40 lowercase hex)'],['budget','Budget cap in cents'],['scope','Scope'],['criteria','Acceptance criteria (one per line)']] as const).map(([key,label])=><label key={key} className="block">{label}<textarea aria-label={label} disabled={busy||!!success} required value={fields[key]} onChange={e=>setFields(prev=>({...prev,[key]:e.target.value}))} className="block border p-2 w-full rounded-lg"/></label>)}
|
||||
<p>Operation mode: PREPARE · Environment: staging. Budget: 20..10000 cents. Scope: nonblank, at most 2000 characters. Acceptance criteria: 1..20 nonblank lines, at most 500 characters each. Human scope and criteria are persisted unchanged; the workflow restriction is server-owned.</p><p>Creation does not authorize work. Gate1 is a separate immutable decision. Production proposals are disabled. After an ambiguous failure, refresh and reconcile the unique task key before resubmitting.</p>
|
||||
{error&&<p role="alert">{error}</p>}{success&&<p role="status">{success}</p>}<Button type="submit" disabled={!valid||busy||!!success}>Create bounded task</Button>
|
||||
</form></details>;
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
import {render,screen,fireEvent,waitFor} from '@testing-library/react';
|
||||
import {beforeEach,it,expect,vi} from 'vitest';
|
||||
import api from '../services/api';
|
||||
import RuntimeReadPanel from './RuntimeReadPanel';
|
||||
const provenance={source:'snapshot',capturedAt:'2020-01-01T00:00:00Z',ageSeconds:5000,freshness:'stale',liveProcess:'unknown'};
|
||||
it('labels snapshot inventory without presenting stopped gateway, zero agents or active sessions as live',async()=>{
|
||||
vi.mocked(api.get).mockImplementation(async url=>({data:url.endsWith('/health')?{ok:true,version:'v1',commit:'abc',auth_required:true,provenance}:url.endsWith('/profiles')?[{name:'dev',is_default:false,gateway_running:false,liveGatewayState:'unknown',provenance}]:url.endsWith('/status')?{version:'v1',gateway_running:false,overall:'unknown',active_agents:0,gateway_state:null,liveGatewayState:'unknown',liveActiveAgents:null,provenance}:url.endsWith('/sessions')?{total:1,sessions:[{id:'session-1',profile:'dev',source:'cli',is_active:true}],provenance}:{period_days:30,totals:{total_input:1,total_output:2,total_sessions:1,total_estimated_cost:0},provenance}}));
|
||||
render(<RuntimeReadPanel/>);
|
||||
expect(await screen.findByText('Inventory service: responsive (not live Hermes health)')).toBeInTheDocument();
|
||||
screen.getAllByText(/Captured at: 2020-01-01T00:00:00Z/).forEach(row=>expect(row).toHaveTextContent('stale'));
|
||||
fireEvent.change(screen.getByLabelText('Authorized runtime profile'),{target:{value:'dev'}});
|
||||
expect(await screen.findByText(/Active agents: unknown/)).toBeInTheDocument();
|
||||
expect(await screen.findByText(/historical snapshot flag: active; live activity unknown/)).toBeInTheDocument();
|
||||
expect(screen.queryByText(/Gateway.*stopped/)).not.toBeInTheDocument();
|
||||
expect(screen.queryByText(/Active agents: 0/)).not.toBeInTheDocument();
|
||||
expect(screen.queryByText(/runtime health: healthy/)).not.toBeInTheDocument();
|
||||
});
|
||||
vi.mock('../services/api',()=>({default:{get:vi.fn()}}));
|
||||
beforeEach(()=>{vi.resetAllMocks();vi.mocked(api.get).mockImplementation(async url=>{
|
||||
if(url.endsWith('/health'))return {data:{ok:true,version:'v1',commit:'abc',auth_required:true}};
|
||||
if(url.endsWith('/profiles'))return {data:[{name:'dev',is_default:false,gateway_running:false}]};
|
||||
if(url.endsWith('/status'))return {data:{version:'v1',gateway_running:false,overall:'degraded',active_agents:0,gateway_state:null}};
|
||||
if(url.endsWith('/sessions'))return {data:{total:0,sessions:[]}};
|
||||
if(url.endsWith('/usage'))return {data:{period_days:30,totals:{total_input:0,total_output:0,total_sessions:0,total_estimated_cost:0}}};
|
||||
throw new Error('Runtime unavailable');
|
||||
});});
|
||||
it('shows actual health identity, explicit stopped gateway, zero usage and profile-bound reads',async()=>{
|
||||
render(<RuntimeReadPanel/>);expect(await screen.findByText('v1 · abc')).toBeInTheDocument();
|
||||
fireEvent.change(await screen.findByLabelText('Authorized runtime profile'),{target:{value:'dev'}});
|
||||
expect(await screen.findByText('degraded')).toBeInTheDocument();expect(await screen.findByText('No sessions returned.')).toBeInTheDocument();
|
||||
expect(screen.getByText('Input tokens: 0')).toBeInTheDocument();expect(screen.getByText(/Upstream estimated cost: 0/)).toBeInTheDocument();
|
||||
expect(api.get).toHaveBeenCalledWith('/admin/orchestration/runtime/sessions',{params:{profile:'dev',limit:20,offset:0}});
|
||||
});
|
||||
it('shows unavailable rather than healthy when runtime cannot be authenticated',async()=>{
|
||||
vi.mocked(api.get).mockRejectedValue(new Error('Runtime unavailable'));
|
||||
render(<RuntimeReadPanel/>);expect(await screen.findByText(/Health unavailable/)).toBeInTheDocument();expect(screen.queryByText('Authenticated runtime health: healthy')).not.toBeInTheDocument();
|
||||
});
|
||||
it('rejects malformed health and never renders unknown secret fields',async()=>{
|
||||
vi.mocked(api.get).mockImplementation(async url=>({data:url.endsWith('/health')?{ok:true,version:'v1',commit:'abc',auth_required:false,password:'secret-value'}:[]}));
|
||||
render(<RuntimeReadPanel/>);expect(await screen.findByText(/Health unavailable/)).toBeInTheDocument();expect(screen.queryByText(/secret-value/)).not.toBeInTheDocument();
|
||||
});
|
||||
it('refreshes the authenticated runtime reads without a browser runtime connection',async()=>{
|
||||
render(<RuntimeReadPanel/>);await screen.findByText('v1 · abc');fireEvent.click(screen.getByRole('button',{name:'Refresh runtime'}));await waitFor(()=>expect(vi.mocked(api.get).mock.calls.filter(([u])=>u.endsWith('/health'))).toHaveLength(2));
|
||||
});
|
||||
@@ -0,0 +1,35 @@
|
||||
import {useEffect,useState} from 'react';
|
||||
import Button from './ui/Button';
|
||||
import {runtime} from '../services/runtime';
|
||||
import type {RuntimeProvenance,RuntimeHealth,RuntimeProfile,RuntimeStatus,RuntimeSessions,RuntimeUsage} from '../services/runtime';
|
||||
function Snapshot({value}:{value?:RuntimeProvenance}){return value?<p>Snapshot inventory · Captured at: {value.capturedAt} · Age: {value.ageSeconds} seconds · {value.freshness} · Live process: {value.liveProcess}. Even fresh snapshots are not live readiness.</p>:null}
|
||||
export default function RuntimeReadPanel(){
|
||||
const [revision,setRevision]=useState(0);const [profile,setProfile]=useState('');const [offset,setOffset]=useState(0);
|
||||
const [health,setHealth]=useState<RuntimeHealth|null>(null);const [profiles,setProfiles]=useState<RuntimeProfile[]>([]);const [errors,setErrors]=useState<Record<string,string>>({});
|
||||
const [status,setStatus]=useState<RuntimeStatus|null>(null);const [sessions,setSessions]=useState<RuntimeSessions|null>(null);const [usage,setUsage]=useState<RuntimeUsage|null>(null);
|
||||
const [observed,setObserved]=useState('');
|
||||
useEffect(()=>{let active=true;setHealth(null);setProfiles([]);setErrors({});
|
||||
runtime.health().then(value=>{if(active){setHealth(value);setObserved(new Date().toISOString())}}).catch(()=>{if(active)setErrors(e=>({...e,health:'Health unavailable — authenticated backend runtime read failed.'}))});
|
||||
runtime.profiles().then(value=>{if(active){setProfiles(value);setProfile(current=>value.some(p=>p.name===current)?current:'')}}).catch(()=>{if(active){setProfile('');setErrors(e=>({...e,profiles:'Profiles unavailable — no authorized runtime identity list.'}))}});
|
||||
return ()=>{active=false};
|
||||
},[revision]);
|
||||
useEffect(()=>{let active=true;setStatus(null);setSessions(null);setUsage(null);setErrors(e=>({health:e.health,profiles:e.profiles}));if(!profile)return;
|
||||
const fail=(name:string)=>{if(active)setErrors(e=>({...e,[name]:name+' unavailable — no fallback values.'}))};
|
||||
runtime.status(profile).then(v=>{if(active)setStatus(v)}).catch(()=>fail('Status'));
|
||||
runtime.sessions(profile,offset).then(v=>{if(active)setSessions(v)}).catch(()=>fail('Sessions'));
|
||||
runtime.usage(profile).then(v=>{if(active)setUsage(v)}).catch(()=>fail('Usage'));
|
||||
return ()=>{active=false};
|
||||
},[profile,offset,revision]);
|
||||
return <section aria-label="Authenticated runtime" className="bg-white border rounded-2xl p-5 space-y-4">
|
||||
<div className="flex justify-between"><h2 className="font-bold">Hermes runtime — backend authenticated reads</h2><Button onClick={()=>setRevision(r=>r+1)}>Refresh runtime</Button></div>
|
||||
<p>No browser runtime credentials or direct Hermes connection. Refresh observations are not heartbeat timestamps.</p>
|
||||
{Object.entries(errors).filter(([,message])=>message).map(([name,message])=><p key={name} role="alert">{message}</p>)}
|
||||
{health&&<div><p>Inventory service: {health.ok?'responsive':'not responsive'} (not live Hermes health)</p><Snapshot value={health.provenance}/><p>{health.version} · {health.commit}</p><p>Client observed at: {observed}. Client receipt time only; not capture time or live heartbeat.</p></div>}
|
||||
<label>Authorized runtime profile<select aria-label="Authorized runtime profile" value={profile} onChange={e=>{setProfile(e.target.value);setOffset(0)}}><option value="">Select authorized profile</option>{profiles.map(p=><option key={p.name} value={p.name}>{p.name}</option>)}</select></label>
|
||||
{!profiles.length&&!errors.profiles&&<p>No authorized profiles returned.</p>}
|
||||
{profiles.map(p=><div key={p.name}>{p.name} · Gateway {p.provenance?'unknown (snapshot; not process readiness)':p.gateway_running?'running':'stopped'} · {p.is_default?'default profile':'non-default profile'}<Snapshot value={p.provenance}/></div>)}
|
||||
{status&&<div><h3>Profile status</h3><Snapshot value={status.provenance}/><p>{status.overall}</p><p>Gateway: {status.provenance?'unknown':status.gateway_running?'running':'stopped'} · State: {status.provenance?'unknown':status.gateway_state??'Unknown'} · Active agents: {status.provenance?'unknown':status.active_agents}</p></div>}
|
||||
{usage&&<div><h3>Runtime usage — {usage.period_days} days, selected profile only</h3><Snapshot value={usage.provenance}/><p>Input tokens: {usage.totals.total_input}</p><p>Output tokens: {usage.totals.total_output}</p><p>Sessions: {usage.totals.total_sessions}</p><p>Upstream estimated cost: {usage.totals.total_estimated_cost} (currency unspecified by source; not settled billing)</p><p>Do not add this to 9router totals; attribution overlaps and is not reconciled.</p></div>}
|
||||
{sessions&&<div><h3>Sessions — {sessions.total} returned total</h3><Snapshot value={sessions.provenance}/>{!sessions.sessions.length&&<p>No sessions returned.</p>}{sessions.sessions.map(s=><article key={s.id} className="border rounded p-2"><p>{s.id} · {s.profile} · {s.source} · {sessions.provenance?`historical snapshot flag: ${s.is_active?'active':'inactive'}; live activity unknown`:s.is_active?'active':'inactive'}</p></article>)}<Button disabled={offset===0} onClick={()=>setOffset(n=>Math.max(0,n-20))}>Previous sessions</Button><Button disabled={offset+20>=sessions.total} onClick={()=>setOffset(n=>n+20)}>Next sessions</Button></div>}
|
||||
</section>;
|
||||
}
|
||||
@@ -6,7 +6,7 @@ vi.mock('../services/api', () => ({ default: {get:vi.fn(),post:vi.fn(),put:vi.fn
|
||||
const task={id:'t1',key:'bounded-1',title:'Bounded SEO audit',department:'SEO',state:'GATE1_PENDING',version:4,operationMode:'PREPARE',budgetLimitCents:500,availableActions:[{id:'g1',label:'Approve bounded work',gate:1,decision:'APPROVED'}],runs:[],metadata:{scope:'Only staging',acceptanceCriteria:['Audit schema']}};
|
||||
describe('command center', () => {
|
||||
beforeEach(() => { vi.resetAllMocks(); vi.mocked(api.get).mockImplementation(async url => {
|
||||
if(url.endsWith('/overview'))return {data:{items:[],measuredAt:'2026-10-22T00:00:00Z',capabilities:{mutations:true,chat:true,runtime:false}}};
|
||||
if(url.endsWith('/overview'))return {data:{items:[],measuredAt:'2026-10-22T00:00:00Z',capabilities:{mutations:true,killSwitch:true,chat:true,runtime:false}}};
|
||||
if(url.endsWith('/tasks')) return {data:[task]};
|
||||
if(url.endsWith('/tasks/t1')) return {data:task};
|
||||
throw new Error('Unavailable: '+url);
|
||||
@@ -60,8 +60,8 @@ describe('command center', () => {
|
||||
expect(screen.getByText('BLOCKED')).toBeInTheDocument();
|
||||
});
|
||||
it('binds Gate2 using registered evidence, never editable commit or environment',async()=>{
|
||||
const release={...task,state:'GATE2_PENDING',availableActions:[{id:'g2',label:'Approve release',gate:2,decision:'APPROVED'}],runs:[{id:'run-1',status:'VERIFIED',commitSha:'abc123',environment:'staging',artifacts:[{id:'artifact-1',sha256:'a'.repeat(64)}]}]};
|
||||
vi.mocked(api.get).mockImplementation(async url=>({data:url.endsWith('/tasks')?[release]:release}));
|
||||
const release={...task,state:'GATE2_PENDING',availableActions:[{id:'g2',label:'Approve release',gate:2,decision:'APPROVED'}],runs:[{id:'run-1',status:'SUCCEEDED',commitSha:'abc123',environment:'staging',artifacts:[{id:'artifact-1',sha256:'a'.repeat(64)}]}]};
|
||||
vi.mocked(api.get).mockImplementation(async url=>({data:url.endsWith('/overview')?{items:[],measuredAt:'2026-10-23T00:00:00Z',capabilities:{mutations:true,chat:false,runtime:false}}:url.endsWith('/tasks')?[release]:release}));
|
||||
vi.mocked(api.post).mockResolvedValue({data:{id:'decision-2'}});
|
||||
render(<OrchestrationCommandCenter/>);
|
||||
fireEvent.click(await screen.findByRole('button',{name:'Bounded SEO audit'}));
|
||||
@@ -114,3 +114,23 @@ describe('command center', () => {
|
||||
});
|
||||
|
||||
});
|
||||
|
||||
it('requires explicit kill capability rather than generic mutations',async()=>{
|
||||
vi.mocked(api.get).mockImplementation(async url=>({data:url.endsWith('/overview')?{items:[],measuredAt:'now',capabilities:{mutations:true}}:url.endsWith('/tasks')?[task]:task}));
|
||||
render(<OrchestrationCommandCenter/>);await screen.findByRole('button',{name:'Bounded SEO audit'});
|
||||
fireEvent.click(screen.getByRole('button',{name:'Kill switch'}));
|
||||
expect(screen.queryByRole('button',{name:'Apply kill switch'})).not.toBeInTheDocument();expect(api.put).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each([{}, {scope:'STAGE_PREPARE'}, {scope:'Human scope',acceptanceCriteria:[]}])('does not claim incomplete or legacy metadata is scope evidence: %j',async metadata=>{
|
||||
vi.mocked(api.get).mockImplementation(async url=>({data:url.endsWith('/tasks')?[{...task,metadata}]:{...task,metadata}}));
|
||||
render(<OrchestrationCommandCenter/>);fireEvent.click(await screen.findByRole('button',{name:'Bounded SEO audit'}));
|
||||
expect(await screen.findByText(/Scope \/ acceptance criteria unavailable/)).toBeInTheDocument();
|
||||
expect(screen.getByText('Server workflow scope: unavailable in safe projection.')).toBeInTheDocument();
|
||||
});
|
||||
it('separates human evidence from server workflow scope without inventing a marker',async()=>{
|
||||
vi.mocked(api.get).mockImplementation(async url=>({data:url.endsWith('/tasks')?[task]:task}));
|
||||
render(<OrchestrationCommandCenter/>);fireEvent.click(await screen.findByRole('button',{name:'Bounded SEO audit'}));
|
||||
expect(await screen.findByText('Backend-reported scope / acceptance criteria')).toBeInTheDocument();
|
||||
expect(screen.getByText('Server workflow scope: unavailable in safe projection.')).toBeInTheDocument();
|
||||
});
|
||||
@@ -1,17 +1,19 @@
|
||||
import { useEffect, useState } from 'react';
|
||||
import { useEffect, useState, useRef } from 'react';
|
||||
import Button from '../components/ui/Button';
|
||||
import RouterUsagePanel from '../components/RouterUsagePanel';
|
||||
import RuntimeReadPanel from '../components/RuntimeReadPanel';
|
||||
import CreateOrchestrationTask from '../components/CreateOrchestrationTask';
|
||||
import { orchestration, killLevels } from '../services/orchestration';
|
||||
import { orchestration, killLevels, actionBlockReason, commandFailure } from '../services/orchestration';
|
||||
import type { Task, ServerAction, Resource, ResourceSnapshot, KillLevel, ChatMessage } from '../services/orchestration';
|
||||
const sections = ['Overview','Tasks & approvals','Runs & artifacts','Agents & departments','Models & budgets','Schedules','Audit & releases','Kill switch','9router usage & quota','Chat'] as const;
|
||||
const sections = ['Overview','Tasks & approvals','Runs & artifacts','Agents & departments','Models & budgets','Schedules','Audit & releases','Kill switch','9router usage & quota','Runtime health & sessions','Chat'] as const;
|
||||
type Section = typeof sections[number];
|
||||
const resources:Partial<Record<Section,Resource>> = {'Agents & departments':'agents','Models & budgets':'model-policy','Schedules':'schedules','Audit & releases':'releases'};
|
||||
const errorText=(e:unknown)=> e instanceof Error ? e.message : 'Unknown backend error';
|
||||
function Json({value}:{value:unknown}) {return <pre className="text-xs whitespace-pre-wrap break-all bg-gray-50 border rounded-xl p-3" dir="ltr">{JSON.stringify(value,null,2)}</pre>}
|
||||
export default function OrchestrationCommandCenter() {
|
||||
const detailEpoch=useRef(0);
|
||||
const [section,setSection]=useState<Section>('Overview');
|
||||
const [capabilities,setCapabilities]=useState({mutations:false,chat:false,runtime:false});
|
||||
const [capabilities,setCapabilities]=useState({mutations:false,chat:false,runtime:false,killSwitch:false});
|
||||
const [tasks,setTasks]=useState<Task[]>([]);
|
||||
const [selected,setSelected]=useState<Task|null>(null);
|
||||
const [snapshot,setSnapshot]=useState<ResourceSnapshot|null>(null);
|
||||
@@ -22,12 +24,15 @@ export default function OrchestrationCommandCenter() {
|
||||
const [binding,setBinding]=useState(''); const [confirmed,setConfirmed]=useState(false);
|
||||
const [attempted,setAttempted]=useState(false); const [busy,setBusy]=useState(false); const [commandError,setCommandError]=useState('');
|
||||
const [success,setSuccess]=useState('');
|
||||
const [blocked,setBlocked]=useState(false);
|
||||
const [now,setNow]=useState(Date.now());
|
||||
useEffect(()=>{const timer=setInterval(()=>setNow(Date.now()),1000);return ()=>clearInterval(timer)},[]);
|
||||
const [kill,setKill]=useState<KillLevel>('FREEZE_NEW_WORK');
|
||||
const [session,setSession]=useState(''); const [message,setMessage]=useState('');
|
||||
const [messages,setMessages]=useState<ChatMessage[]>([]);
|
||||
const [chatKey,setChatKey]=useState(()=>crypto.randomUUID());
|
||||
const loadTasks=async()=> {setLoading(true);setError(''); try {setTasks(await orchestration.tasks())} catch(e){setTasks([]);setSelected(null);setError(errorText(e))} finally {setLoading(false)}};
|
||||
useEffect(()=>{void loadTasks(); orchestration.resource('overview').then(s=>setCapabilities({mutations:s.capabilities?.mutations===true,chat:s.capabilities?.chat===true,runtime:s.capabilities?.runtime===true})).catch(()=>setCapabilities({mutations:false,chat:false,runtime:false}));},[]);
|
||||
useEffect(()=>{void loadTasks(); orchestration.resource('overview').then(s=>setCapabilities({mutations:s.capabilities?.mutations===true,chat:s.capabilities?.chat===true,runtime:s.capabilities?.runtime===true,killSwitch:s.capabilities?.killSwitch===true})).catch(()=>setCapabilities({mutations:false,chat:false,runtime:false,killSwitch:false}));},[]);
|
||||
useEffect(()=> {
|
||||
const resource=resources[section]; setSnapshot(null);setError('');
|
||||
if(!resource)return;
|
||||
@@ -35,21 +40,21 @@ export default function OrchestrationCommandCenter() {
|
||||
orchestration.resource(resource).then(data=>{if(active)setSnapshot(data)}).catch(e=>{if(active)setError(errorText(e))}).finally(()=>{if(active)setLoading(false)});
|
||||
return ()=>{active=false};
|
||||
},[section]);
|
||||
const select=async(task:Task)=>{setSelected(null);setError('');try {setSelected(await orchestration.task(task.id))}catch(e){setError(errorText(e))}};
|
||||
const open=(task:Task,action:ServerAction)=>{setAttempted(false);setPending({task,action,key:crypto.randomUUID()});setExpires('');setReason('');setConfirmed(false);setBinding('');setCommandError('');setSuccess('')};
|
||||
const select=async(task:Task)=>{const epoch=++detailEpoch.current;setSelected(null);setError('');try {const detail=await orchestration.task(task.id);if(epoch===detailEpoch.current)setSelected(detail)}catch(e){if(epoch===detailEpoch.current)setError(errorText(e))}};
|
||||
const open=(task:Task,action:ServerAction)=>{if(!capabilities.mutations||actionBlockReason(task,action))return;setBlocked(false);setAttempted(false);setPending({task,action,key:crypto.randomUUID()});setExpires('');setReason('');setConfirmed(false);setBinding('');setCommandError('');setSuccess('')};
|
||||
const execute=async()=> {
|
||||
if(!pending||busy||!confirmed)return;
|
||||
if(!pending||busy||!confirmed||blocked||!capabilities.mutations)return;
|
||||
setAttempted(true);setBusy(true);setCommandError('');
|
||||
try { const [runId,artifactHash]=binding.split('|');
|
||||
await orchestration.command(pending.task,pending.action,{idempotencyKey:pending.key,reason,expiresAt:expires?new Date(expires).toISOString():undefined,runId,artifactHash});
|
||||
setPending(null);setSelected(null);setSuccess('Command accepted by backend; refreshed authoritative state.');await loadTasks();
|
||||
} catch(e) {setCommandError(errorText(e)+' — Refresh before retrying a version conflict; network retries keep the same key.')} finally {setBusy(false)}
|
||||
} catch(e) {const failure=commandFailure(e);setBlocked(!failure.retryable);setCommandError(failure.message)} finally {setBusy(false)}
|
||||
};
|
||||
const refresh=()=>{setSelected(null);setPending(null);void loadTasks()};
|
||||
const refresh=()=>{detailEpoch.current++;setSelected(null);setPending(null);void loadTasks()};
|
||||
const filtered=tasks.filter(t=>(t.title+' '+t.department+' '+t.state).toLowerCase().includes(search.toLowerCase()));
|
||||
return <div className="space-y-5 font-vazir pb-12">
|
||||
<header className="bg-white border rounded-2xl p-5 flex justify-between gap-4"><div><h1 className="text-xl font-black">Orchestration command center</h1><p className="text-sm text-gray-600">Backend-authoritative operations · Gate1 bounded work · Gate2 immutable release binding</p></div><Button onClick={refresh} disabled={busy||loading}>Refresh authoritative state</Button></header>
|
||||
<nav aria-label="Orchestration sections" className="flex flex-wrap gap-2">{sections.map(s=><Button key={s} onClick={()=>{setSection(s);setSuccess('');setReason('');setConfirmed(false)}} variant={section===s?'primary':'outline'}>{s}</Button>)}</nav>
|
||||
<nav aria-label="Orchestration sections" className="flex flex-wrap gap-2">{sections.map(s=><Button key={s} disabled={!!pending||busy} onClick={()=>{setSection(s);setSuccess('');setReason('');setConfirmed(false)}} variant={section===s?'primary':'outline'}>{s}</Button>)}</nav>
|
||||
{loading&&<p role="status">Loading measured backend state…</p>}
|
||||
{error&&<p role="alert" className="border border-red-200 bg-red-50 rounded-xl p-4">Unknown / unavailable: {error}. No fallback data. <button onClick={refresh}>Reload tasks</button></p>}
|
||||
{success&&<p role="status">{success}</p>}
|
||||
@@ -59,24 +64,25 @@ export default function OrchestrationCommandCenter() {
|
||||
{!loading&&!error&&!tasks.length&&<p>No tasks returned by backend.</p>}
|
||||
<div className="overflow-x-auto"><table className="w-full text-sm"><thead><tr>{['Task','Department','State','Mode','Budget cap (cents)','Version'].map(h=><th key={h} className="text-start p-2">{h}</th>)}</tr></thead><tbody>{filtered.map(t=><tr key={t.id} className="border-t"><td className="p-2"><button className="text-purple-700 underline" onClick={()=>void select(t)}>{t.title}</button></td><td>{t.department}</td><td>{t.state}</td><td>{t.operationMode}</td><td>{t.budgetLimitCents??'Unknown'}</td><td>{t.version}</td></tr>)}</tbody></table></div>
|
||||
</section>}
|
||||
{selected&&<section aria-label="Task detail" className="bg-white border rounded-2xl p-5 space-y-4"><h2 className="font-bold">{selected.title} · v{selected.version}</h2>{selected.metadata?<h3>Backend-reported scope / acceptance criteria</h3>:<p>Scope / acceptance criteria unavailable — omitted by the safe backend projection.</p>}<Json value={{operationMode:selected.operationMode,budgetLimitCents:selected.budgetLimitCents,metadata:selected.metadata}}/>
|
||||
<h3>Server-authorized actions</h3>{!selected.availableActions.length&&<p>No actions advertised. Read-only; the browser does not infer transitions.</p>}<div className="flex flex-wrap gap-2">{selected.availableActions.map(a=><Button key={a.id} disabled={!!a.disabledReason} title={a.disabledReason} onClick={()=>open(selected,a)}>{a.label}</Button>)}</div>
|
||||
{selected&&<section aria-label="Task detail" className="bg-white border rounded-2xl p-5 space-y-4"><h2 className="font-bold">{selected.title} · v{selected.version}</h2>{typeof selected.metadata?.scope==='string'&&selected.metadata.scope.trim()&&Array.isArray(selected.metadata.acceptanceCriteria)&&selected.metadata.acceptanceCriteria.length>0&&selected.metadata.acceptanceCriteria.every(value=>typeof value==='string'&&value.trim())?<h3>Backend-reported scope / acceptance criteria</h3>:<p>Scope / acceptance criteria unavailable — omitted by the safe backend projection.</p>}<p>Server workflow scope: unavailable in safe projection.</p><Json value={{operationMode:selected.operationMode,budgetLimitCents:selected.budgetLimitCents,metadata:selected.metadata}}/>
|
||||
<h3>Server-authorized actions</h3>{!selected.availableActions.length&&<p>No actions advertised. Read-only; the browser does not infer transitions.</p>}<div className="flex flex-wrap gap-2">{selected.availableActions.map(a=><Button key={a.id} disabled={!capabilities.mutations||!!actionBlockReason(selected,a)} title={!capabilities.mutations?'Backend mutations unavailable':actionBlockReason(selected,a)} onClick={()=>open(selected,a)}>{a.label}</Button>)}</div>
|
||||
<h3>Immutable approval decisions</h3><Json value={selected.decisions??[]}/>
|
||||
<h3>Runs & registered artifacts</h3>{!selected.runs?.length&&<p>No registered runs.</p>}{selected.runs?.map(r=><article key={r.id} className="border rounded-xl p-3 space-y-2"><h4>{r.id} · {r.status}</h4><p>Commit: {r.commitSha} · Environment: {r.environment}</p>{r.artifacts.map(a=><Json key={a.id} value={a}/>)}</article>)}
|
||||
<h3>Budget ledger (cents; not inferred usage)</h3><Json value={selected.budgetLedger??[]}/><h3>Ordered audit events</h3><Json value={selected.events??[]}/>
|
||||
</section>}
|
||||
{snapshot&&<section className="bg-white border rounded-2xl p-5 space-y-4"><h2>{section}</h2><p>Measured at: {snapshot.measuredAt}</p>{!snapshot.items.length&&<p>No records returned by backend.</p>}<div className="grid md:grid-cols-2 gap-3">{snapshot.items.map(item=><article key={item.id} className="border rounded-xl p-4"><h3>{item.title??item.id}</h3><dl>{Object.entries(item).filter(([k])=>k!=='id'&&k!=='title').map(([k,v])=><div key={k} className="flex justify-between gap-3"><dt>{k}</dt><dd>{String(v??'Unknown')}</dd></div>)}</dl></article>)}</div><p>Configuration mutations require backend-advertised commands; none are inferred by this client.</p></section>}
|
||||
{section==='Kill switch'&&!capabilities.mutations&&<p>Mutation controls unavailable: backend security hardening pending.</p>}
|
||||
{section==='Kill switch'&&capabilities.mutations&&<section className="border border-red-300 rounded-2xl bg-white p-5 space-y-3"><h2>Global kill switch — four escalating levels</h2><p>Applies globally, not only to selected task. No implicit resume. Current active level: unknown until backend returns state.</p><label>Level<select aria-label="Kill level" value={kill} onChange={e=>{setKill(e.target.value as KillLevel);setConfirmed(false)}}>{killLevels.map(l=><option key={l}>{l}</option>)}</select></label><p>Freeze new work → stop agents → disable mutations → emergency halt.</p><label>Reason<input aria-label="Kill reason" value={reason} onChange={e=>setReason(e.target.value)} className="border p-2"/></label><label className="block"><input type="checkbox" checked={confirmed} onChange={e=>setConfirmed(e.target.checked)}/>I confirm global operational impact</label><Button disabled={busy||!reason.trim()||!confirmed} onClick={async()=>{setBusy(true);setError('');try{const result=await orchestration.kill(kill,reason);setSuccess('Backend acknowledged kill switch: '+JSON.stringify(result));setConfirmed(false)}catch(e){setError(errorText(e))}finally{setBusy(false)}}}>Apply kill switch</Button></section>}
|
||||
{section==='Kill switch'&&!capabilities.killSwitch&&<p>Mutation controls unavailable: backend security hardening pending.</p>}
|
||||
{section==='Kill switch'&&capabilities.killSwitch&&<section className="border border-red-300 rounded-2xl bg-white p-5 space-y-3"><h2>Global kill switch — four escalating levels</h2><p>Applies globally, not only to selected task. No implicit resume. Current active level: unknown until backend returns state.</p><label>Level<select aria-label="Kill level" value={kill} onChange={e=>{setKill(e.target.value as KillLevel);setConfirmed(false)}}>{killLevels.map(l=><option key={l}>{l}</option>)}</select></label><p>Freeze new work → stop agents → disable mutations → emergency halt.</p><label>Reason<input aria-label="Kill reason" value={reason} onChange={e=>setReason(e.target.value)} className="border p-2"/></label><label className="block"><input type="checkbox" checked={confirmed} onChange={e=>setConfirmed(e.target.checked)}/>I confirm global operational impact</label><Button disabled={busy||!reason.trim()||!confirmed} onClick={async()=>{setBusy(true);setError('');try{const result=await orchestration.kill(kill,reason);setSuccess('Backend acknowledged kill switch: '+JSON.stringify(result));setConfirmed(false)}catch(e){setError(errorText(e))}finally{setBusy(false)}}}>Apply kill switch</Button></section>}
|
||||
{section==='9router usage & quota'&&<RouterUsagePanel/>}
|
||||
{section==='Runtime health & sessions'&&<RuntimeReadPanel/>}
|
||||
{section==='Chat'&&!capabilities.chat&&<p>Chat unsupported: trusted runtime bridge unavailable.</p>}
|
||||
{section==='Chat'&&capabilities.chat&&<section className="bg-white border rounded-2xl p-5 space-y-3"><h2>Typed backend chat bridge</h2><p>Advisory text only. No terminal, PTY, runtime token, or direct Hermes connection. Decisions use explicit approval controls.</p><label>Session ID<input value={session} onChange={e=>{setSession(e.target.value);setMessages([])}} className="border p-2"/></label><div role="log">{messages.map(m=><p key={m.id}>{m.role}: {m.content}</p>)}</div><label>Message<textarea value={message} onChange={e=>{setMessage(e.target.value);setChatKey(crypto.randomUUID())}} className="border p-2 block w-full"/></label><Button disabled={busy||!session.trim()||!message.trim()} onClick={async()=>{setBusy(true);setError('');try{const reply=await orchestration.chat(session,message,chatKey);setMessages(prev=>[...prev,{id:chatKey,role:'user',content:message},reply]);setMessage('');setChatKey(crypto.randomUUID())}catch(e){setError(errorText(e))}finally{setBusy(false)}}}>Send advisory message</Button></section>}
|
||||
{pending&&<div role="dialog" aria-modal="true" aria-label="Confirm immutable command" className="fixed inset-0 z-50 bg-black/50 overflow-y-auto p-8"><div className="bg-white rounded-2xl p-6 max-w-2xl mx-auto space-y-4"><h2>{pending.action.label}</h2><Json value={{taskId:pending.task.id,expectedVersion:pending.task.version,idempotencyKey:pending.key,scope:pending.task.metadata,budgetLimitCents:pending.task.budgetLimitCents,operationMode:pending.task.operationMode}}/>
|
||||
{pending.action.gate&&<label className="block">Expires at<input aria-label="Expires at" type="datetime-local" value={expires} onChange={e=>setExpires(e.target.value)} disabled={busy||attempted}/></label>}
|
||||
{pending.action.gate===2&&pending.action.decision==='APPROVED'&&<label>Registered run / artifact binding<select aria-label="Registered run / artifact binding" value={binding} onChange={e=>{setBinding(e.target.value);setConfirmed(false)}} disabled={busy||attempted}><option value="">Select immutable evidence</option>{pending.task.runs?.flatMap(r=>r.artifacts.map(a=><option key={r.id+'|'+a.sha256} value={r.id+'|'+a.sha256}>{r.id} · {r.commitSha} · {r.environment} · {a.sha256}</option>))}</select></label>}
|
||||
{pending.action.gate===2&&pending.action.decision==='APPROVED'&&<label>Registered run / artifact binding<select aria-label="Registered run / artifact binding" value={binding} onChange={e=>{setBinding(e.target.value);setConfirmed(false)}} disabled={busy||attempted}><option value="">Select immutable evidence</option>{pending.task.runs?.filter(r=>r.status==='SUCCEEDED').flatMap(r=>r.artifacts.map(a=><option key={r.id+'|'+a.sha256} value={r.id+'|'+a.sha256}>{r.id} · {r.commitSha} · {r.environment} · {a.sha256}</option>))}</select></label>}
|
||||
<label className="block">Reason<input aria-label="Reason" value={reason} onChange={e=>setReason(e.target.value)} disabled={busy||attempted} className="border p-2"/></label>
|
||||
<label className="block"><input type="checkbox" checked={confirmed} onChange={e=>setConfirmed(e.target.checked)} disabled={busy}/>I reviewed the immutable scope and binding</label>
|
||||
{commandError&&<p role="alert">{commandError}</p>}<div className="flex gap-3"><Button disabled={busy||!confirmed||!reason.trim()|| (!!pending.action.gate&&!Number.isFinite(Date.parse(expires))) || (pending.action.gate===2&&pending.action.decision==='APPROVED'&&!binding)} onClick={()=>void execute()}>Confirm command</Button><Button variant="outline" disabled={busy} onClick={()=>setPending(null)}>Cancel</Button></div>
|
||||
{commandError&&<p role="alert">{commandError}</p>}<div className="flex gap-3"><Button disabled={busy||blocked||!capabilities.mutations||!confirmed||!reason.trim()|| (!!pending.action.gate&&!(Date.parse(expires)>now)) || (['APPROVE_GATE1','REJECT_GATE1'].includes(pending.action.id)&&Date.parse(expires)>now+86400000) || (pending.action.gate===2&&pending.action.decision==='APPROVED'&&!binding)} onClick={()=>void execute()}>Confirm command</Button><Button variant="outline" disabled={busy} onClick={()=>setPending(null)}>Cancel</Button></div>
|
||||
</div></div>}
|
||||
</div>;
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
import {render,screen,fireEvent,within,waitFor,act} from '@testing-library/react';
|
||||
import {it,expect,vi} from 'vitest';
|
||||
import api from '../services/api';
|
||||
import OrchestrationCommandCenter from './OrchestrationCommandCenter';
|
||||
vi.mock('../services/api',()=>({default:{get:vi.fn(),post:vi.fn(),put:vi.fn()}}));
|
||||
it('does not restore stale detail after authoritative refresh while detail request is pending',async()=>{
|
||||
const t={id:'t',key:'t',title:'Task identity',department:'DEV',state:'GATE1_PENDING',version:2,operationMode:'PREPARE',budgetLimitCents:0,availableActions:[]};
|
||||
let resolve!:(v:unknown)=>void;
|
||||
vi.mocked(api.get).mockImplementation(async url=>{if(url.endsWith('/overview'))return {data:{items:[],measuredAt:'2026-10-23',capabilities:{mutations:true}}};if(url.endsWith('/tasks'))return {data:[t]};return await new Promise<unknown>(r=>{resolve=r}) as never});
|
||||
render(<OrchestrationCommandCenter/>);fireEvent.click(await screen.findByRole('button',{name:'Task identity'}));fireEvent.click(screen.getByRole('button',{name:'Refresh authoritative state'}));
|
||||
await act(async()=>resolve({data:t}));await waitFor(()=>expect(screen.queryByText('Loading measured backend state…')).not.toBeInTheDocument());
|
||||
expect(screen.queryByRole('region',{name:'Task detail'})).not.toBeInTheDocument();
|
||||
});
|
||||
it('keeps payload immutable when navigating after an ambiguous network failure',async()=>{
|
||||
vi.resetAllMocks();const t={id:'t',key:'t',title:'Task identity',department:'DEV',state:'GATE1_PENDING',version:2,operationMode:'PREPARE',budgetLimitCents:0,availableActions:[{id:'g1',label:'Approve safe work',gate:1,decision:'APPROVED'}]};
|
||||
vi.mocked(api.get).mockImplementation(async url=>({data:url.endsWith('/overview')?{items:[],measuredAt:'2026-10-23',capabilities:{mutations:true}}:url.endsWith('/tasks')?[t]:t}));vi.mocked(api.post).mockRejectedValue(new Error('network'));
|
||||
render(<OrchestrationCommandCenter/>);fireEvent.click(await screen.findByRole('button',{name:'Task identity'}));fireEvent.click(await screen.findByRole('button',{name:'Approve safe work'}));const d=within(screen.getByRole('dialog'));fireEvent.change(d.getByLabelText('Expires at'),{target:{value:'2099-01-01T12:00'}});fireEvent.change(d.getByLabelText('Reason'),{target:{value:'Reviewed'}});fireEvent.click(d.getByLabelText('I reviewed the immutable scope and binding'));fireEvent.click(d.getByRole('button',{name:'Confirm command'}));await d.findByRole('alert');
|
||||
fireEvent.click(screen.getByRole('button',{name:'Tasks & approvals'}));expect(d.getByLabelText('Reason')).toHaveValue('Reviewed');expect(d.getByLabelText('I reviewed the immutable scope and binding')).toBeChecked();
|
||||
});
|
||||
@@ -0,0 +1,47 @@
|
||||
import {render,screen,fireEvent,within,waitFor} from '@testing-library/react';
|
||||
import {beforeEach,it,expect,vi} from 'vitest';
|
||||
import api from '../services/api';
|
||||
import {orchestration} from '../services/orchestration';
|
||||
import type {CreateTaskInput} from '../services/orchestration';
|
||||
import OrchestrationCommandCenter from './OrchestrationCommandCenter';
|
||||
vi.mock('../services/api',()=>({default:{get:vi.fn(),post:vi.fn(),put:vi.fn()}}));
|
||||
const action={id:'g1',label:'Review Gate1',gate:1 as const,decision:'APPROVED' as const};
|
||||
const task={id:'t',key:'t',title:'Safe proposal',department:'DEV',state:'GATE1_PENDING',version:2,operationMode:'PREPARE',budgetLimitCents:0,availableActions:[action],runs:[]};
|
||||
beforeEach(()=>{vi.resetAllMocks();vi.mocked(api.get).mockImplementation(async url=>({data:url.endsWith('/overview')?{items:[],measuredAt:'2026-10-23T00:00:00Z',capabilities:{mutations:true,chat:false,runtime:false}}:url.endsWith('/tasks')?[task]:task}));});
|
||||
it('refuses Production creation even when called outside form',async()=>{
|
||||
await expect(orchestration.create({key:'x',title:'x',department:'DEV',operationMode:'MUTATE_PRODUCTION',budgetLimitCents:0,metadata:{commitSha:'a'.repeat(40)}} as unknown as CreateTaskInput)).rejects.toThrow(/PREPARE|advertised/);
|
||||
expect(api.post).not.toHaveBeenCalled();
|
||||
});
|
||||
it('rejects ambiguous advertised action rather than selecting a mutation path',async()=>{
|
||||
const ambiguous={...action,toState:'READY'};
|
||||
await expect(orchestration.command({...task,availableActions:[ambiguous]},ambiguous,{idempotencyKey:'key',expiresAt:'2099-01-01T00:00:00Z'})).rejects.toThrow(/advertised/);
|
||||
expect(api.post).not.toHaveBeenCalled();
|
||||
});
|
||||
it('blocks release transitions even when advertised by server',async()=>{
|
||||
const release={id:'release',label:'Release',toState:'RELEASING'};
|
||||
await expect(orchestration.command({...task,availableActions:[release]},release,{idempotencyKey:'key'})).rejects.toThrow(/PREPARE|advertised/);
|
||||
expect(api.put).not.toHaveBeenCalled();
|
||||
});
|
||||
it('rejects Gate2 evidence from failed runs',async()=>{
|
||||
const g2={...action,gate:2 as const};
|
||||
await expect(orchestration.command({...task,availableActions:[g2],runs:[{id:'r',status:'FAILED',commitSha:'abc',environment:'stage',artifacts:[{id:'a',sha256:'a'.repeat(64)}]}]},g2,{idempotencyKey:'key',expiresAt:'2099-01-01T00:00:00Z',runId:'r',artifactHash:'a'.repeat(64)})).rejects.toThrow(/successful/);
|
||||
expect(api.post).not.toHaveBeenCalled();
|
||||
});
|
||||
async function dialog(){render(<OrchestrationCommandCenter/>);fireEvent.click(await screen.findByRole('button',{name:'Safe proposal'}));fireEvent.click(await screen.findByRole('button',{name:'Review Gate1'}));const d=within(screen.getByRole('dialog'));fireEvent.change(d.getByLabelText('Expires at'),{target:{value:'2099-01-01T12:00'}});fireEvent.change(d.getByLabelText('Reason'),{target:{value:'Reviewed bounds'}});fireEvent.click(d.getByLabelText('I reviewed the immutable scope and binding'));return d;}
|
||||
it('locks stale 409 command until authoritative refresh instead of retrying stale payload',async()=>{
|
||||
vi.mocked(api.post).mockRejectedValue({response:{status:409,data:{message:'Version conflict'}}});
|
||||
const d=await dialog();fireEvent.click(d.getByRole('button',{name:'Confirm command'}));
|
||||
await waitFor(()=>expect(d.getByRole('button',{name:'Confirm command'})).toBeDisabled());
|
||||
expect(await d.findByRole('alert')).toHaveTextContent(/stale/i);
|
||||
fireEvent.click(d.getByRole('button',{name:'Confirm command'}));expect(api.post).toHaveBeenCalledTimes(1);
|
||||
fireEvent.click(screen.getByRole('button',{name:'Refresh authoritative state'}));expect(screen.queryByRole('dialog')).not.toBeInTheDocument();await waitFor(()=>expect(screen.queryByText('Loading measured backend state…')).not.toBeInTheDocument());
|
||||
});
|
||||
it('does not expose authorized task mutations when capabilities are unavailable',async()=>{
|
||||
vi.mocked(api.get).mockImplementation(async url=>{if(url.endsWith('/overview'))throw new Error('offline');return {data:url.endsWith('/tasks')?[task]:task};});
|
||||
render(<OrchestrationCommandCenter/>);fireEvent.click(await screen.findByRole('button',{name:'Safe proposal'}));await screen.findByRole('region',{name:'Task detail'});
|
||||
expect(screen.getByRole('button',{name:'Review Gate1'})).toBeDisabled();
|
||||
});
|
||||
it('prevents confirming an already expired approval',async()=>{
|
||||
const d=await dialog();fireEvent.change(d.getByLabelText('Expires at'),{target:{value:'2020-01-01T12:00'}});
|
||||
expect(d.getByRole('button',{name:'Confirm command'})).toBeDisabled();
|
||||
});
|
||||
@@ -18,7 +18,7 @@ describe('orchestration adapter', () => {
|
||||
expect(api.put).not.toHaveBeenCalled(); expect(api.post).not.toHaveBeenCalled();
|
||||
});
|
||||
it('binds Gate2 to immutable registered run and sends version and retry-stable key', async () => {
|
||||
const run={id:'r',commitSha:'abc',environment:'staging',status:'VERIFIED',artifacts:[{id:'a',sha256:'a'.repeat(64)}]};
|
||||
const run={id:'r',commitSha:'abc',environment:'staging',status:'SUCCEEDED',artifacts:[{id:'a',sha256:'a'.repeat(64)}]};
|
||||
const input={idempotencyKey:'same-key',expiresAt:'2099-01-01T00:00:00Z',runId:'r',artifactHash:'a'.repeat(64)};
|
||||
vi.mocked(api.post).mockRejectedValueOnce(new Error('network')).mockResolvedValueOnce({data:{id:'decision'}});
|
||||
await expect(orchestration.command({...task,runs:[run]},task.availableActions[0],input)).rejects.toThrow('network');
|
||||
@@ -46,9 +46,84 @@ describe('orchestration adapter', () => {
|
||||
expect(api.post).toHaveBeenCalledWith('/admin/orchestration/chat/messages',{sessionId:'session',message:'Review task',idempotencyKey:'key'});
|
||||
});
|
||||
it('creates bounded work without permitting invalid caps or client-supplied state',async()=>{
|
||||
await expect(orchestration.create({key:'t',title:'Audit',department:'SEO',operationMode:'PREPARE',budgetLimitCents:-1,metadata:{scope:'staging',acceptanceCriteria:['report']}})).rejects.toThrow(/budget/);
|
||||
await expect(orchestration.create({key:'t',title:'Audit',department:'SEO',operationMode:'PREPARE',budgetLimitCents:-1,metadata:{commitSha:'a'.repeat(40),scope:'Audit staging only',acceptanceCriteria:['Schema checked']}})).rejects.toThrow(/budget/);
|
||||
vi.mocked(api.post).mockResolvedValue({data:{id:'new-task'}});
|
||||
await orchestration.create({key:'t',title:'Audit',department:'SEO',operationMode:'PREPARE',budgetLimitCents:0,metadata:{scope:'staging',acceptanceCriteria:['report']}});
|
||||
expect(api.post).toHaveBeenCalledWith('/admin/orchestration/tasks',{key:'t',title:'Audit',department:'SEO',operationMode:'PREPARE',budgetLimitCents:0,metadata:{scope:'staging',acceptanceCriteria:['report']}});
|
||||
await orchestration.create({key:'t',title:'Audit',department:'SEO',operationMode:'PREPARE',budgetLimitCents:20,metadata:{commitSha:'a'.repeat(40),scope:'Audit staging only',acceptanceCriteria:['Schema checked']}});
|
||||
expect(api.post).toHaveBeenCalledWith('/admin/orchestration/tasks',{key:'t',title:'Audit',department:'SEO',operationMode:'PREPARE',budgetLimitCents:20,metadata:{commitSha:'a'.repeat(40),scope:'Audit staging only',acceptanceCriteria:['Schema checked']}});
|
||||
});
|
||||
});
|
||||
|
||||
describe('actual PREPARE backend contract',()=>{
|
||||
beforeEach(()=>vi.resetAllMocks());
|
||||
it('normalizes strings by fixed mapping, drops unknown and tampered objects, never infers from state',async()=>{
|
||||
vi.mocked(api.get).mockResolvedValue({data:{...task,state:'DRAFT',availableActions:['SUBMIT_GATE1','APPROVE_GATE1','REJECT_GATE1','QUEUE_PREPARE','RELEASE_AUTHORIZE',{id:'SUBMIT_GATE1',label:'bad',toState:'RELEASING'},null]}});
|
||||
const detail=await orchestration.task('t/1');
|
||||
expect(detail.availableActions.map(a=>a.id)).toEqual(['SUBMIT_GATE1','APPROVE_GATE1','REJECT_GATE1','QUEUE_PREPARE']);
|
||||
vi.mocked(api.put).mockResolvedValue({data:{}});
|
||||
await orchestration.command(detail,detail.availableActions[0],{idempotencyKey:'submit'});
|
||||
expect(api.put).toHaveBeenCalledWith('/admin/orchestration/tasks/t%2F1/transition',{toState:'GATE1_PENDING',expectedVersion:7,idempotencyKey:'submit',reason:undefined});
|
||||
vi.mocked(api.get).mockResolvedValue({data:[{...task,state:'DRAFT',availableActions:[]}]});
|
||||
expect((await orchestration.tasks())[0].availableActions).toEqual([]);
|
||||
});
|
||||
it('rejects arbitrary object transitions and conflicting known mappings',async()=>{
|
||||
vi.mocked(api.get).mockResolvedValue({data:{...task,availableActions:[{id:'evil',label:'go',toState:'RUNNING'},{id:'SUBMIT_GATE1',label:'go',toState:'READY'}]}});
|
||||
expect((await orchestration.task('t/1')).availableActions).toEqual([]);
|
||||
});
|
||||
it.each([0,19,10001,20.5])('rejects invalid budget %s',async budget=>{
|
||||
await expect(orchestration.create({key:'k',title:'t',department:'SEO',operationMode:'PREPARE',budgetLimitCents:budget,metadata:{commitSha:'a'.repeat(40),scope:'Audit staging only',acceptanceCriteria:['Schema checked']}})).rejects.toThrow(/budget/i);
|
||||
expect(api.post).not.toHaveBeenCalled();
|
||||
});
|
||||
it.each(['abc','A'.repeat(40),'a'.repeat(64),''])('rejects invalid commit %s',async commitSha=>{
|
||||
await expect(orchestration.create({key:'k',title:'t',department:'SEO',operationMode:'PREPARE',budgetLimitCents:20,metadata:{commitSha,scope:'Audit staging only',acceptanceCriteria:['Schema checked']}})).rejects.toThrow(/commit/i);
|
||||
});
|
||||
it.each([20,10000])('posts exact DTO at boundary %s',async budget=>{
|
||||
vi.mocked(api.post).mockResolvedValue({data:{}});
|
||||
const dto={key:'k',title:'t',department:'SEO',operationMode:'PREPARE' as const,budgetLimitCents:budget,metadata:{commitSha:'a'.repeat(40),scope:'Audit staging only',acceptanceCriteria:['Schema checked']}};
|
||||
await orchestration.create(dto);expect(api.post).toHaveBeenCalledWith('/admin/orchestration/tasks',dto);
|
||||
});
|
||||
});
|
||||
|
||||
it.each([
|
||||
['APPROVE_GATE1','post','/gates/1/decide',{decision:'APPROVED'}],
|
||||
['REJECT_GATE1','post','/gates/1/decide',{decision:'REJECTED'}],
|
||||
['QUEUE_PREPARE','put','/transition',{toState:'READY'}],
|
||||
])('routes actual advertised %s regardless of state',async(id,method,path,payload)=>{
|
||||
vi.resetAllMocks();vi.mocked(api.get).mockResolvedValue({data:{...task,state:'DRAFT',availableActions:[id]}});
|
||||
vi.mocked(api.post).mockResolvedValue({data:{}});vi.mocked(api.put).mockResolvedValue({data:{}});
|
||||
const detail=await orchestration.task('t/1');
|
||||
await orchestration.command(detail,detail.availableActions[0],{idempotencyKey:'k',expiresAt:new Date(Date.now()+3600000).toISOString()});
|
||||
expect(method==='post'?api.post:api.put).toHaveBeenCalledWith('/admin/orchestration/tasks/t%2F1'+path,expect.objectContaining({...payload,expectedVersion:7,idempotencyKey:'k'}));
|
||||
});
|
||||
it('rejects canonical Gate1 decisions beyond backend 24 hour limit',async()=>{
|
||||
vi.resetAllMocks();vi.mocked(api.get).mockResolvedValue({data:{...task,availableActions:['APPROVE_GATE1']}});
|
||||
const detail=await orchestration.task('t/1');
|
||||
await expect(orchestration.command(detail,detail.availableActions[0],{idempotencyKey:'k',expiresAt:'2099-01-01T00:00:00Z'})).rejects.toThrow(/24 hours/);
|
||||
expect(api.post).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
const validProposal={key:'scope-1',title:'Review staging',department:'SEO',operationMode:'PREPARE' as const,budgetLimitCents:20,metadata:{commitSha:'a'.repeat(40),scope:' Human scope ',acceptanceCriteria:[' Evidence reviewed ']}};
|
||||
it('preserves immutable human metadata exactly and strips server-owned fields from exact DTO',async()=>{
|
||||
vi.resetAllMocks();
|
||||
const projected={id:'new',...validProposal,state:'DRAFT',version:1,metadata:{...validProposal.metadata,environment:'staging'},availableActions:['SUBMIT_GATE1']};
|
||||
vi.mocked(api.post).mockResolvedValue({data:projected});
|
||||
await expect(orchestration.create({...validProposal,metadata:{...validProposal.metadata,workflowScope:'PRODUCTION',createdBy:'forged'}} as unknown as import('./orchestration').CreateTaskInput)).resolves.toEqual(projected);
|
||||
expect(api.post).toHaveBeenCalledWith('/admin/orchestration/tasks',validProposal);
|
||||
vi.mocked(api.get).mockResolvedValue({data:projected});
|
||||
expect((await orchestration.task('new')).metadata).toEqual(projected.metadata);
|
||||
vi.mocked(api.get).mockResolvedValue({data:[projected]});
|
||||
expect((await orchestration.tasks())[0].metadata).toEqual(projected.metadata);
|
||||
});
|
||||
it.each(['', ' ', 'x'.repeat(2001),undefined,42])('rejects invalid human scope %s before HTTP',async scope=>{
|
||||
vi.resetAllMocks();await expect(orchestration.create({...validProposal,metadata:{...validProposal.metadata,scope}} as unknown as import('./orchestration').CreateTaskInput)).rejects.toThrow(/scope/i);expect(api.post).not.toHaveBeenCalled();
|
||||
});
|
||||
it.each([[],Array(21).fill('x'),[''],[' '],['x'.repeat(501)],undefined,'criterion',[42]])('rejects invalid acceptance criteria %s before HTTP',async acceptanceCriteria=>{
|
||||
vi.resetAllMocks();await expect(orchestration.create({...validProposal,metadata:{...validProposal.metadata,acceptanceCriteria}} as unknown as import('./orchestration').CreateTaskInput)).rejects.toThrow(/criteria/i);expect(api.post).not.toHaveBeenCalled();
|
||||
});
|
||||
it('rejects array-coerced commit hashes',async()=>{
|
||||
vi.resetAllMocks();await expect(orchestration.create({...validProposal,metadata:{...validProposal.metadata,commitSha:['a'.repeat(40)]}} as unknown as import('./orchestration').CreateTaskInput)).rejects.toThrow(/commit/i);expect(api.post).not.toHaveBeenCalled();
|
||||
});
|
||||
it('accepts scope and criteria at exact backend limits',async()=>{
|
||||
vi.resetAllMocks();vi.mocked(api.post).mockResolvedValue({data:{}});
|
||||
const dto={...validProposal,metadata:{...validProposal.metadata,scope:'x'.repeat(2000),acceptanceCriteria:Array(20).fill('x'.repeat(500))}};
|
||||
await orchestration.create(dto);expect(api.post).toHaveBeenCalledWith('/admin/orchestration/tasks',dto);
|
||||
});
|
||||
@@ -1,26 +1,82 @@
|
||||
import api from './api';
|
||||
export interface ServerAction { id: string; label: string; toState?: string; gate?: 1 | 2; decision?: 'APPROVED' | 'REJECTED'; disabledReason?: string }
|
||||
// Only advertised IDs select commands; task state never manufactures authorization.
|
||||
const actionCommands: Record<string, ServerAction> = {
|
||||
SUBMIT_GATE1:{id:'SUBMIT_GATE1',label:'Submit Gate1',toState:'GATE1_PENDING'},
|
||||
APPROVE_GATE1:{id:'APPROVE_GATE1',label:'Approve Gate1',gate:1,decision:'APPROVED'},
|
||||
REJECT_GATE1:{id:'REJECT_GATE1',label:'Reject Gate1',gate:1,decision:'REJECTED'},
|
||||
QUEUE_PREPARE:{id:'QUEUE_PREPARE',label:'Queue PREPARE',toState:'READY'},
|
||||
};
|
||||
export function normalizeActions(value:unknown):ServerAction[] {
|
||||
if(!Array.isArray(value))return [];
|
||||
const result:ServerAction[]=[];
|
||||
for(const raw of value){
|
||||
let action:ServerAction|undefined;
|
||||
if(typeof raw==='string')action=Object.hasOwn(actionCommands,raw)?actionCommands[raw]:undefined;
|
||||
else if(raw&&typeof raw==='object'&&typeof raw.id==='string'){
|
||||
const fixed=Object.hasOwn(actionCommands,raw.id)?actionCommands[raw.id]:undefined;
|
||||
if(fixed){
|
||||
if(['toState','gate','decision'].some(k=>raw[k]!==undefined&&raw[k]!==fixed[k as keyof ServerAction]))continue;
|
||||
action={...fixed};
|
||||
}else if(!raw.toState&&[1,2].includes(raw.gate)&&['APPROVED','REJECTED'].includes(raw.decision)){
|
||||
// Legacy object format may describe only dedicated gate decisions, never arbitrary transitions.
|
||||
action={id:raw.id,label:raw.id,gate:raw.gate,decision:raw.decision};
|
||||
}
|
||||
if(action){
|
||||
if(typeof raw.label==='string'&&raw.label.trim())action={...action,label:raw.label};
|
||||
if(typeof raw.disabledReason==='string')action={...action,disabledReason:raw.disabledReason};
|
||||
}
|
||||
}
|
||||
if(action&&!result.some(a=>a.id===action.id))result.push({...action});
|
||||
}
|
||||
return result;
|
||||
}
|
||||
export interface Artifact { id: string; sha256: string; kind?: string; uri?: string }
|
||||
export interface Run { id: string; commitSha: string; environment: string; status: string; artifacts: Artifact[] }
|
||||
export interface Task { id: string; key: string; title: string; department: string; state: string; version: number; operationMode: string; budgetLimitCents: number; metadata?: Record<string, unknown>; availableActions: ServerAction[]; runs?: Run[]; decisions?: Record<string, unknown>[]; events?: Record<string, unknown>[]; budgetLedger?: Record<string, unknown>[] }
|
||||
export interface CommandInput { idempotencyKey: string; reason?: string; expiresAt?: string; runId?: string; artifactHash?: string }
|
||||
export interface TransitionDto {toState:string;expectedVersion:number;idempotencyKey:string;reason?:string}
|
||||
export interface GateDecisionDto {decision:'APPROVED'|'REJECTED';expectedVersion:number;idempotencyKey:string;expiresAt:string;notes?:string;runId?:string;artifactHash?:string;commitSha?:string;environment?:string}
|
||||
export function actionBlockReason(task:Task,action:ServerAction):string|undefined {
|
||||
if(action.disabledReason)return action.disabledReason;
|
||||
if(task.operationMode==='MUTATE_PRODUCTION'||['RELEASING','PRODUCTION_VERIFYING','LIVE_VERIFIED'].includes(action.toState??''))return 'Production actions are unavailable in this panel';
|
||||
if(task.state==='EXPIRED')return 'Expired proposal: create a fresh proposal';
|
||||
if((action.toState&&action.gate)||(!action.toState&&(![1,2].includes(action.gate??0)||!['APPROVED','REJECTED'].includes(action.decision??''))))return 'Unsupported server action';
|
||||
return undefined;
|
||||
}
|
||||
export function commandFailure(error:unknown):{message:string;retryable:boolean} {
|
||||
const status=(error as {response?:{status?:number}})?.response?.status;
|
||||
if(status===409)return {message:'Stale version or idempotency conflict. Refresh authoritative state and review a fresh decision.',retryable:false};
|
||||
if(status===410)return {message:'Proposal or approval expired. Refresh; expired proposals require a new proposal.',retryable:false};
|
||||
if(status===401||status===403)return {message:'Backend denied this action. Refresh permissions before continuing.',retryable:false};
|
||||
if(status&&status>=400&&status<500)return {message:'Backend rejected this action (HTTP '+status+'). Refresh and review the proposal.',retryable:false};
|
||||
return {message:(error instanceof Error?error.message:'Backend request unavailable')+' — Outcome unknown. Retry only the identical payload/key, or refresh to reconcile.',retryable:true};
|
||||
}
|
||||
export type Resource = 'agents' | 'model-policy' | 'schedules' | 'releases' | 'overview';
|
||||
export interface ResourceSnapshot { capabilities?: {mutations:boolean;chat:boolean;runtime:boolean}; items: Array<{id:string; title?:string; status?:string; department?:string; readiness?:string; model?:string; budgetLimitCents?:number; spentCents?:number; cron?:string; timezone?:string; environment?:string; commitSha?:string; reason?:string}>; measuredAt: string }
|
||||
export interface ResourceSnapshot { capabilities?: {mutations:boolean;chat:boolean;runtime:boolean;killSwitch?:boolean}; items: Array<{id:string; title?:string; status?:string; department?:string; readiness?:string; model?:string; budgetLimitCents?:number; spentCents?:number; cron?:string; timezone?:string; environment?:string; commitSha?:string; reason?:string}>; measuredAt: string }
|
||||
export const killLevels = ['FREEZE_NEW_WORK','STOP_AGENTS','DISABLE_MUTATIONS','EMERGENCY_HALT'] as const;
|
||||
export type KillLevel = typeof killLevels[number];
|
||||
export interface ChatMessage {id:string;role:'assistant'|'user';content:string}
|
||||
function unwrap<T>(data: T | {success:boolean;data:T}): T { if (data && typeof data === 'object' && 'success' in data) {if (!data.success) throw new Error('Orchestration request failed'); return data.data;} return data as T; }
|
||||
export interface CreateTaskInput {key:string;title:string;department:string;operationMode:'OBSERVE'|'PREPARE'|'MUTATE_PRODUCTION';budgetLimitCents:number;metadata:{scope:string;acceptanceCriteria:string[]}}
|
||||
export interface CreateTaskInput {key:string;title:string;department:string;operationMode:'PREPARE';budgetLimitCents:number;metadata:{commitSha:string;scope:string;acceptanceCriteria:string[]}}
|
||||
export function proposalError(input:CreateTaskInput):string|undefined {
|
||||
if(input.operationMode!=='PREPARE')return 'Only bounded Stage PREPARE proposals are available';
|
||||
if(!Number.isSafeInteger(input.budgetLimitCents)||input.budgetLimitCents<20||input.budgetLimitCents>10000)return 'PREPARE budget must be an integer 20..10000 cents';
|
||||
if(typeof input.metadata?.commitSha!=='string'||!/^[a-f0-9]{40}$/.test(input.metadata.commitSha))return 'Immutable commit must be 40 lowercase hex characters';
|
||||
if(typeof input.metadata?.scope!=='string'||!input.metadata.scope.trim()||input.metadata.scope.length>2000)return 'Scope must be nonblank and at most 2000 characters';
|
||||
if(!Array.isArray(input.metadata?.acceptanceCriteria)||input.metadata.acceptanceCriteria.length<1||input.metadata.acceptanceCriteria.length>20||input.metadata.acceptanceCriteria.some(value=>typeof value!=='string'||!value.trim()||value.length>500))return 'Acceptance criteria must contain 1..20 nonblank strings of at most 500 characters';
|
||||
if(!input.key.trim()||input.key.length>120||!input.title.trim()||input.title.length>255||!input.department.trim()||input.department.length>50)return 'Proposal requires key (1..120), title (1..255), department (1..50)';
|
||||
}
|
||||
export const orchestration = {
|
||||
async create(input:CreateTaskInput):Promise<unknown> {
|
||||
if(!Number.isSafeInteger(input.budgetLimitCents)||input.budgetLimitCents<0)throw new Error('Nonnegative integer budget required');
|
||||
if(!input.key.trim()||!input.title.trim()||!input.department.trim()||!input.metadata.scope.trim()||!input.metadata.acceptanceCriteria.length)throw new Error('Bounded scope and acceptance criteria required');
|
||||
return (await api.post('/admin/orchestration/tasks',{key:input.key,title:input.title,department:input.department,operationMode:input.operationMode,budgetLimitCents:input.budgetLimitCents,metadata:input.metadata})).data;
|
||||
const error=proposalError(input);if(error)throw new Error(error);
|
||||
// Preserve human evidence exactly; workflowScope/environment/identity remain server-owned.
|
||||
return (await api.post('/admin/orchestration/tasks',{key:input.key,title:input.title,department:input.department,operationMode:'PREPARE',budgetLimitCents:input.budgetLimitCents,metadata:{commitSha:input.metadata.commitSha,scope:input.metadata.scope,acceptanceCriteria:[...input.metadata.acceptanceCriteria]}})).data;
|
||||
},
|
||||
async task(id:string):Promise<Task> {
|
||||
const data=unwrap<Task>((await api.get('/admin/orchestration/tasks/'+encodeURIComponent(id))).data);
|
||||
if (!data?.id || !Number.isInteger(data.version)) throw new Error('Invalid task detail');
|
||||
return {...data,availableActions:data.availableActions??[]};
|
||||
return {...data,availableActions:normalizeActions(data.availableActions)};
|
||||
},
|
||||
async resource(name:Resource):Promise<ResourceSnapshot> {
|
||||
const data=unwrap<ResourceSnapshot>((await api.get('/admin/orchestration/'+name)).data);
|
||||
@@ -38,15 +94,18 @@ export const orchestration = {
|
||||
return data;
|
||||
},
|
||||
async command(task: Task, requested: ServerAction, input: CommandInput): Promise<unknown> {
|
||||
const action = task.availableActions.find(a => a.id === requested.id);
|
||||
const action = normalizeActions(task.availableActions).find(a => a.id === requested.id);
|
||||
if (!action || action.disabledReason) throw new Error('Action is not advertised or is disabled');
|
||||
const blocked=actionBlockReason(task,action);if(blocked)throw new Error(blocked);
|
||||
if (!input.idempotencyKey || !Number.isInteger(task.version)) throw new Error('Version and idempotency key required');
|
||||
const base = '/admin/orchestration/tasks/' + encodeURIComponent(task.id);
|
||||
if (action.gate && action.decision) {
|
||||
if (!input.expiresAt || !(Date.parse(input.expiresAt) > Date.now())) throw new Error('Future expiry required');
|
||||
if(['APPROVE_GATE1','REJECT_GATE1'].includes(action.id)&&Date.parse(input.expiresAt)>Date.now()+86400000)throw new Error('Gate1 expiry must be within 24 hours');
|
||||
let binding = {};
|
||||
if (action.gate === 2 && action.decision === 'APPROVED') {
|
||||
const run = task.runs?.find(r => r.id === input.runId);
|
||||
if(run?.status!=='SUCCEEDED')throw new Error('Registered successful run binding required for Gate2 evidence');
|
||||
if (!run?.commitSha || !run.environment || !run.artifacts.some(a => a.sha256 === input.artifactHash)) throw new Error('Registered immutable Gate2 binding required');
|
||||
binding = {runId:run.id, artifactHash:input.artifactHash, commitSha:run.commitSha, environment:run.environment};
|
||||
}
|
||||
@@ -59,6 +118,6 @@ export const orchestration = {
|
||||
const { data } = await api.get('/admin/orchestration/tasks');
|
||||
const rows = Array.isArray(data) ? data : data.data;
|
||||
if (!Array.isArray(rows)) throw new Error('Invalid orchestration task response');
|
||||
return rows.map(row => ({ ...row, availableActions: row.availableActions ?? [] }));
|
||||
return rows.map(row => ({ ...row, availableActions: normalizeActions(row.availableActions) }));
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,21 @@
|
||||
import {it,expect,vi} from 'vitest';
|
||||
import api from './api';
|
||||
import {runtime} from './runtime';
|
||||
vi.mock('./api',()=>({default:{get:vi.fn()}}));
|
||||
const provenance={source:'snapshot',capturedAt:'2020-01-01T00:00:00Z',ageSeconds:100,freshness:'stale',liveProcess:'unknown'};
|
||||
it.each([{source:'live'},{capturedAt:'not-a-date'},{capturedAt:'2999-01-01T00:00:00Z'},{ageSeconds:-1},{ageSeconds:0.5},{freshness:'live'},{liveProcess:'running'}])('rejects invalid provenance %j',async invalid=>{
|
||||
vi.mocked(api.get).mockResolvedValue({data:{ok:true,version:'1',commit:'x',auth_required:true,provenance:{...provenance,...invalid}}});
|
||||
await expect(runtime.health()).rejects.toThrow('Invalid snapshot provenance');
|
||||
});
|
||||
it('retains fresh snapshots as inventory with unknown process state',async()=>{
|
||||
vi.mocked(api.get).mockResolvedValue({data:{ok:true,version:'1',commit:'x',auth_required:true,provenance:{...provenance,freshness:'fresh'}}});
|
||||
expect(await runtime.health()).toMatchObject({provenance:{freshness:'fresh',liveProcess:'unknown'}});
|
||||
});
|
||||
it('retains validated snapshot provenance and unknown live telemetry across inventory reads',async()=>{
|
||||
vi.mocked(api.get).mockImplementation(async url=>({data:url.endsWith('/health')?{ok:true,version:'1',commit:'x',auth_required:true,provenance}:url.endsWith('/profiles')?[{name:'dev',is_default:false,gateway_running:false,liveGatewayState:'unknown',provenance}]:url.endsWith('/status')?{version:'1',overall:'unknown',gateway_running:false,active_agents:0,gateway_state:null,liveGatewayState:'unknown',liveActiveAgents:null,provenance}:url.endsWith('/sessions')?{total:0,sessions:[],provenance}:{period_days:30,totals:{total_input:0,total_output:0,total_sessions:0,total_estimated_cost:0},provenance}}));
|
||||
expect(await runtime.health()).toMatchObject({provenance});
|
||||
expect(await runtime.profiles()).toMatchObject([{provenance,liveGatewayState:'unknown'}]);
|
||||
expect(await runtime.status('dev')).toMatchObject({provenance,liveGatewayState:'unknown',liveActiveAgents:null});
|
||||
expect(await runtime.sessions('dev')).toMatchObject({provenance});
|
||||
expect(await runtime.usage('dev')).toMatchObject({provenance});
|
||||
});
|
||||
@@ -0,0 +1,28 @@
|
||||
import api from './api';
|
||||
export interface RuntimeProvenance {source:'snapshot';capturedAt:string;ageSeconds:number;freshness:'fresh'|'stale';liveProcess:'unknown'}
|
||||
interface Inventory {provenance?:RuntimeProvenance}
|
||||
export interface RuntimeHealth extends Inventory {ok:boolean;version:string;commit:string;auth_required:true}
|
||||
export interface RuntimeProfile extends Inventory {name:string;is_default:boolean;gateway_running:boolean;liveGatewayState?:'unknown'}
|
||||
export interface RuntimeStatus extends Inventory {version:string;gateway_running:boolean;overall:string;active_agents:number;gateway_state:string|null;liveGatewayState?:'unknown';liveActiveAgents?:null}
|
||||
export interface RuntimeSession {id:string;profile:string;source:string;is_active:boolean}
|
||||
export interface RuntimeSessions extends Inventory {total:number;sessions:RuntimeSession[]}
|
||||
export interface RuntimeUsage extends Inventory {period_days:number;totals:{total_input:number;total_output:number;total_sessions:number;total_estimated_cost:number}}
|
||||
function object(value:unknown):Record<string,unknown>{if(!value||typeof value!=='object'||Array.isArray(value))throw new Error('Invalid runtime contract');return value as Record<string,unknown>}
|
||||
function text(value:unknown):string{if(typeof value!=='string')throw new Error('Invalid runtime text');return value}
|
||||
function bool(value:unknown):boolean{if(typeof value!=='boolean')throw new Error('Invalid runtime flag');return value}
|
||||
function count(value:unknown):number{if(typeof value!=='number'||!Number.isFinite(value)||value<0)throw new Error('Invalid runtime count');return value}
|
||||
function list(value:unknown):unknown[]{if(!Array.isArray(value)||value.length>1000)throw new Error('Invalid runtime list');return value}
|
||||
async function read(path:string,params?:Record<string,string|number>):Promise<unknown>{const {data}=await api.get('/admin/orchestration/runtime/'+path,...(params?[{params}]:[]));const row=data&&typeof data==='object'&&!Array.isArray(data)?data:null;if(row&&'success' in row){if(row.success!==true)throw new Error('Runtime unavailable');return row.data}return data}
|
||||
function provenance(r:Record<string,unknown>):Inventory {
|
||||
if(r.provenance===undefined)return {};
|
||||
const p=object(r.provenance);
|
||||
if(p.source!=='snapshot'||p.liveProcess!=='unknown'||typeof p.capturedAt!=='string'||!p.capturedAt.endsWith('Z')||!Number.isFinite(Date.parse(p.capturedAt))||Date.parse(p.capturedAt)>Date.now()||typeof p.ageSeconds!=='number'||!Number.isInteger(p.ageSeconds)||p.ageSeconds<0||!['fresh','stale'].includes(String(p.freshness)))throw new Error('Invalid snapshot provenance');
|
||||
return {provenance:{source:'snapshot',capturedAt:p.capturedAt,ageSeconds:p.ageSeconds,freshness:p.freshness as 'fresh'|'stale',liveProcess:'unknown'}};
|
||||
}
|
||||
export const runtime={
|
||||
async health():Promise<RuntimeHealth>{const r=object(await read('health'));if(r.auth_required!==true)throw new Error('Authenticated runtime required');return {...provenance(r),ok:bool(r.ok),version:text(r.version),commit:text(r.commit),auth_required:true}},
|
||||
async profiles():Promise<RuntimeProfile[]>{return list(await read('profiles')).map(value=>{const r=object(value);return {...provenance(r),...(r.provenance?{liveGatewayState:'unknown' as const}:{}),name:text(r.name),is_default:bool(r.is_default),gateway_running:bool(r.gateway_running)}})},
|
||||
async status(profile:string):Promise<RuntimeStatus>{const r=object(await read('status',{profile}));return {...provenance(r),...(r.provenance?{liveGatewayState:'unknown' as const,liveActiveAgents:null}:{}),version:text(r.version),gateway_running:bool(r.gateway_running),overall:text(r.overall),active_agents:count(r.active_agents),gateway_state:r.gateway_state===null?null:text(r.gateway_state)}},
|
||||
async sessions(profile:string,offset=0):Promise<RuntimeSessions>{const r=object(await read('sessions',{profile,limit:20,offset}));return {...provenance(r),total:count(r.total),sessions:list(r.sessions).map(value=>{const s=object(value);if(s.profile!==profile)throw new Error('Runtime profile mismatch');return {id:text(s.id),profile:text(s.profile),source:text(s.source),is_active:bool(s.is_active)}})}},
|
||||
async usage(profile:string):Promise<RuntimeUsage>{const r=object(await read('usage',{profile,days:30}));const t=object(r.totals);return {...provenance(r),period_days:count(r.period_days),totals:{total_input:count(t.total_input),total_output:count(t.total_output),total_sessions:count(t.total_sessions),total_estimated_cost:count(t.total_estimated_cost)}}},
|
||||
};
|
||||
@@ -0,0 +1,78 @@
|
||||
# Real bounded Tech/Dev Hermes pilot (experimental, NOT enabled)
|
||||
|
||||
Owned files only in `infra/hermes-pilot`; no existing core, module, tracer, profile or runtime config changed. No push/deploy/RELEASE_AUTHORIZE. Do not wire this into the existing tracer completion endpoint: that endpoint deliberately accepts only deterministic JSON.
|
||||
|
||||
## Fixed task and boundaries
|
||||
|
||||
Task `tech-dev-clamp-quantity-v1`: produce one <=2048-byte Python `clamp_quantity(quantity)` implementation, integer range 0..99. No input/browser prompt is accepted. The model has **zero tools**, no shell/browser/Git/deploy credentials. Trusted code alone writes `quantity.py`, runs separate AST-allowlisted QA (9 boundary cases), and commits an isolated, remote-free fixture repository. This is real generated-code QA, not application QA. Source commit is recorded as provenance, not represented as the fixture's base commit.
|
||||
|
||||
`infer.py` uses installed `run_agent.AIAgent`, verified from the installed signature and `agent/agent_init.py`; `enabled_toolsets=[]` is additionally enforced by asserting empty `agent.tools`. Existing approved `OPENAI_API_KEY` is read in memory only; no messaging/API-server credentials/auth pools copied. Separate temporary HERMES_HOME, no memories/context/background review/fallback. Provider/model pinned to current canina-hermes `custom`, `https://nr.parsaaghayi.ir/v1`, `Free`; no provider installation or account changes. Existing configs remain untouched. Model output is untrusted and never directly shell-executed; QA rejects all syntax except the single nested min/max return expression before isolated Python execution.
|
||||
|
||||
Installed image identity: `sha256:373ddd5e6649f525b17993e1c885cdf1352127f8e98fd329140a73237125e52f`, install stamp commit `818c13be1dc4fd28987e1e881a9408224afd4535`, v0.21.6. Discovery: `/opt/hermes/hermes chat --help` supports `--oneshot`, `--max-turns`, `--run-budget`; `/opt/hermes/.venv/bin/python` loads installed API. Official skills and https://hermes-agent.nousresearch.com/docs/llms.txt consulted. The initially guessed developer API docs URL returned404; installed source/signature is authoritative for this pinned build.
|
||||
|
||||
## PREPARE mapping — parent integration required
|
||||
|
||||
`executor.execute(lease, artifact_root, infer, heartbeat)` only accepts a fresh staging, first-attempt, 20-cent, 40hex-source lease and exact lease fields; heartbeat failure aborts before inference and before QA/commit. Trusted adapter must obtain this lease via existing authenticated claim AFTER independent Gate1/READY, verify the explicitly allowlisted task identity/scope/criteria server-side, and supply heartbeat connected to the authenticated backend. A caller-created dict is NOT an authorization. Never expose this Python interface to browser requests.
|
||||
|
||||
## Heartbeat runner and parent API (this revision)
|
||||
|
||||
After independent READY/Gate1, allowlist and provider-cap admission, parent constructs
|
||||
`runner = executor.pinned_inference(executor.IMAGE)` and calls
|
||||
`executor.execute(authenticated_lease, artifact_root, runner, authenticated_heartbeat)`.
|
||||
Only trusted Python may construct runners. Callback receives an unchanged COPY of the
|
||||
original lease binding (taskId/runId/leaseToken), must use authenticated server transport,
|
||||
and must raise on any non-2xx, ownership/fencing/expiry denial or malformed response.
|
||||
Explicit `False` also denies; `None` means success for existing raise-on-error adapters.
|
||||
Callback MUST impose <=5s network timeout, no retries; a blocked callback thread cannot
|
||||
be forcibly killed by Python. It is daemonized; its result cannot restart inference or
|
||||
extend a local lease. Parent must serialize executions per server lease; the runner is
|
||||
single-use and concurrent reuse of that runner is locked out. This is not a browser API.
|
||||
|
||||
Immediate initial heartbeat; dedicated ongoing thread waits 5s between acknowledged
|
||||
calls. With required <=5s transport ceiling, heartbeat starts are <=10s apart. A stalled
|
||||
heartbeat (10s without acknowledgment), exception or denial aborts actual subprocess
|
||||
inference, not just its caller. No leaseUntil mutation/local extension occurs. Fresh
|
||||
server-authorized heartbeats retain fencing; this code cannot create authorization.
|
||||
Guard checks also fence QA/artifact commit/report return. A denial during QA prevents
|
||||
report return; any partial local fixture is not accepted completion evidence.
|
||||
|
||||
Monotonic wall ceiling 90s with cancellation reserve: work aborts at85s, remote kill
|
||||
command <=2s and local reap <=1s; bounded QA3s/Git1s. Pinned runner uses one model
|
||||
call, max_iterations1/max_tokens160, run_budget80s, tools0, no retries. An independent
|
||||
in-container timeout85s survives loss of the Docker client. Cancellation addresses
|
||||
the actual in-container process session via a random PID file plus cancellation
|
||||
marker (handles cancellation before PID publication); it kills that process group.
|
||||
Cancellation transport failure raises and prevents success, but cannot prove immediate
|
||||
remote death; independent inner timeout is the fallback. Small random marker files
|
||||
in container /tmp are intentional fencing tombstones, not configuration changes.
|
||||
|
||||
`rehearse.py` remains synthetic, never a live lease. It uses this cancellable runner
|
||||
but is NOT admission evidence. Do not run it until provider-cap approval. Existing
|
||||
20-cent reservation remains UNKNOWN, never zero, and no completion/claim/release or
|
||||
backend transport is implemented here. Existing tracer endpoint still rejects model
|
||||
reports; parent must separately implement authenticated allowlisted artifact ingestion.
|
||||
No changes to installed Hermes or runtime/profile configuration.
|
||||
|
||||
## Actual evidence and blockers
|
||||
|
||||
Evidence directory `/opt/services/canina/validation/v4-real-pilot`:
|
||||
- `red.log`: missing executor assertion failed before implementation.
|
||||
- `inference-private.log`: actual installed Hermes/provider inference exit0 under75s diagnostic ceiling; generated source saved, not fabricated. This ceiling was ONLY diagnostic, not accepted as a PREPARE lease run.
|
||||
- `model-artifact/quantity.py`: actual model artifact; independent QA passes9 cases, SHA256 `0125246b551896c5f9ee98fabeb02982a6690a9974c0eb24ef3ae676d46380d5`; isolated Git commit `95cb3d27d636ae817d6a778e8053e2f87b50e57b`.
|
||||
- `actual.log`: bounded full rehearsal failed with inner timeout exit137 at18s; no completion/QA success was assumed. Previous client-only timeout was corrected to inner process kill. **Installed model/startup cannot currently complete reliably inside30s PREPARE lease without authenticated ongoing heartbeat.** Do not extend lease locally or call the75s diagnostic a bounded lease success.
|
||||
|
||||
Contract tests: `python3 infra/hermes-pilot/test_contract.py`. Parent still owns backend integration, authenticated heartbeat orchestration, allowlist admission, billing caps, signed artifact transport and Stage CI. This commit is a real generation/independent-QA proof and fail-closed executor seam, not a live autonomous Tech department or released product.
|
||||
|
||||
## Verification of heartbeat revision
|
||||
|
||||
`python3 -m unittest discover -s infra/hermes-pilot -v`: 11 passing tests, including
|
||||
real OS child/descendant cancellation, ongoing rejection, stalled heartbeat, wall
|
||||
ceiling, concurrent runner reuse, immutable lease and unknown-cost preservation.
|
||||
The artifact success test is explicitly a deterministic process fixture, NOT model
|
||||
inference. TDD red observed for missing ongoing runner and explicit false denial;
|
||||
additional cancellation/concurrency regression cases were added and executed.
|
||||
Previously generated real model artifact was rechecked by current AST allowlist QA:
|
||||
9 cases pass, SHA256 `0125246b551896c5f9ee98fabeb02982a6690a9974c0eb24ef3ae676d46380d5`.
|
||||
No new real inference was run: README provider pricing/cap gate remains unresolved.
|
||||
Docker cancellation path is implemented but not runtime-exercised here; OS process
|
||||
group cancellation is exercised. No live backend/claim/release, push or deploy.
|
||||
@@ -0,0 +1,64 @@
|
||||
"""Fixed Tech/Dev pilot. Adapter must supply a fresh authenticated PREPARE lease."""
|
||||
import datetime, hashlib, json, os, pathlib, re, subprocess, tempfile, time
|
||||
TASK='tech-dev-clamp-quantity-v1'
|
||||
IMAGE='sha256:373ddd5e6649f525b17993e1c885cdf1352127f8e98fd329140a73237125e52f'
|
||||
|
||||
def validate_lease(lease):
|
||||
if (set(lease) != {'taskId','runId','leaseToken','leaseUntil','commitSha','environment','maxCostCents','attempt'}
|
||||
or lease['environment']!='staging' or lease['maxCostCents']!=20 or lease['attempt']!=1
|
||||
or not re.fullmatch('[a-f0-9]{40}',lease['commitSha'])):
|
||||
raise ValueError('Bounded PREPARE lease required')
|
||||
for k in ('taskId','runId','leaseToken'):
|
||||
if not re.fullmatch('[a-f0-9-]{36}',lease[k]): raise ValueError('Invalid lease binding')
|
||||
expiry=datetime.datetime.fromisoformat(lease['leaseUntil'].replace('Z','+00:00')).timestamp()
|
||||
if not 5 < expiry-time.time() <= 31: raise ValueError('Fresh 30-second lease required')
|
||||
return lease
|
||||
|
||||
def _execute(lease, artifact_root, infer, heartbeat):
|
||||
"""No claim/approval generation. heartbeat must call trusted server; errors abort.
|
||||
infer is the pinned runner, not a browser-supplied command or prompt.
|
||||
Completion is deliberately NOT sent: existing transport accepts tracer JSON only.
|
||||
"""
|
||||
validate_lease(lease)
|
||||
heartbeat(lease)
|
||||
root=pathlib.Path(tempfile.mkdtemp(prefix='real-pilot-',dir=artifact_root))
|
||||
def git(*args):
|
||||
return subprocess.check_output(['git','-C',str(root),*args],stderr=subprocess.DEVNULL,text=True,timeout=1).strip()
|
||||
git('init','-q'); git('config','user.email','pilot@invalid'); git('config','user.name','Bounded Hermes Pilot')
|
||||
(root/'README').write_text('Disposable fixture; PREPARE source '+lease['commitSha']+'\n')
|
||||
git('add','README'); git('commit','-qm','Seed isolated pilot fixture')
|
||||
base=git('rev-parse','HEAD')
|
||||
# One cancellable inference, no tools; ongoing authenticated guard.
|
||||
source=infer()
|
||||
heartbeat(lease)
|
||||
if not isinstance(source,str) or len(source.encode())>2048: raise ValueError('Invalid generated artifact')
|
||||
(root/'quantity.py').write_text(source)
|
||||
qa=subprocess.check_output(['python3','-I',str(pathlib.Path(__file__).with_name('qa.py')),str(root)],timeout=3,text=True)
|
||||
heartbeat(lease)
|
||||
git('add','quantity.py'); git('commit','-qm','Hermes generated quantity clamp; independent QA passed')
|
||||
patch=git('diff',base,'HEAD','--','quantity.py')+'\n'
|
||||
(root/'artifact.patch').write_text(patch)
|
||||
report={'kind':'REAL_HERMES_TECH_DEV_PILOT','task':TASK,'taskId':lease['taskId'],'runId':lease['runId'],
|
||||
'sourceCommit':lease['commitSha'],'artifactCommit':git('rev-parse','HEAD'),'baseCommit':base,
|
||||
'patchSha256':hashlib.sha256(patch.encode()).hexdigest(),'qa':json.loads(qa),
|
||||
'costCents':None,'costStatus':'UNKNOWN_RETAIN_FULL_20_CENT_RESERVATION',
|
||||
'gate1':'EXTERNAL_LEASE_REQUIRED','published':False,'deploymentExecuted':False}
|
||||
(root/'evidence.json').write_text(json.dumps(report,indent=2)+'\n')
|
||||
return root,report
|
||||
|
||||
from runtime import LeaseGuard, ProcessInference, pinned_inference
|
||||
|
||||
def execute(lease, artifact_root, infer, heartbeat):
|
||||
validate_lease(lease)
|
||||
with LeaseGuard(lease, heartbeat) as guard:
|
||||
guard.check()
|
||||
if not isinstance(infer, ProcessInference):
|
||||
raise TypeError("Cancellable ProcessInference required")
|
||||
def generate():
|
||||
response = json.loads(infer.run(guard))
|
||||
if not isinstance(response,dict) or response.get("tools") != 0 or response.get("model") != "Free":
|
||||
raise ValueError("Pinned tool-less model required")
|
||||
return response["source"]
|
||||
result = _execute(lease, artifact_root, generate, lambda _: guard.check())
|
||||
guard.check()
|
||||
return result
|
||||
@@ -0,0 +1,22 @@
|
||||
"""Executed only inside pinned canina-hermes image by controlled runner."""
|
||||
import contextlib, io, json, os, pathlib, tempfile
|
||||
# Copy ONE existing inference credential in memory; no auth pools or messaging keys.
|
||||
from dotenv import dotenv_values
|
||||
key=dotenv_values('/opt/data/.env').get('OPENAI_API_KEY')
|
||||
if not key: raise RuntimeError('Existing OPENAI_API_KEY unavailable')
|
||||
home=tempfile.TemporaryDirectory(prefix='canina-real-pilot-')
|
||||
os.environ.clear()
|
||||
os.environ.update({'PATH':'/usr/bin:/bin','HOME':home.name,'HERMES_HOME':home.name,'OPENAI_API_KEY':key,'OPENAI_BASE_URL':'https://nr.parsaaghayi.ir/v1','HERMES_DISABLE_PLUGINS':'1'})
|
||||
# Installed API verified by inspect.signature at pinned image commit 818c13be.
|
||||
with contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(io.StringIO()):
|
||||
from run_agent import AIAgent
|
||||
agent=AIAgent(model='Free',provider='custom',base_url='https://nr.parsaaghayi.ir/v1',api_key=key,
|
||||
enabled_toolsets=[],max_iterations=1,max_tokens=160,run_budget_seconds=80,
|
||||
skip_context_files=True,skip_memory=True,skip_background_review=True,load_soul_identity=False,
|
||||
quiet_mode=True,fallback_model={},cwd=home.name)
|
||||
if agent.tools: raise RuntimeError('Tool-less invariant failed')
|
||||
result=agent.run_conversation('Write only Python source, no markdown: define clamp_quantity(quantity) returning integer quantity clamped to inclusive 0..99. Use one return with nested min/max calls only. No annotations, comments, imports, decorators, docstrings or other statements.')
|
||||
final=result.get('final_response')
|
||||
if not isinstance(final,str) or not final.strip(): raise RuntimeError('No model artifact; result keys='+','.join(result.keys()))
|
||||
print(json.dumps({'source':final.strip()+'\n','model':'Free','tools':0}))
|
||||
home.cleanup()
|
||||
@@ -0,0 +1,26 @@
|
||||
"""Independent structural + behavior QA. Never executes arbitrary model Python."""
|
||||
import ast, hashlib, json, pathlib, sys
|
||||
|
||||
def verify(root):
|
||||
p=pathlib.Path(root)/'quantity.py'
|
||||
if p.is_symlink() or not p.is_file() or p.stat().st_size>2048: raise ValueError('Invalid file')
|
||||
raw=p.read_bytes(); tree=ast.parse(raw)
|
||||
if len(tree.body)!=1 or not isinstance(tree.body[0],ast.FunctionDef): raise ValueError('Single function required')
|
||||
fn=tree.body[0]
|
||||
if fn.name!='clamp_quantity' or fn.decorator_list or fn.returns or fn.args.defaults or fn.args.kw_defaults or fn.args.kwonlyargs or fn.args.posonlyargs or fn.args.vararg or fn.args.kwarg or len(fn.args.args)!=1 or fn.args.args[0].arg!='quantity' or fn.args.args[0].annotation or len(fn.body)!=1 or not isinstance(fn.body[0],ast.Return): raise ValueError('Unexpected function structure')
|
||||
def expr(n):
|
||||
if isinstance(n,ast.Constant) and type(n.value) is int and 0<=n.value<=99: return
|
||||
if isinstance(n,ast.Name) and n.id=='quantity': return
|
||||
if isinstance(n,ast.Call) and isinstance(n.func,ast.Name) and n.func.id in ('min','max') and len(n.args)==2 and not n.keywords:
|
||||
for a in n.args: expr(a)
|
||||
return
|
||||
raise ValueError('Nonallowlisted syntax')
|
||||
expr(fn.body[0].value)
|
||||
scope={'__builtins__':{},'min':min,'max':max}
|
||||
exec(compile(tree,'generated-quantity','exec'),scope)
|
||||
cases=[-1000,-1,0,1,50,98,99,100,10**30]
|
||||
for q in cases:
|
||||
actual=scope['clamp_quantity'](q)
|
||||
if type(actual) is not int or actual!=max(0,min(q,99)): raise ValueError('Behavior QA failed')
|
||||
return {'passed':True,'cases':len(cases),'sha256':hashlib.sha256(raw).hexdigest(),'scope':'quantity clamp fixture only'}
|
||||
if __name__=='__main__': print(json.dumps(verify(sys.argv[1])))
|
||||
@@ -0,0 +1,18 @@
|
||||
"""Disposable model/artifact exercise, NOT a live Gate1 approval or service completion."""
|
||||
import datetime, json, pathlib, subprocess, time, uuid
|
||||
from executor import execute, IMAGE, pinned_inference
|
||||
|
||||
def main():
|
||||
image=subprocess.check_output(['docker','inspect','--format','{{.Image}}','canina-hermes'],text=True).strip()
|
||||
if image!=IMAGE: raise RuntimeError('Pinned installed image mismatch')
|
||||
infer=pinned_inference(IMAGE)
|
||||
deadline=time.monotonic()+90
|
||||
lease={k:str(uuid.uuid4()) for k in ('taskId','runId','leaseToken')}
|
||||
lease.update(leaseUntil=(datetime.datetime.now(datetime.timezone.utc)+datetime.timedelta(seconds=30)).isoformat(),commitSha=subprocess.check_output(['git','rev-parse','HEAD'],text=True).strip(),environment='staging',maxCostCents=20,attempt=1)
|
||||
def fixture_heartbeat(binding):
|
||||
if time.monotonic()>deadline: raise TimeoutError('Fixture hard deadline')
|
||||
root,report=execute(lease,'/opt/services/canina/validation/v4-real-pilot',infer,fixture_heartbeat)
|
||||
report['executionContext']='DISPOSABLE_FIXTURE_NOT_LIVE_PREPARE_LEASE'
|
||||
(root/'evidence.json').write_text(json.dumps(report,indent=2)+'\n')
|
||||
print(json.dumps({'artifactDirectory':str(root),'report':report}))
|
||||
if __name__=='__main__': main()
|
||||
@@ -0,0 +1,127 @@
|
||||
"""Trusted subprocess boundary; no claim, lease extension or completion transport."""
|
||||
import json
|
||||
import os
|
||||
import pathlib
|
||||
import signal
|
||||
import subprocess
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
|
||||
HEARTBEAT_INTERVAL = 5.0
|
||||
HEARTBEAT_TIMEOUT = 5.0
|
||||
MAX_WALL_SECONDS = 90.0
|
||||
|
||||
class LeaseGuard:
|
||||
def __init__(self, lease, heartbeat):
|
||||
self.lease, self.heartbeat = lease, heartbeat
|
||||
self.stop = threading.Event()
|
||||
self.ready = threading.Event()
|
||||
self.error = None
|
||||
self.started = time.monotonic()
|
||||
self.last = self.started
|
||||
self.thread = threading.Thread(target=self._run, daemon=True)
|
||||
|
||||
def _run(self):
|
||||
while not self.stop.is_set():
|
||||
try:
|
||||
if self.heartbeat(dict(self.lease)) is False:
|
||||
raise PermissionError("Authenticated heartbeat denied")
|
||||
self.last = time.monotonic()
|
||||
self.ready.set()
|
||||
except BaseException as exc:
|
||||
self.error = exc
|
||||
self.ready.set()
|
||||
return
|
||||
if self.stop.wait(HEARTBEAT_INTERVAL):
|
||||
return
|
||||
|
||||
def check(self):
|
||||
if self.error is not None:
|
||||
raise self.error
|
||||
now = time.monotonic()
|
||||
if now - self.started >= MAX_WALL_SECONDS - 5:
|
||||
raise TimeoutError('Pilot wall deadline (cancellation reserve included)')
|
||||
if now - self.last >= HEARTBEAT_INTERVAL + HEARTBEAT_TIMEOUT:
|
||||
raise TimeoutError('Authenticated heartbeat stalled')
|
||||
|
||||
def __enter__(self):
|
||||
self.thread.start()
|
||||
if not self.ready.wait(HEARTBEAT_TIMEOUT):
|
||||
self.stop.set()
|
||||
raise TimeoutError('Initial authenticated heartbeat stalled')
|
||||
try:
|
||||
self.check()
|
||||
except BaseException:
|
||||
self.stop.set()
|
||||
raise
|
||||
return self
|
||||
|
||||
def __exit__(self, *args):
|
||||
self.stop.set()
|
||||
self.thread.join(timeout=.05)
|
||||
|
||||
class ProcessInference:
|
||||
"""Trusted argv only. cancel_remote MUST kill the in-container process group."""
|
||||
def __init__(self, argv, input_text=None, cancel_remote=None):
|
||||
self.argv, self.input_text, self.cancel_remote = argv, input_text, cancel_remote
|
||||
self.used = False
|
||||
self.lock = threading.Lock()
|
||||
|
||||
def run(self, guard):
|
||||
with self.lock:
|
||||
if self.used:
|
||||
raise RuntimeError('Exactly one inference allowed; no retry')
|
||||
self.used = True
|
||||
guard.check()
|
||||
# Files avoid pipe deadlocks and unbounded captured output in memory.
|
||||
with tempfile.TemporaryFile(mode='w+') as incoming, tempfile.TemporaryFile(mode='w+') as out, tempfile.TemporaryFile(mode='w+') as err:
|
||||
incoming.write(self.input_text or '')
|
||||
incoming.seek(0)
|
||||
child = subprocess.Popen(self.argv, stdin=incoming, stdout=out, stderr=err, start_new_session=True)
|
||||
try:
|
||||
while child.poll() is None:
|
||||
guard.check()
|
||||
time.sleep(.02)
|
||||
guard.check()
|
||||
if child.returncode:
|
||||
raise RuntimeError('Inference failed (exit %d)' % child.returncode)
|
||||
out.seek(0)
|
||||
result = out.read(4097)
|
||||
if len(result) > 4096:
|
||||
raise ValueError('Oversized inference response')
|
||||
return result
|
||||
finally:
|
||||
# Always cancel remote, even when docker exec itself failed.
|
||||
try:
|
||||
if self.cancel_remote:
|
||||
self.cancel_remote()
|
||||
finally:
|
||||
try:
|
||||
os.killpg(child.pid, signal.SIGKILL)
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
child.wait(timeout=1)
|
||||
|
||||
|
||||
def pinned_inference(image):
|
||||
actual = subprocess.check_output(['docker','inspect','--format','{{.Image}}','canina-hermes'], text=True, timeout=2).strip()
|
||||
if actual != image:
|
||||
raise RuntimeError('Pinned installed image mismatch')
|
||||
token = uuid.uuid4().hex
|
||||
pidfile = '/tmp/canina-pilot-' + token + '.pid'
|
||||
script = pathlib.Path(__file__).with_name('infer.py').read_text()
|
||||
# Session leader is created INSIDE the container. Inner kill is independent of client.
|
||||
wrapper = ('import os,signal,subprocess,sys; '
|
||||
'p=subprocess.Popen(["timeout","--signal=KILL","85","/opt/hermes/.venv/bin/python","-"],stdin=sys.stdin,start_new_session=True); '
|
||||
'open(%r,"w").write(str(p.pid)); '
|
||||
'os.path.exists(%r) and os.killpg(p.pid,signal.SIGKILL); sys.exit(p.wait())') % (pidfile, pidfile+'.cancel')
|
||||
cancel = ('import os,signal; p=%r; '
|
||||
'\nopen(p+".cancel","w").close()\ntry:\n pid=int(open(p).read()); os.killpg(pid,signal.SIGKILL)\n'
|
||||
'except (FileNotFoundError,ProcessLookupError): pass\n'
|
||||
'finally:\n'
|
||||
' try: os.unlink(p)\n except FileNotFoundError: pass\n') % pidfile
|
||||
def cancel_remote():
|
||||
subprocess.run(['docker','exec','canina-hermes','/opt/hermes/.venv/bin/python','-c',cancel], check=True, timeout=2, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
||||
return ProcessInference(['docker','exec','-i','canina-hermes','/opt/hermes/.venv/bin/python','-c',wrapper], script, cancel_remote)
|
||||
@@ -0,0 +1,68 @@
|
||||
import json, pathlib, sys, tempfile, threading, time, unittest
|
||||
from unittest.mock import patch
|
||||
sys.path.insert(0,str(pathlib.Path(__file__).parent))
|
||||
import executor as m
|
||||
import runtime
|
||||
from test_contract import lease
|
||||
|
||||
class Cancellation(unittest.TestCase):
|
||||
def test_wall_deadline_kills_descendant(self):
|
||||
with tempfile.TemporaryDirectory() as root:
|
||||
marker=pathlib.Path(root,'late')
|
||||
child=f'import pathlib,time; time.sleep(.7); pathlib.Path({str(marker)!r}).touch()'
|
||||
code=f'import subprocess,sys,time; subprocess.Popen([sys.executable,"-c",{child!r}]); time.sleep(5)'
|
||||
with patch.object(runtime,'MAX_WALL_SECONDS',5.2):
|
||||
with self.assertRaises(TimeoutError): m.execute(lease(),root,m.ProcessInference([sys.executable,'-c',code]),lambda _: None)
|
||||
time.sleep(.8)
|
||||
self.assertFalse(marker.exists())
|
||||
|
||||
def test_stalled_heartbeat_cancels_child(self):
|
||||
with tempfile.TemporaryDirectory() as root:
|
||||
marker=pathlib.Path(root,'late'); release=threading.Event(); calls=[]
|
||||
def heartbeat(_):
|
||||
calls.append(time.monotonic())
|
||||
if len(calls)>1: release.wait(3)
|
||||
code=f'import time,pathlib; time.sleep(.7); pathlib.Path({str(marker)!r}).touch()'
|
||||
try:
|
||||
with patch.object(runtime,'HEARTBEAT_INTERVAL',.05),patch.object(runtime,'HEARTBEAT_TIMEOUT',.1):
|
||||
with self.assertRaises(TimeoutError): m.execute(lease(),root,m.ProcessInference([sys.executable,'-c',code]),heartbeat)
|
||||
finally: release.set()
|
||||
time.sleep(.8); self.assertFalse(marker.exists())
|
||||
|
||||
def test_fixture_artifact_qa_and_cost_reservation(self):
|
||||
# Explicit deterministic PROCESS fixture, never reported as real model inference.
|
||||
source='def clamp_quantity(quantity):\n return min(99, max(0, quantity))\n'
|
||||
response=json.dumps(dict(source=source,model='Free',tools=0))
|
||||
beats=[]; original=lease(); snapshot=dict(original)
|
||||
def heartbeat(binding):
|
||||
beats.append(time.monotonic()); binding['leaseUntil']='untrusted mutation'
|
||||
with tempfile.TemporaryDirectory() as root:
|
||||
infer=m.ProcessInference([sys.executable,'-c',f'import time; time.sleep(.25); print({response!r})'])
|
||||
with patch.object(runtime,'HEARTBEAT_INTERVAL',.05):
|
||||
_,report=m.execute(original,root,infer,heartbeat)
|
||||
self.assertEqual(original,snapshot)
|
||||
self.assertGreater(len(beats),2)
|
||||
self.assertIsNone(report['costCents'])
|
||||
self.assertEqual(report['costStatus'],'UNKNOWN_RETAIN_FULL_20_CENT_RESERVATION')
|
||||
with self.assertRaises(RuntimeError):
|
||||
with runtime.LeaseGuard(lease(),lambda _:None) as guard: infer.run(guard)
|
||||
|
||||
def test_concurrent_reuse_rejected(self):
|
||||
infer=m.ProcessInference([sys.executable,"-c","import time; time.sleep(.3); print(1)"])
|
||||
errors=[]
|
||||
with runtime.LeaseGuard(lease(),lambda _:None) as guard:
|
||||
t=threading.Thread(target=lambda:infer.run(guard)); t.start()
|
||||
time.sleep(.05)
|
||||
with self.assertRaises(RuntimeError): infer.run(guard)
|
||||
t.join()
|
||||
|
||||
def test_explicit_false_heartbeat_denied(self):
|
||||
with tempfile.TemporaryDirectory() as root:
|
||||
with self.assertRaises(PermissionError):
|
||||
m.execute(lease(),root,m.ProcessInference([sys.executable,"-c","print(1)"]),lambda _:False)
|
||||
|
||||
def test_callable_not_cancellable_rejected(self):
|
||||
with tempfile.TemporaryDirectory() as root:
|
||||
with self.assertRaises(TypeError): m.execute(lease(),root,lambda:'',lambda _:None)
|
||||
|
||||
if __name__=='__main__': unittest.main()
|
||||
@@ -0,0 +1,23 @@
|
||||
import datetime, importlib.util, pathlib, tempfile, unittest, uuid
|
||||
p=pathlib.Path(__file__).with_name('executor.py')
|
||||
spec=importlib.util.spec_from_file_location('pilot',p); m=importlib.util.module_from_spec(spec); spec.loader.exec_module(m)
|
||||
qs=importlib.util.spec_from_file_location('qa',p.with_name('qa.py')); qa=importlib.util.module_from_spec(qs); qs.loader.exec_module(qa)
|
||||
def lease():
|
||||
d={k:str(uuid.uuid4()) for k in ('taskId','runId','leaseToken')}; d.update(environment='staging',maxCostCents=20,attempt=1,commitSha='a'*40,leaseUntil=(datetime.datetime.now(datetime.timezone.utc)+datetime.timedelta(seconds=30)).isoformat()); return d
|
||||
class Contract(unittest.TestCase):
|
||||
def test_missing_lease_rejected(self):
|
||||
with self.assertRaises(ValueError): m.validate_lease({})
|
||||
def test_bounds(self):
|
||||
for k,v in [('environment','production'),('attempt',2),('maxCostCents',21),('commitSha','main'),('leaseUntil','2000-01-01T00:00:00+00:00')]:
|
||||
d=lease(); d[k]=v
|
||||
with self.assertRaises(ValueError): m.validate_lease(d)
|
||||
def test_heartbeat_denial_prevents_inference(self):
|
||||
with tempfile.TemporaryDirectory() as root:
|
||||
def deny(_): raise PermissionError('killed')
|
||||
with self.assertRaises(PermissionError): m.execute(lease(),root,lambda:self.fail('must not infer'),deny)
|
||||
def test_qa_rejects_malicious_and_wrong_behavior(self):
|
||||
for source in ['import os\n','def clamp_quantity(quantity):\n return quantity\n','def clamp_quantity(quantity):\n return __import__("os").system("true")\n']:
|
||||
with tempfile.TemporaryDirectory() as root:
|
||||
pathlib.Path(root,'quantity.py').write_text(source)
|
||||
with self.assertRaises(ValueError): qa.verify(root)
|
||||
if __name__=='__main__': unittest.main()
|
||||
@@ -0,0 +1,24 @@
|
||||
import datetime, pathlib, sys, tempfile, threading, time, unittest
|
||||
from unittest.mock import patch
|
||||
sys.path.insert(0, str(pathlib.Path(__file__).parent))
|
||||
import executor as m
|
||||
import runtime
|
||||
from test_contract import lease
|
||||
|
||||
class Heartbeats(unittest.TestCase):
|
||||
def test_ongoing_denial_kills_real_child(self):
|
||||
with tempfile.TemporaryDirectory() as root:
|
||||
marker=pathlib.Path(root,'late')
|
||||
code=f'import time,pathlib; time.sleep(1); pathlib.Path({str(marker)!r}).touch()'
|
||||
beats=[]
|
||||
def beat(binding):
|
||||
beats.append(time.monotonic())
|
||||
if len(beats)>1: raise PermissionError('revoked')
|
||||
with patch.object(runtime,'HEARTBEAT_INTERVAL',.1):
|
||||
with self.assertRaises(PermissionError):
|
||||
m.execute(lease(),root,m.ProcessInference([sys.executable,'-c',code]),beat)
|
||||
time.sleep(1.1)
|
||||
self.assertFalse(marker.exists())
|
||||
self.assertGreaterEqual(len(beats),2)
|
||||
|
||||
if __name__=='__main__': unittest.main()
|
||||
@@ -0,0 +1,17 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
# Invoked ONLY by an approved CI job on the target host; never by application startup.
|
||||
[[ "${CI:-}" == true && "${RUNTIME_READ_APPROVED_SHA:-}" =~ ^[0-9a-f]{40}$ ]] || exit 64
|
||||
root=$(git rev-parse --show-toplevel)
|
||||
[[ "$(git rev-parse HEAD)" == "$RUNTIME_READ_APPROVED_SHA" ]] || exit 65
|
||||
[[ -z "$(git status --porcelain)" ]] || exit 65
|
||||
[[ "${RUNTIME_SNAPSHOT_REVIEWED:-}" == true ]] || exit 66
|
||||
# Gateway remains untouched. Its original writable data volume is NEVER mounted.
|
||||
# The external snapshot must already exist; do not silently initialize empty data.
|
||||
docker volume inspect canina_hermes_inventory_snapshot >/dev/null
|
||||
docker network inspect canina_hermes_backend >/dev/null
|
||||
for name in hermes_username hermes_password hermes_signing_key; do
|
||||
test -s "/opt/services/canina/secrets/$name"
|
||||
done
|
||||
docker compose -f "$root/infra/hermes-runtime/compose.readonly.yml" config --quiet
|
||||
docker compose -f "$root/infra/hermes-runtime/compose.readonly.yml" up -d --no-build
|
||||
@@ -0,0 +1,16 @@
|
||||
# Merge this job into the owning Stage workflow only after provisioning review.
|
||||
# No production trigger or secret material belongs in this file.
|
||||
runtime-read-acceptance:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Pinned Hermes HTTP acceptance
|
||||
run: |
|
||||
set -euo pipefail
|
||||
image=nousresearch/hermes-agent@sha256:9774f4f39a9bb8c2f68ce728ed5e99ddbad282163be56764afacf88ed952b784
|
||||
for test in test_http_acceptance.py test_serve.py test_ws_denial.py; do
|
||||
docker run --rm --network none --entrypoint /opt/hermes/.venv/bin/python3 -v "$PWD/infra/hermes-runtime:/contract:ro" "$image" "/contract/$test"
|
||||
done
|
||||
# Apply job must require approved SHA, Stage environment approval, provisioned external
|
||||
# snapshot/network/secrets, and call bash infra/hermes-runtime/ci-apply.sh on target.
|
||||
# Not enabled here: parent owns actual workflow and Stage promotion.
|
||||
@@ -0,0 +1,36 @@
|
||||
name: canina-hermes-inventory
|
||||
services:
|
||||
runtime-inventory:
|
||||
image: nousresearch/hermes-agent@sha256:9774f4f39a9bb8c2f68ce728ed5e99ddbad282163be56764afacf88ed952b784
|
||||
entrypoint: [/opt/hermes/.venv/bin/python3, /runtime/serve.py]
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
HERMES_HOME: /inventory
|
||||
HERMES_SERVE_HEADLESS: "1"
|
||||
volumes:
|
||||
- ./serve.py:/runtime/serve.py:ro
|
||||
- inventory_snapshot:/inventory
|
||||
secrets: [hermes_username, hermes_password, hermes_signing_key]
|
||||
networks:
|
||||
backend:
|
||||
aliases: [canina-hermes-inventory]
|
||||
security_opt: [no-new-privileges:true]
|
||||
cap_drop: [ALL]
|
||||
pids_limit: 128
|
||||
mem_limit: 1g
|
||||
# No ports, host socket, original home, or egress network.
|
||||
volumes:
|
||||
inventory_snapshot:
|
||||
external: true
|
||||
name: canina_hermes_inventory_snapshot
|
||||
networks:
|
||||
backend:
|
||||
external: true
|
||||
name: canina_hermes_backend
|
||||
secrets:
|
||||
hermes_username:
|
||||
file: /opt/services/canina/secrets/hermes_username
|
||||
hermes_password:
|
||||
file: /opt/services/canina/secrets/hermes_password
|
||||
hermes_signing_key:
|
||||
file: /opt/services/canina/secrets/hermes_signing_key
|
||||
@@ -0,0 +1,11 @@
|
||||
# Disposable acceptance fixture only; not production entrypoint.
|
||||
import os,tempfile,importlib.util
|
||||
os.environ['HERMES_HOME']=tempfile.mkdtemp()
|
||||
os.environ['HERMES_SERVE_HEADLESS']='1'
|
||||
from pathlib import Path
|
||||
import json
|
||||
(Path(os.environ["HERMES_HOME"])/"snapshot-manifest.json").write_text(json.dumps({"schemaVersion":1,"source":"snapshot","capturedAt":"2020-01-01T00:00:00Z"}))
|
||||
spec=importlib.util.spec_from_file_location('runtime_serve','/contract/serve.py');m=importlib.util.module_from_spec(spec);spec.loader.exec_module(m)
|
||||
m.secret=lambda name:'isolated-test-only-'+name+'x'*32
|
||||
import uvicorn
|
||||
uvicorn.run(m.build_app(),host='0.0.0.0',port=9119,lifespan='off',access_log=False,log_level='warning')
|
||||
@@ -0,0 +1,94 @@
|
||||
"""Dedicated read-inventory process: never starts cron, gateway, rooms, MCP or PTY."""
|
||||
from pathlib import Path
|
||||
import os, json
|
||||
from datetime import datetime, timezone
|
||||
from fastapi import Request
|
||||
from fastapi.responses import JSONResponse
|
||||
from plugins.dashboard_auth.basic import BasicAuthProvider, hash_password
|
||||
from hermes_cli.dashboard_auth.registry import register_global_provider
|
||||
from hermes_cli import web_server
|
||||
import uvicorn
|
||||
|
||||
def secret(name):
|
||||
p=Path('/run/secrets')/name
|
||||
if not p.is_file() or p.stat().st_size>4096: raise RuntimeError('Runtime credential unavailable')
|
||||
value=p.read_text().strip()
|
||||
if not value or '\n' in value or '\r' in value: raise RuntimeError('Runtime credential unavailable')
|
||||
return value
|
||||
|
||||
def snapshot_provenance():
|
||||
try:
|
||||
path=Path(os.environ['HERMES_HOME'])/'snapshot-manifest.json'
|
||||
if path.is_symlink() or not path.is_file() or path.stat().st_size>4096:
|
||||
raise ValueError()
|
||||
data=json.loads(path.read_text())
|
||||
if set(data)!={'schemaVersion','source','capturedAt'} or data['schemaVersion']!=1 or data['source']!='snapshot':
|
||||
raise ValueError()
|
||||
captured=data['capturedAt']
|
||||
if not isinstance(captured,str) or not captured.endswith('Z'):
|
||||
raise ValueError()
|
||||
at=datetime.fromisoformat(captured.replace('Z','+00:00'))
|
||||
age=(datetime.now(timezone.utc)-at).total_seconds()
|
||||
if age<0: raise ValueError()
|
||||
return {'source':'snapshot','capturedAt':captured,'ageSeconds':int(age),
|
||||
'freshness':'fresh' if age<=3600 else 'stale','liveProcess':'unknown'}
|
||||
except (OSError, ValueError, TypeError, KeyError):
|
||||
raise RuntimeError('Valid snapshot manifest required') from None
|
||||
|
||||
def build_app():
|
||||
snapshot_provenance()
|
||||
# Never execute installed status: it probes PIDs, gateway HTTP and platform health.
|
||||
web_server.app.router.routes[:] = [route for route in web_server.app.router.routes
|
||||
if getattr(route,'path',None)!='/api/status']
|
||||
@web_server.app.get('/api/status')
|
||||
async def snapshot_status():
|
||||
from hermes_cli.web_server import get_version_info
|
||||
return {'version':get_version_info().base_version,'gateway_running':False,
|
||||
'gateway_state':None,'active_agents':0,'overall':'unknown',
|
||||
'liveGatewayState':'unknown','liveActiveAgents':None}
|
||||
# Place before installed catch-all UI route.
|
||||
web_server.app.router.routes.insert(0, web_server.app.router.routes.pop())
|
||||
register_global_provider(BasicAuthProvider(username=secret('hermes_username'),password_hash=hash_password(secret('hermes_password')),secret=secret('hermes_signing_key').encode(),ttl_seconds=300))
|
||||
web_server.app.state.auth_required=True
|
||||
web_server.app.state.bound_host='0.0.0.0'
|
||||
# Enforce an outer exact route/method allowlist. Installed public health/status stay
|
||||
# internal; protected inventory still uses installed provider middleware.
|
||||
allowed={'/api/health','/api/profiles','/api/status','/api/analytics/usage','/api/sessions','/api/cron/jobs','/api/model/info'}
|
||||
@web_server.app.middleware('http')
|
||||
async def readonly_gate(request:Request,call_next):
|
||||
path=request.url.path
|
||||
if not ((request.method=='GET' and path in allowed) or (request.method=='POST' and path=='/auth/password-login')):
|
||||
return JSONResponse({'detail':'Unavailable'},status_code=404)
|
||||
response=await call_next(request)
|
||||
if request.method!='GET' or response.status_code!=200: return response
|
||||
try:
|
||||
provenance=snapshot_provenance()
|
||||
except RuntimeError:
|
||||
return JSONResponse({'detail':'Snapshot unavailable'},status_code=503)
|
||||
body=b''.join([chunk async for chunk in response.body_iterator])
|
||||
data=json.loads(body)
|
||||
def annotate(row):
|
||||
return {**row,'provenance':provenance}
|
||||
if isinstance(data,list):
|
||||
data=[annotate(row) for row in data]
|
||||
else:
|
||||
data=annotate(data)
|
||||
if path=='/api/profiles':
|
||||
data['profiles']=[{**annotate(row),'gateway_running':False,'liveGatewayState':'unknown'} for row in data['profiles']]
|
||||
if path=='/api/status':
|
||||
# Legacy required scalars are conservative placeholders, NOT measurements.
|
||||
data.update(gateway_running=False,active_agents=0,overall='unknown',gateway_state=None,
|
||||
liveGatewayState='unknown',liveActiveAgents=None)
|
||||
return JSONResponse(data,headers={'Cache-Control':'no-store'})
|
||||
class NoSockets:
|
||||
async def __call__(self, scope, receive, send):
|
||||
if scope['type']=='websocket':
|
||||
await send({'type':'websocket.close','code':1008})
|
||||
return
|
||||
await web_server.app(scope,receive,send)
|
||||
return NoSockets()
|
||||
|
||||
if __name__=='__main__':
|
||||
# No upstream lifespan: no eager schema reconciliation or hosted-room recovery.
|
||||
# Data is an operator-prepared independent snapshot, NEVER the gateway's volume.
|
||||
uvicorn.run(build_app(),host='0.0.0.0',port=9119,lifespan='off',access_log=False,log_level='warning')
|
||||
@@ -0,0 +1,21 @@
|
||||
// Execute inside disposable compiled backend runner sharing fixture network namespace.
|
||||
const assert=require('node:assert/strict');
|
||||
const {HermesSessionBroker}=require('/opt/backend/dist/hermes-management/hermes-session-broker');
|
||||
const {HermesAdapter}=require('/opt/backend/dist/hermes-management/hermes.adapter');
|
||||
const {HermesIntegrationService}=require('/opt/backend/dist/hermes-management/hermes-integration.service');
|
||||
const fetcher=(input,options)=>{const url=new URL(String(input));url.hostname='127.0.0.1';return fetch(url,options);};
|
||||
(async()=>{
|
||||
let reads=0;
|
||||
const broker=new HermesSessionBroker(async()=>{reads++;return {username:'isolated-test-only-hermes_username'+'x'.repeat(32),password:'isolated-test-only-hermes_password'+'x'.repeat(32)};},fetcher);
|
||||
const adapter=new HermesAdapter({origin:'http://canina-hermes-inventory:9119',token:()=>broker.token()},fetcher);
|
||||
const runtime=new HermesIntegrationService(adapter,{authorize:async(p,action,profile)=>p.subject==='test-admin'&&action==='runtime.read'&&(!profile||profile==='default')});
|
||||
await assert.rejects(()=>runtime.health({subject:'denied',profiles:[]}),{code:'FORBIDDEN'});assert.equal(reads,0);
|
||||
const principal={subject:'test-admin',profiles:['default']};
|
||||
assert.equal((await runtime.health(principal)).auth_required,true);
|
||||
const profiles=await runtime.profiles(principal);assert(profiles.some(p=>p.name==='default'));assert(profiles.every(p=>p.name==='default'));
|
||||
await assert.rejects(()=>runtime.status(principal,'outside'),{code:'FORBIDDEN'});
|
||||
const status=await runtime.status(principal,'default');assert.equal(status.overall,'unknown');assert.equal(status.liveActiveAgents,null);assert.equal(status.provenance.source,'snapshot');
|
||||
assert.equal(profiles[0].provenance.source,'snapshot');assert.equal(profiles[0].liveGatewayState,'unknown');
|
||||
assert.equal(reads,1);
|
||||
console.log('PASS real Node broker -> pinned TCP Hermes login -> adapter -> policy-filtered profiles; deny performs zero secret reads; cache coalesced');
|
||||
})().catch(e=>{console.error(e.code||'contract failure');process.exitCode=1;});
|
||||
@@ -0,0 +1,30 @@
|
||||
"""Pinned image HTTP contract, no live volumes/network or provider calls."""
|
||||
import os, secrets, tempfile
|
||||
from pathlib import Path
|
||||
os.environ['HERMES_HOME']=tempfile.mkdtemp(prefix='runtime-contract-')
|
||||
os.environ['HERMES_SERVE_HEADLESS']='1'
|
||||
from fastapi.testclient import TestClient
|
||||
from plugins.dashboard_auth.basic import BasicAuthProvider, hash_password
|
||||
from hermes_cli.dashboard_auth.registry import register_global_provider
|
||||
from hermes_cli import web_server
|
||||
password=secrets.token_urlsafe(32)
|
||||
provider=BasicAuthProvider(username='contract',password_hash=hash_password(password),secret=secrets.token_bytes(32))
|
||||
register_global_provider(provider)
|
||||
web_server.app.state.auth_required=True
|
||||
web_server.app.state.bound_host='0.0.0.0'
|
||||
# No lifespan: avoids hosted-room recovery, cron/MCP and schema reconciliation threads.
|
||||
client=TestClient(web_server.app,base_url='http://localhost')
|
||||
def test_protected_inventory():
|
||||
assert client.get('/api/profiles').status_code==401
|
||||
assert client.post('/auth/password-login',json={'provider':'basic','username':'contract','password':'wrong'}).status_code==401
|
||||
response=client.post('/auth/password-login',json={'provider':'basic','username':'contract','password':password})
|
||||
assert response.status_code==200 and response.json()['ok'] is True
|
||||
token=client.cookies.get('hermes_session_at')
|
||||
assert token and provider.verify_session(access_token=token)
|
||||
assert 'HttpOnly' in response.headers['set-cookie']
|
||||
response=client.get('/api/profiles',headers={'Authorization':'Bearer '+token})
|
||||
assert response.status_code==200 and any(p['name']=='default' for p in response.json()['profiles'])
|
||||
client.cookies.clear()
|
||||
assert client.get('/api/profiles',headers={'Authorization':'Bearer tampered'}).status_code==401
|
||||
print('PASS exact-image HTTP wrong-login=401 login=200 HttpOnly verified-token protected-profiles=200 tampered=401')
|
||||
if __name__=='__main__':test_protected_inventory()
|
||||
@@ -0,0 +1,65 @@
|
||||
import importlib.util, os, secrets, tempfile, threading, socket, time
|
||||
from pathlib import Path
|
||||
from fastapi.testclient import TestClient
|
||||
os.environ['HERMES_HOME']=tempfile.mkdtemp(prefix='inventory-snapshot-')
|
||||
os.environ['HERMES_SERVE_HEADLESS']='1'
|
||||
spec=importlib.util.spec_from_file_location('runtime_serve','/contract/serve.py')
|
||||
module=importlib.util.module_from_spec(spec);spec.loader.exec_module(module)
|
||||
values={'hermes_username':'contract','hermes_password':secrets.token_urlsafe(32),'hermes_signing_key':secrets.token_urlsafe(32)}
|
||||
module.secret=lambda name:values[name]
|
||||
# A capture timestamp is supplied by a reviewed exporter, never inferred at boot.
|
||||
import json
|
||||
manifest=Path(os.environ['HERMES_HOME'])/'snapshot-manifest.json'
|
||||
for invalid in [None, {}, {'schemaVersion':1,'source':'snapshot','capturedAt':'not-a-date'}, {'schemaVersion':1,'source':'snapshot','capturedAt':'2999-01-01T00:00:00Z'}]:
|
||||
if invalid is None:
|
||||
manifest.unlink(missing_ok=True)
|
||||
else:
|
||||
manifest.write_text(json.dumps(invalid))
|
||||
try:
|
||||
module.build_app()
|
||||
except (RuntimeError, ValueError):
|
||||
pass
|
||||
else:
|
||||
raise AssertionError('invalid or missing manifest accepted')
|
||||
manifest.write_text(json.dumps({'schemaVersion':1,'source':'snapshot','capturedAt':'2020-01-01T00:00:00Z'}))
|
||||
app=module.build_app()
|
||||
import uvicorn
|
||||
server=uvicorn.Server(uvicorn.Config(app,host='0.0.0.0',port=9119,lifespan='off',access_log=False,log_level='warning'))
|
||||
thread=threading.Thread(target=server.run,daemon=True);thread.start()
|
||||
for _ in range(200):
|
||||
if server.started:break
|
||||
time.sleep(.05)
|
||||
assert server.started
|
||||
import httpx
|
||||
with httpx.Client(base_url='http://127.0.0.1:9119') as c:
|
||||
assert c.get('/api/profiles').status_code==401
|
||||
assert c.post('/auth/password-login',json={'provider':'basic','username':values['hermes_username'],'password':'wrong'}).status_code==401
|
||||
r=c.post('/auth/password-login',json={'provider':'basic','username':values['hermes_username'],'password':values['hermes_password']})
|
||||
assert r.status_code==200 and r.json()['ok'] is True
|
||||
assert c.get('/api/profiles').status_code==200
|
||||
health=c.get('/api/health').json()
|
||||
assert health['auth_required'] is True
|
||||
assert health['provenance']['source']=='snapshot'
|
||||
assert health['provenance']['capturedAt']=='2020-01-01T00:00:00Z'
|
||||
assert health['provenance']['freshness']=='stale'
|
||||
assert health['provenance']['liveProcess']=='unknown'
|
||||
profiles=c.get('/api/profiles').json()['profiles']
|
||||
assert profiles and all(p['liveGatewayState']=='unknown' and p['gateway_running'] is False and p['provenance']['source']=='snapshot' for p in profiles)
|
||||
status=c.get('/api/status',params={'profile':'default'}).json()
|
||||
assert status['overall']=='unknown' and status['gateway_state'] is None
|
||||
assert status['gateway_running'] is False and status['active_agents']==0
|
||||
assert status['liveActiveAgents'] is None and status['liveGatewayState']=='unknown'
|
||||
for path in ['/api/model/info','/api/analytics/usage','/api/sessions','/api/cron/jobs']:
|
||||
response=c.get(path,params={'profile':'default'})
|
||||
if response.status_code==200:
|
||||
data=response.json()
|
||||
if isinstance(data,list):
|
||||
assert all(r['provenance']['source']=='snapshot' for r in data)
|
||||
else:
|
||||
assert data['provenance']['source']=='snapshot'
|
||||
assert c.post('/api/cron/jobs/example/resume').status_code==404
|
||||
assert c.get('/api/config').status_code==404
|
||||
assert c.get('/api/ws').status_code==404
|
||||
print('PASS real TCP pinned serve: unauth401 wrong-password401 login200 inventory200 health.auth_required=true mutations/config/ws404')
|
||||
server.should_exit=True;thread.join(10)
|
||||
assert not thread.is_alive()
|
||||
@@ -0,0 +1,21 @@
|
||||
import os,tempfile,importlib.util
|
||||
os.environ['HERMES_HOME']=tempfile.mkdtemp()
|
||||
from pathlib import Path
|
||||
import json
|
||||
(Path(os.environ['HERMES_HOME'])/'snapshot-manifest.json').write_text(json.dumps({'schemaVersion':1,'source':'snapshot','capturedAt':'2020-01-01T00:00:00Z'}))
|
||||
from fastapi.testclient import TestClient
|
||||
from starlette.websockets import WebSocketDisconnect
|
||||
spec=importlib.util.spec_from_file_location('runtime_serve','/contract/serve.py');m=importlib.util.module_from_spec(spec);spec.loader.exec_module(m)
|
||||
m.secret=lambda name:'test-only-'+'x'*32
|
||||
app=m.build_app()
|
||||
# A permitted session must NOT unlock powerful WebSocket routes.
|
||||
client=TestClient(app)
|
||||
r=client.post('/auth/password-login',json={'provider':'basic','username':m.secret('hermes_username'),'password':m.secret('hermes_password')})
|
||||
assert r.status_code==200
|
||||
for path in ['/api/ws','/api/pty','/api/events']:
|
||||
try:
|
||||
with client.websocket_connect(path):
|
||||
raise AssertionError('Read-only boundary admitted socket '+path)
|
||||
except WebSocketDisconnect as e:
|
||||
assert e.code==1008, (path,e.code)
|
||||
print('PASS authenticated sockets denied1008')
|
||||
@@ -0,0 +1,28 @@
|
||||
# Bounded PREPARE worker transport (not an LLM pilot)
|
||||
|
||||
This is an opt-in Nest dynamic module and one-shot deterministic Node executor. Nothing imports it into AppModule. No live service, credential, dispatch schedule or deployment is created.
|
||||
|
||||
## Required parent wiring
|
||||
|
||||
- Import `WorkerTransportModule.register(config)` from `backend/src/orchestration-worker/worker-transport.module.ts` in the parent application only after review. Default disabled; provide empty grants until secret provisioning. The module owns its own PrepareWorkflowService/config; do not broaden existing orchestration providers.
|
||||
- Provision >=32 random-byte service credential through a mounted secret `/run/secrets/prepare_worker_credential` to the pilot, and provision only its SHA256 in server grants. Never accept identity/role from request bodies, JWT browser users, headers other than this credential, or a snapshot inventory.
|
||||
- `grants` is an explicit digest -> `{id,role}` mapping with independent `workerIds`/`stageCiIds`. Worker and CI identities/digests cannot overlap. Use a separate CI-only secret for `STAGE_CI`; do not give it to this executor.
|
||||
- Fixed config: absolute private writable `artifactDirectory`, authenticated HTTPS `artifactBaseUrl` pointing to `/internal/prepare-worker/artifacts`, separate persistent artifact volume. The fixture host `isolated.invalid` is NOT a real uploaded artifact URI. TLS routing/URI reachability must be verified by parent Stage CI.
|
||||
- Restrict routes to service network, TLS, ingress body limit (8KiB recommended), rate limit, no browser CORS credentials, no public user JWT access. The module validates strict DTO fields, but production ingress/body limits/rate controls remain parent's wiring.
|
||||
- Run `node infra/hermes-worker/pilot.mjs` with explicit `PREPARE_TASK_ID` UUID and `PREPARE_WORKER_BASE_URL` HTTPS origin. It consumes exactly one task/lease, has 5s per HTTP timeout, does not retry, and exits on SIGINT/SIGTERM. No polling or global dispatch. Disable all egress except backend; do not mount shell/Docker/browser credentials or repo write access. Loopback HTTP is available only via the exported test API option, not CLI.
|
||||
|
||||
## Contract and bounds
|
||||
|
||||
POST `/internal/prepare-worker/claim`: `{taskId}` -> `{lease:null|{...}}`; heartbeat: `{taskId,runId,leaseToken}`; complete: same binding plus `report` (max 2048 chars). Server accepts only byte-exact deterministic tracer JSON derived from the immutable run; it computes SHA256, writes real content, and chooses artifact URI/cost (0 cents), not client-supplied costs/URIs/pass flags. It does **not** inspect repository contents or validate application acceptance criteria. QA here means this harmless tracer succeeded, not application QA.
|
||||
|
||||
Core service still owns transactional Gate1, scope, kill checks, task locks, lease fencing, reservations and VERIFYING event. Transport requires exactly 20-cent task budgets and explicit task IDs. One expired unknown-cost reservation exhausts this pilot budget; do not release it to permit retries. Interrupt means cancellation/non-RUNNING state or STOP_AGENTS; heartbeat before compute, before complete, and atomic completion recheck. File writes may leave unregistered orphan content if the final transaction is denied; retrieval requires a registered QA artifact and verifies its bytes against the requested digest.
|
||||
|
||||
POST `/internal/prepare-worker/stage-evidence` has a separate strict CI DTO `{taskId,runId,commitSha,sha256,evidenceSha256,uri}` and calls independently authenticated core Stage evidence ingestion. This tracer never submits it. Valid CI evidence retrieval/provenance remains CI's responsibility. No Gate2 approval/release transport exists.
|
||||
|
||||
Shared department budgets are not implemented by core and are not claimed here. Do not expand beyond a single bounded pilot until aggregate budgeting and live provisioning are reviewed.
|
||||
|
||||
## Real isolated verification
|
||||
|
||||
Run `bash infra/hermes-worker/test-isolated.sh green`. It creates no-network disposable PostgreSQL16, applies only the three repository v4 migrations, copies readonly source and Node22 dependency donor into private writable storage, regenerates Prisma, runs actual Nest TCP HTTP and a child Node executor, then full backend compilation. Docker DB/anonymous volume removed on exit. No live tables/containers/volumes, RELEASE_AUTHORIZE, push or deploy touched.
|
||||
|
||||
Evidence: `/opt/services/canina/validation/v4-worker-transport/`: `red.log` (missing module initial RED), `integrity-red.log` (real 200 vs expected409 corruption regression), `green.log`, `build.log`, `generate.log`, `tracer-evidence.json` (actual artifact content/hash). Final tests cover credential/body identity denial, disabled/overlap config, Gate1 no-dispatch, competing claims, expired/token fencing, retained20-cent budget, interrupt/kill rechecks, CI separation, real executor to VERIFYING, authenticated artifact GET and tamper rejection. Fixture commit is 40 `a` characters; this is not a repository QA claim or live Stage deployment.
|
||||
@@ -0,0 +1,34 @@
|
||||
# Worker transport security review
|
||||
|
||||
Scope: f7048ab (parent equivalent aa99216). Narrow transport-only hardening; no orchestration/core/AppModule changes, release authority, live provisioning or deployment.
|
||||
|
||||
## Fixed
|
||||
|
||||
- Authenticated artifact retrieval now requires a local fixed-base QA registration, successful staging PREPARE run, matching immutable commit, 20-cent task, and DONE outbox bound to that exact run. Workers can read only their own outbox/run; separately provisioned Stage CI can read bounded completed tracer artifacts for independent verification. Foreign QA registrations and arbitrary authenticated workers do not suffice. Returned bytes must match both content hash and deterministic run-bound tracer report.
|
||||
- Artifact base is canonical HTTPS with the exact `/internal/prepare-worker/artifacts` path: no userinfo, query, fragment, alternate port, IP literals, localhost/private-name suffixes, escaped or normalized arbitrary paths. This is static reviewed configuration, not a fetched URL. DNS resolution/reachability is deliberately not attempted; the operator must review the fixed origin and DNS/TLS routing.
|
||||
- Configuration requires an actual boolean enable flag, non-overlapping valid allowlists and valid unique credential grants. Both transport snapshot and workflow config are frozen, including arrays and grant objects; caller mutation cannot change the snapshot.
|
||||
- Linux descriptor-relative directory traversal with O_DIRECTORY/O_NOFOLLOW prevents symlinked path components and pins directories across renames. Final directory must be private and owned by process UID. File creation uses exclusive O_CREAT/O_EXCL/O_NOFOLLOW, never truncating existing symlinks, hardlinks or regular files. Reads use a pinned O_NOFOLLOW/O_NONBLOCK descriptor, require a private owned regular single-link file, and are bounded to 2049 bytes before digest/run binding verification.
|
||||
|
||||
## Preserved and verified
|
||||
|
||||
Bearer service digest identity, strict DTO/body impersonation rejection, independent CI role, explicit task-only claim (no queue polling), exact 20-cent cap, Gate1 gating, lease owner/token/task/run fences, unknown-cost reservation exhaustion, STOP_AGENTS and cancellation checks, atomic core completion recheck. No network fetch of artifact URLs was introduced.
|
||||
|
||||
The executor remains a deterministic JSON tracer: no LLM, shell, browser, deployment, repository inspection or real application QA. Successful tracer QA is not acceptance-criteria verification.
|
||||
|
||||
## Evidence
|
||||
|
||||
Actual disposable PostgreSQL16/no-network Docker fixture, Nest TCP HTTP listener, child Node22 executor and full backend `tsc -b` build: `/opt/services/canina/validation/v4-worker-security/`.
|
||||
|
||||
- `auth-red.log`: unrelated registered worker received 200 instead of expected 404.
|
||||
- `config-red.log`: unsafe credential-bearing base accepted.
|
||||
- `file-red.log`: symlink target sentinel overwritten by deterministic report.
|
||||
- `freeze-red.log`: workflow config remained mutable.
|
||||
- `security-green.log`: all 7 tests pass, including own-worker/CI retrieval, foreign URI denial, directory/file symlink and hardlink rejection, tamper integrity rejection, kill/lease/task fencing and actual child executor to VERIFYING.
|
||||
- `build.log`: full backend `tsc -b` exit 0.
|
||||
- `tracer-evidence.json`: actual fixture task/run/hash/content, GET200, cost0. External URI reachability explicitly NOT_TESTED_ISOLATED_FIXTURE_HOST.
|
||||
|
||||
## Limits / residual responsibilities
|
||||
|
||||
Linux `/proc/self/fd` is required; unsupported platforms fail closed. Protect the artifact mount and its ancestry from same-UID/root attackers; they remain outside this process boundary. File and database cannot commit atomically: a killed/denied/competing completion can leave an unregistered orphan, deliberately unreadable and never overwritten automatically. No retry/cleanup redesign was added. Descriptor pinning prevents path redirection but does not promise availability during hostile renames. Symlink/hardlink swaps are tested; exhaustive adversarial scheduling is not claimed.
|
||||
|
||||
Parent still owns service-network restriction, reviewed origin/DNS/TLS reachability, body/rate limits, secret provisioning, Stage CI evidence provenance, aggregate department budgets and integration tests. All workers remain unwired/default-disabled; no live secrets or RELEASE_AUTHORIZE used.
|
||||
@@ -0,0 +1,37 @@
|
||||
import { readFile } from 'node:fs/promises';
|
||||
import { pathToFileURL } from 'node:url';
|
||||
|
||||
/** One explicit task, one attempt, fixed computation; no shell, prompt, browser or LLM. */
|
||||
export async function runOnce({baseUrl, credential, taskId, allowLoopbackHttp=false, signal}) {
|
||||
const url = new URL(baseUrl);
|
||||
if (url.username || url.password || url.search || url.hash || (url.protocol !== 'https:' && !(allowLoopbackHttp && url.protocol === 'http:' && ['127.0.0.1','[::1]'].includes(url.hostname)))) throw new Error('HTTPS service origin required');
|
||||
if (!credential || credential.length < 32) throw new Error('Service credential required');
|
||||
async function post(route, body) {
|
||||
signal?.throwIfAborted();
|
||||
const timeout=AbortSignal.timeout(5000);
|
||||
const r=await fetch(new URL('/internal/prepare-worker/'+route,url), {method:'POST',redirect:'error',signal:signal ? AbortSignal.any([signal,timeout]) : timeout,headers:{authorization:'Bearer '+credential,'content-type':'application/json'},body:JSON.stringify(body)});
|
||||
const text=await r.text();
|
||||
if(text.length>8192) throw new Error('Oversized service response');
|
||||
if(!r.ok) throw new Error('Worker request denied: '+r.status);
|
||||
return JSON.parse(text);
|
||||
}
|
||||
const {lease}=await post('claim',{taskId});
|
||||
if(!lease) return {state:'IDLE'};
|
||||
if(lease.taskId!==taskId || lease.environment!=='staging' || lease.maxCostCents!==20 || !/^[a-f0-9]{40}$/.test(lease.commitSha)) throw new Error('Unbounded lease rejected');
|
||||
const binding={taskId,runId:lease.runId,leaseToken:lease.leaseToken};
|
||||
await post('heartbeat',binding); // stop/interrupt recheck before any computation
|
||||
signal?.throwIfAborted();
|
||||
const report=JSON.stringify({kind:'DETERMINISTIC_PREPARE_TRACER',schemaVersion:1,taskId,runId:lease.runId,commitSha:lease.commitSha,environment:'staging',llmPilot:false,browserExecuted:false,shellExecuted:false,deploymentExecuted:false,checks:['lease-bound identity','deterministic JSON serialization'],actualCents:0});
|
||||
await post('heartbeat',binding); // stop/interrupt recheck immediately before completion
|
||||
return post('complete',{...binding,report});
|
||||
}
|
||||
if (process.argv[1] && import.meta.url===pathToFileURL(process.argv[1]).href) {
|
||||
const controller=new AbortController();
|
||||
process.once('SIGINT',()=>controller.abort());
|
||||
process.once('SIGTERM',()=>controller.abort());
|
||||
try {
|
||||
const credential=(await readFile('/run/secrets/prepare_worker_credential','utf8')).trim();
|
||||
const result=await runOnce({baseUrl:process.env.PREPARE_WORKER_BASE_URL,credential,taskId:process.env.PREPARE_TASK_ID,signal:controller.signal});
|
||||
console.log(JSON.stringify(result));
|
||||
} catch { console.error('Bounded PREPARE tracer failed; no completion assumed'); process.exitCode=1; }
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
||||
E="${WORKER_EVIDENCE_DIR:-/opt/services/canina/validation/v4-worker-transport}"
|
||||
mkdir -p "$E"
|
||||
DB="v4-worker-test-${RANDOM}"
|
||||
trap 'docker rm -fv "$DB" >/dev/null 2>&1 || true' EXIT
|
||||
docker run -d --name "$DB" --network none -e POSTGRES_PASSWORD=disposable -e POSTGRES_DB=worker postgres:16-alpine >/dev/null
|
||||
for n in $(seq 1 60); do if docker exec "$DB" pg_isready -U postgres >/dev/null 2>&1; then break; fi; sleep 1; done
|
||||
for migration in 20261022000000_add_orchestration_v4_core 20261022190000_harden_orchestration_ledger 20261023170000_prepare_workflow_outbox; do
|
||||
docker exec -i "$DB" psql -v ON_ERROR_STOP=1 -U postgres -d worker < "$ROOT/backend/prisma/migrations/$migration/migration.sql" > "$E/$migration.log"
|
||||
done
|
||||
docker run --rm --userns=host --network "container:$DB" -e DATABASE_URL=postgresql://postgres:disposable@127.0.0.1:5432/worker -e WORKER_TRANSPORT_DB=1 -e TRACER_EVIDENCE_PATH=/evidence/tracer-evidence.json -e TMPDIR=/work/tmp -v "$ROOT:/source:ro" -v /opt/services/canina/worktrees/orchestration-v4-runtime/backend/node_modules:/deps:ro -v "$E:/evidence" node:22-bookworm bash -c '
|
||||
set -euo pipefail
|
||||
mkdir -p /work/backend /work/infra /work/tmp
|
||||
cp -a /source/backend/. /work/backend/
|
||||
cp -a /source/infra/. /work/infra/
|
||||
cp -a /deps /work/backend/node_modules
|
||||
cd /work/backend
|
||||
./node_modules/.bin/prisma generate > /evidence/generate.log 2>&1
|
||||
npm test -- --runInBand orchestration-worker > /evidence/'"${1:-green}"'.log 2>&1
|
||||
npm run build > /evidence/build.log 2>&1
|
||||
'
|
||||
@@ -0,0 +1,45 @@
|
||||
#!/usr/bin/env bash
|
||||
# CI job side only. Never execute host deploy logic in the runner filesystem.
|
||||
set -euo pipefail
|
||||
set +x
|
||||
fail() { printf 'ERROR: %s\n' "$1" >&2; exit 2; }
|
||||
[[ ${GITHUB_ACTIONS:-${GITEA_ACTIONS:-}} == true && ${BRANCH:-} == develop && $# == 0 ]] || fail 'CI-only Stage deployment; Production is blocked'
|
||||
[[ ${DEPLOY_SHA:-} =~ ^[0-9a-f]{40}$ ]] || fail 'immutable DEPLOY_SHA required'
|
||||
# Repository secret DEPLOY_KEY is the historical approved devops SSH credential.
|
||||
# Never fall back to baked-in runner keys, agent forwarding or docker.sock.
|
||||
[[ -n ${DEPLOY_KNOWN_HOSTS:-} ]] || fail 'provision DEPLOY_KNOWN_HOSTS: independently verified [87.248.133.138]:2234 host key; never use ssh-keyscan trust-on-first-use'
|
||||
[[ -n ${DEPLOY_KEY:-} ]] || fail 'provision existing approved DEPLOY_KEY for devops@87.248.133.138:2234'
|
||||
for tool in ssh ssh-keygen mktemp; do command -v "$tool" >/dev/null || fail "job context missing $tool"; done
|
||||
umask 077
|
||||
scratch=$(mktemp -d)
|
||||
trap 'rm -rf "$scratch"' EXIT
|
||||
printf '%s\n' "$DEPLOY_KNOWN_HOSTS" > "$scratch/known_hosts"
|
||||
printf '%s\n' "$DEPLOY_KEY" > "$scratch/key"
|
||||
unset DEPLOY_KEY DEPLOY_KNOWN_HOSTS
|
||||
ssh-keygen -F '[87.248.133.138]:2234' -f "$scratch/known_hosts" >/dev/null || fail 'DEPLOY_KNOWN_HOSTS has no pinned deployment endpoint'
|
||||
# Only the validated SHA enters the remote command; no user-controlled shell text.
|
||||
ssh -F /dev/null -T -p 2234 -i "$scratch/key" \
|
||||
-o BatchMode=yes -o IdentitiesOnly=yes -o IdentityAgent=none \
|
||||
-o StrictHostKeyChecking=yes -o UserKnownHostsFile="$scratch/known_hosts" \
|
||||
-o GlobalKnownHostsFile=/dev/null -o ConnectTimeout=15 \
|
||||
devops@87.248.133.138 "sudo -n bash -s -- $DEPLOY_SHA" <<'HOST'
|
||||
set -euo pipefail
|
||||
set +x
|
||||
umask 077
|
||||
fail() { printf 'ERROR: host context: %s\n' "$1" >&2; exit 2; }
|
||||
[[ $# == 1 && $1 =~ ^[0-9a-f]{40}$ ]] || fail 'immutable SHA required'
|
||||
[[ ! -e /.dockerenv && ! -e /run/.containerenv ]] || fail 'refusing container filesystem'
|
||||
[[ $(id -u) == 0 ]] || fail 'noninteractive approved sudo required'
|
||||
for tool in bash python3 docker git flock realpath install wget; do command -v "$tool" >/dev/null || fail "missing $tool"; done
|
||||
[[ -d /opt/services/canina/stage/.git && -d /opt/services/proxy/certs/canina ]] || fail 'Stage operational checkout/certificates unavailable'
|
||||
[[ -s /opt/services/canina/stage/.support-chat.env && ! -L /opt/services/canina/stage/.support-chat.env ]] || fail 'Stage runtime env unavailable'
|
||||
docker compose version >/dev/null
|
||||
docker volume inspect stage_canina_stage_db stage_canina_stage_uploads >/dev/null
|
||||
docker network inspect traefik_public >/dev/null
|
||||
printf 'Host context validated; Stage commit=%s; runtime env present; Production blocked\n' "$1"
|
||||
script=$(mktemp)
|
||||
trap 'rm -f "$script"' EXIT
|
||||
wget --no-proxy -qO "$script" "https://git.parsaaghayi.ir/parsa/canina/raw/commit/${1}/scripts/deploy.sh"
|
||||
# CI marker exists only across this authenticated, fixed Stage boundary.
|
||||
GITHUB_ACTIONS=true DEPLOY_SHA="$1" bash "$script" develop
|
||||
HOST
|
||||
+85
-119
@@ -1,137 +1,103 @@
|
||||
#!/bin/bash
|
||||
set -euo pipefail
|
||||
set +x
|
||||
# CI/CD-only Stage rollout; Production awaits separate reviewed authorization.
|
||||
if [ "${GITHUB_ACTIONS:-${GITEA_ACTIONS:-}}" != true ] || [ "${1:-}" != develop ] || [ "$#" != 1 ]; then
|
||||
echo 'ERROR: refusing deployment outside CI/CD Stage rollout' >&2
|
||||
exit 2
|
||||
fi
|
||||
if [[ ! "${DEPLOY_SHA:-}" =~ ^[0-9a-f]{40}$ ]]; then
|
||||
echo "ERROR: refusing deployment without an immutable commit SHA" >&2
|
||||
echo 'ERROR: refusing deployment without an immutable commit SHA' >&2
|
||||
exit 2
|
||||
fi
|
||||
# END ROLLOUT GUARD
|
||||
exec 9>/opt/services/canina/stage-backend-deploy.lock
|
||||
flock -n 9 || { echo 'Stage deployment already in progress' >&2; exit 1; }
|
||||
export DOCKER_BUILDKIT=1
|
||||
export COMPOSE_DOCKER_CLI_BUILD=1
|
||||
|
||||
# Ensure nexus is connected to default bridge network so docker build can reach it
|
||||
docker network connect bridge nexus 2>/dev/null || true
|
||||
|
||||
# Nexus npm registry
|
||||
NEXUS_IP=$(docker inspect -f '{{.NetworkSettings.Networks.bridge.IPAddress}}' nexus 2>/dev/null || true)
|
||||
if [ -z "$NEXUS_IP" ]; then
|
||||
NEXUS_IP="172.17.0.3"
|
||||
fi
|
||||
NPM_REGISTRY="http://${NEXUS_IP}:8081/repository/bitchpedar/"
|
||||
|
||||
BRANCH="${1:-develop}"
|
||||
echo "Starting Deployment for branch: ${BRANCH}"
|
||||
|
||||
if [ "$BRANCH" == "develop" ]; then
|
||||
COMPOSE_FILE="compose.stage.yml"
|
||||
TARGET_DIR="/opt/services/canina/stage"
|
||||
BACKEND_HOST="backend_stage"
|
||||
APP_HOST="stage.canina.ir"
|
||||
ADMIN_HOST="stageadmin.canina.ir"
|
||||
API_URL="https://stageapi.canina.ir"
|
||||
CONTAINER_PREFIX="canino_backend_stage"
|
||||
else
|
||||
COMPOSE_FILE="compose.prod.yml"
|
||||
TARGET_DIR="/opt/services/canina/prod"
|
||||
BACKEND_HOST="backend_prod"
|
||||
APP_HOST="canina.ir"
|
||||
ADMIN_HOST="admin.canina.ir"
|
||||
API_URL="https://api.canina.ir"
|
||||
CONTAINER_PREFIX="canino_backend_prod"
|
||||
fi
|
||||
|
||||
mkdir -p "$TARGET_DIR"
|
||||
|
||||
if [ ! -d "$TARGET_DIR/.git" ]; then
|
||||
echo "Cloning repository into $TARGET_DIR..."
|
||||
git clone -b "$BRANCH" https://git.parsaaghayi.ir/parsa/canina.git "$TARGET_DIR"
|
||||
git -C "$TARGET_DIR" cat-file -e "${DEPLOY_SHA}^{commit}"
|
||||
git -C "$TARGET_DIR" merge-base --is-ancestor "$DEPLOY_SHA" "origin/$BRANCH"
|
||||
git -C "$TARGET_DIR" reset --hard "$DEPLOY_SHA"
|
||||
else
|
||||
echo "Pulling latest changes in $TARGET_DIR..."
|
||||
cd "$TARGET_DIR"
|
||||
git fetch origin "$BRANCH"
|
||||
git cat-file -e "${DEPLOY_SHA}^{commit}"
|
||||
git merge-base --is-ancestor "$DEPLOY_SHA" "origin/$BRANCH"
|
||||
git reset --hard "$DEPLOY_SHA"
|
||||
cd -
|
||||
fi
|
||||
|
||||
umask 077
|
||||
export DOCKER_BUILDKIT=1 COMPOSE_DOCKER_CLI_BUILD=1
|
||||
# Stage is an operational reference, NEVER a checkout/build target.
|
||||
STAGE_DIR=/opt/services/canina/stage
|
||||
RELEASE_ROOT=/opt/services/canina/releases
|
||||
TARGET_DIR="$RELEASE_ROOT/$DEPLOY_SHA"
|
||||
mkdir -p "$RELEASE_ROOT"
|
||||
[ ! -L "$RELEASE_ROOT" ] && [ "$(realpath "$RELEASE_ROOT")" = "$RELEASE_ROOT" ]
|
||||
# Do not overwrite a prior release or retry with stale generated configuration.
|
||||
mkdir "$TARGET_DIR"
|
||||
git clone --no-checkout --single-branch --branch develop https://git.parsaaghayi.ir/parsa/canina.git "$TARGET_DIR"
|
||||
cd "$TARGET_DIR"
|
||||
|
||||
# Capture the bot key from Docker before recreating the bot; runner mounts may not include the host env file.
|
||||
if docker inspect canina-bot >/dev/null 2>&1; then
|
||||
LIVE_BOT_KEY=$(docker inspect -f '{{range .Config.Env}}{{println .}}{{end}}' canina-bot | awk -F= '$1 == "SUPPORT_INTERNAL_KEY" {print substr($0, index($0, "=") + 1); exit}')
|
||||
if [ -n "$LIVE_BOT_KEY" ]; then
|
||||
grep -v '^SUPPORT_CHAT_KEY=' .support-chat.env > .support-chat.env.tmp 2>/dev/null || true
|
||||
printf 'SUPPORT_CHAT_KEY=%s
|
||||
' "$LIVE_BOT_KEY" >> .support-chat.env.tmp
|
||||
mv .support-chat.env.tmp .support-chat.env
|
||||
chmod 600 .support-chat.env
|
||||
git cat-file -e "${DEPLOY_SHA}^{commit}"
|
||||
git merge-base --is-ancestor "$DEPLOY_SHA" origin/develop
|
||||
git checkout --detach "$DEPLOY_SHA"
|
||||
[ "$(git rev-parse HEAD)" = "$DEPLOY_SHA" ]
|
||||
# Keep operational files out of status/accidental commits in this disposable clone.
|
||||
printf '\n.support-chat.env\n.revalidation.env\n.stage-base.json\n.stage-compose.json\n' >> .git/info/exclude
|
||||
# Runtime secrets/config must never enter the frontend Docker build context.
|
||||
printf '\n.support-chat.env\n.revalidation.env\n.stage-*.json\n' >> .dockerignore
|
||||
# Only these backend/frontend runtime env files are required. Do not copy the
|
||||
# dirty compose, .env, uploads, bot state, certificates, or arbitrary Stage files.
|
||||
for name in .support-chat.env .revalidation.env; do
|
||||
if [ -e "$STAGE_DIR/$name" ] || [ -L "$STAGE_DIR/$name" ]; then
|
||||
[ -f "$STAGE_DIR/$name" ] && [ ! -L "$STAGE_DIR/$name" ]
|
||||
[ -z "$(git -C "$STAGE_DIR" ls-files -- "$name")" ]
|
||||
[ -z "$(git ls-files -- "$name")" ]
|
||||
install -m 600 "$STAGE_DIR/$name" "$TARGET_DIR/$name"
|
||||
fi
|
||||
unset LIVE_BOT_KEY
|
||||
fi
|
||||
|
||||
# Inject support-chat runtime configuration from CI/CD without committing secrets.
|
||||
# The bot is the runtime owner of the internal key; synchronize only the
|
||||
# matching value into the colocated backend env file, never into Git or logs.
|
||||
BOT_ENV="/opt/services/ai/canina-bot/support.env"
|
||||
if [ -r "$BOT_ENV" ]; then
|
||||
BOT_KEY=$(awk -F= '$1 == "SUPPORT_INTERNAL_KEY" {print substr($0, index($0, "=") + 1); exit}' "$BOT_ENV")
|
||||
if [ -n "$BOT_KEY" ]; then
|
||||
touch .support-chat.env
|
||||
grep -v '^SUPPORT_CHAT_KEY=' .support-chat.env > .support-chat.env.tmp || true
|
||||
printf "SUPPORT_CHAT_KEY=%s\n" "$BOT_KEY" >> .support-chat.env.tmp
|
||||
mv .support-chat.env.tmp .support-chat.env
|
||||
chmod 600 .support-chat.env
|
||||
done
|
||||
[ -s .support-chat.env ] || { echo 'ERROR: missing Stage support runtime env' >&2; exit 1; }
|
||||
# Never inspect/recreate/mutate the shared support bot or infer keys from it.
|
||||
# Certs remain the explicit operational absolute bind from tracked compose.
|
||||
python3 - <<'PY'
|
||||
from pathlib import Path
|
||||
p = Path('nginx.conf')
|
||||
s = p.read_text()
|
||||
for key, value in {'__BACKEND_HOST__': 'backend_stage', '__APP_HOST__': 'stage.canina.ir', '__ADMIN_HOST__': 'stageadmin.canina.ir'}.items():
|
||||
s = s.replace(key, value)
|
||||
p.write_text(s)
|
||||
PY
|
||||
# Wrapper also covers every compose call in the sourced rollout helper. Explicit
|
||||
# project/files/env-file prevent release-SHA projects and ambient .env loading.
|
||||
docker() {
|
||||
if [ "${1:-}" = compose ]; then
|
||||
shift
|
||||
command docker compose -p stage --env-file /dev/null --project-directory "$TARGET_DIR" -f "$TARGET_DIR/.stage-compose.json" "$@"
|
||||
else
|
||||
command docker "$@"
|
||||
fi
|
||||
unset BOT_KEY
|
||||
fi
|
||||
if [ -n "${SUPPORT_CHAT_KEY:-}" ]; then
|
||||
touch .support-chat.env
|
||||
grep -v '^SUPPORT_CHAT_KEY=' .support-chat.env > .support-chat.env.tmp || true
|
||||
printf 'SUPPORT_CHAT_KEY=%s\n' "$SUPPORT_CHAT_KEY" >> .support-chat.env.tmp
|
||||
mv .support-chat.env.tmp .support-chat.env
|
||||
chmod 600 .support-chat.env
|
||||
fi
|
||||
if [ ! -s .support-chat.env ]; then
|
||||
echo "ERROR: missing .support-chat.env / SUPPORT_CHAT_KEY; refusing deployment" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# ✅ فایلهای کانفیگ را از پوشهی scripts مخزن به TARGET_DIR کپی میکنیم
|
||||
echo "Copying configuration files in $TARGET_DIR..."
|
||||
cp "scripts/$COMPOSE_FILE" "docker-compose.yml"
|
||||
sed -i 's/- REDIS_PORT: 6379/- REDIS_PORT=6379/g' "docker-compose.yml" 2>/dev/null || true
|
||||
|
||||
echo "Patching nginx.conf for environment..."
|
||||
sed -i "s/__BACKEND_HOST__/${BACKEND_HOST}/g" "nginx.conf"
|
||||
sed -i "s/__APP_HOST__/${APP_HOST}/g" "nginx.conf"
|
||||
sed -i "s/__ADMIN_HOST__/${ADMIN_HOST}/g" "nginx.conf"
|
||||
|
||||
echo "Using Nexus npm proxy: $NPM_REGISTRY"
|
||||
NPM_REGISTRY_ARG="--build-arg NPM_REGISTRY=${NPM_REGISTRY}"
|
||||
|
||||
echo "Building Docker images..."
|
||||
docker compose --progress plain build \
|
||||
--build-arg NEXT_PUBLIC_API_URL="${API_URL}/api" \
|
||||
--build-arg VITE_API_URL="${API_URL}" \
|
||||
${NPM_REGISTRY_ARG}
|
||||
|
||||
# Never recreate the shared Production support bot during a Stage rollout.
|
||||
}
|
||||
# Render privately: config contains env secrets, so never emit it to CI logs.
|
||||
command docker compose -p stage --env-file /dev/null --project-directory "$TARGET_DIR" -f scripts/compose.stage.yml config --format json > .stage-base.json
|
||||
python3 - <<'PY'
|
||||
import json
|
||||
from pathlib import Path
|
||||
config = json.loads(Path('.stage-base.json').read_text())
|
||||
expected = {'canina_stage_db': 'stage_canina_stage_db', 'canina_stage_uploads': 'stage_canina_stage_uploads'}
|
||||
if set(config['services']) != {'db_stage', 'redis_stage', 'backend_stage', 'frontend_stage'}:
|
||||
raise SystemExit('ERROR: unexpected Stage service set')
|
||||
for key, name in expected.items():
|
||||
if config['volumes'][key].get('name') != name:
|
||||
raise SystemExit('ERROR: Stage volume identity mismatch')
|
||||
config['volumes'][key] = {'name': name, 'external': True}
|
||||
if config['networks']['traefik_public'].get('name') != 'traefik_public' or not config['networks']['traefik_public'].get('external'):
|
||||
raise SystemExit('ERROR: Stage network identity mismatch')
|
||||
backend = config['services']['backend_stage']
|
||||
if backend['container_name'] != 'canino_backend_stage' or config['services']['frontend_stage']['container_name'] != 'canino_frontend_stage':
|
||||
raise SystemExit('ERROR: Stage container identity mismatch')
|
||||
mounts = backend.get('volumes', [])
|
||||
if not any(m.get('type') == 'volume' and m.get('source') == 'canina_stage_uploads' and m.get('target') == '/app/uploads' for m in mounts):
|
||||
raise SystemExit('ERROR: missing Stage uploads volume')
|
||||
for service in config['services'].values():
|
||||
for mount in service.get('volumes', []):
|
||||
if mount.get('type') == 'bind' and (mount.get('source'), mount.get('target')) != ('/opt/services/proxy/certs/canina', '/app/ssl_certs'):
|
||||
raise SystemExit('ERROR: unexpected Stage bind mount')
|
||||
Path('.stage-compose.json').write_text(json.dumps(config))
|
||||
Path('.stage-base.json').unlink()
|
||||
PY
|
||||
# Read-only checks: missing external storage must fail, never provision blank DB.
|
||||
docker volume inspect stage_canina_stage_db stage_canina_stage_uploads >/dev/null
|
||||
docker network inspect traefik_public >/dev/null
|
||||
[ -d /opt/services/proxy/certs/canina ]
|
||||
# Builds only app images; DB/Redis and support-bot remain untouched.
|
||||
docker compose --progress plain build backend_stage frontend_stage
|
||||
source scripts/rollout-stage.sh
|
||||
stage_rollout
|
||||
|
||||
# Auto-cleanup old docker builder cache to keep disk healthy (keeps latest 6GB)
|
||||
echo "Cleaning up old docker build cache..."
|
||||
docker builder prune -af --keep-storage 6GB 2>/dev/null || true
|
||||
docker image prune -f 2>/dev/null || true
|
||||
|
||||
echo "Deployment for $BRANCH completed successfully!"
|
||||
echo 'Deployment for develop completed successfully!'
|
||||
@@ -27,6 +27,8 @@ with tempfile.TemporaryDirectory(prefix='canina-rollout-') as d:
|
||||
query = "const {PrismaClient}=require('@prisma/client');const p=new PrismaClient();(async()=>{await p.product.findMany({take:1,include:{ingredientList:true,symptoms:true}});await p.supportConversation.findMany({take:1});await p.orchestrationTask.findMany({take:1,include:{runs:true,events:true,budgetLedger:true}});console.log('PASS: actual Prisma legacy catalog/support and v4 relation queries');})().catch(()=>{console.error('Prisma compatibility failed');process.exitCode=1}).finally(()=>p.$disconnect());"
|
||||
run(cmd[:-3]+['node','-e',query])
|
||||
after=sql("SELECT table_name,column_name,data_type,is_nullable,column_default FROM information_schema.columns WHERE table_schema='public' AND table_name <> '_prisma_migrations' AND table_name NOT LIKE 'orchestration_%' ORDER BY 1,2;")
|
||||
# Compare only the exact pre-existing table set; new additive tables are not legacy.
|
||||
after='\n'.join(line for line in after.splitlines() if line.split('|',1)[0] in tables)+'\n'
|
||||
assert before==after,'Legacy column definitions changed'
|
||||
for table in tables: sql(f'SELECT * FROM "{table}" LIMIT 0;')
|
||||
v4=sql("SELECT tablename FROM pg_tables WHERE schemaname='public' AND tablename LIKE 'orchestration_%' ORDER BY 1;").splitlines()
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
import os, pathlib, subprocess, tempfile, unittest
|
||||
ROOT = pathlib.Path(__file__).resolve().parents[2]
|
||||
class BoundaryTests(unittest.TestCase):
|
||||
def run_wrapper(self, **updates):
|
||||
env = dict(os.environ, GITHUB_ACTIONS='true', BRANCH='develop', DEPLOY_SHA='a'*40)
|
||||
for name in ('DEPLOY_KEY','DEPLOY_KNOWN_HOSTS'): env.pop(name, None)
|
||||
env.update(updates)
|
||||
return subprocess.run(['bash', str(ROOT/'scripts/ci-host-deploy.sh')], env=env, capture_output=True, text=True)
|
||||
def test_missing_pinned_host_identity_fails_before_network(self):
|
||||
result = self.run_wrapper(DEPLOY_KEY='sentinel-secret')
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertIn('DEPLOY_KNOWN_HOSTS', result.stderr)
|
||||
self.assertNotIn('sentinel-secret', result.stdout+result.stderr)
|
||||
|
||||
def test_guards_refuse_non_ci_production_and_injection(self):
|
||||
for updates in ({'GITHUB_ACTIONS':'false'}, {'BRANCH':'main'}, {'DEPLOY_SHA':'a'*40+';touch bad'}, {'DEPLOY_SHA':''}):
|
||||
result = self.run_wrapper(**updates)
|
||||
self.assertEqual(result.returncode, 2)
|
||||
self.assertNotIn('provision', result.stderr)
|
||||
def test_authentication_is_noninteractive_and_sha_is_only_remote_argument(self):
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
tools = pathlib.Path(d)
|
||||
# Deliberate transport double: validates invocation, never a live deploy.
|
||||
(tools/'ssh').write_text('#!/bin/bash\nprintf "%s\\n" "$@" > "$TRACE"\n/bin/cat > "$PAYLOAD"\nexit "${SSH_RESULT:-0}"\n')
|
||||
(tools/'ssh').chmod(0o755)
|
||||
known = '[87.248.133.138]:2234 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA'
|
||||
for code in ('0','255'):
|
||||
result = self.run_wrapper(DEPLOY_KEY='sentinel-secret', DEPLOY_KNOWN_HOSTS=known, PATH=d+':'+os.environ['PATH'], TRACE=d+'/trace', PAYLOAD=d+'/payload', SSH_RESULT=code)
|
||||
self.assertEqual(result.returncode, int(code), result.stderr)
|
||||
trace = (tools/'trace').read_text()
|
||||
for option in ('StrictHostKeyChecking=yes','BatchMode=yes','IdentitiesOnly=yes','IdentityAgent=none','devops@87.248.133.138','sudo -n bash -s -- '+'a'*40): self.assertIn(option,trace)
|
||||
self.assertNotIn('sentinel-secret', trace+result.stdout+result.stderr)
|
||||
lines = trace.splitlines()
|
||||
key = pathlib.Path(lines[lines.index('-i')+1])
|
||||
self.assertFalse(key.exists(), 'temporary credential must be cleaned even on SSH failure')
|
||||
payload = (tools/'payload').read_text()
|
||||
self.assertIn('/raw/commit/${1}/scripts/deploy.sh', payload)
|
||||
self.assertIn('python3',payload)
|
||||
self.assertIn('refusing container filesystem',payload)
|
||||
self.assertNotIn('docker run', payload)
|
||||
self.assertNotIn('git reset', payload)
|
||||
def test_workflow_uses_only_authenticated_wrapper(self):
|
||||
workflow = (ROOT/'.gitea/workflows/deploy.yml').read_text()
|
||||
self.assertIn('scripts/ci-host-deploy.sh',workflow)
|
||||
self.assertIn('DEPLOY_KNOWN_HOSTS: ${{ secrets.DEPLOY_KNOWN_HOSTS }}',workflow)
|
||||
self.assertIn('DEPLOY_KEY: ${{ secrets.DEPLOY_KEY }}',workflow)
|
||||
self.assertNotIn('cd scripts',workflow)
|
||||
self.assertNotIn('./deploy.sh',workflow)
|
||||
|
||||
if __name__ == '__main__': unittest.main()
|
||||
@@ -13,9 +13,12 @@ class RolloutTests(unittest.TestCase):
|
||||
script=(ROOT/'scripts/deploy.sh').read_text()
|
||||
workflow=(ROOT/'.gitea/workflows/deploy.yml').read_text()
|
||||
self.assertIn('DEPLOY_SHA: ${{ github.sha }}',workflow)
|
||||
self.assertIn('/raw/commit/${DEPLOY_SHA}/scripts/deploy.sh',workflow)
|
||||
self.assertIn('git reset --hard "$DEPLOY_SHA"',script)
|
||||
self.assertNotIn('git reset --hard "origin/$BRANCH"',script)
|
||||
self.assertIn('/raw/commit/${DEPLOY_SHA}/scripts/ci-host-deploy.sh',workflow)
|
||||
wrapper=(ROOT/'scripts/ci-host-deploy.sh').read_text()
|
||||
self.assertIn('/raw/commit/${1}/scripts/deploy.sh',wrapper)
|
||||
self.assertNotIn('git reset',script)
|
||||
self.assertIn('git checkout --detach "$DEPLOY_SHA"',script)
|
||||
self.assertIn('/opt/services/canina/releases',script)
|
||||
|
||||
def test_commit_guard_rejects_missing_and_malformed_sha_before_side_effects(self):
|
||||
guard=(ROOT/'scripts/deploy.sh').read_text().split('# END ROLLOUT GUARD')[0]
|
||||
@@ -33,7 +36,91 @@ class RolloutTests(unittest.TestCase):
|
||||
workflow=(ROOT/'.gitea/workflows/deploy.yml').read_text()
|
||||
run=workflow.split(' run: |',1)[1]
|
||||
self.assertEqual(run.strip().splitlines()[0].strip(),'set -euo pipefail')
|
||||
self.assertIn('./deploy.sh "${BRANCH}"',run)
|
||||
self.assertIn('bash "$wrapper"',run)
|
||||
import textwrap
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
tools=pathlib.Path(d)
|
||||
(tools/'wget').write_text('#!/bin/sh\nexit 42\n')
|
||||
(tools/'wget').chmod(0o755)
|
||||
# Execute the real job block; failed download must not launch wrapper.
|
||||
result=subprocess.run(['bash','-c',textwrap.dedent(run)], env=dict(os.environ, PATH=d+':'+os.environ['PATH'], BRANCH='develop', DEPLOY_SHA='a'*40, DEPLOY_KEY='test-not-a-key', DEPLOY_KNOWN_HOSTS='test-pin'), capture_output=True,text=True)
|
||||
self.assertEqual(result.returncode,42,result.stderr)
|
||||
|
||||
def test_isolated_deploy_preserves_dirty_stage_and_stable_storage(self):
|
||||
import json, shutil
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
base = pathlib.Path(directory)
|
||||
source, stage, releases, tools = [base / n for n in ('source', 'stage', 'releases', 'tools')]
|
||||
for p in (source, stage, tools): p.mkdir()
|
||||
def git(*args, cwd=source):
|
||||
return subprocess.check_output(['git', *args], cwd=cwd, text=True).strip()
|
||||
git('init', '-b', 'develop')
|
||||
git('config', 'user.email', 'test@example.invalid')
|
||||
git('config', 'user.name', 'Test')
|
||||
shutil.copytree(ROOT/'scripts', source/'scripts')
|
||||
shutil.copyfile(ROOT/'.dockerignore', source/'.dockerignore')
|
||||
(source/'nginx.conf').write_text('__BACKEND_HOST__')
|
||||
git('add', '.')
|
||||
git('commit', '-qm', 'fixture')
|
||||
sha = git('rev-parse', 'HEAD')
|
||||
git('init', '-b', 'develop', cwd=stage)
|
||||
git('config', 'user.email', 'test@example.invalid', cwd=stage)
|
||||
git('config', 'user.name', 'Test', cwd=stage)
|
||||
(stage/'dirty').write_text('original')
|
||||
git('add', '.', cwd=stage)
|
||||
git('commit', '-qm', 'stage', cwd=stage)
|
||||
(stage/'dirty').write_text('staged')
|
||||
git('add', '.', cwd=stage)
|
||||
(stage/'dirty').write_text('unstaged')
|
||||
for n in ('.support-chat.env', '.revalidation.env'):
|
||||
(stage/n).write_text('TEST_SECRET=never-log-this\n')
|
||||
before = git('status', '--porcelain', cwd=stage), git('diff', '--binary', 'HEAD', cwd=stage), (stage/'.git/index').read_bytes()
|
||||
config = {'services': {
|
||||
'db_stage': {'volumes': [{'type': 'volume', 'source': 'canina_stage_db', 'target': '/var/lib/postgresql/data'}]},
|
||||
'redis_stage': {},
|
||||
'backend_stage': {'container_name': 'canino_backend_stage', 'volumes': [
|
||||
{'type': 'volume', 'source': 'canina_stage_uploads', 'target': '/app/uploads'},
|
||||
{'type': 'bind', 'source': '/opt/services/proxy/certs/canina', 'target': '/app/ssl_certs'}]},
|
||||
'frontend_stage': {'container_name': 'canino_frontend_stage'}},
|
||||
'volumes': {n: {'name': 'stage_'+n} for n in ('canina_stage_db','canina_stage_uploads')},
|
||||
'networks': {'traefik_public': {'name': 'traefik_public', 'external': True}}}
|
||||
(base/'config.json').write_text(json.dumps(config))
|
||||
(tools/'docker').write_text('#!/bin/bash\nprintf "%s\\n" "$*" >> "$TRACE"\nif [[ "$*" == *"config --format json"* ]]; then /bin/cat "$FIXTURE"; fi\nif [[ "$*" == *"run --rm"* && "${FAIL_MIGRATION:-}" == 1 ]]; then exit 42; fi\n')
|
||||
(tools/'docker').chmod(0o755)
|
||||
script = (ROOT/'scripts/deploy.sh').read_text()
|
||||
script = script.replace('/opt/services/canina/stage-backend-deploy.lock', str(base/'lock')).replace('/opt/services/canina/stage', str(stage)).replace('/opt/services/canina/releases', str(releases)).replace('https://git.parsaaghayi.ir/parsa/canina.git', str(source)).replace('[ -d /opt/services/proxy/certs/canina ]', 'true')
|
||||
deploy = base/'deploy.sh'
|
||||
deploy.write_text(script)
|
||||
env = dict(os.environ, GITHUB_ACTIONS='true', DEPLOY_SHA=sha, PATH=str(tools)+':'+os.environ['PATH'], TRACE=str(base/'trace'), FIXTURE=str(base/'config.json'))
|
||||
for fail in ('1', '0'):
|
||||
if releases.exists(): shutil.rmtree(releases)
|
||||
(base/'trace').write_text('')
|
||||
result = subprocess.run(['bash', str(deploy), 'develop'], env=dict(env, FAIL_MIGRATION=fail), capture_output=True, text=True)
|
||||
self.assertEqual(result.returncode, 42 if fail == '1' else 0, result.stderr)
|
||||
trace = (base/'trace').read_text()
|
||||
compose = [line for line in trace.splitlines() if line.startswith('compose ')]
|
||||
self.assertTrue(compose)
|
||||
self.assertTrue(all(line.startswith('compose -p stage --env-file /dev/null --project-directory '+str(releases/sha)) for line in compose))
|
||||
self.assertNotIn('canina-bot', trace)
|
||||
self.assertNotIn('remove-orphans', trace)
|
||||
self.assertNotIn('up -d --no-deps db_stage', trace)
|
||||
self.assertNotIn('never-log-this', result.stdout+result.stderr)
|
||||
if fail == '1': self.assertNotIn('up -d', trace)
|
||||
else:
|
||||
self.assertLess(trace.index('migrate deploy'), trace.index('up -d --no-deps backend_stage'))
|
||||
self.assertLess(trace.index('exec -T backend_stage'), trace.index('up -d --no-deps frontend_stage'))
|
||||
release = releases/sha
|
||||
self.assertEqual(git('rev-parse', 'HEAD', cwd=release), sha)
|
||||
self.assertEqual((release/'.support-chat.env').stat().st_mode & 0o777, 0o600)
|
||||
self.assertIn('.stage-*.json', (release/'.dockerignore').read_text())
|
||||
self.assertIn('.support-chat.env', (release/'.dockerignore').read_text())
|
||||
self.assertIn('.revalidation.env', (release/'.dockerignore').read_text())
|
||||
self.assertEqual((release/'.stage-compose.json').stat().st_mode & 0o777, 0o600)
|
||||
effective = json.loads((release/'.stage-compose.json').read_text())
|
||||
self.assertEqual(effective['volumes'], {n: {'name': 'stage_'+n, 'external': True} for n in ('canina_stage_db','canina_stage_uploads')})
|
||||
self.assertEqual(before, (git('status', '--porcelain', cwd=stage), git('diff', '--binary', 'HEAD', cwd=stage), (stage/'.git/index').read_bytes()))
|
||||
retry = subprocess.run(['bash', str(deploy), 'develop'], env=env, capture_output=True, text=True)
|
||||
self.assertNotEqual(retry.returncode, 0)
|
||||
|
||||
def test_migration_failure_never_starts_apps(self):
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
@@ -60,4 +147,3 @@ class RolloutTests(unittest.TestCase):
|
||||
self.assertIn('exec -T backend_stage node',trace)
|
||||
self.assertNotIn('up -d --no-deps frontend_stage',trace)
|
||||
self.assertNotIn('SUCCESS',result.stdout)
|
||||
if __name__=='__main__': unittest.main()
|
||||
Reference in new issue
Block a user