fix(infra): integrate private revalidation provisioning and narrow release gates
Deploy Canina / deploy (push) Failing after 1m6s
Deploy Canina / deploy (push) Failing after 1m6s
This commit is contained in:
1 parent
1bb6274b35
commit
f94ef580fb
6 files changed
+79
-12
No files matched your search
+2
-1
@@ -51,7 +51,7 @@ printf '\n.support-chat.env\n.revalidation.env\n.stage-base.json\n.stage-compose
|
||||
printf '\n.support-chat.env\n.revalidation.env\n.stage-*.json\n' >> .dockerignore
|
||||
# Only these backend/frontend runtime env files are required. Do not copy the
|
||||
# dirty compose, .env, uploads, bot state, certificates, or arbitrary Stage files.
|
||||
for name in .support-chat.env .revalidation.env; do
|
||||
for name in .support-chat.env; do
|
||||
if [ -e "$STAGE_DIR/$name" ] || [ -L "$STAGE_DIR/$name" ]; then
|
||||
[ -f "$STAGE_DIR/$name" ] && [ ! -L "$STAGE_DIR/$name" ]
|
||||
[ -z "$(git -C "$STAGE_DIR" ls-files -- "$name")" ]
|
||||
@@ -59,6 +59,7 @@ for name in .support-chat.env .revalidation.env; do
|
||||
install -m 600 "$STAGE_DIR/$name" "$TARGET_DIR/$name"
|
||||
fi
|
||||
done
|
||||
stage_revalidation_provision
|
||||
[ -s .support-chat.env ] || { echo 'ERROR: missing Stage support runtime env' >&2; exit 1; }
|
||||
# Never inspect/recreate/mutate the shared support bot or infer keys from it.
|
||||
# Certs remain the explicit operational absolute bind from tracked compose.
|
||||
|
||||
@@ -13,10 +13,19 @@ ALLOWED = {'nginx.conf', 'Dockerfile', 'backend/Dockerfile', '.dockerignore',
|
||||
'scripts/rollout-production.sh', 'scripts/production-release.py', 'scripts/test_production_release.py',
|
||||
'scripts/compose.stage.yml', 'scripts/rollout-stage.sh', 'scripts/revalidation-runtime.py',
|
||||
'scripts/provision-revalidation-env.py', 'scripts/tests/test_revalidation_runtime.py',
|
||||
'scripts/tests/test_ci_host_boundary.py', 'scripts/tests/test_stage_provision_wiring.py',
|
||||
'scripts/tests/test_stage_rollout.py',
|
||||
'backend/src/common/revalidation/revalidation.service.ts',
|
||||
'backend/src/common/revalidation/revalidation.service.spec.ts',
|
||||
'backend/src/orchestration-worker/worker-registration.spec.ts',
|
||||
'backend/src/support-chat/support-chat.http.spec.ts',
|
||||
# Exact baseline gate repairs reviewed in 1bb6274; never a frontend wildcard.
|
||||
'frontend/admin-panel/src/pages/OrchestrationCommandCenter.tsx',
|
||||
'frontend/admin-panel/src/pages/selectionSafety.test.tsx',
|
||||
'frontend/admin-panel/src/pages/workflowSafety.test.tsx',
|
||||
'frontend/admin-panel/src/test/setup.ts',
|
||||
'frontend/admin-panel/src/test/storage.test.ts',
|
||||
'frontend/application/components/AnalyticsDeferred.tsx',
|
||||
'frontend/application/components/BlogPage.tsx',
|
||||
'frontend/application/components/__tests__/BlogPage.sorting.test.tsx'}
|
||||
|
||||
@@ -89,6 +98,9 @@ def main():
|
||||
s.setdefault('labels',{})['ir.canina.release.sha']=sha
|
||||
s['env_file']=[{'path':str(Path.cwd()/'.revalidation.env'),'required':True}]
|
||||
s['environment']=[v for v in s['environment'] if not v.startswith('REVALIDATION_SECRET=')]
|
||||
if name=='backend_prod':
|
||||
s['environment']=[v for v in s['environment'] if not v.startswith('FRONTEND_INTERNAL_URL=')]
|
||||
s['environment'].append('FRONTEND_INTERNAL_URL=http://frontend_prod:3000')
|
||||
s['build']={'context':str(Path.cwd() / ('backend' if name=='backend_prod' else '.')),'dockerfile':'Dockerfile'}
|
||||
if name=='frontend_prod': s['build']['args']={'NEXT_PUBLIC_API_URL':'https://api.canina.ir/api','VITE_API_URL':'https://api.canina.ir'}
|
||||
Path('.prod-compose.json').write_text(json.dumps(c))
|
||||
|
||||
@@ -11,6 +11,21 @@ class ProductionGuards(unittest.TestCase):
|
||||
def receipt(self):
|
||||
return dict(sha='a'*40, baseline='b'*40, stage_run_url='https://git.parsaaghayi.ir/parsa/canina/actions/runs/123', reviewer='CEO delegated acceptance', accepted=True, scope='reliability-only', paths=['nginx.conf'])
|
||||
|
||||
def test_exact_six_baseline_gate_repairs_allowed_not_frontend_wildcard(self):
|
||||
files = {
|
||||
'frontend/admin-panel/src/pages/OrchestrationCommandCenter.tsx',
|
||||
'frontend/admin-panel/src/pages/selectionSafety.test.tsx',
|
||||
'frontend/admin-panel/src/pages/workflowSafety.test.tsx',
|
||||
'frontend/admin-panel/src/test/setup.ts',
|
||||
'frontend/admin-panel/src/test/storage.test.ts',
|
||||
'frontend/application/components/AnalyticsDeferred.tsx',
|
||||
}
|
||||
r=self.receipt();r['paths']=sorted(files)
|
||||
m.validate_receipt(r,'a'*40,'b'*40,sorted(files))
|
||||
for path in ['frontend/admin-panel/src/pages/Other.tsx', 'frontend/application/components/Other.tsx']:
|
||||
r['paths']=[path]
|
||||
with self.assertRaises(ValueError): m.validate_receipt(r,'a'*40,'b'*40,[path])
|
||||
|
||||
def test_exact_stage_sha(self):
|
||||
r=self.receipt()
|
||||
m.validate_receipt(r, 'a'*40, 'b'*40, ['nginx.conf'])
|
||||
@@ -37,6 +52,23 @@ class ProductionGuards(unittest.TestCase):
|
||||
x['Mounts'][0]['Name']='stage_canina_stage_uploads'
|
||||
with self.assertRaises(ValueError): m.snapshot([x])
|
||||
|
||||
def test_production_snapshot_forces_direct_next_url(self):
|
||||
import os, json
|
||||
from unittest.mock import patch
|
||||
c={'services':{n:{'environment':['FRONTEND_INTERNAL_URL=http://legacy:8080','REVALIDATION_SECRET=old'], 'labels':{}} for n in ['backend_prod','frontend_prod']}}
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
old=os.getcwd()
|
||||
try:
|
||||
os.chdir(d)
|
||||
with patch.object(m.sys,'argv',['production-release.py','snapshot']), patch.dict(os.environ,DEPLOY_SHA='a'*40), patch.object(m.subprocess,'check_output',return_value=b'[]'), patch.object(m,'snapshot',return_value=c):
|
||||
m.main()
|
||||
services=json.loads(Path('.prod-compose.json').read_text())['services']
|
||||
self.assertIn('FRONTEND_INTERNAL_URL=http://frontend_prod:3000',services['backend_prod']['environment'])
|
||||
for s in services.values():
|
||||
self.assertFalse(any(v.startswith('REVALIDATION_SECRET=') for v in s['environment']))
|
||||
self.assertTrue(s['env_file'][0]['required'])
|
||||
finally: os.chdir(old)
|
||||
|
||||
def test_private_receipt_permissions(self):
|
||||
import os
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
|
||||
@@ -12,8 +12,8 @@ class BoundaryTests(unittest.TestCase):
|
||||
self.assertIn('DEPLOY_KNOWN_HOSTS', result.stderr)
|
||||
self.assertNotIn('sentinel-secret', result.stdout+result.stderr)
|
||||
|
||||
def test_guards_refuse_non_ci_production_and_injection(self):
|
||||
for updates in ({'GITHUB_ACTIONS':'false'}, {'BRANCH':'main'}, {'DEPLOY_SHA':'a'*40+';touch bad'}, {'DEPLOY_SHA':''}):
|
||||
def test_guards_refuse_non_ci_unknown_branch_and_injection(self):
|
||||
for updates in ({'GITHUB_ACTIONS':'false'}, {'BRANCH':'unreviewed'}, {'DEPLOY_SHA':'a'*40+';touch bad'}, {'DEPLOY_SHA':''}):
|
||||
result = self.run_wrapper(**updates)
|
||||
self.assertEqual(result.returncode, 2)
|
||||
self.assertNotIn('provision', result.stderr)
|
||||
|
||||
@@ -7,6 +7,12 @@ class WiringTests(unittest.TestCase):
|
||||
self.assertIn('stage_runtime_provision',s)
|
||||
self.assertLess(s.index('stage_runtime_provision'),s.index('>> .dockerignore'))
|
||||
self.assertIn('CANINA_RUNTIME_READ_ENABLED', (ROOT/'scripts/rollout-stage.sh').read_text())
|
||||
def test_revalidation_provision_before_compose_without_copy(self):
|
||||
s=(ROOT/'scripts/deploy.sh').read_text()
|
||||
self.assertIn('\nstage_revalidation_provision\n',s)
|
||||
self.assertLess(s.index('\nstage_revalidation_provision\n'),s.index('# Render privately'))
|
||||
self.assertNotIn('for name in .support-chat.env .revalidation.env',s)
|
||||
|
||||
def test_inventory_started_before_backend(self):
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
p=pathlib.Path(d)/'docker'; p.write_text('#!/bin/sh\nprintf "%s\\n" "$*" >> "$TRACE"\n');p.chmod(0o755)
|
||||
@@ -23,23 +29,31 @@ class WiringTests(unittest.TestCase):
|
||||
shutil.copytree(ROOT/'scripts',repo/'scripts',ignore=shutil.ignore_patterns('__pycache__'))
|
||||
shutil.copytree(ROOT/'infra',repo/'infra')
|
||||
root=base/'runtime';home=base/'home';home.mkdir();tools=base/'tools';tools.mkdir()
|
||||
policy=json.loads((ROOT/'infra/hermes-editorial/model-policy.json').read_text())
|
||||
registered=set(policy['profiles'])-set(policy['additionalProfiles'])
|
||||
(home/policy['registry']).write_text('departments:\n'+''.join(' d%d: {profile: %s}\n' % (i,p) for i,p in enumerate(sorted(registered))))
|
||||
for profile in policy['profiles']:
|
||||
config=home/'profiles'/profile/'config.yaml';config.parent.mkdir(parents=True)
|
||||
config.write_text('model:\n default: old\n provider: custom\n');config.chmod(0o600)
|
||||
db=sqlite3.connect(home/'state.db')
|
||||
db.execute('CREATE TABLE sessions(source,started_at,input_tokens,output_tokens,estimated_cost_usd,title)')
|
||||
db.execute("INSERT INTO sessions VALUES ('cli',1,2,3,0.1,'PRIVATE TITLE')");db.commit();db.close()
|
||||
helper=repo/'scripts/rollout-stage.sh'
|
||||
helper.write_text(helper.read_text().replace('/opt/services/canina/stage-runtime',str(root)))
|
||||
# The fixture Docker backend runs in this test namespace, not the host's
|
||||
# /proc user namespace; bind projected ownership to this test process.
|
||||
helper.write_text(helper.read_text().replace('/opt/services/canina/stage-runtime',str(root)).replace("uid,gid=mapped('uid'),mapped('gid')", 'uid,gid=os.getuid(),os.getgid()'))
|
||||
editorial=repo/'scripts/provision-editorial-runtime.py'
|
||||
editorial.write_text(editorial.read_text().replace('/opt/services/canina/stage-runtime',str(root)))
|
||||
def git(*args):return subprocess.check_output(['git',*args],cwd=repo,text=True).strip()
|
||||
git('init','-q');git('config','user.email','fixture@example.invalid');git('config','user.name','Fixture');git('add','.');git('commit','-qm','fixture')
|
||||
sha=git('rev-parse','HEAD')
|
||||
subject='12345678-1234-1234-1234-123456789abc'
|
||||
subject='12345678-1234-1234-1234-123456789012'
|
||||
fixture={'backend':{'Config':{'User':'node'},'HostConfig':{'UsernsMode':''},'State':{'Running':True,'Pid':os.getpid()}},'hermes':{'State':{'Running':True},'HostConfig':{'UsernsMode':''},'Config':{'Env':['HERMES_HOME=/home/runtime']},'Mounts':[{'Type':'bind','Destination':'/home','Source':str(base)}]}}
|
||||
# /home/runtime resolves to base/runtime; point home at the SQLite fixture.
|
||||
fixture['hermes']['Mounts'][0]['Source']=str(home)
|
||||
fixture['hermes']['Config']['Env']=['HERMES_HOME=/home']
|
||||
(base/'inspect.json').write_text(json.dumps(fixture))
|
||||
(tools/'docker').write_text('#!/usr/bin/env python3\nimport os,sys,json,subprocess\na=sys.argv[1:]\nif a[0]=="compose": os.execv("/usr/bin/docker",["docker",*a])\nf=json.load(open(os.environ["INSPECT"]))\nif a[0]=="inspect": print(json.dumps([f["backend" if a[1]=="canino_backend_stage" else "hermes"]]))\nelif a[0]=="exec" and "-i" in a:\n p=json.load(sys.stdin); p["home"]=f["hermes"]["Mounts"][0]["Source"]; subprocess.run([sys.executable,"-B","-c",a[-1]],input=json.dumps(p),text=True,check=True)\nelif a[0]=="exec" and "skills" in a:\n print("canina-editorial-v4")\nelif a[0]=="exec":\n assert "SET TRANSACTION READ ONLY" in a[-1] and "role::text" in a[-1]\n print(json.dumps([{ "id":os.environ["SUBJECT"]}]))\nelse: raise RuntimeError(a)\n');(tools/'docker').chmod(0o755)
|
||||
(tools/'docker').write_text('#!/usr/bin/env python3\nimport os,sys,json,subprocess\na=sys.argv[1:]\nif a[0]=="compose": os.execv("/usr/bin/docker",["docker",*a])\nf=json.load(open(os.environ["INSPECT"]))\nif a[0]=="inspect": print(json.dumps([f["backend" if a[1]=="canino_backend_stage" else "hermes"]]))\nelif a[0]=="exec" and "ci_ownership" in a[-1]:\n assert a[1:4] == ["--user","0:0","-i"]\n p=json.load(sys.stdin); assert "def migrate(" in p["source"]\n # Root container ownership migration is covered in its dedicated suite.\nelif a[0]=="exec" and "-i" in a:\n p=json.load(sys.stdin); p["home"]=f["hermes"]["Mounts"][0]["Source"]; subprocess.run([sys.executable,"-B","-c",a[-1]],input=json.dumps(p),text=True,check=True)\nelif a[0]=="exec" and "pwd.getpwnam" in a[-1]:\n print(json.dumps([10000,10000]))\nelif a[0]=="exec" and "skills" in a:\n print("canina-editorial-v4")\nelif a[0]=="exec":\n assert "SET TRANSACTION READ ONLY" in a[-1] and "role::text" in a[-1]\n print(json.dumps([{ "id":os.environ["SUBJECT"], "role":os.environ.get("ROLE","Admin")}]))\nelse: raise RuntimeError(a)\n');(tools/'docker').chmod(0o755)
|
||||
env=dict(os.environ,PATH=str(tools)+':'+os.environ['PATH'],INSPECT=str(base/'inspect.json'),SUBJECT=subject,GITHUB_ACTIONS='true',STAGE_PROVISION_ENABLED='true',PYTHONDONTWRITEBYTECODE='1')
|
||||
command='set -euo pipefail; source scripts/rollout-stage.sh; DEPLOY_SHA='+sha+'; unset STAGE_PROVISION_APPROVED_SHA; stage_runtime_provision'
|
||||
# DEPLOY_SHA is deliberately shell-local: helper must export it.
|
||||
@@ -62,6 +76,11 @@ class WiringTests(unittest.TestCase):
|
||||
self.assertNotEqual(r.returncode,0)
|
||||
self.assertIn('Verified admin subjects required',r.stderr)
|
||||
self.assertEqual(before,set((root/'snapshots').iterdir()))
|
||||
for updates in [dict(SUBJECT='12345678-1234-1234-1234-123456789abc'),dict(ROLE='User')]:
|
||||
r=subprocess.run(['bash','-c',command],cwd=repo,env=dict(env,**updates),capture_output=True,text=True)
|
||||
self.assertNotEqual(r.returncode,0)
|
||||
self.assertIn('Pinned Stage owner must exist with Admin role',r.stderr)
|
||||
self.assertEqual(before,set((root/'snapshots').iterdir()))
|
||||
(home/'state.db-wal').symlink_to(home/'state.db')
|
||||
r=subprocess.run(['bash','-c',command],cwd=repo,env=env,capture_output=True,text=True)
|
||||
self.assertNotEqual(r.returncode,0)
|
||||
@@ -90,10 +109,10 @@ class WiringTests(unittest.TestCase):
|
||||
script=(ROOT/'scripts/deploy.sh').read_text()
|
||||
validation=script.split("config = json.loads(Path('.stage-base.json').read_text())",1)[1].split("\nPY",1)[0]
|
||||
code="import json\nfrom pathlib import Path\nconfig=json.loads(Path('.stage-base.json').read_text())"+validation
|
||||
config={'services':{'db_stage':{},'redis_stage':{},'backend_stage':{'container_name':'canino_backend_stage','volumes':[{'type':'volume','source':'canina_stage_uploads','target':'/app/uploads'},{'type':'bind','source':'/etc','target':'/stolen'}]},'frontend_stage':{'container_name':'canino_frontend_stage'}},'volumes':{n:{'name':'stage_'+n} for n in ['canina_stage_db','canina_stage_uploads']},'networks':{'traefik_public':{'external':True,'name':'traefik_public'}}}
|
||||
config={'services':{'db_stage':{},'redis_stage':{},'backend_stage':{'build':{'context':'./backend'},'container_name':'canino_backend_stage','volumes':[{'type':'volume','source':'canina_stage_uploads','target':'/app/uploads'},{'type':'bind','source':'/etc','target':'/stolen'}]},'frontend_stage':{'container_name':'canino_frontend_stage'}},'volumes':{n:{'name':'stage_'+n} for n in ['canina_stage_db','canina_stage_uploads']},'networks':{'traefik_public':{'external':True,'name':'traefik_public'}}}
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
p=pathlib.Path(directory);(p/'.stage-base.json').write_text(json.dumps(config))
|
||||
result=subprocess.run(['python3','-c',code],cwd=p,capture_output=True,text=True)
|
||||
result=subprocess.run(['python3','-c',code],cwd=p,env=dict(os.environ,DEPLOY_SHA='a'*40),capture_output=True,text=True)
|
||||
self.assertNotEqual(result.returncode,0)
|
||||
self.assertIn('unexpected Stage bind mount',result.stderr)
|
||||
self.assertFalse((p/'.stage-compose.json').exists())
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
import os, pathlib, subprocess, tempfile, unittest
|
||||
ROOT = pathlib.Path(__file__).resolve().parents[2]
|
||||
class RolloutTests(unittest.TestCase):
|
||||
def test_non_ci_and_production_refused_before_docker(self):
|
||||
def test_non_ci_and_unpinned_production_refused_before_docker(self):
|
||||
script=(ROOT/'scripts/deploy.sh').read_text()
|
||||
self.assertIn('CI/CD-only Stage rollout',script)
|
||||
self.assertIn('authenticated CI/CD rollout',script)
|
||||
guard=script.split('# END ROLLOUT GUARD')[0]
|
||||
for args,env in [(['develop'],{}),(['develop','stage-backend-only'],{'GITHUB_ACTIONS':'true'}),(['main'],{'GITHUB_ACTIONS':'true'})]:
|
||||
r=subprocess.run(['bash','-c',guard,'deploy.sh',*args],cwd=ROOT,env=dict(os.environ,**env),capture_output=True,text=True)
|
||||
@@ -58,6 +58,9 @@ class RolloutTests(unittest.TestCase):
|
||||
git('config', 'user.email', 'test@example.invalid')
|
||||
git('config', 'user.name', 'Test')
|
||||
shutil.copytree(ROOT/'scripts', source/'scripts')
|
||||
# This test exercises rollout/Compose, never host systemd installation.
|
||||
# The installer's guarded install contract has its own full unit suite.
|
||||
(source/'scripts/install-stage-principal-cleanup.py').write_text('raise SystemExit(0)\n')
|
||||
shutil.copyfile(ROOT/'.dockerignore', source/'.dockerignore')
|
||||
(source/'nginx.conf').write_text('__BACKEND_HOST__')
|
||||
git('add', '.')
|
||||
@@ -73,12 +76,12 @@ class RolloutTests(unittest.TestCase):
|
||||
git('add', '.', cwd=stage)
|
||||
(stage/'dirty').write_text('unstaged')
|
||||
for n in ('.support-chat.env', '.revalidation.env'):
|
||||
(stage/n).write_text('TEST_SECRET=never-log-this\n')
|
||||
(stage/n).write_text('REVALIDATION_SECRET='+'s'*48+'\n' if n == '.revalidation.env' else 'TEST_SECRET=never-log-this\n')
|
||||
before = git('status', '--porcelain', cwd=stage), git('diff', '--binary', 'HEAD', cwd=stage), (stage/'.git/index').read_bytes()
|
||||
config = {'services': {
|
||||
'db_stage': {'volumes': [{'type': 'volume', 'source': 'canina_stage_db', 'target': '/var/lib/postgresql/data'}]},
|
||||
'redis_stage': {},
|
||||
'backend_stage': {'container_name': 'canino_backend_stage', 'volumes': [
|
||||
'backend_stage': {'build': {'context': './backend'}, 'container_name': 'canino_backend_stage', 'volumes': [
|
||||
{'type': 'volume', 'source': 'canina_stage_uploads', 'target': '/app/uploads'},
|
||||
{'type': 'bind', 'source': '/opt/services/proxy/certs/canina', 'target': '/app/ssl_certs'}]},
|
||||
'frontend_stage': {'container_name': 'canino_frontend_stage'}},
|
||||
|
||||
Reference in new issue
Block a user