fix(ci): renew only existing unstarted Stage task 1a00
Bounded Stage v4 credential acceptance window / acceptance-window (push) Failing after 3m22s
Bounded Stage v4 credential acceptance window / acceptance-window (push) Failing after 3m22s
This commit is contained in:
1 parent
0837042865
commit
aff1da02f4
6 files changed
+56
-44
No files matched your search
@@ -2,10 +2,10 @@ name: Bounded Stage v4 credential acceptance window
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- ci/v4-stage-acceptance-timeout-860324e
|
||||
- ci/v4-stage-resume-1a00-860324e
|
||||
jobs:
|
||||
acceptance-window:
|
||||
if: github.ref == 'refs/heads/ci/v4-stage-acceptance-timeout-860324e'
|
||||
if: github.ref == 'refs/heads/ci/v4-stage-resume-1a00-860324e'
|
||||
runs-on: canina
|
||||
timeout-minutes: 12
|
||||
steps:
|
||||
@@ -20,7 +20,7 @@ jobs:
|
||||
run: |
|
||||
set -euo pipefail
|
||||
set +x
|
||||
[[ "$BRANCH" == ci/v4-stage-acceptance-timeout-860324e && "$DEPLOY_SHA" =~ ^[a-f0-9]{40}$ && "$GITHUB_RUN_ATTEMPT" == 1 ]] || exit 2
|
||||
[[ "$BRANCH" == ci/v4-stage-resume-1a00-860324e && "$DEPLOY_SHA" =~ ^[a-f0-9]{40}$ && "$GITHUB_RUN_ATTEMPT" == 1 ]] || exit 2
|
||||
umask 077
|
||||
scratch=$(mktemp -d)
|
||||
trap 'rm -rf "$scratch"' EXIT
|
||||
|
||||
@@ -2,11 +2,11 @@
|
||||
# One bounded Stage acceptance only; never deploy, retry, reconcile or publish.
|
||||
set -euo pipefail
|
||||
set +x
|
||||
[[ $# == 0 && ${GITHUB_ACTIONS:-${GITEA_ACTIONS:-}} == true && ${BRANCH:-} == ci/v4-stage-acceptance-timeout-860324e && ${GITHUB_RUN_ATTEMPT:-} == 1 && ${DEPLOY_SHA:-} =~ ^[a-f0-9]{40}$ ]] || exit 2
|
||||
[[ $# == 0 && ${GITHUB_ACTIONS:-${GITEA_ACTIONS:-}} == true && ${BRANCH:-} == ci/v4-stage-resume-1a00-860324e && ${GITHUB_RUN_ATTEMPT:-} == 1 && ${DEPLOY_SHA:-} =~ ^[a-f0-9]{40}$ ]] || exit 2
|
||||
# Separate workflow source SHA from the immutable, already-deployed Stage artifact.
|
||||
export DEPLOY_SHA=860324e3a91b31c2b6d4e784e458f6d120582cb8
|
||||
export STAGE_TEST_APPROVAL=owner-authorized-stage-editorial
|
||||
export STAGE_TEST_SESSION=v4-ci-timeout-860324e-20261024
|
||||
export STAGE_TEST_SESSION=v4-ci-resume-1a00-20261024
|
||||
export STAGE_TEST_TTL_SECONDS=600
|
||||
wrapper="$(dirname "${BASH_SOURCE[0]}")/ci-stage-test-principal.sh"
|
||||
close() {
|
||||
|
||||
@@ -6,12 +6,12 @@ fail() { printf 'ERROR: %s\n' "$1" >&2; exit 2; }
|
||||
[[ ${GITHUB_ACTIONS:-${GITEA_ACTIONS:-}} == true && $# == 0 ]] || fail 'CI-only Stage operation'
|
||||
case ${BRANCH:-} in
|
||||
develop) ;;
|
||||
ci/v4-stage-acceptance-timeout-860324e)
|
||||
[[ ${DEPLOY_SHA:-} == 860324e3a91b31c2b6d4e784e458f6d120582cb8 && ${STAGE_TEST_SESSION:-} == v4-ci-timeout-860324e-20261024 && ${STAGE_TEST_TTL_SECONDS:-} == 600 ]] || fail 'fixed acceptance scope required'
|
||||
ci/v4-stage-resume-1a00-860324e)
|
||||
[[ ${DEPLOY_SHA:-} == 860324e3a91b31c2b6d4e784e458f6d120582cb8 && ${STAGE_TEST_SESSION:-} == v4-ci-resume-1a00-20261024 && ${STAGE_TEST_TTL_SECONDS:-} == 600 ]] || fail 'fixed acceptance scope required'
|
||||
;;
|
||||
*) fail 'Stage branch required; Production is blocked' ;;
|
||||
esac
|
||||
[[ ${STAGE_TEST_ACTION:-} != open || ${BRANCH:-} != ci/v4-stage-acceptance-timeout-860324e || ${GITHUB_RUN_ATTEMPT:-} == 1 ]] || fail 'acceptance reruns blocked'
|
||||
[[ ${STAGE_TEST_ACTION:-} != open || ${BRANCH:-} != ci/v4-stage-resume-1a00-860324e || ${GITHUB_RUN_ATTEMPT:-} == 1 ]] || fail 'acceptance reruns blocked'
|
||||
[[ ${DEPLOY_SHA:-} =~ ^[0-9a-f]{40}$ ]] || fail 'immutable DEPLOY_SHA required'
|
||||
# Repository secret DEPLOY_KEY is the historical approved devops SSH credential.
|
||||
# Never fall back to baked-in runner keys, agent forwarding or docker.sock.
|
||||
@@ -28,7 +28,7 @@ ssh-keygen -F '[87.248.133.138]:2234' -f "$scratch/known_hosts" >/dev/null || fa
|
||||
[[ ${STAGE_TEST_SESSION:-} =~ ^[a-z0-9-]{8,64}$ && ${STAGE_TEST_TTL_SECONDS:-} =~ ^[0-9]+$ && ${STAGE_TEST_ACTION:-} =~ ^(open|close|accept)$ && ${STAGE_TEST_APPROVAL:-} == owner-authorized-stage-editorial ]] || fail 'explicit owner Stage test grant required'
|
||||
[[ $STAGE_TEST_TTL_SECONDS -ge 60 && $STAGE_TEST_TTL_SECONDS -le 3600 ]] || exit 2
|
||||
if [[ $STAGE_TEST_ACTION == accept ]]; then
|
||||
[[ $BRANCH == ci/v4-stage-acceptance-timeout-860324e && $GITHUB_RUN_ATTEMPT == 1 ]] || fail 'acceptance scope required'
|
||||
[[ $BRANCH == ci/v4-stage-resume-1a00-860324e && $GITHUB_RUN_ATTEMPT == 1 ]] || fail 'acceptance scope required'
|
||||
ssh -F /dev/null -T -p 2234 -i "$scratch/key" -o BatchMode=yes -o IdentitiesOnly=yes -o IdentityAgent=none -o StrictHostKeyChecking=yes -o UserKnownHostsFile="$scratch/known_hosts" -o GlobalKnownHostsFile=/dev/null -o ConnectTimeout=15 devops@87.248.133.138 "sudo -n timeout --signal=TERM --kill-after=10 480 python3 -" < "$(dirname "${BASH_SOURCE[0]}")/stage-editorial-acceptance.py"
|
||||
exit $?
|
||||
fi
|
||||
|
||||
@@ -11,16 +11,20 @@ import urllib.request
|
||||
import uuid
|
||||
|
||||
SHA='860324e3a91b31c2b6d4e784e458f6d120582cb8'
|
||||
SESSION='v4-ci-timeout-860324e-20261024'
|
||||
REQUEST='6e82f843-45e4-49e3-9164-11a6879e39ba'
|
||||
SESSION='v4-ci-resume-1a00-20261024'
|
||||
TASK='1a00bc08-3ca4-4519-ae1e-9eec7b098756'
|
||||
|
||||
def proposal(prior):
|
||||
if prior.get('state')!='FAILED' or prior.get('version')!=5: raise ValueError('Prior must remain FAILED v5')
|
||||
scope=prior['metadata']['scope']
|
||||
prefix='editorial-research-to-brief-v1\nTopic: '
|
||||
suffix='\nResearch and evidence-backed brief only. No publication, release, production changes or execution authorization.'
|
||||
if not scope.startswith(prefix) or not scope.endswith(suffix): raise ValueError('Exact prior scope required')
|
||||
return dict(requestId=REQUEST,pilotKind='editorial-research-to-brief-v1',operationMode='PREPARE',environment='staging',scope='EDITORIAL_RESEARCH_TO_BRIEF',researchTopic=scope[len(prefix):-len(suffix)],acceptanceCriteria=prior['metadata']['acceptanceCriteria'],commitSha=SHA,budgetLimitCents=20)
|
||||
def renew_existing(a):
|
||||
route='tasks/'+TASK
|
||||
row=a(route)
|
||||
if row['id']!=TASK or row['state']!='READY' or row['runs'] or row['budgetLedger'] or row['budgetLimitCents']!=20 or row['metadata']['commitSha']!=SHA:
|
||||
raise ValueError('Only exact unstarted zero-reservation READY task allowed')
|
||||
approved=a(route+'/gates/1/decide',{'decision':'APPROVED','expectedVersion':row['version'],'idempotencyKey':SESSION+':renew-gate1','expiresAt':time.strftime('%Y-%m-%dT%H:%M:%SZ',time.gmtime(time.time()+450)),'notes':'Owner-authorized continuation of existing unstarted task only; max20 existing budget; no QA/publication/Production authority.'})
|
||||
if approved['state']!='READY': raise ValueError('Gate1 must enqueue READY')
|
||||
refreshed=a(route)
|
||||
if refreshed['state']!='READY' or refreshed['version']!=approved['version'] or refreshed['runs'] or refreshed['budgetLedger']:
|
||||
raise ValueError('Renewal refresh mismatch')
|
||||
return TASK
|
||||
|
||||
def command(args,timeout=15):
|
||||
return subprocess.check_output(args,stderr=subprocess.DEVNULL,timeout=timeout)
|
||||
@@ -67,20 +71,13 @@ def main():
|
||||
a=lambda route,body=None,method=None:http(base,admin,'admin/orchestration/'+route,body,method)
|
||||
w=lambda route,body=None:http(base,worker,'internal/prepare-worker/'+route,body)
|
||||
tasks=a('tasks')
|
||||
prior=[t for t in tasks if t['id'].startswith('4a109')]
|
||||
if len(prior)!=1: raise ValueError('Exact prior task missing')
|
||||
prior=a('tasks/'+prior[0]['id'])
|
||||
# Preserve all old unknown costs and fence previous failed attempts.
|
||||
failed=[t for t in tasks if t.get('key','').startswith('editorial-v1:') and t['state']=='FAILED']
|
||||
frozen={t['id']:a('tasks/'+t['id'])['budgetLedger'] for t in failed}
|
||||
unknown=sum(sum(x['amountCents'] for x in ledger) for ledger in frozen.values())
|
||||
if unknown!=60: raise ValueError('Prior unknown reservation must be 60')
|
||||
p=w('editorial-proposals',proposal(prior))
|
||||
if p['state']!='GATE1_PENDING' or p['version']!=2: raise ValueError('Fresh proposal only; never replay execution')
|
||||
task=p['taskId']; route='tasks/'+task
|
||||
approved=a(route+'/gates/1/decide',{'decision':'APPROVED','expectedVersion':p['version'],'idempotencyKey':SESSION+':gate1','expiresAt':time.strftime('%Y-%m-%dT%H:%M:%SZ',time.gmtime(time.time()+450)),'notes':'Explicit owner-authorized bounded Stage acceptance only; no QA/publication/Production authority.'})
|
||||
# Gate1 approval atomically queues PREPARE tasks; no second queue request.
|
||||
if approved['state']!='READY': raise ValueError('Gate1 must enqueue READY')
|
||||
task=renew_existing(a); route='tasks/'+task
|
||||
# Existing Gate1 renewal atomically retains READY; no queue or proposal request.
|
||||
# Exactly one process, as hermes. Installed runner has 120-second profile cap,
|
||||
# CLI headroom30 and allowlisted timing evidence. Total330 fits window480.
|
||||
code="import sys,json;sys.path.insert(0,'/opt/data/editorial-v4/runtime-package');import runner;from pathlib import Path;runner.run(Path('/opt/data'),sys.argv[1],total_seconds=330)"
|
||||
|
||||
@@ -8,7 +8,7 @@ ROOT = pathlib.Path(__file__).resolve().parent
|
||||
SHA = '860324e3a91b31c2b6d4e784e458f6d120582cb8'
|
||||
|
||||
class AcceptanceTests(unittest.TestCase):
|
||||
def run_driver(self, branch='ci/v4-stage-acceptance-timeout-860324e', fail_open=False, attempt='1', fail_accept=False):
|
||||
def run_driver(self, branch='ci/v4-stage-resume-1a00-860324e', fail_open=False, attempt='1', fail_accept=False):
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
p = pathlib.Path(d)
|
||||
(p/'ssh').write_text('#!/bin/bash\nprintf "%s\\n" "${@: -1}" >> "$CALLS"\n[[ "$FAIL_OPEN" != true || "${@: -1}" != *" open" ]] || exit 1\n[[ "$FAIL_ACCEPT" != true || "${@: -1}" != *"480 python3 -" ]]\n')
|
||||
@@ -28,7 +28,7 @@ class AcceptanceTests(unittest.TestCase):
|
||||
def test_fixed_stage_open_then_finally_close(self):
|
||||
result,calls = self.run_driver()
|
||||
self.assertEqual(result.returncode,0,result.stderr)
|
||||
prefix = 'sudo -n env CI=true STAGE_TEST_APPROVAL=owner-authorized-stage-editorial bash /opt/services/canina/releases/'+SHA+'/scripts/stage-test-principal-host.sh '+SHA+' v4-ci-timeout-860324e-20261024 600 '
|
||||
prefix = 'sudo -n env CI=true STAGE_TEST_APPROVAL=owner-authorized-stage-editorial bash /opt/services/canina/releases/'+SHA+'/scripts/stage-test-principal-host.sh '+SHA+' v4-ci-resume-1a00-20261024 600 '
|
||||
self.assertEqual(calls[0],prefix+'open')
|
||||
self.assertIn('timeout --signal=TERM --kill-after=10 480 python3 -',calls[1])
|
||||
self.assertEqual(calls[2],prefix+'close')
|
||||
@@ -59,10 +59,10 @@ class AcceptanceTests(unittest.TestCase):
|
||||
text = (ROOT/'ci-stage-test-principal.sh').read_text()
|
||||
for value in ('StrictHostKeyChecking=yes','IdentitiesOnly=yes','IdentityAgent=none','GlobalKnownHostsFile=/dev/null'):
|
||||
self.assertIn(value,text)
|
||||
for branch,sha,session,ttl in [('main',SHA,'v4-ci-timeout-860324e-20261024','600'),
|
||||
('ci/v4-stage-acceptance-timeout-860324e','b'*40,'v4-ci-timeout-860324e-20261024','600'),
|
||||
('ci/v4-stage-acceptance-timeout-860324e',SHA,'different-session','600'),
|
||||
('ci/v4-stage-acceptance-timeout-860324e',SHA,'v4-ci-timeout-860324e-20261024','601')]:
|
||||
for branch,sha,session,ttl in [('main',SHA,'v4-ci-resume-1a00-20261024','600'),
|
||||
('ci/v4-stage-resume-1a00-860324e','b'*40,'v4-ci-resume-1a00-20261024','600'),
|
||||
('ci/v4-stage-resume-1a00-860324e',SHA,'different-session','600'),
|
||||
('ci/v4-stage-resume-1a00-860324e',SHA,'v4-ci-resume-1a00-20261024','601')]:
|
||||
env = dict(os.environ,GITHUB_ACTIONS='true',BRANCH=branch,DEPLOY_SHA=sha,
|
||||
STAGE_TEST_SESSION=session,STAGE_TEST_TTL_SECONDS=ttl,STAGE_TEST_ACTION='open')
|
||||
result = subprocess.run(['bash',str(ROOT/'ci-stage-test-principal.sh')],env=env,capture_output=True)
|
||||
|
||||
@@ -14,17 +14,32 @@ class AcceptanceTests(unittest.TestCase):
|
||||
self.assertIn("approved['state']!='READY'",source)
|
||||
self.assertNotIn("approved['state']!='GATE1_APPROVED'",source)
|
||||
self.assertNotIn("w('editorial-queue'",source)
|
||||
def test_prior_contract_copied_exactly(self):
|
||||
prior={'state':'FAILED','version':5,'title':'Editorial research: Persian owner topic','metadata':{'scope':'editorial-research-to-brief-v1\nTopic: Persian owner topic\nResearch and evidence-backed brief only. No publication, release, production changes or execution authorization.','acceptanceCriteria':['Do not invent QA']}}
|
||||
self.assertTrue(hasattr(m,'proposal'), 'missing bounded proposal builder')
|
||||
body=m.proposal(prior)
|
||||
self.assertEqual(body['researchTopic'],'Persian owner topic')
|
||||
self.assertEqual(body['acceptanceCriteria'],prior['metadata']['acceptanceCriteria'])
|
||||
self.assertEqual(body['budgetLimitCents'],20)
|
||||
self.assertEqual(body['commitSha'],m.SHA)
|
||||
def test_wrong_prior_state_denied(self):
|
||||
self.assertTrue(hasattr(m,'proposal'), 'missing bounded proposal builder')
|
||||
with self.assertRaises(ValueError): m.proposal({'state':'READY'})
|
||||
def test_no_new_proposal_or_budget(self):
|
||||
source=p.read_text()
|
||||
self.assertNotIn("w('editorial-proposals',",source)
|
||||
self.assertNotIn('def proposal(',source)
|
||||
self.assertIn("row['budgetLedger']",source)
|
||||
|
||||
class ContinuationTests(unittest.TestCase):
|
||||
def test_renew_existing_ready_refreshes_version_and_never_proposes(self):
|
||||
task='1a00bc08-3ca4-4519-ae1e-9eec7b098756'
|
||||
calls=[]
|
||||
row={'id':task,'state':'READY','version':3,'runs':[],'budgetLedger':[],'budgetLimitCents':20,'metadata':{'commitSha':m.SHA}}
|
||||
def api(route,body=None):
|
||||
calls.append((route,body))
|
||||
if body is None: return row.copy()
|
||||
self.assertEqual(body['expectedVersion'],3)
|
||||
self.assertEqual(body['decision'],'APPROVED')
|
||||
self.assertIn('renew',body['idempotencyKey'])
|
||||
row['version']=4
|
||||
return {'state':'READY','version':4}
|
||||
self.assertTrue(hasattr(m,'renew_existing'),'missing existing READY renewal')
|
||||
self.assertEqual(m.renew_existing(api),task)
|
||||
self.assertEqual([r for r,b in calls],['tasks/'+task,'tasks/'+task+'/gates/1/decide','tasks/'+task])
|
||||
def test_started_task_is_denied_before_approval(self):
|
||||
row={'id':'1a00bc08-3ca4-4519-ae1e-9eec7b098756','state':'READY','version':3,'runs':[{'id':'started'}],'budgetLedger':[],'budgetLimitCents':20,'metadata':{'commitSha':m.SHA}}
|
||||
self.assertTrue(hasattr(m,'renew_existing'),'missing existing READY renewal')
|
||||
with self.assertRaises(ValueError): m.renew_existing(lambda route:row)
|
||||
|
||||
class ArtifactTests(unittest.TestCase):
|
||||
def test_digest_and_binding_must_match(self):
|
||||
|
||||
Reference in new issue
Block a user