feat(orchestration): implement atomic claims, root task selection and truthful completion gates

This commit is contained in:
devops committed 2026-10-20 19:42:53 +00:00
1 parent ee90284129
commit 344f79039f
3 files changed
+184 -139

No files matched your search

+117 -137
View File
@@ -12,6 +12,8 @@ import json
import os import os
import re import re
import subprocess import subprocess
import uuid
from dataclasses import dataclass
from datetime import datetime, timezone from datetime import datetime, timezone
from pathlib import Path from pathlib import Path
from typing import Any from typing import Any
@@ -85,11 +87,84 @@ def build_child_tasks(parent_code: str, proposal: dict[str, Any], depth: int) ->
def determine_parent_status(*, has_children: bool, deliverable_url: str | None, verified: bool) -> str: def determine_parent_status(*, has_children: bool, deliverable_url: str | None, verified: bool) -> str:
if has_children: if has_children:
return "WAITING_DEPENDENCIES" return "WAITING_DEPENDENCIES"
if deliverable_url and verified: # A model URL and its self-assessed verification cannot satisfy independent QA.
return "DONE"
return "TEST" return "TEST"
@dataclass(frozen=True)
class QAResult:
passed: bool
reason: str
def verify_artifact(deliverable: dict[str, Any], verification: dict[str, Any]) -> QAResult:
"""Offline gate: claimed URLs and model assertions are not artifact evidence."""
return QAResult(False, "Independent artifact QA and approval required")
def _literal(value: Any) -> str:
return "'" + str(value).replace("'", "''") + "'"
def claim_sql(token: str) -> str:
"""Select and lease one root or child in a single locked statement."""
event = json.dumps({"status": "CLAIMED", "claim_token": token})
return f"""
WITH candidate AS (
SELECT id FROM department_tasks
WHERE status = 'TODO' OR (
status = 'IN_PROGRESS' AND execution_logs->-1->>'status' = 'CLAIMED'
AND updated_at < NOW() - INTERVAL '15 minutes'
)
ORDER BY CASE priority WHEN 'CEO_EXPEDITE' THEN 0 WHEN 'CRITICAL' THEN 1
WHEN 'HIGH' THEN 2 WHEN 'MEDIUM' THEN 3 ELSE 4 END,
updated_at ASC, id ASC
FOR UPDATE SKIP LOCKED LIMIT 1
), claimed AS (
UPDATE department_tasks t SET status = 'IN_PROGRESS', updated_at = NOW(),
execution_logs = COALESCE(t.execution_logs, '[]'::jsonb) || {_literal(event)}::jsonb
FROM candidate WHERE t.id = candidate.id
RETURNING t.id, t.code, t.department, t.title, t.description,
t.acceptance_criteria, t.execution_logs, t.assignee, t.priority
) SELECT COALESCE(json_agg(claimed), '[]'::json)::text FROM claimed;
"""
def complete_sql(task_id: str, token: str, status: str, event: dict[str, Any],
children: list[dict[str, Any]] | None = None) -> str:
"""Fenced transition and idempotent child insert in one atomic SQL statement."""
if status not in ('TODO', 'TEST', 'WAITING_DEPENDENCIES', 'REJECTED'):
raise ValueError('Invalid worker transition')
entries = []
for child in children or []:
entries.append('(' + ', '.join(_literal(x) for x in (
child['code'], child['department'], child['title'], child['description'],
child['priority'], child['assignee'], json.dumps(child['acceptanceCriteria'], ensure_ascii=False)
)) + ')')
insert = ''
if entries:
insert = f""", inserted AS (
INSERT INTO department_tasks (id, code, department, title, description,
status, priority, assignee, acceptance_criteria, execution_logs, created_at, updated_at)
SELECT gen_random_uuid(), v.code, v.department, v.title, v.description,
'TODO', v.priority, v.assignee, v.criteria::jsonb, '[]'::jsonb, NOW(), NOW()
FROM (VALUES {', '.join(entries)}) AS v(code, department, title, description, priority, assignee, criteria)
WHERE EXISTS (SELECT 1 FROM completed)
ON CONFLICT (code) DO NOTHING RETURNING id
)"""
return f"""
WITH completed AS (
UPDATE department_tasks SET status = {_literal(status)}, updated_at = NOW(),
execution_logs = COALESCE(execution_logs, '[]'::jsonb) || {_literal(json.dumps(event, ensure_ascii=False))}::jsonb
WHERE id = {_literal(task_id)}::uuid AND status = 'IN_PROGRESS'
AND updated_at >= NOW() - INTERVAL '15 minutes'
AND execution_logs->-1->>'claim_token' = {_literal(token)}
RETURNING id
){insert}
SELECT id FROM completed;
"""
class Database: class Database:
def __init__(self) -> None: def __init__(self) -> None:
self.ssh = ["ssh", "-F", "/opt/data/.ssh/config", "parsa-host"] self.ssh = ["ssh", "-F", "/opt/data/.ssh/config", "parsa-host"]
@@ -99,6 +174,8 @@ class Database:
] ]
def sql(self, statement: str) -> str: def sql(self, statement: str) -> str:
if os.environ.get('CANINA_ORCHESTRATION_LIVE_WRITE') != 'I_ACCEPT_LIVE_MUTATION':
raise RuntimeError('Live database writes disabled; use injected mock database for dry runs')
result = subprocess.run( result = subprocess.run(
self.ssh + self.psql, self.ssh + self.psql,
input=statement, input=statement,
@@ -114,42 +191,16 @@ class Database:
raw = self.sql(wrapped) raw = self.sql(wrapped)
return json.loads(raw or "[]") return json.loads(raw or "[]")
def append_event(self, task_id: str, event: dict[str, Any], *, status: str | None = None) -> None: def claim_next(self) -> dict[str, Any] | None:
event_json = json.dumps(event, ensure_ascii=False).replace("$event$", "") token = str(uuid.uuid4())
status_sql = f", status = '{status}'" if status else "" rows = json.loads(self.sql(claim_sql(token)) or '[]')
self.sql( if not rows:
f""" return None
UPDATE department_tasks return {**rows[0], 'claim_token': token}
SET execution_logs = COALESCE(execution_logs, '[]'::jsonb) || $event${event_json}$event$::jsonb,
updated_at = NOW(){status_sql}
WHERE id = '{task_id}';
"""
)
def insert_children(self, parent: dict[str, Any], children: list[dict[str, Any]]) -> int:
inserted = 0
for child in children:
criteria = json.dumps(child["acceptanceCriteria"], ensure_ascii=False).replace("$criteria$", "")
title = child["title"].replace("'", "''")
description = child["description"].replace("'", "''")
assignee = child["assignee"].replace("'", "''")
out = self.sql(
f"""
INSERT INTO department_tasks
(id, code, department, title, description, status, priority, assignee,
acceptance_criteria, execution_logs, created_at, updated_at)
VALUES
(gen_random_uuid(), '{child['code']}', '{child['department']}', '{title}',
'{description}', 'TODO', '{child['priority']}', '{assignee}',
$criteria${criteria}$criteria$::jsonb, '[]'::jsonb, NOW(), NOW())
ON CONFLICT (code) DO NOTHING
RETURNING code;
"""
)
if out:
inserted += 1
return inserted
def finish_claim(self, task: dict[str, Any], event: dict[str, Any], status: str,
children: list[dict[str, Any]] | None = None) -> bool:
return bool(self.sql(complete_sql(task['id'], task['claim_token'], status, event, children)))
class ModelClient: class ModelClient:
def __init__(self) -> None: def __init__(self) -> None:
@@ -244,67 +295,44 @@ def execution_prompt(task: dict[str, Any]) -> str:
""".strip() """.strip()
def execute_next_child(db: Database, model: ModelClient) -> bool: def execute_next_task(db: Database, model: ModelClient) -> bool:
tasks = db.rows_json( """Produce reviewable drafts; never publish or trust model QA assertions."""
""" task = db.claim_next()
SELECT id, code, department, title, description, status, acceptance_criteria, if not task:
execution_logs, deliverable_url
FROM department_tasks
WHERE status = 'TODO'
AND acceptance_criteria->>'parentTaskCode' IS NOT NULL
ORDER BY created_at ASC
LIMIT 1
"""
)
if not tasks:
return False return False
task = tasks[0]
prompt = execution_prompt(task) prompt = execution_prompt(task)
db.append_event(task["id"], {
"timestamp": utc_now(), "status": "AGENT_EXECUTION_STARTED",
"agent": task.get("assignee") or f"Canina-{task['department']}-Agent",
"prompt_to_model": prompt,
}, status="IN_PROGRESS")
try: try:
raw = model.complete(prompt) raw = model.complete(prompt)
result = parse_structured_response(raw) result = parse_structured_response(raw)
criteria = task.get('acceptance_criteria') or {}
children = build_child_tasks(task['code'], result, int(criteria.get('depth', 0)))
deliverable = result.get('deliverable') or {}
qa = verify_artifact(deliverable, result.get('verification') or {})
status = determine_parent_status(has_children=bool(children),
deliverable_url=deliverable.get('url'), verified=qa.passed)
event = {'timestamp': utc_now(), 'status': 'QA_PENDING', 'prompt_to_model': prompt,
'model_response': raw, 'deliverable': deliverable,
'qa': qa.reason, 'childrenProposed': len(children)}
if not db.finish_claim(task, event, status, children):
raise RuntimeError('Claim lost before completion; no result committed')
except Exception as exc: except Exception as exc:
db.append_event(task["id"], { db.finish_claim(task, {'timestamp': utc_now(), 'status': 'AGENT_EXECUTION_FAILED',
"timestamp": utc_now(), "status": "AGENT_EXECUTION_FAILED", 'error': str(exc)}, 'TODO')
"error": str(exc),
}, status="TODO")
raise raise
criteria = task.get("acceptance_criteria") or {}
depth = int(criteria.get("depth", 1))
children = build_child_tasks(task["code"], result, depth)
inserted = db.insert_children(task, children)
deliverable = result.get("deliverable") or {}
verification = result.get("verification") or {}
verified = bool(deliverable.get("verified")) and bool(verification.get("passed"))
next_status = determine_parent_status(
has_children=bool(children),
deliverable_url=deliverable.get("url"),
verified=verified,
)
db.append_event(task["id"], {
"timestamp": utc_now(), "status": "AGENT_EXECUTION_COMPLETED",
"agent": task.get("assignee") or f"Canina-{task['department']}-Agent",
"model_response": raw,
"summary": result.get("summary"),
"deliverable": deliverable,
"verification": verification,
"childrenCreated": inserted,
}, status=next_status)
return True return True
execute_next_child = execute_next_task
def reconcile_parent_states(db: Database) -> bool: def reconcile_parent_states(db: Database) -> bool:
parents = db.rows_json( parents = db.rows_json(
""" """
SELECT id, code, status SELECT id, code, status
FROM department_tasks p FROM department_tasks p
WHERE status = 'WAITING_DEPENDENCIES' WHERE status = 'WAITING_DEPENDENCIES'
AND EXISTS (SELECT 1 FROM department_tasks c
WHERE c.acceptance_criteria->>'parentTaskCode' = p.code)
AND NOT EXISTS ( AND NOT EXISTS (
SELECT 1 FROM department_tasks c SELECT 1 FROM department_tasks c
WHERE c.acceptance_criteria->>'parentTaskCode' = p.code WHERE c.acceptance_criteria->>'parentTaskCode' = p.code
@@ -316,71 +344,23 @@ def reconcile_parent_states(db: Database) -> bool:
) )
if not parents: if not parents:
return False return False
parent = parents[0] # Dependency completion alone cannot pass parent QA. Leave it blocked pending
db.append_event(parent["id"], { # independently persisted attestations; no synthetic URL or DONE transition.
"timestamp": utc_now(), "status": "DEPENDENCIES_COMPLETED", return False
"agent": "Canina-PMO-Agent",
"note": "تمام child taskها تکمیل و راستی‌آزمایی شده‌اند؛ والد وارد QA نهایی شد.",
}, status="TEST")
return True
def reconcile_pseudo_done(db: Database, model: ModelClient) -> bool: def reconcile_pseudo_done(db: Database, model: ModelClient) -> bool:
tasks = db.rows_json( # Legacy DONE records require a separate human audit, not autonomous rewrites.
""" return False
SELECT id, code, department, title, description, status, acceptance_criteria,
execution_logs, deliverable_url
FROM department_tasks
WHERE status = 'DONE'
AND code LIKE 'TSK-%'
AND (deliverable_url IS NULL OR deliverable_url = '')
AND NOT EXISTS (
SELECT 1 FROM department_tasks child
WHERE child.acceptance_criteria->>'parentTaskCode' = department_tasks.code
)
ORDER BY created_at ASC
LIMIT 1
"""
)
if not tasks:
return False
task = tasks[0]
prompt = decomposition_prompt(task)
db.append_event(task["id"], {
"timestamp": utc_now(), "status": "PMO_DECOMPOSITION_STARTED",
"agent": "Canina-PMO-Agent", "prompt_to_model": prompt,
}, status="IN_PROGRESS")
raw = model.complete(prompt)
proposal = parse_structured_response(raw)
depth = int((task.get("acceptance_criteria") or {}).get("depth", 0))
children = build_child_tasks(task["code"], proposal, depth)
inserted = db.insert_children(task, children)
deliverable = proposal.get("deliverable") or {}
next_status = determine_parent_status(
has_children=bool(children),
deliverable_url=deliverable.get("url"),
verified=bool(deliverable.get("verified")),
)
db.append_event(task["id"], {
"timestamp": utc_now(), "status": "PMO_DECOMPOSITION_COMPLETED",
"agent": "Canina-PMO-Agent", "model_response": raw,
"summary": proposal.get("summary"), "childrenCreated": inserted,
}, status=next_status)
return True
def main() -> None: def main() -> None:
db = Database() db = Database()
model = ModelClient() model = ModelClient()
if reconcile_pseudo_done(db, model): if execute_next_task(db, model):
print("Reconciled one pseudo-DONE root task into cross-department dependencies.")
return
if execute_next_child(db, model):
print("Executed one child task; unverified model output remains in TEST, never DONE.") print("Executed one child task; unverified model output remains in TEST, never DONE.")
return return
if reconcile_parent_states(db): # Parent reconciliation requires independent persisted QA attestations.
print("Moved one parent with verified completed children into TEST.")
return
print("No actionable orchestration task found.") print("No actionable orchestration task found.")
@@ -0,0 +1,65 @@
import json
import unittest
from unittest.mock import Mock
from orchestration_worker import (
claim_sql, complete_sql, execute_next_task, reconcile_parent_states,
determine_parent_status, verify_artifact,
)
class ContractTests(unittest.TestCase):
def test_claim_is_atomic_priority_first_and_includes_roots(self):
sql = claim_sql("token-1")
self.assertIn("FOR UPDATE SKIP LOCKED", sql)
self.assertIn("CEO_EXPEDITE", sql)
self.assertIn("updated_at ASC", sql)
self.assertNotIn("parentTaskCode' IS NOT NULL", sql)
self.assertIn("RETURNING", sql)
def test_completion_is_fenced_by_claim_token(self):
sql = complete_sql("task-id", "token-1", "TEST", {"status": "QA_PENDING"})
self.assertIn("IN_PROGRESS", sql)
self.assertIn("token-1", sql)
self.assertIn("RETURNING", sql)
self.assertIn("claim_token", sql)
self.assertIn("updated_at >= NOW() - INTERVAL '15 minutes'", sql)
def test_model_verified_url_cannot_mark_done(self):
self.assertEqual(determine_parent_status(has_children=False, deliverable_url="https://canina.ir/fake", verified=True), "TEST")
def test_independent_qa_rejects_unverified_claimed_url(self):
self.assertFalse(verify_artifact({"url": "https://canina.ir/fake", "verified": True}, {"passed": True}).passed)
def test_unapproved_content_publish_is_not_attempted(self):
db = Mock()
db.claim_next.return_value = {"id": "1", "code": "CEO-2", "department": "CONTENT", "title": "Draft", "description": "Do", "acceptance_criteria": {}, "claim_token": "abc"}
db.finish_claim.return_value = True
model = Mock()
model.complete.return_value = json.dumps({"deliverable": {"type": "draft", "content": "clinical " * 200, "url": "https://canina.ir/blog/fake", "verified": True}, "verification": {"passed": True}})
self.assertTrue(execute_next_task(db, model))
self.assertEqual(db.finish_claim.call_args.args[2], "TEST")
self.assertNotIn("published", db.finish_claim.call_args.args[1])
def test_root_executes_without_live_side_effects(self):
db = Mock()
db.claim_next.return_value = {"id": "1", "code": "CEO-1", "department": "PMO", "title": "Plan", "description": "Do", "acceptance_criteria": {}, "claim_token": "abc"}
db.finish_claim.return_value = True
model = Mock()
model.complete.return_value = json.dumps({"deliverable": {"url": "https://canina.ir/fake", "verified": True}, "verification": {"passed": True}, "child_tasks": []})
self.assertTrue(execute_next_task(db, model))
db.finish_claim.assert_called_once()
self.assertEqual(db.finish_claim.call_args.args[2], "TEST")
db.insert_children.assert_not_called()
def test_rejected_child_never_unblocks_parent(self):
db = Mock()
db.rows_json.return_value = []
self.assertFalse(reconcile_parent_states(db))
# REJECTED is not DONE, therefore the anti-join still blocks the parent.
self.assertIn("NOT IN ('DONE')", db.rows_json.call_args.args[0])
self.assertNotIn("NOT IN ('DONE', 'REJECTED')", db.rows_json.call_args.args[0])
if __name__ == "__main__":
unittest.main()
@@ -19,10 +19,10 @@ class OrchestrationWorkerTests(unittest.TestCase):
"TEST", "TEST",
) )
def test_verified_live_deliverable_may_be_done(self): def test_model_claimed_live_deliverable_still_requires_independent_qa(self):
self.assertEqual( self.assertEqual(
determine_parent_status(has_children=False, deliverable_url="https://canina.ir/x", verified=True), determine_parent_status(has_children=False, deliverable_url="https://canina.ir/x", verified=True),
"DONE", "TEST",
) )
def test_children_force_waiting_dependencies(self): def test_children_force_waiting_dependencies(self):