feat(orchestration): implement atomic claims, root task selection and truthful completion gates

This commit is contained in:
devops committed 2026-10-20 19:42:53 +00:00
1 parent ee90284129
commit 344f79039f
3 files changed
+184 -139

No files matched your search

+117 -137
View File
@@ -12,6 +12,8 @@ import json
import os
import re
import subprocess
import uuid
from dataclasses import dataclass
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
@@ -85,11 +87,84 @@ def build_child_tasks(parent_code: str, proposal: dict[str, Any], depth: int) ->
def determine_parent_status(*, has_children: bool, deliverable_url: str | None, verified: bool) -> str:
if has_children:
return "WAITING_DEPENDENCIES"
if deliverable_url and verified:
return "DONE"
# A model URL and its self-assessed verification cannot satisfy independent QA.
return "TEST"
@dataclass(frozen=True)
class QAResult:
passed: bool
reason: str
def verify_artifact(deliverable: dict[str, Any], verification: dict[str, Any]) -> QAResult:
"""Offline gate: claimed URLs and model assertions are not artifact evidence."""
return QAResult(False, "Independent artifact QA and approval required")
def _literal(value: Any) -> str:
return "'" + str(value).replace("'", "''") + "'"
def claim_sql(token: str) -> str:
"""Select and lease one root or child in a single locked statement."""
event = json.dumps({"status": "CLAIMED", "claim_token": token})
return f"""
WITH candidate AS (
SELECT id FROM department_tasks
WHERE status = 'TODO' OR (
status = 'IN_PROGRESS' AND execution_logs->-1->>'status' = 'CLAIMED'
AND updated_at < NOW() - INTERVAL '15 minutes'
)
ORDER BY CASE priority WHEN 'CEO_EXPEDITE' THEN 0 WHEN 'CRITICAL' THEN 1
WHEN 'HIGH' THEN 2 WHEN 'MEDIUM' THEN 3 ELSE 4 END,
updated_at ASC, id ASC
FOR UPDATE SKIP LOCKED LIMIT 1
), claimed AS (
UPDATE department_tasks t SET status = 'IN_PROGRESS', updated_at = NOW(),
execution_logs = COALESCE(t.execution_logs, '[]'::jsonb) || {_literal(event)}::jsonb
FROM candidate WHERE t.id = candidate.id
RETURNING t.id, t.code, t.department, t.title, t.description,
t.acceptance_criteria, t.execution_logs, t.assignee, t.priority
) SELECT COALESCE(json_agg(claimed), '[]'::json)::text FROM claimed;
"""
def complete_sql(task_id: str, token: str, status: str, event: dict[str, Any],
children: list[dict[str, Any]] | None = None) -> str:
"""Fenced transition and idempotent child insert in one atomic SQL statement."""
if status not in ('TODO', 'TEST', 'WAITING_DEPENDENCIES', 'REJECTED'):
raise ValueError('Invalid worker transition')
entries = []
for child in children or []:
entries.append('(' + ', '.join(_literal(x) for x in (
child['code'], child['department'], child['title'], child['description'],
child['priority'], child['assignee'], json.dumps(child['acceptanceCriteria'], ensure_ascii=False)
)) + ')')
insert = ''
if entries:
insert = f""", inserted AS (
INSERT INTO department_tasks (id, code, department, title, description,
status, priority, assignee, acceptance_criteria, execution_logs, created_at, updated_at)
SELECT gen_random_uuid(), v.code, v.department, v.title, v.description,
'TODO', v.priority, v.assignee, v.criteria::jsonb, '[]'::jsonb, NOW(), NOW()
FROM (VALUES {', '.join(entries)}) AS v(code, department, title, description, priority, assignee, criteria)
WHERE EXISTS (SELECT 1 FROM completed)
ON CONFLICT (code) DO NOTHING RETURNING id
)"""
return f"""
WITH completed AS (
UPDATE department_tasks SET status = {_literal(status)}, updated_at = NOW(),
execution_logs = COALESCE(execution_logs, '[]'::jsonb) || {_literal(json.dumps(event, ensure_ascii=False))}::jsonb
WHERE id = {_literal(task_id)}::uuid AND status = 'IN_PROGRESS'
AND updated_at >= NOW() - INTERVAL '15 minutes'
AND execution_logs->-1->>'claim_token' = {_literal(token)}
RETURNING id
){insert}
SELECT id FROM completed;
"""
class Database:
def __init__(self) -> None:
self.ssh = ["ssh", "-F", "/opt/data/.ssh/config", "parsa-host"]
@@ -99,6 +174,8 @@ class Database:
]
def sql(self, statement: str) -> str:
if os.environ.get('CANINA_ORCHESTRATION_LIVE_WRITE') != 'I_ACCEPT_LIVE_MUTATION':
raise RuntimeError('Live database writes disabled; use injected mock database for dry runs')
result = subprocess.run(
self.ssh + self.psql,
input=statement,
@@ -114,42 +191,16 @@ class Database:
raw = self.sql(wrapped)
return json.loads(raw or "[]")
def append_event(self, task_id: str, event: dict[str, Any], *, status: str | None = None) -> None:
event_json = json.dumps(event, ensure_ascii=False).replace("$event$", "")
status_sql = f", status = '{status}'" if status else ""
self.sql(
f"""
UPDATE department_tasks
SET execution_logs = COALESCE(execution_logs, '[]'::jsonb) || $event${event_json}$event$::jsonb,
updated_at = NOW(){status_sql}
WHERE id = '{task_id}';
"""
)
def insert_children(self, parent: dict[str, Any], children: list[dict[str, Any]]) -> int:
inserted = 0
for child in children:
criteria = json.dumps(child["acceptanceCriteria"], ensure_ascii=False).replace("$criteria$", "")
title = child["title"].replace("'", "''")
description = child["description"].replace("'", "''")
assignee = child["assignee"].replace("'", "''")
out = self.sql(
f"""
INSERT INTO department_tasks
(id, code, department, title, description, status, priority, assignee,
acceptance_criteria, execution_logs, created_at, updated_at)
VALUES
(gen_random_uuid(), '{child['code']}', '{child['department']}', '{title}',
'{description}', 'TODO', '{child['priority']}', '{assignee}',
$criteria${criteria}$criteria$::jsonb, '[]'::jsonb, NOW(), NOW())
ON CONFLICT (code) DO NOTHING
RETURNING code;
"""
)
if out:
inserted += 1
return inserted
def claim_next(self) -> dict[str, Any] | None:
token = str(uuid.uuid4())
rows = json.loads(self.sql(claim_sql(token)) or '[]')
if not rows:
return None
return {**rows[0], 'claim_token': token}
def finish_claim(self, task: dict[str, Any], event: dict[str, Any], status: str,
children: list[dict[str, Any]] | None = None) -> bool:
return bool(self.sql(complete_sql(task['id'], task['claim_token'], status, event, children)))
class ModelClient:
def __init__(self) -> None:
@@ -244,67 +295,44 @@ def execution_prompt(task: dict[str, Any]) -> str:
""".strip()
def execute_next_child(db: Database, model: ModelClient) -> bool:
tasks = db.rows_json(
"""
SELECT id, code, department, title, description, status, acceptance_criteria,
execution_logs, deliverable_url
FROM department_tasks
WHERE status = 'TODO'
AND acceptance_criteria->>'parentTaskCode' IS NOT NULL
ORDER BY created_at ASC
LIMIT 1
"""
)
if not tasks:
def execute_next_task(db: Database, model: ModelClient) -> bool:
"""Produce reviewable drafts; never publish or trust model QA assertions."""
task = db.claim_next()
if not task:
return False
task = tasks[0]
prompt = execution_prompt(task)
db.append_event(task["id"], {
"timestamp": utc_now(), "status": "AGENT_EXECUTION_STARTED",
"agent": task.get("assignee") or f"Canina-{task['department']}-Agent",
"prompt_to_model": prompt,
}, status="IN_PROGRESS")
try:
raw = model.complete(prompt)
result = parse_structured_response(raw)
criteria = task.get('acceptance_criteria') or {}
children = build_child_tasks(task['code'], result, int(criteria.get('depth', 0)))
deliverable = result.get('deliverable') or {}
qa = verify_artifact(deliverable, result.get('verification') or {})
status = determine_parent_status(has_children=bool(children),
deliverable_url=deliverable.get('url'), verified=qa.passed)
event = {'timestamp': utc_now(), 'status': 'QA_PENDING', 'prompt_to_model': prompt,
'model_response': raw, 'deliverable': deliverable,
'qa': qa.reason, 'childrenProposed': len(children)}
if not db.finish_claim(task, event, status, children):
raise RuntimeError('Claim lost before completion; no result committed')
except Exception as exc:
db.append_event(task["id"], {
"timestamp": utc_now(), "status": "AGENT_EXECUTION_FAILED",
"error": str(exc),
}, status="TODO")
db.finish_claim(task, {'timestamp': utc_now(), 'status': 'AGENT_EXECUTION_FAILED',
'error': str(exc)}, 'TODO')
raise
criteria = task.get("acceptance_criteria") or {}
depth = int(criteria.get("depth", 1))
children = build_child_tasks(task["code"], result, depth)
inserted = db.insert_children(task, children)
deliverable = result.get("deliverable") or {}
verification = result.get("verification") or {}
verified = bool(deliverable.get("verified")) and bool(verification.get("passed"))
next_status = determine_parent_status(
has_children=bool(children),
deliverable_url=deliverable.get("url"),
verified=verified,
)
db.append_event(task["id"], {
"timestamp": utc_now(), "status": "AGENT_EXECUTION_COMPLETED",
"agent": task.get("assignee") or f"Canina-{task['department']}-Agent",
"model_response": raw,
"summary": result.get("summary"),
"deliverable": deliverable,
"verification": verification,
"childrenCreated": inserted,
}, status=next_status)
return True
execute_next_child = execute_next_task
def reconcile_parent_states(db: Database) -> bool:
parents = db.rows_json(
"""
SELECT id, code, status
FROM department_tasks p
WHERE status = 'WAITING_DEPENDENCIES'
AND EXISTS (SELECT 1 FROM department_tasks c
WHERE c.acceptance_criteria->>'parentTaskCode' = p.code)
AND NOT EXISTS (
SELECT 1 FROM department_tasks c
WHERE c.acceptance_criteria->>'parentTaskCode' = p.code
@@ -316,71 +344,23 @@ def reconcile_parent_states(db: Database) -> bool:
)
if not parents:
return False
parent = parents[0]
db.append_event(parent["id"], {
"timestamp": utc_now(), "status": "DEPENDENCIES_COMPLETED",
"agent": "Canina-PMO-Agent",
"note": "تمام child taskها تکمیل و راستی‌آزمایی شده‌اند؛ والد وارد QA نهایی شد.",
}, status="TEST")
return True
# Dependency completion alone cannot pass parent QA. Leave it blocked pending
# independently persisted attestations; no synthetic URL or DONE transition.
return False
def reconcile_pseudo_done(db: Database, model: ModelClient) -> bool:
tasks = db.rows_json(
"""
SELECT id, code, department, title, description, status, acceptance_criteria,
execution_logs, deliverable_url
FROM department_tasks
WHERE status = 'DONE'
AND code LIKE 'TSK-%'
AND (deliverable_url IS NULL OR deliverable_url = '')
AND NOT EXISTS (
SELECT 1 FROM department_tasks child
WHERE child.acceptance_criteria->>'parentTaskCode' = department_tasks.code
)
ORDER BY created_at ASC
LIMIT 1
"""
)
if not tasks:
return False
task = tasks[0]
prompt = decomposition_prompt(task)
db.append_event(task["id"], {
"timestamp": utc_now(), "status": "PMO_DECOMPOSITION_STARTED",
"agent": "Canina-PMO-Agent", "prompt_to_model": prompt,
}, status="IN_PROGRESS")
raw = model.complete(prompt)
proposal = parse_structured_response(raw)
depth = int((task.get("acceptance_criteria") or {}).get("depth", 0))
children = build_child_tasks(task["code"], proposal, depth)
inserted = db.insert_children(task, children)
deliverable = proposal.get("deliverable") or {}
next_status = determine_parent_status(
has_children=bool(children),
deliverable_url=deliverable.get("url"),
verified=bool(deliverable.get("verified")),
)
db.append_event(task["id"], {
"timestamp": utc_now(), "status": "PMO_DECOMPOSITION_COMPLETED",
"agent": "Canina-PMO-Agent", "model_response": raw,
"summary": proposal.get("summary"), "childrenCreated": inserted,
}, status=next_status)
return True
# Legacy DONE records require a separate human audit, not autonomous rewrites.
return False
def main() -> None:
db = Database()
model = ModelClient()
if reconcile_pseudo_done(db, model):
print("Reconciled one pseudo-DONE root task into cross-department dependencies.")
return
if execute_next_child(db, model):
if execute_next_task(db, model):
print("Executed one child task; unverified model output remains in TEST, never DONE.")
return
if reconcile_parent_states(db):
print("Moved one parent with verified completed children into TEST.")
return
# Parent reconciliation requires independent persisted QA attestations.
print("No actionable orchestration task found.")
@@ -0,0 +1,65 @@
import json
import unittest
from unittest.mock import Mock
from orchestration_worker import (
claim_sql, complete_sql, execute_next_task, reconcile_parent_states,
determine_parent_status, verify_artifact,
)
class ContractTests(unittest.TestCase):
def test_claim_is_atomic_priority_first_and_includes_roots(self):
sql = claim_sql("token-1")
self.assertIn("FOR UPDATE SKIP LOCKED", sql)
self.assertIn("CEO_EXPEDITE", sql)
self.assertIn("updated_at ASC", sql)
self.assertNotIn("parentTaskCode' IS NOT NULL", sql)
self.assertIn("RETURNING", sql)
def test_completion_is_fenced_by_claim_token(self):
sql = complete_sql("task-id", "token-1", "TEST", {"status": "QA_PENDING"})
self.assertIn("IN_PROGRESS", sql)
self.assertIn("token-1", sql)
self.assertIn("RETURNING", sql)
self.assertIn("claim_token", sql)
self.assertIn("updated_at >= NOW() - INTERVAL '15 minutes'", sql)
def test_model_verified_url_cannot_mark_done(self):
self.assertEqual(determine_parent_status(has_children=False, deliverable_url="https://canina.ir/fake", verified=True), "TEST")
def test_independent_qa_rejects_unverified_claimed_url(self):
self.assertFalse(verify_artifact({"url": "https://canina.ir/fake", "verified": True}, {"passed": True}).passed)
def test_unapproved_content_publish_is_not_attempted(self):
db = Mock()
db.claim_next.return_value = {"id": "1", "code": "CEO-2", "department": "CONTENT", "title": "Draft", "description": "Do", "acceptance_criteria": {}, "claim_token": "abc"}
db.finish_claim.return_value = True
model = Mock()
model.complete.return_value = json.dumps({"deliverable": {"type": "draft", "content": "clinical " * 200, "url": "https://canina.ir/blog/fake", "verified": True}, "verification": {"passed": True}})
self.assertTrue(execute_next_task(db, model))
self.assertEqual(db.finish_claim.call_args.args[2], "TEST")
self.assertNotIn("published", db.finish_claim.call_args.args[1])
def test_root_executes_without_live_side_effects(self):
db = Mock()
db.claim_next.return_value = {"id": "1", "code": "CEO-1", "department": "PMO", "title": "Plan", "description": "Do", "acceptance_criteria": {}, "claim_token": "abc"}
db.finish_claim.return_value = True
model = Mock()
model.complete.return_value = json.dumps({"deliverable": {"url": "https://canina.ir/fake", "verified": True}, "verification": {"passed": True}, "child_tasks": []})
self.assertTrue(execute_next_task(db, model))
db.finish_claim.assert_called_once()
self.assertEqual(db.finish_claim.call_args.args[2], "TEST")
db.insert_children.assert_not_called()
def test_rejected_child_never_unblocks_parent(self):
db = Mock()
db.rows_json.return_value = []
self.assertFalse(reconcile_parent_states(db))
# REJECTED is not DONE, therefore the anti-join still blocks the parent.
self.assertIn("NOT IN ('DONE')", db.rows_json.call_args.args[0])
self.assertNotIn("NOT IN ('DONE', 'REJECTED')", db.rows_json.call_args.args[0])
if __name__ == "__main__":
unittest.main()
@@ -19,10 +19,10 @@ class OrchestrationWorkerTests(unittest.TestCase):
"TEST",
)
def test_verified_live_deliverable_may_be_done(self):
def test_model_claimed_live_deliverable_still_requires_independent_qa(self):
self.assertEqual(
determine_parent_status(has_children=False, deliverable_url="https://canina.ir/x", verified=True),
"DONE",
"TEST",
)
def test_children_force_waiting_dependencies(self):