From 344f79039f65678bdf1ecd2c53f9fa00a0189d0e Mon Sep 17 00:00:00 2001 From: devops Date: Tue, 20 Oct 2026 19:42:53 +0000 Subject: [PATCH] feat(orchestration): implement atomic claims, root task selection and truthful completion gates --- scripts/orchestration/orchestration_worker.py | 254 ++++++++---------- .../test_orchestration_contract.py | 65 +++++ .../test_orchestration_worker.py | 4 +- 3 files changed, 184 insertions(+), 139 deletions(-) create mode 100644 scripts/orchestration/test_orchestration_contract.py diff --git a/scripts/orchestration/orchestration_worker.py b/scripts/orchestration/orchestration_worker.py index f10c1b86..6f3e98e1 100644 --- a/scripts/orchestration/orchestration_worker.py +++ b/scripts/orchestration/orchestration_worker.py @@ -12,6 +12,8 @@ import json import os import re import subprocess +import uuid +from dataclasses import dataclass from datetime import datetime, timezone from pathlib import Path from typing import Any @@ -85,11 +87,84 @@ def build_child_tasks(parent_code: str, proposal: dict[str, Any], depth: int) -> def determine_parent_status(*, has_children: bool, deliverable_url: str | None, verified: bool) -> str: if has_children: return "WAITING_DEPENDENCIES" - if deliverable_url and verified: - return "DONE" + # A model URL and its self-assessed verification cannot satisfy independent QA. return "TEST" +@dataclass(frozen=True) +class QAResult: + passed: bool + reason: str + + +def verify_artifact(deliverable: dict[str, Any], verification: dict[str, Any]) -> QAResult: + """Offline gate: claimed URLs and model assertions are not artifact evidence.""" + return QAResult(False, "Independent artifact QA and approval required") + + +def _literal(value: Any) -> str: + return "'" + str(value).replace("'", "''") + "'" + + +def claim_sql(token: str) -> str: + """Select and lease one root or child in a single locked statement.""" + event = json.dumps({"status": "CLAIMED", "claim_token": token}) + return f""" + WITH candidate AS ( + SELECT id FROM department_tasks + WHERE status = 'TODO' OR ( + status = 'IN_PROGRESS' AND execution_logs->-1->>'status' = 'CLAIMED' + AND updated_at < NOW() - INTERVAL '15 minutes' + ) + ORDER BY CASE priority WHEN 'CEO_EXPEDITE' THEN 0 WHEN 'CRITICAL' THEN 1 + WHEN 'HIGH' THEN 2 WHEN 'MEDIUM' THEN 3 ELSE 4 END, + updated_at ASC, id ASC + FOR UPDATE SKIP LOCKED LIMIT 1 + ), claimed AS ( + UPDATE department_tasks t SET status = 'IN_PROGRESS', updated_at = NOW(), + execution_logs = COALESCE(t.execution_logs, '[]'::jsonb) || {_literal(event)}::jsonb + FROM candidate WHERE t.id = candidate.id + RETURNING t.id, t.code, t.department, t.title, t.description, + t.acceptance_criteria, t.execution_logs, t.assignee, t.priority + ) SELECT COALESCE(json_agg(claimed), '[]'::json)::text FROM claimed; + """ + + +def complete_sql(task_id: str, token: str, status: str, event: dict[str, Any], + children: list[dict[str, Any]] | None = None) -> str: + """Fenced transition and idempotent child insert in one atomic SQL statement.""" + if status not in ('TODO', 'TEST', 'WAITING_DEPENDENCIES', 'REJECTED'): + raise ValueError('Invalid worker transition') + entries = [] + for child in children or []: + entries.append('(' + ', '.join(_literal(x) for x in ( + child['code'], child['department'], child['title'], child['description'], + child['priority'], child['assignee'], json.dumps(child['acceptanceCriteria'], ensure_ascii=False) + )) + ')') + insert = '' + if entries: + insert = f""", inserted AS ( + INSERT INTO department_tasks (id, code, department, title, description, + status, priority, assignee, acceptance_criteria, execution_logs, created_at, updated_at) + SELECT gen_random_uuid(), v.code, v.department, v.title, v.description, + 'TODO', v.priority, v.assignee, v.criteria::jsonb, '[]'::jsonb, NOW(), NOW() + FROM (VALUES {', '.join(entries)}) AS v(code, department, title, description, priority, assignee, criteria) + WHERE EXISTS (SELECT 1 FROM completed) + ON CONFLICT (code) DO NOTHING RETURNING id + )""" + return f""" + WITH completed AS ( + UPDATE department_tasks SET status = {_literal(status)}, updated_at = NOW(), + execution_logs = COALESCE(execution_logs, '[]'::jsonb) || {_literal(json.dumps(event, ensure_ascii=False))}::jsonb + WHERE id = {_literal(task_id)}::uuid AND status = 'IN_PROGRESS' + AND updated_at >= NOW() - INTERVAL '15 minutes' + AND execution_logs->-1->>'claim_token' = {_literal(token)} + RETURNING id + ){insert} + SELECT id FROM completed; + """ + + class Database: def __init__(self) -> None: self.ssh = ["ssh", "-F", "/opt/data/.ssh/config", "parsa-host"] @@ -99,6 +174,8 @@ class Database: ] def sql(self, statement: str) -> str: + if os.environ.get('CANINA_ORCHESTRATION_LIVE_WRITE') != 'I_ACCEPT_LIVE_MUTATION': + raise RuntimeError('Live database writes disabled; use injected mock database for dry runs') result = subprocess.run( self.ssh + self.psql, input=statement, @@ -114,42 +191,16 @@ class Database: raw = self.sql(wrapped) return json.loads(raw or "[]") - def append_event(self, task_id: str, event: dict[str, Any], *, status: str | None = None) -> None: - event_json = json.dumps(event, ensure_ascii=False).replace("$event$", "") - status_sql = f", status = '{status}'" if status else "" - self.sql( - f""" - UPDATE department_tasks - SET execution_logs = COALESCE(execution_logs, '[]'::jsonb) || $event${event_json}$event$::jsonb, - updated_at = NOW(){status_sql} - WHERE id = '{task_id}'; - """ - ) - - def insert_children(self, parent: dict[str, Any], children: list[dict[str, Any]]) -> int: - inserted = 0 - for child in children: - criteria = json.dumps(child["acceptanceCriteria"], ensure_ascii=False).replace("$criteria$", "") - title = child["title"].replace("'", "''") - description = child["description"].replace("'", "''") - assignee = child["assignee"].replace("'", "''") - out = self.sql( - f""" - INSERT INTO department_tasks - (id, code, department, title, description, status, priority, assignee, - acceptance_criteria, execution_logs, created_at, updated_at) - VALUES - (gen_random_uuid(), '{child['code']}', '{child['department']}', '{title}', - '{description}', 'TODO', '{child['priority']}', '{assignee}', - $criteria${criteria}$criteria$::jsonb, '[]'::jsonb, NOW(), NOW()) - ON CONFLICT (code) DO NOTHING - RETURNING code; - """ - ) - if out: - inserted += 1 - return inserted + def claim_next(self) -> dict[str, Any] | None: + token = str(uuid.uuid4()) + rows = json.loads(self.sql(claim_sql(token)) or '[]') + if not rows: + return None + return {**rows[0], 'claim_token': token} + def finish_claim(self, task: dict[str, Any], event: dict[str, Any], status: str, + children: list[dict[str, Any]] | None = None) -> bool: + return bool(self.sql(complete_sql(task['id'], task['claim_token'], status, event, children))) class ModelClient: def __init__(self) -> None: @@ -244,67 +295,44 @@ def execution_prompt(task: dict[str, Any]) -> str: """.strip() -def execute_next_child(db: Database, model: ModelClient) -> bool: - tasks = db.rows_json( - """ - SELECT id, code, department, title, description, status, acceptance_criteria, - execution_logs, deliverable_url - FROM department_tasks - WHERE status = 'TODO' - AND acceptance_criteria->>'parentTaskCode' IS NOT NULL - ORDER BY created_at ASC - LIMIT 1 - """ - ) - if not tasks: +def execute_next_task(db: Database, model: ModelClient) -> bool: + """Produce reviewable drafts; never publish or trust model QA assertions.""" + task = db.claim_next() + if not task: return False - task = tasks[0] prompt = execution_prompt(task) - db.append_event(task["id"], { - "timestamp": utc_now(), "status": "AGENT_EXECUTION_STARTED", - "agent": task.get("assignee") or f"Canina-{task['department']}-Agent", - "prompt_to_model": prompt, - }, status="IN_PROGRESS") try: raw = model.complete(prompt) result = parse_structured_response(raw) + criteria = task.get('acceptance_criteria') or {} + children = build_child_tasks(task['code'], result, int(criteria.get('depth', 0))) + deliverable = result.get('deliverable') or {} + qa = verify_artifact(deliverable, result.get('verification') or {}) + status = determine_parent_status(has_children=bool(children), + deliverable_url=deliverable.get('url'), verified=qa.passed) + event = {'timestamp': utc_now(), 'status': 'QA_PENDING', 'prompt_to_model': prompt, + 'model_response': raw, 'deliverable': deliverable, + 'qa': qa.reason, 'childrenProposed': len(children)} + if not db.finish_claim(task, event, status, children): + raise RuntimeError('Claim lost before completion; no result committed') except Exception as exc: - db.append_event(task["id"], { - "timestamp": utc_now(), "status": "AGENT_EXECUTION_FAILED", - "error": str(exc), - }, status="TODO") + db.finish_claim(task, {'timestamp': utc_now(), 'status': 'AGENT_EXECUTION_FAILED', + 'error': str(exc)}, 'TODO') raise - - criteria = task.get("acceptance_criteria") or {} - depth = int(criteria.get("depth", 1)) - children = build_child_tasks(task["code"], result, depth) - inserted = db.insert_children(task, children) - deliverable = result.get("deliverable") or {} - verification = result.get("verification") or {} - verified = bool(deliverable.get("verified")) and bool(verification.get("passed")) - next_status = determine_parent_status( - has_children=bool(children), - deliverable_url=deliverable.get("url"), - verified=verified, - ) - db.append_event(task["id"], { - "timestamp": utc_now(), "status": "AGENT_EXECUTION_COMPLETED", - "agent": task.get("assignee") or f"Canina-{task['department']}-Agent", - "model_response": raw, - "summary": result.get("summary"), - "deliverable": deliverable, - "verification": verification, - "childrenCreated": inserted, - }, status=next_status) return True +execute_next_child = execute_next_task + + def reconcile_parent_states(db: Database) -> bool: parents = db.rows_json( """ SELECT id, code, status FROM department_tasks p WHERE status = 'WAITING_DEPENDENCIES' + AND EXISTS (SELECT 1 FROM department_tasks c + WHERE c.acceptance_criteria->>'parentTaskCode' = p.code) AND NOT EXISTS ( SELECT 1 FROM department_tasks c WHERE c.acceptance_criteria->>'parentTaskCode' = p.code @@ -316,71 +344,23 @@ def reconcile_parent_states(db: Database) -> bool: ) if not parents: return False - parent = parents[0] - db.append_event(parent["id"], { - "timestamp": utc_now(), "status": "DEPENDENCIES_COMPLETED", - "agent": "Canina-PMO-Agent", - "note": "تمام child taskها تکمیل و راستی‌آزمایی شده‌اند؛ والد وارد QA نهایی شد.", - }, status="TEST") - return True + # Dependency completion alone cannot pass parent QA. Leave it blocked pending + # independently persisted attestations; no synthetic URL or DONE transition. + return False def reconcile_pseudo_done(db: Database, model: ModelClient) -> bool: - tasks = db.rows_json( - """ - SELECT id, code, department, title, description, status, acceptance_criteria, - execution_logs, deliverable_url - FROM department_tasks - WHERE status = 'DONE' - AND code LIKE 'TSK-%' - AND (deliverable_url IS NULL OR deliverable_url = '') - AND NOT EXISTS ( - SELECT 1 FROM department_tasks child - WHERE child.acceptance_criteria->>'parentTaskCode' = department_tasks.code - ) - ORDER BY created_at ASC - LIMIT 1 - """ - ) - if not tasks: - return False - task = tasks[0] - prompt = decomposition_prompt(task) - db.append_event(task["id"], { - "timestamp": utc_now(), "status": "PMO_DECOMPOSITION_STARTED", - "agent": "Canina-PMO-Agent", "prompt_to_model": prompt, - }, status="IN_PROGRESS") - raw = model.complete(prompt) - proposal = parse_structured_response(raw) - depth = int((task.get("acceptance_criteria") or {}).get("depth", 0)) - children = build_child_tasks(task["code"], proposal, depth) - inserted = db.insert_children(task, children) - deliverable = proposal.get("deliverable") or {} - next_status = determine_parent_status( - has_children=bool(children), - deliverable_url=deliverable.get("url"), - verified=bool(deliverable.get("verified")), - ) - db.append_event(task["id"], { - "timestamp": utc_now(), "status": "PMO_DECOMPOSITION_COMPLETED", - "agent": "Canina-PMO-Agent", "model_response": raw, - "summary": proposal.get("summary"), "childrenCreated": inserted, - }, status=next_status) - return True + # Legacy DONE records require a separate human audit, not autonomous rewrites. + return False def main() -> None: db = Database() model = ModelClient() - if reconcile_pseudo_done(db, model): - print("Reconciled one pseudo-DONE root task into cross-department dependencies.") - return - if execute_next_child(db, model): + if execute_next_task(db, model): print("Executed one child task; unverified model output remains in TEST, never DONE.") return - if reconcile_parent_states(db): - print("Moved one parent with verified completed children into TEST.") - return + # Parent reconciliation requires independent persisted QA attestations. print("No actionable orchestration task found.") diff --git a/scripts/orchestration/test_orchestration_contract.py b/scripts/orchestration/test_orchestration_contract.py new file mode 100644 index 00000000..d37bcd76 --- /dev/null +++ b/scripts/orchestration/test_orchestration_contract.py @@ -0,0 +1,65 @@ +import json +import unittest +from unittest.mock import Mock + +from orchestration_worker import ( + claim_sql, complete_sql, execute_next_task, reconcile_parent_states, + determine_parent_status, verify_artifact, +) + + +class ContractTests(unittest.TestCase): + def test_claim_is_atomic_priority_first_and_includes_roots(self): + sql = claim_sql("token-1") + self.assertIn("FOR UPDATE SKIP LOCKED", sql) + self.assertIn("CEO_EXPEDITE", sql) + self.assertIn("updated_at ASC", sql) + self.assertNotIn("parentTaskCode' IS NOT NULL", sql) + self.assertIn("RETURNING", sql) + + def test_completion_is_fenced_by_claim_token(self): + sql = complete_sql("task-id", "token-1", "TEST", {"status": "QA_PENDING"}) + self.assertIn("IN_PROGRESS", sql) + self.assertIn("token-1", sql) + self.assertIn("RETURNING", sql) + self.assertIn("claim_token", sql) + self.assertIn("updated_at >= NOW() - INTERVAL '15 minutes'", sql) + + def test_model_verified_url_cannot_mark_done(self): + self.assertEqual(determine_parent_status(has_children=False, deliverable_url="https://canina.ir/fake", verified=True), "TEST") + + def test_independent_qa_rejects_unverified_claimed_url(self): + self.assertFalse(verify_artifact({"url": "https://canina.ir/fake", "verified": True}, {"passed": True}).passed) + + def test_unapproved_content_publish_is_not_attempted(self): + db = Mock() + db.claim_next.return_value = {"id": "1", "code": "CEO-2", "department": "CONTENT", "title": "Draft", "description": "Do", "acceptance_criteria": {}, "claim_token": "abc"} + db.finish_claim.return_value = True + model = Mock() + model.complete.return_value = json.dumps({"deliverable": {"type": "draft", "content": "clinical " * 200, "url": "https://canina.ir/blog/fake", "verified": True}, "verification": {"passed": True}}) + self.assertTrue(execute_next_task(db, model)) + self.assertEqual(db.finish_claim.call_args.args[2], "TEST") + self.assertNotIn("published", db.finish_claim.call_args.args[1]) + + def test_root_executes_without_live_side_effects(self): + db = Mock() + db.claim_next.return_value = {"id": "1", "code": "CEO-1", "department": "PMO", "title": "Plan", "description": "Do", "acceptance_criteria": {}, "claim_token": "abc"} + db.finish_claim.return_value = True + model = Mock() + model.complete.return_value = json.dumps({"deliverable": {"url": "https://canina.ir/fake", "verified": True}, "verification": {"passed": True}, "child_tasks": []}) + self.assertTrue(execute_next_task(db, model)) + db.finish_claim.assert_called_once() + self.assertEqual(db.finish_claim.call_args.args[2], "TEST") + db.insert_children.assert_not_called() + + def test_rejected_child_never_unblocks_parent(self): + db = Mock() + db.rows_json.return_value = [] + self.assertFalse(reconcile_parent_states(db)) + # REJECTED is not DONE, therefore the anti-join still blocks the parent. + self.assertIn("NOT IN ('DONE')", db.rows_json.call_args.args[0]) + self.assertNotIn("NOT IN ('DONE', 'REJECTED')", db.rows_json.call_args.args[0]) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/orchestration/test_orchestration_worker.py b/scripts/orchestration/test_orchestration_worker.py index 1c02bc6d..6b4a266d 100644 --- a/scripts/orchestration/test_orchestration_worker.py +++ b/scripts/orchestration/test_orchestration_worker.py @@ -19,10 +19,10 @@ class OrchestrationWorkerTests(unittest.TestCase): "TEST", ) - def test_verified_live_deliverable_may_be_done(self): + def test_model_claimed_live_deliverable_still_requires_independent_qa(self): self.assertEqual( determine_parent_status(has_children=False, deliverable_url="https://canina.ir/x", verified=True), - "DONE", + "TEST", ) def test_children_force_waiting_dependencies(self):