merge(worker): integrate orchestration worker contract
Deploy Canina / deploy (push) Successful in 2m56s
Deploy Canina / deploy (push) Successful in 2m56s
This commit is contained in:
commit
19c4bb451d
3 files changed
+184
-139
No files matched your search
@@ -12,6 +12,8 @@ import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import uuid
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
@@ -85,11 +87,84 @@ def build_child_tasks(parent_code: str, proposal: dict[str, Any], depth: int) ->
|
||||
def determine_parent_status(*, has_children: bool, deliverable_url: str | None, verified: bool) -> str:
|
||||
if has_children:
|
||||
return "WAITING_DEPENDENCIES"
|
||||
if deliverable_url and verified:
|
||||
return "DONE"
|
||||
# A model URL and its self-assessed verification cannot satisfy independent QA.
|
||||
return "TEST"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class QAResult:
|
||||
passed: bool
|
||||
reason: str
|
||||
|
||||
|
||||
def verify_artifact(deliverable: dict[str, Any], verification: dict[str, Any]) -> QAResult:
|
||||
"""Offline gate: claimed URLs and model assertions are not artifact evidence."""
|
||||
return QAResult(False, "Independent artifact QA and approval required")
|
||||
|
||||
|
||||
def _literal(value: Any) -> str:
|
||||
return "'" + str(value).replace("'", "''") + "'"
|
||||
|
||||
|
||||
def claim_sql(token: str) -> str:
|
||||
"""Select and lease one root or child in a single locked statement."""
|
||||
event = json.dumps({"status": "CLAIMED", "claim_token": token})
|
||||
return f"""
|
||||
WITH candidate AS (
|
||||
SELECT id FROM department_tasks
|
||||
WHERE status = 'TODO' OR (
|
||||
status = 'IN_PROGRESS' AND execution_logs->-1->>'status' = 'CLAIMED'
|
||||
AND updated_at < NOW() - INTERVAL '15 minutes'
|
||||
)
|
||||
ORDER BY CASE priority WHEN 'CEO_EXPEDITE' THEN 0 WHEN 'CRITICAL' THEN 1
|
||||
WHEN 'HIGH' THEN 2 WHEN 'MEDIUM' THEN 3 ELSE 4 END,
|
||||
updated_at ASC, id ASC
|
||||
FOR UPDATE SKIP LOCKED LIMIT 1
|
||||
), claimed AS (
|
||||
UPDATE department_tasks t SET status = 'IN_PROGRESS', updated_at = NOW(),
|
||||
execution_logs = COALESCE(t.execution_logs, '[]'::jsonb) || {_literal(event)}::jsonb
|
||||
FROM candidate WHERE t.id = candidate.id
|
||||
RETURNING t.id, t.code, t.department, t.title, t.description,
|
||||
t.acceptance_criteria, t.execution_logs, t.assignee, t.priority
|
||||
) SELECT COALESCE(json_agg(claimed), '[]'::json)::text FROM claimed;
|
||||
"""
|
||||
|
||||
|
||||
def complete_sql(task_id: str, token: str, status: str, event: dict[str, Any],
|
||||
children: list[dict[str, Any]] | None = None) -> str:
|
||||
"""Fenced transition and idempotent child insert in one atomic SQL statement."""
|
||||
if status not in ('TODO', 'TEST', 'WAITING_DEPENDENCIES', 'REJECTED'):
|
||||
raise ValueError('Invalid worker transition')
|
||||
entries = []
|
||||
for child in children or []:
|
||||
entries.append('(' + ', '.join(_literal(x) for x in (
|
||||
child['code'], child['department'], child['title'], child['description'],
|
||||
child['priority'], child['assignee'], json.dumps(child['acceptanceCriteria'], ensure_ascii=False)
|
||||
)) + ')')
|
||||
insert = ''
|
||||
if entries:
|
||||
insert = f""", inserted AS (
|
||||
INSERT INTO department_tasks (id, code, department, title, description,
|
||||
status, priority, assignee, acceptance_criteria, execution_logs, created_at, updated_at)
|
||||
SELECT gen_random_uuid(), v.code, v.department, v.title, v.description,
|
||||
'TODO', v.priority, v.assignee, v.criteria::jsonb, '[]'::jsonb, NOW(), NOW()
|
||||
FROM (VALUES {', '.join(entries)}) AS v(code, department, title, description, priority, assignee, criteria)
|
||||
WHERE EXISTS (SELECT 1 FROM completed)
|
||||
ON CONFLICT (code) DO NOTHING RETURNING id
|
||||
)"""
|
||||
return f"""
|
||||
WITH completed AS (
|
||||
UPDATE department_tasks SET status = {_literal(status)}, updated_at = NOW(),
|
||||
execution_logs = COALESCE(execution_logs, '[]'::jsonb) || {_literal(json.dumps(event, ensure_ascii=False))}::jsonb
|
||||
WHERE id = {_literal(task_id)}::uuid AND status = 'IN_PROGRESS'
|
||||
AND updated_at >= NOW() - INTERVAL '15 minutes'
|
||||
AND execution_logs->-1->>'claim_token' = {_literal(token)}
|
||||
RETURNING id
|
||||
){insert}
|
||||
SELECT id FROM completed;
|
||||
"""
|
||||
|
||||
|
||||
class Database:
|
||||
def __init__(self) -> None:
|
||||
self.ssh = ["ssh", "-F", "/opt/data/.ssh/config", "parsa-host"]
|
||||
@@ -99,6 +174,8 @@ class Database:
|
||||
]
|
||||
|
||||
def sql(self, statement: str) -> str:
|
||||
if os.environ.get('CANINA_ORCHESTRATION_LIVE_WRITE') != 'I_ACCEPT_LIVE_MUTATION':
|
||||
raise RuntimeError('Live database writes disabled; use injected mock database for dry runs')
|
||||
result = subprocess.run(
|
||||
self.ssh + self.psql,
|
||||
input=statement,
|
||||
@@ -114,42 +191,16 @@ class Database:
|
||||
raw = self.sql(wrapped)
|
||||
return json.loads(raw or "[]")
|
||||
|
||||
def append_event(self, task_id: str, event: dict[str, Any], *, status: str | None = None) -> None:
|
||||
event_json = json.dumps(event, ensure_ascii=False).replace("$event$", "")
|
||||
status_sql = f", status = '{status}'" if status else ""
|
||||
self.sql(
|
||||
f"""
|
||||
UPDATE department_tasks
|
||||
SET execution_logs = COALESCE(execution_logs, '[]'::jsonb) || $event${event_json}$event$::jsonb,
|
||||
updated_at = NOW(){status_sql}
|
||||
WHERE id = '{task_id}';
|
||||
"""
|
||||
)
|
||||
|
||||
def insert_children(self, parent: dict[str, Any], children: list[dict[str, Any]]) -> int:
|
||||
inserted = 0
|
||||
for child in children:
|
||||
criteria = json.dumps(child["acceptanceCriteria"], ensure_ascii=False).replace("$criteria$", "")
|
||||
title = child["title"].replace("'", "''")
|
||||
description = child["description"].replace("'", "''")
|
||||
assignee = child["assignee"].replace("'", "''")
|
||||
out = self.sql(
|
||||
f"""
|
||||
INSERT INTO department_tasks
|
||||
(id, code, department, title, description, status, priority, assignee,
|
||||
acceptance_criteria, execution_logs, created_at, updated_at)
|
||||
VALUES
|
||||
(gen_random_uuid(), '{child['code']}', '{child['department']}', '{title}',
|
||||
'{description}', 'TODO', '{child['priority']}', '{assignee}',
|
||||
$criteria${criteria}$criteria$::jsonb, '[]'::jsonb, NOW(), NOW())
|
||||
ON CONFLICT (code) DO NOTHING
|
||||
RETURNING code;
|
||||
"""
|
||||
)
|
||||
if out:
|
||||
inserted += 1
|
||||
return inserted
|
||||
def claim_next(self) -> dict[str, Any] | None:
|
||||
token = str(uuid.uuid4())
|
||||
rows = json.loads(self.sql(claim_sql(token)) or '[]')
|
||||
if not rows:
|
||||
return None
|
||||
return {**rows[0], 'claim_token': token}
|
||||
|
||||
def finish_claim(self, task: dict[str, Any], event: dict[str, Any], status: str,
|
||||
children: list[dict[str, Any]] | None = None) -> bool:
|
||||
return bool(self.sql(complete_sql(task['id'], task['claim_token'], status, event, children)))
|
||||
|
||||
class ModelClient:
|
||||
def __init__(self) -> None:
|
||||
@@ -244,67 +295,44 @@ def execution_prompt(task: dict[str, Any]) -> str:
|
||||
""".strip()
|
||||
|
||||
|
||||
def execute_next_child(db: Database, model: ModelClient) -> bool:
|
||||
tasks = db.rows_json(
|
||||
"""
|
||||
SELECT id, code, department, title, description, status, acceptance_criteria,
|
||||
execution_logs, deliverable_url
|
||||
FROM department_tasks
|
||||
WHERE status = 'TODO'
|
||||
AND acceptance_criteria->>'parentTaskCode' IS NOT NULL
|
||||
ORDER BY created_at ASC
|
||||
LIMIT 1
|
||||
"""
|
||||
)
|
||||
if not tasks:
|
||||
def execute_next_task(db: Database, model: ModelClient) -> bool:
|
||||
"""Produce reviewable drafts; never publish or trust model QA assertions."""
|
||||
task = db.claim_next()
|
||||
if not task:
|
||||
return False
|
||||
task = tasks[0]
|
||||
prompt = execution_prompt(task)
|
||||
db.append_event(task["id"], {
|
||||
"timestamp": utc_now(), "status": "AGENT_EXECUTION_STARTED",
|
||||
"agent": task.get("assignee") or f"Canina-{task['department']}-Agent",
|
||||
"prompt_to_model": prompt,
|
||||
}, status="IN_PROGRESS")
|
||||
try:
|
||||
raw = model.complete(prompt)
|
||||
result = parse_structured_response(raw)
|
||||
criteria = task.get('acceptance_criteria') or {}
|
||||
children = build_child_tasks(task['code'], result, int(criteria.get('depth', 0)))
|
||||
deliverable = result.get('deliverable') or {}
|
||||
qa = verify_artifact(deliverable, result.get('verification') or {})
|
||||
status = determine_parent_status(has_children=bool(children),
|
||||
deliverable_url=deliverable.get('url'), verified=qa.passed)
|
||||
event = {'timestamp': utc_now(), 'status': 'QA_PENDING', 'prompt_to_model': prompt,
|
||||
'model_response': raw, 'deliverable': deliverable,
|
||||
'qa': qa.reason, 'childrenProposed': len(children)}
|
||||
if not db.finish_claim(task, event, status, children):
|
||||
raise RuntimeError('Claim lost before completion; no result committed')
|
||||
except Exception as exc:
|
||||
db.append_event(task["id"], {
|
||||
"timestamp": utc_now(), "status": "AGENT_EXECUTION_FAILED",
|
||||
"error": str(exc),
|
||||
}, status="TODO")
|
||||
db.finish_claim(task, {'timestamp': utc_now(), 'status': 'AGENT_EXECUTION_FAILED',
|
||||
'error': str(exc)}, 'TODO')
|
||||
raise
|
||||
|
||||
criteria = task.get("acceptance_criteria") or {}
|
||||
depth = int(criteria.get("depth", 1))
|
||||
children = build_child_tasks(task["code"], result, depth)
|
||||
inserted = db.insert_children(task, children)
|
||||
deliverable = result.get("deliverable") or {}
|
||||
verification = result.get("verification") or {}
|
||||
verified = bool(deliverable.get("verified")) and bool(verification.get("passed"))
|
||||
next_status = determine_parent_status(
|
||||
has_children=bool(children),
|
||||
deliverable_url=deliverable.get("url"),
|
||||
verified=verified,
|
||||
)
|
||||
db.append_event(task["id"], {
|
||||
"timestamp": utc_now(), "status": "AGENT_EXECUTION_COMPLETED",
|
||||
"agent": task.get("assignee") or f"Canina-{task['department']}-Agent",
|
||||
"model_response": raw,
|
||||
"summary": result.get("summary"),
|
||||
"deliverable": deliverable,
|
||||
"verification": verification,
|
||||
"childrenCreated": inserted,
|
||||
}, status=next_status)
|
||||
return True
|
||||
|
||||
|
||||
execute_next_child = execute_next_task
|
||||
|
||||
|
||||
def reconcile_parent_states(db: Database) -> bool:
|
||||
parents = db.rows_json(
|
||||
"""
|
||||
SELECT id, code, status
|
||||
FROM department_tasks p
|
||||
WHERE status = 'WAITING_DEPENDENCIES'
|
||||
AND EXISTS (SELECT 1 FROM department_tasks c
|
||||
WHERE c.acceptance_criteria->>'parentTaskCode' = p.code)
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM department_tasks c
|
||||
WHERE c.acceptance_criteria->>'parentTaskCode' = p.code
|
||||
@@ -316,71 +344,23 @@ def reconcile_parent_states(db: Database) -> bool:
|
||||
)
|
||||
if not parents:
|
||||
return False
|
||||
parent = parents[0]
|
||||
db.append_event(parent["id"], {
|
||||
"timestamp": utc_now(), "status": "DEPENDENCIES_COMPLETED",
|
||||
"agent": "Canina-PMO-Agent",
|
||||
"note": "تمام child taskها تکمیل و راستیآزمایی شدهاند؛ والد وارد QA نهایی شد.",
|
||||
}, status="TEST")
|
||||
return True
|
||||
# Dependency completion alone cannot pass parent QA. Leave it blocked pending
|
||||
# independently persisted attestations; no synthetic URL or DONE transition.
|
||||
return False
|
||||
|
||||
|
||||
def reconcile_pseudo_done(db: Database, model: ModelClient) -> bool:
|
||||
tasks = db.rows_json(
|
||||
"""
|
||||
SELECT id, code, department, title, description, status, acceptance_criteria,
|
||||
execution_logs, deliverable_url
|
||||
FROM department_tasks
|
||||
WHERE status = 'DONE'
|
||||
AND code LIKE 'TSK-%'
|
||||
AND (deliverable_url IS NULL OR deliverable_url = '')
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM department_tasks child
|
||||
WHERE child.acceptance_criteria->>'parentTaskCode' = department_tasks.code
|
||||
)
|
||||
ORDER BY created_at ASC
|
||||
LIMIT 1
|
||||
"""
|
||||
)
|
||||
if not tasks:
|
||||
return False
|
||||
task = tasks[0]
|
||||
prompt = decomposition_prompt(task)
|
||||
db.append_event(task["id"], {
|
||||
"timestamp": utc_now(), "status": "PMO_DECOMPOSITION_STARTED",
|
||||
"agent": "Canina-PMO-Agent", "prompt_to_model": prompt,
|
||||
}, status="IN_PROGRESS")
|
||||
raw = model.complete(prompt)
|
||||
proposal = parse_structured_response(raw)
|
||||
depth = int((task.get("acceptance_criteria") or {}).get("depth", 0))
|
||||
children = build_child_tasks(task["code"], proposal, depth)
|
||||
inserted = db.insert_children(task, children)
|
||||
deliverable = proposal.get("deliverable") or {}
|
||||
next_status = determine_parent_status(
|
||||
has_children=bool(children),
|
||||
deliverable_url=deliverable.get("url"),
|
||||
verified=bool(deliverable.get("verified")),
|
||||
)
|
||||
db.append_event(task["id"], {
|
||||
"timestamp": utc_now(), "status": "PMO_DECOMPOSITION_COMPLETED",
|
||||
"agent": "Canina-PMO-Agent", "model_response": raw,
|
||||
"summary": proposal.get("summary"), "childrenCreated": inserted,
|
||||
}, status=next_status)
|
||||
return True
|
||||
# Legacy DONE records require a separate human audit, not autonomous rewrites.
|
||||
return False
|
||||
|
||||
|
||||
def main() -> None:
|
||||
db = Database()
|
||||
model = ModelClient()
|
||||
if reconcile_pseudo_done(db, model):
|
||||
print("Reconciled one pseudo-DONE root task into cross-department dependencies.")
|
||||
return
|
||||
if execute_next_child(db, model):
|
||||
if execute_next_task(db, model):
|
||||
print("Executed one child task; unverified model output remains in TEST, never DONE.")
|
||||
return
|
||||
if reconcile_parent_states(db):
|
||||
print("Moved one parent with verified completed children into TEST.")
|
||||
return
|
||||
# Parent reconciliation requires independent persisted QA attestations.
|
||||
print("No actionable orchestration task found.")
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
import json
|
||||
import unittest
|
||||
from unittest.mock import Mock
|
||||
|
||||
from orchestration_worker import (
|
||||
claim_sql, complete_sql, execute_next_task, reconcile_parent_states,
|
||||
determine_parent_status, verify_artifact,
|
||||
)
|
||||
|
||||
|
||||
class ContractTests(unittest.TestCase):
|
||||
def test_claim_is_atomic_priority_first_and_includes_roots(self):
|
||||
sql = claim_sql("token-1")
|
||||
self.assertIn("FOR UPDATE SKIP LOCKED", sql)
|
||||
self.assertIn("CEO_EXPEDITE", sql)
|
||||
self.assertIn("updated_at ASC", sql)
|
||||
self.assertNotIn("parentTaskCode' IS NOT NULL", sql)
|
||||
self.assertIn("RETURNING", sql)
|
||||
|
||||
def test_completion_is_fenced_by_claim_token(self):
|
||||
sql = complete_sql("task-id", "token-1", "TEST", {"status": "QA_PENDING"})
|
||||
self.assertIn("IN_PROGRESS", sql)
|
||||
self.assertIn("token-1", sql)
|
||||
self.assertIn("RETURNING", sql)
|
||||
self.assertIn("claim_token", sql)
|
||||
self.assertIn("updated_at >= NOW() - INTERVAL '15 minutes'", sql)
|
||||
|
||||
def test_model_verified_url_cannot_mark_done(self):
|
||||
self.assertEqual(determine_parent_status(has_children=False, deliverable_url="https://canina.ir/fake", verified=True), "TEST")
|
||||
|
||||
def test_independent_qa_rejects_unverified_claimed_url(self):
|
||||
self.assertFalse(verify_artifact({"url": "https://canina.ir/fake", "verified": True}, {"passed": True}).passed)
|
||||
|
||||
def test_unapproved_content_publish_is_not_attempted(self):
|
||||
db = Mock()
|
||||
db.claim_next.return_value = {"id": "1", "code": "CEO-2", "department": "CONTENT", "title": "Draft", "description": "Do", "acceptance_criteria": {}, "claim_token": "abc"}
|
||||
db.finish_claim.return_value = True
|
||||
model = Mock()
|
||||
model.complete.return_value = json.dumps({"deliverable": {"type": "draft", "content": "clinical " * 200, "url": "https://canina.ir/blog/fake", "verified": True}, "verification": {"passed": True}})
|
||||
self.assertTrue(execute_next_task(db, model))
|
||||
self.assertEqual(db.finish_claim.call_args.args[2], "TEST")
|
||||
self.assertNotIn("published", db.finish_claim.call_args.args[1])
|
||||
|
||||
def test_root_executes_without_live_side_effects(self):
|
||||
db = Mock()
|
||||
db.claim_next.return_value = {"id": "1", "code": "CEO-1", "department": "PMO", "title": "Plan", "description": "Do", "acceptance_criteria": {}, "claim_token": "abc"}
|
||||
db.finish_claim.return_value = True
|
||||
model = Mock()
|
||||
model.complete.return_value = json.dumps({"deliverable": {"url": "https://canina.ir/fake", "verified": True}, "verification": {"passed": True}, "child_tasks": []})
|
||||
self.assertTrue(execute_next_task(db, model))
|
||||
db.finish_claim.assert_called_once()
|
||||
self.assertEqual(db.finish_claim.call_args.args[2], "TEST")
|
||||
db.insert_children.assert_not_called()
|
||||
|
||||
def test_rejected_child_never_unblocks_parent(self):
|
||||
db = Mock()
|
||||
db.rows_json.return_value = []
|
||||
self.assertFalse(reconcile_parent_states(db))
|
||||
# REJECTED is not DONE, therefore the anti-join still blocks the parent.
|
||||
self.assertIn("NOT IN ('DONE')", db.rows_json.call_args.args[0])
|
||||
self.assertNotIn("NOT IN ('DONE', 'REJECTED')", db.rows_json.call_args.args[0])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -19,10 +19,10 @@ class OrchestrationWorkerTests(unittest.TestCase):
|
||||
"TEST",
|
||||
)
|
||||
|
||||
def test_verified_live_deliverable_may_be_done(self):
|
||||
def test_model_claimed_live_deliverable_still_requires_independent_qa(self):
|
||||
self.assertEqual(
|
||||
determine_parent_status(has_children=False, deliverable_url="https://canina.ir/x", verified=True),
|
||||
"DONE",
|
||||
"TEST",
|
||||
)
|
||||
|
||||
def test_children_force_waiting_dependencies(self):
|
||||
|
||||
Reference in new issue
Block a user