merge(market): integrate feat/safe-market-proposals-20261020
This commit is contained in:
commit
f71b79a17f
3 files changed
+157
No files matched your search
@@ -0,0 +1,9 @@
|
||||
# Market opportunity proposal gate (inactive)
|
||||
|
||||
Hermes cron `482f37952b74` (`canina-market-opportunity-scout`) currently produces reports in `/opt/data/cron/output/482f37952b74/`; its schedule is enabled. Cron `e95025c84a31` (`canina-opportunity-decomposer-daemon`) is **paused**, with `script: market_opportunity_decomposer.py`, resolved to the untracked Hermes script `/opt/data/scripts/market_opportunity_decomposer.py`. Neither cron definition nor that script is part of this repository. The latter directly inserts model-generated `approval_requests`, uses title-only deduplication, and increments its success count even when `ON CONFLICT DO NOTHING` returns no row. Do not re-enable it.
|
||||
|
||||
`market_proposals.py` is a versioned, **unwired** validation/insertion gate. It accepts only paired digital route identifiers, and deliberately ignores model-supplied title, prose, pricing, clinical claims, and payload. Evidence URL and verbatim quote must coexist in a scout report paragraph; this is a syntactic provenance check, **not** independent verification that the URL is genuine or medically correct. The resulting review-only payload links the source and report; no automatic approval, task execution, or publication exists. A transaction-scoped PostgreSQL advisory lock serializes proposals by source URL + exact quote + action fingerprint and the SQL counts only returned UUIDs. The application-level deduplication assumes every writer uses this gate. Consider a schema-level partial unique index on `(payload->>'sourceFingerprint')` after examining existing rows before production rollout; coordinate schema changes with backend ownership.
|
||||
|
||||
Before activation, migrate the Hermes entry point into this repository under `scripts/orchestration/`, add an explicit offline parser for actual scout output format and bounded model schema, and have the scheduler invoke the **versioned** script by pinned revision/absolute path after review. Replace its direct SQL write path with this gate; inspect sample scout outputs and test malformed, mismatched, stale, and false-source citations. Add a dry-run mode and staging integration tests with a disposable database, review permissions and audit logging, and require human clinical/pricing review. Do not infer clinical or price facts from model prose. Keep `e95025c84a31` paused until rollout approval; do not change the active scout job as part of this patch.
|
||||
|
||||
Tests: `cd scripts/orchestration && python3 -m unittest discover -p 'test_*.py' -v` (no DB access).
|
||||
@@ -0,0 +1,77 @@
|
||||
"""Conservative, offline-testable approval proposal gateway for market-scout evidence.
|
||||
|
||||
Not scheduled or activated. Only explicitly allowlisted digital audit work is accepted.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import uuid
|
||||
from urllib.parse import urlsplit, urlunsplit
|
||||
|
||||
|
||||
ROUTES = {
|
||||
"CATALOG_AUDIT": ("CONTENT", "CAMPAIGN", "Audit cited catalog evidence"),
|
||||
"SEO_AUDIT": ("SEO", "SEO", "Audit cited search evidence"),
|
||||
"WEB_FEATURE_REVIEW": ("DEV", "FEATURE", "Review cited website feature gap"),
|
||||
}
|
||||
|
||||
|
||||
def validate_proposal(item: dict, report: str, report_name: str) -> dict:
|
||||
"""Discard untrusted prose and construct a bounded review-only proposal."""
|
||||
if not isinstance(item, dict) or not isinstance(report, str):
|
||||
raise ValueError("Invalid proposal or report")
|
||||
action = item.get("action_type")
|
||||
route = ROUTES.get(action)
|
||||
if route is None or item.get("department") != route[0]:
|
||||
raise ValueError("Only paired digital routes are supported")
|
||||
url = item.get("source_url")
|
||||
quote = item.get("evidence_quote")
|
||||
if not isinstance(url, str) or not isinstance(quote, str):
|
||||
raise ValueError("A source URL and exact evidence quote are required")
|
||||
parsed = urlsplit(url)
|
||||
if parsed.scheme not in ("https", "http") or not parsed.hostname or parsed.username or parsed.password or len(url) > 1000:
|
||||
raise ValueError("Invalid evidence URL")
|
||||
canonical_url = urlunsplit((parsed.scheme.lower(), parsed.netloc.lower(), parsed.path.rstrip("/") or "/", parsed.query, ""))
|
||||
if not quote.strip() or len(quote) > 500 or not any(
|
||||
url in block and quote in block for block in re.split(r"\n\s*\n", report)
|
||||
):
|
||||
raise ValueError("Evidence and URL must occur verbatim in the same scout report block")
|
||||
fingerprint = hashlib.sha256(json.dumps([canonical_url, quote.strip(), action], ensure_ascii=False, separators=(",", ":")).encode()).hexdigest()
|
||||
return {
|
||||
"department": route[0], "category": route[1],
|
||||
"title": f"{route[2]} [{fingerprint[:12]}]",
|
||||
"fingerprint": fingerprint,
|
||||
"payload": {
|
||||
"sourceUrl": canonical_url, "evidenceQuote": quote.strip(),
|
||||
"sourceFingerprint": fingerprint, "sourceReport": str(report_name).split("/")[-1][:255],
|
||||
"actionType": action, "reviewRequired": True,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _literal(value: str) -> str:
|
||||
return "'" + value.replace("'", "''") + "'"
|
||||
|
||||
|
||||
def insert_proposal(db, proposal: dict) -> bool:
|
||||
"""Atomically serialize fingerprint insertion and count only returned IDs."""
|
||||
fp = proposal["fingerprint"]
|
||||
if not re.fullmatch(r"[0-9a-f]{64}", fp):
|
||||
raise ValueError("Invalid fingerprint")
|
||||
payload = json.dumps(proposal["payload"], ensure_ascii=False)
|
||||
sql = f"""BEGIN;
|
||||
SELECT pg_advisory_xact_lock(hashtextextended({_literal('market-proposal:' + fp)}, 0));
|
||||
INSERT INTO approval_requests (id, department, category, title, payload, status, created_at, updated_at)
|
||||
SELECT gen_random_uuid(), {_literal(proposal['department'])}, {_literal(proposal['category'])},
|
||||
{_literal(proposal['title'])}, {_literal(payload)}::jsonb, 'PENDING', NOW(), NOW()
|
||||
WHERE NOT EXISTS (
|
||||
SELECT 1 FROM approval_requests
|
||||
WHERE payload->>'sourceFingerprint' = {_literal(fp)}
|
||||
)
|
||||
RETURNING id;
|
||||
COMMIT;"""
|
||||
result = db.sql(sql)
|
||||
return any(re.fullmatch(r"[0-9a-fA-F]{8}-(?:[0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}", line.strip())
|
||||
for line in result.splitlines())
|
||||
@@ -0,0 +1,71 @@
|
||||
import unittest
|
||||
|
||||
from market_proposals import validate_proposal, insert_proposal
|
||||
|
||||
|
||||
REPORT = "https://example.org/research\nEvidence: product listing lacks dosage instructions."
|
||||
BASE = {"source_url": "https://example.org/research", "evidence_quote": "product listing lacks dosage instructions", "action_type": "CATALOG_AUDIT", "department": "CONTENT"}
|
||||
|
||||
|
||||
class FakeDB:
|
||||
def __init__(self, output):
|
||||
self.output = output
|
||||
self.statements = []
|
||||
|
||||
def sql(self, statement):
|
||||
self.statements.append(statement)
|
||||
return self.output
|
||||
|
||||
|
||||
class MarketProposalTests(unittest.TestCase):
|
||||
def test_verified_citation_produces_stable_fingerprint_and_pending_payload(self):
|
||||
first = validate_proposal(BASE, REPORT, "scout-2026-10-20.md")
|
||||
second = validate_proposal(BASE, REPORT, "scout-2026-10-22.md")
|
||||
self.assertEqual(first["fingerprint"], second["fingerprint"])
|
||||
self.assertEqual(first["payload"]["sourceUrl"], BASE["source_url"])
|
||||
self.assertEqual(first["payload"]["evidenceQuote"], BASE["evidence_quote"])
|
||||
self.assertEqual(first["payload"]["reviewRequired"], True)
|
||||
self.assertNotIn("price", str(first).lower())
|
||||
|
||||
def test_unsubstantiated_or_unsafe_sources_rejected(self):
|
||||
for item in ({**BASE, "source_url": "https://other.example/page"},
|
||||
{**BASE, "evidence_quote": "invented price 123"},
|
||||
{**BASE, "source_url": "file:///etc/passwd"}):
|
||||
with self.subTest(item=item), self.assertRaises(ValueError):
|
||||
validate_proposal(item, REPORT, "scout.md")
|
||||
|
||||
def test_quote_must_be_in_same_citation_block_as_url(self):
|
||||
report = "https://example.org/research\nSome unrelated finding.\n\nhttps://example.org/other\nproduct listing lacks dosage instructions"
|
||||
with self.assertRaises(ValueError):
|
||||
validate_proposal(BASE, report, "scout.md")
|
||||
|
||||
def test_human_and_physical_routes_rejected(self):
|
||||
for item in ({**BASE, "department": "B2B"},
|
||||
{**BASE, "department": "CLINICAL_RD"},
|
||||
{**BASE, "action_type": "FIELD_VISIT"},
|
||||
{**BASE, "action_type": "CATALOG_AUDIT", "department": "SUPPORT"}):
|
||||
with self.subTest(item=item), self.assertRaises(ValueError):
|
||||
validate_proposal(item, REPORT, "scout.md")
|
||||
|
||||
def test_untrusted_model_narrative_cannot_enter_payload_or_title(self):
|
||||
item = {**BASE, "title": "Guaranteed cure 999", "payload": {"price": "invented"}}
|
||||
result = validate_proposal(item, REPORT, "scout.md")
|
||||
self.assertNotIn("Guaranteed", str(result))
|
||||
self.assertNotIn("invented", str(result))
|
||||
|
||||
def test_insert_counts_only_returned_uuid_and_sql_is_atomic_and_pending(self):
|
||||
proposal = validate_proposal(BASE, REPORT, "scout.md")
|
||||
db = FakeDB("BEGIN\n123e4567-e89b-12d3-a456-426614174000\nINSERT 0 1\nCOMMIT")
|
||||
self.assertTrue(insert_proposal(db, proposal))
|
||||
sql = db.statements[0]
|
||||
self.assertIn("pg_advisory_xact_lock", sql)
|
||||
self.assertIn("sourceFingerprint", sql)
|
||||
self.assertIn("RETURNING id", sql)
|
||||
self.assertIn("'PENDING'", sql)
|
||||
self.assertNotIn("APPROVED", sql)
|
||||
self.assertFalse(insert_proposal(FakeDB("BEGIN\nINSERT 0 0\nCOMMIT"), proposal))
|
||||
self.assertFalse(insert_proposal(FakeDB("BEGIN\nCOMMIT"), proposal))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in new issue
Block a user