merge(market): integrate feat/safe-market-proposals-20261020

This commit is contained in:
devops committed 2026-10-20 19:43:31 +00:00
commit f71b79a17f
3 files changed
+157

No files matched your search

@@ -0,0 +1,9 @@
# Market opportunity proposal gate (inactive)
Hermes cron `482f37952b74` (`canina-market-opportunity-scout`) currently produces reports in `/opt/data/cron/output/482f37952b74/`; its schedule is enabled. Cron `e95025c84a31` (`canina-opportunity-decomposer-daemon`) is **paused**, with `script: market_opportunity_decomposer.py`, resolved to the untracked Hermes script `/opt/data/scripts/market_opportunity_decomposer.py`. Neither cron definition nor that script is part of this repository. The latter directly inserts model-generated `approval_requests`, uses title-only deduplication, and increments its success count even when `ON CONFLICT DO NOTHING` returns no row. Do not re-enable it.
`market_proposals.py` is a versioned, **unwired** validation/insertion gate. It accepts only paired digital route identifiers, and deliberately ignores model-supplied title, prose, pricing, clinical claims, and payload. Evidence URL and verbatim quote must coexist in a scout report paragraph; this is a syntactic provenance check, **not** independent verification that the URL is genuine or medically correct. The resulting review-only payload links the source and report; no automatic approval, task execution, or publication exists. A transaction-scoped PostgreSQL advisory lock serializes proposals by source URL + exact quote + action fingerprint and the SQL counts only returned UUIDs. The application-level deduplication assumes every writer uses this gate. Consider a schema-level partial unique index on `(payload->>'sourceFingerprint')` after examining existing rows before production rollout; coordinate schema changes with backend ownership.
Before activation, migrate the Hermes entry point into this repository under `scripts/orchestration/`, add an explicit offline parser for actual scout output format and bounded model schema, and have the scheduler invoke the **versioned** script by pinned revision/absolute path after review. Replace its direct SQL write path with this gate; inspect sample scout outputs and test malformed, mismatched, stale, and false-source citations. Add a dry-run mode and staging integration tests with a disposable database, review permissions and audit logging, and require human clinical/pricing review. Do not infer clinical or price facts from model prose. Keep `e95025c84a31` paused until rollout approval; do not change the active scout job as part of this patch.
Tests: `cd scripts/orchestration && python3 -m unittest discover -p 'test_*.py' -v` (no DB access).
+77
View File
@@ -0,0 +1,77 @@
"""Conservative, offline-testable approval proposal gateway for market-scout evidence.
Not scheduled or activated. Only explicitly allowlisted digital audit work is accepted.
"""
from __future__ import annotations
import hashlib
import json
import re
import uuid
from urllib.parse import urlsplit, urlunsplit
ROUTES = {
"CATALOG_AUDIT": ("CONTENT", "CAMPAIGN", "Audit cited catalog evidence"),
"SEO_AUDIT": ("SEO", "SEO", "Audit cited search evidence"),
"WEB_FEATURE_REVIEW": ("DEV", "FEATURE", "Review cited website feature gap"),
}
def validate_proposal(item: dict, report: str, report_name: str) -> dict:
"""Discard untrusted prose and construct a bounded review-only proposal."""
if not isinstance(item, dict) or not isinstance(report, str):
raise ValueError("Invalid proposal or report")
action = item.get("action_type")
route = ROUTES.get(action)
if route is None or item.get("department") != route[0]:
raise ValueError("Only paired digital routes are supported")
url = item.get("source_url")
quote = item.get("evidence_quote")
if not isinstance(url, str) or not isinstance(quote, str):
raise ValueError("A source URL and exact evidence quote are required")
parsed = urlsplit(url)
if parsed.scheme not in ("https", "http") or not parsed.hostname or parsed.username or parsed.password or len(url) > 1000:
raise ValueError("Invalid evidence URL")
canonical_url = urlunsplit((parsed.scheme.lower(), parsed.netloc.lower(), parsed.path.rstrip("/") or "/", parsed.query, ""))
if not quote.strip() or len(quote) > 500 or not any(
url in block and quote in block for block in re.split(r"\n\s*\n", report)
):
raise ValueError("Evidence and URL must occur verbatim in the same scout report block")
fingerprint = hashlib.sha256(json.dumps([canonical_url, quote.strip(), action], ensure_ascii=False, separators=(",", ":")).encode()).hexdigest()
return {
"department": route[0], "category": route[1],
"title": f"{route[2]} [{fingerprint[:12]}]",
"fingerprint": fingerprint,
"payload": {
"sourceUrl": canonical_url, "evidenceQuote": quote.strip(),
"sourceFingerprint": fingerprint, "sourceReport": str(report_name).split("/")[-1][:255],
"actionType": action, "reviewRequired": True,
},
}
def _literal(value: str) -> str:
return "'" + value.replace("'", "''") + "'"
def insert_proposal(db, proposal: dict) -> bool:
"""Atomically serialize fingerprint insertion and count only returned IDs."""
fp = proposal["fingerprint"]
if not re.fullmatch(r"[0-9a-f]{64}", fp):
raise ValueError("Invalid fingerprint")
payload = json.dumps(proposal["payload"], ensure_ascii=False)
sql = f"""BEGIN;
SELECT pg_advisory_xact_lock(hashtextextended({_literal('market-proposal:' + fp)}, 0));
INSERT INTO approval_requests (id, department, category, title, payload, status, created_at, updated_at)
SELECT gen_random_uuid(), {_literal(proposal['department'])}, {_literal(proposal['category'])},
{_literal(proposal['title'])}, {_literal(payload)}::jsonb, 'PENDING', NOW(), NOW()
WHERE NOT EXISTS (
SELECT 1 FROM approval_requests
WHERE payload->>'sourceFingerprint' = {_literal(fp)}
)
RETURNING id;
COMMIT;"""
result = db.sql(sql)
return any(re.fullmatch(r"[0-9a-fA-F]{8}-(?:[0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}", line.strip())
for line in result.splitlines())
@@ -0,0 +1,71 @@
import unittest
from market_proposals import validate_proposal, insert_proposal
REPORT = "https://example.org/research\nEvidence: product listing lacks dosage instructions."
BASE = {"source_url": "https://example.org/research", "evidence_quote": "product listing lacks dosage instructions", "action_type": "CATALOG_AUDIT", "department": "CONTENT"}
class FakeDB:
def __init__(self, output):
self.output = output
self.statements = []
def sql(self, statement):
self.statements.append(statement)
return self.output
class MarketProposalTests(unittest.TestCase):
def test_verified_citation_produces_stable_fingerprint_and_pending_payload(self):
first = validate_proposal(BASE, REPORT, "scout-2026-10-20.md")
second = validate_proposal(BASE, REPORT, "scout-2026-10-22.md")
self.assertEqual(first["fingerprint"], second["fingerprint"])
self.assertEqual(first["payload"]["sourceUrl"], BASE["source_url"])
self.assertEqual(first["payload"]["evidenceQuote"], BASE["evidence_quote"])
self.assertEqual(first["payload"]["reviewRequired"], True)
self.assertNotIn("price", str(first).lower())
def test_unsubstantiated_or_unsafe_sources_rejected(self):
for item in ({**BASE, "source_url": "https://other.example/page"},
{**BASE, "evidence_quote": "invented price 123"},
{**BASE, "source_url": "file:///etc/passwd"}):
with self.subTest(item=item), self.assertRaises(ValueError):
validate_proposal(item, REPORT, "scout.md")
def test_quote_must_be_in_same_citation_block_as_url(self):
report = "https://example.org/research\nSome unrelated finding.\n\nhttps://example.org/other\nproduct listing lacks dosage instructions"
with self.assertRaises(ValueError):
validate_proposal(BASE, report, "scout.md")
def test_human_and_physical_routes_rejected(self):
for item in ({**BASE, "department": "B2B"},
{**BASE, "department": "CLINICAL_RD"},
{**BASE, "action_type": "FIELD_VISIT"},
{**BASE, "action_type": "CATALOG_AUDIT", "department": "SUPPORT"}):
with self.subTest(item=item), self.assertRaises(ValueError):
validate_proposal(item, REPORT, "scout.md")
def test_untrusted_model_narrative_cannot_enter_payload_or_title(self):
item = {**BASE, "title": "Guaranteed cure 999", "payload": {"price": "invented"}}
result = validate_proposal(item, REPORT, "scout.md")
self.assertNotIn("Guaranteed", str(result))
self.assertNotIn("invented", str(result))
def test_insert_counts_only_returned_uuid_and_sql_is_atomic_and_pending(self):
proposal = validate_proposal(BASE, REPORT, "scout.md")
db = FakeDB("BEGIN\n123e4567-e89b-12d3-a456-426614174000\nINSERT 0 1\nCOMMIT")
self.assertTrue(insert_proposal(db, proposal))
sql = db.statements[0]
self.assertIn("pg_advisory_xact_lock", sql)
self.assertIn("sourceFingerprint", sql)
self.assertIn("RETURNING id", sql)
self.assertIn("'PENDING'", sql)
self.assertNotIn("APPROVED", sql)
self.assertFalse(insert_proposal(FakeDB("BEGIN\nINSERT 0 0\nCOMMIT"), proposal))
self.assertFalse(insert_proposal(FakeDB("BEGIN\nCOMMIT"), proposal))
if __name__ == "__main__":
unittest.main()