feat(seo-worker): collect cached read-only Clarity aggregates

This commit is contained in:
Canina Automation committed 2026-10-28 08:00:20 +00:00
1 parent 13dc74807a
commit db3979d212
10 files changed
+455 -4

No files matched your search

+17
View File
@@ -0,0 +1,17 @@
# Clarity worker integration
Worker CLI now instantiates one shared Clarity collector and includes independent provenance in `sourceMetadata.clarity`. Uses the existing metadata contract; no backend column/schema changes. Default scope is project, never an article's measurement or keyword-caused conversions. Missing commerce linkage stays null; no commerce activation or purchase-error changes.
Run with `/opt/hermes/.venv/bin/python` (dotenv available) to read the provisioned `/opt/data/.env` token in-process without restart. Never pass the token on the command line. Regular CLI retains explicit local state and read-only behavior. Use the same durable `--state` path on future runs so cache/budget survives. The verification state is `/opt/data/clarity-worker-state.sqlite` outside Git; worker default `tracker.sqlite` is separate unless explicitly selected.
Read-only standalone verification:
```
/opt/hermes/.venv/bin/python verify_clarity.py --state /opt/data/clarity-worker-state.sqlite --receipt evidence/clarity-live-receipt.json
```
Only official GET endpoint, Bearer header, redirects off, connect/read timeouts 5/30s. No retries or pagination. API reports rolling 24/48/72h (default 3 days), UTC, not GSC's 14-day period. Max three valid dimensions, 1000 rows, ten attempted calls per UTC day/project **in the shared local state**. External callers/setup requests are not observable by this budget; HTTP429 is cached unavailable. Failed/pending collections are cached for the day too; crashed reservations intentionally remain unavailable rather than silently retrying. Project identifier is explicit; all callers for a project must use the same identifier/state.
Persist only allowlisted numeric aggregate fields; omit categorical/popular-page reports and arbitrary text. URL breakdown is opt-in via constructor; only an exact URL dimension hash can produce article scope. No URL normalization/inference, query strings or raw provider response persisted/logged. A row-limit boundary is disclosed as possibly truncated, not paginated. Missing/malformed responses remain unavailable with null metrics, not zero.
Evidence: seven initial RED failures for missing collector; full final worker suite recorded in `evidence/clarity-tests.txt`. Single real API request returned HTTP200 with nine allowlisted metric groups / nine aggregate rows. Sanitized receipt `evidence/clarity-live-receipt.json` contains shape/status only. Actual backend delivery of Clarity metadata was not attempted/verified; no deployment, push, cron changes or runtime restart. No article-specific Clarity coverage verified by the project-scoped live probe.
@@ -0,0 +1,107 @@
"""Clarity rolling-window export. Persist sanitized aggregates only, never raw bodies."""
import datetime as dt
import hashlib
import json
import math
import os
import requests
CLARITY_ENDPOINT='https://www.clarity.ms/export-data/api/v1/project-live-insights'
DIMENSIONS={'Browser','Device','Country/Region','OS','Source','Medium','Campaign','Channel','URL'}
FIELDS={'distinctUserCount','pagesPerSessionPercentage','pagesViews','totalSessionCount','totalBotSessionCount','distantUserCount','PagesPerSessionPercentage','averageScrollDepth','scrollDepth','engagementTime','activeTime','totalTime','totalCount','sessionsCount','sessionsWithMetricPercentage','sessionsWithoutMetricPercentage','pagesCount','subTotal','total','percentage','average','count'}
def stamp(): return dt.datetime.now(dt.timezone.utc).isoformat()
def hashed(value): return hashlib.sha256(value.encode()).hexdigest()
def load_token(env_path='/opt/data/.env'):
# No dependency on a gateway restart or its previously inherited environment.
try:
from dotenv import dotenv_values
return dotenv_values(env_path).get('CLARITY_API_TOKEN') or os.getenv('CLARITY_API_TOKEN')
except ImportError:
return os.getenv('CLARITY_API_TOKEN')
class ClarityCollector:
def __init__(self,store,session,token,project,days=3,dimensions=(),day=None,project_suffix=''):
if days not in (1,2,3) or len(dimensions)>3 or len(set(dimensions))!=len(dimensions) or any(d not in DIMENSIONS for d in dimensions):
raise ValueError('invalid Clarity window/dimensions')
self.db,self.session,self.token=store.db,session,token
self.project=hashed(project+project_suffix); self.days=days; self.dimensions=tuple(dimensions)
self.day=str(day or dt.datetime.now(dt.timezone.utc).date())
self.db.execute('CREATE TABLE IF NOT EXISTS clarity_cache (key TEXT PRIMARY KEY, project TEXT, day TEXT, payload TEXT NOT NULL)')
def base(self):
return {'source':'clarity','status':'unavailable','metrics':None,'scope':'project','observedAt':stamp(),
'window':{'kind':'rolling','numOfDays':self.days,'timezone':'UTC','asOf':stamp(),'independentOfGooglePeriod':True},
'dimensions':list(self.dimensions),'coverage':{'maxRows':1000,'paginated':False,'possiblyTruncated':False,'selection':'allowlisted numeric aggregates only; categorical and URL reports omitted'},
'provenance':{'endpoint':CLARITY_ENDPOINT,'projectFingerprint':self.project}}
def sanitize(self,data,result):
if not isinstance(data,list) or not data: raise ValueError()
groups={}; row_count=0
for item in data:
if not isinstance(item,dict) or not isinstance(item.get('metricName'),str) or not isinstance(item.get('information'),list) or not item['information']: raise ValueError()
# Only known metrics names; never persist arbitrary provider text.
name=item['metricName']
if name not in {'Traffic','ScrollDepth','Scroll Depth','EngagementTime','Engagement Time','DeadClickCount','Dead Click Count','ExcessiveScroll','Excessive Scroll','RageClickCount','Rage Click Count','QuickbackClick','Quickback Click','ScriptErrorCount','Script Error Count','ErrorClickCount','Error Click Count'}: continue
if name in groups: raise ValueError()
rows=[]
for raw in item['information']:
if not isinstance(raw,dict): raise ValueError()
clean={}
for key in FIELDS & raw.keys():
value=raw[key]
if isinstance(value,bool): raise ValueError()
number=float(value)
if not math.isfinite(number) or number<0: raise ValueError()
clean[key]=int(number) if number.is_integer() else number
if not clean: raise ValueError()
if self.dimensions:
# URL hash permits exact matching without retaining URL/query PII.
if any(not isinstance(raw.get(d),str) for d in self.dimensions): raise ValueError()
clean['_dimensionHashes']={d:hashed(raw[d]) for d in self.dimensions}
rows.append(clean)
if len(rows)>1000: raise ValueError()
row_count+=len(rows); groups[name]=rows
if len(rows)==1000: result['coverage']['possiblyTruncated']=True
if not groups: raise ValueError()
result.update(status='available',metrics=groups,httpStatus=200)
result['coverage']['sanitizedRowCount']=row_count
return result
def collect(self,url=None):
result=self.base()
if not self.token:
result['error']='credentials_not_configured'; return result
key=hashed(json.dumps([self.project,self.day,self.days,self.dimensions]))
# Reserve before HTTP, including failures. Concurrent callers reuse pending
# unavailable result rather than each consuming a project API request.
with self.db:
self.db.execute('BEGIN IMMEDIATE')
row=self.db.execute('SELECT payload FROM clarity_cache WHERE key=?',(key,)).fetchone()
if row: result=json.loads(row[0])
else:
count=self.db.execute('SELECT COUNT(*) FROM clarity_cache WHERE project=? AND day=?',(self.project,self.day)).fetchone()[0]
if count>=10:
result['error']='daily_budget_exhausted'; return result
result['error']='collection_pending'
self.db.execute('INSERT INTO clarity_cache VALUES (?,?,?,?)',(key,self.project,self.day,json.dumps(result)))
if not row:
try:
params={'numOfDays':self.days,**{f'dimension{i+1}':d for i,d in enumerate(self.dimensions)}}
response=self.session.get(CLARITY_ENDPOINT,params=params,headers={'Authorization':'Bearer '+self.token},timeout=(5,30),allow_redirects=False)
result['httpStatus']=response.status_code
if response.status_code!=200: result['error']=f'http_{response.status_code}'
else:
result.pop('error',None)
result=self.sanitize(response.json(),result)
except requests.RequestException: result.update(status='unavailable',metrics=None,error='network_error')
except (ValueError,TypeError,KeyError,OverflowError): result.update(status='unavailable',metrics=None,error='malformed_response')
with self.db: self.db.execute('UPDATE clarity_cache SET payload=? WHERE key=?',(json.dumps(result),key))
result=json.loads(json.dumps(result))
if 'URL' in self.dimensions and result['status']=='available':
matches={name:[{k:v for k,v in r.items() if k!='_dimensionHashes'} for r in rows if r.get('_dimensionHashes',{}).get('URL')==hashed(url or '')] for name,rows in result['metrics'].items()}
matches={name:rows for name,rows in matches.items() if rows}
if url and matches: result.update(scope='article',metrics=matches)
else: result.update(status='unavailable',metrics=None,error='exact_page_not_reported')
elif result.get('metrics'):
# Dimension names are provenance; identities are not report payload.
for rows in result['metrics'].values():
for r in rows: r.pop('_dimensionHashes',None)
return result
@@ -0,0 +1,168 @@
{
"status": "available",
"httpStatus": 200,
"error": null,
"scope": "project",
"window": {
"kind": "rolling",
"numOfDays": 3,
"timezone": "UTC",
"asOf": "2026-10-28T07:57:38.557869+00:00",
"independentOfGooglePeriod": true
},
"coverage": {
"maxRows": 1000,
"paginated": false,
"possiblyTruncated": false,
"sanitizedRowCount": 9
},
"metricNames": [
"DeadClickCount",
"EngagementTime",
"ErrorClickCount",
"ExcessiveScroll",
"QuickbackClick",
"RageClickCount",
"ScriptErrorCount",
"ScrollDepth",
"Traffic"
],
"aggregateShape": [
{
"rowCount": 1,
"fieldNames": [
"pagesViews",
"sessionsCount",
"sessionsWithMetricPercentage",
"sessionsWithoutMetricPercentage",
"subTotal"
]
},
{
"rowCount": 1,
"fieldNames": [
"pagesViews",
"sessionsCount",
"sessionsWithMetricPercentage",
"sessionsWithoutMetricPercentage",
"subTotal"
]
},
{
"rowCount": 1,
"fieldNames": [
"pagesViews",
"sessionsCount",
"sessionsWithMetricPercentage",
"sessionsWithoutMetricPercentage",
"subTotal"
]
},
{
"rowCount": 1,
"fieldNames": [
"pagesViews",
"sessionsCount",
"sessionsWithMetricPercentage",
"sessionsWithoutMetricPercentage",
"subTotal"
]
},
{
"rowCount": 1,
"fieldNames": [
"pagesViews",
"sessionsCount",
"sessionsWithMetricPercentage",
"sessionsWithoutMetricPercentage",
"subTotal"
]
},
{
"rowCount": 1,
"fieldNames": [
"pagesViews",
"sessionsCount",
"sessionsWithMetricPercentage",
"sessionsWithoutMetricPercentage",
"subTotal"
]
},
{
"rowCount": 1,
"fieldNames": [
"averageScrollDepth"
]
},
{
"rowCount": 1,
"fieldNames": [
"distinctUserCount",
"pagesPerSessionPercentage",
"totalBotSessionCount",
"totalSessionCount"
]
},
{
"rowCount": 1,
"fieldNames": [
"activeTime",
"totalTime"
]
},
{
"rowCount": 3,
"fieldNames": [
"name",
"sessionsCount"
]
},
{
"rowCount": 2,
"fieldNames": [
"name",
"sessionsCount"
]
},
{
"rowCount": 4,
"fieldNames": [
"name",
"sessionsCount"
]
},
{
"rowCount": 4,
"fieldNames": [
"name",
"sessionsCount"
]
},
{
"rowCount": 10,
"fieldNames": [
"name",
"sessionsCount"
]
},
{
"rowCount": 2,
"fieldNames": [
"name",
"sessionsCount"
]
},
{
"rowCount": 10,
"fieldNames": [
"url",
"visitsCount"
]
}
],
"networkRequests": 1,
"remoteWrites": false,
"backendDelivery": "not_attempted",
"catalogMode": "unchanged",
"secretLogged": false
}
@@ -0,0 +1,32 @@
test_budget (test_clarity.ClarityTests.test_budget) ... ok
test_exact_url_only (test_clarity.ClarityTests.test_exact_url_only) ... ok
test_failures_are_null_cached_sanitized (test_clarity.ClarityTests.test_failures_are_null_cached_sanitized) ... ok
test_finding_uses_shared_collector (test_clarity.ClarityTests.test_finding_uses_shared_collector) ... ok
test_hard_daily_cap_and_day_rollover (test_clarity.ClarityTests.test_hard_daily_cap_and_day_rollover) ... ok
test_limits_missing_credentials (test_clarity.ClarityTests.test_limits_missing_credentials) ... ok
test_malformed_partial (test_clarity.ClarityTests.test_malformed_partial) ... ok
test_metadata_and_catalog (test_clarity.ClarityTests.test_metadata_and_catalog) ... ok
test_project_cache_survives_reopen_and_no_article_attribution (test_clarity.ClarityTests.test_project_cache_survives_reopen_and_no_article_attribution) ... ok
test_real_numeric_shape (test_clarity.ClarityTests.test_real_numeric_shape) ... ok
test_contract_null (test_delivery.DeliveryTests.test_contract_null) ... ok
test_retry_and_ack (test_delivery.DeliveryTests.test_retry_and_ack) ... ok
test_stale_feedback_blocks_delivery (test_delivery.DeliveryTests.test_stale_feedback_blocks_delivery) ... ok
test_http_pagination_collection_and_delivery (test_integration.Integration.test_http_pagination_collection_and_delivery) ... ok
test_aggregate_sales_do_not_prove_keyword_causality (test_tracker.TestConservativeDecisions.test_aggregate_sales_do_not_prove_keyword_causality) ... ok
test_directive_is_carried_without_inventing_action (test_tracker.TestConservativeDecisions.test_directive_is_carried_without_inventing_action) ... ok
test_backend_pagination (test_worker.WorkerTests.test_backend_pagination) ... ok
test_conservative_and_directives (test_worker.WorkerTests.test_conservative_and_directives) ... ok
test_exact_ga4 (test_worker.WorkerTests.test_exact_ga4) ... ok
test_ga4_pagination_and_row_count (test_worker.WorkerTests.test_ga4_pagination_and_row_count) ... ok
test_gsc_pagination (test_worker.WorkerTests.test_gsc_pagination) ... ok
test_idempotent_local_and_revision (test_worker.WorkerTests.test_idempotent_local_and_revision) ... ok
test_malformed_json_sanitized (test_worker.WorkerTests.test_malformed_json_sanitized) ... ok
test_partial_pagination_discards_aggregates (test_worker.WorkerTests.test_partial_pagination_discards_aggregates) ... ok
test_period (test_worker.WorkerTests.test_period) ... ok
test_timeout (test_worker.WorkerTests.test_timeout) ... ok
test_unavailable_is_null (test_worker.WorkerTests.test_unavailable_is_null) ... ok
----------------------------------------------------------------------
Ran 27 tests in 1.568s
OK
@@ -9,6 +9,7 @@ from pathlib import Path
import sqlite3
from urllib.parse import quote, urlsplit
import requests
from clarity_collector import ClarityCollector, CLARITY_ENDPOINT, load_token
GSC_CREDS_PATH='/opt/data/credentials/gsc-service-account.json'
GA4_PROPERTY_ID='552453291'
@@ -114,13 +115,13 @@ class Backend:
def decision(finding,sources):
return {'action':'INSUFFICIENT_EVIDENCE','reason':'No approved comparative decision policy; unavailable signals cannot justify scaling or pruning. Correlation is not keyword-caused sales.','mandatoryDirective':finding.get('ceoFeedback'),'feedbackRevision':finding.get('feedbackVersion',finding.get('feedbackRevision',finding.get('feedbackSubmittedAt')))}
def collect_finding(finding,collector,dates,store):
def collect_finding(finding,collector,dates,store,clarity=None):
blog=finding.get('blog') or {}; url=finding.get('articleUrl') or blog.get('canonicalUrl')
if not url and blog.get('slug'): url='https://canina.ir/blog/'+quote(blog['slug'],safe='')
if not url or urlsplit(url).hostname!='canina.ir' or urlsplit(url).scheme!='https':
store.alert('BACKEND','missing_or_invalid_article_url',finding['id']); return None
sources={'gsc':collector.gsc_page(url,dates),'gscQueries':collector.gsc(url,dates),'ga4':collector.ga4(url,dates),'clarity':unavailable('reporting_credentials_not_configured'),'commerce':unavailable('article_to_order_linkage_not_verified')}
payload={'findingId':finding['id'],'articleUrl':url,'periodStart':dates[0],'periodEnd':dates[1],'feedbackRevision':finding.get('feedbackVersion',finding.get('feedbackRevision',finding.get('feedbackSubmittedAt'))),'sources':sources,'decision':decision(finding,sources),'observedAt':now(),'periodAlignment':'same date labels; GSC Pacific and GA4 property timezone may differ; not session-level alignment'}
sources={'gsc':collector.gsc_page(url,dates),'gscQueries':collector.gsc(url,dates),'ga4':collector.ga4(url,dates),'clarity':clarity.collect(url) if clarity else unavailable('collector_not_configured'),'commerce':unavailable('article_to_order_linkage_not_verified')}
payload={'findingId':finding['id'],'articleUrl':url,'periodStart':dates[0],'periodEnd':dates[1],'feedbackRevision':finding.get('feedbackVersion',finding.get('feedbackRevision',finding.get('feedbackSubmittedAt'))),'sources':sources,'decision':decision(finding,sources),'observedAt':now(),'periodAlignment':'same date labels; GSC Pacific and GA4 property timezone may differ; not session-level alignment; Clarity has its own independent rolling UTC window'}
for source,data in sources.items():
if data['status']=='unavailable': store.alert(source.upper(),data['error'],finding['id'])
store.snapshot(payload); return payload
@@ -135,6 +136,8 @@ def snapshot_body(payload,key):
if available:
for k,v in mapping.items(): body[v]=metrics[k]
else: metadata[source]['error']=data.get('error','no_reported_metrics')
clarity=sources.get('clarity') or unavailable('missing_source')
metadata['clarity']=dict(clarity,source='clarity',fetchedAt=clarity.get('observedAt',now()))
metadata['worker']={'feedbackVersion':payload.get('feedbackRevision'),'mandatoryDirective':payload.get('decision',{}).get('mandatoryDirective'),'periodAlignment':payload.get('periodAlignment'),'sources':sources}
body['sourceMetadata']=metadata
return body
@@ -183,8 +186,9 @@ def main():
except Exception as exc:
store.alert('BACKEND',type(exc).__name__,'global'); Path(args.receipt).write_text(json.dumps({'status':'blocked','error':type(exc).__name__,'observedAt':now()})); return 1
else: p.error('--backend or --probe-url required')
clarity=ClarityCollector(store,requests.Session(),load_token(),'canina.ir')
for finding in findings:
if finding.get('blog') or finding.get('articleUrl'): results.append(collect_finding(finding,collector,dates,store))
if finding.get('blog') or finding.get('articleUrl'): results.append(collect_finding(finding,collector,dates,store,clarity))
Path(args.receipt).write_text(json.dumps({'observedAt':now(),'period':dates,'findingCount':len(findings),'collectedCount':sum(x is not None for x in results),'results':results,'remoteWrites':False},ensure_ascii=False,indent=2))
return 0
if __name__=='__main__': raise SystemExit(main())
@@ -0,0 +1,2 @@
# Plan
Implement separate ClarityCollector with SQLite durable cache/budget and sanitized numeric metric schema. Wire optional collector to collect_finding and CLI using same state database; provide independent read-only probe for single real call. Backend accepts sourceMetadata object and allows clarity; add clarity there, not new backend columns. Unit tests use transport doubles; real probe separately proves official API connectivity, not backend delivery.
@@ -0,0 +1,7 @@
# Clarity collection
Scope: only scripts/seo-performance; backend/storefront/schema paths are owned by concurrent developers. Base source SHA 312fcc2acc8f07dcf20d3d6ee0165ba8e619122e; shared worktree dirty outside this scope. No frontend/backend changes or graph regeneration required; direct worker and DTO/service contracts inspected. Existing graph is unbound navigation only.
Collect real read-only Clarity dashboard aggregates from the official endpoint with Bearer token, redirects disabled and bounded timeout. Token is loaded in-process from provisioned dotenv, never printed. Keep rolling UTC 1–3 day window independent of GSC/GA4 dates. No historical reconstruction or keyword/order attribution. Preserve catalog-only commerce state.
Acceptance: durable project/UTC-day/window/dimensions cache reuses one request across articles and process reopen; account for attempted calls before transport and cap at ten per project/day; fail closed on invalid/missing credentials, HTTP, network or malformed/partial data; cache failures too. Store only allowlisted numeric aggregate fields and hashed exact URL identities, no raw URLs, PII, bodies or token. Default project aggregate collection is explicitly not article data. Optional URL breakdown may be article-scoped only for exact URL matches; no guessed or normalized matching, no pagination and 1000-row truncation disclosed. Backend sourceMetadata receives clarity provenance without assigning it to the snapshot's Google dates. No remote backend mutations, deployment, scheduling or restart authorized.
@@ -0,0 +1,6 @@
# Tasks
1. Write collector contract tests; capture behavioral RED.
2. Implement sanitization, bounded official transport, durable caching/budget and worker metadata integration.
3. Run GREEN plus complete worker suite.
4. Single real read-only Clarity probe with dotenv in process, durable state and sanitized receipt; no raw data output.
5. Commit explicit owned paths only. Report backend delivery and rollout as unverified/not attempted.
+72
View File
@@ -0,0 +1,72 @@
import datetime as dt
import json
import tempfile
import unittest
from pathlib import Path
from unittest.mock import Mock
import requests
import seo_performance_tracker as w
DATA=[{'metricName':'Traffic','information':[{'totalSessionCount':'12','distantUserCount':'8','email':'private@example.invalid'}]}]
class ClarityTests(unittest.TestCase):
def setUp(self):
self.tmp=tempfile.TemporaryDirectory(); self.store=w.Store(Path(self.tmp.name)/'state.db'); self.session=Mock()
self.session.get.return_value.status_code=200; self.session.get.return_value.json.return_value=DATA
def tearDown(self): self.store.db.close(); self.tmp.cleanup()
def collector(self,**kw): return w.ClarityCollector(self.store,self.session,'test-token','project',**kw)
def test_project_cache_survives_reopen_and_no_article_attribution(self):
a=self.collector().collect('https://canina.ir/blog/a'); self.store.db.close(); self.store=w.Store(Path(self.tmp.name)/'state.db')
b=self.collector().collect('https://canina.ir/blog/b')
self.assertEqual(self.session.get.call_count,1); self.assertEqual(a,b)
self.assertEqual(a['scope'],'project'); self.assertEqual(a['metrics']['Traffic'][0]['totalSessionCount'],12)
self.assertNotIn('private',json.dumps(a)); self.assertEqual(a['window']['numOfDays'],3)
args=self.session.get.call_args
self.assertEqual(args.args[0],w.CLARITY_ENDPOINT); self.assertFalse(args.kwargs['allow_redirects']); self.assertEqual(args.kwargs['timeout'],(5,30))
self.assertEqual(args.kwargs['headers'],{'Authorization':'Bearer test-token'})
def test_failures_are_null_cached_sanitized(self):
for code in (401,403,429,302,500):
self.session.get.return_value.status_code=code
r=self.collector(project_suffix= str(code)).collect()
self.assertEqual(r['error'],f'http_{code}'); self.assertIsNone(r['metrics'])
self.session.get.side_effect=requests.Timeout('test-token private')
r=self.collector(project_suffix='timeout').collect(); self.assertEqual(r['error'],'network_error'); self.assertNotIn('test-token',json.dumps(r))
def test_malformed_partial(self):
for i,data in enumerate(({},[{'metricName':'Traffic','information':[]}],DATA+[{'metricName':'Oops'}], [{'metricName':'Traffic','information':[{'totalSessionCount':'NaN'}]}])):
self.session.get.return_value.json.return_value=data
r=self.collector(project_suffix=str(i)).collect(); self.assertEqual(r['status'],'unavailable'); self.assertIsNone(r['metrics'])
def test_exact_url_only(self):
self.session.get.return_value.json.return_value=[{'metricName':'Traffic','information':[{'URL':'https://canina.ir/blog/a','totalSessionCount':'2'},{'URL':'https://canina.ir/blog/b?email=private','totalSessionCount':'9'}]}]
c=self.collector(dimensions=('URL',)); a=c.collect('https://canina.ir/blog/a'); b=c.collect('https://canina.ir/blog/a/')
self.assertEqual(a['scope'],'article'); self.assertEqual(a['metrics']['Traffic'][0]['totalSessionCount'],2)
self.assertIsNone(b['metrics']); self.assertEqual(b['error'],'exact_page_not_reported'); self.assertNotIn('private',json.dumps(a)); self.assertEqual(self.session.get.call_count,1)
def test_limits_missing_credentials(self):
for days in (0,4):
with self.assertRaises(ValueError): self.collector(days=days)
with self.assertRaises(ValueError): self.collector(dimensions=('URL','Browser','OS','Device'))
self.assertEqual(w.ClarityCollector(self.store,self.session,None,'project').collect()['error'],'credentials_not_configured'); self.session.get.assert_not_called()
def test_budget(self):
for n in range(11): r=self.collector(dimensions=('URL',) if n%2 else (),project_suffix='budget',day=dt.date(2026,10,28)+dt.timedelta(days=0),days=(n%3)+1).collect()
# Six unique supported windows consume six requests; repeats reuse cache.
self.assertEqual(self.session.get.call_count,6)
def test_hard_daily_cap_and_day_rollover(self):
c=self.collector(day=dt.date(2026,10,28))
with self.store.db:
for n in range(10): self.store.db.execute('INSERT INTO clarity_cache VALUES (?,?,?,?)',(str(n),c.project,c.day,'{}'))
self.assertEqual(c.collect()['error'],'daily_budget_exhausted'); self.session.get.assert_not_called()
self.assertEqual(self.collector(day=dt.date(2026,10,29)).collect()['status'],'available')
def test_real_numeric_shape(self):
self.session.get.return_value.json.return_value=[{'metricName':'Traffic','information':[{'totalSessionCount':'12','distinctUserCount':'8','pagesPerSessionPercentage':1.2}]},{'metricName':'DeadClickCount','information':[{'pagesViews':'3','sessionsCount':'2','subTotal':'1'}]}]
r=self.collector().collect()
self.assertEqual(r['metrics']['Traffic'][0]['distinctUserCount'],8)
self.assertEqual(r['metrics']['DeadClickCount'][0]['pagesViews'],3)
def test_finding_uses_shared_collector(self):
google=Mock(); google.gsc_page.return_value=w.unavailable('test'); google.gsc.return_value=w.unavailable('test'); google.ga4.return_value=w.unavailable('test')
c=self.collector()
for fid in ('a','b'):
p=w.collect_finding({'id':fid,'articleUrl':'https://canina.ir/blog/'+fid},google,('2026-10-01','2026-10-14'),self.store,c)
self.assertEqual(p['sources']['clarity']['scope'],'project')
self.assertEqual(self.session.get.call_count,1)
def test_metadata_and_catalog(self):
p={'periodStart':'2026-10-01','periodEnd':'2026-10-14','sources':{'clarity':self.collector().collect()},'feedbackRevision':1}
b=w.snapshot_body(p,'k'); self.assertEqual(b['sourceMetadata']['clarity']['scope'],'project'); self.assertNotIn('attributedOrdersCount',b)
if __name__=='__main__': unittest.main()
+36
View File
@@ -0,0 +1,36 @@
#!/usr/bin/env python3
"""Single official read-only request, durable cached, aggregate receipt only."""
import argparse
import json
from pathlib import Path
import re
import requests
from clarity_collector import ClarityCollector, load_token
from seo_performance_tracker import Store
class ShapeSession:
def __init__(self): self.session=requests.Session(); self.shape=[]; self.calls=0
def get(self,*args,**kwargs):
self.calls+=1
response=self.session.get(*args,**kwargs)
if response.status_code==200:
try:
data=response.json()
if isinstance(data,list):
for group in data:
if not isinstance(group,dict): continue
rows=group.get('information')
keys=set().union(*(r.keys() for r in rows if isinstance(r,dict))) if isinstance(rows,list) else set()
self.shape.append({'rowCount':len(rows) if isinstance(rows,list) else None,'fieldNames':sorted(k for k in keys if re.fullmatch(r'[A-Za-z]{1,60}',k))})
except (ValueError,TypeError): pass
return response
def main():
p=argparse.ArgumentParser(); p.add_argument('--state',required=True); p.add_argument('--receipt',required=True); a=p.parse_args()
store=Store(a.state); session=ShapeSession()
result=ClarityCollector(store,session,load_token(),'canina.ir').collect()
receipt={'status':result['status'],'httpStatus':result.get('httpStatus'),'error':result.get('error'),'scope':result['scope'],'window':result['window'],'coverage':result['coverage'],'metricNames':sorted((result.get('metrics') or {}).keys()),'aggregateShape':session.shape,'networkRequests':session.calls,'remoteWrites':False,'backendDelivery':'not_attempted','catalogMode':'unchanged','secretLogged':False}
Path(a.receipt).write_text(json.dumps(receipt,indent=2)+'\n')
print(json.dumps(receipt,indent=2)); store.db.close()
return 0 if result['status']=='available' else 1
if __name__=='__main__': raise SystemExit(main())