diff --git a/scripts/seo-performance/CLARITY.md b/scripts/seo-performance/CLARITY.md new file mode 100644 index 00000000..17b2dfc3 --- /dev/null +++ b/scripts/seo-performance/CLARITY.md @@ -0,0 +1,17 @@ +# Clarity worker integration + +Worker CLI now instantiates one shared Clarity collector and includes independent provenance in `sourceMetadata.clarity`. Uses the existing metadata contract; no backend column/schema changes. Default scope is project, never an article's measurement or keyword-caused conversions. Missing commerce linkage stays null; no commerce activation or purchase-error changes. + +Run with `/opt/hermes/.venv/bin/python` (dotenv available) to read the provisioned `/opt/data/.env` token in-process without restart. Never pass the token on the command line. Regular CLI retains explicit local state and read-only behavior. Use the same durable `--state` path on future runs so cache/budget survives. The verification state is `/opt/data/clarity-worker-state.sqlite` outside Git; worker default `tracker.sqlite` is separate unless explicitly selected. + +Read-only standalone verification: + +``` +/opt/hermes/.venv/bin/python verify_clarity.py --state /opt/data/clarity-worker-state.sqlite --receipt evidence/clarity-live-receipt.json +``` + +Only official GET endpoint, Bearer header, redirects off, connect/read timeouts 5/30s. No retries or pagination. API reports rolling 24/48/72h (default 3 days), UTC, not GSC's 14-day period. Max three valid dimensions, 1000 rows, ten attempted calls per UTC day/project **in the shared local state**. External callers/setup requests are not observable by this budget; HTTP429 is cached unavailable. Failed/pending collections are cached for the day too; crashed reservations intentionally remain unavailable rather than silently retrying. Project identifier is explicit; all callers for a project must use the same identifier/state. + +Persist only allowlisted numeric aggregate fields; omit categorical/popular-page reports and arbitrary text. URL breakdown is opt-in via constructor; only an exact URL dimension hash can produce article scope. No URL normalization/inference, query strings or raw provider response persisted/logged. A row-limit boundary is disclosed as possibly truncated, not paginated. Missing/malformed responses remain unavailable with null metrics, not zero. + +Evidence: seven initial RED failures for missing collector; full final worker suite recorded in `evidence/clarity-tests.txt`. Single real API request returned HTTP200 with nine allowlisted metric groups / nine aggregate rows. Sanitized receipt `evidence/clarity-live-receipt.json` contains shape/status only. Actual backend delivery of Clarity metadata was not attempted/verified; no deployment, push, cron changes or runtime restart. No article-specific Clarity coverage verified by the project-scoped live probe. diff --git a/scripts/seo-performance/clarity_collector.py b/scripts/seo-performance/clarity_collector.py new file mode 100644 index 00000000..3521b03a --- /dev/null +++ b/scripts/seo-performance/clarity_collector.py @@ -0,0 +1,107 @@ +"""Clarity rolling-window export. Persist sanitized aggregates only, never raw bodies.""" +import datetime as dt +import hashlib +import json +import math +import os +import requests + +CLARITY_ENDPOINT='https://www.clarity.ms/export-data/api/v1/project-live-insights' +DIMENSIONS={'Browser','Device','Country/Region','OS','Source','Medium','Campaign','Channel','URL'} +FIELDS={'distinctUserCount','pagesPerSessionPercentage','pagesViews','totalSessionCount','totalBotSessionCount','distantUserCount','PagesPerSessionPercentage','averageScrollDepth','scrollDepth','engagementTime','activeTime','totalTime','totalCount','sessionsCount','sessionsWithMetricPercentage','sessionsWithoutMetricPercentage','pagesCount','subTotal','total','percentage','average','count'} + +def stamp(): return dt.datetime.now(dt.timezone.utc).isoformat() +def hashed(value): return hashlib.sha256(value.encode()).hexdigest() +def load_token(env_path='/opt/data/.env'): + # No dependency on a gateway restart or its previously inherited environment. + try: + from dotenv import dotenv_values + return dotenv_values(env_path).get('CLARITY_API_TOKEN') or os.getenv('CLARITY_API_TOKEN') + except ImportError: + return os.getenv('CLARITY_API_TOKEN') + +class ClarityCollector: + def __init__(self,store,session,token,project,days=3,dimensions=(),day=None,project_suffix=''): + if days not in (1,2,3) or len(dimensions)>3 or len(set(dimensions))!=len(dimensions) or any(d not in DIMENSIONS for d in dimensions): + raise ValueError('invalid Clarity window/dimensions') + self.db,self.session,self.token=store.db,session,token + self.project=hashed(project+project_suffix); self.days=days; self.dimensions=tuple(dimensions) + self.day=str(day or dt.datetime.now(dt.timezone.utc).date()) + self.db.execute('CREATE TABLE IF NOT EXISTS clarity_cache (key TEXT PRIMARY KEY, project TEXT, day TEXT, payload TEXT NOT NULL)') + def base(self): + return {'source':'clarity','status':'unavailable','metrics':None,'scope':'project','observedAt':stamp(), + 'window':{'kind':'rolling','numOfDays':self.days,'timezone':'UTC','asOf':stamp(),'independentOfGooglePeriod':True}, + 'dimensions':list(self.dimensions),'coverage':{'maxRows':1000,'paginated':False,'possiblyTruncated':False,'selection':'allowlisted numeric aggregates only; categorical and URL reports omitted'}, + 'provenance':{'endpoint':CLARITY_ENDPOINT,'projectFingerprint':self.project}} + def sanitize(self,data,result): + if not isinstance(data,list) or not data: raise ValueError() + groups={}; row_count=0 + for item in data: + if not isinstance(item,dict) or not isinstance(item.get('metricName'),str) or not isinstance(item.get('information'),list) or not item['information']: raise ValueError() + # Only known metrics names; never persist arbitrary provider text. + name=item['metricName'] + if name not in {'Traffic','ScrollDepth','Scroll Depth','EngagementTime','Engagement Time','DeadClickCount','Dead Click Count','ExcessiveScroll','Excessive Scroll','RageClickCount','Rage Click Count','QuickbackClick','Quickback Click','ScriptErrorCount','Script Error Count','ErrorClickCount','Error Click Count'}: continue + if name in groups: raise ValueError() + rows=[] + for raw in item['information']: + if not isinstance(raw,dict): raise ValueError() + clean={} + for key in FIELDS & raw.keys(): + value=raw[key] + if isinstance(value,bool): raise ValueError() + number=float(value) + if not math.isfinite(number) or number<0: raise ValueError() + clean[key]=int(number) if number.is_integer() else number + if not clean: raise ValueError() + if self.dimensions: + # URL hash permits exact matching without retaining URL/query PII. + if any(not isinstance(raw.get(d),str) for d in self.dimensions): raise ValueError() + clean['_dimensionHashes']={d:hashed(raw[d]) for d in self.dimensions} + rows.append(clean) + if len(rows)>1000: raise ValueError() + row_count+=len(rows); groups[name]=rows + if len(rows)==1000: result['coverage']['possiblyTruncated']=True + if not groups: raise ValueError() + result.update(status='available',metrics=groups,httpStatus=200) + result['coverage']['sanitizedRowCount']=row_count + return result + def collect(self,url=None): + result=self.base() + if not self.token: + result['error']='credentials_not_configured'; return result + key=hashed(json.dumps([self.project,self.day,self.days,self.dimensions])) + # Reserve before HTTP, including failures. Concurrent callers reuse pending + # unavailable result rather than each consuming a project API request. + with self.db: + self.db.execute('BEGIN IMMEDIATE') + row=self.db.execute('SELECT payload FROM clarity_cache WHERE key=?',(key,)).fetchone() + if row: result=json.loads(row[0]) + else: + count=self.db.execute('SELECT COUNT(*) FROM clarity_cache WHERE project=? AND day=?',(self.project,self.day)).fetchone()[0] + if count>=10: + result['error']='daily_budget_exhausted'; return result + result['error']='collection_pending' + self.db.execute('INSERT INTO clarity_cache VALUES (?,?,?,?)',(key,self.project,self.day,json.dumps(result))) + if not row: + try: + params={'numOfDays':self.days,**{f'dimension{i+1}':d for i,d in enumerate(self.dimensions)}} + response=self.session.get(CLARITY_ENDPOINT,params=params,headers={'Authorization':'Bearer '+self.token},timeout=(5,30),allow_redirects=False) + result['httpStatus']=response.status_code + if response.status_code!=200: result['error']=f'http_{response.status_code}' + else: + result.pop('error',None) + result=self.sanitize(response.json(),result) + except requests.RequestException: result.update(status='unavailable',metrics=None,error='network_error') + except (ValueError,TypeError,KeyError,OverflowError): result.update(status='unavailable',metrics=None,error='malformed_response') + with self.db: self.db.execute('UPDATE clarity_cache SET payload=? WHERE key=?',(json.dumps(result),key)) + result=json.loads(json.dumps(result)) + if 'URL' in self.dimensions and result['status']=='available': + matches={name:[{k:v for k,v in r.items() if k!='_dimensionHashes'} for r in rows if r.get('_dimensionHashes',{}).get('URL')==hashed(url or '')] for name,rows in result['metrics'].items()} + matches={name:rows for name,rows in matches.items() if rows} + if url and matches: result.update(scope='article',metrics=matches) + else: result.update(status='unavailable',metrics=None,error='exact_page_not_reported') + elif result.get('metrics'): + # Dimension names are provenance; identities are not report payload. + for rows in result['metrics'].values(): + for r in rows: r.pop('_dimensionHashes',None) + return result diff --git a/scripts/seo-performance/evidence/clarity-live-receipt.json b/scripts/seo-performance/evidence/clarity-live-receipt.json new file mode 100644 index 00000000..eda16616 --- /dev/null +++ b/scripts/seo-performance/evidence/clarity-live-receipt.json @@ -0,0 +1,168 @@ +{ + "status": "available", + "httpStatus": 200, + "error": null, + "scope": "project", + "window": { + "kind": "rolling", + "numOfDays": 3, + "timezone": "UTC", + "asOf": "2026-10-28T07:57:38.557869+00:00", + "independentOfGooglePeriod": true + }, + "coverage": { + "maxRows": 1000, + "paginated": false, + "possiblyTruncated": false, + "sanitizedRowCount": 9 + }, + "metricNames": [ + "DeadClickCount", + "EngagementTime", + "ErrorClickCount", + "ExcessiveScroll", + "QuickbackClick", + "RageClickCount", + "ScriptErrorCount", + "ScrollDepth", + "Traffic" + ], + "aggregateShape": [ + { + "rowCount": 1, + "fieldNames": [ + "pagesViews", + "sessionsCount", + "sessionsWithMetricPercentage", + "sessionsWithoutMetricPercentage", + "subTotal" + ] + }, + { + "rowCount": 1, + "fieldNames": [ + "pagesViews", + "sessionsCount", + "sessionsWithMetricPercentage", + "sessionsWithoutMetricPercentage", + "subTotal" + ] + }, + { + "rowCount": 1, + "fieldNames": [ + "pagesViews", + "sessionsCount", + "sessionsWithMetricPercentage", + "sessionsWithoutMetricPercentage", + "subTotal" + ] + }, + { + "rowCount": 1, + "fieldNames": [ + "pagesViews", + "sessionsCount", + "sessionsWithMetricPercentage", + "sessionsWithoutMetricPercentage", + "subTotal" + ] + }, + { + "rowCount": 1, + "fieldNames": [ + "pagesViews", + "sessionsCount", + "sessionsWithMetricPercentage", + "sessionsWithoutMetricPercentage", + "subTotal" + ] + }, + { + "rowCount": 1, + "fieldNames": [ + "pagesViews", + "sessionsCount", + "sessionsWithMetricPercentage", + "sessionsWithoutMetricPercentage", + "subTotal" + ] + }, + { + "rowCount": 1, + "fieldNames": [ + "averageScrollDepth" + ] + }, + { + "rowCount": 1, + "fieldNames": [ + "distinctUserCount", + "pagesPerSessionPercentage", + "totalBotSessionCount", + "totalSessionCount" + ] + }, + { + "rowCount": 1, + "fieldNames": [ + "activeTime", + "totalTime" + ] + }, + { + "rowCount": 3, + "fieldNames": [ + "name", + "sessionsCount" + ] + }, + { + "rowCount": 2, + "fieldNames": [ + "name", + "sessionsCount" + ] + }, + { + "rowCount": 4, + "fieldNames": [ + "name", + "sessionsCount" + ] + }, + { + "rowCount": 4, + "fieldNames": [ + "name", + "sessionsCount" + ] + }, + { + "rowCount": 10, + "fieldNames": [ + "name", + "sessionsCount" + ] + }, + { + "rowCount": 2, + "fieldNames": [ + "name", + "sessionsCount" + ] + }, + { + "rowCount": 10, + "fieldNames": [ + "url", + "visitsCount" + ] + } + ], + "networkRequests": 1, + "remoteWrites": false, + "backendDelivery": "not_attempted", + "catalogMode": "unchanged", + "secretLogged": false +} diff --git a/scripts/seo-performance/evidence/clarity-tests.txt b/scripts/seo-performance/evidence/clarity-tests.txt new file mode 100644 index 00000000..4bfdd5c8 --- /dev/null +++ b/scripts/seo-performance/evidence/clarity-tests.txt @@ -0,0 +1,32 @@ +test_budget (test_clarity.ClarityTests.test_budget) ... ok +test_exact_url_only (test_clarity.ClarityTests.test_exact_url_only) ... ok +test_failures_are_null_cached_sanitized (test_clarity.ClarityTests.test_failures_are_null_cached_sanitized) ... ok +test_finding_uses_shared_collector (test_clarity.ClarityTests.test_finding_uses_shared_collector) ... ok +test_hard_daily_cap_and_day_rollover (test_clarity.ClarityTests.test_hard_daily_cap_and_day_rollover) ... ok +test_limits_missing_credentials (test_clarity.ClarityTests.test_limits_missing_credentials) ... ok +test_malformed_partial (test_clarity.ClarityTests.test_malformed_partial) ... ok +test_metadata_and_catalog (test_clarity.ClarityTests.test_metadata_and_catalog) ... ok +test_project_cache_survives_reopen_and_no_article_attribution (test_clarity.ClarityTests.test_project_cache_survives_reopen_and_no_article_attribution) ... ok +test_real_numeric_shape (test_clarity.ClarityTests.test_real_numeric_shape) ... ok +test_contract_null (test_delivery.DeliveryTests.test_contract_null) ... ok +test_retry_and_ack (test_delivery.DeliveryTests.test_retry_and_ack) ... ok +test_stale_feedback_blocks_delivery (test_delivery.DeliveryTests.test_stale_feedback_blocks_delivery) ... ok +test_http_pagination_collection_and_delivery (test_integration.Integration.test_http_pagination_collection_and_delivery) ... ok +test_aggregate_sales_do_not_prove_keyword_causality (test_tracker.TestConservativeDecisions.test_aggregate_sales_do_not_prove_keyword_causality) ... ok +test_directive_is_carried_without_inventing_action (test_tracker.TestConservativeDecisions.test_directive_is_carried_without_inventing_action) ... ok +test_backend_pagination (test_worker.WorkerTests.test_backend_pagination) ... ok +test_conservative_and_directives (test_worker.WorkerTests.test_conservative_and_directives) ... ok +test_exact_ga4 (test_worker.WorkerTests.test_exact_ga4) ... ok +test_ga4_pagination_and_row_count (test_worker.WorkerTests.test_ga4_pagination_and_row_count) ... ok +test_gsc_pagination (test_worker.WorkerTests.test_gsc_pagination) ... ok +test_idempotent_local_and_revision (test_worker.WorkerTests.test_idempotent_local_and_revision) ... ok +test_malformed_json_sanitized (test_worker.WorkerTests.test_malformed_json_sanitized) ... ok +test_partial_pagination_discards_aggregates (test_worker.WorkerTests.test_partial_pagination_discards_aggregates) ... ok +test_period (test_worker.WorkerTests.test_period) ... ok +test_timeout (test_worker.WorkerTests.test_timeout) ... ok +test_unavailable_is_null (test_worker.WorkerTests.test_unavailable_is_null) ... ok + +---------------------------------------------------------------------- +Ran 27 tests in 1.568s + +OK diff --git a/scripts/seo-performance/seo_performance_tracker.py b/scripts/seo-performance/seo_performance_tracker.py index cf6554d2..9ced8cbe 100644 --- a/scripts/seo-performance/seo_performance_tracker.py +++ b/scripts/seo-performance/seo_performance_tracker.py @@ -9,6 +9,7 @@ from pathlib import Path import sqlite3 from urllib.parse import quote, urlsplit import requests +from clarity_collector import ClarityCollector, CLARITY_ENDPOINT, load_token GSC_CREDS_PATH='/opt/data/credentials/gsc-service-account.json' GA4_PROPERTY_ID='552453291' @@ -114,13 +115,13 @@ class Backend: def decision(finding,sources): return {'action':'INSUFFICIENT_EVIDENCE','reason':'No approved comparative decision policy; unavailable signals cannot justify scaling or pruning. Correlation is not keyword-caused sales.','mandatoryDirective':finding.get('ceoFeedback'),'feedbackRevision':finding.get('feedbackVersion',finding.get('feedbackRevision',finding.get('feedbackSubmittedAt')))} -def collect_finding(finding,collector,dates,store): +def collect_finding(finding,collector,dates,store,clarity=None): blog=finding.get('blog') or {}; url=finding.get('articleUrl') or blog.get('canonicalUrl') if not url and blog.get('slug'): url='https://canina.ir/blog/'+quote(blog['slug'],safe='') if not url or urlsplit(url).hostname!='canina.ir' or urlsplit(url).scheme!='https': store.alert('BACKEND','missing_or_invalid_article_url',finding['id']); return None - sources={'gsc':collector.gsc_page(url,dates),'gscQueries':collector.gsc(url,dates),'ga4':collector.ga4(url,dates),'clarity':unavailable('reporting_credentials_not_configured'),'commerce':unavailable('article_to_order_linkage_not_verified')} - payload={'findingId':finding['id'],'articleUrl':url,'periodStart':dates[0],'periodEnd':dates[1],'feedbackRevision':finding.get('feedbackVersion',finding.get('feedbackRevision',finding.get('feedbackSubmittedAt'))),'sources':sources,'decision':decision(finding,sources),'observedAt':now(),'periodAlignment':'same date labels; GSC Pacific and GA4 property timezone may differ; not session-level alignment'} + sources={'gsc':collector.gsc_page(url,dates),'gscQueries':collector.gsc(url,dates),'ga4':collector.ga4(url,dates),'clarity':clarity.collect(url) if clarity else unavailable('collector_not_configured'),'commerce':unavailable('article_to_order_linkage_not_verified')} + payload={'findingId':finding['id'],'articleUrl':url,'periodStart':dates[0],'periodEnd':dates[1],'feedbackRevision':finding.get('feedbackVersion',finding.get('feedbackRevision',finding.get('feedbackSubmittedAt'))),'sources':sources,'decision':decision(finding,sources),'observedAt':now(),'periodAlignment':'same date labels; GSC Pacific and GA4 property timezone may differ; not session-level alignment; Clarity has its own independent rolling UTC window'} for source,data in sources.items(): if data['status']=='unavailable': store.alert(source.upper(),data['error'],finding['id']) store.snapshot(payload); return payload @@ -135,6 +136,8 @@ def snapshot_body(payload,key): if available: for k,v in mapping.items(): body[v]=metrics[k] else: metadata[source]['error']=data.get('error','no_reported_metrics') + clarity=sources.get('clarity') or unavailable('missing_source') + metadata['clarity']=dict(clarity,source='clarity',fetchedAt=clarity.get('observedAt',now())) metadata['worker']={'feedbackVersion':payload.get('feedbackRevision'),'mandatoryDirective':payload.get('decision',{}).get('mandatoryDirective'),'periodAlignment':payload.get('periodAlignment'),'sources':sources} body['sourceMetadata']=metadata return body @@ -183,8 +186,9 @@ def main(): except Exception as exc: store.alert('BACKEND',type(exc).__name__,'global'); Path(args.receipt).write_text(json.dumps({'status':'blocked','error':type(exc).__name__,'observedAt':now()})); return 1 else: p.error('--backend or --probe-url required') + clarity=ClarityCollector(store,requests.Session(),load_token(),'canina.ir') for finding in findings: - if finding.get('blog') or finding.get('articleUrl'): results.append(collect_finding(finding,collector,dates,store)) + if finding.get('blog') or finding.get('articleUrl'): results.append(collect_finding(finding,collector,dates,store,clarity)) Path(args.receipt).write_text(json.dumps({'observedAt':now(),'period':dates,'findingCount':len(findings),'collectedCount':sum(x is not None for x in results),'results':results,'remoteWrites':False},ensure_ascii=False,indent=2)) return 0 if __name__=='__main__': raise SystemExit(main()) diff --git a/scripts/seo-performance/specs/clarity/plan.md b/scripts/seo-performance/specs/clarity/plan.md new file mode 100644 index 00000000..a818848c --- /dev/null +++ b/scripts/seo-performance/specs/clarity/plan.md @@ -0,0 +1,2 @@ +# Plan +Implement separate ClarityCollector with SQLite durable cache/budget and sanitized numeric metric schema. Wire optional collector to collect_finding and CLI using same state database; provide independent read-only probe for single real call. Backend accepts sourceMetadata object and allows clarity; add clarity there, not new backend columns. Unit tests use transport doubles; real probe separately proves official API connectivity, not backend delivery. diff --git a/scripts/seo-performance/specs/clarity/spec.md b/scripts/seo-performance/specs/clarity/spec.md new file mode 100644 index 00000000..72a549f1 --- /dev/null +++ b/scripts/seo-performance/specs/clarity/spec.md @@ -0,0 +1,7 @@ +# Clarity collection + +Scope: only scripts/seo-performance; backend/storefront/schema paths are owned by concurrent developers. Base source SHA 312fcc2acc8f07dcf20d3d6ee0165ba8e619122e; shared worktree dirty outside this scope. No frontend/backend changes or graph regeneration required; direct worker and DTO/service contracts inspected. Existing graph is unbound navigation only. + +Collect real read-only Clarity dashboard aggregates from the official endpoint with Bearer token, redirects disabled and bounded timeout. Token is loaded in-process from provisioned dotenv, never printed. Keep rolling UTC 1–3 day window independent of GSC/GA4 dates. No historical reconstruction or keyword/order attribution. Preserve catalog-only commerce state. + +Acceptance: durable project/UTC-day/window/dimensions cache reuses one request across articles and process reopen; account for attempted calls before transport and cap at ten per project/day; fail closed on invalid/missing credentials, HTTP, network or malformed/partial data; cache failures too. Store only allowlisted numeric aggregate fields and hashed exact URL identities, no raw URLs, PII, bodies or token. Default project aggregate collection is explicitly not article data. Optional URL breakdown may be article-scoped only for exact URL matches; no guessed or normalized matching, no pagination and 1000-row truncation disclosed. Backend sourceMetadata receives clarity provenance without assigning it to the snapshot's Google dates. No remote backend mutations, deployment, scheduling or restart authorized. diff --git a/scripts/seo-performance/specs/clarity/tasks.md b/scripts/seo-performance/specs/clarity/tasks.md new file mode 100644 index 00000000..661bce9e --- /dev/null +++ b/scripts/seo-performance/specs/clarity/tasks.md @@ -0,0 +1,6 @@ +# Tasks +1. Write collector contract tests; capture behavioral RED. +2. Implement sanitization, bounded official transport, durable caching/budget and worker metadata integration. +3. Run GREEN plus complete worker suite. +4. Single real read-only Clarity probe with dotenv in process, durable state and sanitized receipt; no raw data output. +5. Commit explicit owned paths only. Report backend delivery and rollout as unverified/not attempted. diff --git a/scripts/seo-performance/test_clarity.py b/scripts/seo-performance/test_clarity.py new file mode 100644 index 00000000..bea3c229 --- /dev/null +++ b/scripts/seo-performance/test_clarity.py @@ -0,0 +1,72 @@ +import datetime as dt +import json +import tempfile +import unittest +from pathlib import Path +from unittest.mock import Mock +import requests +import seo_performance_tracker as w + +DATA=[{'metricName':'Traffic','information':[{'totalSessionCount':'12','distantUserCount':'8','email':'private@example.invalid'}]}] +class ClarityTests(unittest.TestCase): + def setUp(self): + self.tmp=tempfile.TemporaryDirectory(); self.store=w.Store(Path(self.tmp.name)/'state.db'); self.session=Mock() + self.session.get.return_value.status_code=200; self.session.get.return_value.json.return_value=DATA + def tearDown(self): self.store.db.close(); self.tmp.cleanup() + def collector(self,**kw): return w.ClarityCollector(self.store,self.session,'test-token','project',**kw) + def test_project_cache_survives_reopen_and_no_article_attribution(self): + a=self.collector().collect('https://canina.ir/blog/a'); self.store.db.close(); self.store=w.Store(Path(self.tmp.name)/'state.db') + b=self.collector().collect('https://canina.ir/blog/b') + self.assertEqual(self.session.get.call_count,1); self.assertEqual(a,b) + self.assertEqual(a['scope'],'project'); self.assertEqual(a['metrics']['Traffic'][0]['totalSessionCount'],12) + self.assertNotIn('private',json.dumps(a)); self.assertEqual(a['window']['numOfDays'],3) + args=self.session.get.call_args + self.assertEqual(args.args[0],w.CLARITY_ENDPOINT); self.assertFalse(args.kwargs['allow_redirects']); self.assertEqual(args.kwargs['timeout'],(5,30)) + self.assertEqual(args.kwargs['headers'],{'Authorization':'Bearer test-token'}) + def test_failures_are_null_cached_sanitized(self): + for code in (401,403,429,302,500): + self.session.get.return_value.status_code=code + r=self.collector(project_suffix= str(code)).collect() + self.assertEqual(r['error'],f'http_{code}'); self.assertIsNone(r['metrics']) + self.session.get.side_effect=requests.Timeout('test-token private') + r=self.collector(project_suffix='timeout').collect(); self.assertEqual(r['error'],'network_error'); self.assertNotIn('test-token',json.dumps(r)) + def test_malformed_partial(self): + for i,data in enumerate(({},[{'metricName':'Traffic','information':[]}],DATA+[{'metricName':'Oops'}], [{'metricName':'Traffic','information':[{'totalSessionCount':'NaN'}]}])): + self.session.get.return_value.json.return_value=data + r=self.collector(project_suffix=str(i)).collect(); self.assertEqual(r['status'],'unavailable'); self.assertIsNone(r['metrics']) + def test_exact_url_only(self): + self.session.get.return_value.json.return_value=[{'metricName':'Traffic','information':[{'URL':'https://canina.ir/blog/a','totalSessionCount':'2'},{'URL':'https://canina.ir/blog/b?email=private','totalSessionCount':'9'}]}] + c=self.collector(dimensions=('URL',)); a=c.collect('https://canina.ir/blog/a'); b=c.collect('https://canina.ir/blog/a/') + self.assertEqual(a['scope'],'article'); self.assertEqual(a['metrics']['Traffic'][0]['totalSessionCount'],2) + self.assertIsNone(b['metrics']); self.assertEqual(b['error'],'exact_page_not_reported'); self.assertNotIn('private',json.dumps(a)); self.assertEqual(self.session.get.call_count,1) + def test_limits_missing_credentials(self): + for days in (0,4): + with self.assertRaises(ValueError): self.collector(days=days) + with self.assertRaises(ValueError): self.collector(dimensions=('URL','Browser','OS','Device')) + self.assertEqual(w.ClarityCollector(self.store,self.session,None,'project').collect()['error'],'credentials_not_configured'); self.session.get.assert_not_called() + def test_budget(self): + for n in range(11): r=self.collector(dimensions=('URL',) if n%2 else (),project_suffix='budget',day=dt.date(2026,10,28)+dt.timedelta(days=0),days=(n%3)+1).collect() + # Six unique supported windows consume six requests; repeats reuse cache. + self.assertEqual(self.session.get.call_count,6) + def test_hard_daily_cap_and_day_rollover(self): + c=self.collector(day=dt.date(2026,10,28)) + with self.store.db: + for n in range(10): self.store.db.execute('INSERT INTO clarity_cache VALUES (?,?,?,?)',(str(n),c.project,c.day,'{}')) + self.assertEqual(c.collect()['error'],'daily_budget_exhausted'); self.session.get.assert_not_called() + self.assertEqual(self.collector(day=dt.date(2026,10,29)).collect()['status'],'available') + def test_real_numeric_shape(self): + self.session.get.return_value.json.return_value=[{'metricName':'Traffic','information':[{'totalSessionCount':'12','distinctUserCount':'8','pagesPerSessionPercentage':1.2}]},{'metricName':'DeadClickCount','information':[{'pagesViews':'3','sessionsCount':'2','subTotal':'1'}]}] + r=self.collector().collect() + self.assertEqual(r['metrics']['Traffic'][0]['distinctUserCount'],8) + self.assertEqual(r['metrics']['DeadClickCount'][0]['pagesViews'],3) + def test_finding_uses_shared_collector(self): + google=Mock(); google.gsc_page.return_value=w.unavailable('test'); google.gsc.return_value=w.unavailable('test'); google.ga4.return_value=w.unavailable('test') + c=self.collector() + for fid in ('a','b'): + p=w.collect_finding({'id':fid,'articleUrl':'https://canina.ir/blog/'+fid},google,('2026-10-01','2026-10-14'),self.store,c) + self.assertEqual(p['sources']['clarity']['scope'],'project') + self.assertEqual(self.session.get.call_count,1) + def test_metadata_and_catalog(self): + p={'periodStart':'2026-10-01','periodEnd':'2026-10-14','sources':{'clarity':self.collector().collect()},'feedbackRevision':1} + b=w.snapshot_body(p,'k'); self.assertEqual(b['sourceMetadata']['clarity']['scope'],'project'); self.assertNotIn('attributedOrdersCount',b) +if __name__=='__main__': unittest.main() diff --git a/scripts/seo-performance/verify_clarity.py b/scripts/seo-performance/verify_clarity.py new file mode 100644 index 00000000..31a0b8b9 --- /dev/null +++ b/scripts/seo-performance/verify_clarity.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python3 +"""Single official read-only request, durable cached, aggregate receipt only.""" +import argparse +import json +from pathlib import Path +import re +import requests +from clarity_collector import ClarityCollector, load_token +from seo_performance_tracker import Store + +class ShapeSession: + def __init__(self): self.session=requests.Session(); self.shape=[]; self.calls=0 + def get(self,*args,**kwargs): + self.calls+=1 + response=self.session.get(*args,**kwargs) + if response.status_code==200: + try: + data=response.json() + if isinstance(data,list): + for group in data: + if not isinstance(group,dict): continue + rows=group.get('information') + keys=set().union(*(r.keys() for r in rows if isinstance(r,dict))) if isinstance(rows,list) else set() + self.shape.append({'rowCount':len(rows) if isinstance(rows,list) else None,'fieldNames':sorted(k for k in keys if re.fullmatch(r'[A-Za-z]{1,60}',k))}) + except (ValueError,TypeError): pass + return response + +def main(): + p=argparse.ArgumentParser(); p.add_argument('--state',required=True); p.add_argument('--receipt',required=True); a=p.parse_args() + store=Store(a.state); session=ShapeSession() + result=ClarityCollector(store,session,load_token(),'canina.ir').collect() + receipt={'status':result['status'],'httpStatus':result.get('httpStatus'),'error':result.get('error'),'scope':result['scope'],'window':result['window'],'coverage':result['coverage'],'metricNames':sorted((result.get('metrics') or {}).keys()),'aggregateShape':session.shape,'networkRequests':session.calls,'remoteWrites':False,'backendDelivery':'not_attempted','catalogMode':'unchanged','secretLogged':False} + Path(a.receipt).write_text(json.dumps(receipt,indent=2)+'\n') + print(json.dumps(receipt,indent=2)); store.db.close() + return 0 if result['status']=='available' else 1 +if __name__=='__main__': raise SystemExit(main())