Files
canina/scripts/seo-performance/clarity_collector.py
T

108 lines
7.3 KiB
Python

"""Clarity rolling-window export. Persist sanitized aggregates only, never raw bodies."""
import datetime as dt
import hashlib
import json
import math
import os
import requests
CLARITY_ENDPOINT='https://www.clarity.ms/export-data/api/v1/project-live-insights'
DIMENSIONS={'Browser','Device','Country/Region','OS','Source','Medium','Campaign','Channel','URL'}
FIELDS={'distinctUserCount','pagesPerSessionPercentage','pagesViews','totalSessionCount','totalBotSessionCount','distantUserCount','PagesPerSessionPercentage','averageScrollDepth','scrollDepth','engagementTime','activeTime','totalTime','totalCount','sessionsCount','sessionsWithMetricPercentage','sessionsWithoutMetricPercentage','pagesCount','subTotal','total','percentage','average','count'}
def stamp(): return dt.datetime.now(dt.timezone.utc).isoformat()
def hashed(value): return hashlib.sha256(value.encode()).hexdigest()
def load_token(env_path='/opt/data/.env'):
# No dependency on a gateway restart or its previously inherited environment.
try:
from dotenv import dotenv_values
return dotenv_values(env_path).get('CLARITY_API_TOKEN') or os.getenv('CLARITY_API_TOKEN')
except ImportError:
return os.getenv('CLARITY_API_TOKEN')
class ClarityCollector:
def __init__(self,store,session,token,project,days=3,dimensions=(),day=None,project_suffix=''):
if days not in (1,2,3) or len(dimensions)>3 or len(set(dimensions))!=len(dimensions) or any(d not in DIMENSIONS for d in dimensions):
raise ValueError('invalid Clarity window/dimensions')
self.db,self.session,self.token=store.db,session,token
self.project=hashed(project+project_suffix); self.days=days; self.dimensions=tuple(dimensions)
self.day=str(day or dt.datetime.now(dt.timezone.utc).date())
self.db.execute('CREATE TABLE IF NOT EXISTS clarity_cache (key TEXT PRIMARY KEY, project TEXT, day TEXT, payload TEXT NOT NULL)')
def base(self):
return {'source':'clarity','status':'unavailable','metrics':None,'scope':'project','observedAt':stamp(),
'window':{'kind':'rolling','numOfDays':self.days,'timezone':'UTC','asOf':stamp(),'independentOfGooglePeriod':True},
'dimensions':list(self.dimensions),'coverage':{'maxRows':1000,'paginated':False,'possiblyTruncated':False,'selection':'allowlisted numeric aggregates only; categorical and URL reports omitted'},
'provenance':{'endpoint':CLARITY_ENDPOINT,'projectFingerprint':self.project}}
def sanitize(self,data,result):
if not isinstance(data,list) or not data: raise ValueError()
groups={}; row_count=0
for item in data:
if not isinstance(item,dict) or not isinstance(item.get('metricName'),str) or not isinstance(item.get('information'),list) or not item['information']: raise ValueError()
# Only known metrics names; never persist arbitrary provider text.
name=item['metricName']
if name not in {'Traffic','ScrollDepth','Scroll Depth','EngagementTime','Engagement Time','DeadClickCount','Dead Click Count','ExcessiveScroll','Excessive Scroll','RageClickCount','Rage Click Count','QuickbackClick','Quickback Click','ScriptErrorCount','Script Error Count','ErrorClickCount','Error Click Count'}: continue
if name in groups: raise ValueError()
rows=[]
for raw in item['information']:
if not isinstance(raw,dict): raise ValueError()
clean={}
for key in FIELDS & raw.keys():
value=raw[key]
if isinstance(value,bool): raise ValueError()
number=float(value)
if not math.isfinite(number) or number<0: raise ValueError()
clean[key]=int(number) if number.is_integer() else number
if not clean: raise ValueError()
if self.dimensions:
# URL hash permits exact matching without retaining URL/query PII.
if any(not isinstance(raw.get(d),str) for d in self.dimensions): raise ValueError()
clean['_dimensionHashes']={d:hashed(raw[d]) for d in self.dimensions}
rows.append(clean)
if len(rows)>1000: raise ValueError()
row_count+=len(rows); groups[name]=rows
if len(rows)==1000: result['coverage']['possiblyTruncated']=True
if not groups: raise ValueError()
result.update(status='available',metrics=groups,httpStatus=200)
result['coverage']['sanitizedRowCount']=row_count
return result
def collect(self,url=None):
result=self.base()
if not self.token:
result['error']='credentials_not_configured'; return result
key=hashed(json.dumps([self.project,self.day,self.days,self.dimensions]))
# Reserve before HTTP, including failures. Concurrent callers reuse pending
# unavailable result rather than each consuming a project API request.
with self.db:
self.db.execute('BEGIN IMMEDIATE')
row=self.db.execute('SELECT payload FROM clarity_cache WHERE key=?',(key,)).fetchone()
if row: result=json.loads(row[0])
else:
count=self.db.execute('SELECT COUNT(*) FROM clarity_cache WHERE project=? AND day=?',(self.project,self.day)).fetchone()[0]
if count>=10:
result['error']='daily_budget_exhausted'; return result
result['error']='collection_pending'
self.db.execute('INSERT INTO clarity_cache VALUES (?,?,?,?)',(key,self.project,self.day,json.dumps(result)))
if not row:
try:
params={'numOfDays':self.days,**{f'dimension{i+1}':d for i,d in enumerate(self.dimensions)}}
response=self.session.get(CLARITY_ENDPOINT,params=params,headers={'Authorization':'Bearer '+self.token},timeout=(5,30),allow_redirects=False)
result['httpStatus']=response.status_code
if response.status_code!=200: result['error']=f'http_{response.status_code}'
else:
result.pop('error',None)
result=self.sanitize(response.json(),result)
except requests.RequestException: result.update(status='unavailable',metrics=None,error='network_error')
except (ValueError,TypeError,KeyError,OverflowError): result.update(status='unavailable',metrics=None,error='malformed_response')
with self.db: self.db.execute('UPDATE clarity_cache SET payload=? WHERE key=?',(json.dumps(result),key))
result=json.loads(json.dumps(result))
if 'URL' in self.dimensions and result['status']=='available':
matches={name:[{k:v for k,v in r.items() if k!='_dimensionHashes'} for r in rows if r.get('_dimensionHashes',{}).get('URL')==hashed(url or '')] for name,rows in result['metrics'].items()}
matches={name:rows for name,rows in matches.items() if rows}
if url and matches: result.update(scope='article',metrics=matches)
else: result.update(status='unavailable',metrics=None,error='exact_page_not_reported')
elif result.get('metrics'):
# Dimension names are provenance; identities are not report payload.
for rows in result['metrics'].values():
for r in rows: r.pop('_dimensionHashes',None)
return result