"""Clarity rolling-window export. Persist sanitized aggregates only, never raw bodies.""" import datetime as dt import hashlib import json import math import os import requests CLARITY_ENDPOINT='https://www.clarity.ms/export-data/api/v1/project-live-insights' DIMENSIONS={'Browser','Device','Country/Region','OS','Source','Medium','Campaign','Channel','URL'} FIELDS={'distinctUserCount','pagesPerSessionPercentage','pagesViews','totalSessionCount','totalBotSessionCount','distantUserCount','PagesPerSessionPercentage','averageScrollDepth','scrollDepth','engagementTime','activeTime','totalTime','totalCount','sessionsCount','sessionsWithMetricPercentage','sessionsWithoutMetricPercentage','pagesCount','subTotal','total','percentage','average','count'} def stamp(): return dt.datetime.now(dt.timezone.utc).isoformat() def hashed(value): return hashlib.sha256(value.encode()).hexdigest() def load_token(env_path='/opt/data/.env'): # No dependency on a gateway restart or its previously inherited environment. try: from dotenv import dotenv_values return dotenv_values(env_path).get('CLARITY_API_TOKEN') or os.getenv('CLARITY_API_TOKEN') except ImportError: return os.getenv('CLARITY_API_TOKEN') class ClarityCollector: def __init__(self,store,session,token,project,days=3,dimensions=(),day=None,project_suffix=''): if days not in (1,2,3) or len(dimensions)>3 or len(set(dimensions))!=len(dimensions) or any(d not in DIMENSIONS for d in dimensions): raise ValueError('invalid Clarity window/dimensions') self.db,self.session,self.token=store.db,session,token self.project=hashed(project+project_suffix); self.days=days; self.dimensions=tuple(dimensions) self.day=str(day or dt.datetime.now(dt.timezone.utc).date()) self.db.execute('CREATE TABLE IF NOT EXISTS clarity_cache (key TEXT PRIMARY KEY, project TEXT, day TEXT, payload TEXT NOT NULL)') def base(self): return {'source':'clarity','status':'unavailable','metrics':None,'scope':'project','observedAt':stamp(), 'window':{'kind':'rolling','numOfDays':self.days,'timezone':'UTC','asOf':stamp(),'independentOfGooglePeriod':True}, 'dimensions':list(self.dimensions),'coverage':{'maxRows':1000,'paginated':False,'possiblyTruncated':False,'selection':'allowlisted numeric aggregates only; categorical and URL reports omitted'}, 'provenance':{'endpoint':CLARITY_ENDPOINT,'projectFingerprint':self.project}} def sanitize(self,data,result): if not isinstance(data,list) or not data: raise ValueError() groups={}; row_count=0 for item in data: if not isinstance(item,dict) or not isinstance(item.get('metricName'),str) or not isinstance(item.get('information'),list) or not item['information']: raise ValueError() # Only known metrics names; never persist arbitrary provider text. name=item['metricName'] if name not in {'Traffic','ScrollDepth','Scroll Depth','EngagementTime','Engagement Time','DeadClickCount','Dead Click Count','ExcessiveScroll','Excessive Scroll','RageClickCount','Rage Click Count','QuickbackClick','Quickback Click','ScriptErrorCount','Script Error Count','ErrorClickCount','Error Click Count'}: continue if name in groups: raise ValueError() rows=[] for raw in item['information']: if not isinstance(raw,dict): raise ValueError() clean={} for key in FIELDS & raw.keys(): value=raw[key] if isinstance(value,bool): raise ValueError() number=float(value) if not math.isfinite(number) or number<0: raise ValueError() clean[key]=int(number) if number.is_integer() else number if not clean: raise ValueError() if self.dimensions: # URL hash permits exact matching without retaining URL/query PII. if any(not isinstance(raw.get(d),str) for d in self.dimensions): raise ValueError() clean['_dimensionHashes']={d:hashed(raw[d]) for d in self.dimensions} rows.append(clean) if len(rows)>1000: raise ValueError() row_count+=len(rows); groups[name]=rows if len(rows)==1000: result['coverage']['possiblyTruncated']=True if not groups: raise ValueError() result.update(status='available',metrics=groups,httpStatus=200) result['coverage']['sanitizedRowCount']=row_count return result def collect(self,url=None): result=self.base() if not self.token: result['error']='credentials_not_configured'; return result key=hashed(json.dumps([self.project,self.day,self.days,self.dimensions])) # Reserve before HTTP, including failures. Concurrent callers reuse pending # unavailable result rather than each consuming a project API request. with self.db: self.db.execute('BEGIN IMMEDIATE') row=self.db.execute('SELECT payload FROM clarity_cache WHERE key=?',(key,)).fetchone() if row: result=json.loads(row[0]) else: count=self.db.execute('SELECT COUNT(*) FROM clarity_cache WHERE project=? AND day=?',(self.project,self.day)).fetchone()[0] if count>=10: result['error']='daily_budget_exhausted'; return result result['error']='collection_pending' self.db.execute('INSERT INTO clarity_cache VALUES (?,?,?,?)',(key,self.project,self.day,json.dumps(result))) if not row: try: params={'numOfDays':self.days,**{f'dimension{i+1}':d for i,d in enumerate(self.dimensions)}} response=self.session.get(CLARITY_ENDPOINT,params=params,headers={'Authorization':'Bearer '+self.token},timeout=(5,30),allow_redirects=False) result['httpStatus']=response.status_code if response.status_code!=200: result['error']=f'http_{response.status_code}' else: result.pop('error',None) result=self.sanitize(response.json(),result) except requests.RequestException: result.update(status='unavailable',metrics=None,error='network_error') except (ValueError,TypeError,KeyError,OverflowError): result.update(status='unavailable',metrics=None,error='malformed_response') with self.db: self.db.execute('UPDATE clarity_cache SET payload=? WHERE key=?',(json.dumps(result),key)) result=json.loads(json.dumps(result)) if 'URL' in self.dimensions and result['status']=='available': matches={name:[{k:v for k,v in r.items() if k!='_dimensionHashes'} for r in rows if r.get('_dimensionHashes',{}).get('URL')==hashed(url or '')] for name,rows in result['metrics'].items()} matches={name:rows for name,rows in matches.items() if rows} if url and matches: result.update(scope='article',metrics=matches) else: result.update(status='unavailable',metrics=None,error='exact_page_not_reported') elif result.get('metrics'): # Dimension names are provenance; identities are not report payload. for rows in result['metrics'].values(): for r in rows: r.pop('_dimensionHashes',None) return result