108 lines
7.3 KiB
Python
108 lines
7.3 KiB
Python
"""Clarity rolling-window export. Persist sanitized aggregates only, never raw bodies."""
|
|
import datetime as dt
|
|
import hashlib
|
|
import json
|
|
import math
|
|
import os
|
|
import requests
|
|
|
|
CLARITY_ENDPOINT='https://www.clarity.ms/export-data/api/v1/project-live-insights'
|
|
DIMENSIONS={'Browser','Device','Country/Region','OS','Source','Medium','Campaign','Channel','URL'}
|
|
FIELDS={'distinctUserCount','pagesPerSessionPercentage','pagesViews','totalSessionCount','totalBotSessionCount','distantUserCount','PagesPerSessionPercentage','averageScrollDepth','scrollDepth','engagementTime','activeTime','totalTime','totalCount','sessionsCount','sessionsWithMetricPercentage','sessionsWithoutMetricPercentage','pagesCount','subTotal','total','percentage','average','count'}
|
|
|
|
def stamp(): return dt.datetime.now(dt.timezone.utc).isoformat()
|
|
def hashed(value): return hashlib.sha256(value.encode()).hexdigest()
|
|
def load_token(env_path='/opt/data/.env'):
|
|
# No dependency on a gateway restart or its previously inherited environment.
|
|
try:
|
|
from dotenv import dotenv_values
|
|
return dotenv_values(env_path).get('CLARITY_API_TOKEN') or os.getenv('CLARITY_API_TOKEN')
|
|
except ImportError:
|
|
return os.getenv('CLARITY_API_TOKEN')
|
|
|
|
class ClarityCollector:
|
|
def __init__(self,store,session,token,project,days=3,dimensions=(),day=None,project_suffix=''):
|
|
if days not in (1,2,3) or len(dimensions)>3 or len(set(dimensions))!=len(dimensions) or any(d not in DIMENSIONS for d in dimensions):
|
|
raise ValueError('invalid Clarity window/dimensions')
|
|
self.db,self.session,self.token=store.db,session,token
|
|
self.project=hashed(project+project_suffix); self.days=days; self.dimensions=tuple(dimensions)
|
|
self.day=str(day or dt.datetime.now(dt.timezone.utc).date())
|
|
self.db.execute('CREATE TABLE IF NOT EXISTS clarity_cache (key TEXT PRIMARY KEY, project TEXT, day TEXT, payload TEXT NOT NULL)')
|
|
def base(self):
|
|
return {'source':'clarity','status':'unavailable','metrics':None,'scope':'project','observedAt':stamp(),
|
|
'window':{'kind':'rolling','numOfDays':self.days,'timezone':'UTC','asOf':stamp(),'independentOfGooglePeriod':True},
|
|
'dimensions':list(self.dimensions),'coverage':{'maxRows':1000,'paginated':False,'possiblyTruncated':False,'selection':'allowlisted numeric aggregates only; categorical and URL reports omitted'},
|
|
'provenance':{'endpoint':CLARITY_ENDPOINT,'projectFingerprint':self.project}}
|
|
def sanitize(self,data,result):
|
|
if not isinstance(data,list) or not data: raise ValueError()
|
|
groups={}; row_count=0
|
|
for item in data:
|
|
if not isinstance(item,dict) or not isinstance(item.get('metricName'),str) or not isinstance(item.get('information'),list) or not item['information']: raise ValueError()
|
|
# Only known metrics names; never persist arbitrary provider text.
|
|
name=item['metricName']
|
|
if name not in {'Traffic','ScrollDepth','Scroll Depth','EngagementTime','Engagement Time','DeadClickCount','Dead Click Count','ExcessiveScroll','Excessive Scroll','RageClickCount','Rage Click Count','QuickbackClick','Quickback Click','ScriptErrorCount','Script Error Count','ErrorClickCount','Error Click Count'}: continue
|
|
if name in groups: raise ValueError()
|
|
rows=[]
|
|
for raw in item['information']:
|
|
if not isinstance(raw,dict): raise ValueError()
|
|
clean={}
|
|
for key in FIELDS & raw.keys():
|
|
value=raw[key]
|
|
if isinstance(value,bool): raise ValueError()
|
|
number=float(value)
|
|
if not math.isfinite(number) or number<0: raise ValueError()
|
|
clean[key]=int(number) if number.is_integer() else number
|
|
if not clean: raise ValueError()
|
|
if self.dimensions:
|
|
# URL hash permits exact matching without retaining URL/query PII.
|
|
if any(not isinstance(raw.get(d),str) for d in self.dimensions): raise ValueError()
|
|
clean['_dimensionHashes']={d:hashed(raw[d]) for d in self.dimensions}
|
|
rows.append(clean)
|
|
if len(rows)>1000: raise ValueError()
|
|
row_count+=len(rows); groups[name]=rows
|
|
if len(rows)==1000: result['coverage']['possiblyTruncated']=True
|
|
if not groups: raise ValueError()
|
|
result.update(status='available',metrics=groups,httpStatus=200)
|
|
result['coverage']['sanitizedRowCount']=row_count
|
|
return result
|
|
def collect(self,url=None):
|
|
result=self.base()
|
|
if not self.token:
|
|
result['error']='credentials_not_configured'; return result
|
|
key=hashed(json.dumps([self.project,self.day,self.days,self.dimensions]))
|
|
# Reserve before HTTP, including failures. Concurrent callers reuse pending
|
|
# unavailable result rather than each consuming a project API request.
|
|
with self.db:
|
|
self.db.execute('BEGIN IMMEDIATE')
|
|
row=self.db.execute('SELECT payload FROM clarity_cache WHERE key=?',(key,)).fetchone()
|
|
if row: result=json.loads(row[0])
|
|
else:
|
|
count=self.db.execute('SELECT COUNT(*) FROM clarity_cache WHERE project=? AND day=?',(self.project,self.day)).fetchone()[0]
|
|
if count>=10:
|
|
result['error']='daily_budget_exhausted'; return result
|
|
result['error']='collection_pending'
|
|
self.db.execute('INSERT INTO clarity_cache VALUES (?,?,?,?)',(key,self.project,self.day,json.dumps(result)))
|
|
if not row:
|
|
try:
|
|
params={'numOfDays':self.days,**{f'dimension{i+1}':d for i,d in enumerate(self.dimensions)}}
|
|
response=self.session.get(CLARITY_ENDPOINT,params=params,headers={'Authorization':'Bearer '+self.token},timeout=(5,30),allow_redirects=False)
|
|
result['httpStatus']=response.status_code
|
|
if response.status_code!=200: result['error']=f'http_{response.status_code}'
|
|
else:
|
|
result.pop('error',None)
|
|
result=self.sanitize(response.json(),result)
|
|
except requests.RequestException: result.update(status='unavailable',metrics=None,error='network_error')
|
|
except (ValueError,TypeError,KeyError,OverflowError): result.update(status='unavailable',metrics=None,error='malformed_response')
|
|
with self.db: self.db.execute('UPDATE clarity_cache SET payload=? WHERE key=?',(json.dumps(result),key))
|
|
result=json.loads(json.dumps(result))
|
|
if 'URL' in self.dimensions and result['status']=='available':
|
|
matches={name:[{k:v for k,v in r.items() if k!='_dimensionHashes'} for r in rows if r.get('_dimensionHashes',{}).get('URL')==hashed(url or '')] for name,rows in result['metrics'].items()}
|
|
matches={name:rows for name,rows in matches.items() if rows}
|
|
if url and matches: result.update(scope='article',metrics=matches)
|
|
else: result.update(status='unavailable',metrics=None,error='exact_page_not_reported')
|
|
elif result.get('metrics'):
|
|
# Dimension names are provenance; identities are not report payload.
|
|
for rows in result['metrics'].values():
|
|
for r in rows: r.pop('_dimensionHashes',None)
|
|
return result
|