Files
canina/scripts/seo-performance/seo_performance_tracker.py

195 lines
14 KiB
Python

#!/usr/bin/env python3
"""Read-only collection by default. Local durable snapshots; delivery is explicit."""
import argparse
import datetime as dt
import hashlib
import json
import os
from pathlib import Path
import sqlite3
from urllib.parse import quote, urlsplit
import requests
from clarity_collector import ClarityCollector, CLARITY_ENDPOINT, load_token
GSC_CREDS_PATH='/opt/data/credentials/gsc-service-account.json'
GA4_PROPERTY_ID='552453291'
GSC_SITE='sc-domain:canina.ir'
def now(): return dt.datetime.now(dt.timezone.utc).isoformat()
def period(today=None,days=14):
if days<1: raise ValueError('days must be positive')
end=(today or dt.datetime.now(dt.timezone.utc).date())-dt.timedelta(days=3)
return str(end-dt.timedelta(days=days-1)),str(end)
def digest(value): return hashlib.sha256(json.dumps(value,sort_keys=True,ensure_ascii=False).encode()).hexdigest()
def get_google_session():
from google.oauth2 import service_account
from google.auth.transport.requests import AuthorizedSession
return AuthorizedSession(service_account.Credentials.from_service_account_file(GSC_CREDS_PATH,scopes=['https://www.googleapis.com/auth/webmasters.readonly','https://www.googleapis.com/auth/analytics.readonly']))
def unavailable(reason): return {'status':'unavailable','metrics':None,'error':reason,'observedAt':now()}
class GoogleCollector:
def __init__(self,session,page_size=25000): self.session,self.page_size=session,page_size
def collect(self,source,url,body,dates):
rows=[]; offset=0; metadata={}; seen=set()
try:
while True:
payload=dict(body)
if source=='GSC': payload.update(rowLimit=self.page_size,startRow=offset)
else: payload.update(limit=str(self.page_size),offset=str(offset))
response=self.session.post(url,json=payload,timeout=30)
if response.status_code!=200: return unavailable(f'http_{response.status_code}')
data=response.json(); batch=data.get('rows',[])
if not isinstance(batch,list): raise ValueError('invalid rows')
metadata.update(data.get('metadata',{}))
if batch:
signature=digest(batch)
if signature in seen: raise ValueError('repeated pagination')
seen.add(signature)
rows.extend(batch); offset+=len(batch)
if source=='GA4':
if offset>=int(data.get('rowCount',offset)): break
if not batch: raise ValueError('incomplete pagination')
elif len(batch)<self.page_size: break
return {'status':'available' if rows else 'no_data','metrics':None,'rows':rows,'periodStart':dates[0],'periodEnd':dates[1],'observedAt':now(),'metadata':metadata,'coverage':'GSC top rows; anonymized queries omitted' if source=='GSC' else 'GA4 reporting rows; inspect metadata for thresholding','source':source,'httpStatus':200}
except Exception as exc:
return unavailable(type(exc).__name__)
def gsc(self,url,dates):
result = self.collect('GSC',f'https://www.googleapis.com/webmasters/v3/sites/{quote(GSC_SITE,safe="")}/searchAnalytics/query',{'startDate':dates[0],'endDate':dates[1],'dataState':'final','dimensions':['query'],'dimensionFilterGroups':[{'filters':[{'dimension':'page','operator':'equals','expression':url}]}]},dates)
return result
def gsc_page(self,url,dates):
result=self.collect('GSC',f'https://www.googleapis.com/webmasters/v3/sites/{quote(GSC_SITE,safe="")}/searchAnalytics/query',{'startDate':dates[0],'endDate':dates[1],'dataState':'final','dimensions':['page'],'dimensionFilterGroups':[{'filters':[{'dimension':'page','operator':'equals','expression':url}]}]},dates)
if result['status']=='available':
try:
row=result['rows'][0]
if len(result['rows'])!=1 or row['keys']!=[url]: raise ValueError('exact URL mismatch')
result['metrics']={k:row[k] for k in ('clicks','impressions','position')}
except Exception as exc: return unavailable(type(exc).__name__)
return result
def ga4(self,url,dates):
result=self.collect('GA4',f'https://analyticsdata.googleapis.com/v1beta/properties/{GA4_PROPERTY_ID}:runReport',{'dateRanges':[{'startDate':dates[0],'endDate':dates[1]}],'dimensions':[{'name':'pageLocation'}],'metrics':[{'name':'activeUsers'},{'name':'userEngagementDuration'},{'name':'bounceRate'}],'dimensionFilter':{'filter':{'fieldName':'pageLocation','stringFilter':{'matchType':'EXACT','value':url,'caseSensitive':True}}}},dates)
if result['status']=='available':
try:
rows=result['rows']
if len(rows)!=1 or rows[0]['dimensionValues'][0]['value']!=url: raise ValueError('exact URL mismatch')
vals=rows[0]['metricValues']; users=int(vals[0]['value']); duration=float(vals[1]['value'])
result['metrics']={'activeUsers':users,'userEngagementDuration':duration,'avgEngagementSec':duration/users if users else None,'bounceRate':float(vals[2]['value']),'scrollDepthRate':None}
except Exception as exc: return unavailable(type(exc).__name__)
return result
class Store:
def __init__(self,path):
self.db=sqlite3.connect(path)
self.db.execute('CREATE TABLE IF NOT EXISTS snapshots (key TEXT PRIMARY KEY, payload TEXT NOT NULL, delivered INTEGER DEFAULT 0)')
self.db.execute('CREATE TABLE IF NOT EXISTS outbox (key TEXT PRIMARY KEY,payload TEXT NOT NULL, delivered INTEGER DEFAULT 0)')
def snapshot(self,payload):
key=digest([payload.get(k) for k in ('findingId','periodStart','periodEnd','feedbackRevision')]); payload=dict(payload,idempotencyKey=key)
with self.db:
cur=self.db.execute('INSERT OR IGNORE INTO snapshots(key,payload) VALUES (?,?)',(key,json.dumps(payload,ensure_ascii=False)))
return bool(cur.rowcount)
def alert(self,source,error,finding):
payload={'source':source,'error':error,'findingId':finding,'department':'TECHNICAL_SEO' if source=='GSC' else 'ENGINEERING','observedAt':now(),'status':'pending'}
key=digest([source,error,finding,dt.datetime.now(dt.timezone.utc).date().isoformat()])
with self.db: self.db.execute('INSERT OR IGNORE INTO outbox(key,payload) VALUES (?,?)',(key,json.dumps(payload)))
def pending(self): return self.db.execute('SELECT key,payload FROM outbox WHERE delivered=0').fetchall()
class Backend:
def __init__(self,base,session): self.base,self.session=base.rstrip('/')+'/api/admin/seo-tracking',session
def findings(self):
page=1; found={}; total=None
while True:
r=self.session.get(self.base,params={'page':page,'limit':100},timeout=30)
if r.status_code!=200: raise RuntimeError(f'backend_http_{r.status_code}')
data=r.json(); meta=data['meta']; batch=data['data']
if int(meta['page'])!=page: raise ValueError('pagination mismatch')
if total is None: total=int(meta['total'])
if int(meta['total'])!=total: raise ValueError('findings changed during pagination')
for item in batch:
if item['id'] in found: raise ValueError('duplicate finding')
found[item['id']]=item
if page>=int(meta['totalPages']): break
if not batch: raise ValueError('incomplete pagination')
page+=1
if len(found)!=total: raise ValueError('finding count mismatch')
return list(found.values())
def decision(finding,sources):
return {'action':'INSUFFICIENT_EVIDENCE','reason':'No approved comparative decision policy; unavailable signals cannot justify scaling or pruning. Correlation is not keyword-caused sales.','mandatoryDirective':finding.get('ceoFeedback'),'feedbackRevision':finding.get('feedbackVersion',finding.get('feedbackRevision',finding.get('feedbackSubmittedAt')))}
def collect_finding(finding,collector,dates,store,clarity=None):
blog=finding.get('blog') or {}; url=finding.get('articleUrl') or blog.get('canonicalUrl')
if not url and blog.get('slug'): url='https://canina.ir/blog/'+quote(blog['slug'],safe='')
if not url or urlsplit(url).hostname!='canina.ir' or urlsplit(url).scheme!='https':
store.alert('BACKEND','missing_or_invalid_article_url',finding['id']); return None
sources={'gsc':collector.gsc_page(url,dates),'gscQueries':collector.gsc(url,dates),'ga4':collector.ga4(url,dates),'clarity':clarity.collect(url) if clarity else unavailable('collector_not_configured'),'commerce':unavailable('article_to_order_linkage_not_verified')}
payload={'findingId':finding['id'],'articleUrl':url,'periodStart':dates[0],'periodEnd':dates[1],'feedbackRevision':finding.get('feedbackVersion',finding.get('feedbackRevision',finding.get('feedbackSubmittedAt'))),'sources':sources,'decision':decision(finding,sources),'observedAt':now(),'periodAlignment':'same date labels; GSC Pacific and GA4 property timezone may differ; not session-level alignment; Clarity has its own independent rolling UTC window'}
for source,data in sources.items():
if data['status']=='unavailable': store.alert(source.upper(),data['error'],finding['id'])
store.snapshot(payload); return payload
def snapshot_body(payload,key):
sources=payload['sources']; metadata={}; body={'ingestionKey':key,'periodStart':payload['periodStart']+'T00:00:00.000Z','periodEnd':str(dt.date.fromisoformat(payload['periodEnd'])+dt.timedelta(days=1))+'T00:00:00.000Z'}
mappings={'gsc':{'clicks':'gscClicks','impressions':'gscImpressions','position':'gscPosition'},'ga4':{'activeUsers':'ga4ActiveUsers','avgEngagementSec':'ga4AvgEngagementSec','bounceRate':'ga4BounceRate'},'commerce':{'orders':'attributedOrdersCount','revenue':'attributedSalesValue'}}
for source,mapping in mappings.items():
data=sources.get(source) or unavailable('missing_source'); metrics=data.get('metrics')
available=data['status']=='available' and metrics is not None and all(metrics.get(k) is not None for k in mapping)
metadata[source]={'status':'available' if available else 'unavailable','source':source,'fetchedAt':data.get('observedAt',now())}
if available:
for k,v in mapping.items(): body[v]=metrics[k]
else: metadata[source]['error']=data.get('error','no_reported_metrics')
clarity=sources.get('clarity') or unavailable('missing_source')
metadata['clarity']=dict(clarity,source='clarity',fetchedAt=clarity.get('observedAt',now()))
metadata['worker']={'feedbackVersion':payload.get('feedbackRevision'),'mandatoryDirective':payload.get('decision',{}).get('mandatoryDirective'),'periodAlignment':payload.get('periodAlignment'),'sources':sources}
body['sourceMetadata']=metadata
return body
def deliver_snapshots(store,backend):
"""Explicit mutation function, never invoked in read-only CLI. Backend dedupes ingestionKey."""
count=0
for key,raw in store.db.execute('SELECT key,payload FROM snapshots WHERE delivered=0').fetchall():
payload=json.loads(raw)
try:
response=backend.session.get(backend.base+'/'+payload['findingId'],timeout=30)
if response.status_code!=200: raise RuntimeError('detail_unavailable')
if response.json().get('feedbackVersion')!=payload['feedbackRevision']:
store.alert('BACKEND','stale_feedback',payload['findingId']); continue
# Measurement snapshots do not apply directives or advance article status.
response=backend.session.post(backend.base+'/'+payload['findingId']+'/snapshots',json=snapshot_body(payload,key),timeout=30)
if response.status_code not in (200,201): raise RuntimeError(f'http_{response.status_code}')
with store.db: store.db.execute('UPDATE snapshots SET delivered=1 WHERE key=?',(key,))
count+=1
except Exception as exc: store.alert('BACKEND',type(exc).__name__,payload['findingId'])
return count
def deliver_alerts(store,session,endpoint):
"""Send structured department events to configured receiver; require durable receiver ack."""
count=0
for key,raw in store.pending():
try:
response=session.post(endpoint,json=dict(json.loads(raw),idempotencyKey=key),timeout=30)
if response.status_code not in (200,201,202): continue
with store.db: store.db.execute('UPDATE outbox SET delivered=1 WHERE key=?',(key,))
count+=1
except Exception: pass # event remains pending; no secret-bearing exception logging
return count
def main():
p=argparse.ArgumentParser(); p.add_argument('--backend'); p.add_argument('--probe-url'); p.add_argument('--days',type=int,default=14); p.add_argument('--state',default='tracker.sqlite'); p.add_argument('--receipt',default='receipt.json'); args=p.parse_args()
store=Store(args.state); dates=period(days=args.days); results=[]
try: collector=GoogleCollector(get_google_session())
except Exception as exc:
store.alert('GOOGLE',type(exc).__name__,'global'); Path(args.receipt).write_text(json.dumps({'status':'blocked','error':type(exc).__name__,'observedAt':now()})); return 1
if args.probe_url: findings=[{'id':'read-only-probe','articleUrl':args.probe_url}]
elif args.backend:
session=requests.Session(); token=os.getenv('CANINA_ADMIN_TOKEN')
if token: session.headers['Authorization']='Bearer '+token
try: findings=Backend(args.backend,session).findings()
except Exception as exc:
store.alert('BACKEND',type(exc).__name__,'global'); Path(args.receipt).write_text(json.dumps({'status':'blocked','error':type(exc).__name__,'observedAt':now()})); return 1
else: p.error('--backend or --probe-url required')
clarity=ClarityCollector(store,requests.Session(),load_token(),'canina.ir')
for finding in findings:
if finding.get('blog') or finding.get('articleUrl'): results.append(collect_finding(finding,collector,dates,store,clarity))
Path(args.receipt).write_text(json.dumps({'observedAt':now(),'period':dates,'findingCount':len(findings),'collectedCount':sum(x is not None for x in results),'results':results,'remoteWrites':False},ensure_ascii=False,indent=2))
return 0
if __name__=='__main__': raise SystemExit(main())