195 lines
14 KiB
Python
195 lines
14 KiB
Python
#!/usr/bin/env python3
|
|
"""Read-only collection by default. Local durable snapshots; delivery is explicit."""
|
|
import argparse
|
|
import datetime as dt
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import sqlite3
|
|
from urllib.parse import quote, urlsplit
|
|
import requests
|
|
from clarity_collector import ClarityCollector, CLARITY_ENDPOINT, load_token
|
|
|
|
GSC_CREDS_PATH='/opt/data/credentials/gsc-service-account.json'
|
|
GA4_PROPERTY_ID='552453291'
|
|
GSC_SITE='sc-domain:canina.ir'
|
|
|
|
def now(): return dt.datetime.now(dt.timezone.utc).isoformat()
|
|
def period(today=None,days=14):
|
|
if days<1: raise ValueError('days must be positive')
|
|
end=(today or dt.datetime.now(dt.timezone.utc).date())-dt.timedelta(days=3)
|
|
return str(end-dt.timedelta(days=days-1)),str(end)
|
|
def digest(value): return hashlib.sha256(json.dumps(value,sort_keys=True,ensure_ascii=False).encode()).hexdigest()
|
|
def get_google_session():
|
|
from google.oauth2 import service_account
|
|
from google.auth.transport.requests import AuthorizedSession
|
|
return AuthorizedSession(service_account.Credentials.from_service_account_file(GSC_CREDS_PATH,scopes=['https://www.googleapis.com/auth/webmasters.readonly','https://www.googleapis.com/auth/analytics.readonly']))
|
|
def unavailable(reason): return {'status':'unavailable','metrics':None,'error':reason,'observedAt':now()}
|
|
|
|
class GoogleCollector:
|
|
def __init__(self,session,page_size=25000): self.session,self.page_size=session,page_size
|
|
def collect(self,source,url,body,dates):
|
|
rows=[]; offset=0; metadata={}; seen=set()
|
|
try:
|
|
while True:
|
|
payload=dict(body)
|
|
if source=='GSC': payload.update(rowLimit=self.page_size,startRow=offset)
|
|
else: payload.update(limit=str(self.page_size),offset=str(offset))
|
|
response=self.session.post(url,json=payload,timeout=30)
|
|
if response.status_code!=200: return unavailable(f'http_{response.status_code}')
|
|
data=response.json(); batch=data.get('rows',[])
|
|
if not isinstance(batch,list): raise ValueError('invalid rows')
|
|
metadata.update(data.get('metadata',{}))
|
|
if batch:
|
|
signature=digest(batch)
|
|
if signature in seen: raise ValueError('repeated pagination')
|
|
seen.add(signature)
|
|
rows.extend(batch); offset+=len(batch)
|
|
if source=='GA4':
|
|
if offset>=int(data.get('rowCount',offset)): break
|
|
if not batch: raise ValueError('incomplete pagination')
|
|
elif len(batch)<self.page_size: break
|
|
return {'status':'available' if rows else 'no_data','metrics':None,'rows':rows,'periodStart':dates[0],'periodEnd':dates[1],'observedAt':now(),'metadata':metadata,'coverage':'GSC top rows; anonymized queries omitted' if source=='GSC' else 'GA4 reporting rows; inspect metadata for thresholding','source':source,'httpStatus':200}
|
|
except Exception as exc:
|
|
return unavailable(type(exc).__name__)
|
|
def gsc(self,url,dates):
|
|
result = self.collect('GSC',f'https://www.googleapis.com/webmasters/v3/sites/{quote(GSC_SITE,safe="")}/searchAnalytics/query',{'startDate':dates[0],'endDate':dates[1],'dataState':'final','dimensions':['query'],'dimensionFilterGroups':[{'filters':[{'dimension':'page','operator':'equals','expression':url}]}]},dates)
|
|
return result
|
|
def gsc_page(self,url,dates):
|
|
result=self.collect('GSC',f'https://www.googleapis.com/webmasters/v3/sites/{quote(GSC_SITE,safe="")}/searchAnalytics/query',{'startDate':dates[0],'endDate':dates[1],'dataState':'final','dimensions':['page'],'dimensionFilterGroups':[{'filters':[{'dimension':'page','operator':'equals','expression':url}]}]},dates)
|
|
if result['status']=='available':
|
|
try:
|
|
row=result['rows'][0]
|
|
if len(result['rows'])!=1 or row['keys']!=[url]: raise ValueError('exact URL mismatch')
|
|
result['metrics']={k:row[k] for k in ('clicks','impressions','position')}
|
|
except Exception as exc: return unavailable(type(exc).__name__)
|
|
return result
|
|
def ga4(self,url,dates):
|
|
result=self.collect('GA4',f'https://analyticsdata.googleapis.com/v1beta/properties/{GA4_PROPERTY_ID}:runReport',{'dateRanges':[{'startDate':dates[0],'endDate':dates[1]}],'dimensions':[{'name':'pageLocation'}],'metrics':[{'name':'activeUsers'},{'name':'userEngagementDuration'},{'name':'bounceRate'}],'dimensionFilter':{'filter':{'fieldName':'pageLocation','stringFilter':{'matchType':'EXACT','value':url,'caseSensitive':True}}}},dates)
|
|
if result['status']=='available':
|
|
try:
|
|
rows=result['rows']
|
|
if len(rows)!=1 or rows[0]['dimensionValues'][0]['value']!=url: raise ValueError('exact URL mismatch')
|
|
vals=rows[0]['metricValues']; users=int(vals[0]['value']); duration=float(vals[1]['value'])
|
|
result['metrics']={'activeUsers':users,'userEngagementDuration':duration,'avgEngagementSec':duration/users if users else None,'bounceRate':float(vals[2]['value']),'scrollDepthRate':None}
|
|
except Exception as exc: return unavailable(type(exc).__name__)
|
|
return result
|
|
|
|
class Store:
|
|
def __init__(self,path):
|
|
self.db=sqlite3.connect(path)
|
|
self.db.execute('CREATE TABLE IF NOT EXISTS snapshots (key TEXT PRIMARY KEY, payload TEXT NOT NULL, delivered INTEGER DEFAULT 0)')
|
|
self.db.execute('CREATE TABLE IF NOT EXISTS outbox (key TEXT PRIMARY KEY,payload TEXT NOT NULL, delivered INTEGER DEFAULT 0)')
|
|
def snapshot(self,payload):
|
|
key=digest([payload.get(k) for k in ('findingId','periodStart','periodEnd','feedbackRevision')]); payload=dict(payload,idempotencyKey=key)
|
|
with self.db:
|
|
cur=self.db.execute('INSERT OR IGNORE INTO snapshots(key,payload) VALUES (?,?)',(key,json.dumps(payload,ensure_ascii=False)))
|
|
return bool(cur.rowcount)
|
|
def alert(self,source,error,finding):
|
|
payload={'source':source,'error':error,'findingId':finding,'department':'TECHNICAL_SEO' if source=='GSC' else 'ENGINEERING','observedAt':now(),'status':'pending'}
|
|
key=digest([source,error,finding,dt.datetime.now(dt.timezone.utc).date().isoformat()])
|
|
with self.db: self.db.execute('INSERT OR IGNORE INTO outbox(key,payload) VALUES (?,?)',(key,json.dumps(payload)))
|
|
def pending(self): return self.db.execute('SELECT key,payload FROM outbox WHERE delivered=0').fetchall()
|
|
|
|
class Backend:
|
|
def __init__(self,base,session): self.base,self.session=base.rstrip('/')+'/api/admin/seo-tracking',session
|
|
def findings(self):
|
|
page=1; found={}; total=None
|
|
while True:
|
|
r=self.session.get(self.base,params={'page':page,'limit':100},timeout=30)
|
|
if r.status_code!=200: raise RuntimeError(f'backend_http_{r.status_code}')
|
|
data=r.json(); meta=data['meta']; batch=data['data']
|
|
if int(meta['page'])!=page: raise ValueError('pagination mismatch')
|
|
if total is None: total=int(meta['total'])
|
|
if int(meta['total'])!=total: raise ValueError('findings changed during pagination')
|
|
for item in batch:
|
|
if item['id'] in found: raise ValueError('duplicate finding')
|
|
found[item['id']]=item
|
|
if page>=int(meta['totalPages']): break
|
|
if not batch: raise ValueError('incomplete pagination')
|
|
page+=1
|
|
if len(found)!=total: raise ValueError('finding count mismatch')
|
|
return list(found.values())
|
|
|
|
def decision(finding,sources):
|
|
return {'action':'INSUFFICIENT_EVIDENCE','reason':'No approved comparative decision policy; unavailable signals cannot justify scaling or pruning. Correlation is not keyword-caused sales.','mandatoryDirective':finding.get('ceoFeedback'),'feedbackRevision':finding.get('feedbackVersion',finding.get('feedbackRevision',finding.get('feedbackSubmittedAt')))}
|
|
|
|
def collect_finding(finding,collector,dates,store,clarity=None):
|
|
blog=finding.get('blog') or {}; url=finding.get('articleUrl') or blog.get('canonicalUrl')
|
|
if not url and blog.get('slug'): url='https://canina.ir/blog/'+quote(blog['slug'],safe='')
|
|
if not url or urlsplit(url).hostname!='canina.ir' or urlsplit(url).scheme!='https':
|
|
store.alert('BACKEND','missing_or_invalid_article_url',finding['id']); return None
|
|
sources={'gsc':collector.gsc_page(url,dates),'gscQueries':collector.gsc(url,dates),'ga4':collector.ga4(url,dates),'clarity':clarity.collect(url) if clarity else unavailable('collector_not_configured'),'commerce':unavailable('article_to_order_linkage_not_verified')}
|
|
payload={'findingId':finding['id'],'articleUrl':url,'periodStart':dates[0],'periodEnd':dates[1],'feedbackRevision':finding.get('feedbackVersion',finding.get('feedbackRevision',finding.get('feedbackSubmittedAt'))),'sources':sources,'decision':decision(finding,sources),'observedAt':now(),'periodAlignment':'same date labels; GSC Pacific and GA4 property timezone may differ; not session-level alignment; Clarity has its own independent rolling UTC window'}
|
|
for source,data in sources.items():
|
|
if data['status']=='unavailable': store.alert(source.upper(),data['error'],finding['id'])
|
|
store.snapshot(payload); return payload
|
|
|
|
def snapshot_body(payload,key):
|
|
sources=payload['sources']; metadata={}; body={'ingestionKey':key,'periodStart':payload['periodStart']+'T00:00:00.000Z','periodEnd':str(dt.date.fromisoformat(payload['periodEnd'])+dt.timedelta(days=1))+'T00:00:00.000Z'}
|
|
mappings={'gsc':{'clicks':'gscClicks','impressions':'gscImpressions','position':'gscPosition'},'ga4':{'activeUsers':'ga4ActiveUsers','avgEngagementSec':'ga4AvgEngagementSec','bounceRate':'ga4BounceRate'},'commerce':{'orders':'attributedOrdersCount','revenue':'attributedSalesValue'}}
|
|
for source,mapping in mappings.items():
|
|
data=sources.get(source) or unavailable('missing_source'); metrics=data.get('metrics')
|
|
available=data['status']=='available' and metrics is not None and all(metrics.get(k) is not None for k in mapping)
|
|
metadata[source]={'status':'available' if available else 'unavailable','source':source,'fetchedAt':data.get('observedAt',now())}
|
|
if available:
|
|
for k,v in mapping.items(): body[v]=metrics[k]
|
|
else: metadata[source]['error']=data.get('error','no_reported_metrics')
|
|
clarity=sources.get('clarity') or unavailable('missing_source')
|
|
metadata['clarity']=dict(clarity,source='clarity',fetchedAt=clarity.get('observedAt',now()))
|
|
metadata['worker']={'feedbackVersion':payload.get('feedbackRevision'),'mandatoryDirective':payload.get('decision',{}).get('mandatoryDirective'),'periodAlignment':payload.get('periodAlignment'),'sources':sources}
|
|
body['sourceMetadata']=metadata
|
|
return body
|
|
|
|
def deliver_snapshots(store,backend):
|
|
"""Explicit mutation function, never invoked in read-only CLI. Backend dedupes ingestionKey."""
|
|
count=0
|
|
for key,raw in store.db.execute('SELECT key,payload FROM snapshots WHERE delivered=0').fetchall():
|
|
payload=json.loads(raw)
|
|
try:
|
|
response=backend.session.get(backend.base+'/'+payload['findingId'],timeout=30)
|
|
if response.status_code!=200: raise RuntimeError('detail_unavailable')
|
|
if response.json().get('feedbackVersion')!=payload['feedbackRevision']:
|
|
store.alert('BACKEND','stale_feedback',payload['findingId']); continue
|
|
# Measurement snapshots do not apply directives or advance article status.
|
|
response=backend.session.post(backend.base+'/'+payload['findingId']+'/snapshots',json=snapshot_body(payload,key),timeout=30)
|
|
if response.status_code not in (200,201): raise RuntimeError(f'http_{response.status_code}')
|
|
with store.db: store.db.execute('UPDATE snapshots SET delivered=1 WHERE key=?',(key,))
|
|
count+=1
|
|
except Exception as exc: store.alert('BACKEND',type(exc).__name__,payload['findingId'])
|
|
return count
|
|
|
|
def deliver_alerts(store,session,endpoint):
|
|
"""Send structured department events to configured receiver; require durable receiver ack."""
|
|
count=0
|
|
for key,raw in store.pending():
|
|
try:
|
|
response=session.post(endpoint,json=dict(json.loads(raw),idempotencyKey=key),timeout=30)
|
|
if response.status_code not in (200,201,202): continue
|
|
with store.db: store.db.execute('UPDATE outbox SET delivered=1 WHERE key=?',(key,))
|
|
count+=1
|
|
except Exception: pass # event remains pending; no secret-bearing exception logging
|
|
return count
|
|
|
|
def main():
|
|
p=argparse.ArgumentParser(); p.add_argument('--backend'); p.add_argument('--probe-url'); p.add_argument('--days',type=int,default=14); p.add_argument('--state',default='tracker.sqlite'); p.add_argument('--receipt',default='receipt.json'); args=p.parse_args()
|
|
store=Store(args.state); dates=period(days=args.days); results=[]
|
|
try: collector=GoogleCollector(get_google_session())
|
|
except Exception as exc:
|
|
store.alert('GOOGLE',type(exc).__name__,'global'); Path(args.receipt).write_text(json.dumps({'status':'blocked','error':type(exc).__name__,'observedAt':now()})); return 1
|
|
if args.probe_url: findings=[{'id':'read-only-probe','articleUrl':args.probe_url}]
|
|
elif args.backend:
|
|
session=requests.Session(); token=os.getenv('CANINA_ADMIN_TOKEN')
|
|
if token: session.headers['Authorization']='Bearer '+token
|
|
try: findings=Backend(args.backend,session).findings()
|
|
except Exception as exc:
|
|
store.alert('BACKEND',type(exc).__name__,'global'); Path(args.receipt).write_text(json.dumps({'status':'blocked','error':type(exc).__name__,'observedAt':now()})); return 1
|
|
else: p.error('--backend or --probe-url required')
|
|
clarity=ClarityCollector(store,requests.Session(),load_token(),'canina.ir')
|
|
for finding in findings:
|
|
if finding.get('blog') or finding.get('articleUrl'): results.append(collect_finding(finding,collector,dates,store,clarity))
|
|
Path(args.receipt).write_text(json.dumps({'observedAt':now(),'period':dates,'findingCount':len(findings),'collectedCount':sum(x is not None for x in results),'results':results,'remoteWrites':False},ensure_ascii=False,indent=2))
|
|
return 0
|
|
if __name__=='__main__': raise SystemExit(main())
|