#!/usr/bin/env python3 """Read-only collection by default. Local durable snapshots; delivery is explicit.""" import argparse import datetime as dt import hashlib import json import os from pathlib import Path import sqlite3 from urllib.parse import quote, urlsplit import requests GSC_CREDS_PATH='/opt/data/credentials/gsc-service-account.json' GA4_PROPERTY_ID='552453291' GSC_SITE='sc-domain:canina.ir' def now(): return dt.datetime.now(dt.timezone.utc).isoformat() def period(today=None,days=14): if days<1: raise ValueError('days must be positive') end=(today or dt.datetime.now(dt.timezone.utc).date())-dt.timedelta(days=3) return str(end-dt.timedelta(days=days-1)),str(end) def digest(value): return hashlib.sha256(json.dumps(value,sort_keys=True,ensure_ascii=False).encode()).hexdigest() def get_google_session(): from google.oauth2 import service_account from google.auth.transport.requests import AuthorizedSession return AuthorizedSession(service_account.Credentials.from_service_account_file(GSC_CREDS_PATH,scopes=['https://www.googleapis.com/auth/webmasters.readonly','https://www.googleapis.com/auth/analytics.readonly'])) def unavailable(reason): return {'status':'unavailable','metrics':None,'error':reason,'observedAt':now()} class GoogleCollector: def __init__(self,session,page_size=25000): self.session,self.page_size=session,page_size def collect(self,source,url,body,dates): rows=[]; offset=0; metadata={}; seen=set() try: while True: payload=dict(body) if source=='GSC': payload.update(rowLimit=self.page_size,startRow=offset) else: payload.update(limit=str(self.page_size),offset=str(offset)) response=self.session.post(url,json=payload,timeout=30) if response.status_code!=200: return unavailable(f'http_{response.status_code}') data=response.json(); batch=data.get('rows',[]) if not isinstance(batch,list): raise ValueError('invalid rows') metadata.update(data.get('metadata',{})) if batch: signature=digest(batch) if signature in seen: raise ValueError('repeated pagination') seen.add(signature) rows.extend(batch); offset+=len(batch) if source=='GA4': if offset>=int(data.get('rowCount',offset)): break if not batch: raise ValueError('incomplete pagination') elif len(batch)=int(meta['totalPages']): break if not batch: raise ValueError('incomplete pagination') page+=1 if len(found)!=total: raise ValueError('finding count mismatch') return list(found.values()) def decision(finding,sources): return {'action':'INSUFFICIENT_EVIDENCE','reason':'No approved comparative decision policy; unavailable signals cannot justify scaling or pruning. Correlation is not keyword-caused sales.','mandatoryDirective':finding.get('ceoFeedback'),'feedbackRevision':finding.get('feedbackVersion',finding.get('feedbackRevision',finding.get('feedbackSubmittedAt')))} def collect_finding(finding,collector,dates,store): blog=finding.get('blog') or {}; url=finding.get('articleUrl') or blog.get('canonicalUrl') if not url and blog.get('slug'): url='https://canina.ir/blog/'+quote(blog['slug'],safe='') if not url or urlsplit(url).hostname!='canina.ir' or urlsplit(url).scheme!='https': store.alert('BACKEND','missing_or_invalid_article_url',finding['id']); return None sources={'gsc':collector.gsc_page(url,dates),'gscQueries':collector.gsc(url,dates),'ga4':collector.ga4(url,dates),'clarity':unavailable('reporting_credentials_not_configured'),'commerce':unavailable('article_to_order_linkage_not_verified')} payload={'findingId':finding['id'],'articleUrl':url,'periodStart':dates[0],'periodEnd':dates[1],'feedbackRevision':finding.get('feedbackVersion',finding.get('feedbackRevision',finding.get('feedbackSubmittedAt'))),'sources':sources,'decision':decision(finding,sources),'observedAt':now(),'periodAlignment':'same date labels; GSC Pacific and GA4 property timezone may differ; not session-level alignment'} for source,data in sources.items(): if data['status']=='unavailable': store.alert(source.upper(),data['error'],finding['id']) store.snapshot(payload); return payload def snapshot_body(payload,key): sources=payload['sources']; metadata={}; body={'ingestionKey':key,'periodStart':payload['periodStart']+'T00:00:00.000Z','periodEnd':str(dt.date.fromisoformat(payload['periodEnd'])+dt.timedelta(days=1))+'T00:00:00.000Z'} mappings={'gsc':{'clicks':'gscClicks','impressions':'gscImpressions','position':'gscPosition'},'ga4':{'activeUsers':'ga4ActiveUsers','avgEngagementSec':'ga4AvgEngagementSec','bounceRate':'ga4BounceRate'},'commerce':{'orders':'attributedOrdersCount','revenue':'attributedSalesValue'}} for source,mapping in mappings.items(): data=sources.get(source) or unavailable('missing_source'); metrics=data.get('metrics') available=data['status']=='available' and metrics is not None and all(metrics.get(k) is not None for k in mapping) metadata[source]={'status':'available' if available else 'unavailable','source':source,'fetchedAt':data.get('observedAt',now())} if available: for k,v in mapping.items(): body[v]=metrics[k] else: metadata[source]['error']=data.get('error','no_reported_metrics') metadata['worker']={'feedbackVersion':payload.get('feedbackRevision'),'mandatoryDirective':payload.get('decision',{}).get('mandatoryDirective'),'periodAlignment':payload.get('periodAlignment'),'sources':sources} body['sourceMetadata']=metadata return body def deliver_snapshots(store,backend): """Explicit mutation function, never invoked in read-only CLI. Backend dedupes ingestionKey.""" count=0 for key,raw in store.db.execute('SELECT key,payload FROM snapshots WHERE delivered=0').fetchall(): payload=json.loads(raw) try: response=backend.session.get(backend.base+'/'+payload['findingId'],timeout=30) if response.status_code!=200: raise RuntimeError('detail_unavailable') if response.json().get('feedbackVersion')!=payload['feedbackRevision']: store.alert('BACKEND','stale_feedback',payload['findingId']); continue # Measurement snapshots do not apply directives or advance article status. response=backend.session.post(backend.base+'/'+payload['findingId']+'/snapshots',json=snapshot_body(payload,key),timeout=30) if response.status_code not in (200,201): raise RuntimeError(f'http_{response.status_code}') with store.db: store.db.execute('UPDATE snapshots SET delivered=1 WHERE key=?',(key,)) count+=1 except Exception as exc: store.alert('BACKEND',type(exc).__name__,payload['findingId']) return count def deliver_alerts(store,session,endpoint): """Send structured department events to configured receiver; require durable receiver ack.""" count=0 for key,raw in store.pending(): try: response=session.post(endpoint,json=dict(json.loads(raw),idempotencyKey=key),timeout=30) if response.status_code not in (200,201,202): continue with store.db: store.db.execute('UPDATE outbox SET delivered=1 WHERE key=?',(key,)) count+=1 except Exception: pass # event remains pending; no secret-bearing exception logging return count def main(): p=argparse.ArgumentParser(); p.add_argument('--backend'); p.add_argument('--probe-url'); p.add_argument('--days',type=int,default=14); p.add_argument('--state',default='tracker.sqlite'); p.add_argument('--receipt',default='receipt.json'); args=p.parse_args() store=Store(args.state); dates=period(days=args.days); results=[] try: collector=GoogleCollector(get_google_session()) except Exception as exc: store.alert('GOOGLE',type(exc).__name__,'global'); Path(args.receipt).write_text(json.dumps({'status':'blocked','error':type(exc).__name__,'observedAt':now()})); return 1 if args.probe_url: findings=[{'id':'read-only-probe','articleUrl':args.probe_url}] elif args.backend: session=requests.Session(); token=os.getenv('CANINA_ADMIN_TOKEN') if token: session.headers['Authorization']='Bearer '+token try: findings=Backend(args.backend,session).findings() except Exception as exc: store.alert('BACKEND',type(exc).__name__,'global'); Path(args.receipt).write_text(json.dumps({'status':'blocked','error':type(exc).__name__,'observedAt':now()})); return 1 else: p.error('--backend or --probe-url required') for finding in findings: if finding.get('blog') or finding.get('articleUrl'): results.append(collect_finding(finding,collector,dates,store)) Path(args.receipt).write_text(json.dumps({'observedAt':now(),'period':dates,'findingCount':len(findings),'collectedCount':sum(x is not None for x in results),'results':results,'remoteWrites':False},ensure_ascii=False,indent=2)) return 0 if __name__=='__main__': raise SystemExit(main())