252 lines
13 KiB
Python
252 lines
13 KiB
Python
#!/usr/bin/env python3
|
|
"""Manual deterministic department audits. Persisted bytes are data, never instructions."""
|
|
from html.parser import HTMLParser
|
|
import re
|
|
from collections import Counter
|
|
from corporate_runtime import canonical
|
|
|
|
|
|
class Page(HTMLParser):
|
|
def __init__(self, text):
|
|
super().__init__(convert_charrefs=True)
|
|
self.title, self.h1, self.descriptions = [], [], []
|
|
self.active = None
|
|
self.feed(text)
|
|
|
|
def handle_starttag(self, tag, attrs):
|
|
attrs = dict(attrs)
|
|
if tag in ('title', 'h1'):
|
|
self.active = tag
|
|
if tag == 'meta' and (attrs.get('name') or '').lower() == 'description':
|
|
self.descriptions.append((attrs.get('content') or '').strip())
|
|
|
|
def handle_endtag(self, tag):
|
|
if self.active == tag:
|
|
self.active = None
|
|
|
|
def handle_data(self, data):
|
|
if self.active and data.strip():
|
|
getattr(self, self.active).append(data.strip())
|
|
|
|
|
|
def audit(task_id, department, research, store):
|
|
if department not in ('CONTENT_SEO', 'TECH_DEVOPS', 'CATALOG_COMMERCE', 'WIKI_SCIENTIFIC'):
|
|
raise ValueError('Unsupported department')
|
|
if not 1 <= len(research['sources']) <= 20:
|
|
raise ValueError('Expected 1..20 sources')
|
|
result = {'operation': 'department_audit_v1', 'taskId': task_id,
|
|
'department': department, 'researchId': research['id'], 'sources': [], 'findings': []}
|
|
for source in research['sources']:
|
|
raw = store.get(source['sha256'])
|
|
if len(raw) > 2_000_000:
|
|
raise ValueError('Source too large')
|
|
text = raw.decode('utf-8')
|
|
if department == 'TECH_DEVOPS':
|
|
headings = re.findall(r'^#{1,6}\s+(.+)$', text, re.M)
|
|
anchors = [re.sub(r'[^\w -]', '', x.lower()).replace(' ', '-') for x in headings]
|
|
extracted = {'headings': headings, 'links': re.findall(r'\[[^\]\n]+\]\((https?://[^\s)]+)\)', text)}
|
|
result['sources'].append({'sha256': source['sha256'], 'url': source['url'], 'availability': 'AVAILABLE' if headings else 'UNSUPPORTED_FORMAT', 'extracted': extracted})
|
|
for anchor, count in sorted(Counter(anchors).items()):
|
|
if count > 1:
|
|
result['findings'].append({'code': 'duplicate_heading_anchor', 'sourceSha256': source['sha256'], 'url': source['url'], 'observation': {'anchor': anchor, 'count': count}})
|
|
continue
|
|
if not re.search(r'<(?:html|head|title|body|h1)\b', text, re.I):
|
|
result['sources'].append({'sha256': source['sha256'], 'url': source['url'], 'availability': 'UNSUPPORTED_FORMAT', 'extracted': {}})
|
|
continue
|
|
if department in ('WIKI_SCIENTIFIC', 'CATALOG_COMMERCE'):
|
|
import json
|
|
findings = []
|
|
if department == 'WIKI_SCIENTIFIC':
|
|
extracted = {'headings': [{'level': int(level), 'text': re.sub('<[^>]+>', '', body).strip()}
|
|
for level, body in re.findall(r'<h([1-6])\b[^>]*>(.*?)</h\1\s*>', text, re.I | re.S)]}
|
|
previous = 0
|
|
for heading in extracted['headings']:
|
|
if heading['level'] > previous + 1:
|
|
findings.append(('heading_level_skip', {'previousLevel': previous, **heading}))
|
|
previous = heading['level']
|
|
else:
|
|
products = []
|
|
for attrs, body in re.findall(r'<script\b([^>]*)>(.*?)</script\s*>', text, re.I | re.S):
|
|
if not re.search(r'type\s*=\s*[\"\']application/ld\+json[\"\']', attrs, re.I):
|
|
continue
|
|
try:
|
|
value = json.loads(body)
|
|
except ValueError:
|
|
findings.append(('invalid_json_ld', {'parseable': False}))
|
|
continue
|
|
queue = value if isinstance(value, list) else [value]
|
|
while queue:
|
|
node = queue.pop()
|
|
if not isinstance(node, dict):
|
|
continue
|
|
queue.extend(node.get('@graph', []) if isinstance(node.get('@graph', []), list) else [])
|
|
if node.get('@type') == 'Product' or (isinstance(node.get('@type'), list) and 'Product' in node['@type']):
|
|
products.append({'name': node.get('name'), 'description': node.get('description')})
|
|
extracted = {'products': products}
|
|
for index, product in enumerate(products):
|
|
if not isinstance(product['name'], str) or not product['name'].strip():
|
|
findings.append(('product_schema_missing_name', {'productIndex': index, 'name': product['name']}))
|
|
result['sources'].append({'sha256': source['sha256'], 'url': source['url'], 'availability': 'AVAILABLE', 'extracted': extracted})
|
|
for code, observation in findings:
|
|
result['findings'].append({'code': code, 'sourceSha256': source['sha256'], 'url': source['url'], 'observation': observation})
|
|
continue
|
|
page = Page(text)
|
|
extracted = {'title': page.title, 'h1': page.h1, 'metaDescription': page.descriptions}
|
|
result['sources'].append({'sha256': source['sha256'], 'url': source['url'], 'extracted': extracted})
|
|
for code, values in [('missing_title', page.title), ('missing_meta_description', page.descriptions), ('missing_h1', page.h1)]:
|
|
if not any(values):
|
|
result['findings'].append({'code': code, 'sourceSha256': source['sha256'], 'url': source['url'], 'observation': values})
|
|
return result
|
|
|
|
|
|
def research_lookup(client, research_id, max_pages=3):
|
|
if not 1 <= max_pages <= 5:
|
|
raise ValueError('Expected 1..5 research pages')
|
|
for page in range(1, max_pages + 1):
|
|
response = client.call('market-insights?limit=100&page=' + str(page))
|
|
if len(response['data']) > 100:
|
|
raise ValueError('Oversized research page')
|
|
for research in response['data']:
|
|
if research['id'] == research_id:
|
|
return research
|
|
if not response['meta']['hasNextPage']:
|
|
break
|
|
raise ValueError('Research absent within bounded history')
|
|
|
|
|
|
def execute(client, store, department, key):
|
|
import json
|
|
from corporate_runtime import now
|
|
task = client.call('claims', {'department': department, 'leaseSeconds': 300}, key + ':claim')['task']
|
|
if task is None:
|
|
return {'task': None}
|
|
op = json.loads(task['description'])
|
|
if (task['actionType'] != 'RESEARCH' or task['department'] != department
|
|
or set(op) != {'operation', 'researchId'} or op['operation'] != 'department_audit_v1'):
|
|
raise ValueError('Unsupported task; left leased without evidence')
|
|
started = now()
|
|
artifact = audit(task['id'], department, research_lookup(client, op['researchId']), store)
|
|
digest = store.put(canonical(artifact))
|
|
receipt = {'operation': 'department_audit_v1', 'startedAt': started, 'finishedAt': now(),
|
|
'sha256': digest, 'findingCount': len(artifact['findings']), 'sourceCount': len(artifact['sources'])}
|
|
body = {'claimToken': task['claimToken'], 'artifactUri': 'urn:sha256:' + digest,
|
|
'sha256': digest, 'summary': 'Deterministic persisted-source audit; no publishing or external effects',
|
|
'tool': 'python:department_audit_v1', 'result': json.dumps(receipt)}
|
|
checkpoint = client.checkpoint(store, key + ':evidence', 'tasks/' + task['id'] + '/evidence', body)
|
|
return {'taskId': task['id'], 'artifactSha256': digest, 'checkpoint': checkpoint,
|
|
'evidence': client.call('tasks/' + task['id'] + '/evidence', body, key + ':evidence')}
|
|
|
|
|
|
def plan_day(tasks, day, complete=True, store=None):
|
|
from datetime import date
|
|
date.fromisoformat(day)
|
|
output = {'day': day, 'operation': 'pmo_daily_outcomes_v1', 'historyComplete': complete,
|
|
'outcomes': [], 'existingFollowup': [], 'dailyNew': [], 'limitations': []}
|
|
for task in tasks:
|
|
lineage = {'taskId': task['id'], 'department': task['department'],
|
|
'evidence': [{'id': e['id'], 'sha256': e['sha256']} for e in task['evidence']]}
|
|
truncated = any(x.get('truncated') for x in task.get('relationsMeta', {}).values())
|
|
if truncated:
|
|
output['historyComplete'] = False
|
|
output['outcomes'].append(lineage | {'status': task['status'], 'effects': task['effects'], 'reviews': task['reviews'], 'historyTruncated': truncated})
|
|
reason = None
|
|
if task['status'] == 'QA_REVIEW' and task['evidence'] and not task['reviews']:
|
|
reason = 'evidence_awaits_review'
|
|
elif task['status'] == 'BLOCKED':
|
|
reason = 'task_blocked'
|
|
elif task['status'] in ('TODO', 'IN_PROGRESS'):
|
|
reason = 'existing_work_not_complete'
|
|
if reason:
|
|
output['existingFollowup'].append(lineage | {'reason': reason})
|
|
if store and task['status'] == 'DONE' and not truncated:
|
|
import json
|
|
for evidence in task['evidence']:
|
|
try:
|
|
artifact = json.loads(store.get(evidence['sha256']))
|
|
except (ValueError, OSError):
|
|
output['limitations'].append({'taskId': task['id'], 'reason': 'audit_artifact_unavailable'})
|
|
continue
|
|
if artifact.get('operation') != 'department_audit_v1' or artifact.get('taskId') != task['id']:
|
|
continue
|
|
for finding in artifact['findings']:
|
|
output['dailyNew'].append(lineage | {'reason': 'completed_audit_has_unresolved_finding', 'finding': finding,
|
|
'action': 'Verify current source and resolve observed structural gap; do not publish automatically'})
|
|
active_departments = {t['department'] for t in tasks if t['status'] not in ('DONE', 'CANCELLED')}
|
|
unique = {}
|
|
for action in output['dailyNew']:
|
|
if action['department'] not in active_departments:
|
|
unique[canonical({'department': action['department'], 'finding': action['finding']})] = action
|
|
output['dailyNew'] = list(unique.values())
|
|
if not output['historyComplete']:
|
|
output['dailyNew'] = []
|
|
output['limitations'].append({'reason': 'incomplete_history_no_new_work_inferred'})
|
|
return output
|
|
|
|
|
|
def history(client, max_pages=3):
|
|
if not 1 <= max_pages <= 5:
|
|
raise ValueError('Expected 1..5 pages')
|
|
tasks, seen = [], set()
|
|
for page in range(1, max_pages + 1):
|
|
response = client.call('tasks?limit=100&page=' + str(page))
|
|
if len(response['data']) > 100:
|
|
raise ValueError('Oversized task page')
|
|
for row in response['data']:
|
|
if row['id'] in seen:
|
|
continue
|
|
seen.add(row['id'])
|
|
if not re.fullmatch(r'[a-zA-Z0-9-]{1,36}', row['id']):
|
|
raise ValueError('Invalid task ID')
|
|
tasks.append(client.call('tasks/' + row['id']))
|
|
if not response['meta']['hasNextPage']:
|
|
return tasks, True
|
|
return tasks, False
|
|
|
|
|
|
def main():
|
|
import argparse
|
|
import json
|
|
from pathlib import Path
|
|
from corporate_runtime import Client, Store, sha
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('command', choices=['audit', 'execute', 'plan'])
|
|
parser.add_argument('--store', required=True)
|
|
parser.add_argument('--department', choices=['CONTENT_SEO', 'TECH_DEVOPS', 'CATALOG_COMMERCE', 'WIKI_SCIENTIFIC'])
|
|
parser.add_argument('--research', help='Persisted API research JSON for offline audit')
|
|
parser.add_argument('--task-id', default='offline-audit')
|
|
parser.add_argument('--snapshot', help='Task detail array; always treated as incomplete offline history')
|
|
parser.add_argument('--day')
|
|
parser.add_argument('--api', default='http://127.0.0.1:3001/api/admin/hermes')
|
|
parser.add_argument('--credential-env', default='CORPORATE_API_KEY')
|
|
parser.add_argument('--key')
|
|
args = parser.parse_args()
|
|
store = Store(args.store)
|
|
def read_json(path):
|
|
raw = Path(path).read_bytes()
|
|
if len(raw) > 2_000_000:
|
|
raise ValueError('Input too large')
|
|
return json.loads(raw), sha(raw)
|
|
if args.command == 'audit':
|
|
research, digest = read_json(args.research)
|
|
result = audit(args.task_id, args.department, research, store)
|
|
result.update(mode='OFFLINE_PERSISTED_SOURCE', inputSha256=digest)
|
|
elif args.command == 'execute':
|
|
if not args.key or not args.department:
|
|
parser.error('--key and --department required')
|
|
result = execute(Client(args.api, args.credential_env), store, args.department, args.key)
|
|
else:
|
|
if args.snapshot:
|
|
tasks, digest = read_json(args.snapshot)
|
|
complete, mode = False, 'OFFLINE_SNAPSHOT'
|
|
else:
|
|
tasks, complete = history(Client(args.api, args.credential_env))
|
|
digest, mode = store.put(canonical(tasks)), 'API_GET_ONLY'
|
|
result = plan_day(tasks, args.day, complete=complete, store=store)
|
|
result.update(mode=mode, inputSha256=digest)
|
|
artifact = store.put(canonical(result))
|
|
print(json.dumps(result | {'artifactSha256': artifact}, ensure_ascii=False, sort_keys=True))
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|