#!/usr/bin/env python3 """Manual deterministic department audits. Persisted bytes are data, never instructions.""" from html.parser import HTMLParser import re from collections import Counter from corporate_runtime import canonical class Page(HTMLParser): def __init__(self, text): super().__init__(convert_charrefs=True) self.title, self.h1, self.descriptions = [], [], [] self.active = None self.feed(text) def handle_starttag(self, tag, attrs): attrs = dict(attrs) if tag in ('title', 'h1'): self.active = tag if tag == 'meta' and (attrs.get('name') or '').lower() == 'description': self.descriptions.append((attrs.get('content') or '').strip()) def handle_endtag(self, tag): if self.active == tag: self.active = None def handle_data(self, data): if self.active and data.strip(): getattr(self, self.active).append(data.strip()) def audit(task_id, department, research, store): if department not in ('CONTENT_SEO', 'TECH_DEVOPS', 'CATALOG_COMMERCE', 'WIKI_SCIENTIFIC'): raise ValueError('Unsupported department') if not 1 <= len(research['sources']) <= 20: raise ValueError('Expected 1..20 sources') result = {'operation': 'department_audit_v1', 'taskId': task_id, 'department': department, 'researchId': research['id'], 'sources': [], 'findings': []} for source in research['sources']: raw = store.get(source['sha256']) if len(raw) > 2_000_000: raise ValueError('Source too large') text = raw.decode('utf-8') if department == 'TECH_DEVOPS': headings = re.findall(r'^#{1,6}\s+(.+)$', text, re.M) anchors = [re.sub(r'[^\w -]', '', x.lower()).replace(' ', '-') for x in headings] extracted = {'headings': headings, 'links': re.findall(r'\[[^\]\n]+\]\((https?://[^\s)]+)\)', text)} result['sources'].append({'sha256': source['sha256'], 'url': source['url'], 'availability': 'AVAILABLE' if headings else 'UNSUPPORTED_FORMAT', 'extracted': extracted}) for anchor, count in sorted(Counter(anchors).items()): if count > 1: result['findings'].append({'code': 'duplicate_heading_anchor', 'sourceSha256': source['sha256'], 'url': source['url'], 'observation': {'anchor': anchor, 'count': count}}) continue if not re.search(r'<(?:html|head|title|body|h1)\b', text, re.I): result['sources'].append({'sha256': source['sha256'], 'url': source['url'], 'availability': 'UNSUPPORTED_FORMAT', 'extracted': {}}) continue if department in ('WIKI_SCIENTIFIC', 'CATALOG_COMMERCE'): import json findings = [] if department == 'WIKI_SCIENTIFIC': extracted = {'headings': [{'level': int(level), 'text': re.sub('<[^>]+>', '', body).strip()} for level, body in re.findall(r']*>(.*?)', text, re.I | re.S)]} previous = 0 for heading in extracted['headings']: if heading['level'] > previous + 1: findings.append(('heading_level_skip', {'previousLevel': previous, **heading})) previous = heading['level'] else: products = [] for attrs, body in re.findall(r']*)>(.*?)', text, re.I | re.S): if not re.search(r'type\s*=\s*[\"\']application/ld\+json[\"\']', attrs, re.I): continue try: value = json.loads(body) except ValueError: findings.append(('invalid_json_ld', {'parseable': False})) continue queue = value if isinstance(value, list) else [value] while queue: node = queue.pop() if not isinstance(node, dict): continue queue.extend(node.get('@graph', []) if isinstance(node.get('@graph', []), list) else []) if node.get('@type') == 'Product' or (isinstance(node.get('@type'), list) and 'Product' in node['@type']): products.append({'name': node.get('name'), 'description': node.get('description')}) extracted = {'products': products} for index, product in enumerate(products): if not isinstance(product['name'], str) or not product['name'].strip(): findings.append(('product_schema_missing_name', {'productIndex': index, 'name': product['name']})) result['sources'].append({'sha256': source['sha256'], 'url': source['url'], 'availability': 'AVAILABLE', 'extracted': extracted}) for code, observation in findings: result['findings'].append({'code': code, 'sourceSha256': source['sha256'], 'url': source['url'], 'observation': observation}) continue page = Page(text) extracted = {'title': page.title, 'h1': page.h1, 'metaDescription': page.descriptions} result['sources'].append({'sha256': source['sha256'], 'url': source['url'], 'extracted': extracted}) for code, values in [('missing_title', page.title), ('missing_meta_description', page.descriptions), ('missing_h1', page.h1)]: if not any(values): result['findings'].append({'code': code, 'sourceSha256': source['sha256'], 'url': source['url'], 'observation': values}) return result def research_lookup(client, research_id, max_pages=3): if not 1 <= max_pages <= 5: raise ValueError('Expected 1..5 research pages') for page in range(1, max_pages + 1): response = client.call('market-insights?limit=100&page=' + str(page)) if len(response['data']) > 100: raise ValueError('Oversized research page') for research in response['data']: if research['id'] == research_id: return research if not response['meta']['hasNextPage']: break raise ValueError('Research absent within bounded history') def execute(client, store, department, key): import json from corporate_runtime import now task = client.call('claims', {'department': department, 'leaseSeconds': 300}, key + ':claim')['task'] if task is None: return {'task': None} op = json.loads(task['description']) if (task['actionType'] != 'RESEARCH' or task['department'] != department or set(op) != {'operation', 'researchId'} or op['operation'] != 'department_audit_v1'): raise ValueError('Unsupported task; left leased without evidence') started = now() artifact = audit(task['id'], department, research_lookup(client, op['researchId']), store) digest = store.put(canonical(artifact)) receipt = {'operation': 'department_audit_v1', 'startedAt': started, 'finishedAt': now(), 'sha256': digest, 'findingCount': len(artifact['findings']), 'sourceCount': len(artifact['sources'])} body = {'claimToken': task['claimToken'], 'artifactUri': 'urn:sha256:' + digest, 'sha256': digest, 'summary': 'Deterministic persisted-source audit; no publishing or external effects', 'tool': 'python:department_audit_v1', 'result': json.dumps(receipt)} checkpoint = client.checkpoint(store, key + ':evidence', 'tasks/' + task['id'] + '/evidence', body) return {'taskId': task['id'], 'artifactSha256': digest, 'checkpoint': checkpoint, 'evidence': client.call('tasks/' + task['id'] + '/evidence', body, key + ':evidence')} def plan_day(tasks, day, complete=True, store=None): from datetime import date date.fromisoformat(day) output = {'day': day, 'operation': 'pmo_daily_outcomes_v1', 'historyComplete': complete, 'outcomes': [], 'existingFollowup': [], 'dailyNew': [], 'limitations': []} for task in tasks: lineage = {'taskId': task['id'], 'department': task['department'], 'evidence': [{'id': e['id'], 'sha256': e['sha256']} for e in task['evidence']]} truncated = any(x.get('truncated') for x in task.get('relationsMeta', {}).values()) if truncated: output['historyComplete'] = False output['outcomes'].append(lineage | {'status': task['status'], 'effects': task['effects'], 'reviews': task['reviews'], 'historyTruncated': truncated}) reason = None if task['status'] == 'QA_REVIEW' and task['evidence'] and not task['reviews']: reason = 'evidence_awaits_review' elif task['status'] == 'BLOCKED': reason = 'task_blocked' elif task['status'] in ('TODO', 'IN_PROGRESS'): reason = 'existing_work_not_complete' if reason: output['existingFollowup'].append(lineage | {'reason': reason}) if store and task['status'] == 'DONE' and not truncated: import json for evidence in task['evidence']: try: artifact = json.loads(store.get(evidence['sha256'])) except (ValueError, OSError): output['limitations'].append({'taskId': task['id'], 'reason': 'audit_artifact_unavailable'}) continue if artifact.get('operation') != 'department_audit_v1' or artifact.get('taskId') != task['id']: continue for finding in artifact['findings']: output['dailyNew'].append(lineage | {'reason': 'completed_audit_has_unresolved_finding', 'finding': finding, 'action': 'Verify current source and resolve observed structural gap; do not publish automatically'}) active_departments = {t['department'] for t in tasks if t['status'] not in ('DONE', 'CANCELLED')} unique = {} for action in output['dailyNew']: if action['department'] not in active_departments: unique[canonical({'department': action['department'], 'finding': action['finding']})] = action output['dailyNew'] = list(unique.values()) if not output['historyComplete']: output['dailyNew'] = [] output['limitations'].append({'reason': 'incomplete_history_no_new_work_inferred'}) return output def history(client, max_pages=3): if not 1 <= max_pages <= 5: raise ValueError('Expected 1..5 pages') tasks, seen = [], set() for page in range(1, max_pages + 1): response = client.call('tasks?limit=100&page=' + str(page)) if len(response['data']) > 100: raise ValueError('Oversized task page') for row in response['data']: if row['id'] in seen: continue seen.add(row['id']) if not re.fullmatch(r'[a-zA-Z0-9-]{1,36}', row['id']): raise ValueError('Invalid task ID') tasks.append(client.call('tasks/' + row['id'])) if not response['meta']['hasNextPage']: return tasks, True return tasks, False def main(): import argparse import json from pathlib import Path from corporate_runtime import Client, Store, sha parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('command', choices=['audit', 'execute', 'plan']) parser.add_argument('--store', required=True) parser.add_argument('--department', choices=['CONTENT_SEO', 'TECH_DEVOPS', 'CATALOG_COMMERCE', 'WIKI_SCIENTIFIC']) parser.add_argument('--research', help='Persisted API research JSON for offline audit') parser.add_argument('--task-id', default='offline-audit') parser.add_argument('--snapshot', help='Task detail array; always treated as incomplete offline history') parser.add_argument('--day') parser.add_argument('--api', default='http://127.0.0.1:3001/api/admin/hermes') parser.add_argument('--credential-env', default='CORPORATE_API_KEY') parser.add_argument('--key') args = parser.parse_args() store = Store(args.store) def read_json(path): raw = Path(path).read_bytes() if len(raw) > 2_000_000: raise ValueError('Input too large') return json.loads(raw), sha(raw) if args.command == 'audit': research, digest = read_json(args.research) result = audit(args.task_id, args.department, research, store) result.update(mode='OFFLINE_PERSISTED_SOURCE', inputSha256=digest) elif args.command == 'execute': if not args.key or not args.department: parser.error('--key and --department required') result = execute(Client(args.api, args.credential_env), store, args.department, args.key) else: if args.snapshot: tasks, digest = read_json(args.snapshot) complete, mode = False, 'OFFLINE_SNAPSHOT' else: tasks, complete = history(Client(args.api, args.credential_env)) digest, mode = store.put(canonical(tasks)), 'API_GET_ONLY' result = plan_day(tasks, args.day, complete=complete, store=store) result.update(mode=mode, inputSha256=digest) artifact = store.put(canonical(result)) print(json.dumps(result | {'artifactSha256': artifact}, ensure_ascii=False, sort_keys=True)) if __name__ == '__main__': main()