#!/usr/bin/env python3
"""Offline recount for the YC identity-gate study.

Inputs are the saved September 18 roster, September 19 guarded homepage
inventory, and the bounded September 21 title/headline verification. The
recount does not make network requests.
"""
from __future__ import annotations
import argparse, json, re, sys, unicodedata
from collections import Counter, defaultdict
from pathlib import Path
from urllib.parse import urlsplit

ROOT = Path(__file__).resolve().parent
DEFAULT_ROSTER = None
# This existing public compact dataset retains URL/status fields but no raw HTML.
DEFAULT_INVENTORY = ROOT.parent / 'yc-homepage-links-2026-09-19.json'
PRIMARY = ROOT.parent / 'yc-identity-gate-primary-verification-2026-09-21.json'

# This is intentionally a small public-suffix approximation for these saved
# hosts. The inputs were checked for the listed exceptions before use.
SECOND_LEVEL = {'co.uk','com.au','co.jp','co.in','com.br','com.mx','co.nz','co.za'}
def hostname(url: str | None) -> str:
    return (urlsplit(url or '').hostname or '').lower().removeprefix('www.')
def registrable(host: str) -> str:
    labels = host.split('.')
    suffix = '.'.join(labels[-2:]) if len(labels) >= 2 else host
    return '.'.join(labels[-3:]) if suffix in SECOND_LEVEL and len(labels) >= 3 else suffix

def tokens(name: str) -> list[str]:
    text = unicodedata.normalize('NFKD', name).encode('ascii','ignore').decode().lower()
    return [x for x in re.findall(r'[a-z0-9]+', text) if len(x) >= 4 and x not in {'company','incorporated','technologies','technology','systems','automation','insurance','health','labs','lab','group','capital','network','services'}]
def has_row_token(row: dict, verification: dict) -> bool:
    # Whole-word matching avoids treating a generic substring as a brand.
    # Compact matching preserves names whose public typography inserts dots or spaces.
    haystack = ' '.join([verification.get('title') or '', verification.get('meta_description') or ''] + [h.get('text','') for h in verification.get('headings') or []]).lower()
    words = set(re.findall(r'[a-z0-9]+', haystack))
    compact_haystack = ''.join(re.findall(r'[a-z0-9]+', haystack))
    compact_name = ''.join(re.findall(r'[a-z0-9]+', unicodedata.normalize('NFKD', row['name']).encode('ascii','ignore').decode().lower()))
    return any(token in words for token in tokens(row['name'])) or bool(compact_name and compact_name in compact_haystack)

def main(argv: list[str] | None = None) -> int:
    parser = argparse.ArgumentParser(description='Offline YC identity-gate recount; no network requests.')
    parser.add_argument('--roster', type=Path, default=DEFAULT_ROSTER, help='Optional archived roster JSON input; the public compact inventory supplies names/cohorts when omitted.')
    parser.add_argument('--inventory', type=Path, default=DEFAULT_INVENTORY, help='Public compact homepage URL/status dataset or saved homepage inventory JSON input.')
    parser.add_argument('--primary', type=Path, default=PRIMARY, help='Bounded primary verification JSON input.')
    parser.add_argument('--output', type=Path, default=ROOT / 'recount-output.json', help='Output JSON path.')
    args = parser.parse_args(argv)
    inventory = json.loads(args.inventory.read_text())['companies']
    verified = json.loads(args.primary.read_text())['companies']
    if args.roster:
        roster = json.loads(args.roster.read_text())['rows']
    else:
        # The public compact inventory keeps the fields needed for this study.
        roster = [{'source_object_id': r['companyID'], 'name': r['publicCompanyName'], 'batch': r['cohort']} for r in inventory]
    assert len(roster) == 1586, len(roster)
    assert len(inventory) == len(roster) == 1586
    by_id = {int(r['source_object_id']): r for r in roster}
    inv_by_id = {int(r['companyID']): r for r in inventory}
    verify_by_id = {int(r['companyID']): r for r in verified}
    assert len(by_id) == len(roster)
    assert set(inv_by_id) == set(by_id)
    assert all(by_id[i]['name'] == inv_by_id[i]['publicCompanyName'] and by_id[i]['batch'] == inv_by_id[i]['cohort'] for i in by_id), 'roster name/cohort must match inventory'
    observed = [r for r in inventory if r.get('htmlObserved') and r.get('homepageStatus') == 200]
    unknown = [r for r in inventory if r not in observed]
    # Host continuity is a triage signal, not proof of legal ownership or a rebrand.
    states = Counter()
    queue = []
    for r in observed:
        listed, final = hostname(r.get('listedWebsite')), hostname(r.get('finalHomepage'))
        if not listed or not final:
            state = 'unclassifiable_host'
        elif listed == final:
            state = 'same_host'
        elif registrable(listed) == registrable(final):
            state = 'same_registrable_domain'
        else:
            state = 'cross_registrable_domain_manual_identity'
            queue.append(r)
        states[state] += 1
    assert states == Counter({'same_host': 1476, 'same_registrable_domain': 5, 'cross_registrable_domain_manual_identity': 34}), states
    assert len(queue) == 34
    assert len(verified) == 39
    assert all(v.get('status') == 200 and not v.get('error') for v in verified)
    assert all((v.get('robots') or {}).get('allowed') is True for v in verified), 'robots policy must be verified'
    assert all(all(h.get('allowed') is True for h in v.get('robots_chain', [])) for v in verified), 'every redirect policy must be verified'
    assert all(v.get('observed_url') == v.get('finalHomepage') or any(h.get('target') == v.get('observed_url') and h.get('allowed') is True for h in v.get('robots_chain', [])) for v in verified), 'redirect target policy must be verified'
    same_brand, different_brand = [], []
    for r in queue:
        v = verify_by_id[int(r['companyID'])]
        target = same_brand if has_row_token(by_id[int(r['companyID'])], v) else different_brand
        target.append({
            'company_id': int(r['companyID']), 'name': r['publicCompanyName'], 'cohort': r['cohort'],
            'listed_website': r['listedWebsite'], 'observed_homepage': r['finalHomepage'],
            'title': v.get('title'), 'headings': v.get('headings', []), 'verification_sha256': v.get('body_sha256'),
            'status': 'same_brand_signal_in_title_or_heading' if target is same_brand else 'different_brand_signal_or_no_row_token',
        })
    assert len(same_brand) == 29, len(same_brand)
    assert len(different_brand) == 5, len(different_brand)
    by_cohort = defaultdict(lambda: {'rows': 0, 'html_observed': 0, 'unknown': 0, 'same_host': 0, 'same_registrable_domain': 0, 'cross_registrable_domain_manual_identity': 0})
    for r in roster: by_cohort[r['batch']]['rows'] += 1
    for r in inventory:
        c = by_cohort[r['cohort']]
        if r in observed:
            c['html_observed'] += 1
            listed, final = hostname(r.get('listedWebsite')), hostname(r.get('finalHomepage'))
            if listed == final: c['same_host'] += 1
            elif registrable(listed) == registrable(final): c['same_registrable_domain'] += 1
            else: c['cross_registrable_domain_manual_identity'] += 1
        else: c['unknown'] += 1
    for c in by_cohort.values(): assert c['rows'] == c['html_observed'] + c['unknown']
    result = {
        'study': 'yc-identity-gate',
        'recount_scope': {'roster_rows': len(roster), 'observed_2xx_html': len(observed), 'unknown_html': len(unknown), 'unknown_rule': 'no saved 2xx HTML observation; includes 54 no-status and 17 non-2xx rows'},
        'states': dict(states),
        'manual_queue': {'cross_registrable_domain_rows': len(queue), 'same_brand_signal_in_verified_title_or_heading': len(same_brand), 'different_brand_signal_or_no_row_token': len(different_brand)},
        'by_cohort': {k: by_cohort[k] for k in sorted(by_cohort)},
        'same_brand_examples': sorted(same_brand, key=lambda x:(x['cohort'],x['name'])),
        'different_brand_examples': sorted(different_brand, key=lambda x:(x['cohort'],x['name'])),
    }
    args.output.write_text(json.dumps(result, indent=2, ensure_ascii=False)+'\n')
    print(json.dumps({'roster_rows':len(roster),'observed_2xx_html':len(observed),'unknown_html':len(unknown),'same_host':states['same_host'],'same_registrable_domain':states['same_registrable_domain'],'cross_registrable_domain_manual_identity':states['cross_registrable_domain_manual_identity'],'same_brand_signal':len(same_brand),'different_brand_signal_or_no_row_token':len(different_brand)}))
    return 0
if __name__ == '__main__': raise SystemExit(main())