#!/usr/bin/env python3
"""Bounded, read-only verification of saved cross-host YC redirects.

Fetches only the 39 final public homepage URLs already recorded by the
September 19 link-followup. Stores public title/headings plus response hashes;
never stores page bodies and never submits forms.
"""
from __future__ import annotations
import argparse, hashlib, html, http.client, ipaddress, json, pathlib, re, socket, sys
from concurrent.futures import ThreadPoolExecutor, as_completed
from datetime import datetime, timezone
from html.parser import HTMLParser
from urllib.error import URLError, HTTPError
from urllib.parse import urljoin, urlsplit
from urllib.request import Request, build_opener, HTTPRedirectHandler, HTTPHandler, HTTPSHandler, ProxyHandler
from urllib.robotparser import RobotFileParser

BASE = pathlib.Path(__file__).resolve().parent
DEFAULT_INVENTORY = BASE.parent / 'yc-homepage-links-2026-09-19.json'
USER_AGENT = 'MudpiePublicResearch/1.0 (+https://mudpie.ai; credential-free read-only study)'
PRIVATE_PATH = re.compile(r'(?:^|/)(?:private|admin|account|accounts|login|logout|log-in|log-out|signin|signout|sign-in|sign-out|signup|sign-up|auth|oauth|settings|credentials?|api[-_]?keys?|tokens?|sessions?|reset-password)(?:/|$)', re.I)

class PageParser(HTMLParser):
    def __init__(self):
        super().__init__(convert_charrefs=True)
        self.title = []
        self.meta_description = None
        self.headings = []
        self._tag = None
        self._buffer = []
    def handle_starttag(self, tag, attrs):
        tag = tag.lower(); attrs = dict(attrs)
        if tag == 'title': self._tag = 'title'; self._buffer = []
        elif tag in {'h1','h2'}: self._tag = tag; self._buffer = []
        elif tag == 'meta':
            name = (attrs.get('name') or attrs.get('property') or '').lower()
            if name == 'description' and attrs.get('content'): self.meta_description = clean(attrs['content'])[:300]
    def handle_endtag(self, tag):
        tag = tag.lower()
        if self._tag == tag:
            value = clean(' '.join(self._buffer))
            if value:
                if tag == 'title': self.title = [value]
                elif tag in {'h1','h2'} and len(self.headings) < 8: self.headings.append({'tag': tag, 'text': value[:300]})
            self._tag = None; self._buffer = []
    def handle_data(self, data):
        if self._tag: self._buffer.append(data)

def clean(s): return re.sub(r'\s+', ' ', html.unescape(str(s))).strip()
def public_url(u):
    try:
        p = urlsplit(u)
        if p.scheme not in {'http','https'} or p.username or p.password or not p.hostname or '.' not in p.hostname.rstrip('.'): return None
        if p.port not in {None, 80 if p.scheme == 'http' else 443}: return None
        if PRIVATE_PATH.search(p.path): return None
        return u
    except Exception: return None
class SafeRedirect(HTTPRedirectHandler):
    def redirect_request(self, req, fp, code, msg, headers, newurl):
        # fetch_bytes follows redirects explicitly, after destination policy checks.
        return None
def public_socket(host, port, timeout):
    """Resolve once, reject the entire answer set if unsafe, and connect to a pinned address."""
    if port not in {80, 443}: raise URLError('unsafe_port')
    addresses = socket.getaddrinfo(host, port, type=socket.SOCK_STREAM)
    if not addresses: raise URLError('no_public_address')
    for _, _, _, _, address in addresses:
        ip = ipaddress.ip_address(address[0])
        if not ip.is_global or ip.is_multicast or ip.is_unspecified or getattr(ip, 'is_site_local', False) or '%' in address[0]:
            raise URLError('non_public_address')
        if isinstance(ip, ipaddress.IPv6Address) and (ip.ipv4_mapped or ip.sixtofour or ip.teredo):
            raise URLError('unsupported_transition_address')
    last_error = None
    for family, kind, protocol, _, address in addresses:
        sock = socket.socket(family, kind, protocol)
        try:
            sock.settimeout(timeout)
            sock.connect(address)
            return sock
        except OSError as exc:
            sock.close()
            last_error = exc
    raise URLError('public_connection_failed') from last_error

class PublicHTTPConnection(http.client.HTTPConnection):
    def connect(self):
        if self._tunnel_host: raise URLError('tunnel_not_supported')
        self.sock = public_socket(self.host, self.port, self.timeout)

class PublicHTTPSConnection(http.client.HTTPSConnection):
    def connect(self):
        if self._tunnel_host: raise URLError('tunnel_not_supported')
        sock = public_socket(self.host, self.port, self.timeout)
        try:
            self.sock = self._context.wrap_socket(sock, server_hostname=self.host)
        except Exception:
            sock.close()
            raise

class PublicHTTPHandler(HTTPHandler):
    def http_open(self, req):
        if not public_url(req.full_url): raise URLError('unsafe_target')
        return self.do_open(PublicHTTPConnection, req)

class PublicHTTPSHandler(HTTPSHandler):
    def https_open(self, req):
        if not public_url(req.full_url): raise URLError('unsafe_target')
        return self.do_open(PublicHTTPSConnection, req, context=self._context)

# Ignore ambient proxies: all requests must use the validated destination socket.
OPENER = build_opener(ProxyHandler({}), PublicHTTPHandler, PublicHTTPSHandler, SafeRedirect)
def fetch_bytes(url, max_bytes=2_000_000, *, respect_robots=False, policies=None):
    for _ in range(8):
        if not public_url(url): raise URLError('unsafe_target')
        if respect_robots:
            policy = check_robots(url)
            if policies is not None: policies.append({'target': url, **policy})
            if policy.get('allowed') is False: raise PermissionError('robots_disallowed')
            if policy.get('allowed') is not True: raise URLError('robots_unverified')
        req = Request(url, headers={'User-Agent': USER_AGENT, 'Accept': 'text/html,application/xhtml+xml;q=0.9,*/*;q=0.1'})
        try:
            response = OPENER.open(req, timeout=16)
        except HTTPError as exc:
            if exc.code in {301, 302, 303, 307, 308} and exc.headers.get('Location'):
                url = urljoin(url, exc.headers['Location'])
                exc.close()
                continue
            exc.close()
            raise
        with response as resp:
            chunks=[]; total=0
            while True:
                chunk=resp.read(min(65536,max_bytes-total))
                if not chunk: break
                chunks.append(chunk); total += len(chunk)
                if total >= max_bytes: break
            return {'status': getattr(resp,'status',None), 'url': resp.geturl(), 'content_type': resp.headers.get('Content-Type',''), 'body': b''.join(chunks), 'truncated': total>=max_bytes}
    raise URLError('redirect_limit')
def check_robots(url):
    p = urlsplit(url); robots_url = f'{p.scheme}://{p.netloc}/robots.txt'
    try:
        r = fetch_bytes(robots_url, 512_000)
        if r['truncated']:
            return {'url': robots_url, 'status': r['status'], 'allowed': None, 'error': 'robots_truncated'}
        text = r['body'].decode('utf-8','replace') if r['status'] and r['status'] < 400 else ''
        # Stdlib group-merging behavior varies across Python patch releases.
        # Conservatively leave multiple matching declarations unverified.
        product = USER_AGENT.split('/')[0].lower()
        agents = re.findall(r'(?im)^\s*user-agent\s*:\s*([^\s#]+)', text)
        if sum(agent == '*' or agent.lower() in product for agent in agents) > 1:
            return {'url': robots_url, 'status': r['status'], 'allowed': None, 'error': 'robots_multiple_matching_agents', 'body_sha256': hashlib.sha256(r['body']).hexdigest()}
        rp = RobotFileParser(); rp.set_url(robots_url); rp.parse(text.splitlines())
        return {'url': robots_url, 'status': r['status'], 'allowed': rp.can_fetch(USER_AGENT, url), 'body_sha256': hashlib.sha256(r['body']).hexdigest()}
    except HTTPError as exc:
        allowed = True if exc.code in {404, 410} else False if exc.code in {401, 403} else None
        exc.close()
        return {'url': robots_url, 'status': exc.code, 'allowed': allowed, 'error': 'HTTPError'}
    except Exception as exc: return {'url': robots_url, 'status': None, 'allowed': None, 'error': type(exc).__name__}
def one(row):
    out = {k: row.get(k) for k in ['companyID','publicCompanyName','cohort','primaryIndustry','listedWebsite','finalHomepage','homepageStatus']}
    out.update({'verification_started_at': datetime.now(timezone.utc).isoformat(), 'robots': None, 'status': None, 'observed_url': None, 'content_type': None, 'bytes': 0, 'body_sha256': None, 'truncated': False, 'title': None, 'headings': [], 'meta_description': None, 'error': None})
    try:
        if sys.version_info < (3, 14, 6):
            raise RuntimeError('network_verification_requires_python_3_14_6_for_robots_matching')
        url = row['finalHomepage']
        if not public_url(url): raise ValueError('unsafe_target')
        out['robots_chain'] = []
        try:
            f = fetch_bytes(url, respect_robots=True, policies=out['robots_chain'])
        finally:
            out['robots'] = out['robots_chain'][0] if out['robots_chain'] else None
        body=f['body']
        out.update(status=f['status'], observed_url=f['url'], content_type=f['content_type'], bytes=len(body), body_sha256=hashlib.sha256(body).hexdigest(), truncated=f['truncated'])
        if f['status'] and f['status'] < 400 and 'html' in f['content_type'].lower():
            parser=PageParser(); parser.feed(body.decode('utf-8','replace'))
            out.update(title=(parser.title[0] if parser.title else None), headings=parser.headings, meta_description=parser.meta_description)
    except Exception as exc: out['error'] = f'{type(exc).__name__}:{exc}'
    out['verification_finished_at'] = datetime.now(timezone.utc).isoformat()
    return out
def main(argv=None):
    parser = argparse.ArgumentParser(description='Credential-free bounded verification of saved cross-host homepage URLs.')
    parser.add_argument('--inventory', type=pathlib.Path, default=DEFAULT_INVENTORY, help='Saved homepage inventory JSON input.')
    parser.add_argument('--output', type=pathlib.Path, default=BASE / 'primary-verification.json', help='Output metadata JSON path.')
    args = parser.parse_args(argv)
    inventory = args.inventory
    d=json.loads(inventory.read_text()); rows=[]
    for r in d['companies']:
        a=(urlsplit(r.get('listedWebsite') or '').hostname or '').lower().removeprefix('www.')
        b=(urlsplit(r.get('finalHomepage') or '').hostname or '').lower().removeprefix('www.')
        if a and b and a != b: rows.append(r)
    assert len(rows)==39, len(rows)
    with ThreadPoolExecutor(max_workers=3) as pool: results=[f.result() for f in as_completed([pool.submit(one,r) for r in rows])]
    results.sort(key=lambda x:(x['cohort'],x['publicCompanyName']))
    payload={'metadata':{'source_inventory':inventory.name,'source_inventory_sha256':hashlib.sha256(inventory.read_bytes()).hexdigest(),'scope':'39 saved final-host-change rows only','fetched_at':datetime.now(timezone.utc).isoformat(),'network_scope':'credential-free GET final homepage and robots.txt only; no forms or reference pages','body_storage':'discarded; only response hash, size, public title/headings/meta description retained','user_agent':USER_AGENT},'companies':results}
    args.output.write_text(json.dumps(payload,indent=2,ensure_ascii=False)+'\n')
    print(json.dumps({'rows':len(results),'html_titles':sum(bool(x['title']) for x in results),'robots_disallowed':sum(str(x.get('error') or '').startswith('PermissionError') for x in results),'errors':sum(bool(x.get('error')) for x in results)}))
if __name__=='__main__': main()