#!/usr/bin/env python3
import json, re, html, os
from pathlib import Path
from urllib.parse import urljoin, urlparse, urldefrag, parse_qsl, urlencode, urlunparse
from urllib.request import Request, urlopen
from urllib.error import HTTPError, URLError
from collections import deque

ROOT=Path(__file__).resolve().parents[1]
BASE=os.environ.get('MPC_AUDIT_BASE','http://127.0.0.1:43104').rstrip('/')
base_parsed=urlparse(BASE)
HOSTS={'marlothparkcentral.com','www.marlothparkcentral.com',base_parsed.netloc.lower()}
BAD_CODES={402,403,404,405}
MAX=2500
ATTR_RE=re.compile(r'''\b(?:href|src|action|poster)\s*=\s*["']([^"']+)["']''',re.I)
SRCSET_RE=re.compile(r'''\bsrcset\s*=\s*["']([^"']+)["']''',re.I)
URL_CSS_RE=re.compile(r'''url\(([^)]+)\)''',re.I)
ignore_schemes=('mailto:','tel:','sms:','whatsapp:','javascript:','data:','blob:','#')
api_skip_prefixes=('/api/mpc/contact-message','/api/mpc/accommodation-enquiry','/api/mpc/stay-enquiry','/api/mpc/order-pack','/api/mpc/free-guide-lead','/api/mpc/gated-delivery/request-code','/api/mpc/gated-delivery/verify','/api/mpc/guide-delivery','/api/mpc/alerts-subscribe','/api/mpc/preferences','/api/admin/')
external=set(); visited={}; bad=[]; discovered_by={}

def normalise(raw,current='/'):
    if raw is None: return None
    raw=html.unescape(str(raw).strip()).replace('&','&')
    if not raw or raw.startswith(ignore_schemes): return None
    # skip template/compiler fragments
    if any(x in raw for x in ['${','function','=>','window.','location.','currentTarget','.href','||','&&','?','.map(']):
        return None
    u=urljoin(BASE+current,raw)
    u,_=urldefrag(u)
    p=urlparse(u)
    if p.scheme not in ('http','https'): return None
    if p.netloc.lower() not in HOSTS:
        external.add(u); return None
    path=p.path or '/'
    if any(path.startswith(x) for x in api_skip_prefixes): return None
    if path.startswith(('/_next/','/images/','/uploads/','/fonts/')): return None
    if re.search(r'\.(?:webp|png|jpe?g|gif|svg|ico|woff2?|ttf|mp4|webm|pdf|zip)$', path, re.I): return None
    query=urlencode(sorted(parse_qsl(p.query,keep_blank_values=True))) if p.query else ''
    return urlunparse((base_parsed.scheme,base_parsed.netloc,path,'',query,''))

def fetch(url):
    try:
        with urlopen(Request(url,headers={'User-Agent':'MPC-deep-launch-audit/2.0','Accept':'text/html,application/xhtml+xml,application/xml,text/css,image/*,*/*;q=0.8'}),timeout=15) as r:
            return r.status,r.headers.get('content-type','').split(';')[0].lower(),r.read(3_000_000),str(r.url)
    except HTTPError as e:
        try: body=e.read(1_000_000)
        except Exception: body=b''
        return e.code,e.headers.get('content-type','').split(';')[0].lower(),body,url
    except Exception as e:
        return 'ERR','',repr(e).encode(),url

def extract(body,ctype,path):
    out=[]
    if ctype not in ('text/html','application/xhtml+xml','text/css','application/xml','text/xml') and not path.endswith(('.css','.xml','.html')):
        return out
    text=body.decode('utf-8','ignore')
    if ctype in ('text/html','application/xhtml+xml') or path.endswith('.html') or path=='/':
        for m in ATTR_RE.finditer(text): out.append(m.group(1))
        for m in SRCSET_RE.finditer(text):
            for part in m.group(1).split(','):
                bits=part.strip().split()
                if bits: out.append(bits[0])
    if ctype=='text/css' or path.endswith('.css'):
        for m in URL_CSS_RE.finditer(text): out.append(m.group(1).strip(' "\''))
    if ctype in ('application/xml','text/xml') or path.endswith('.xml'):
        for loc in re.findall(r'<loc>(.*?)</loc>', text): out.append(loc)
    return out

def seed_urls():
    seeds=['/','/robots.txt','/sitemap.xml']
    rm=ROOT/'.next/routes-manifest.json'
    if rm.exists():
        data=json.load(open(rm))
        for r in data.get('staticRoutes',[]):
            page=r.get('page')
            if page and not page.startswith('/api') and page != '/_not-found': seeds.append(page)
    # Source literal href checks, only href/url values that start with slash
    for p in ROOT.rglob('*'):
        if not p.is_file(): continue
        if 'node_modules' in p.parts or '.next' in p.parts: continue
        if p.suffix not in ('.tsx','.ts','.js','.css','.md','.json'): continue
        try: txt=p.read_text(errors='ignore')
        except Exception: continue
        for pat in [r'''href\s*=\s*["'`](/[^"'`{}\s]+)["'`]''', r'''href\s*:\s*["'`](/[^"'`{}\s]+)["'`]''', r'''url\s*:\s*["'`](/[^"'`{}\s]+)["'`]''', r'''redirect\s*\(\s*["'`](/[^"'`{}\s]+)["'`]''']:
            for raw in re.findall(pat, txt):
                seeds.append(raw)
    out=[]; seen=set()
    for s in seeds:
        n=normalise(s,'/')
        if n and n not in seen:
            out.append(n); seen.add(n)
    return out

q=deque(seed_urls())
for u in q: discovered_by.setdefault(u,'seed/source')
while q and len(visited)<MAX:
    u=q.popleft()
    if u in visited: continue
    status,ctype,body,final=fetch(u)
    visited[u]={'status':status,'ctype':ctype,'bytes':len(body),'by':discovered_by.get(u,''),'final':final}
    isbad=(status=='ERR') or (isinstance(status,int) and (status in BAD_CODES or status>=500))
    if isbad:
        bad.append({'url':u,'status':status,'ctype':ctype,'by':discovered_by.get(u,''),'sample':body[:240].decode('utf-8','ignore')})
        continue
    if isinstance(status,int) and status==200:
        path=urlparse(u).path
        for raw in extract(body,ctype,path):
            n=normalise(raw,path)
            if n and n not in visited and n not in q:
                discovered_by[n]=u; q.append(n)

version=os.environ.get('NEXT_PUBLIC_BUILD_VERSION','6.82.173')
out=ROOT/f'operations/proof/MPC_v{version}_DEEP_ROUTE_LINK_CRAWL.json'
out.parent.mkdir(parents=True,exist_ok=True)
out.write_text(json.dumps({'checked_count':len(visited),'bad_count':len(bad),'bad':bad,'visited':visited,'external_skipped_count':len(external),'external_sample':sorted(external)[:100]},indent=2),encoding='utf-8')
print(f'checked_count={len(visited)} bad_count={len(bad)} external_skipped={len(external)}')
for b in bad[:100]: print('BAD', b['status'], b['url'], 'by', b['by'])
if not bad: print('NO_BAD_ROUTE_LINK_STATUS')