#!/usr/bin/env python3
import json, re, sys, urllib.parse, os
from collections import deque
from pathlib import Path
import requests
from bs4 import BeautifulSoup

BASE=os.environ.get('MPC_AUDIT_BASE','http://127.0.0.1:43180').rstrip('/')
HOST=os.environ.get('MPC_AUDIT_HOST','marlothparkcentral.com')
root=Path(__file__).resolve().parents[1]
s=requests.Session(); s.headers.update({'Host':HOST,'User-Agent':'MPC-v6.82.190-proof-crawler/1.0'})
seeds=['/','/robots.txt','/sitemap.xml','/api/build-info','/route-atlas','/kidz','/free-guides','/international-visitors-guide','/accommodation','/services','/local-options','/contact']
# Public routes are sourced from sitemap and recursive internal links.
# Add sitemap URLs.
try:
    r=s.get(BASE+'/sitemap.xml',timeout=15,allow_redirects=True)
    for loc in re.findall(r'<loc>(.*?)</loc>',r.text):
        u=urllib.parse.urlparse(loc)
        if u.path: seeds.append(u.path + (('?'+u.query) if u.query else ''))
except Exception: pass

q=deque(dict.fromkeys(seeds)); seen=set(); results=[]; assets=set(); external=0
while q and len(seen)<600:
    path=q.popleft()
    if not path.startswith('/'): continue
    key=path.split('#',1)[0]
    if key in seen: continue
    seen.add(key)
    try:
        r=s.get(BASE+key,timeout=25,allow_redirects=True)
        final=urllib.parse.urlparse(r.url).path
        results.append({'path':key,'status':r.status_code,'final':final,'content_type':r.headers.get('content-type','')})
        if r.status_code>=400: continue
        ctype=r.headers.get('content-type','')
        if 'text/html' in ctype:
            soup=BeautifulSoup(r.text,'html.parser')
            for tag,attr in [('a','href'),('img','src'),('script','src'),('link','href'),('source','src'),('video','poster')]:
                for el in soup.find_all(tag):
                    val=el.get(attr)
                    if not val: continue
                    val=val.strip()
                    if val.startswith(('mailto:','tel:','javascript:','data:','blob:','#')): continue
                    absu=urllib.parse.urljoin('https://'+HOST+key,val)
                    u=urllib.parse.urlparse(absu)
                    if u.hostname not in (HOST,'www.'+HOST):
                        external+=1; continue
                    p=u.path + (('?'+u.query) if u.query else '')
                    if tag=='a':
                        if p not in seen: q.append(p)
                    else:
                        assets.add(p)
    except Exception as e:
        results.append({'path':key,'status':'ERR','final':'','error':str(e)})

asset_results=[]
for p in sorted(assets):
    try:
        r=s.get(BASE+p,timeout=25,allow_redirects=True,stream=True)
        asset_results.append({'path':p,'status':r.status_code,'content_type':r.headers.get('content-type','')})
        r.close()
    except Exception as e:
        asset_results.append({'path':p,'status':'ERR','error':str(e)})

bad=[x for x in results if x['status'] not in (200,301,302,307,308)]
bad_assets=[x for x in asset_results if x['status'] not in (200,206,301,302,307,308)]
proof={'base':BASE,'host':HOST,'pages_checked':len(results),'assets_checked':len(asset_results),'external_skipped':external,'bad_pages':bad,'bad_assets':bad_assets,'pages':results,'assets':asset_results}
out=root/'operations/proof/MPC_v6.82.190_RUNTIME_CRAWL.json'; out.parent.mkdir(parents=True,exist_ok=True); out.write_text(json.dumps(proof,indent=2))
print(json.dumps({'pages_checked':len(results),'assets_checked':len(asset_results),'bad_pages':len(bad),'bad_assets':len(bad_assets),'external_skipped':external},indent=2))
if bad:
    print('BAD PAGES',file=sys.stderr)
    for x in bad[:50]: print(x,file=sys.stderr)
if bad_assets:
    print('BAD ASSETS',file=sys.stderr)
    for x in bad_assets[:50]: print(x,file=sys.stderr)
sys.exit(1 if bad or bad_assets else 0)