#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""Scrape full Lieferando menus for all partners delivering to 47829 with headless chromium.
Usage: python3 scrape_lieferando.py [slug ...]   (default: all partners)
Re-running overwrites that service's previously scraped rows."""
import subprocess, sys, os, re, json, sqlite3
from bs4 import BeautifulSoup

SNAP="2026-11-17"
HERE=os.path.dirname(os.path.abspath(__file__))
CACHE=os.path.join(HERE,"cache"); os.makedirs(CACHE,exist_ok=True)
DB=os.path.join(HERE,"lieferdienste_47829.sqlite3")
LF="https://www.lieferando.de"
UA="Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"

# slug -> service_id, display name
SHOPS = {
 "master-pizza-more":("master_pizza_more","Master Pizza & More"),
 "dopi-47829":("dopi","DöPi"),
 "sorrento-trattoria-pizzeria":("sorrento","Sorrento Trattoria Pizzeria"),
 "pizza-live-traarer-strasse":("pizza_live","Pizza Live"),
 "boom-smash":("boom_smash","BOOM Smash"),
 "pizzeria-ciao-bockumer-platz":("pizzeria_ciao","Pizzeria Ciao"),
 "pizza-king-krefeld":("pizza_king","Pizza King"),
 "pizza-burger-bulls-krefeld":("pizza_bulls","Pizza & Burger Bulls"),
 "rumelner-grill":("rumelner_grill","Rumelner Grill"),
 "melt-n-munch-krefeld-uerdinger-strasse":("melt_n_munch","Melt n Munch"),
 "indische-kuche-krefeld":("indische_kueche","Indische Küche"),
 "chicken-chuck-krefeld":("chicken_chuck","Chicken Chuck"),
 "pizza-cab-alte-linner-strasse":("pizza_cab","Pizza Cab"),
 "pizza-hero-1":("pizza_hero","Pizza Hero"),
 "pizza-boys-krefeld":("pizza_boyz","Pizza Boyz"),
 "loco-chicken-krefeld-mitte-1":("loco_chicken","Loco Chicken Krefeld Mitte"),
 "tiger-food":("tiger_snacks","Tiger Snacks"),
 "just-smashed-krefeld-prinz-ferdinand-strasse":("just_smashed","Just Smashed"),
 "pizza-xpo-krefeld-47798":("pizza_xpo","Pizza XPO"),
 "thunderbuns-smash-burgers-krefeld-krefeld":("thunderbuns","Thunderbuns"),
 "bodrum-pizza-doner-marktstrasse-121-krefeld":("bodrum","Bodrum Pizza & Döner"),
 "sorrento-bros-krefeld":("sorrento_bros","Sorrento Bros"),
 "bei-genc-47829":("bei_genc","Bei Genc"),
 "mcdonalds-krefeld-bruchfeld":("mcdonalds","McDonald's Bruchfeld"),
 "pizza-xpo-krefeld-47798":("pizza_xpo","Pizza XPO"),
}
slugs = sys.argv[1:] or list(SHOPS)

def chromium(url, out):
    cmd=["chromium","--headless=new","--no-sandbox","--disable-gpu",
         "--window-size=1400,2500","--virtual-time-budget=20000",
         f"--user-agent={UA}","--dump-dom",url]
    with open(out,"w") as f:
        subprocess.run(cmd,stdout=f,stderr=subprocess.DEVNULL,timeout=120)
    return os.path.getsize(out)

def parse_menu(html):
    soup=BeautifulSoup(html,"html.parser")
    items=[]
    for ul in soup.select('ul[data-qa="item-category-list"]'):
        h2=ul.find_previous("h2")
        cat=None
        if h2:
            s=h2.select_one('span[aria-hidden="true"]')
            cat=(s or h2).get_text(" ",strip=True)
        for li in ul.select('li[data-item-id]'):
            name_el=li.select_one('span[data-qa="item-name"]')
            if not name_el: continue
            name=name_el.get_text(" ",strip=True)
            price_el=li.select_one('div[data-qa="item-price"]')
            price=None; ptype="standard"
            if price_el:
                t=price_el.get_text(" ",strip=True)
                m=re.search(r"([\d.,]+)\s*€",t)
                if m:
                    price=float(m.group(1).replace(".","").replace(",","."))
                if t.strip().lower().startswith("ab"): ptype="from"
            desc_el=li.select_one("span[class*='description']")
            desc=None
            if desc_el:
                ah=desc_el.select_one('span[aria-hidden="true"]')
                desc=(ah or desc_el).get_text(" ",strip=True) or None
            # sold-out?
            badge=li.get_text(" ",strip=True).lower()
            avail=0 if "ausverkauft" in badge or "sold out" in badge else 1
            items.append(dict(item_id=li["data-item-id"],category=cat,name=name,
                              description=desc,price=price,price_type=ptype,available=avail))
    return items

con=sqlite3.connect(DB); cur=con.cursor()
# ensure schema fields (idempotent)
cur.executescript("""
CREATE TABLE IF NOT EXISTS menu_items(
  id INTEGER PRIMARY KEY AUTOINCREMENT, service_id TEXT REFERENCES services(id),
  category TEXT, name TEXT, description TEXT, size TEXT, price REAL,
  price_type TEXT, available INTEGER DEFAULT 1, source_url TEXT, item_uid TEXT);
CREATE INDEX IF NOT EXISTS idx_items_service ON menu_items(service_id);
CREATE INDEX IF NOT EXISTS idx_items_name ON menu_items(name);
""")
try: cur.execute("ALTER TABLE menu_items ADD COLUMN item_uid TEXT")
except sqlite3.OperationalError: pass
# ensure services exist for each shop, pull min/fee from platform_partners
for slug,(sid,name) in SHOPS.items():
    row=cur.execute("SELECT 1 FROM services WHERE id=?",(sid,)).fetchone()
    p=cur.execute("SELECT min_order,free_delivery,rating,rating_count FROM platform_partners WHERE url LIKE ? LIMIT 1",("%"+slug+"%",)).fetchone()
    mino,free,rt,rc=p if p else (None,0,None,None)
    fee=0.0 if free else None
    if not row:
        cur.execute("""INSERT INTO services(id,name,kind,website,min_order,delivery_fee,rating,rating_count,notes,snapshot_date)
          VALUES(?,?,?,?,?,?,?,?,?,?)""",(sid,name,"platform:Lieferando",f"{LF}/speisekarte/{slug}",mino,fee,rt,rc,
          f"Automatisch gescraped (chromium headless). Mindestbestellwert/Liefergebuehr aus Lieferando-47829-Liste.",SNAP))
    else:
        cur.execute("""UPDATE services SET
          min_order=COALESCE(min_order,?), delivery_fee=COALESCE(delivery_fee,?),
          rating=COALESCE(rating,?), rating_count=COALESCE(rating_count,?) WHERE id=?""",
          (mino,fee,rt,rc,sid))

summary=[]
for slug in slugs:
    if slug not in SHOPS: print("skip",slug); continue
    sid,name=SHOPS[slug]
    url=f"{LF}/speisekarte/{slug}"
    cache=os.path.join(CACHE,f"lf_{slug}.html")
    if not os.path.exists(cache) or os.environ.get("REFETCH"):
        size=chromium(url,cache)
    else:
        size=os.path.getsize(cache)
    html=open(cache,encoding="utf-8",errors="ignore").read()
    if "Just a moment" in html or "challenge" in html[:20000].lower():
        print(f"{slug}: CHALLENGE page ({size}b)"); summary.append((slug,-1)); continue
    items=parse_menu(html)
    # replace old scraped rows for this service
    cur.execute("DELETE FROM menu_items WHERE service_id=? AND item_uid IS NOT NULL",(sid,))
    n=0
    for it in items:
        cur.execute("""INSERT INTO menu_items(service_id,category,name,description,size,price,price_type,available,source_url,item_uid)
          VALUES(?,?,?,?,?,?,?,?,?,?)""",
          (sid,it["category"],it["name"],it["description"],None,it["price"],it["price_type"],it["available"],url,it["item_id"]))
        n+=1
    cur.execute("INSERT OR REPLACE INTO sources(service_id,url,what,fetched_date) VALUES(?,?,?,?)",
      (sid,url,f"Vollstaendige Lieferando-Speisekarte gescraped (headless chromium): {n} Items, Cache cache/lf_{slug}.html",SNAP))
    con.commit()
    cats=sorted(set(i['category'] for i in items if i['category']))
    print(f"{slug}: {n} items, {len(cats)} Kategorien, cache {size//1024}KB")
    summary.append((slug,n))
    with open(os.path.join(HERE,"scrape_summary.json"),"w") as f:
        json.dump({s:c for s,c in summary},f,indent=1)
con.close()
print("DONE", summary)
