119 lines
4.3 KiB
Python
119 lines
4.3 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Template: Chefkoch Nightly Scraper (async Playwright)
|
|
|
|
Scrapes recipe metadata from Chefkoch.de via headless browser.
|
|
Intended to be customized per search query and run as background job.
|
|
|
|
Prerequisite: playwright install chromium
|
|
"""
|
|
import asyncio, json, re
|
|
from datetime import datetime
|
|
from pathlib import Path
|
|
from playwright.async_api import async_playwright
|
|
|
|
# ─── Config ─────────────────────────────────────────────────────────────────
|
|
QUERY = "asiatisch einfach familie"
|
|
MAX_RESULTS = 10
|
|
OUTPUT_DIR = Path.home() / "Documents/Essensplanung"
|
|
OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
|
|
OUTPUT = OUTPUT_DIR / f"scrape_{re.sub(r'[^a-z0-9]', '_', QUERY.lower())}_{datetime.now().strftime('%Y%m%d')}.json"
|
|
|
|
# ─── Playwright ─────────────────────────────────────────────────────────────
|
|
|
|
async def accept_cookies(page):
|
|
try:
|
|
await page.click("button[title='Zustimmen']", timeout=2000)
|
|
except:
|
|
pass
|
|
|
|
async def scrape_page(browser, query, max_results):
|
|
results = []
|
|
url = f"https://www.chefkoch.de/rs/s0/{query.replace(' ', '%20')}/Rezepte.html"
|
|
|
|
page = await browser.new_page()
|
|
try:
|
|
await page.goto(url, wait_until="networkidle", timeout=30000)
|
|
await accept_cookies(page)
|
|
|
|
# Scrolle bis genug Karten geladen
|
|
for _ in range(15):
|
|
cards = await page.query_selector_all(".ds-recipe-card")
|
|
if len(cards) >= max_results * 2:
|
|
break
|
|
await page.evaluate("window.scrollBy(0, 800)")
|
|
await asyncio.sleep(1.5)
|
|
|
|
# Extrahiere Links
|
|
links = await page.eval_on_selector_all(
|
|
"a.ds-recipe-card__title-link",
|
|
"els => [...new Set(els.map(e => e.href))]"
|
|
)
|
|
|
|
print(f" {len(links)} Links gefunden")
|
|
|
|
# Lade Details
|
|
for link in list(dict.fromkeys(links))[:max_results]:
|
|
detail = await get_recipe_detail(browser, link)
|
|
if detail:
|
|
results.append(detail)
|
|
print(f" + {detail['name'][:40]}")
|
|
await asyncio.sleep(0.5)
|
|
finally:
|
|
await page.close()
|
|
|
|
return results
|
|
|
|
async def get_recipe_detail(browser, url):
|
|
page = await browser.new_page()
|
|
try:
|
|
await page.goto(url, wait_until="networkidle", timeout=15000)
|
|
await accept_cookies(page)
|
|
|
|
# Name
|
|
name = await page.eval_on_selector("h1", "e => e?.innerText.trim() || 'Unbekannt'")
|
|
|
|
# Zutaten
|
|
ings = []
|
|
rows = await page.query_selector_all("table.ingredients tbody tr")
|
|
for row in rows[:15]:
|
|
try:
|
|
amt = await row.eval_on_selector("td.amount", "e => e.innerText.trim()")
|
|
item = await row.eval_on_selector("td.name", "e => e.innerText.trim()")
|
|
if item: ings.append({"amount": amt, "item": item})
|
|
except: pass
|
|
|
|
# Zeit
|
|
time = await page.eval_on_selector(
|
|
".ds-recipe-meta__icon-text, .bi-recipe-meta-duration",
|
|
"e => e?.innerText.trim() || '?'"
|
|
)
|
|
|
|
return {
|
|
"name": name,
|
|
"url": url,
|
|
"ingredients": ings,
|
|
"prep_time_text": time,
|
|
}
|
|
except Exception as e:
|
|
print(f" ⚠️ Fehler: {e}")
|
|
return None
|
|
finally:
|
|
await page.close()
|
|
|
|
# ─── Main ───────────────────────────────────────────────────────────────────
|
|
|
|
async def main():
|
|
print(f"🔍 Scrape: '{QUERY}' → {MAX_RESULTS}x")
|
|
|
|
async with async_playwright() as p:
|
|
browser = await p.chromium.launch(headless=True)
|
|
results = await scrape_page(browser, QUERY, MAX_RESULTS)
|
|
await browser.close()
|
|
|
|
OUTPUT.write_text(json.dumps(results, indent=2, ensure_ascii=False), encoding="utf-8")
|
|
print(f"💾 {len(results)} Rezepte → {OUTPUT}")
|
|
|
|
if __name__ == "__main__":
|
|
asyncio.run(main())
|