Sephora - Category Listings & Product Details

import json, re, time
 
LIST_URL = "https://www.sephora.com/shop/face-serum"   # a /shop/<category> or /brand/<brand> page
MAX_PAGES = 3      # 60 products per page
PAUSE = 5          # seconds between page loads; robots.txt asks for Crawl-delay: 5
 
if not re.fullmatch(r"https://www\.sephora\.com/(shop|brand)/[a-z0-9-]+", LIST_URL):
    raise SystemExit("LIST_URL must be https://www.sephora.com/shop/<slug> or /brand/<slug>, no query string")
 
def load(url, timeout=30):
    """Navigate the attached tab once, wait until location.href is exactly url, return the page's linkStore JSON."""
    deadline = time.time() + timeout
    try:
        goto_url(url)
    except TimeoutError:   # the harness stops waiting for Page.navigate after 5 s; the page may still arrive
        pass
    while True:
        try:
            state = js("""(() => ({href: location.href, title: document.title, ready: document.readyState,
                           store: document.getElementById('linkStore')?.textContent || ''}))()""") or {}
        except TimeoutError:   # page still busy with the navigation
            state = {}
        if "Access Denied" in (state.get("title") or ""):
            raise SystemExit(f"Sephora's CDN answered {url} with 'Access Denied' (HTTP 403). "
                             "Stop and back off; do not retry in a loop.")
        if state.get("href") == url and state.get("ready") == "complete" and state.get("store"):
            return json.loads(state["store"])
        if time.time() >= deadline:
            raise RuntimeError(f"{url} did not load with a linkStore in {timeout} s, tab is on {state.get('href')}")
        time.sleep(0.5)
 
def parse(p):
    sku = p.get("currentSku") or {}
    return {
        "product_id": p.get("productId"),                 # "P480284"
        "sku_id": sku.get("skuId"),                      # default SKU shown on the tile
        "brand": p.get("brandName"),
        "name": p.get("displayName"),
        "price": sku.get("listPrice"),                   # "$129.00" or a range "$96.00 - $142.00"
        "value_price": sku.get("valuePrice"),            # sets only: combined value of the items
        "rating": float(p["rating"]) if p.get("rating") else None,
        "reviews": int(p["reviews"]) if p.get("reviews") else 0,
        "variants": p.get("numberOfVariants"),
        "variation_type": p.get("variationType"),        # "Size", "Color", ...
        "sponsored": bool(p.get("sponsored")),
        "url": "https://www.sephora.com" + p["targetUrl"] if p.get("targetUrl") else None,
    }
 
# Reuse a tab already on sephora.com, otherwise enter via the homepage
# (in a fresh browser the first direct /shop/ load was denied in testing, the homepage was not).
opened_tab = None   # set only if this block opened the tab; it never closes a tab it did not open
href = js("location.href") or ""
if not href.startswith("https://www.sephora.com/"):
    opened_tab = new_tab("https://www.sephora.com/")
    wait_for_load()
    if "Access Denied" in (js("document.title") or ""):
        raise SystemExit("Sephora denied the homepage for this browser. Stop and back off.")
 
rows, seen, listing, total, size, pages_read = [], set(), {}, 0, 60, 0
for page in range(1, MAX_PAGES + 1):
    time.sleep(PAUSE)
    url = LIST_URL if page == 1 else f"{LIST_URL}?currentPage={page}"
    pg = load(url).get("page") or {}
    listing = pg.get("nthCategory") or pg.get("nthBrand")
    if not listing:
        raise RuntimeError(f"{url} has no page.nthCategory / page.nthBrand in linkStore: {list(pg)}")
    if int(listing.get("currentPage") or 1) != page:   # page 1 has currentPage None
        raise RuntimeError(f"asked for page {page}, linkStore says currentPage={listing.get('currentPage')}")
    if listing.get("totalProducts") is None:
        raise RuntimeError(f"{url}: listing has no totalProducts, cannot tell when the walk is complete")
    total, size = int(listing["totalProducts"]), int(listing.get("pageSize") or 60)
    products = listing.get("products") or []
    if not products and (page - 1) * size < total:
        raise RuntimeError(f"{url} has 0 products but totalProducts={total} means page {page} should not be empty")
    for p in products:
        if not p.get("productId") or not p.get("targetUrl"):   # required: identity and link
            raise RuntimeError(f"{url}: product card without productId/targetUrl, keys {sorted(p)[:12]}")
        row = parse(p)
        if row["product_id"] in seen:
            continue
        seen.add(row["product_id"])
        rows.append(row)
    pages_read = page
    print(f"page {page}/{-(-total // size)}: {len(products)} products, {len(rows)} unique so far")
    if page * size >= total:
        break
 
pages_total = -(-total // size)
print(listing.get("displayName"), listing.get("categoryId") or listing.get("brandId"), "totalProducts:", total)
if pages_read >= pages_total:
    if len(rows) != total:   # e.g. the listing shifted mid-walk: duplicates skipped, other products never seen
        raise RuntimeError(f"read {pages_read}/{pages_total} pages but got {len(rows)} unique products, "
                           f"totalProducts={total}; the walk is not complete")
    print(f"status: complete, {pages_read}/{pages_total} pages read, {len(rows)} unique rows")
else:
    print(f"status: PARTIAL, MAX_PAGES={MAX_PAGES} reached at page {pages_read}/{pages_total}: "
          f"{len(rows)} of {total} products, raise MAX_PAGES for the rest")
print(json.dumps(rows[:2], indent=1, ensure_ascii=False))
print(f"tab: opened {opened_tab}" if opened_tab else "tab: reused the existing sephora.com tab")

import html, json, re, time
 
PRODUCT_URL = "https://www.sephora.com/product/studio-fix-powder-plus-foundation-24hr-oil-control-blur-matte-finish-12hr-buildable-coverage-P516457"
PAUSE = 5   # seconds between page loads (robots.txt Crawl-delay: 5)
 
PRODUCT_URL = PRODUCT_URL.split("?")[0].split("#")[0]   # ?skuId=... from listing URLs is optional
m = re.fullmatch(r"https://www\.sephora\.com/product/[^/?#]*?(P\d+)", PRODUCT_URL)
if not m:
    raise SystemExit("PRODUCT_URL must look like https://www.sephora.com/product/<slug>-P<digits>")
PRODUCT_ID = m.group(1)
 
def text(s):
    return re.sub(r"\s+", " ", html.unescape(re.sub(r"<[^>]+>", " ", s or ""))).strip()
 
# Reuse a tab already on sephora.com (e.g. after the category block), otherwise enter via the homepage.
opened_tab = None   # set only if this block opened the tab; it never closes a tab it did not open
href = js("location.href") or ""
if not href.startswith("https://www.sephora.com/"):
    opened_tab = new_tab("https://www.sephora.com/")
    wait_for_load()
    if "Access Denied" in (js("document.title") or ""):
        raise SystemExit("Sephora denied the homepage for this browser. Stop and back off.")
time.sleep(PAUSE)
 
deadline = time.time() + 30
try:
    goto_url(PRODUCT_URL)
except TimeoutError:   # the harness stops waiting for Page.navigate after 5 s; the page may still arrive
    pass
while True:
    try:
        state = js("""(() => ({href: location.href, title: document.title, ready: document.readyState,
                       store: document.getElementById('linkStore')?.textContent || '',
                       ld: [...document.querySelectorAll('script[type="application/ld+json"]')].map(s => s.textContent)}))()""") or {}
    except TimeoutError:   # page still busy with the navigation
        state = {}
    if "Access Denied" in (state.get("title") or ""):
        raise SystemExit(f"'Access Denied' (HTTP 403) for {PRODUCT_URL}. Stop and back off.")
    on_url = re.fullmatch(re.escape(PRODUCT_URL) + r"(\?skuId=\d+)?", state.get("href") or "")
    if on_url and state.get("ready") == "complete" and state.get("store"):
        break
    if time.time() >= deadline:
        raise RuntimeError(f"{PRODUCT_URL} did not load with a linkStore in 30 s, tab is on {state.get('href')}")
    time.sleep(0.5)
 
prod = (json.loads(state["store"]).get("page") or {}).get("product")
if not prod or prod.get("productId") != PRODUCT_ID:
    raise RuntimeError(f"linkStore has no page.product for {PRODUCT_ID} (got {prod and prod.get('productId')})")
if not all(s.get("skuId") for s in prod.get("regularChildSkus") or [prod.get("currentSku") or {}]):
    raise RuntimeError(f"{PRODUCT_ID}: a variant SKU has no skuId")
 
# JSON-LD ProductGroup: per-variant price + availability (skip its 'reviews', they carry reviewer handles)
offers = {}
for raw in state.get("ld") or []:
    try:
        d = json.loads(raw)
    except ValueError:
        continue
    if d.get("@type") in ("ProductGroup", "Product"):
        for v in d.get("hasVariant") or [d]:
            o = v.get("offers") or {}
            if v.get("sku"):
                offers[v["sku"]] = {"price": o.get("price"), "currency": o.get("priceCurrency"),
                                    "availability": (o.get("availability") or "").rsplit("/", 1)[-1]}
 
pd = prod.get("productDetails") or {}
cur = prod.get("currentSku") or {}
variants = []
for s in prod.get("regularChildSkus") or [cur]:
    variants.append({
        "sku_id": s.get("skuId"),
        "value": s.get("variationValue"),             # shade name / size
        "description": s.get("variationDesc"),
        "size": s.get("size"),
        "list_price": s.get("listPrice"),
        "out_of_stock": s.get("isOutOfStock"),
        "few_left": s.get("isOnlyFewLeft"),
        "availability": (offers.get(s.get("skuId")) or {}).get("availability"),
        "image": (s.get("skuImages") or {}).get("imageUrl"),
        "url": "https://www.sephora.com" + s["targetUrl"] if s.get("targetUrl") else None,
    })
 
detail = {
    "product_id": prod.get("productId"),
    "brand": cur.get("brandName"),
    "brand_id": (pd.get("brand") or {}).get("brandId"),
    "name": pd.get("displayName"),
    "category": (prod.get("parentCategory") or {}).get("displayName"),
    "rating": pd.get("rating"),
    "reviews": pd.get("reviews"),
    "loves": pd.get("lovesCount"),
    "variation_type": prod.get("variationType"),      # "Color", "Size", ...
    "default_sku": cur.get("skuId"),
    "price": cur.get("listPrice"),
    "short_description": text(pd.get("shortDescription"))[:300],
    "ingredients": text(cur.get("ingredientDesc") or next(   # per SKU; some SKUs have none
        (s["ingredientDesc"] for s in prod.get("regularChildSkus") or [] if s.get("ingredientDesc")), ""))[:300],
    "how_to_use": text(pd.get("suggestedUsage"))[:300],
    "url": PRODUCT_URL,
    "variant_count": len(variants),
    "variants": variants,
}
print(json.dumps({**detail, "variants": variants[:3]}, indent=1, ensure_ascii=False))
print("variants with JSON-LD offer:", sum(1 for v in variants if v["availability"]), "of", len(variants))
print(f"tab: opened {opened_tab}" if opened_tab else "tab: reused the existing sephora.com tab")