Best Buy - Product Search, Prices, Availability & Specs

# NEEDS A VISIBLE TAB: Best Buy never renders a hidden (background) tab, and new_tab() opens a
# hidden one. Run this in a dedicated or cloud browser with FOREGROUND_TAB = True, which brings
# the tab to the front. Left False, a hidden tab raises instead of taking over your window.
FOREGROUND_TAB = False
 
import json, math, time
from urllib.parse import quote_plus, urlparse, parse_qs
 
EXTRACT_CARDS = r"""(() => {
  const money = s => { const m = (s || '').match(/\$([\d,]+\.\d{2})/); return m ? parseFloat(m[1].replace(/,/g, '')) : null; };
  const txt = (el, sel) => { const e = el.querySelector(sel); return e ? e.innerText.trim() : null; };
  return [...document.querySelectorAll('.product-list-item[data-product-id]')].map(card => {
    const h3 = card.querySelector('h3');
    const link = h3 ? h3.closest('a') : null;
    const rnr = txt(card, '[data-testid="rnr-stats-link"] p') || '';
    const rm = rnr.match(/rating,? ([\d.]+) out of 5 stars with ([\d,]+) review/i);
    const price = money(txt(card, '[data-testid="price-block-customer-price"]'));
    const regular = money(txt(card, '[data-testid="price-block-regular-price"]'));
    // plp-add-to-cart-<sku>-label, plp-sold-out-<sku>-label, plp-unavailable-<sku>-label, plp_openbox-unavailable-<sku>-label
    const btn = card.querySelector('[data-testid^="plp"][data-testid$="-label"]');
    const button = btn ? btn.innerText.trim() : null;
    const status = button === 'Add to cart' ? 'in_stock' : button === 'Sold Out' ? 'sold_out'
      : button === 'Unavailable' ? 'unavailable' : /High demand product/.test(card.innerText) ? 'high_demand'
      : button ? button.toLowerCase() : null;
    return {
      sku: card.getAttribute('data-product-id'),
      title: h3 ? (h3.getAttribute('title') || h3.innerText).trim() : null,
      url: link ? link.href.split('?')[0] : null,
      price,                       // what the customer pays now
      regular_price: regular,      // "Comp. Value", only shown when discounted
      on_sale: regular !== null && price !== null && price < regular,
      savings: txt(card, '[data-testid="price-block-total-savings-text"]'),
      rating: rm ? parseFloat(rm[1]) : null,
      review_count: rm ? parseInt(rm[2].replace(/,/g, ''), 10) : (/Not yet reviewed/i.test(card.innerText) ? 0 : null),
      availability: status,
      button,
      pickup: txt(card, '[data-testid^="plp-pickup-messaging-"]'),
      shipping: txt(card, '[data-testid^="plp-shipping-messaging-"]'),
      sponsored: !!card.closest('li.slContainer'),
    };
  });
})()"""
 
def safe_js(expr, tries=6, deadline=None):
    # The page's main thread is busy for a few seconds after navigation and
    # Runtime.evaluate can time out; retry instead of failing, but not past the deadline.
    for _ in range(tries):
        try:
            return js(expr)
        except (RuntimeError, TimeoutError):
            if deadline is not None and time.monotonic() + 2 > deadline:
                break
            wait(2)
    return None
 
class NotReady(RuntimeError):
    """The page did not render, or showed an error, in this attempt."""
 
def nav(url, href_ok, timeout):
    # Best Buy keeps the main thread busy; give Page.navigate more than the 5 s default.
    # A tab can hang mid-navigation: stop it and navigate once more (a second timeout raises).
    end = time.monotonic() + timeout
    left = lambda: max(1, min(30, end - time.monotonic()))  # no CDP wait runs past the caller's deadline
    # Record where the tab was and mark that document, so a reload of the same URL is told apart.
    before = safe_js("(window.__bhOldDoc = true, location.href)", tries=1)
    try:
        cdp("Page.navigate", url=url, _response_timeout=left())
    except TimeoutError:
        cdp("Page.stopLoading", _response_timeout=left())
        cdp("Page.navigate", url=url, _response_timeout=left())
    here = None
    while time.monotonic() < end:  # read nothing until the tab is on a new document at the requested URL
        wait(1)
        here = safe_js("({href: location.href, old: !!window.__bhOldDoc,"
                       " status: (performance.getEntriesByType('navigation')[0] || {}).responseStatus})", tries=1) or here
        if here and not here["old"] and href_ok(here["href"]):
            if here["status"] in (403, 429):  # a block: stop, never retry
                raise RuntimeError(f"HTTP {here['status']} at {here['href']}: blocked, stop and back off")
            return here["href"]
    raise NotReady(f"navigation to {url} not confirmed within {timeout:.0f} s "
                   f"(was {before}, now {here['href'] if here else 'unreadable'})")
 
def ensure_visible():
    if safe_js("document.visibilityState") == "visible":
        return
    if not FOREGROUND_TAB:
        raise RuntimeError("tab is hidden and Best Buy does not render hidden tabs: run this in a dedicated or "
                           "cloud browser with FOREGROUND_TAB = True (brings the tab to the front), or show the tab yourself")
    activate_tab(current_tab())
 
# robots.txt (User-agent: *) disallows any URL containing cp=1 (cp=1, cp=10-19), cp=20 and cp=6..9.
# Page 1 is the URL without cp; pages 2-5 are the only other pages it allows in sequence.
ROBOTS_PAGES = (1, 2, 3, 4, 5)
 
def search_url(query, page):
    if page not in ROBOTS_PAGES:
        raise ValueError(f"page {page}: robots.txt disallows cp={page} for User-agent: *")
    url = f"https://www.bestbuy.com/site/searchpage.jsp?st={quote_plus(query)}"
    return url if page == 1 else f"{url}&cp={page}"
 
def search_href_ok(query, page):
    def ok(href):  # Best Buy may add id=pcat17071 and reorder the parameters; st and cp must match
        u = urlparse(href)
        q = parse_qs(u.query)
        return (u.netloc == "www.bestbuy.com" and u.path == "/site/searchpage.jsp"
                and q.get("st") == [query] and q.get("cp", ["1"]) == [str(page)])
    return ok
 
def load_search_page(query, page, deadline):
    def check_time(step):
        if time.monotonic() > deadline:
            raise NotReady(f"out of time while {step}")
    nav(search_url(query, page), search_href_ok(query, page), timeout=min(45, deadline - time.monotonic()))
    for _ in range(20):  # wait until product titles have rendered
        wait(1.5)
        if (safe_js("document.querySelectorAll('.product-list-item h3').length", deadline=deadline) or 0) > 0:
            break
        # The block never asks for a page past the end (see PAGE_SIZE), so this is an error page.
        if safe_js("document.body.innerText.includes('unable to display products')", deadline=deadline):
            raise NotReady("Best Buy showed \"unable to display products\"")
        check_time("waiting for product titles")
    else:
        raise NotReady("no product titles rendered in 30 s")
    # Cards and price blocks render lazily when they scroll into view.
    for _ in range(25):
        check_time("scrolling")
        safe_js("window.scrollBy(0, 800)", deadline=deadline)
        wait(0.6)
        if safe_js("window.innerHeight + window.scrollY >= document.body.scrollHeight - 50", deadline=deadline):
            break
    for _ in range(3):  # revisit cards whose title or price block has not rendered yet
        missing = safe_js("""[...document.querySelectorAll('.product-list-item[data-product-id]')]
            .filter(c => !c.querySelector('h3') || !c.querySelector('[data-testid="price-block-customer-price"]'))
            .map(c => c.getAttribute('data-product-id'))""", deadline=deadline) or []
        if not missing:
            break
        for sku in missing:
            check_time("revisiting unrendered cards")
            safe_js(f"document.querySelector('.product-list-item[data-product-id=\"{sku}\"]')?.scrollIntoView({{block: 'center'}})", deadline=deadline)
            wait(0.8)
        wait(1.5)
    cards = safe_js(EXTRACT_CARDS, deadline=deadline)
    if not cards:
        raise NotReady("no product cards could be read")
    # A card shell that never hydrated has a SKU but no h3 or link; sponsored cards render both too.
    shells = [c["sku"] for c in cards if not (c["sku"] and c["title"] and c["url"])]
    if shells:
        raise NotReady(f"{len(shells)} of {len(cards)} cards still lack SKU, title or URL: {shells[:5]}")
    # listResult.count is sometimes {} and searchResultsNum is "" or a numeric string: accept numbers only
    meta = safe_js("""(() => {
      const num = v => (v === '' || v === null || v === undefined || typeof v === 'object') ? null
        : (Number.isFinite(Number(v)) ? Number(v) : null);
      const lr = (window._BBY_DATA_ || {}).listResult || {}, d = window.dioPageData || {};
      return {total: num(lr.count) ?? num(d.searchResultsNum), page: num(d.listPageNum) ?? num(lr.pageNumber)};
    })()""", deadline=deadline) or {}
    if meta.get("page") is None or meta.get("total") is None:
        raise NotReady(f"Best Buy did not report the page number and result count ({meta})")
    if meta["page"] != page:  # also confirms the cp-less URL is page 1
        raise RuntimeError(f"asked for page {page}, Best Buy rendered page {meta['page']}")
    return cards, meta["total"], meta["page"]
 
RETRY_BUDGET_S = 120  # one deadline per result page, covering every attempt and every backoff
MIN_ATTEMPT_S = 40    # do not back off and retry unless an attempt this long still fits
 
def fetch_page(query, page):
    deadline = time.monotonic() + RETRY_BUDGET_S
    attempt = 0
    while True:
        attempt += 1
        try:
            return load_search_page(query, page, deadline)
        except NotReady as e:
            backoff = 15 * attempt
            if time.monotonic() + backoff + MIN_ATTEMPT_S > deadline:  # last attempt: raise now, no sleep
                raise RuntimeError(f"page {page} failed after {attempt} attempt(s) within {RETRY_BUDGET_S} s: {e}. "
                                   "Results so far are incomplete; do not report them as the full crawl") from None
            print(f"page {page}: {e}; backing off {backoff} s, then retrying")
            time.sleep(backoff)
 
# Best Buy paginates 18 organic results per page (24 cards = 18 organic + 6 sponsored; 1236 results
# gave a last-page link of 69 = ceil(1236 / 18)). The crawl stops at the last page the count implies.
PAGE_SIZE = 18
 
new_tab("about:blank")
ensure_visible()
 
query = "gaming laptop"
all_items, seen = [], set()
for page in ROBOTS_PAGES:
    cards, total, shown_page = fetch_page(query, page)  # raises rather than return a partial crawl
    new = 0
    for c in cards:  # the same SKU can show up as a sponsored and an organic card
        if c["sku"] and c["sku"] not in seen:
            seen.add(c["sku"])
            c["page"] = page
            all_items.append(c)
            new += 1
    print(f"page {page} (site says page {shown_page}): {len(cards)} cards, {new} new, "
          f"{sum(c['price'] is not None for c in cards)} priced, total results {total}")
    if page >= math.ceil(total / PAGE_SIZE):
        print(f"page {page} is the last page for {total} results: done")
        break
    if page != ROBOTS_PAGES[-1]:
        time.sleep(3)  # polite pacing between result pages
else:
    print(f"stopped after page {ROBOTS_PAGES[-1]}: robots.txt disallows cp=6 to cp=20 for User-agent: *, "
          f"so the rest of the {total} results are not fetched")
 
from collections import Counter
print(len(all_items), "unique products;", Counter(c["availability"] for c in all_items))
for item in all_items[:3]:
    print(json.dumps(item))

# NEEDS A VISIBLE TAB, like the search block: in a hidden tab the main price block never renders.
# Run it in a dedicated or cloud browser with FOREGROUND_TAB = True, which brings the tab to the
# front. Left False, a hidden tab raises instead of taking over your window.
FOREGROUND_TAB = False
 
import json, time
from urllib.parse import urlparse, parse_qs
 
PRODUCT_LD = r"""(() => {
  for (const s of document.querySelectorAll('script[type="application/ld+json"]')) {
    try { const d = JSON.parse(s.textContent); if (d['@type'] === 'Product') return d; } catch (e) {}
  }
  return null;
})()"""
 
SPEC_SHEET = r"""(() => {
  const root = document.querySelector('[data-testid="brix-sheet-content"]');
  if (!root) return null;
  const out = {};
  for (const h3 of root.querySelectorAll('h3')) {
    const rows = {};
    for (const row of h3.parentElement.querySelectorAll(':scope > div')) {
      if (row.children.length < 2) continue;
      const k = row.children[0].innerText.trim(), v = row.children[1].innerText.trim();
      if (k) rows[k] = v;
    }
    out[h3.innerText.trim()] = rows;
  }
  return out;
})()"""
 
def safe_js(expr, tries=6):
    for _ in range(tries):
        try:
            return js(expr)
        except (RuntimeError, TimeoutError):  # main thread busy right after navigation
            wait(2)
    return None
 
def nav(url, href_ok, timeout=45):
    # Best Buy pages keep the main thread busy; give Page.navigate more than the 5 s default.
    # A tab can hang mid-navigation: stop it and navigate once more (a second timeout raises).
    end = time.monotonic() + timeout
    left = lambda: max(1, min(30, end - time.monotonic()))
    # Record where the tab was and mark that document, so a reload of the same URL is told apart.
    before = safe_js("(window.__bhOldDoc = true, location.href)", tries=1)
    try:
        cdp("Page.navigate", url=url, _response_timeout=left())
    except TimeoutError:
        cdp("Page.stopLoading", _response_timeout=left())
        cdp("Page.navigate", url=url, _response_timeout=left())
    here = None
    while time.monotonic() < end:  # read nothing until the tab is on a new document at the requested URL
        wait(1)
        here = safe_js("({href: location.href, old: !!window.__bhOldDoc,"
                       " status: (performance.getEntriesByType('navigation')[0] || {}).responseStatus})", tries=1) or here
        if here and not here["old"] and href_ok(here["href"]):
            if here["status"] in (403, 429):  # a block: stop, never retry
                raise RuntimeError(f"HTTP {here['status']} at {here['href']}: blocked, stop and back off")
            return here["href"]
    raise RuntimeError(f"navigation to {url} not confirmed within {timeout:.0f} s "
                       f"(was {before}, now {here['href'] if here else 'unreadable'})")
 
def ensure_visible():
    if safe_js("document.visibilityState") == "visible":
        return
    if not FOREGROUND_TAB:
        raise RuntimeError("tab is hidden and Best Buy does not render hidden tabs: run this in a dedicated or "
                           "cloud browser with FOREGROUND_TAB = True (brings the tab to the front), or show the tab yourself")
    activate_tab(current_tab())
 
def click_button_named(name):
    try:  # every CDP call here can exceed the 5 s default while the page is busy
        nodes = cdp("Accessibility.getFullAXTree", _response_timeout=30)["nodes"]
        for n in nodes:
            if n.get("role", {}).get("value") == "button" and (n.get("name", {}).get("value") or "").strip().lower() == name.lower():
                bid = n["backendDOMNodeId"]
                cdp("DOM.scrollIntoViewIfNeeded", backendNodeId=bid, _response_timeout=30)
                wait(0.5)
                q = cdp("DOM.getBoxModel", backendNodeId=bid, _response_timeout=30)["model"]["content"]
                click_at_xy(sum(q[0::2]) / 4, sum(q[1::2]) / 4)
                return True
    except (RuntimeError, TimeoutError):
        pass  # the caller retries
    return False
 
# The main product's price block. Accessory, warranty, sponsored and "consider" carousels
# also use data-testid="price-block" and "price-block-regular-price", but sit outside this
# container, so never read those test ids page-wide.
MAIN_PRICE = '[data-testid="bestbuy-price-ui-price-experience"] [data-testid="price-block"]'
 
MAIN_PRICES = r"""(() => {
  const money = e => { const m = e && e.innerText.match(/\$([\d,]+\.\d{2})/); return m ? parseFloat(m[1].replace(/,/g, '')) : null; };
  const block = document.querySelector('%s');
  if (!block) return null;
  return {price: money(block.querySelector('[data-testid="price-block-customer-price"]')),
          regular: money(block.querySelector('[data-testid="price-block-regular-price"]'))};
})()""" % MAIN_PRICE
 
def get_product(sku):
    # /site/<sku>.p?skuId=<sku> redirects to the canonical /product/<slug>/<code> URL
    nav(f"https://www.bestbuy.com/site/{sku}.p?skuId={sku}",
        lambda h: urlparse(h).netloc == "www.bestbuy.com" and urlparse(h).path.startswith("/product/"))
    ld, state = None, None
    for _ in range(30):  # JSON-LD arrives with the HTML; the page state and price block hydrate later
        wait(1.5)
        ld = ld or safe_js(PRODUCT_LD)
        # a second isPDP entry for the same SKU can appear with priceType null; take the filled one
        state = safe_js(f"((window._BBY_DATA_ || {{}}).skus || []).find(s => s.id === '{sku}' && s.isPDP && s.priceType) || null")
        if ld and state and safe_js(f"!!document.querySelector('{MAIN_PRICE} [data-testid=\"price-block-customer-price\"]')"):
            break
    if not ld:
        return None
    if str(ld.get("sku")) != str(sku):  # the redirect target must be the product that was asked for
        raise RuntimeError(f"asked for SKU {sku}, the page is SKU {ld.get('sku')}")
    state = state or {}
    offers = ld.get("offers") or []
    if isinstance(offers, dict):
        offers = [offers]
    new = next((o for o in offers if "NewCondition" in (o.get("itemCondition") or "")), offers[0] if offers else {})
    # Main product only: its "Comp. Value" line shows when it is discounted, else regular is None
    main = safe_js(MAIN_PRICES) or {}
    regular = main.get("regular")
    specs = None
    for _ in range(3):  # a click before hydration finishes does nothing, so retry
        if not click_button_named("See all specifications"):
            wait(2)
            continue
        for _ in range(8):
            wait(1)
            specs = safe_js(SPEC_SHEET)
            if specs:
                break
        if specs:
            break
    try:
        press_key("Escape")  # close the sheet
    except (RuntimeError, TimeoutError):
        pass
    rating = ld.get("aggregateRating") or {}
    return {
        "sku": ld.get("sku"),
        "title": ld.get("name"),
        "brand": (ld.get("brand") or {}).get("name"),
        "model": ld.get("model"),
        "url": ld.get("url"),
        "price": new.get("price"),
        "price_block_price": main.get("price"),          # should equal price
        "regular_price": regular,                        # None when the main product is not discounted
        "on_sale": regular is not None and main.get("price") is not None and main["price"] < regular,
        "availability": (new.get("availability") or "").rsplit("/", 1)[-1],
        "price_type": state.get("priceType"),           # "regular" / "onSale"; None if the state never hydrated
        "button_state": state.get("primaryButtonState"),  # e.g. "ADD_TO_CART"
        "rating": rating.get("ratingValue"),
        "review_count": rating.get("reviewCount"),
        "other_offers": [{"condition": o.get("description"), "price": o.get("price")} for o in offers if o is not new],
        "specs": specs,
    }
 
ensure_visible()
# Pick a SKU from a live search so this block does not depend on a hard-coded product.
nav("https://www.bestbuy.com/site/searchpage.jsp?st=laptop",
    lambda h: urlparse(h).path == "/site/searchpage.jsp" and parse_qs(urlparse(h).query).get("st") == ["laptop"]
    and "cp" not in parse_qs(urlparse(h).query))
skus = []
for _ in range(20):
    wait(1.5)
    skus = safe_js("[...new Set(((window._BBY_DATA_ || {}).skus || []).filter(s => s.id && !s.isSponsored).map(s => s.id))]") or []
    if len(skus) >= 5:  # the list fills in as the page hydrates
        break
print("organic SKUs on page:", skus[:5])
# Scrape organic SKUs until there is one discounted and one full-price product to show both cases.
shown = set()
for sku in skus[:12]:
    product = get_product(sku)
    if not product:
        print(sku, "no product data")
        continue
    specs = product.pop("specs") or {}
    print(sku, "on_sale", product["on_sale"], "price", product["price"], "price block", product["price_block_price"],
          "regular", product["regular_price"], "price_type", product["price_type"])
    if product["on_sale"] in shown:
        continue
    shown.add(product["on_sale"])
    print(json.dumps(product, indent=1))
    print(sum(len(v) for v in specs.values()), "spec rows in", len(specs), "sections")
    print(json.dumps({k: specs[k] for k in list(specs)[:3]}, indent=1))
    if shown == {True, False}:
        break