Tripadvisor - Scraping hotels and restaurants for a city

import re, time
 
QUERY = "Lisbon"
 
new_tab("https://www.tripadvisor.com/")
time.sleep(2)
wait_for_load(30)
time.sleep(2)
xy = js("""(() => {
  const i = document.querySelector('input[aria-label="Search"], input[type="search"]');
  if (!i) return null;
  i.scrollIntoView({block: 'center'});
  const r = i.getBoundingClientRect();
  return [r.x + r.width / 2, r.y + r.height / 2];
})()""")
if not xy:
    raise SystemExit("search box not found")
click_at_xy(*xy)
time.sleep(1)
type_text(QUERY)
time.sleep(4)   # typeahead suggestions render as role="option" elements
options = js("""[...document.querySelectorAll('[role="option"]')].map(o => ({
  label: o.innerText.replace(/\\n/g, ' | ').trim(),
  href: (o.querySelector('a') || o.closest('a') || {}).href || null,
}))""") or []
cities = [o for o in options if o["href"] and "/Tourism-g" in o["href"]]
for o in cities:
    m = re.search(r"/Tourism-(g\d+)-(.+)-Vacations\.html", o["href"])
    o["restaurants_url"] = f"https://www.tripadvisor.com/Restaurants-{m.group(1)}-{m.group(2)}.html"
    o["hotels_url"] = f"https://www.tripadvisor.com/Hotels-{m.group(1)}-{m.group(2)}-Hotels.html"
    print(o)
press_key("Escape")

import json, re, time
from urllib.parse import urljoin
 
CITY_URL = "https://www.tripadvisor.com/Restaurants-g60745-Boston_Massachusetts.html"
PAGES = 3          # 30 ranked restaurants per page
PER_PAGE = 30
 
def page_url(city_url, page):
    # page 1 = city URL; page N inserts -oa<30*(N-1)>- right after the g<geoId> segment
    if page == 1:
        return city_url
    return re.sub(r"-(g\d+)-", lambda m: f"-{m.group(1)}-oa{PER_PAGE * (page - 1)}-", city_url, count=1)
 
def open_page(url, first=False):
    new_tab(url) if first else goto_url(url)
    time.sleep(2)
    wait_for_load(30)
    wait_for_element('[data-automation="restaurantCard"]', timeout=20)
    time.sleep(1.5)
 
def ld_itemlist():
    for s in js("[...document.querySelectorAll('script[type=\"application/ld+json\"]')].map(s => s.textContent)") or []:
        try:
            d = json.loads(s)
        except ValueError:
            continue
        if isinstance(d, dict) and d.get("@type") == "ItemList":
            return [e.get("item", {}) for e in d.get("itemListElement", [])]
    return []
 
CARDS_JS = r"""
[...document.querySelectorAll('[data-automation="restaurantCard"]')].map(c => {
  const a = c.querySelector('a[href*="Restaurant_Review-"]');
  const lines = c.innerText.split('\n').map(s => s.trim()).filter(Boolean);
  const title = lines.find(l => /^\d+\.\s/.test(l)) || null;
  return {
    href: a ? a.getAttribute('href') : null,
    rank: title ? parseInt(title) : null,           // null = Sponsored card
    name: title ? title.replace(/^\d+\.\s*/, '') : lines.find(l => l !== 'MICHELIN'),
    rating: (c.querySelector('[data-automation="bubbleRatingValue"]') || {}).innerText || null,
    reviews: (c.querySelector('[data-automation="bubbleReviewCount"]') || {}).innerText || null,
    price: lines.find(l => /^\$+( - \$+)?$/.test(l)) || null,
  };
})
"""
 
def loc_id(url):
    m = re.search(r"-d(\d+)-", url or "")
    return m.group(1) if m else None
 
rows, seen = [], set()
for p in range(1, PAGES + 1):
    open_page(page_url(CITY_URL, p), first=(p == 1))
    ld = {loc_id(i.get("url")): i for i in ld_itemlist()}
    total = js("(document.querySelector('[data-automation=\"resultsTotal\"]') || {}).innerText || null")
    for c in js(CARDS_JS) or []:
        lid = loc_id(c["href"])
        if c["rank"] is None or not lid or lid in seen:
            continue                                   # skip sponsored cards and repeats
        seen.add(lid)
        j = ld.get(lid, {})
        agg = j.get("aggregateRating") or {}
        addr = j.get("address") or {}
        rows.append({
            "rank": c["rank"],
            "name": j.get("name") or c["name"],
            "rating": float(agg.get("ratingValue") or c["rating"] or 0) or None,
            "review_count": agg.get("reviewCount") or int(re.sub(r"\D", "", c["reviews"] or "") or 0) or None,
            "price_level": j.get("priceRange") or c["price"],
            "cuisine": j.get("servesCuisine"),
            "address": addr.get("streetAddress"),
            "telephone": j.get("telephone"),
            "url": urljoin("https://www.tripadvisor.com", c["href"]).split("?")[0].split("#")[0],
            "location_id": lid,
        })
    print(f"page {p}: {js('location.href')} total={total} rows so far={len(rows)}")
    time.sleep(4)   # polite pacing between pages
 
print(json.dumps(rows[:2], indent=1))
print([(r["rank"], r["name"]) for r in rows[-3:]])
print("ranks contiguous:", [r["rank"] for r in rows] == list(range(1, len(rows) + 1)))

import json, re, time
from urllib.parse import urljoin
 
CITY_URL = "https://www.tripadvisor.com/Hotels-g60745-Boston_Massachusetts-Hotels.html"
PAGES = 2          # 30 ranked hotels per page
PER_PAGE = 30
TITLES = '[data-automation="hotel-card-title"]'
 
def page_url(city_url, page):
    if page == 1:
        return city_url
    return re.sub(r"-(g\d+)-", lambda m: f"-{m.group(1)}-oa{PER_PAGE * (page - 1)}-", city_url, count=1)
 
def ranked_count():
    return js(f"[...document.querySelectorAll('{TITLES}')].filter(e => /^\\d+\\./.test(e.innerText)).length") or 0
 
def open_page(url, first=False):
    new_tab(url) if first else goto_url(url)
    time.sleep(2)
    wait_for_load(30)
    wait_for_element(TITLES, timeout=20)
    time.sleep(2)
 
SEE_ALL_JS = """[...document.querySelectorAll('button')].find(b => /^See all [\\d,]+ hotels/.test(b.innerText.trim()))"""
 
def expand_first_page():
    """Page 1 renders only 5 ranked hotels until 'See all N hotels' is clicked. Raises instead of scraping 5."""
    m = re.search(r"([\d,]+) properties in ", js("document.body.innerText") or "")
    if not m:
        raise RuntimeError("'N properties in City' total not found on page 1")
    want = min(PER_PAGE, int(m.group(1).replace(",", "")))   # small cities have fewer than 30
    if ranked_count() >= want:
        return
    if not js(f"(() => {{ const b = {SEE_ALL_JS}; if (!b) return false; b.scrollIntoView({{block: 'center'}}); return true; }})()"):
        raise RuntimeError(f"'See all N hotels' button not found; page 1 has {ranked_count()} of {want} ranked hotels")
    time.sleep(1.5)
    press_key("Escape")          # scrolling can pop the sticky date picker over the button
    time.sleep(1)
    xy = js(f"(() => {{ const r = ({SEE_ALL_JS}).getBoundingClientRect(); return [r.x + r.width / 2, r.y + r.height / 2]; }})()")
    click_at_xy(*xy)
    for _ in range(20):
        if ranked_count() >= want:
            return
        time.sleep(0.5)
    raise RuntimeError(f"'See all' click did not expand page 1: {ranked_count()} of {want} ranked hotels after 10 s")
 
def ld_itemlist():
    for s in js("[...document.querySelectorAll('script[type=\"application/ld+json\"]')].map(s => s.textContent)") or []:
        try:
            d = json.loads(s)
        except ValueError:
            continue
        if isinstance(d, dict) and d.get("@type") == "ItemList":
            return [e.get("item", {}) for e in d.get("itemListElement", [])]
    return []
 
CARDS_JS = r"""
[...document.querySelectorAll('[data-automation="hotel-card-title"]')].map(t => {
  const c = t.closest('li') || t.parentElement;
  const a = c.querySelector('a[href*="Hotel_Review-"]');
  const text = c.innerText;
  const price = text.match(/(?:^|\n)from\n([^\n]*\d[^\n]*)/);   // "from\n$165"; descriptions also say "from South Station"
  return {
    title: t.innerText.trim(),
    href: a ? a.getAttribute('href') : null,
    rating: (c.querySelector('[data-automation="bubbleRatingValue"]') || {}).innerText || null,
    reviews: (c.querySelector('[data-automation="bubbleReviewCount"]') || {}).innerText || null,
    rank_text: (text.match(/#\d[\d,]* [^\n]* of [\d,]+ [^\n]*/) || [null])[0],
    price_from: price ? price[1] : null,
  };
})
"""
 
def loc_id(url):
    m = re.search(r"-d(\d+)-", url or "")
    return m.group(1) if m else None
 
rows, seen = [], set()
for p in range(1, PAGES + 1):
    open_page(page_url(CITY_URL, p), first=(p == 1))
    if p == 1:
        expand_first_page()
    ld = {loc_id(i.get("url")): i for i in ld_itemlist()}
    for c in js(CARDS_JS) or []:
        m = re.match(r"(\d+)\.\s*(.+)", c["title"])
        lid = loc_id(c["href"])
        if not m or not lid or lid in seen:
            continue                                   # no "N." prefix = Sponsored card
        seen.add(lid)
        j = ld.get(lid, {})
        addr = j.get("address") or {}
        rows.append({
            "rank": int(m.group(1)),
            "name": m.group(2),
            "rating": float(c["rating"]) if c["rating"] else None,
            "review_count": int(re.sub(r"\D", "", c["reviews"] or "") or 0) or None,
            "rank_text": c["rank_text"],
            "price_from": c["price_from"],
            "price_tier": j.get("priceRange"),
            "address": ", ".join(filter(None, [addr.get("streetAddress"), addr.get("addressLocality")])) or None,
            "url": urljoin("https://www.tripadvisor.com", c["href"]).split("?")[0].split("#")[0],
            "location_id": lid,
        })
    print(f"page {p}: {js('location.href')} rows so far={len(rows)}")
    time.sleep(4)   # polite pacing between pages
 
print(json.dumps(rows[:2], indent=1))
print([(r["rank"], r["name"], r["price_from"]) for r in rows[-3:]])
print("ranks contiguous:", [r["rank"] for r in rows] == list(range(1, len(rows) + 1)))
print("rows joined to JSON-LD:", sum(1 for r in rows if r["address"]))
print("rows with a price:", sum(1 for r in rows if r["price_from"]))

import json, re, time
 
URLS = [
    "https://www.tripadvisor.com/Hotel_Review-g60745-d15618152-Reviews-Cambria_Hotel_Boston_Downtown-Boston_Massachusetts.html",
    "https://www.tripadvisor.com/Restaurant_Review-g60745-d19799010-Reviews-Cafe_Bonjour-Boston_Massachusetts.html",
]
TYPES = {"LodgingBusiness", "Hotel", "FoodEstablishment", "Restaurant"}
 
def detail(url, first=False):
    new_tab(url) if first else goto_url(url)
    time.sleep(2)
    wait_for_load(30)
    time.sleep(2)
    j = {}
    for s in js("[...document.querySelectorAll('script[type=\"application/ld+json\"]')].map(s => s.textContent)") or []:
        try:
            d = json.loads(s)
        except ValueError:
            continue
        if isinstance(d, dict) and d.get("@type") in TYPES:
            j = d
            break
    text = js("document.body.innerText") or ""
    rank = re.search(r"#\d[\d,]* of [\d,]+ [^\n]+", text)
    stars = re.search(r"(\d(?:\.\d)?) of 5 stars", text)
    agg = j.get("aggregateRating") or {}
    addr = j.get("address") or {}
    cuisine = j.get("servesCuisine")
    return {
        "type": j.get("@type"),
        "name": j.get("name"),
        "rating": float(agg["ratingValue"]) if agg.get("ratingValue") else None,
        "review_count": agg.get("reviewCount"),
        "popularity_rank": rank.group(0) if rank else None,
        "price_level": j.get("priceRange"),
        "hotel_class": float(stars.group(1)) if stars and j.get("@type") in ("LodgingBusiness", "Hotel") else None,
        "cuisine": ", ".join(cuisine) if isinstance(cuisine, list) else cuisine,
        "address": ", ".join(filter(None, [addr.get("streetAddress"), addr.get("addressLocality"), addr.get("postalCode")])),
        "telephone": j.get("telephone"),
        "geo": [j["geo"]["latitude"], j["geo"]["longitude"]] if j.get("geo") else None,
        "amenities": [a["name"] for a in j.get("amenityFeatures") or [] if a.get("value")][:10],
        "opening_hours": [(h.get("dayOfWeek"), h.get("opens"), h.get("closes")) for h in j.get("openingHoursSpecification") or []],
        "url": j.get("url") or url,
        "location_id": (re.search(r"-d(\d+)-", url) or [None, None])[1],
    }
 
for i, u in enumerate(URLS):
    print(json.dumps(detail(u, first=(i == 0)), indent=1))
    time.sleep(4)