Agoda - Hotel search results and hotel pages

import json, re, time, urllib.error, urllib.request
 
UA = "agoda-city-id-lookup/1.0 (python-urllib)"   # honest client name, no browser spoofing
 
def city_id(slug):
    """slug like 'bangkok-th' or 'tokyo-jp' (the /city/<slug>.html landing page)."""
    url = f"https://www.agoda.com/city/{slug}.html"
    deadline, delay = time.time() + 120, 5        # one deadline for every attempt and backoff
    while True:
        try:
            req = urllib.request.Request(url, headers={"User-Agent": UA, "Accept-Language": "en-US"})
            with urllib.request.urlopen(req, timeout=max(1, min(30, deadline - time.time()))) as r:
                if r.geturl() != url:
                    raise ValueError(f"{url} redirected to {r.geturl()} - unknown city slug?")
                html = r.read().decode("utf-8", "replace")
            break
        except urllib.error.HTTPError as e:      # 403/429 = blocked or rate limited: stop, do not retry
            if e.code not in (500, 502, 503, 504) or time.time() + delay + 1 > deadline:
                raise
            time.sleep(delay); delay *= 2
    m = re.search(r'"defaultSearchURL":"/search\?city=(\d+)', html)
    if not m:
        raise ValueError(f"no city id on {url}")
    return int(m.group(1))
 
for slug in ["bangkok-th", "tokyo-jp"]:
    print(slug, city_id(slug))
    time.sleep(2)

import datetime, json, re, time, urllib.parse
 
CITY_ID = 9395       # Bangkok; get others with city_id("tokyo-jp") from the block above
PAGES = 3            # page cap; each results page holds ~100 properties
FOREGROUND = False   # opt-in: True may bring this tab to the front (visible to whoever uses that Chrome)
check_in = (datetime.date.today() + datetime.timedelta(days=30)).isoformat()
params = {"city": CITY_ID, "checkIn": check_in, "los": 2, "rooms": 1, "adults": 2,
          "children": 0, "locale": "en-us", "currency": "USD"}
SEARCH_URL = "https://www.agoda.com/search?" + urllib.parse.urlencode(params)
 
CARDS = r"""
JSON.stringify([...document.querySelectorAll('li[data-selenium="hotel-item"]')].map(li => {
  const q = s => li.querySelector(s);
  const id = li.getAttribute('data-hotelid');
  const a = q('a[data-selenium="hotel-name"]');
  const name = a ? a.innerText.trim() : '', href = a ? a.getAttribute('href') : null;
  const price = q('[data-selenium="display-price"]');
  const sold_out = /Sold out on your dates/i.test(li.innerText);
  // virtualized: a card counts only once id, name, url and (price or sold-out note) are all there
  if (!id || !name || !href || (!price && !sold_out)) return {id, empty: true};
  const star = q('[data-testid="rating-container"]');
  const sm = star && star.innerText.match(/([\d.]+) stars? out of 5/);
  // two layouts seen: data-element-name + aria-label, or data-testid + screen-reader text
  const rev = q('[data-element-name="property-card-review"],[data-testid="property-card-review"]');
  const rm = rev && ((rev.getAttribute('aria-label') || '') + ' ' + rev.innerText)
    .match(/([\d.]+) out of 10 with ([\d,]+) review/);
  // two href forms seen: /<slug>/hotel/<city>.html?... and /accom/property?propertyId=<id>&...
  const u = new URL(href, location.origin);
  const pid = u.searchParams.get('propertyId');
  return {
    hotel_id: id,
    name,
    stars: sm ? parseFloat(sm[1]) : null,
    review_score: rm ? parseFloat(rm[1]) : null,
    review_count: rm ? parseInt(rm[2].replace(/,/g, '')) : null,
    price: price ? parseFloat(price.innerText.replace(/,/g, '')) : null,
    currency: q('[data-selenium="hotel-currency"]')?.innerText || null,
    sold_out,
    promoted: /\bPromoted\b/.test(li.innerText),
    area: q('[data-selenium="area-city-text"]')?.innerText || null,
    url: u.pathname === '/accom/property' && pid ? u.origin + u.pathname + '?propertyId=' + pid
                                                 : u.origin + u.pathname,
  };
}))
"""
 
# Generic block-page markers. No challenge was seen on Agoda in these runs, so these are not Agoda-specific.
CHALLENGE = r"""
(() => {
  const text = (document.title + ' ' + (document.body ? document.body.innerText : '')).slice(0, 20000);
  return /captcha|are you a robot|verify (that )?you are (a )?human|unusual traffic|access denied|request blocked|pardon our interruption/i.test(text)
    || !!document.querySelector('iframe[src*="captcha"], iframe[src*="challenge"], #px-captcha, .g-recaptcha, .h-captcha');
})()
"""
LABEL = "document.querySelector('[data-selenium=\"pagination-text\"]')?.innerText || ''"
 
def try_js(expr, default=None):
    """Agoda pages can block the main thread for several seconds; evaluate then times out.
    No sleep here: every caller is a deadline loop that already sleeps between tries."""
    try:
        return js(expr)
    except RuntimeError:
        return default
 
def check_challenge():
    if try_js(CHALLENGE):
        raise RuntimeError(f"challenge or block page at {try_js('location.href')}: stop here, do not retry")
 
def on_requested_url(href, url):
    """Agoda appends ds=... and sort=...; every parameter we asked for must still be there unchanged."""
    want, got = urllib.parse.urlsplit(url), urllib.parse.urlsplit(href or "")
    got_q = dict(urllib.parse.parse_qsl(got.query))
    return (got.netloc, got.path) == (want.netloc, want.path) and \
        all(got_q.get(k) == v for k, v in urllib.parse.parse_qsl(want.query))
 
def page_numbers(label):
    m = re.fullmatch(r"Page (\d+) of (\d+)", (label or "").strip())
    return (int(m[1]), int(m[2])) if m else None
 
def wait_for_cards(timeout=60):
    deadline, href = time.time() + timeout, None
    while time.time() < deadline:
        check_challenge()
        href = try_js("location.href") or href
        if on_requested_url(href, SEARCH_URL) and \
                try_js("document.querySelector('li[data-selenium=hotel-item] a[data-selenium=hotel-name]') !== null"):
            return
        time.sleep(1)
    text = try_js("document.body.innerText.replace(/\\s+/g, ' ').trim().slice(0, 120)")
    raise RuntimeError(f"no hotel cards for the requested search after {timeout}s; at {href}; page text {text!r}")
 
def harvest_page(timeout=120):
    """Cards are virtualized: only ones near the viewport have content. Scroll and merge by id.
    If ids seen earlier are no longer in the DOM, the list was replaced under us: start the page over.
    One deadline covers every attempt; a challenge or the deadline raises, never a partial page."""
    deadline, attempt = time.time() + timeout, 1
    while True:
        check_challenge()
        rows, missing, gone = harvest_once(deadline)
        if not missing:
            return rows
        print(f"  {len(missing)} cards missing ({len(gone)} no longer in the DOM), attempt {attempt}")
        if not gone or time.time() + 3 >= deadline:   # no sleep after the last attempt
            raise RuntimeError(f"{len(missing)} cards never rendered: {missing[:5]}, "
                               f"visibility={try_js('document.visibilityState')}")
        time.sleep(3); attempt += 1
 
def harvest_once(deadline):
    rows, order = {}, []
    def collect():
        cards = json.loads(js(CARDS))
        if not cards:
            check_challenge()
            raise RuntimeError(f"no result cards on {try_js('location.href')}")
        for r in cards:
            hid = r.get("hotel_id") or r.get("id")
            if not hid:
                raise RuntimeError("result card without data-hotelid: markup changed?")
            if hid not in rows:
                order.append(hid); rows[hid] = None
            if not r.get("empty"):
                rows[hid] = r
    js("window.scrollTo(0, 0)"); time.sleep(1)
    bottom_hits = 0
    while bottom_hits < 3:                       # 3 checks at the bottom = lazy loading finished
        if time.time() > deadline:
            raise RuntimeError(f"page harvest passed its deadline at {len(order)} cards")
        collect()
        at_bottom = js("scrollY + innerHeight >= document.documentElement.scrollHeight - 5")
        bottom_hits = bottom_hits + 1 if at_bottom else 0
        js("window.scrollBy(0, 600)"); time.sleep(0.8)
    for hid in [h for h in order if rows[h] is None]:   # skipped or slow cards: visit each, wait up to 10 s
        js(f"document.querySelector('li[data-hotelid=\"{hid}\"]')?.scrollIntoView({{block: 'center'}})")
        card_deadline = min(deadline, time.time() + 10)
        while rows[hid] is None and time.time() < card_deadline:
            time.sleep(1); collect()
    missing = [h for h in order if rows[h] is None]
    gone = [h for h in missing if not js(f"!!document.querySelector('li[data-hotelid=\"{h}\"]')")]
    return [rows[h] for h in order if rows[h] is not None], missing, gone
 
def next_page(cur):
    """Click Next and wait until the label reads page cur + 1. Raises if there is no Next button."""
    js("document.querySelector('[data-selenium=\"pagination-next-btn\"]')?.scrollIntoView({block: 'center'})")
    time.sleep(1)
    box = js("(() => { const b = document.querySelector('[data-selenium=\"pagination-next-btn\"]'); "
             "if (!b) return null; const r = b.getBoundingClientRect(); return [r.x + r.width/2, r.y + r.height/2]; })()")
    if not box:
        raise RuntimeError(f"no Next button on page {cur} (label {try_js(LABEL)!r})")
    click_at_xy(*box)
    deadline = time.time() + 60
    while time.time() < deadline:
        time.sleep(1)
        check_challenge()
        nums = page_numbers(try_js(LABEL))
        if nums and nums[0] == cur + 1:
            wait_for_cards(); return nums
    raise RuntimeError(f"page did not advance from {cur} (label {try_js(LABEL)!r})")
 
new_tab(SEARCH_URL)
wait_for_cards()
if js("document.visibilityState") != "visible":   # hidden tabs never render the lazy cards
    if not FOREGROUND:
        raise RuntimeError("tab is hidden, so cards will not render: use a dedicated browser "
                           "or set FOREGROUND = True to bring this tab to the front")
    activate_tab(current_tab())
print("search:", js("location.href"))
nums = page_numbers(try_js(LABEL))
if not nums:
    raise RuntimeError(f"unexpected pagination label {try_js(LABEL)!r}")
cur, total = nums
results, seen, dupes = [], set(), 0
while True:
    rows = harvest_page()
    print(f"Page {cur} of {total}: {len(rows)} properties")
    for r in rows:                                # the same hotel can show on two pages; keep the first
        if r["hotel_id"] in seen:
            dupes += 1; continue
        seen.add(r["hotel_id"]); results.append(dict(r, page=cur))
    if cur >= total or cur >= PAGES:
        break
    time.sleep(3)                                 # polite pacing between pages
    cur, total = next_page(cur)
 
status = {"complete": cur >= total, "pages_read": cur, "pages_total": total,
          "search_url": SEARCH_URL, "params": params}
print("unique", len(results), "duplicates dropped", dupes)
for r in results[:3]:
    print(r)
print("sold_out", sum(r["sold_out"] for r in results), "promoted", sum(r["promoted"] for r in results),
      "no price", sum(r["price"] is None for r in results),
      "no stars", sum(r["stars"] is None for r in results),
      "no review score", sum(r["review_score"] is None for r in results))
if not status["complete"]:
    print(f"PARTIAL: stopped at the PAGES = {PAGES} cap on page {cur} of {total}; more results exist")
json.dump(dict(status, results=results), open("agoda_results.json", "w"), indent=1, ensure_ascii=False)

import datetime, json, re, time, urllib.parse
 
HOTEL_URLS = [   # any `url` values from the search rows
    "https://www.agoda.com/bless-residence/hotel/bangkok-th.html",
    "https://www.agoda.com/accom/property?propertyId=14654101",   # the other href form
    "https://www.agoda.com/accom/property?propertyId=14651963",   # had no rate for 2026-11-23 when tested; other dates may have one
]
FOREGROUND = False   # opt-in: True may bring this tab to the front (visible to whoever uses that Chrome)
check_in = (datetime.date.today() + datetime.timedelta(days=30)).isoformat()
STAY = {"checkIn": check_in, "los": 2, "rooms": 1, "adults": 2, "currencyCode": "USD"}
 
HOTEL = r"""
(() => {
  const t = s => document.querySelector(s)?.innerText.trim() || null;
  const ld = [...document.querySelectorAll('script[type="application/ld+json"]')]
    .map(s => { try { return JSON.parse(s.textContent); } catch (e) { return null; } })
    .find(d => d && d['@type'] === 'Hotel') || {};
  const star = (t('[data-selenium="mosaic-hotel-rating"]') || '').match(/([\d.]+) stars? out of 5/);
  const rev = (t('[data-selenium="review-score-with-test"],[data-testid="review-score-with-test"],[data-element-name="review-score-with-test"]') || '').match(/([\d.]+)[\s\S]*?([\d,]+) reviews/);
  const from = (t('[data-element-name="cheapest-room-price-property-nav-bar"]') || '').match(/from ([A-Z]{3}) ([\d,.]+)/);
  // no rate for these dates: no "from" bar, and the room grid's own h2 reads "Sold out!" or
  // "Unavailable on Agoda" (both seen). Other "sold out" text on the page (one room type) does not count.
  const grid = document.querySelector('#property-room-grid-root');
  const sold_out = !from && !!grid && [...grid.querySelectorAll('h2')]
    .some(h => /^(Sold out!?|Unavailable on Agoda)$/i.test(h.innerText.trim()));
  const pid = new URLSearchParams(location.search).get('propertyId');
  const geo = (ld.hasMap || '').match(/center=([-\d.]+)%2c([-\d.]+)/i);
  // free text: drop email addresses and phone-like numbers (9+ digits) before printing
  const redact = s => s.replace(/[\w.+-]+@[\w-]+\.[\w.-]+/g, '[email]')
    .replace(/\+?\d[\d ().-]{7,}\d/g, m => m.replace(/\D/g, '').length >= 9 ? '[phone]' : m);
  return JSON.stringify({
    jsonld_loaded: ld['@type'] === 'Hotel',   // the script tag exists empty at first, filled ~2 s later
    name: t('h1[data-selenium="hotel-header-name"]') || ld.name || null,
    // on /accom/property?propertyId=... pages JSON-LD url is the bare "/accom/property"
    url: ld.url && !ld.url.endsWith('/accom/property') ? ld.url
         : location.origin + location.pathname + (pid ? '?propertyId=' + pid : ''),
    stars: star ? parseFloat(star[1]) : null,
    address: t('[data-selenium="hotel-address-map"]'),
    postal: ld.address || null,
    lat: geo ? parseFloat(geo[1]) : null,
    lng: geo ? parseFloat(geo[2]) : null,
    review_score: rev ? parseFloat(rev[1]) : (ld.aggregateRating?.ratingValue ?? null),
    review_count: rev ? parseInt(rev[2].replace(/,/g, '')) : null,
    jsonld_rating: ld.aggregateRating || null,
    from_price: from ? {currency: from[1], amount: parseFloat(from[2].replace(/,/g, ''))} : null,
    sold_out,
    top_amenities: [...document.querySelectorAll('[data-element-name="atf-top-amenities-item"]')].map(e => e.innerText.trim()),
    image: ld.image ? new URL(ld.image, location.origin).href : null,
    description: redact(ld.description || '').slice(0, 300),
  });
})()
"""
 
# Generic block-page markers. No challenge was seen on Agoda in these runs, so these are not Agoda-specific.
CHALLENGE = r"""
(() => {
  const text = (document.title + ' ' + (document.body ? document.body.innerText : '')).slice(0, 20000);
  return /captcha|are you a robot|verify (that )?you are (a )?human|unusual traffic|access denied|request blocked|pardon our interruption/i.test(text)
    || !!document.querySelector('iframe[src*="captcha"], iframe[src*="challenge"], #px-captcha, .g-recaptcha, .h-captcha');
})()
"""
 
def try_js(expr, default=None):
    """The hotel page can block its main thread for 5-15 s while it renders; evaluate then times out.
    No sleep here: the loops below sleep between tries, and the error path must not wait."""
    try:
        return js(expr)
    except RuntimeError:
        return default
 
def check_challenge():
    if try_js(CHALLENGE):
        raise RuntimeError(f"challenge or block page at {try_js('location.href')}: stop here, do not retry")
 
def on_requested_url(href, url):
    """Agoda appends ds=...; every parameter we asked for must still be there unchanged."""
    want, got = urllib.parse.urlsplit(url), urllib.parse.urlsplit(href or "")
    got_q = dict(urllib.parse.parse_qsl(got.query))
    return (got.netloc, got.path) == (want.netloc, want.path) and \
        all(got_q.get(k) == v for k, v in urllib.parse.parse_qsl(want.query))
 
def open_hotel(url, timeout=90):
    u = urllib.parse.urlparse(url)
    query = dict(urllib.parse.parse_qsl(u.query), **STAY)
    target = u._replace(query=urllib.parse.urlencode(query)).geturl()
    if current_tab()["url"].startswith("https://www.agoda.com/"):
        goto_url(target)                          # reuse this skill's own Agoda tab
    else:
        new_tab(target)                           # never navigate a tab that is on another site
    href, deadline = None, time.time() + timeout
    while time.time() < deadline:
        check_challenge()
        href = try_js("location.href") or href
        if on_requested_url(href, target) and \
                try_js("document.querySelector('h1[data-selenium=hotel-header-name]') !== null"):
            break
        time.sleep(1)
    else:
        raise RuntimeError(f"hotel page {target} not rendered after {timeout}s; last location {href}")
    if js("document.visibilityState") != "visible":
        if not FOREGROUND:
            raise RuntimeError("tab is hidden: use a dedicated browser or set FOREGROUND = True")
        activate_tab(current_tab())
 
def read_hotel(timeout=60):
    """Accept the page once name, url and JSON-LD are there and the price state is known:
    a "from" price, or the room grid's sold-out heading (then from_price is None and sold_out is True)."""
    d, deadline = None, time.time() + timeout
    while time.time() < deadline:
        check_challenge()
        raw = try_js(HOTEL)
        d = json.loads(raw) if raw else d
        if d and d["name"] and d["url"] and d["jsonld_loaded"] and (d["from_price"] or d["sold_out"]):
            return d
        time.sleep(2)
    raise RuntimeError(f"incomplete hotel page at {try_js('location.href')}: {d} "
                       "(no 'from' price and no sold-out heading in the room grid)")
 
for i, u in enumerate(HOTEL_URLS):
    if i:
        time.sleep(3)   # polite pacing between hotel pages
    open_hotel(u)
    print(json.dumps(read_hotel(), indent=1, ensure_ascii=False))