Google Maps - Scraping places from a search

import gzip, html, json, os, re, tempfile, time, urllib.error, urllib.parse, urllib.request
 
HEADERS = {
    "User-Agent": "browser-harness-skill-example",   # a truthful client UA was enough; no browser UA needed
    "Accept-Language": "en-US,en;q=0.9",
    "Accept-Encoding": "gzip",
}
 
def get(url, tries=3):
    for i in range(tries):                            # retry transient errors with backoff, then raise
        try:
            with urllib.request.urlopen(urllib.request.Request(url, headers=HEADERS), timeout=30) as r:
                data = r.read()
                if r.headers.get("Content-Encoding") == "gzip":
                    data = gzip.decompress(data)
                return data.decode("utf-8")
        except (urllib.error.URLError, ConnectionError, TimeoutError) as e:   # ConnectionError covers resets mid-read
            if getattr(e, "code", None) in (403, 429):   # blocked or rate-limited: stop, never retry
                raise RuntimeError(f"blocked: HTTP {e.code} on {url}; stop and back off") from e
            if i == tries - 1:
                raise
            time.sleep(2 ** (i + 1))
 
def at(o, *path):
    """Safe nested index: at(p, 4, 7) -> p[4][7] or None."""
    for i in path:
        if not isinstance(o, list) or i >= len(o):
            return None
        o = o[i]
    return o
 
def parse_place(p):
    hours = {}
    for day in at(p, 203, 0) or []:
        slots = [s[0] for s in (at(day, 3) or []) if s and s[0]]
        if at(day, 0):
            hours[day[0]] = ", ".join(slots).replace(" ", " ").replace(" ", " ")
    pid = at(p, 78)
    web = at(p, 7, 0)
    if web and web.startswith("/url?"):               # non-browser UAs get a /url?q=<site> redirect: unwrap it
        web = urllib.parse.parse_qs(urllib.parse.urlsplit(web).query).get("q", [web])[0]
    ftid = at(p, 10) or ""
    cid = str(int(ftid.split(":")[1], 16)) if ":" in ftid else None
    return {
        "name": at(p, 11),
        "rating": at(p, 4, 7),
        "reviews": at(p, 4, 8),
        "category": at(p, 13, 0),
        "categories": at(p, 13) or [],
        "address": at(p, 39),
        "neighborhood": at(p, 14),
        "phone": at(p, 178, 0, 0),
        "phone_e164": at(p, 178, 0, 3),
        "website": web,
        "lat": at(p, 9, 2),
        "lng": at(p, 9, 3),
        "place_id": pid,
        "cid": cid,
        "maps_url": f"https://www.google.com/maps/place/?q=place_id:{pid}" if pid else None,
        "open_now": (at(p, 203, 1, 4, 0) or "").replace(" ", " ") or None,
        "hours": hours,
    }
 
def search_places(query, max_results=200, pause=2.0):
    q = urllib.parse.quote_plus(query)
    page = get(f"https://www.google.com/maps/search/{q}?hl=en&gl=us")
    m = re.search(r'<link href="(/search\?tbm=map[^"]+)"', page)
    if not m:
        raise RuntimeError("no /search?tbm=map preload link (consent page or layout change?)")
    base = "https://www.google.com" + html.unescape(m.group(1))
    # page-size token: served percent-encoded (%217i20) on 2026-10-23; accept a literal !7i20 too
    size = next((t for t in ("%217i20", "!7i20") if t in base), None)
    if not size:
        raise RuntimeError("no !7i20 page-size token in the preload URL; cannot page")
    bang = "%21" if size.startswith("%") else "!"
    places, offset = {}, 0
    while len(places) < max_results:
        url = base if offset == 0 else base.replace(size, f"{size}{bang}8i{offset}", 1)
        raw = get(url)
        d = json.loads(raw[raw.index("\n") + 1:])          # strip the )]}' guard line
        rows = at(d, 64) or []
        if not rows:
            break                                         # past the last page
        before = len(places)
        for r in rows:
            p = at(r, 1)
            if not p or not at(p, 11):                    # defensive: skip any row without a named place
                continue
            place = parse_place(p)
            places.setdefault(place["place_id"] or place["name"], place)
        if len(places) == before:                         # paging token ignored: fail instead of looping on page 1
            raise RuntimeError(f"offset {offset} added no new places")
        offset += 20
        time.sleep(pause)
    return list(places.values())[:max_results]
 
places = search_places("coffee shops in Austin TX")
print(len(places), "places")
print(json.dumps(places[:2], indent=1, ensure_ascii=False))
print("with phone:", sum(1 for p in places if p["phone"]),
      "with website:", sum(1 for p in places if p["website"]),
      "with hours:", sum(1 for p in places if p["hours"]))
out = os.path.join(tempfile.gettempdir(), "google_maps_places.json")   # temp dir, not the cwd
with open(out, "w", encoding="utf-8") as f:
    json.dump(places, f, indent=1, ensure_ascii=False)
print("saved", out)

175 places
[
 {
  "name": "Jo's Coffee – South Congress",
  "rating": 4.4,
  "reviews": 2004,
  "category": "Coffee shop",
  "categories": ["Coffee shop", "Cafe", "Takeout Restaurant", "Sandwich shop"],
  "address": "1300 S Congress Ave, Austin, TX 78704",
  "neighborhood": "Bouldin Creek",
  "phone": "(512) 852-2300",
  "phone_e164": "+15128522300",
  "website": "https://www.joscoffee.com/south-congress-jos",
  "lat": 30.2510458,
  "lng": -97.7493717,
  "place_id": "ChIJ1S_Bov20RIYRe7MiR8tYawY",
  "cid": "462561016830669691",
  "maps_url": "https://www.google.com/maps/place/?q=place_id:ChIJ1S_Bov20RIYRe7MiR8tYawY",
  "open_now": "Open · Closes 7 PM",
  "hours": {"Friday": "7 AM–7 PM", "Saturday": "7 AM–7 PM", "Sunday": "7 AM–7 PM", ...}
 },
 { "name": "Epoch Coffee", ... }
]
with phone: 145 with website: 167 with hours: 168
saved <temp dir>/google_maps_places.json

import json, re, time, urllib.parse
 
query = "coffee shops in Austin TX"
# Opt-in: the feed only lazy-loads in a visible tab (a hidden tab stopped at 6-12 cards).
# True brings this tab to the front of the browser. Leave False on someone's own Chrome
# unless they agreed to it; method 1 needs no tab at all.
FOREGROUND = False
tid = new_tab()            # blank tab first
if FOREGROUND:
    activate_tab(tid)      # make it visible BEFORE loading the search
goto_url("https://www.google.com/maps/search/" + urllib.parse.quote_plus(query) + "?hl=en")
wait_for_load()
wait(3)
 
def try_js(expr):
    try:
        return js(expr)
    except RuntimeError:     # js() raised "Runtime.evaluate timed out" once while a batch of cards rendered
        return None
 
COUNT = "document.querySelectorAll('div[role=feed] a[href*=\"/maps/place/\"]').length"
last, stalls, stop = 0, 0, "scroll cap (80) reached"
for _ in range(80):
    try_js("(()=>{const f=document.querySelector('div[role=feed]'); if(f) f.scrollTop=f.scrollHeight})()")
    time.sleep(2)
    n = try_js(COUNT) or last
    if try_js("(document.querySelector('div[role=feed]')||{innerText:''}).innerText.includes('end of the list')"):
        stop = "end of the list"
        break
    stalls = stalls + 1 if n == last else 0
    if stalls >= 8:          # ~16 s with no new cards: the feed stopped loading
        stop = "feed stalled, PARTIAL list" + ("" if FOREGROUND else " (hidden tab: set FOREGROUND = True)")
        break
    last = n
print("stopped:", stop)
 
cards = js(r"""[...document.querySelectorAll('div[role=feed] a[href*="/maps/place/"]')].map(a => {
  const card = a.parentElement;
  const star = card.querySelector('[role=img][aria-label*="stars"]');
  return {name: a.getAttribute('aria-label'), href: a.href,
          stars: star ? star.getAttribute('aria-label') : null,
          lines: card.innerText.split('\n').map(s => s.trim()).filter(Boolean)};
})""") or []
 
places = {}                  # the feed repeats some places: key by place_id, fall back to the URL
for c in cards:
    m = re.match(r"([\d.]+) stars?\s*([\d,]*)", c["stars"] or "")
    info = next((l for l in c["lines"] if " · " in l and not l.startswith(("Open", "Closed"))), "")
    parts = [s.strip() for s in info.split("·") if s.strip() and not s.strip().startswith("")]
    status = next((l for l in c["lines"] if l.startswith(("Open", "Closed", "Temporarily", "Permanently"))), None)
    pid = re.search(r"!19s(ChIJ[\w-]+)", c["href"])
    ll = re.search(r"!3d(-?[\d.]+)!4d(-?[\d.]+)", c["href"])
    maps_url = c["href"].split("?")[0]
    places.setdefault(pid.group(1) if pid else maps_url, {
        "name": c["name"],
        "rating": float(m.group(1)) if m else None,
        "reviews": int(m.group(2).replace(",", "")) if m and m.group(2) else None,
        "category": parts[0] if parts else None,
        "address": parts[-1] if len(parts) > 1 else None,
        "status": status.replace(" ", " ") if status else None,
        "place_id": pid.group(1) if pid else None,
        "lat": float(ll.group(1)) if ll else None,
        "lng": float(ll.group(2)) if ll else None,
        "maps_url": maps_url,
    })
places = list(places.values())
print(len(cards), "cards ->", len(places), "unique places")
print(json.dumps(places[:2], indent=1, ensure_ascii=False))

stopped: feed stalled, PARTIAL list (hidden tab: set FOREGROUND = True)
6 cards -> 6 unique places
[ ... same card fields as below ... ]

stopped: end of the list
120 cards -> 116 unique places
[
 {
  "name": "Jo's Coffee – South Congress",
  "rating": 4.4,
  "reviews": 2004,
  "category": "Coffee shop",
  "address": "1300 S Congress Ave",
  "status": "Open · Closes 7 PM",
  "place_id": "ChIJ1S_Bov20RIYRe7MiR8tYawY",
  "lat": 30.2510458,
  "lng": -97.7493717,
  "maps_url": "https://www.google.com/maps/place/Jo%27s+Coffee+%E2%80%93+South+Congress/data=!4m7!3m6!1s0x8644b4fda2c12fd5:0x66b58cb4722b37b!8m2!3d30.2510458!4d-97.7493717!16s%2Fm%2F020z01l!19sChIJ1S_Bov20RIYRe7MiR8tYawY"
 },
 ...
]

import json
 
def place_details(place_id):
    goto_url(f"https://www.google.com/maps/place/?q=place_id:{place_id}&hl=en")
    wait_for_load()
    wait(3)
    return js(r"""(() => {
      const item = sel => document.querySelector(`[data-item-id${sel}]`);
      const label = (el, prefix) => el ? (el.getAttribute('aria-label') || '').replace(prefix, '').trim() : null;
      const phone = item('^="phone:tel:"');
      const site = item('="authority"');
      return {
        name: (document.querySelector('h1') || {}).innerText || null,
        address: label(item('="address"'), 'Address:'),
        phone: label(phone, 'Phone:'),
        website: site ? site.href : null,
        hours: [...document.querySelectorAll('[aria-label$="Copy open hours"]')]
                 .map(e => e.getAttribute('aria-label').replace(', Copy open hours', '').replace(/ /g, ' ')),
      };
    })()""")
 
tid = new_tab()
for pid in ["ChIJ1S_Bov20RIYRe7MiR8tYawY", "ChIJP8p2kJC1RIYR2qrGoXZtZCk"]:
    print(json.dumps(place_details(pid), indent=1, ensure_ascii=False))
    wait(2)

{
 "name": "Jo's Coffee – South Congress",
 "address": "1300 S Congress Ave, Austin, TX 78704",
 "phone": "(512) 852-2300",
 "website": "https://www.joscoffee.com/south-congress-jos",
 "hours": ["Friday, 7 AM to 7 PM", "Saturday, 7 AM to 7 PM", ...]
}
{
 "name": "Terrible Love Coffee",
 "address": "3908 Avenue B, Austin, TX 78751",
 "phone": null,
 "website": "http://terriblelovecoffee.com/",
 "hours": ["Friday, 7:30 AM to 2 PM", ...]
}