Redfin - For-sale listings by city or ZIP, plus listing details

import json, time, csv, os, re, tempfile
 
SEARCH = "78704"     # a ZIP, or a Redfin search page path such as "/city/30818/TX/Austin"
MAX_PAGES = 200      # safety cap on list pages (raises if hit); Austin needed 150
PAUSE = 3.0          # seconds between page loads
MAX_SHORT = 0.01     # raise if more than this share of the header count is missing (a live count drifts a little)
BASE = "https://www.redfin.com"
 
def search_path(s):
    s = s.strip()
    if re.fullmatch(r"\d{5}", s):
        return f"/zipcode/{s}"
    s = re.sub(r"/page-\d+$", "", s.replace(BASE, "").split("?")[0].rstrip("/"))
    if not re.match(r"^/(city|zipcode|neighborhood|county)/", s):
        raise SystemExit(f"SEARCH must be a ZIP or a Redfin /city/, /zipcode/, /neighborhood/ or /county/ path, got {s!r}")
    return s
 
BLOCKED = re.compile(r"request could not be satisfied|ratelimited|access denied|are you a human|captcha", re.I)
CARDS = '[data-rf-test-name="basicNode-homeCard"]'
ZERO = ".ZeroResultsText, .nearbyHomesTitle"   # "We didn't find any exact matches" / "Homes within 1 mile of ..."
POLL_JS = ("JSON.stringify({old: !!window.__bhOld, u: location.href, t: document.title, r: document.readyState, "
           "n: document.querySelectorAll('%s').length, z: !!document.querySelector('%s')})" % (CARDS.replace("'", "\\'"), ZERO))
 
def open_page(url, budget=120, per_try=30):
    """Load url in the tab and wait for the new document. Returns the URL it landed on: url itself once result
    cards (or Redfin's no-exact-matches view) have rendered, or the redirect target, which the caller must check.
    A block or challenge page raises at once (stop and back off). A page that never renders is retried after
    10 s, then 20 s, all inside one budget-second deadline; then it raises."""
    end, pause, attempt = time.time() + budget, 10, 0
    while True:
        attempt += 1
        before = js("window.__bhOld = 1, location.href")   # tag the old document; the new one will not carry the tag
        try:
            goto_url(url)
        except Exception as e:       # Page.navigate often outlives the 5 s daemon wait on these heavy pages
            print("navigate still running:", str(e)[:80])
        stop, n, z, last = min(end, time.time() + per_try), -1, False, None
        while time.time() + 1 < stop:   # the last poll starts before the deadline
            time.sleep(1)
            try:
                st = last = json.loads(js(POLL_JS))
            except Exception:
                continue                 # page still busy; poll again
            if st["old"]:
                continue                 # still the document from before the navigation
            if BLOCKED.search(st["t"]) or "ratelimited" in st["u"]:
                raise RuntimeError(f"Redfin served a block/challenge page ({st['t']!r} at {st['u']}). Stop and back off.")
            if st["r"] != "complete":
                continue
            if st["u"].rstrip("/") != url.rstrip("/"):
                return st["u"]           # a new document at another URL: redirected
            if (st["n"] > 0 and st["n"] == n) or (st["z"] and z):
                return st["u"]           # card count (or the zero-results view) stable across two polls
            n, z = st["n"], st["z"]
        print(f"try {attempt}: {url} not ready (navigated from {before}; last state {last})")
        if time.time() + pause + 15 > end:
            break                        # no time left for a backoff and another try; no sleep after the last one
        time.sleep(pause)
        pause *= 2
    raise RuntimeError(f"{url} did not render within the {budget} s budget ({attempt} tries)")
 
def header_total():
    """'323 homes + 9 early access' -> 332."""
    t = js("(document.querySelector('.homes.summary, [data-rf-test-id=\"homes-description\"]') || {}).innerText || ''")
    nums = [int(x.replace(",", "")) for x in re.findall(r"([\d,]+) (?:homes?|early access)", t)]
    if not nums:
        if js("!!document.querySelector('%s')" % ZERO):
            return 0, "no exact matches"
        raise RuntimeError(f"could not read the result count from the page header: {t!r}")
    return sum(nums), t
 
# The search page HTML embeds the server-rendered map data (up to 350 homes) in __reactServerState.
GIS_JS = r"""(() => {
  const s = [...document.scripts].find(s => !s.src && s.textContent.includes('__reactServerState.InitialContext = '));
  if (!s) return null;
  const p = 'root.__reactServerState.InitialContext = ';
  const line = s.textContent.split('\n').find(l => l.startsWith(p));
  const dc = JSON.parse(line.slice(p.length, -1))['ReactServerAgent.cache'].dataCache;
  const k = Object.keys(dc).find(k => k.startsWith('/stingray/api/gis?'));
  return k ? dc[k].res.text : null;
})()"""
 
def embedded_homes():
    t = js(GIS_JS)
    if not t or not t.startswith("{}&&"):
        raise RuntimeError("no embedded map data on the search page: " + str(t)[:120])
    d = json.loads(t[4:])                # strip the "{}&&" prefix
    if d.get("resultCode") != 0 or not isinstance((d.get("payload") or {}).get("homes"), list):
        raise RuntimeError("unexpected embedded map data: " + t[:160])
    return {h["propertyId"]: h for h in d["payload"]["homes"]}
 
CARD_JS = r"""JSON.stringify([...document.querySelectorAll('[data-rf-test-name="basicNode-homeCard"]')].map(c => {
  const t = s => { const e = c.querySelector(s); return e ? e.textContent.trim() : null; };
  const a = c.querySelector('a.bp-Homecard__Address');
  let ld = null; const s = c.querySelector('script[type="application/ld+json"]');
  if (s) { try { ld = JSON.parse(s.textContent); } catch (e) {} }
  return {href: a && a.getAttribute('href'), address: a && a.textContent.trim(), price: t('.bp-Homecard__Price--value'),
    beds: t('.bp-Homecard__Stats--beds'), baths: t('.bp-Homecard__Stats--baths'), sqft: t('.bp-Homecard__Stats--sqft'),
    sashes: [...c.querySelectorAll(':scope > .bp-Homecard__Sashes [data-rf-test-id="home-sash"]')].map(e => e.textContent.trim()),
    ld};
}))"""
 
def num(s):
    m = re.search(r"[\d,.]+", s or "")
    return float(m.group().replace(",", "")) if m and m.group().strip(".,") else None
 
def v(x):
    return x.get("value") if isinstance(x, dict) else x
 
def from_embedded(h):
    ll = v(h.get("latLong")) or {}
    return {
        "property_id": h.get("propertyId"), "listing_id": h.get("listingId"), "mls_id": v(h.get("mlsId")),
        "address": v(h.get("streetLine")), "city": h.get("city"), "state": h.get("state"), "zip": h.get("zip"),
        "price": v(h.get("price")), "beds": h.get("beds"), "baths": h.get("baths"), "sqft": v(h.get("sqFt")),
        "lot_sqft": v(h.get("lotSize")), "year_built": v(h.get("yearBuilt")), "price_per_sqft": v(h.get("pricePerSqFt")),
        "hoa_month": v(h.get("hoa")), "days_on_market": v(h.get("dom")), "status": h.get("mlsStatus"),
        "neighborhood": v(h.get("location")), "lat": ll.get("latitude"), "lon": ll.get("longitude"),
        "url": BASE + h["url"] if h.get("url") else None, "source": "embedded",
    }
 
def from_card(c, pid):
    ld = c.get("ld") or []
    ld = [x for x in (ld if isinstance(ld, list) else [ld]) if isinstance(x, dict)]   # a card's JSON-LD can be one object or a list
    home = next((x for x in ld if x.get("address")), {})
    offer = next((x.get("offers") for x in ld if x.get("offers")), {}) or {}
    addr, geo = home.get("address") or {}, home.get("geo") or {}
    price = offer.get("price") or num(c.get("price"))
    sq, sqft, lot = c.get("sqft") or "", None, None
    if "(lot)" in sq:                    # land cards say "Land" and "0.38 acre (lot)" or "9,557 sq ft (lot)"
        n = num(sq)
        lot = None if n is None else round(n * 43560 if "acre" in sq else n)
    else:
        sqft = num(sq)
    return {
        "property_id": pid, "listing_id": None, "mls_id": None,
        "address": addr.get("streetAddress") or (c.get("address") or "").split(",")[0],
        "city": addr.get("addressLocality"), "state": addr.get("addressRegion"), "zip": addr.get("postalCode"),
        "price": int(float(price)) if price else None, "beds": num(c.get("beds")), "baths": num(c.get("baths")),
        "sqft": sqft, "lot_sqft": lot, "year_built": None, "price_per_sqft": None, "hoa_month": None,
        "days_on_market": None, "status": None, "neighborhood": None,
        "lat": geo.get("latitude"), "lon": geo.get("longitude"), "url": BASE + c["href"], "source": "card",
    }
 
path = search_path(SEARCH)
new_tab()
landed = open_page(BASE + path)
if landed.rstrip("/") != BASE + path:
    raise RuntimeError(f"{BASE + path} redirected to {landed}; check SEARCH")
total, header = header_total()
emb = embedded_homes() if total else {}
print(f"{path}: header says {header!r} -> {total}; embedded map data has {len(emb)} homes")
 
rows = {}
if len(emb) >= total:                     # the whole result set is in the page's embedded map data
    for pid, h in emb.items():
        rows[pid] = from_embedded(h)
else:                                     # page through the list (about 41 cards per page)
    page = 1
    while True:
        cards = json.loads(js(CARD_JS))
        new = 0
        for c in cards:
            m = re.search(r"/home/(\d+)", c.get("href") or "")
            if not m:
                continue
            pid = int(m.group(1))
            if pid not in rows:
                rows[pid] = from_embedded(emb[pid]) if pid in emb else from_card(c, pid)
                new += 1
        print(f"page {page}: {len(cards)} cards, {new} new, {len(rows)} unique so far")
        if len(rows) >= total or new == 0:
            break
        page += 1
        if page > MAX_PAGES:
            raise RuntimeError(f"hit MAX_PAGES={MAX_PAGES}; {len(rows)} of {total} rows so far. Raise MAX_PAGES and rerun.")
        time.sleep(PAUSE)
        want = f"{BASE}{path}/page-{page}"
        landed = open_page(want)
        if landed.rstrip("/") != want:
            print(f"page-{page} redirected to {landed}; treating it as the end of the list")
            break
    for pid, h in emb.items():            # embedded homes no card showed are still part of the result
        rows.setdefault(pid, from_embedded(h))
 
rows = list(rows.values())
short = total - len(rows)
if short > max(2, total * MAX_SHORT):
    raise RuntimeError(f"collected {len(rows)} of the {total} homes the page header reports ({header!r}), {short} missing. "
                       "Raise MAX_SHORT only if you accept an incomplete result.")
 
out = os.path.join(tempfile.gettempdir(), "redfin_listings")
with open(out + ".json", "w") as f:
    json.dump(rows, f)
with open(out + ".csv", "w", newline="") as f:
    w = csv.DictWriter(f, fieldnames=list(from_embedded({})))   # same columns even when rows is empty
    w.writeheader()
    w.writerows(rows)
print("saved", len(rows), "rows to", out + ".json", "and .csv;", sum(r["source"] == "card" for r in rows), "from cards only;",
      f"header count {total}, {max(short, 0)} short")
print(json.dumps(rows[:2], indent=1))

/zipcode/78704: header says '323 homes + 9 early access' -> 332; embedded map data has 332 homes
saved 332 rows to /tmp/.../redfin_listings.json and .csv; 0 from cards only; header count 332, 0 short
[
 {"property_id": 31703545, "listing_id": 224677425, "mls_id": "3820245",
  "address": "503 S Park Dr", "city": "Austin", "state": "TX", "zip": "78704",
  "price": 1120000, "beds": 3, "baths": 2.5, "sqft": 1640, "lot_sqft": 11704, "year_built": 2023,
  "price_per_sqft": 683, "hoa_month": null, "days_on_market": 1, "status": "Active",
  "neighborhood": "SOUTH PARK SEC 2", "lat": 30.232246, "lon": -97.7638868,
  "url": "https://www.redfin.com/TX/Austin/503-S-Park-Dr-78704/home/31703545", "source": "embedded"},
 {"property_id": 31205095, "address": "1818 Margaret St", "price": 2995000, "beds": 3, "baths": 4.0,
  "status": "Coming Soon", "days_on_market": null, ...}
]

/city/30818/TX/Austin: header says '5,788 homes + 120 early access' -> 5908; embedded map data has 350 homes
page 1: 41 cards, 41 new, 41 unique so far
...
page 9: 22 cards, 22 new, 350 unique so far
page 10: 41 cards, 41 new, 391 unique so far
...
page 144: 22 cards, 22 new, 5597 unique so far
page 145: 41 cards, 40 new, 5637 unique so far
...
page 149: 36 cards, 36 new, 5796 unique so far
page 150: 41 cards, 0 new, 5796 unique so far
saved 5796 rows to /tmp/.../redfin_listings.json and .csv; 5446 from cards only

import json, os, re, time, tempfile
 
# Take a listing URL from the search output (block above); any redfin.com/.../home/<id> URL works.
saved = os.path.join(tempfile.gettempdir(), "redfin_listings.json")
listings = json.load(open(saved)) if os.path.exists(saved) else []
URL = listings[0]["url"] if listings else None
if not URL:
    raise SystemExit("Run the search block first, or set URL to a redfin.com listing URL")
 
BLOCKED = re.compile(r"request could not be satisfied|ratelimited|access denied|are you a human|captcha", re.I)
READY_JS = ("[...document.querySelectorAll('script[type=\"application/ld+json\"]')].some(s => s.textContent.includes('RealEstateListing'))"
            " && [...document.scripts].some(s => !s.src && s.textContent.includes('__reactServerState.InitialContext = '))")
 
def open_listing(url, budget=120, per_try=30):
    """Load a listing page and wait for the new document. Returns the URL it landed on: url itself once its JSON-LD
    and embedded data are there, or the redirect target (a gone listing redirects away). A block or challenge page
    raises at once (stop and back off). A page that never finishes is retried after 10 s, then 20 s, all inside
    one budget-second deadline; then it raises."""
    end, pause, attempt = time.time() + budget, 10, 0
    while True:
        attempt += 1
        before = js("window.__bhOld = 1, location.href")   # tag the old document; the new one will not carry the tag
        try:
            goto_url(url)
        except Exception as e:       # Page.navigate often outlives the 5 s daemon wait on these heavy pages
            print("navigate still running:", str(e)[:80])
        stop, last = min(end, time.time() + per_try), None
        while time.time() + 1 < stop:   # the last poll starts before the deadline
            time.sleep(1)
            try:
                st = last = json.loads(js("JSON.stringify({old: !!window.__bhOld, u: location.href, t: document.title, "
                                          "r: document.readyState, ok: %s})" % READY_JS))
            except Exception:
                continue                 # page still busy; poll again
            if st["old"]:
                continue                 # still the document from before the navigation
            if BLOCKED.search(st["t"]) or "ratelimited" in st["u"]:
                raise RuntimeError(f"Redfin served a block/challenge page ({st['t']!r} at {st['u']}). Stop and back off.")
            if st["r"] == "complete" and (st["ok"] or st["u"].rstrip("/") != url.rstrip("/")):
                return st["u"]           # ready, or a new document at another URL (redirected)
        print(f"try {attempt}: {url} not ready (navigated from {before}; last state {last})")
        if time.time() + pause + 15 > end:
            break                        # no time left for a backoff and another try; no sleep after the last one
        time.sleep(pause)
        pause *= 2
    raise RuntimeError(f"{url} did not finish loading within the {budget} s budget ({attempt} tries)")
 
# The listing page HTML embeds the server-rendered detail data in __reactServerState.
STATE_JS = r"""(() => {
  const s = [...document.scripts].find(s => !s.src && s.textContent.includes('__reactServerState.InitialContext = '));
  const p = 'root.__reactServerState.InitialContext = ';
  const line = s.textContent.split('\n').find(l => l.startsWith(p));
  const dc = JSON.parse(line.slice(p.length, -1))['ReactServerAgent.cache'].dataCache;
  const pick = part => { const k = Object.keys(dc).find(k => k.split('?')[0].endsWith(part)); return k ? dc[k].res.text : null; };
  return JSON.stringify({info: pick('/home/details/initialInfo'), atf: pick('/home/details/aboveTheFold'),
                         btf: pick('/home/details/belowTheFold')});
})()"""
 
def payload(text, name):
    if not text or not text.startswith("{}&&"):
        raise RuntimeError(f"{name} is missing from the page's embedded data: {str(text)[:120]}")
    d = json.loads(text[4:])             # strip the "{}&&" prefix
    if d.get("resultCode") != 0 or not isinstance(d.get("payload"), dict):
        raise RuntimeError(f"unexpected embedded {name}: {text[:160]}")
    return d["payload"]
 
def strip_tags(s):
    return re.sub(r"<[^>]+>", "", s or "").strip()
 
def listing_details(url):
    landed = open_listing(url)
    if landed.rstrip("/") != url.rstrip("/"):
        print(f"{url} redirected to {landed}")
        return None                      # listing gone or moved: never read another page's facts
    # 1) JSON-LD block typed ["Product", "RealEstateListing"]
    ld = None
    for t in js("[...document.querySelectorAll('script[type=\"application/ld+json\"]')].map(s => s.textContent)") or []:
        try:
            d = json.loads(t)
        except ValueError:
            continue
        if isinstance(d, dict) and "RealEstateListing" in (d.get("@type") or []):
            ld = d
    if not ld:
        raise RuntimeError("no RealEstateListing JSON-LD on " + landed)
    me, offer = ld.get("mainEntity", {}), ld.get("offers", {})
    facts = {
        "url": url,
        "address": me.get("address", {}),
        "price": offer.get("price"),
        "availability": (offer.get("availability") or "").rsplit("/", 1)[-1],
        "date_posted": ld.get("datePosted"),
        "home_type": me.get("accommodationCategory"),
        "beds": me.get("numberOfBedrooms"),
        "baths": me.get("numberOfBathroomsTotal"),
        "sqft": (me.get("floorSize") or {}).get("value"),
        "year_built": me.get("yearBuilt"),
        "lat": (me.get("geo") or {}).get("latitude"),
        "lon": (me.get("geo") or {}).get("longitude"),
        "amenities": [a.get("name") for a in me.get("amenityFeature", [])],
        "photos": [i.get("url") for i in me.get("image", [])],
        "description": (ld.get("description") or "").strip(),
    }
    # 2) the same page's embedded detail data (no extra requests)
    st = json.loads(js(STATE_JS))
    p = payload(st["info"], "initialInfo")
    atf = payload(st["atf"], "aboveTheFold").get("addressSectionInfo")
    if not isinstance(atf, dict):
        raise RuntimeError("aboveTheFold has no addressSectionInfo on " + landed)
    btf = payload(st["btf"], "belowTheFold")
    facts.update({
        "property_id": p.get("propertyId"), "listing_id": p.get("listingId"), "mls_id": p.get("mlsId"),
        "status": (atf.get("status") or {}).get("displayValue"),
        "lot_sqft": atf.get("lotSize"),
        "price_per_sqft": atf.get("pricePerSqFt"),
        "full_baths": atf.get("numFullBaths"), "half_baths": atf.get("numPartialBaths"),
    })
    groups = {}
    for sg in (btf.get("amenitiesInfo") or {}).get("superGroups", []):
        for g in sg.get("amenityGroups", []):
            for e in g.get("amenityEntries", []):
                vals = ", ".join(strip_tags(x) for x in e.get("amenityValues", []))
                groups.setdefault(g.get("groupTitle") or "untitled", []).append(f'{e.get("amenityName") or e.get("referenceName")}: {vals}')
    facts["mls_details"] = groups
    return facts
 
d = listing_details(URL)
if d is None:
    print("listing page not available:", URL)
else:
    out = dict(d, description=d["description"][:120] + "...", photos=f'{len(d["photos"])} urls, first: {d["photos"][0] if d["photos"] else None}',
               mls_details={k: v[:3] for k, v in list(d["mls_details"].items())[:6]} if d.get("mls_details") else None)
    print(json.dumps(out, indent=1))

{
 "url": "https://www.redfin.com/TX/Austin/503-S-Park-Dr-78704/home/31703545",
 "address": {"@type": "PostalAddress", "streetAddress": "503 S Park Dr", "addressLocality": "Austin",
             "addressRegion": "TX", "postalCode": "78704", "addressCountry": "US"},
 "price": 1120000, "availability": "InStock", "date_posted": "2026-10-23T18:45:28.294Z",
 "home_type": "Single Family Residential", "beds": 3, "baths": 2.5, "sqft": 1640, "year_built": 2023,
 "lat": 30.232246, "lon": -97.7638868,
 "amenities": ["Heating & cooling: Has A/C"],
 "photos": "22 urls, first: https://ssl.cdn-redfin.com/photo/92/bigphoto/245/3820245_0.jpg",
 "property_id": 31703545, "listing_id": 224677425, "mls_id": "3820245",
 "status": "Active", "lot_sqft": 11704, "price_per_sqft": 683, "full_baths": 2, "half_baths": 1,
 "mls_details": {
  "Bedroom Information": ["# of Main Level Bedrooms: 3"],
  "Bathroom Information": ["# of Full Bathrooms: 2", "# of Half Bathrooms: 1"],
  "Room 1 Information": ["Level: First"],
  "Laundry Information": ["Features: Inside"],
  ...
 }
}