Apartments.com - Rental listings and floor plans

import json, os, re, tempfile, time
 
CITY = "austin-tx"   # slug from the site URL: austin-tx, chicago-il, houses/austin-tx
MAX_PAGES = 3        # safety cap; the site serves at most 18 pages (700 results) per search
PAUSE = 5            # seconds between page loads; see "Rate limits" below
TRIES = 3            # navigation attempts per page
MAX_BACKOFF = 120    # never sleep longer than this between attempts
 
class Blocked(Exception):
    pass
 
def _norm(u):
    return (u or "").split("#")[0].split("?")[0].rstrip("/")
 
def open_page(url, first=False):
    """Navigate, then return only when THIS url is the loaded, complete document."""
    want, here = _norm(url), None
    for attempt in range(1, TRIES + 1):
        before = "" if first and attempt == 1 else None
        if before is None:
            try:
                before = _norm(js("location.href"))
            except (TimeoutError, RuntimeError):
                pass
        try:
            if first and attempt == 1:
                new_tab(url)
            else:
                goto_url(url)
        except TimeoutError as e:  # browser-harness IPC timeouts subclass TimeoutError
            print(f"navigation timed out ({e}); attempt {attempt}/{TRIES}")
        deadline = time.time() + 30
        while time.time() < deadline:
            try:
                s = js("({href: location.href, ready: document.readyState, title: document.title,"
                       " rendered: !!document.querySelector('link[rel=canonical]') && document.body?.childElementCount > 0})")
            except (TimeoutError, RuntimeError):  # js() reports timeouts and mid-navigation errors as RuntimeError
                s = None
            if isinstance(s, dict):
                if "Access Denied" in str(s.get("title")):
                    raise Blocked(f"CDN returned Access Denied for {url}; stop and retry much later")
                here = _norm(s.get("href"))
                if s.get("ready") == "complete" and s.get("rendered"):
                    if here == want:
                        return
                    if before is not None and here != before and not here.startswith(("about:", "chrome:")):
                        raise RuntimeError(f"{url} redirected to {here}")
            time.sleep(0.5)
        print(f"{url} not loaded within 30 s; attempt {attempt}/{TRIES}")
        if attempt < TRIES:
            time.sleep(min(PAUSE * attempt, MAX_BACKOFF))
    raise RuntimeError(f"could not load {url} after {TRIES} attempts (tab is at {here})")
 
EXTRACT = r"""
(() => {
  const clean = v => (typeof v === 'string' && v.replace(/\s+/g, ' ').trim()) || null;
  // .reinforcement = a repeat of an earlier card inserted mid-list; not counted in "Showing N"
  const rows = [...document.querySelectorAll('article.placard[data-listingid]:not(.reinforcement)')].map(a => {
    const q = s => a.querySelector(s);
    const txt = s => clean(q(s)?.textContent);
    let share = [];
    try {
      const v = JSON.parse(q('[data-listing-share-lines]')?.dataset.listingShareLines || '[]');
      if (Array.isArray(v)) share = v;
    } catch (e) {}
    const rentLine = clean(share[7]) || '';
    const tier = ([...a.classList].find(c => c.startsWith('tmpl-')) || '').replace('tmpl-', '') || null;
    // basic-io / premiumplus are single homes, often individual landlords: do not collect their phone
    const personal = ['basic-io', 'premiumplus'].includes(tier);
    return {
      listing_id: a.dataset.listingid || null,
      tier,
      name: txt('.property-title'),
      address: clean(q('.property-address')?.getAttribute('title'))
               || clean(q('[data-listing-address]')?.dataset.listingAddress)
               || [share[2], share[3]].map(clean).filter(Boolean).join(', ') || null,
      beds: clean(share[5]),
      baths: clean(share[6]),
      rent: rentLine.replace(/^Total Monthly Price:\s*/, '') || null,
      rent_includes_fees: rentLine.startsWith('Total Monthly Price'),
      rent_by_bed: [...a.querySelectorAll('.bedRentBox')].map(b => ({
        beds: clean(b.querySelector('.bedTextBox')?.textContent),
        rent: clean(b.querySelector('.priceTextBox')?.textContent)})),
      phone: personal ? null : clean(q('.phone-link')?.getAttribute('phone-data')),
      amenities: txt('.property-amenities'),
      url: q('a.property-link[href]')?.href.split(/[?#]/)[0] || null,
    };
  });
  return {title: document.title,
          summary: clean(document.querySelector('.searchResults')?.textContent),
          next: document.querySelector('#paging a.next')?.href || null,
          rows};
})()
"""
SUMMARY = re.compile(r"Showing (\d+)(?:-(\d+))? of ([\d,]+) Results? - Page (\d+) of (\d+)")
 
listings, seen, repeats = [], set(), 0
url, page = f"https://www.apartments.com/{CITY}/", 1
while True:
    open_page(url, first=(page == 1))
    time.sleep(2)
    res = js(EXTRACT)
    if not isinstance(res, dict) or not isinstance(res.get("rows"), list):
        raise RuntimeError(f"extractor returned nothing on {url}")
    rows, summary = res["rows"], res.get("summary") or ""
    m = SUMMARY.search(summary)
    if m:
        expected = int(m[2]) - int(m[1]) + 1 if m[2] else int(m[1])
        total, page_no, pages_total = int(m[3].replace(",", "")), int(m[4]), int(m[5])
    elif page == 1 and not res.get("next") and (rows or re.search(r"- 0 Rentals", str(res.get("title")))):
        # small searches have one page and no "Showing ..." line
        expected, total, page_no, pages_total = len(rows), len(rows), 1, 1
    else:
        raise RuntimeError(f"no results summary on {url} (title: {res.get('title')})")
    if page_no != page or len(rows) != expected:
        raise RuntimeError(f"{url}: page {page_no} with {len(rows)} cards, expected page {page} with {expected}")
    print(page, summary or "(single page, no pager)", len(rows))
    for r in rows:
        if r["listing_id"] is not None and r["listing_id"] in seen:
            repeats += 1
            continue
        seen.add(r["listing_id"])
        listings.append(r)
    if page >= pages_total:
        print(f"complete: all {pages_total} pages, {total} results")
        if total == 700:
            print("700 is the site's per-search cap; it serves no results past it")
        break
    if page >= MAX_PAGES:
        print(f"stopped at MAX_PAGES={MAX_PAGES}: {pages_total - page} of {pages_total} pages not fetched")
        break
    nxt = res.get("next")
    if not isinstance(nxt, str) or not nxt:
        raise RuntimeError(f"no Next link on page {page} of {pages_total}")
    url, page = nxt, page + 1
    time.sleep(PAUSE)
 
print(len(listings), "unique listings,", repeats, "repeated across pages")
print(json.dumps(listings[:1], indent=1))
out = os.path.join(tempfile.gettempdir(), "apartments_" + CITY.replace("/", "_") + ".json")
json.dump(listings, open(out, "w"), indent=1)
print("saved", out)

1 Showing 40 of 700 Results - Page 1 of 18 40
2 Showing 41-80 of 700 Results - Page 2 of 18 40
3 Showing 81-120 of 700 Results - Page 3 of 18 40
stopped at MAX_PAGES=3: 15 of 18 pages not fetched
120 unique listings, 0 repeated across pages
[
 {
  "listing_id": "4v6169l",
  "tier": "diamond",
  "name": "Ellwood at Lake Travis",
  "address": "7655 N Ranch Road 620, Austin, TX 78726",
  "beds": "1 - 3 bd",
  "baths": "1 - 2 ba",
  "rent": "$1,111 - $2,518",
  "rent_includes_fees": true,
  "rent_by_bed": [{"beds": "1 Bed", "rent": "$1,111+"}, {"beds": "2 Beds", "rent": "$1,249+"}, {"beds": "3 Beds", "rent": "$1,753+"}],
  "phone": "7372379124",
  "amenities": "Pets Allowed, Fitness Center, Pool, Walk-In Closets, Clubhouse, Stainless Steel Appliances, Business Center",
  "url": "https://www.apartments.com/ellwood-at-lake-travis-austin-tx/4v6169l/"
 }
]

import json, re, time
 
CITY = "austin-tx"
PROPERTY_URL = None   # e.g. a listing "url" from the search step; None = first result on the city page
PAUSE = 5
TRIES = 3
MAX_BACKOFF = 120
 
class Blocked(Exception):
    pass
 
def _norm(u):
    return (u or "").split("#")[0].split("?")[0].rstrip("/")
 
def open_page(url, first=False):
    """Navigate, then return only when THIS url is the loaded, complete document."""
    want, here = _norm(url), None
    for attempt in range(1, TRIES + 1):
        before = "" if first and attempt == 1 else None
        if before is None:
            try:
                before = _norm(js("location.href"))
            except (TimeoutError, RuntimeError):
                pass
        try:
            if first and attempt == 1:
                new_tab(url)
            else:
                goto_url(url)
        except TimeoutError as e:
            print(f"navigation timed out ({e}); attempt {attempt}/{TRIES}")
        deadline = time.time() + 30
        while time.time() < deadline:
            try:
                s = js("({href: location.href, ready: document.readyState, title: document.title,"
                       " rendered: !!document.querySelector('link[rel=canonical]') && document.body?.childElementCount > 0})")
            except (TimeoutError, RuntimeError):
                s = None
            if isinstance(s, dict):
                if "Access Denied" in str(s.get("title")):
                    raise Blocked(f"CDN returned Access Denied for {url}; stop and retry much later")
                here = _norm(s.get("href"))
                if s.get("ready") == "complete" and s.get("rendered"):
                    if here == want:
                        return
                    if before is not None and here != before and not here.startswith(("about:", "chrome:")):
                        raise RuntimeError(f"{url} redirected to {here}")
            time.sleep(0.5)
        print(f"{url} not loaded within 30 s; attempt {attempt}/{TRIES}")
        if attempt < TRIES:
            time.sleep(min(PAUSE * attempt, MAX_BACKOFF))
    raise RuntimeError(f"could not load {url} after {TRIES} attempts (tab is at {here})")
 
if not PROPERTY_URL:
    open_page(f"https://www.apartments.com/{CITY}/", first=True)
    time.sleep(2)
    href = js("document.querySelector('article.placard[data-listingid] a.property-link[href]')?.href || null")
    if not isinstance(href, str) or not href:
        raise RuntimeError("no listing card on the city page: " + str(page_info().get("title")))
    PROPERTY_URL = href.split("?")[0].split("#")[0]
    time.sleep(PAUSE)
    open_page(PROPERTY_URL)
else:
    open_page(PROPERTY_URL, first=True)
time.sleep(2)
 
PROPERTY = r"""
(() => {
  const str = v => (typeof v === 'string' && v.replace(/\s+/g, ' ').trim()) || null;
  const num = v => (typeof v === 'number' && isFinite(v)) ? v
                 : (typeof v === 'string' && v.trim() !== '' && isFinite(+v)) ? +v : null;
  const obj = v => (v && typeof v === 'object' && !Array.isArray(v)) ? v : {};
  const one = v => obj(Array.isArray(v) ? v[0] : v);
  // 1) JSON-LD: the first node whose @type includes RealEstateListing (top level or in @graph)
  let listing = null;
  for (const s of document.querySelectorAll('script[type="application/ld+json"]')) {
    let d;
    try { d = JSON.parse(s.textContent); } catch (e) { continue; }
    for (const top of [].concat(d)) {
      for (const n of [].concat(obj(top)['@graph'] || top)) {
        if (!listing && [].concat(obj(n)['@type'] || []).includes('RealEstateListing')) listing = obj(n);
      }
    }
  }
  listing = listing || {};
  const me = one(listing.mainEntity);
  const a = me.address;
  const address = typeof a === 'string' ? str(a)
    : [a?.streetAddress, a?.addressLocality, a?.addressRegion, a?.postalCode].map(str).filter(Boolean).join(', ') || null;
  const geo = one(me.geo), offers = one(listing.offers), rating = one(listing.aggregateRating);
  // 2) Floor plans: the "all" tab holds every plan once; per-bed tabs repeat them
  const root = document.querySelector('[data-tab-content-id="all"]') || document;
  const seen = new Set();
  const plans = [...root.querySelectorAll('.pricingGridItem')].map(g => {
    const key = g.querySelector('.priceGridModelWrapper')?.dataset.rentalkey || null;
    if (key && seen.has(key)) return null;
    if (key) seen.add(key);
    const t = s => str(g.querySelector(s)?.textContent);
    return {
      rental_key: key,
      plan: t('.modelName'),
      rent: t('.rentLabel'),
      details: [...g.querySelectorAll('.detailsTextWrapper > span')].map(s => str(s.textContent)).filter(Boolean),
      availability: t('.availability'),
      units: [...g.querySelectorAll('li.unitContainer')].map(u => ({
        unit: u.dataset.unit || null,
        beds: u.dataset.beds || null, baths: u.dataset.baths || null,
        base_rent: u.dataset.maxrent || null,
        price: u.dataset.applynowPrice || null,
        specs: u.dataset.applynowSpecs || null,
      })),
    };
  }).filter(Boolean);
  // Phone only for apartment communities; single homes (SingleFamilyResidence etc.) are often individual landlords
  const propertyType = str([].concat(me['@type'] || [])[0]);
  return {
    url: location.href.split(/[?#]/)[0],
    name: str(listing.name) || str(document.querySelector('h1')?.textContent),
    property_type: propertyType,
    phone: propertyType === 'ApartmentComplex' ? str(listing.telephone) : null,
    address,
    lat: num(geo.latitude), lon: num(geo.longitude),
    rent_low: num(offers.lowPrice), rent_high: num(offers.highPrice),
    rating: num(rating.ratingValue),
    rating_count: num(rating.ratingCount),
    pets_allowed: typeof me.petsAllowed === 'boolean' ? me.petsAllowed : str(me.petsAllowed),
    amenities_jsonld: [].concat(me.amenityFeature || []).map(f => typeof f === 'string' ? str(f) : str(obj(f).name)).filter(Boolean),
    amenities_page: [...new Set([...document.querySelectorAll('#amenitiesSection li')].map(e => str(e.textContent)).filter(Boolean))],
    floor_plans: plans,
  };
})()
"""
 
prop = js(PROPERTY)
if not isinstance(prop, dict) or not prop.get("name"):
    raise RuntimeError("property extractor returned nothing on " + PROPERTY_URL)
print(json.dumps({k: v for k, v in prop.items() if k != "floor_plans"}, indent=1))
plans = prop["floor_plans"]
print(len(plans), "floor plans,", sum(len(p["units"]) for p in plans), "listed units")
for p in plans:
    print(p["plan"], "|", p["rent"], "|", " / ".join(p["details"]), "|", p["availability"])
# Completeness: a plan that says "N Available units" must list N unit rows
short = [(p["plan"], m[1], len(p["units"])) for p in plans
         for m in [re.match(r"(\d+) Available units?", p["availability"] or "")]
         if m and int(m[1]) != len(p["units"])]
if short:
    raise RuntimeError(f"unit grid incomplete (plan, stated, listed): {short}")
print("unit rows match every plan's stated available count")

{
 "url": "https://www.apartments.com/ellwood-at-lake-travis-austin-tx/4v6169l/",
 "name": "Ellwood at Lake Travis",
 "property_type": "ApartmentComplex",
 "phone": "+1-737-237-9124",
 "address": "7655 N Ranch Road 620, Austin, TX, 78726",
 "lat": 30.40643,
 "lon": -97.85038,
 "rent_low": 951,
 "rent_high": 2358,
 "rating": null,
 "rating_count": null,
 "pets_allowed": true,
 "amenities_jsonld": ["Pool", "Fitness Center", "Clubhouse", "Recycling", "Business Center", "Grill", "Pet Care", "Conference Rooms"],
 "amenities_page": ["Package Service", "Wi-Fi", "Maintenance on site", "Property Manager on Site", ...]
}
49 floor plans, 79 listed units
A2 Lake | $1,111 Total Monthly Price | 1 Bed / 1 Bath / 683 Sq Ft / $250 Deposit | 1 Available unit
A1 Lake | Starting at $1,143 Total Monthly Price | 1 Bed / 1 Bath / 644 Sq Ft / $250 Deposit | 2 Available units
...
C1 Canyon Renovated | Starting at $1,918 Total Monthly Price | 3 Beds / 2 Baths / 1,341 Sq Ft / $250 Deposit | 3 Available units
A4 Lake | Call for Rent | 1 Bed / 1 Bath / 775 Sq Ft | None
...
unit rows match every plan's stated available count