Apple App Store - App Details and Search Results

import email.utils, json, sys, time, urllib.error, urllib.parse, urllib.request
 
UA = f"Python-urllib/{sys.version_info.major}.{sys.version_info.minor}"  # truthful client UA
RETRY = {429, 500, 502, 503, 504}  # 403 and other statuses are not retried
MAX_WAIT = 120  # longest backoff this code sleeps; a longer Retry-After stops the run
 
def retry_after(e, default):
    """Seconds to wait: Retry-After as seconds or HTTP-date, else the default backoff."""
    v = ((getattr(e, "headers", None) or {}).get("Retry-After") or "").strip()
    if v.isdigit():
        return int(v)
    if v:
        try:
            return max(0, email.utils.parsedate_to_datetime(v).timestamp() - time.time())
        except (TypeError, ValueError):
            pass
    return default
 
def fetch(url, retries=4):
    """GET url and return the body text. None only for HTTP 404. Retries 429/5xx and network
    errors with capped backoff, then raises: a failed call never looks like an empty result."""
    for attempt in range(retries):
        try:
            req = urllib.request.Request(url, headers={"User-Agent": UA})
            with urllib.request.urlopen(req, timeout=30) as r:
                return r.read().decode()
        except urllib.error.HTTPError as e:
            if e.code == 404:
                return None
            if e.code not in RETRY or attempt == retries - 1:
                raise
            wait = retry_after(e, 15 * (attempt + 1))
            if wait > MAX_WAIT:
                raise RuntimeError(f"{url}: server asked to wait {wait:.0f} s (> {MAX_WAIT} s); stopping") from e
            time.sleep(wait)
        except OSError:  # URLError, timeouts, connection resets
            if attempt == retries - 1:
                raise
            time.sleep(15 * (attempt + 1))
 
def itunes(path, params):
    """GET https://itunes.apple.com/<path>?<params> and return the parsed JSON object."""
    url = f"https://itunes.apple.com/{path}?" + urllib.parse.urlencode(params)  # quotes every value, '/' included
    body = fetch(url)
    data = json.loads(body) if body else None
    if not isinstance(data, dict) or not isinstance(data.get("results"), list):
        raise RuntimeError(f"unexpected iTunes response from {url}")
    return data
 
def num(v):
    return v if isinstance(v, (int, float)) and not isinstance(v, bool) else None
 
def text(v):
    return v if isinstance(v, str) else None
 
def to_row(r):
    size = r.get("fileSizeBytes")
    size = int(size) if isinstance(size, str) and size.isdigit() else num(size)
    rating = num(r.get("averageUserRating"))
    return {
        "id": num(r.get("trackId")),
        "name": text(r.get("trackName")),
        "developer": text(r.get("artistName")),
        "rating": round(rating, 2) if rating is not None else None,
        "ratings_count": num(r.get("userRatingCount")),
        "price": num(r.get("price")),
        "price_label": text(r.get("formattedPrice")),
        "currency": text(r.get("currency")),
        "category": text(r.get("primaryGenreName")),
        "size_mb": round(size / 1e6, 1) if size is not None else None,
        "version": text(r.get("version")),
        "version_date": text(r.get("currentVersionReleaseDate")),
        "release_notes": (text(r.get("releaseNotes")) or "")[:200],
        "first_release": text(r.get("releaseDate")),
        "bundle_id": text(r.get("bundleId")),
        "url": (text(r.get("trackViewUrl")) or "").split("?")[0],
    }
 
def search_apps(term, country="us", limit=200, entity="software"):
    data = itunes("search", {"term": term, "country": country, "entity": entity, "limit": limit})
    if len(data["results"]) >= limit:
        print(f"WARNING: {term!r} hit the limit={limit} cap; Apple serves no more, so this list is truncated")
    return [to_row(r) for r in data["results"] if r.get("wrapperType") == "software"]
 
terms = ["habit tracker", "budget planner", "meditation"]
apps = {}
for t in terms:
    rows = search_apps(t)
    print(f"{t!r}: {len(rows)} apps")
    for row in rows:
        key = row["id"] if row["id"] is not None else ("no-id", row["name"], row["developer"])
        apps.setdefault(key, row)  # rows without an id are kept, not merged
    time.sleep(3)  # polite pacing between calls
 
print("unique apps:", len(apps))
top = sorted(apps.values(), key=lambda a: a["ratings_count"] or 0, reverse=True)[:5]
for a in top:
    print(a["name"], "|", a["developer"], "|", a["rating"], "|", a["ratings_count"], "|",
          a["price_label"], "|", a["category"], "|", a["size_mb"], "MB |", a["version"], a["version_date"])
print(json.dumps(top[0], indent=1))

import email.utils, json, sys, time, urllib.error, urllib.parse, urllib.request
 
UA = f"Python-urllib/{sys.version_info.major}.{sys.version_info.minor}"  # truthful client UA
RETRY = {429, 500, 502, 503, 504}  # 403 and other statuses are not retried
MAX_WAIT = 120  # longest backoff this code sleeps; a longer Retry-After stops the run
 
def retry_after(e, default):
    v = ((getattr(e, "headers", None) or {}).get("Retry-After") or "").strip()
    if v.isdigit():
        return int(v)
    if v:
        try:
            return max(0, email.utils.parsedate_to_datetime(v).timestamp() - time.time())
        except (TypeError, ValueError):
            pass
    return default
 
def fetch(url, retries=4):
    for attempt in range(retries):
        try:
            req = urllib.request.Request(url, headers={"User-Agent": UA})
            with urllib.request.urlopen(req, timeout=30) as r:
                return r.read().decode()
        except urllib.error.HTTPError as e:
            if e.code == 404:
                return None
            if e.code not in RETRY or attempt == retries - 1:
                raise
            wait = retry_after(e, 15 * (attempt + 1))
            if wait > MAX_WAIT:
                raise RuntimeError(f"{url}: server asked to wait {wait:.0f} s (> {MAX_WAIT} s); stopping") from e
            time.sleep(wait)
        except OSError:
            if attempt == retries - 1:
                raise
            time.sleep(15 * (attempt + 1))
 
def itunes(path, params):
    url = f"https://itunes.apple.com/{path}?" + urllib.parse.urlencode(params)
    body = fetch(url)
    data = json.loads(body) if body else None
    if not isinstance(data, dict) or not isinstance(data.get("results"), list):
        raise RuntimeError(f"unexpected iTunes response from {url}")
    return data
 
def lookup_ids(app_ids, country="us", chunk=100):
    """Return {trackId: raw_result}. Missing/removed IDs are simply absent; compare against the input."""
    if not isinstance(chunk, int) or isinstance(chunk, bool) or not 1 <= chunk <= 100:
        raise ValueError("chunk must be an integer from 1 to 100 (lookup silently drops IDs past about 210 per call)")
    out = {}
    app_ids = [int(i) for i in app_ids]
    for i in range(0, len(app_ids), chunk):
        batch = app_ids[i:i + chunk]
        data = itunes("lookup", {"id": ",".join(map(str, batch)), "country": country})
        for r in data["results"]:
            if r.get("wrapperType") == "software" and isinstance(r.get("trackId"), int):
                out[r["trackId"]] = r
        time.sleep(3)
    return out
 
# 1) Many IDs: take the IDs of the top "productivity" search hits
hits = itunes("search", {"term": "productivity", "entity": "software", "country": "us", "limit": 200})["results"]
ids = [r["trackId"] for r in hits if isinstance(r.get("trackId"), int)]
ids.append(1)  # an ID that does not exist
time.sleep(3)
found = lookup_ids(ids)
print(f"requested {len(ids)} ids, got {len(found)}; missing: {[i for i in ids if i not in found]}")
 
# 2) Single app, all detail fields
app = lookup_ids([389801252]).get(389801252)
if app is None:
    raise SystemExit("app 389801252 not found in this storefront")
print({k: app.get(k) for k in ("trackName", "artistName", "averageUserRating", "userRatingCount",
      "formattedPrice", "primaryGenreName", "fileSizeBytes", "version", "currentVersionReleaseDate",
      "contentAdvisoryRating", "minimumOsVersion")})
 
# 3) By bundle ID
res = itunes("lookup", {"bundleId": "com.burbn.instagram", "country": "us"})["results"]
print("bundleId ->", [(r.get("trackId"), r.get("trackName")) for r in res])
time.sleep(3)
 
# 4) Every app from a developer (artistId comes from any app's 'artistId')
dev = itunes("lookup", {"id": app["artistId"], "entity": "software", "country": "us", "limit": 200})["results"]
artist = next((r for r in dev if r.get("wrapperType") == "artist"), {})
print("developer:", artist.get("artistName"), "->", [r.get("trackName") for r in dev if r.get("wrapperType") == "software"][:8])
time.sleep(3)
 
# 5) Paid app, priced per storefront
for cc in ("us", "gb", "de"):
    r = itunes("lookup", {"id": 963034692, "country": cc})["results"]
    print(cc, [(x.get("trackName"), x.get("price"), x.get("currency"), x.get("formattedPrice")) for x in r])
    time.sleep(3)

import email.utils, json, re, sys, time, urllib.error, urllib.request
 
UA = f"Python-urllib/{sys.version_info.major}.{sys.version_info.minor}"  # truthful client UA
RETRY = {429, 500, 502, 503, 504}  # 403 and other statuses are not retried
MAX_WAIT = 120  # longest backoff this code sleeps; a longer Retry-After stops the run
 
def retry_after(e, default):
    v = ((getattr(e, "headers", None) or {}).get("Retry-After") or "").strip()
    if v.isdigit():
        return int(v)
    if v:
        try:
            return max(0, email.utils.parsedate_to_datetime(v).timestamp() - time.time())
        except (TypeError, ValueError):
            pass
    return default
 
def fetch(url, retries=4):
    for attempt in range(retries):
        try:
            req = urllib.request.Request(url, headers={"User-Agent": UA})
            with urllib.request.urlopen(req, timeout=30) as r:
                return r.read().decode()
        except urllib.error.HTTPError as e:
            if e.code == 404:
                return None
            if e.code not in RETRY or attempt == retries - 1:
                raise
            wait = retry_after(e, 15 * (attempt + 1))
            if wait > MAX_WAIT:
                raise RuntimeError(f"{url}: server asked to wait {wait:.0f} s (> {MAX_WAIT} s); stopping") from e
            time.sleep(wait)
        except OSError:
            if attempt == retries - 1:
                raise
            time.sleep(15 * (attempt + 1))
 
def first(v):
    return v[0] if isinstance(v, list) and v and isinstance(v[0], dict) else {}
 
def obj(v):
    return v if isinstance(v, dict) else {}
 
def app_page(app_id, country="us"):
    """Extras only the web page has. None if the app does not exist (HTTP 404); raises on anything else odd."""
    url = f"https://apps.apple.com/{country}/app/id{app_id}"
    html = fetch(url)
    if html is None:
        return None
    m = re.search(r'<script type="application/json" id="serialized-server-data">(.*?)</script>', html, re.S)
    if not m:
        raise RuntimeError(f"{url}: no serialized-server-data in the page")
    page = obj(first(obj(json.loads(m.group(1))).get("data")).get("data"))
    lockup = obj(page.get("lockup"))
    if str(lockup.get("adamId")) != str(app_id):
        raise RuntimeError(f"{url}: page is for app {lockup.get('adamId')!r}, not {app_id}")
    sm = obj(page.get("shelfMapping"))
    badges = {b.get("key"): obj(b.get("content"))
              for b in obj(sm.get("informationRibbon")).get("items") or [] if isinstance(b, dict)}
    info = {a.get("title"): a.get("summary")
            for a in obj(sm.get("information")).get("items") or [] if isinstance(a, dict)}
    ratings = first(obj(sm.get("productRatings")).get("items"))
    counts = ratings.get("ratingCounts") if isinstance(ratings.get("ratingCounts"), list) else []
    vh = first(obj(obj(obj(sm.get("mostRecentVersion")).get("seeAllAction")).get("pageData")).get("shelves"))
    return {
        "id": app_id,
        "incomplete": page.get("isIncomplete"),  # Apple's own flag; False on every page tested
        "name": lockup.get("title"),
        "subtitle": lockup.get("subtitle"),
        "rating": ratings.get("ratingAverage"),
        "ratings_count": ratings.get("totalNumberOfRatings"),
        "stars_5_to_1": [round(x) for x in counts if isinstance(x, (int, float))],
        "chart_rank": badges.get("chartPosition", {}).get("position"),
        "age_rating": badges.get("contentRating", {}).get("contentRating"),
        "size": info.get("Size"),
        "category": info.get("Category"),
        "in_app_purchases": info.get("In-App Purchases"),
        "versions": [{"version": v.get("primarySubtitle"), "date": v.get("secondarySubtitle"),
                      "notes": (v.get("text") or "")[:120] if isinstance(v.get("text"), str) else ""}
                     for v in vh.get("items") or [] if isinstance(v, dict)],
    }
 
for app_id in [389801252, 963034692, 1438388363, 1]:  # Instagram, Streaks (paid), a habit app, a bad ID
    d = app_page(app_id)
    if d is None:
        print(app_id, "-> None (HTTP 404, no such app)")
        continue
    vs = d.pop("versions")
    print(json.dumps(d, ensure_ascii=False))
    print("   versions embedded:", len(vs), "(Apple embeds at most 25)", vs[:1])
    time.sleep(3)

import json, re, time
 
def browser_timeout(e):
    # cdp/IPC timeouts raise TimeoutError; js() wraps a timed-out Runtime.evaluate in RuntimeError("... timed out ...")
    return isinstance(e, TimeoutError) or (isinstance(e, RuntimeError) and "timed out" in str(e))
 
def open_app(app_id, first_tab, timeout=30):
    """Navigate, then wait until location.href is the requested app and the document is complete."""
    url = f"https://apps.apple.com/us/app/id{app_id}"
    new_tab(url) if first_tab else goto_url(url)
    deadline, last = time.time() + timeout, None
    while time.time() < deadline:
        try:
            state = js("[location.href, document.readyState]")
            if state and re.search(rf"/id{app_id}(?!\d)", state[0]) and state[1] == "complete":
                return state[0]
        except Exception as e:
            if not browser_timeout(e):
                raise
            last = e
        time.sleep(0.5)
    raise TimeoutError(f"tab never reached app {app_id} (last browser timeout: {last})")
 
EXTRACT = r"""(async (url, appId) => {
  const res = await fetch(url, {credentials: 'omit'});
  if (!res.ok) return {status: res.status};
  const doc = new DOMParser().parseFromString(await res.text(), 'text/html');
  const el = doc.getElementById('serialized-server-data');
  if (!el) return {status: res.status, error: 'no serialized-server-data'};
  const page = JSON.parse(el.textContent).data?.[0]?.data || {};
  if (String(page.lockup?.adamId) !== appId) return {status: res.status, error: 'wrong app ' + page.lockup?.adamId};
  const sm = page.shelfMapping || {};
  const badge = k => (sm.informationRibbon?.items || []).find(b => b?.key === k)?.content || {};
  const info = Object.fromEntries((sm.information?.items || []).map(a => [a?.title, a?.summary]));
  const r = sm.productRatings?.items?.[0] || {};
  const vh = sm.mostRecentVersion?.seeAllAction?.pageData?.shelves?.[0]?.items || [];
  return {status: res.status, incomplete: page.isIncomplete ?? null, name: page.lockup?.title ?? null,
          rating: r.ratingAverage ?? null, ratings_count: r.totalNumberOfRatings ?? null,
          stars_5_to_1: (Array.isArray(r.ratingCounts) ? r.ratingCounts : []).filter(x => typeof x === 'number').map(Math.round),
          chart_rank: badge('chartPosition').position ?? null, age_rating: badge('contentRating').contentRating ?? null,
          size: info['Size'] ?? null, category: info['Category'] ?? null, versions: vh.length,
          latest: vh[0]?.primarySubtitle ?? null, latest_date: vh[0]?.secondarySubtitle ?? null};
})"""
 
def extract_in_tab(app_id, retries=3, timeout=45):
    """Page extras for app_id via an in-page fetch. None on HTTP 404; retries 429/5xx and browser timeouts
    with backoff (no sleep after the last try), then raises. 403 and other statuses raise at once."""
    url, out = f"https://apps.apple.com/us/app/id{app_id}", None
    for attempt in range(retries):
        js(f"window.__bhApp = null; ({EXTRACT})({json.dumps(url)}, {json.dumps(str(app_id))})"
           ".then(r => { window.__bhApp = r; }, e => { window.__bhApp = {error: String(e)}; }); true")
        deadline, out = time.time() + timeout, None
        while time.time() < deadline and out is None:
            time.sleep(1)
            try:
                out = js("window.__bhApp")
            except Exception as e:
                if not browser_timeout(e):
                    raise
        status = (out or {}).get("status")
        if out is not None and status == 200 and "error" not in out:
            return out
        if status == 404:
            return None
        if out is not None and status not in (429, 500, 502, 503, 504) and "status" in out:
            raise RuntimeError(f"app {app_id}: {out}")  # e.g. wrong app or missing embedded data
        if attempt < retries - 1:
            time.sleep(15 * (attempt + 1))  # 429/5xx, fetch error or poll timeout: back off and retry
    raise RuntimeError(f"app {app_id}: in-page fetch still failing after {retries} tries: {out}")
 
for n, app_id in enumerate([310633997, 1586435171]):  # WhatsApp, Actions
    href = open_app(app_id, first_tab=(n == 0))
    print(app_id, href, json.dumps(extract_in_tab(app_id), ensure_ascii=False))
    time.sleep(3)