Yahoo Finance - Quote Summary and Historical OHLCV

import json, time, urllib.parse
 
TICKERS = ["AAPL", "MSFT", "^GSPC", "BTC-USD"]   # Yahoo symbols you already know are valid
 
READ_QUOTE = r"""(() => {
  const q = s => document.querySelector(s);
  const txt = s => (q(s) && q(s).innerText.trim()) || null;
  const hdr = q('[data-testid="quote-hdr"]');
  const price = q('[data-testid="quote-price"]');
  const hdrLines = hdr ? hdr.innerText.split('\n').map(s => s.trim()).filter(Boolean) : [];
  const stats = {}, fields = {};   // label -> shown text; label -> Yahoo field name
  for (const li of document.querySelectorAll('[data-testid="quote-statistics"] li')) {
    const label = li.querySelector('.label'), value = li.querySelector('.value');
    if (!label || !value) continue;
    const key = label.innerText.trim();
    stats[key] = value.innerText.trim();
    const fs = value.querySelector('fin-streamer[data-field]');
    if (fs) fields[key] = fs.dataset.field;   // e.g. "Volume" -> "regularMarketVolume"
  }
  return {
    url: location.href,
    title: document.title,
    name: txt('[data-testid="quote-title"] h1'),
    exchange_line: hdrLines[0] || null,
    currency: hdrLines.find(s => /^[A-Z]{3}$/.test(s)) || null,
    price: txt('[data-testid="qsp-price"]'),
    change: txt('[data-testid="qsp-price-change"]'),
    change_pct: txt('[data-testid="qsp-price-change-percent"]'),
    post_price: txt('[data-testid="qsp-post-price"]'),
    post_change: txt('[data-testid="qsp-post-price-change"]'),
    post_change_pct: txt('[data-testid="qsp-post-price-change-percent"]'),
    times: price ? price.innerText.split('\n').map(s => s.trim()).filter(s => /^(At close|As of|After hours|Pre-Market|Overnight)/i.test(s)) : [],
    stats, fields,
  };
})()"""
 
def open_page(url, tries=3):
    # Yahoo pages are heavy (ads, live quotes). goto_url() gives up after ~5 s while the tab is busy;
    # the navigation usually still lands, so wait and retry only on that timeout.
    for i in range(tries):
        try:
            goto_url(url)
            break
        except Exception as e:
            if "timed out" not in str(e) or i == tries - 1:
                raise
            time.sleep(5)
    for i in range(tries):   # wait_for_load() polls js(), which also times out while a big page renders
        try:
            wait_for_load()
            return
        except (TimeoutError, RuntimeError) as e:
            if (not isinstance(e, TimeoutError) and "timed out" not in str(e)) or i == tries - 1:
                raise
            time.sleep(2)
 
def js_retry(expr, tries=3):
    # A busy page makes js() time out (TimeoutError, or RuntimeError "... timed out ..."); retry only that.
    for _ in range(tries):
        try:
            return js(expr)
        except (TimeoutError, RuntimeError) as e:
            if not isinstance(e, TimeoutError) and "timed out" not in str(e):
                raise
            time.sleep(2)
    raise TimeoutError(f"js() timed out {tries} times")
 
def quote_summary(symbol):
    url = f"https://finance.yahoo.com/quote/{urllib.parse.quote(symbol, safe='')}/"   # ^GSPC -> %5EGSPC
    open_page(url)
    d = None
    for _ in range(20):   # the price header and stats list render client-side; wait up to ~20 s
        d = js_retry(READ_QUOTE)
        if not d["url"].startswith(url):
            # consent page (guce.yahoo.com), a redirect for an unknown symbol, or a challenge: stop, do not work around it
            raise RuntimeError(f"{symbol}: landed on {d['url']} ({d['title']!r}); stopping")
        if d["price"] and d["stats"]:
            return d
        time.sleep(1)
    raise RuntimeError(f"{symbol}: no price/stats after 20 s at {d['url']} ({d['title']!r}); stopping")
 
tab = new_tab()
try:
    for i, sym in enumerate(TICKERS):
        if i:
            time.sleep(3)   # polite pacing between pages
        d = quote_summary(sym)
        print(json.dumps({k: d[k] for k in ("name", "exchange_line", "currency", "price", "change", "change_pct", "post_price", "times")}))
        print(f"  {len(d['stats'])} stats:", json.dumps(d["stats"]))
        print("  fields:", json.dumps(dict(list(d["fields"].items())[:4])))
finally:
    close_tab(tab)

import json, time, calendar, urllib.parse
from datetime import date, datetime, timedelta
 
SYMBOL = "AAPL"
START, END = date(2024, 1, 1), date(2024, 3, 31)   # inclusive calendar dates
FREQUENCY = "1d"                                    # the page's own menu values: 1d, 1wk, 1mo
 
READ_TABLE = r"""(() => {
  const t = document.querySelector('[data-testid="history-table"] table');
  if (!t) return {url: location.href, title: document.title, head: null, rows: []};
  return {
    url: location.href,
    title: document.title,
    range_label: ([...document.querySelectorAll('main button')].map(b => b.innerText.trim()).find(s => /\d{4} - /.test(s)) || null),
    head: [...t.querySelectorAll('thead th')].map(th => th.innerText.trim()),
    rows: [...t.querySelectorAll('tbody tr')].map(r => [...r.cells].map(c => c.innerText.trim())),
  };
})()"""
 
def open_page(url, tries=3):
    # Yahoo pages are heavy (ads, live quotes). goto_url() gives up after ~5 s while the tab is busy;
    # the navigation usually still lands, so wait and retry only on that timeout.
    for i in range(tries):
        try:
            goto_url(url)
            break
        except Exception as e:
            if "timed out" not in str(e) or i == tries - 1:
                raise
            time.sleep(5)
    for i in range(tries):   # wait_for_load() polls js(), which also times out while a big page renders
        try:
            wait_for_load()
            return
        except (TimeoutError, RuntimeError) as e:
            if (not isinstance(e, TimeoutError) and "timed out" not in str(e)) or i == tries - 1:
                raise
            time.sleep(2)
 
def js_retry(expr, tries=3):
    # A busy page makes js() time out (TimeoutError, or RuntimeError "... timed out ..."); retry only that.
    for _ in range(tries):
        try:
            return js(expr)
        except (TimeoutError, RuntimeError) as e:
            if not isinstance(e, TimeoutError) and "timed out" not in str(e):
                raise
            time.sleep(2)
    raise TimeoutError(f"js() timed out {tries} times")
 
def num(s):
    s = (s or "").replace(",", "")
    try:
        return float(s)
    except ValueError:
        return None   # "-" = no value on that row
 
def history(symbol, start, end, frequency="1d"):
    """OHLCV rows for start..end (inclusive), newest first, plus dividend/split rows, from /quote/<T>/history/."""
    # Same URL the page's own date picker writes: unix seconds, period2 exclusive.
    p1 = calendar.timegm(start.timetuple())
    p2 = calendar.timegm((end + timedelta(days=1)).timetuple())
    base = f"https://finance.yahoo.com/quote/{urllib.parse.quote(symbol, safe='')}/history/"   # ^GSPC -> %5EGSPC
    url = f"{base}?period1={p1}&period2={p2}"
    if frequency != "1d":
        url += f"&frequency={frequency}"
    open_page(url)
    last_n, d = -1, None
    for _ in range(25):   # rows render client-side; wait until the count is non-zero and stable
        d = js_retry(READ_TABLE)
        if not d["url"].startswith(base):
            raise RuntimeError(f"{symbol}: landed on {d['url']} ({d['title']!r}); stopping")
        if d["rows"] and len(d["rows"]) == last_n:
            break
        last_n = len(d["rows"])
        time.sleep(1)
    else:
        raise RuntimeError(f"{symbol}: table did not settle ({last_n} rows) at {d['url']}; stopping")
    head = [h.lower() for h in d["head"]]
    if head != ["date", "open", "high", "low", "close", "adj close", "volume"]:
        raise RuntimeError(f"{symbol}: unexpected columns {d['head']}; layout changed?")
    prices, events = [], []
    for cells in d["rows"]:
        day = datetime.strptime(cells[0], "%b %d, %Y").date().isoformat()
        if len(cells) == 7:
            o, h, l, c, ac, v = (num(x) for x in cells[1:])
            prices.append({"date": day, "open": o, "high": h, "low": l, "close": c, "adj_close": ac,
                           "volume": int(v) if v is not None else None})
        else:   # "0.25 Dividend" or "4:1 Stock Splits" spans the row
            events.append({"date": day, "event": cells[1]})
    return {"url": d["url"], "range_label": d["range_label"], "prices": prices, "events": events}
 
tab = new_tab()
try:
    h = history(SYMBOL, START, END, FREQUENCY)
    p = h["prices"]
    print(h["url"])
    print("range shown:", h["range_label"], "|", len(p), "price rows,", len(h["events"]), "events")
    print("newest:", json.dumps(p[0]))
    print("oldest:", json.dumps(p[-1]))
    print("events:", json.dumps(h["events"]))
finally:
    close_tab(tab)