Dice - Tech Job Search Scraping

import http.client, json, re, time, urllib.error, urllib.parse, urllib.request
 
UA = "dice-jobs-research/1.0 (python-urllib; personal job search script)"
BASE = "https://www.dice.com"
MAX_PAGE = 25          # Dice's cap: pages past 25 come back as page 25. Hard stop.
PAUSE_S = 2.0          # polite gap between page requests
DEADLINE_S = 120       # one budget per fetch(), across all attempts and backoff
DELAYS = (5, 15, 40)   # backoff between attempts, never after the last one
CHALLENGE = re.compile(r"cf-chl|/cdn-cgi/challenge-platform|px-captcha|captcha-delivery\.com|"
                       r"_Incapsula_Resource|g-recaptcha|h-captcha|"
                       r"<title>\s*(?:Access Denied|Just a moment|Attention Required)", re.I)
UNVERIFIED = "unique jobs reached totalResults, but Dice has no stable snapshot: not verified complete"
 
def search_url(keyword=None, location=None, page=1, filters=None):
    """Public search path /jobs/q-<kw>-l-<loc>-jobs. Hyphens are path separators
    there, so hyphens inside the keyword/location are replaced with spaces."""
    parts = []
    if keyword:
        parts.append("q-" + urllib.parse.quote_plus(keyword.replace("-", " ").strip()))
    if location:
        parts.append("l-" + urllib.parse.quote_plus(location.replace("-", " ").strip()))
    if not parts:
        raise ValueError("need a keyword or a location")
    params = dict(filters or {})
    if page > 1:
        params["page"] = page
    url = f"{BASE}/jobs/{'-'.join(parts)}-jobs"
    return url + ("?" + urllib.parse.urlencode(params) if params else "")
 
def fetch(url, attempts=4, deadline_s=DEADLINE_S):
    """GET under one overall deadline. Retries 429/5xx, timeouts and connection
    errors with DELAYS backoff; raises at once on redirects, other HTTP errors
    and challenge pages."""
    if not 1 <= attempts <= len(DELAYS) + 1:
        raise ValueError(f"attempts must be 1..{len(DELAYS) + 1}")
    if not 0 < deadline_s <= 120:
        raise ValueError("deadline_s must be in (0, 120]")
    end = time.monotonic() + deadline_s
    for i in range(attempts):
        left = end - time.monotonic()
        if left <= 0:
            raise TimeoutError(f"{deadline_s}s deadline used up for {url}")
        try:
            req = urllib.request.Request(url, headers={"User-Agent": UA, "Accept": "text/html"})
            with urllib.request.urlopen(req, timeout=min(30, left)) as r:
                final, html = r.geturl(), r.read().decode("utf-8")
        except urllib.error.HTTPError as e:
            if e.code not in (429, 500, 502, 503, 504) or i == attempts - 1:
                raise
            err = f"HTTP {e.code}"
        except (urllib.error.URLError, TimeoutError, ConnectionError, http.client.HTTPException) as e:
            if i == attempts - 1:
                raise
            err = repr(e)
        else:
            if final != url:
                raise RuntimeError(f"redirected to {final}")
            if CHALLENGE.search(html):
                raise RuntimeError(f"challenge/block page at {url} - stop, do not retry")
            return html
        if time.monotonic() + DELAYS[i] >= end:
            raise TimeoutError(f"{deadline_s}s deadline reached for {url} after {i + 1} attempts ({err})")
        time.sleep(DELAYS[i])
 
def is_count(v):
    """A real non-negative int. bool is an int subclass, so True/False are rejected too."""
    return type(v) is int and v >= 0
 
def parse_joblist(html):
    """Concatenate the RSC strings, decode the object after "jobList":, validate it."""
    chunks = re.findall(r'self\.__next_f\.push\(\[1,(".*?")\]\)</script>', html, re.S)
    flight = "".join(json.loads(c) for c in chunks)
    i = flight.find('"jobList":')
    if i < 0:
        raise RuntimeError("no jobList in page (layout change or block page) - stop and inspect")
    jl = json.JSONDecoder().raw_decode(flight, i + len('"jobList":'))[0]
    meta, data = jl.get("meta"), jl.get("data")
    if (not isinstance(meta, dict) or not isinstance(data, list)
            or any(not is_count(meta.get(k)) for k in ("currentPage", "pageCount", "totalResults"))):
        raise RuntimeError("jobList without valid data/meta counts - layout change, stop")
    bad = sum(1 for j in data if not (j.get("guid") and j.get("title") and j.get("detailsPageUrl")))
    if bad:
        raise RuntimeError(f"{bad} jobs missing guid/title/detailsPageUrl - layout change, stop")
    return jl
 
def normalize(j):
    loc = j.get("jobLocation") or {}
    return {
        "title": j.get("title"),
        "company": j.get("companyName"),
        "location": loc.get("displayName"),
        "workplace": ", ".join(j.get("workplaceTypes") or []),
        "employment_type": j.get("employmentType"),
        "posted": j.get("postedDate"),
        "salary": j.get("salary"),
        "url": j.get("detailsPageUrl"),
        "guid": j.get("guid"),
    }
 
def search_dice(keyword=None, location=None, max_pages=3, filters=None):
    """Returns (jobs, info). info["status"] is never "complete": Dice serves no
    stable snapshot, so a matching count does not prove every job was seen.
    "count_matched" = every requested page came back as asked, totalResults stayed
    the same on every page and at least that many unique jobs were collected
    (still unverified, see reason); otherwise "partial" with a reason."""
    if max_pages < 1:
        raise ValueError("max_pages must be >= 1")
    jobs, seen, meta, reason, totals, counts = [], set(), {}, None, [], []
    for page in range(1, min(max_pages, MAX_PAGE) + 1):
        if page > 1:
            time.sleep(PAUSE_S)
        jl = parse_joblist(fetch(search_url(keyword, location, page, filters)))
        meta = jl["meta"]
        totals.append(meta["totalResults"])
        counts.append(meta["pageCount"])
        if meta["currentPage"] != page:      # count shrank or clamp: stop, do not guess
            reason = f"asked for page {page}, Dice returned page {meta['currentPage']}"
            break
        for j in jl["data"]:
            if j["guid"] not in seen:
                seen.add(j["guid"])
                jobs.append(normalize(j))
        if page >= meta["pageCount"]:
            break
    fetched = page if reason is None else page - 1
    # Largest count seen, so a total that shrank mid-crawl cannot make a short crawl look complete.
    total, pages = max(totals), max(counts)
    # A failed pagination step (reason already set) is always partial, whatever the count says.
    if reason is None and len(jobs) >= total and min(totals) == total:
        # Order reshuffles and postings can swap without changing the count,
        # so a matching count is the best signal available, not proof.
        status, reason = "count_matched", UNVERIFIED
    else:
        status = "partial"
        if reason is None and pages > fetched:
            reason = ("Dice 25-page cap (500 jobs max per search)" if fetched == MAX_PAGE
                      else f"max_pages={max_pages} reached")
        if reason is None and min(totals) != total:
            reason = f"totalResults changed during the crawl ({min(totals)} to {total})"
        reason = reason or "relevance order reshuffled between pages (repeats/skips)"
    return jobs, {"status": status, "reason": reason, "total": total, "page_count": pages,
                  "pages_fetched": fetched, "collected": len(jobs)}
 
jobs, info = search_dice("Python Developer", "Houston, TX", max_pages=3)
print(info)
for r in jobs[:3]:
    print(json.dumps(r))
 
# Narrowing with Dice's own filter params (values come from meta.facetQueryResults):
remote, rinfo = search_dice("DevOps Engineer", max_pages=2,
                            filters={"filters.workplaceTypes": "Remote"})
print("remote devops:", rinfo)
 
# Small search, every page: the best case is "count_matched", never "complete".
full, finfo = search_dice("Python Developer", "Houston, TX", max_pages=30)
print("houston all pages:", finfo)

import json, time, urllib.parse
 
def search_url(keyword=None, location=None, page=1):
    parts = []
    if keyword:
        parts.append("q-" + urllib.parse.quote_plus(keyword.replace("-", " ").strip()))
    if location:
        parts.append("l-" + urllib.parse.quote_plus(location.replace("-", " ").strip()))
    if not parts:
        raise ValueError("need a keyword or a location")
    url = "https://www.dice.com/jobs/" + "-".join(parts) + "-jobs"
    return url if page == 1 else f"{url}?page={page}"
 
READ_JOBLIST = r"""(()=>{
  const pre = 'self.__next_f.push([1,', parts = [];
  for (const s of document.querySelectorAll('script:not([src])')) {
    const t = s.textContent;
    if (!t.startsWith(pre)) continue;
    try { parts.push(JSON.parse(t.slice(pre.length, t.lastIndexOf(']')))); } catch (e) {}
  }
  const f = parts.join(''), i = f.indexOf('"jobList":');
  return i < 0 ? null : f.slice(i + 10);
})()"""
 
IS_CHALLENGE = r"""(()=>!!document.querySelector(
  'script[src*="challenge-platform"],#px-captcha,iframe[src*="captcha"],.g-recaptcha,.h-captcha')
  || /^(Access Denied|Just a moment|Attention Required)/i.test(document.title))()"""
 
def open_page(url, first=False):
    new_tab(url) if first else goto_url(url)
    wait_for_load()
    for _ in range(20):                      # confirm we are on the requested page (incl. query)
        if js("location.href") == url:
            break
        time.sleep(0.5)
    else:
        raise RuntimeError(f"expected {url}, got {js('location.href')}")
    if js(IS_CHALLENGE):
        raise RuntimeError(f"challenge/block page at {url} - stop and tell the user, do not retry")
 
def is_count(v):
    return type(v) is int and v >= 0      # rejects bools and negatives
 
def browser_search(keyword=None, location=None, max_pages=2):
    if max_pages < 1:
        raise ValueError("max_pages must be >= 1")
    rows, seen, meta, page, reason, totals, counts = [], set(), {}, 0, None, [], []
    for page in range(1, min(max_pages, 25) + 1):
        if page > 1:
            time.sleep(2)
        open_page(search_url(keyword, location, page), first=(page == 1))
        raw = js(READ_JOBLIST)
        if not raw:
            raise RuntimeError("no jobList on page - stop and inspect (layout change or block page)")
        jl = json.JSONDecoder().raw_decode(raw)[0]
        meta, data = jl.get("meta"), jl.get("data")
        if (not isinstance(meta, dict) or not isinstance(data, list)
                or any(not is_count(meta.get(k)) for k in ("currentPage", "pageCount", "totalResults"))):
            raise RuntimeError("jobList without valid data/meta counts - layout change, stop")
        if any(not (j.get("guid") and j.get("title") and j.get("detailsPageUrl")) for j in data):
            raise RuntimeError("job without guid/title/detailsPageUrl - layout change, stop")
        totals.append(meta["totalResults"])
        counts.append(meta["pageCount"])
        if meta["currentPage"] != page:
            reason = f"asked for page {page}, Dice returned page {meta['currentPage']}"
            page -= 1
            break
        for j in data:
            if j["guid"] in seen:
                continue
            seen.add(j["guid"])
            loc = j.get("jobLocation") or {}
            rows.append({"title": j["title"], "company": j.get("companyName"),
                         "location": loc.get("displayName"), "employment_type": j.get("employmentType"),
                         "posted": j.get("postedDate"), "url": j["detailsPageUrl"]})
        if page >= meta["pageCount"]:
            break
    total, pages = max(totals), max(counts)   # largest seen: a shrinking total cannot fake a match
    if reason is None and len(rows) >= total and min(totals) == total:   # page mismatch = partial
        # Never "complete": order reshuffles and postings can swap without changing the count.
        status = "count_matched"
        reason = "unique jobs reached totalResults, but Dice has no stable snapshot: not verified complete"
    else:
        status = "partial"
        if reason is None and pages > page:
            reason = "Dice 25-page cap" if page == 25 else f"max_pages={max_pages} reached"
        if reason is None and min(totals) != total:
            reason = f"totalResults changed during the crawl ({min(totals)} to {total})"
        reason = reason or "relevance order reshuffled between pages (repeats/skips)"
    return rows, {"status": status, "reason": reason, "total": total,
                  "pages_fetched": page, "collected": len(rows)}
 
rows, info = browser_search("DevOps Engineer", "Remote", max_pages=2)
print(info)
for r in rows[:3]:
    print(json.dumps(r))

import http.client, json, re, time, urllib.error, urllib.request
 
UA = "dice-jobs-research/1.0 (python-urllib; personal job search script)"
DELAYS = (5, 15, 40)
CHALLENGE = re.compile(r"cf-chl|/cdn-cgi/challenge-platform|px-captcha|captcha-delivery\.com|"
                       r"_Incapsula_Resource|g-recaptcha|h-captcha|"
                       r"<title>\s*(?:Access Denied|Just a moment|Attention Required)", re.I)
 
def fetch(url, attempts=4, deadline_s=120):
    """Same contract as section 1: one deadline, bounded retries, raise on challenge."""
    if not 1 <= attempts <= len(DELAYS) + 1 or not 0 < deadline_s <= 120:
        raise ValueError("attempts must be 1..4 and deadline_s in (0, 120]")
    end = time.monotonic() + deadline_s
    for i in range(attempts):
        left = end - time.monotonic()
        if left <= 0:
            raise TimeoutError(f"{deadline_s}s deadline used up for {url}")
        try:
            req = urllib.request.Request(url, headers={"User-Agent": UA, "Accept": "text/html"})
            with urllib.request.urlopen(req, timeout=min(30, left)) as r:
                final, html = r.geturl(), r.read().decode("utf-8")
        except urllib.error.HTTPError as e:
            if e.code not in (429, 500, 502, 503, 504) or i == attempts - 1:
                raise
        except (urllib.error.URLError, TimeoutError, ConnectionError, http.client.HTTPException):
            if i == attempts - 1:
                raise
        else:
            if final != url:
                raise RuntimeError(f"redirected to {final}")
            if CHALLENGE.search(html):
                raise RuntimeError(f"challenge/block page at {url} - stop, do not retry")
            return html
        if time.monotonic() + DELAYS[i] >= end:
            raise TimeoutError(f"{deadline_s}s deadline reached for {url} after {i + 1} attempts")
        time.sleep(DELAYS[i])
 
def job_detail(detail_url):
    html = fetch(detail_url)
    for block in re.findall(r'<script[^>]*application/ld\+json[^>]*>(.*?)</script>', html, re.S):
        d = json.loads(block)
        if isinstance(d, dict) and d.get("@type") == "JobPosting":
            missing = [k for k in ("title", "datePosted", "hiringOrganization", "description") if not d.get(k)]
            if missing:
                raise RuntimeError(f"JobPosting missing {missing} at {detail_url} - stop and inspect")
            return d
    if re.search(r"<title>Job Not Found \| Dice\.com</title>", html):
        return None
    raise RuntimeError(f"no JobPosting JSON-LD and not a Job Not Found page at {detail_url} - stop and inspect")
 
# Take a fresh URL from a search page so this does not depend on one posting staying live.
first = re.search(r'href="(/job-detail/[0-9a-f-]{36})"', fetch("https://www.dice.com/jobs/q-Python+Developer-jobs"))
if not first:
    raise RuntimeError("no job-detail link on the search page")
jp = job_detail("https://www.dice.com" + first.group(1))
if jp is None:
    print("posting is gone (Job Not Found)")
else:
    addr = (jp.get("jobLocation") or {}).get("address") or {}
    print(jp["title"], "|", jp["hiringOrganization"].get("name"), "|", jp.get("employmentType"),
          "|", jp["datePosted"], "|", jp.get("validThrough"), "|", addr.get("addressLocality"), addr.get("addressRegion"))
    print(len(jp["description"]), "chars of description HTML")
 
print(job_detail("https://www.dice.com/job-detail/00000000-0000-0000-0000-000000000000"))  # unknown guid