Automate a Website With No API: Log In and Download Invoices Daily

// invoices.mjs
import { chromium } from "playwright";
 
const url = "https://portal.example.com/invoices";
const login = process.env.LOGIN === "1";
const context = await chromium.launchPersistentContext("./portal-profile", { headless: !login });
const page = await context.newPage();
await page.goto(url);
if (login) {
  await page.pause(); // log in by hand, then press Resume
  await page.goto(url);
}
if (page.url().includes("/login")) throw new Error("session expired: run again with LOGIN=1");
 
const links = page.getByRole("link", { name: "Download PDF" });
await links.first().waitFor(); // fails the run if no invoice links show up
for (const link of await links.all()) {
  const downloadPromise = page.waitForEvent("download"); // start waiting before the click
  await link.click();
  const download = await downloadPromise;
  await download.saveAs(`invoices/${download.suggestedFilename()}`);
}
await context.close(); // downloads are deleted when the context closes, so save first

# daily_invoices.py
import os, pathlib, time, requests
from browser_use_sdk.v4 import BrowserUse
 
API = "https://api.browser-use.com/api/v4"
KEY = {"X-Browser-Use-API-Key": os.environ["BROWSER_USE_API_KEY"]}
 
def op(alias, field):  # one 1Password field, typed only on the portal's host
    return {"alias": alias, "allowedDomains": ["portal.example.com"],
            "source": {"type": "onepassword", "integrationId": os.environ["OP_INTEGRATION_ID"],
                       "vaultId": os.environ["OP_VAULT_ID"], "itemId": os.environ["OP_ITEM_ID"],
                       "fieldId": field}}
 
def get(path, **params):
    r = requests.get(f"{API}{path}", headers=KEY, params=params, timeout=60)
    r.raise_for_status()
    return r.json()
 
def downloads(browser_id):  # 50 files per page; follow the cursor
    files, params = [], {"includeUrls": "true"}
    while True:
        page = get(f"/browsers/{browser_id}/downloads", **params)
        files += page["files"]
        if not page.get("hasMore"):
            return files
        params["cursor"] = page["nextCursor"]
 
with BrowserUse() as client:
    run = client.runs.create(
        "Open https://portal.example.com/invoices. If asked to log in, use portal_user, "
        "portal_pass and portal_otp. Download every invoice dated yesterday as PDF "
        "and report the invoice numbers.",
        browser_settings={"profileId": os.environ["PORTAL_PROFILE_ID"]},
        secret_bindings=[op("portal_user", "username"), op("portal_pass", "password"),
                         op("portal_otp", "one-time-password")],
    )
    done = client.runs.wait_for_completion(run.id)
 
if done.status.value != "completed":
    raise SystemExit(f"run failed: {done.error or done.status.value}")
 
browsers = get("/browsers", agentSessionId=run.session_id)["items"]
files, last = [], None
for _ in range(18):  # files can land after the run ends: wait until two polls agree (max 3 min)
    files = [f for b in browsers for f in downloads(b["id"])]
    if files and len(files) == last:
        break
    last = len(files)
    time.sleep(10)
if not files or any(f["size"] == 0 for f in files):  # drop if some days have no invoices
    raise SystemExit(f"no invoice files; agent said: {done.result}")
 
out = pathlib.Path("invoices")
out.mkdir(exist_ok=True)
for f in files:  # download links expire after 15 minutes
    r = requests.get(f["url"], timeout=120)
    r.raise_for_status()
    (out / f["path"]).write_bytes(r.content)
print(done.result)

# .github/workflows/invoices.yml
name: Pull invoices
on:
  schedule:
    - cron: "17 6 * * *" # 06:17 UTC daily
  workflow_dispatch:
jobs:
  invoices:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - run: pip install browser-use-sdk requests
      - run: python daily_invoices.py
        env:
          BROWSER_USE_API_KEY: ${{ secrets.BROWSER_USE_API_KEY }}
          PORTAL_PROFILE_ID: ${{ secrets.PORTAL_PROFILE_ID }}
          OP_INTEGRATION_ID: ${{ secrets.OP_INTEGRATION_ID }}
          OP_VAULT_ID: ${{ secrets.OP_VAULT_ID }}
          OP_ITEM_ID: ${{ secrets.OP_ITEM_ID }}