Bulk PDF downloads

curl -s "https://api.sustainabilityreports.com/api/v2/reports?limit=200" -H "Authorization: Bearer $SRC_API_KEY"
curl -s "https://api.sustainabilityreports.com/api/v2/reports?limit=200&cursor=<next_cursor>" -H "Authorization: Bearer $SRC_API_KEY"

{
  "data": [
    {"change": "upsert", "report_id": 529769, "report_key": "XMIL_G_2025_AR2", "company_key": "XMIL_G",
     "changed_at": "2026-10-08T03:12:44Z", "content_sha256": "a1d0c6e8...", "report": { "...": "full Report object" }},
    {"change": "withdrawn", "report_id": 98765, "report_key": "XMIL_G_2025_AR1", "company_key": "XMIL_G",
     "changed_at": "2026-10-08T04:01:02Z", "content_sha256": null, "report": null}
  ],
  "page_info": {"count": 2, "has_more": false, "next_cursor": "eyJ1IjoiMjAyNi0xMC0wOFQwNDowMTowMloiLCJpIjo5ODc2NX0"}
}

import hashlib
import os
import pathlib
import tempfile
import time
from datetime import datetime, timezone

import requests

BASE = "https://api.sustainabilityreports.com/api/v2"
OUT = pathlib.Path("pdfs")
S = requests.Session()
S.headers.update({
    "Authorization": f"Bearer {os.environ['SRC_API_KEY']}",
    "User-Agent": "example-sync/1.0",
})
TRANSIENT = {429, 500, 502, 503, 504}


class SyncError(Exception):
    pass


def call(method, path, **kw):
    """One API call, retried on 429, transient 5xx and network errors."""
    for attempt in range(6):
        try:
            r = S.request(method, BASE + path, timeout=60, **kw)
        except requests.RequestException:
            time.sleep(2 ** attempt)
            continue
        if r.status_code in TRANSIENT:
            time.sleep(int(r.headers.get("Retry-After", 2 ** attempt)))
            continue
        if r.status_code >= 400:
            raise SyncError(f"{method} {path}: HTTP {r.status_code} {r.text[:300]}")
        return r.json()
    raise SyncError(f"{method} {path}: gave up after retries")


def pages(path, params):
    """Yield items; the final next_cursor is returned as StopIteration.value."""
    params = dict(params)
    while True:
        page = call("GET", path, params=params)
        yield from page["data"]
        cursor = page["page_info"].get("next_cursor")
        if not page["page_info"]["has_more"]:
            return cursor
        params = {k: v for k, v in params.items() if k != "since"}
        params["cursor"] = cursor


def sha256_of(path):
    h = hashlib.sha256()
    with open(path, "rb") as f:
        for chunk in iter(lambda: f.read(1 << 20), b""):
            h.update(chunk)
    return h.hexdigest()


def download_pdf(report):
    """Fetch, verify and atomically store one PDF. Never logs the signed URL."""
    OUT.mkdir(exist_ok=True)
    target = OUT / report["filename"]
    expected = report.get("content_sha256")
    if target.exists() and expected and sha256_of(target) == expected:
        return target                                       # unchanged, skip
    for attempt in range(4):
        link = call("GET", f"/reports/{report['report_id']}/download")  # fresh link each try
        tmp = None
        try:
            with requests.get(link["url"], stream=True, timeout=300) as r:  # no API key here
                if r.status_code != 200:
                    raise SyncError(f"PDF fetch for report {report['report_id']}: HTTP {r.status_code}")
                fd, tmp = tempfile.mkstemp(dir=OUT, suffix=".part")
                with os.fdopen(fd, "wb") as f:
                    for chunk in r.iter_content(1 << 20):
                        f.write(chunk)
        except (requests.RequestException, SyncError):
            if tmp and os.path.exists(tmp):
                os.remove(tmp)                              # no partial files left behind
            time.sleep(2 ** attempt)
            continue
        if expected and sha256_of(tmp) != expected:
            os.remove(tmp)
            time.sleep(2 ** attempt)
            continue
        os.replace(tmp, target)                             # atomic
        return target
    raise SyncError(f"PDF for report {report['report_id']} failed verification or download")


def full_load():
    started = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
    for rep in pages("/reports", {"limit": 200}):
        store_metadata(rep)                                 # your code: upsert by report_id
        if rep["has_file"]:
            download_pdf(rep)
    save_state(since=started, cursor=None)                  # your code: persist both


def weekly_sync():
    state = load_state()                                    # your code
    params = {"cursor": state["cursor"]} if state["cursor"] else {"since": state["since"]}
    gen = pages("/reports/changes", {**params, "limit": 100})
    while True:
        try:
            item = next(gen)
        except StopIteration as stop:
            save_state(since=state["since"], cursor=stop.value)  # resume point for next time
            return
        if item["change"] == "withdrawn":
            mark_withdrawn(item["report_id"])               # your code; ignore unknown ids
        else:
            store_metadata(item["report"])
            if item["report"]["has_file"]:
                download_pdf(item["report"])