Scrape bot-protected sites (Cloudflare, PerimeterX, more)

curl --max-time 180 https://api.spicrawl.com/v1/scrape \
  -H "Authorization: Bearer $SPICRAWL_API_KEY" \
  -H "Content-Type: application/json" \
  -d '{"url": "https://example.com/community/thread/42", "proxy": "http://user:pass@your-proxy:8000", "response_format": "markdown"}' \
  -D -

curl --max-time 180 https://api.spicrawl.com/v1/scrape \
  -H "Authorization: Bearer $SPICRAWL_API_KEY" \
  -H "Content-Type: application/json" \
  -d '{"url": "https://example.com/community/thread/42", "js_render": true, "proxy": "http://user:pass@your-proxy:8000", "response_format": "pdf"}' \
  -o thread.pdf
head -c 5 thread.pdf   # %PDF-

curl --max-time 180 https://api.spicrawl.com/v1/scrape \
  -H "Authorization: Bearer $SPICRAWL_API_KEY" \
  -H "Content-Type: application/json" \
  -d '{"url": "https://example.com/community/thread/42", "js_render": true, "proxy": "http://user:pass@your-proxy:8000", "screenshot": true, "screenshot_fullpage": true}' \
  | jq -r '.screenshots[0].data' | base64 -d > thread.png

import os, time, requests
from concurrent.futures import ThreadPoolExecutor

URLS = ["https://example.com/pair/1", "https://example.com/pair/2"]

def fetch(url, tries=3):
    for _ in range(tries):
        r = requests.post(
            "https://api.spicrawl.com/v1/scrape",
            headers={"Authorization": f"Bearer {os.environ['SPICRAWL_API_KEY']}"},
            json={"url": url, "proxy": os.environ["MY_PROXY_URL"], "response_format": "markdown", "cache": False},
            timeout=180,
        )
        if r.ok:
            return url, int(r.headers["X-Credits-Charged"]), r.text
        problem = r.json() if r.headers.get("content-type", "").startswith("application/problem") else {}
        if not problem.get("retryable"):
            break
        time.sleep(int(r.headers.get("Retry-After", 2)))
    return url, 0, None

with ThreadPoolExecutor(max_workers=4) as pool:  # stay under Concurrency-Limit
    for url, credits, text in pool.map(fetch, URLS):
        print(url, credits, "ok" if text else "failed")