Python

# Async scrape example. Requires Python 3; uses only the standard library.
# Set RUNHARVESTER_API_KEY before running. Keep the printed key and body for a manual retry.
import json
import os
import time
import urllib.error
import urllib.parse
import urllib.request
import uuid

API_BASE_URL_ENV = "RUNHARVESTER_API_BASE_URL"
API_KEY_ENV = "RUNHARVESTER_API_KEY"
IDEMPOTENCY_KEY_ENV = "RUNHARVESTER_IDEMPOTENCY_KEY"
REQUEST_TIMEOUT_ENV = "RUNHARVESTER_REQUEST_TIMEOUT_MS"
POLL_INTERVAL_ENV = "RUNHARVESTER_POLL_INTERVAL_MS"
DEFAULT_API_BASE_URL = "https://api.runharvester.com/api/v1/data"
DEFAULT_REQUEST_TIMEOUT_MS = 120000
DEFAULT_POLL_INTERVAL_MS = 1000
HTTP_TIMEOUT_MS = 15000
CREATE_ATTEMPT_LIMIT = 2
RETRY_DELAY_MS = 250
RETRYABLE_STATUS_CODES = {502, 503, 504}
ACTIVE_STATUSES = {"pending", "running", "solving_captcha"}

api_base_url = os.environ.get(API_BASE_URL_ENV, DEFAULT_API_BASE_URL).rstrip("/")
api_key = os.environ.get(API_KEY_ENV)
if not api_key:
    raise RuntimeError("Set RUNHARVESTER_API_KEY before running this async request example.")

idempotency_key = os.environ.get(IDEMPOTENCY_KEY_ENV) or str(uuid.uuid4())
request_body = json.dumps(
    {"url": "https://example.com", "mode": "auto"}, separators=(",", ":")
).encode("utf-8")
print("Idempotency-Key:", idempotency_key)

request_timeout_ms = int(os.environ.get(REQUEST_TIMEOUT_ENV, DEFAULT_REQUEST_TIMEOUT_MS))
poll_interval_ms = int(os.environ.get(POLL_INTERVAL_ENV, DEFAULT_POLL_INTERVAL_MS))
if request_timeout_ms <= 0 or poll_interval_ms <= 0:
    raise RuntimeError("Timeout and poll interval must be positive numbers in milliseconds.")
deadline = time.monotonic() + request_timeout_ms / 1000

def api_request(method, path, body=None):
    remaining = deadline - time.monotonic()
    if remaining <= 0:
        raise TimeoutError("Request exceeded its deadline.")
    headers = {"Accept": "application/json", "X-API-Key": api_key}
    if body is not None:
        headers["Content-Type"] = "application/json"
        headers["Idempotency-Key"] = idempotency_key
    request = urllib.request.Request(api_base_url + path, data=body, headers=headers, method=method)
    timeout = min(HTTP_TIMEOUT_MS / 1000, remaining)
    try:
        response = urllib.request.urlopen(request, timeout=timeout)
    except urllib.error.HTTPError as error:
        response = error
    with response:
        status_code = response.status
        response_body = response.read()
    try:
        payload = json.loads(response_body)
    except json.JSONDecodeError:
        payload = response_body.decode("utf-8", errors="replace")
    return status_code, payload

job_id = None
for attempt in range(1, CREATE_ATTEMPT_LIMIT + 1):
    try:
        status_code, payload = api_request("POST", "/requests", request_body)
    except (TimeoutError, urllib.error.URLError) as error:
        if attempt == CREATE_ATTEMPT_LIMIT:
            raise RuntimeError("Could not create request after retrying with the same key and body.") from error
        time.sleep(min(RETRY_DELAY_MS / 1000, max(0, deadline - time.monotonic())))
        continue
    if status_code in RETRYABLE_STATUS_CODES and attempt < CREATE_ATTEMPT_LIMIT:
        time.sleep(min(RETRY_DELAY_MS / 1000, max(0, deadline - time.monotonic())))
        continue
    if status_code != 202:
        raise RuntimeError("Create request failed with HTTP " + str(status_code) + ": " + str(payload))
    job_id = payload.get("id")
    if not job_id:
        raise RuntimeError("Create response did not include a request id.")
    break
if not job_id:
    raise RuntimeError("Could not create request.")

while True:
    status_code, status_payload = api_request("GET", "/requests/" + urllib.parse.quote(job_id, safe=""))
    if status_code != 200:
        raise RuntimeError("Status request failed with HTTP " + str(status_code) + ": " + str(status_payload))
    status = status_payload.get("status")
    if status == "completed":
        break
    if status in {"failed", "cancelled"}:
        raise RuntimeError("Request ended with status " + status + ": " + str(status_payload.get("error", "")))
    if status not in ACTIVE_STATUSES:
        raise RuntimeError("Unknown request status: " + str(status))
    remaining = deadline - time.monotonic()
    if remaining <= 0:
        raise TimeoutError("Request exceeded its deadline.")
    time.sleep(min(poll_interval_ms / 1000, remaining))

result_path = "/requests/" + urllib.parse.quote(job_id, safe="") + "/result"
status_code, result = api_request("GET", result_path)
if status_code != 200:
    raise RuntimeError("Result request failed with HTTP " + str(status_code) + ": " + str(result))
print(json.dumps(result, indent=2))