cURL

# Async scrape example. Requires Bash 5+, cURL, Python 3, and uuidgen.
# Export RUNHARVESTER_API_KEY before running. Keep the printed key and body for a manual retry.
set -euo pipefail

API_BASE_URL="${RUNHARVESTER_API_BASE_URL:-https://api.runharvester.com/api/v1/data}"
readonly API_KEY="${RUNHARVESTER_API_KEY:?Set RUNHARVESTER_API_KEY before running this example}"
readonly REQUEST_TIMEOUT_ENV="RUNHARVESTER_REQUEST_TIMEOUT_MS"
readonly POLL_INTERVAL_ENV="RUNHARVESTER_POLL_INTERVAL_MS"
readonly IDEMPOTENCY_KEY_ENV="RUNHARVESTER_IDEMPOTENCY_KEY"
readonly DEFAULT_TIMEOUT_MS=120000
readonly DEFAULT_POLL_INTERVAL_MS=1000
readonly CREATE_ATTEMPT_LIMIT=2
readonly RETRY_DELAY_MS=250

while [[ "$API_BASE_URL" == */ ]]; do API_BASE_URL="${API_BASE_URL%/}"; done
request_timeout_ms="${!REQUEST_TIMEOUT_ENV:-$DEFAULT_TIMEOUT_MS}"
poll_interval_ms="${!POLL_INTERVAL_ENV:-$DEFAULT_POLL_INTERVAL_MS}"
if [[ ! "$request_timeout_ms" =~ ^[1-9][0-9]*$ || ! "$poll_interval_ms" =~ ^[1-9][0-9]*$ ]]; then
    printf 'Timeout and poll interval must be positive integers in milliseconds.\n' >&2
    exit 1
fi

idempotency_key="${!IDEMPOTENCY_KEY_ENV:-$(uuidgen)}"
request_body='{"url":"https://example.com","mode":"auto"}'
printf 'Idempotency-Key: %s\n' "$idempotency_key"

now_ms() {
    local value="${EPOCHREALTIME/./}"
    printf '%s' "${value:0:13}"
}

deadline_ms=$(( $(now_ms) + request_timeout_ms ))

remaining_ms() {
    local remaining=$(( deadline_ms - $(now_ms) ))
    (( remaining > 0 )) || return 1
    printf '%s' "$remaining"
}

sleep_ms() {
    local duration_ms="$1"
    sleep "$(printf '%d.%03d' "$((duration_ms / 1000))" "$((duration_ms % 1000))")"
}

retry_delay() {
    local remaining
    remaining="$(remaining_ms)" || return 1
    if (( remaining < RETRY_DELAY_MS )); then
        sleep_ms "$remaining"
    else
        sleep_ms "$RETRY_DELAY_MS"
    fi
}

json_field() {
    python3 -c 'import json, sys; print(json.load(sys.stdin).get(sys.argv[1], ""))' "$1" <<< "$2"
}

send_request() {
    local method="$1"
    local path="$2"
    local body="${3:-}"
    local remaining
    local request_timeout
    local output
    local -a curl_args

    remaining="$(remaining_ms)" || return 2
    request_timeout="$remaining"
    if (( request_timeout > 15000 )); then request_timeout=15000; fi
    curl_args=(
        --silent --show-error
        --max-time "$(printf '%d.%03d' "$((request_timeout / 1000))" "$((request_timeout % 1000))")"
        --request "$method"
        --header 'Accept: application/json'
        --header "X-API-Key: $API_KEY"
    )
    if [[ "$method" == "POST" ]]; then
        curl_args+=(
            --header 'Content-Type: application/json'
            --header "Idempotency-Key: $idempotency_key"
            --data "$body"
        )
    fi
    if ! output="$(curl "${curl_args[@]}" --write-out $'\n%{http_code}' "$API_BASE_URL$path")"; then
        return 1
    fi
    HTTP_STATUS="${output##*$'\n'}"
    HTTP_BODY="${output%$'\n'*}"
}

create_response_received=false
for ((attempt = 1; attempt <= CREATE_ATTEMPT_LIMIT; attempt += 1)); do
    if ! send_request POST /requests "$request_body"; then
        if (( attempt < CREATE_ATTEMPT_LIMIT )); then
            if ! retry_delay; then
                printf 'Request exceeded its deadline.\n' >&2
                exit 1
            fi
            continue
        fi
        printf 'Could not create request after retrying with the same key and body.\n' >&2
        exit 1
    fi
    if [[ "$HTTP_STATUS" =~ ^(502|503|504)$ ]] && (( attempt < CREATE_ATTEMPT_LIMIT )); then
        if ! retry_delay; then
            printf 'Request exceeded its deadline.\n' >&2
            exit 1
        fi
        continue
    fi
    if [[ "$HTTP_STATUS" != "202" ]]; then
        printf 'Create request failed with HTTP %s: %s\n' "$HTTP_STATUS" "$HTTP_BODY" >&2
        exit 1
    fi
    job_id="$(json_field id "$HTTP_BODY")"
    if [[ -z "$job_id" ]]; then
        printf 'Create response did not include a request id.\n' >&2
        exit 1
    fi
    create_response_received=true
    break
done
if [[ "$create_response_received" != true ]]; then
    printf 'Could not create request.\n' >&2
    exit 1
fi

while true; do
    if ! send_request GET "/requests/$job_id"; then
        if ! remaining_ms >/dev/null; then
            printf 'Request exceeded its deadline.\n' >&2
            exit 1
        fi
        printf 'Could not read request status.\n' >&2
        exit 1
    fi
    if [[ "$HTTP_STATUS" != "200" ]]; then
        printf 'Status request failed with HTTP %s: %s\n' "$HTTP_STATUS" "$HTTP_BODY" >&2
        exit 1
    fi
    request_status="$(json_field status "$HTTP_BODY")"
    case "$request_status" in
        completed)
            break
            ;;
        failed|cancelled)
            printf 'Request ended with status %s: %s\n' "$request_status" "$HTTP_BODY" >&2
            exit 1
            ;;
        pending|running|solving_captcha)
            remaining="$(remaining_ms)" || {
                printf 'Request exceeded its deadline.\n' >&2
                exit 1
            }
            if (( remaining < poll_interval_ms )); then
                sleep_ms "$remaining"
            else
                sleep_ms "$poll_interval_ms"
            fi
            ;;
        *)
            printf 'Unknown request status: %s\n' "$request_status" >&2
            exit 1
            ;;
    esac
done

if ! send_request GET "/requests/$job_id/result"; then
    if ! remaining_ms >/dev/null; then
        printf 'Request exceeded its deadline.\n' >&2
        exit 1
    fi
    printf 'Could not retrieve request result.\n' >&2
    exit 1
fi
if [[ "$HTTP_STATUS" != "200" ]]; then
    printf 'Result request failed with HTTP %s: %s\n' "$HTTP_STATUS" "$HTTP_BODY" >&2
    exit 1
fi
printf '%s\n' "$HTTP_BODY"