TypeScript / JavaScript

// Async scrape example. Requires Bun; Node.js with Web Crypto and fetch also works.
// Export RUNHARVESTER_API_KEY before running. Keep the printed key and body for a manual retry.
const API_BASE_URL_ENV = "RUNHARVESTER_API_BASE_URL";
const API_KEY_ENV = "RUNHARVESTER_API_KEY";
const IDEMPOTENCY_KEY_ENV = "RUNHARVESTER_IDEMPOTENCY_KEY";
const REQUEST_TIMEOUT_ENV = "RUNHARVESTER_REQUEST_TIMEOUT_MS";
const POLL_INTERVAL_ENV = "RUNHARVESTER_POLL_INTERVAL_MS";
const DEFAULT_API_BASE_URL = "https://api.runharvester.com/api/v1/data";
const DEFAULT_REQUEST_TIMEOUT_MS = 120000;
const DEFAULT_POLL_INTERVAL_MS = 1000;
const HTTP_TIMEOUT_MS = 15000;
const CREATE_ATTEMPT_LIMIT = 2;
const RETRY_DELAY_MS = 250;
const RETRYABLE_STATUS_CODES = new Set([502, 503, 504]);
const ACTIVE_STATUSES = new Set(["pending", "running", "solving_captcha"]);

const apiBaseUrl = (process.env[API_BASE_URL_ENV] ?? DEFAULT_API_BASE_URL).replace(/\/+$/, "");
const apiKey = process.env[API_KEY_ENV];
if (!apiKey) throw new Error("Set RUNHARVESTER_API_KEY before running this async request example.");

const idempotencyKey = process.env[IDEMPOTENCY_KEY_ENV] ?? crypto.randomUUID();
const requestBody = JSON.stringify({ url: "https://example.com", mode: "auto" });
const headers = {
  Accept: "application/json",
  "X-API-Key": apiKey,
  "Content-Type": "application/json",
  "Idempotency-Key": idempotencyKey,
};
console.log("Idempotency-Key:", idempotencyKey);

const requestTimeoutMs = Number(process.env[REQUEST_TIMEOUT_ENV] ?? DEFAULT_REQUEST_TIMEOUT_MS);
const pollIntervalMs = Number(process.env[POLL_INTERVAL_ENV] ?? DEFAULT_POLL_INTERVAL_MS);
if (!Number.isFinite(requestTimeoutMs) || requestTimeoutMs <= 0 || !Number.isFinite(pollIntervalMs) || pollIntervalMs <= 0) {
  throw new Error("Timeout and poll interval must be positive numbers in milliseconds.");
}
const deadline = Date.now() + requestTimeoutMs;

const delay = (milliseconds: number) => new Promise((resolve) => setTimeout(resolve, milliseconds));
const request = (url: string, init: RequestInit = {}) => {
  const remainingMs = deadline - Date.now();
  if (remainingMs <= 0) throw new Error("Request exceeded its deadline.");
  return fetch(url, {
    ...init,
    signal: AbortSignal.timeout(Math.max(1, Math.min(HTTP_TIMEOUT_MS, remainingMs))),
  });
};
const readJson = async (response: Response) => {
  const text = await response.text();
  let payload: unknown;
  try {
    payload = JSON.parse(text);
  } catch {
    payload = text;
  }
  if (!response.ok) throw new Error("HTTP " + response.status + ": " + JSON.stringify(payload));
  return payload;
};

let jobId = "";
for (let attempt = 1; attempt <= CREATE_ATTEMPT_LIMIT; attempt += 1) {
  let response: Response;
  try {
    response = await request(apiBaseUrl + "/requests", { method: "POST", headers, body: requestBody });
  } catch (error) {
    if (attempt === CREATE_ATTEMPT_LIMIT) throw error;
    await delay(Math.min(RETRY_DELAY_MS, Math.max(0, deadline - Date.now())));
    continue;
  }
  if (RETRYABLE_STATUS_CODES.has(response.status) && attempt < CREATE_ATTEMPT_LIMIT) {
    await response.body?.cancel();
    await delay(Math.min(RETRY_DELAY_MS, Math.max(0, deadline - Date.now())));
    continue;
  }
  const job = await readJson(response) as { id?: string };
  if (response.status !== 202 || !job.id) throw new Error("Create response did not include an accepted request id.");
  jobId = job.id;
  break;
}
if (!jobId) throw new Error("Could not create request.");

while (true) {
  const statusResponse = await request(apiBaseUrl + "/requests/" + encodeURIComponent(jobId), {
    headers: { Accept: "application/json", "X-API-Key": apiKey },
  });
  const statusPayload = await readJson(statusResponse) as { status?: string; error?: unknown; user_error?: unknown };
  const status = statusPayload.status;
  if (status === "completed") break;
  if (status === "failed" || status === "cancelled") {
    const error = statusPayload.error ?? statusPayload.user_error ?? "";
    throw new Error("Request ended with status " + status + ": " + JSON.stringify(error));
  }
  if (!status || !ACTIVE_STATUSES.has(status)) throw new Error("Unknown request status: " + String(status));
  const remainingMs = deadline - Date.now();
  if (remainingMs <= 0) throw new Error("Request exceeded its deadline.");
  await delay(Math.min(pollIntervalMs, remainingMs));
}

const resultResponse = await request(apiBaseUrl + "/requests/" + encodeURIComponent(jobId) + "/result", {
  headers: { Accept: "application/json", "X-API-Key": apiKey },
});
const result = await readJson(resultResponse) as { data?: unknown };
console.log(JSON.stringify(result, null, 2));