How to scrape in bulk with async requests

curl -H "Authorization: Bearer YOUR_API_KEY" \
  -H "Content-Type: application/json" \
  -d '{"input":[{"user_name":"zoobarcelona"}],"limit_per_input":null}' \
  "https://api.brightdata.com/datasets/v3/trigger?dataset_id=gd_l1vikfch901nx3by4&notify=false&include_errors=true&type=discover_new&discover_by=user_name"

import requests

response = requests.post(
    "https://api.brightdata.com/datasets/v3/trigger",
    params={
        "dataset_id": "gd_l1vikfch901nx3by4",
        "notify": "false",
        "include_errors": "true",
        "type": "discover_new",
        "discover_by": "user_name",
    },
    headers={"Authorization": "Bearer YOUR_API_KEY"},
    json={"input": [{"user_name": "zoobarcelona"}], "limit_per_input": None},
)

snapshot = response.json()
print("Snapshot ID:", snapshot["snapshot_id"])

import axios from "axios";

const response = await axios.post(
  "https://api.brightdata.com/datasets/v3/trigger",
  { input: [{ user_name: "zoobarcelona" }], limit_per_input: null },
  {
    params: {
      dataset_id: "gd_l1vikfch901nx3by4",
      notify: false,
      include_errors: true,
      type: "discover_new",
      discover_by: "user_name",
    },
    headers: { Authorization: "Bearer YOUR_API_KEY" },
  }
);

console.log("Snapshot ID:", response.data.snapshot_id);

const params = new URLSearchParams({
  dataset_id: "gd_l1vikfch901nx3by4",
  notify: "false",
  include_errors: "true",
  type: "discover_new",
  discover_by: "user_name",
});

const response = await fetch(`https://api.brightdata.com/datasets/v3/trigger?${params}`, {
  method: "POST",
  headers: {
    "Authorization": "Bearer YOUR_API_KEY",
    "Content-Type": "application/json",
  },
  body: JSON.stringify({ input: [{ user_name: "zoobarcelona" }], limit_per_input: null }),
});

const snapshot = await response.json();
console.log("Snapshot ID:", snapshot.snapshot_id);

curl "https://api.brightdata.com/datasets/v3/progress/sd_m1a2b3c4d5e6f7g8h" \
  -H "Authorization: Bearer YOUR_API_KEY"

import time
import requests

snapshot_id = "sd_m1a2b3c4d5e6f7g8h"

for attempt in range(60):  # stop after about 10 minutes
    status_response = requests.get(
        f"https://api.brightdata.com/datasets/v3/progress/{snapshot_id}",
        headers={"Authorization": "Bearer YOUR_API_KEY"},
    )
    status_response.raise_for_status()
    progress = status_response.json()
    status = progress.get("status")
    print(f"Status: {status}")

    if status == "ready":
        break
    if status in ("failed", "canceled"):
        raise RuntimeError(f"Snapshot {status}: {progress.get('error_message', 'no details')}")
    time.sleep(10)
else:
    raise TimeoutError("Snapshot was not ready after 10 minutes")

const snapshotId = "sd_m1a2b3c4d5e6f7g8h";
const maxAttempts = 60; // stop after about 10 minutes

for (let attempt = 1; ; attempt++) {
  const statusResponse = await fetch(
    `https://api.brightdata.com/datasets/v3/progress/${snapshotId}`,
    { headers: { "Authorization": "Bearer YOUR_API_KEY" } }
  );
  if (!statusResponse.ok) throw new Error(`HTTP ${statusResponse.status}`);
  const progress = await statusResponse.json();
  console.log("Status:", progress.status);

  if (progress.status === "ready") break;
  if (progress.status === "failed" || progress.status === "canceled") {
    throw new Error(`Snapshot ${progress.status}: ${progress.error_message ?? "no details"}`);
  }
  if (attempt === maxAttempts) throw new Error("Snapshot was not ready after 10 minutes");
  await new Promise((r) => setTimeout(r, 10000));
}

curl "https://api.brightdata.com/datasets/v3/snapshot/sd_m1a2b3c4d5e6f7g8h?format=json" \
  -H "Authorization: Bearer YOUR_API_KEY" \
  -o results.json

results_response = requests.get(
    f"https://api.brightdata.com/datasets/v3/snapshot/{snapshot_id}",
    params={"format": "json"},
    headers={"Authorization": "Bearer YOUR_API_KEY"},
)

results = results_response.json()
print(f"Collected {len(results)} records")

const resultsResponse = await fetch(
  `https://api.brightdata.com/datasets/v3/snapshot/${snapshotId}?format=json`,
  { headers: { "Authorization": "Bearer YOUR_API_KEY" } }
);

const results = await resultsResponse.json();
console.log(`Collected ${results.length} records`);

response = requests.post(api_url, json=data)

if response.status_code == 429:
    requests.post(api_url, json=data)  # retrying at once is what gets an IP blacklisted

import time
import requests

resp = requests.post(api_url, json=data)

if resp.status_code == 429:
    retry_after = resp.headers.get("Retry-After")
    wait_s = int(retry_after) if retry_after else 2
    time.sleep(wait_s)
    # retry after waiting, and double the wait if the 429 repeats