如何使用异步请求批量抓取数据

curl -X POST \
  "https://api.brightdata.com/datasets/v3/trigger?dataset_id=gd_l1viktl72bvl7bjuj0&format=json" \
  -H "Authorization: Bearer YOUR_API_KEY" \
  -H "Content-Type: application/json" \
  -d '[
    {"url": "https://www.linkedin.com/in/satyanadella"},
    {"url": "https://www.linkedin.com/in/jeffweiner08"},
    {"url": "https://www.linkedin.com/in/rbranson"},
    {"url": "https://www.linkedin.com/in/sherylsandberg"},
    {"url": "https://www.linkedin.com/in/raboram"}
  ]'

import requests

response = requests.post(
    "https://api.brightdata.com/datasets/v3/trigger",
    params={
        "dataset_id": "gd_l1viktl72bvl7bjuj0",
        "format": "json",
    },
    headers={
        "Authorization": "Bearer YOUR_API_KEY",
        "Content-Type": "application/json",
    },
    json=[
        {"url": "https://www.linkedin.com/in/satyanadella"},
        {"url": "https://www.linkedin.com/in/jeffweiner08"},
        {"url": "https://www.linkedin.com/in/rbranson"},
        {"url": "https://www.linkedin.com/in/sherylsandberg"},
        {"url": "https://www.linkedin.com/in/raboram"},
    ],
)

snapshot = response.json()
print("Snapshot ID:", snapshot["snapshot_id"])

const response = await fetch(
  "https://api.brightdata.com/datasets/v3/trigger?dataset_id=gd_l1viktl72bvl7bjuj0&format=json",
  {
    method: "POST",
    headers: {
      "Authorization": "Bearer YOUR_API_KEY",
      "Content-Type": "application/json",
    },
    body: JSON.stringify([
      { url: "https://www.linkedin.com/in/satyanadella" },
      { url: "https://www.linkedin.com/in/jeffweiner08" },
      { url: "https://www.linkedin.com/in/rbranson" },
      { url: "https://www.linkedin.com/in/sherylsandberg" },
      { url: "https://www.linkedin.com/in/raboram" },
    ]),
  }
);

const snapshot = await response.json();
console.log("Snapshot ID:", snapshot.snapshot_id);

curl "https://api.brightdata.com/datasets/v3/progress/sd_m1a2b3c4d5e6f7g8h" \
  -H "Authorization: Bearer YOUR_API_KEY"

import time

snapshot_id = "sd_m1a2b3c4d5e6f7g8h"

while True:
    status_response = requests.get(
        f"https://api.brightdata.com/datasets/v3/progress/{snapshot_id}",
        headers={"Authorization": "Bearer YOUR_API_KEY"},
    )
    status = status_response.json().get("status")
    print(f"Status: {status}")

    if status == "ready":
        break
    time.sleep(10)

const snapshotId = "sd_m1a2b3c4d5e6f7g8h";

let status = "starting";
while (status !== "ready") {
  const statusResponse = await fetch(
    `https://api.brightdata.com/datasets/v3/progress/${snapshotId}`,
    { headers: { "Authorization": "Bearer YOUR_API_KEY" } }
  );
  const statusData = await statusResponse.json();
  status = statusData.status;
  console.log("Status:", status);

  if (status !== "ready") {
    await new Promise((r) => setTimeout(r, 10000));
  }
}

curl "https://api.brightdata.com/datasets/v3/snapshot/sd_m1a2b3c4d5e6f7g8h?format=json" \
  -H "Authorization: Bearer YOUR_API_KEY" \
  -o results.json

results_response = requests.get(
    f"https://api.brightdata.com/datasets/v3/snapshot/{snapshot_id}",
    params={"format": "json"},
    headers={"Authorization": "Bearer YOUR_API_KEY"},
)

results = results_response.json()
print(f"Collected {len(results)} records")

const resultsResponse = await fetch(
  `https://api.brightdata.com/datasets/v3/snapshot/${snapshotId}?format=json`,
  { headers: { "Authorization": "Bearer YOUR_API_KEY" } }
);

const results = await resultsResponse.json();
console.log(`Collected ${results.length} records`);

response = requests.post(api_url, json=data)

if response.status_code == 429:
    requests.post(api_url, json=data)  # 立即重试正是导致 IP 被列入黑名单的原因

import time
import requests

resp = requests.post(api_url, json=data)

if resp.status_code == 429:
    retry_after = resp.headers.get("Retry-After")
    wait_s = int(retry_after) if retry_after else 2
    time.sleep(wait_s)
    # 等待后重试，如果 429 再次出现则将等待时间加倍