curl --request POST \
--url https://api.brightdata.com/datasets/v3/scrape \
--header 'Authorization: Bearer <token>' \
--header 'Content-Type: application/json' \
--data '
[
{
"url": "https://www.amazon.com/dp/B0FQFB8FMG"
}
]
'import requests
url = "https://api.brightdata.com/datasets/v3/scrape"
payload = [{ "url": "https://www.amazon.com/dp/B0FQFB8FMG" }]
headers = {
"Authorization": "Bearer <token>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
body: JSON.stringify([{url: 'https://www.amazon.com/dp/B0FQFB8FMG'}])
};
fetch('https://api.brightdata.com/datasets/v3/scrape', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.brightdata.com/datasets/v3/scrape",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
[
'url' => 'https://www.amazon.com/dp/B0FQFB8FMG'
]
]),
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>",
"Content-Type: application/json"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://api.brightdata.com/datasets/v3/scrape"
payload := strings.NewReader("[\n {\n \"url\": \"https://www.amazon.com/dp/B0FQFB8FMG\"\n }\n]")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("Authorization", "Bearer <token>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://api.brightdata.com/datasets/v3/scrape")
.header("Authorization", "Bearer <token>")
.header("Content-Type", "application/json")
.body("[\n {\n \"url\": \"https://www.amazon.com/dp/B0FQFB8FMG\"\n }\n]")
.asString();require 'uri'
require 'net/http'
url = URI("https://api.brightdata.com/datasets/v3/scrape")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "[\n {\n \"url\": \"https://www.amazon.com/dp/B0FQFB8FMG\"\n }\n]"
response = http.request(request)
puts response.read_body"OK"{
"snapshot_id": "sd_m1a2b3c4d5e6f7g8h",
"message": "Your request is still in progress and cannot be retrieved in this call. Use the provided Snapshot ID to track progress via the Monitor Snapshot endpoint and download it once ready via the Download Snapshot endpoint."
}Synchronous requests
Use the Bright Data Web Scraper API to synchronous Requests. POST /datasets/v3/scrape returns scraped data synchronously in a single response.
curl --request POST \
--url https://api.brightdata.com/datasets/v3/scrape \
--header 'Authorization: Bearer <token>' \
--header 'Content-Type: application/json' \
--data '
[
{
"url": "https://www.amazon.com/dp/B0FQFB8FMG"
}
]
'import requests
url = "https://api.brightdata.com/datasets/v3/scrape"
payload = [{ "url": "https://www.amazon.com/dp/B0FQFB8FMG" }]
headers = {
"Authorization": "Bearer <token>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
body: JSON.stringify([{url: 'https://www.amazon.com/dp/B0FQFB8FMG'}])
};
fetch('https://api.brightdata.com/datasets/v3/scrape', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.brightdata.com/datasets/v3/scrape",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
[
'url' => 'https://www.amazon.com/dp/B0FQFB8FMG'
]
]),
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>",
"Content-Type: application/json"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://api.brightdata.com/datasets/v3/scrape"
payload := strings.NewReader("[\n {\n \"url\": \"https://www.amazon.com/dp/B0FQFB8FMG\"\n }\n]")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("Authorization", "Bearer <token>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://api.brightdata.com/datasets/v3/scrape")
.header("Authorization", "Bearer <token>")
.header("Content-Type", "application/json")
.body("[\n {\n \"url\": \"https://www.amazon.com/dp/B0FQFB8FMG\"\n }\n]")
.asString();require 'uri'
require 'net/http'
url = URI("https://api.brightdata.com/datasets/v3/scrape")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "[\n {\n \"url\": \"https://www.amazon.com/dp/B0FQFB8FMG\"\n }\n]"
response = http.request(request)
puts response.read_body"OK"{
"snapshot_id": "sd_m1a2b3c4d5e6f7g8h",
"message": "Your request is still in progress and cannot be retrieved in this call. Use the provided Snapshot ID to track progress via the Monitor Snapshot endpoint and download it once ready via the Download Snapshot endpoint."
}How It Works
This synchronous API endpoint allows users to send a scraping request and receive the results in real-time directly in the response, at the point of request - such as a terminal or application - without the need for external storage or manual downloads. This approach streamlines the data collection process by eliminating additional steps for retrieving results. You can specify the desired output format using the format parameter. If no format is provided, the response will default to JSON.Request body
POST /datasets/v3/scrape accepts the inputs in either of two shapes. Both return the same records.
A bare JSON array of input objects. Every quickstart and platform page uses this form:
[{"url": "https://www.amazon.com/dp/B0FQFB8FMG"}]
input array. Use this form when you also pass custom_output_fields to return only the fields you name, or limit_per_input to cap how many records each input returns:
{
"input": [{"url": "https://www.amazon.com/dp/B0FQFB8FMG"}],
"custom_output_fields": "title|final_price|url"
}
limit_per_input matters for discovery requests, whose result count is open-ended. This request discovers posts in a subreddit and stops at three:
{
"input": [{"url": "https://www.reddit.com/r/learnpython/"}],
"limit_per_input": 3
}
type=discover_new&discover_by=subreddit_url on the query string. The Control Panel’s generated cURL uses this same body shape. limit_per_input is ignored when passed as a query parameter on /scrape.
POST /datasets/v3/trigger accepts the same two shapes. The object form is also where a deliver block goes on that endpoint. See Asynchronous requests.
Timeout Limit
Please note that this synchronous request is subject to a 1 minute timeout limit. If the data retrieval process exceeds this limit, the API will return an HTTP 202 response, indicating that the request is still being processed. In such cases, you will receive a snapshot ID to monitor and retrieve the results asynchronously via the Monitor Snapshot and Download Snapshot endpoints. The 202 response carries aretry-after header (10 seconds at the time of writing) telling you how long to wait before polling.
Example response on timeout:
{
"snapshot_id": "sd_m1a2b3c4d5e6f7g8h",
"message": "Your request is still in progress and cannot be retrieved in this call. Use the provided Snapshot ID to track progress via the Monitor Snapshot endpoint and download it once ready via the Download Snapshot endpoint."
}
How to handle a 202 response
A/scrape request that runs past 1 minute answers HTTP 202 with a snapshot_id instead of records, and the job continues. Branch on the status code, then poll Monitor progress and fetch the records with Download snapshot. The polling loop is in How to scrape in bulk with async requests.
import requests
response = requests.post(
"https://api.brightdata.com/datasets/v3/scrape",
params={"dataset_id": "gd_l7q7dkf244hwjntr0", "format": "json"},
headers={"Authorization": "Bearer YOUR_API_KEY"},
json=[{"url": "https://www.amazon.com/dp/B0FQFB8FMG"}],
)
if response.status_code == 200:
print(f"Got {len(response.json())} results")
elif response.status_code == 202:
print(f"Job running. Snapshot ID: {response.json()['snapshot_id']}")
else:
response.raise_for_status()
Custom inputs
You can add custom fields to the input schema. Whatever you send in those fields is returned in the results for each record. Use this to:- Keep a unified output structure across different scrapers and datasets.
- Pass an
id,row_indexor any internal key so you can match results back to your original input rows.
Authorizations
Use your Bright Data API Key as a Bearer token in the Authorization header.
How to authenticate:
- Obtain your API Key from the Bright Data account settings at https://brightdata.com/cp/setting/users
- Include the API Key in the Authorization header of your requests
- Format:
Authorization: Bearer YOUR_API_KEY
Example:
Authorization: Bearer YOUR_API_KEY
Learn how to get your Bright Data API key: https://docs.brightdata.com/api-reference/authentication
Query Parameters
Dataset ID for which data collection is triggered.
List of output columns, separated by | (e.g., url|about.updated_on). Filters the response to include only the specified fields.
"url|about.updated_on"
Include errors report with the results.
Specifies the format of the response (default: ndjson).
ndjson, json, csv Body
- Only inputs · object[]
- Inputs with output filter · object
Response
OK
A JSON array of records. An empty array means the inputs produced no records; check the input URLs.
The response is of type string.
"OK"
Was this page helpful?