Scrape any web page by URL with Fetch API
curl --request POST \
--url https://api.brightdata.com/datasets/v3/scrape \
--header 'Authorization: Bearer <token>' \
--header 'Content-Type: application/json' \
--data '
{
"input": [
{
"url": "<string>",
"country": "<string>"
}
]
}
'import requests
url = "https://api.brightdata.com/datasets/v3/scrape"
payload = { "input": [
{
"url": "<string>",
"country": "<string>"
}
] }
headers = {
"Authorization": "Bearer <token>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
body: JSON.stringify({input: [{url: '<string>', country: '<string>'}]})
};
fetch('https://api.brightdata.com/datasets/v3/scrape', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.brightdata.com/datasets/v3/scrape",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'input' => [
[
'url' => '<string>',
'country' => '<string>'
]
]
]),
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>",
"Content-Type: application/json"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://api.brightdata.com/datasets/v3/scrape"
payload := strings.NewReader("{\n \"input\": [\n {\n \"url\": \"<string>\",\n \"country\": \"<string>\"\n }\n ]\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("Authorization", "Bearer <token>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://api.brightdata.com/datasets/v3/scrape")
.header("Authorization", "Bearer <token>")
.header("Content-Type", "application/json")
.body("{\n \"input\": [\n {\n \"url\": \"<string>\",\n \"country\": \"<string>\"\n }\n ]\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://api.brightdata.com/datasets/v3/scrape")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n \"input\": [\n {\n \"url\": \"<string>\",\n \"country\": \"<string>\"\n }\n ]\n}"
response = http.request(request)
puts response.read_body[
{
"url": "https://example.com",
"final_url": "https://example.com/",
"page_title": "Example Domain",
"country": "us",
"markdown": "# Example Domain\n\nThis domain is for use in illustrative examples in documents.",
"html2text": "Example Domain\n\nThis domain is for use in illustrative examples in documents.",
"page_html": "<!doctype html><html><head><title>Example Domain</title></head><body><div><h1>Example Domain</h1><p>This domain is for use in illustrative examples in documents.</p></div></body></html>",
"ld_json": null
}
]
Web Content APIs
Scrape any web page by URL with Fetch API
Use the Bright Data Fetch API scraper to collect any public web page by URL as Markdown, plain text, raw HTML or JSON-LD. Send POST /datasets/v3/scrape.
POST
/
datasets
/
v3
/
scrape
Scrape any web page by URL with Fetch API
curl --request POST \
--url https://api.brightdata.com/datasets/v3/scrape \
--header 'Authorization: Bearer <token>' \
--header 'Content-Type: application/json' \
--data '
{
"input": [
{
"url": "<string>",
"country": "<string>"
}
]
}
'import requests
url = "https://api.brightdata.com/datasets/v3/scrape"
payload = { "input": [
{
"url": "<string>",
"country": "<string>"
}
] }
headers = {
"Authorization": "Bearer <token>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
body: JSON.stringify({input: [{url: '<string>', country: '<string>'}]})
};
fetch('https://api.brightdata.com/datasets/v3/scrape', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.brightdata.com/datasets/v3/scrape",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'input' => [
[
'url' => '<string>',
'country' => '<string>'
]
]
]),
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>",
"Content-Type: application/json"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://api.brightdata.com/datasets/v3/scrape"
payload := strings.NewReader("{\n \"input\": [\n {\n \"url\": \"<string>\",\n \"country\": \"<string>\"\n }\n ]\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("Authorization", "Bearer <token>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://api.brightdata.com/datasets/v3/scrape")
.header("Authorization", "Bearer <token>")
.header("Content-Type", "application/json")
.body("{\n \"input\": [\n {\n \"url\": \"<string>\",\n \"country\": \"<string>\"\n }\n ]\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://api.brightdata.com/datasets/v3/scrape")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n \"input\": [\n {\n \"url\": \"<string>\",\n \"country\": \"<string>\"\n }\n ]\n}"
response = http.request(request)
puts response.read_body[
{
"url": "https://example.com",
"final_url": "https://example.com/",
"page_title": "Example Domain",
"country": "us",
"markdown": "# Example Domain\n\nThis domain is for use in illustrative examples in documents.",
"html2text": "Example Domain\n\nThis domain is for use in illustrative examples in documents.",
"page_html": "<!doctype html><html><head><title>Example Domain</title></head><body><div><h1>Example Domain</h1><p>This domain is for use in illustrative examples in documents.</p></div></body></html>",
"ld_json": null
}
]
Fetch API (formerly Crawl API) is the Scraper API equivalent of Web Unlocker API. Send any public URL and get back the page as Markdown, plain text, raw HTML and JSON-LD in a single record.
To get only some of these fields, set
Query Parameters
string
default:"gd_m6gjtfmeh43we6cqc"
required
The dataset ID used for this request.
Must be set to
gd_m6gjtfmeh43we6cqc to collect Fetch API - Collect by URL data.boolean
default:false
Whether to send notifications when the request is completed.
boolean
default:true
Whether to include errors in the response.
string
Return only the listed output fields, separated by a pipe (
|). For example, url|markdown returns only the URL and the Markdown version of the page.Request Body
object[]
required
An array of input objects.
Show properties
Show properties
Example
{
"input": [
{"url": "https://example.com"},
{"url": "https://example.com/1", "country": "us"}
]
}
Output fields
Each record holds these fields:| Field | Type | Description |
|---|---|---|
url | url | The input URL |
final_url | url | The URL after redirects |
page_title | text | The page title |
country | country | The country the page was fetched from |
markdown | Markdown | Page content as Markdown, keeping headings, lists and links |
html2text | text | Page text with all HTML tags and formatting removed |
page_html | HTML | The complete HTML source, including tags, scripts and styles |
ld_json | JSON | Structured JSON-LD metadata from the page, such as schema.org data |
custom_output_fields, for example custom_output_fields=url|markdown.
[
{
"url": "https://example.com",
"final_url": "https://example.com/",
"page_title": "Example Domain",
"country": "us",
"markdown": "# Example Domain\n\nThis domain is for use in illustrative examples in documents.",
"html2text": "Example Domain\n\nThis domain is for use in illustrative examples in documents.",
"page_html": "<!doctype html><html><head><title>Example Domain</title></head><body><div><h1>Example Domain</h1><p>This domain is for use in illustrative examples in documents.</p></div></body></html>",
"ld_json": null
}
]
Was this page helpful?