Run a search
curl --request POST \
--url https://api.brightdata.com/webarchive/search \
--header 'Authorization: Bearer <token>' \
--header 'Content-Type: application/json' \
--data '
{
"filters": {
"max_age": "Duration",
"min_date": "YYYY-MM-DD",
"max_date": "YYYY-MM-DD",
"domain_whitelist": [
"example.com"
],
"domain_blacklist": [
"example.com"
],
"domain_regex_whitelist": [
".*example..*"
],
"domain_regex_blacklist": [
".*example..*"
],
"domain_like_whitelist": [
"%.example.%",
"example%"
],
"domain_like_blacklist": [
"%.example.ca"
],
"category_whitelist": [
"Motor Vehicles"
],
"category_blacklist": [
"Motor Vehicles"
],
"url_regex_whitelist": [
".*/products/.*"
],
"url_regex_blacklist": [
".*/products/.*"
],
"url_like_whitelist": [
"%/products/%",
"%/search%"
],
"url_like_blacklist": [
"%/review/%"
],
"language_whitelist": [
"eng"
],
"language_blacklist": [
"eng"
],
"ip_country_whitelist": [
"us",
"ie",
"in"
],
"ip_country_blacklist": [
"mx",
"ae",
"ca"
],
"captcha": true,
"robots_block": true
}
}
'import requests
url = "https://api.brightdata.com/webarchive/search"
payload = { "filters": {
"max_age": "Duration",
"min_date": "YYYY-MM-DD",
"max_date": "YYYY-MM-DD",
"domain_whitelist": ["example.com"],
"domain_blacklist": ["example.com"],
"domain_regex_whitelist": [".*example..*"],
"domain_regex_blacklist": [".*example..*"],
"domain_like_whitelist": ["%.example.%", "example%"],
"domain_like_blacklist": ["%.example.ca"],
"category_whitelist": ["Motor Vehicles"],
"category_blacklist": ["Motor Vehicles"],
"url_regex_whitelist": [".*/products/.*"],
"url_regex_blacklist": [".*/products/.*"],
"url_like_whitelist": ["%/products/%", "%/search%"],
"url_like_blacklist": ["%/review/%"],
"language_whitelist": ["eng"],
"language_blacklist": ["eng"],
"ip_country_whitelist": ["us", "ie", "in"],
"ip_country_blacklist": ["mx", "ae", "ca"],
"captcha": True,
"robots_block": True
} }
headers = {
"Authorization": "Bearer <token>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
body: JSON.stringify({
filters: {
max_age: 'Duration',
min_date: 'YYYY-MM-DD',
max_date: 'YYYY-MM-DD',
domain_whitelist: ['example.com'],
domain_blacklist: ['example.com'],
domain_regex_whitelist: ['.*example..*'],
domain_regex_blacklist: ['.*example..*'],
domain_like_whitelist: ['%.example.%', 'example%'],
domain_like_blacklist: ['%.example.ca'],
category_whitelist: ['Motor Vehicles'],
category_blacklist: ['Motor Vehicles'],
url_regex_whitelist: ['.*/products/.*'],
url_regex_blacklist: ['.*/products/.*'],
url_like_whitelist: ['%/products/%', '%/search%'],
url_like_blacklist: ['%/review/%'],
language_whitelist: ['eng'],
language_blacklist: ['eng'],
ip_country_whitelist: ['us', 'ie', 'in'],
ip_country_blacklist: ['mx', 'ae', 'ca'],
captcha: true,
robots_block: true
}
})
};
fetch('https://api.brightdata.com/webarchive/search', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.brightdata.com/webarchive/search",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'filters' => [
'max_age' => 'Duration',
'min_date' => 'YYYY-MM-DD',
'max_date' => 'YYYY-MM-DD',
'domain_whitelist' => [
'example.com'
],
'domain_blacklist' => [
'example.com'
],
'domain_regex_whitelist' => [
'.*example..*'
],
'domain_regex_blacklist' => [
'.*example..*'
],
'domain_like_whitelist' => [
'%.example.%',
'example%'
],
'domain_like_blacklist' => [
'%.example.ca'
],
'category_whitelist' => [
'Motor Vehicles'
],
'category_blacklist' => [
'Motor Vehicles'
],
'url_regex_whitelist' => [
'.*/products/.*'
],
'url_regex_blacklist' => [
'.*/products/.*'
],
'url_like_whitelist' => [
'%/products/%',
'%/search%'
],
'url_like_blacklist' => [
'%/review/%'
],
'language_whitelist' => [
'eng'
],
'language_blacklist' => [
'eng'
],
'ip_country_whitelist' => [
'us',
'ie',
'in'
],
'ip_country_blacklist' => [
'mx',
'ae',
'ca'
],
'captcha' => true,
'robots_block' => true
]
]),
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>",
"Content-Type: application/json"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://api.brightdata.com/webarchive/search"
payload := strings.NewReader("{\n \"filters\": {\n \"max_age\": \"Duration\",\n \"min_date\": \"YYYY-MM-DD\",\n \"max_date\": \"YYYY-MM-DD\",\n \"domain_whitelist\": [\n \"example.com\"\n ],\n \"domain_blacklist\": [\n \"example.com\"\n ],\n \"domain_regex_whitelist\": [\n \".*example..*\"\n ],\n \"domain_regex_blacklist\": [\n \".*example..*\"\n ],\n \"domain_like_whitelist\": [\n \"%.example.%\",\n \"example%\"\n ],\n \"domain_like_blacklist\": [\n \"%.example.ca\"\n ],\n \"category_whitelist\": [\n \"Motor Vehicles\"\n ],\n \"category_blacklist\": [\n \"Motor Vehicles\"\n ],\n \"url_regex_whitelist\": [\n \".*/products/.*\"\n ],\n \"url_regex_blacklist\": [\n \".*/products/.*\"\n ],\n \"url_like_whitelist\": [\n \"%/products/%\",\n \"%/search%\"\n ],\n \"url_like_blacklist\": [\n \"%/review/%\"\n ],\n \"language_whitelist\": [\n \"eng\"\n ],\n \"language_blacklist\": [\n \"eng\"\n ],\n \"ip_country_whitelist\": [\n \"us\",\n \"ie\",\n \"in\"\n ],\n \"ip_country_blacklist\": [\n \"mx\",\n \"ae\",\n \"ca\"\n ],\n \"captcha\": true,\n \"robots_block\": true\n }\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("Authorization", "Bearer <token>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://api.brightdata.com/webarchive/search")
.header("Authorization", "Bearer <token>")
.header("Content-Type", "application/json")
.body("{\n \"filters\": {\n \"max_age\": \"Duration\",\n \"min_date\": \"YYYY-MM-DD\",\n \"max_date\": \"YYYY-MM-DD\",\n \"domain_whitelist\": [\n \"example.com\"\n ],\n \"domain_blacklist\": [\n \"example.com\"\n ],\n \"domain_regex_whitelist\": [\n \".*example..*\"\n ],\n \"domain_regex_blacklist\": [\n \".*example..*\"\n ],\n \"domain_like_whitelist\": [\n \"%.example.%\",\n \"example%\"\n ],\n \"domain_like_blacklist\": [\n \"%.example.ca\"\n ],\n \"category_whitelist\": [\n \"Motor Vehicles\"\n ],\n \"category_blacklist\": [\n \"Motor Vehicles\"\n ],\n \"url_regex_whitelist\": [\n \".*/products/.*\"\n ],\n \"url_regex_blacklist\": [\n \".*/products/.*\"\n ],\n \"url_like_whitelist\": [\n \"%/products/%\",\n \"%/search%\"\n ],\n \"url_like_blacklist\": [\n \"%/review/%\"\n ],\n \"language_whitelist\": [\n \"eng\"\n ],\n \"language_blacklist\": [\n \"eng\"\n ],\n \"ip_country_whitelist\": [\n \"us\",\n \"ie\",\n \"in\"\n ],\n \"ip_country_blacklist\": [\n \"mx\",\n \"ae\",\n \"ca\"\n ],\n \"captcha\": true,\n \"robots_block\": true\n }\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://api.brightdata.com/webarchive/search")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n \"filters\": {\n \"max_age\": \"Duration\",\n \"min_date\": \"YYYY-MM-DD\",\n \"max_date\": \"YYYY-MM-DD\",\n \"domain_whitelist\": [\n \"example.com\"\n ],\n \"domain_blacklist\": [\n \"example.com\"\n ],\n \"domain_regex_whitelist\": [\n \".*example..*\"\n ],\n \"domain_regex_blacklist\": [\n \".*example..*\"\n ],\n \"domain_like_whitelist\": [\n \"%.example.%\",\n \"example%\"\n ],\n \"domain_like_blacklist\": [\n \"%.example.ca\"\n ],\n \"category_whitelist\": [\n \"Motor Vehicles\"\n ],\n \"category_blacklist\": [\n \"Motor Vehicles\"\n ],\n \"url_regex_whitelist\": [\n \".*/products/.*\"\n ],\n \"url_regex_blacklist\": [\n \".*/products/.*\"\n ],\n \"url_like_whitelist\": [\n \"%/products/%\",\n \"%/search%\"\n ],\n \"url_like_blacklist\": [\n \"%/review/%\"\n ],\n \"language_whitelist\": [\n \"eng\"\n ],\n \"language_blacklist\": [\n \"eng\"\n ],\n \"ip_country_whitelist\": [\n \"us\",\n \"ie\",\n \"in\"\n ],\n \"ip_country_blacklist\": [\n \"mx\",\n \"ae\",\n \"ca\"\n ],\n \"captcha\": true,\n \"robots_block\": true\n }\n}"
response = http.request(request)
puts response.read_body{
"search_id": "ucd_abc123xyz"
}Archive API
Run a search
Use the Bright Data Marketplace Archive API to run a Search. POST /webarchive/search manages web archive snapshots; returns 200 OK with JSON status.
POST
/
webarchive
/
search
Run a search
curl --request POST \
--url https://api.brightdata.com/webarchive/search \
--header 'Authorization: Bearer <token>' \
--header 'Content-Type: application/json' \
--data '
{
"filters": {
"max_age": "Duration",
"min_date": "YYYY-MM-DD",
"max_date": "YYYY-MM-DD",
"domain_whitelist": [
"example.com"
],
"domain_blacklist": [
"example.com"
],
"domain_regex_whitelist": [
".*example..*"
],
"domain_regex_blacklist": [
".*example..*"
],
"domain_like_whitelist": [
"%.example.%",
"example%"
],
"domain_like_blacklist": [
"%.example.ca"
],
"category_whitelist": [
"Motor Vehicles"
],
"category_blacklist": [
"Motor Vehicles"
],
"url_regex_whitelist": [
".*/products/.*"
],
"url_regex_blacklist": [
".*/products/.*"
],
"url_like_whitelist": [
"%/products/%",
"%/search%"
],
"url_like_blacklist": [
"%/review/%"
],
"language_whitelist": [
"eng"
],
"language_blacklist": [
"eng"
],
"ip_country_whitelist": [
"us",
"ie",
"in"
],
"ip_country_blacklist": [
"mx",
"ae",
"ca"
],
"captcha": true,
"robots_block": true
}
}
'import requests
url = "https://api.brightdata.com/webarchive/search"
payload = { "filters": {
"max_age": "Duration",
"min_date": "YYYY-MM-DD",
"max_date": "YYYY-MM-DD",
"domain_whitelist": ["example.com"],
"domain_blacklist": ["example.com"],
"domain_regex_whitelist": [".*example..*"],
"domain_regex_blacklist": [".*example..*"],
"domain_like_whitelist": ["%.example.%", "example%"],
"domain_like_blacklist": ["%.example.ca"],
"category_whitelist": ["Motor Vehicles"],
"category_blacklist": ["Motor Vehicles"],
"url_regex_whitelist": [".*/products/.*"],
"url_regex_blacklist": [".*/products/.*"],
"url_like_whitelist": ["%/products/%", "%/search%"],
"url_like_blacklist": ["%/review/%"],
"language_whitelist": ["eng"],
"language_blacklist": ["eng"],
"ip_country_whitelist": ["us", "ie", "in"],
"ip_country_blacklist": ["mx", "ae", "ca"],
"captcha": True,
"robots_block": True
} }
headers = {
"Authorization": "Bearer <token>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
body: JSON.stringify({
filters: {
max_age: 'Duration',
min_date: 'YYYY-MM-DD',
max_date: 'YYYY-MM-DD',
domain_whitelist: ['example.com'],
domain_blacklist: ['example.com'],
domain_regex_whitelist: ['.*example..*'],
domain_regex_blacklist: ['.*example..*'],
domain_like_whitelist: ['%.example.%', 'example%'],
domain_like_blacklist: ['%.example.ca'],
category_whitelist: ['Motor Vehicles'],
category_blacklist: ['Motor Vehicles'],
url_regex_whitelist: ['.*/products/.*'],
url_regex_blacklist: ['.*/products/.*'],
url_like_whitelist: ['%/products/%', '%/search%'],
url_like_blacklist: ['%/review/%'],
language_whitelist: ['eng'],
language_blacklist: ['eng'],
ip_country_whitelist: ['us', 'ie', 'in'],
ip_country_blacklist: ['mx', 'ae', 'ca'],
captcha: true,
robots_block: true
}
})
};
fetch('https://api.brightdata.com/webarchive/search', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.brightdata.com/webarchive/search",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'filters' => [
'max_age' => 'Duration',
'min_date' => 'YYYY-MM-DD',
'max_date' => 'YYYY-MM-DD',
'domain_whitelist' => [
'example.com'
],
'domain_blacklist' => [
'example.com'
],
'domain_regex_whitelist' => [
'.*example..*'
],
'domain_regex_blacklist' => [
'.*example..*'
],
'domain_like_whitelist' => [
'%.example.%',
'example%'
],
'domain_like_blacklist' => [
'%.example.ca'
],
'category_whitelist' => [
'Motor Vehicles'
],
'category_blacklist' => [
'Motor Vehicles'
],
'url_regex_whitelist' => [
'.*/products/.*'
],
'url_regex_blacklist' => [
'.*/products/.*'
],
'url_like_whitelist' => [
'%/products/%',
'%/search%'
],
'url_like_blacklist' => [
'%/review/%'
],
'language_whitelist' => [
'eng'
],
'language_blacklist' => [
'eng'
],
'ip_country_whitelist' => [
'us',
'ie',
'in'
],
'ip_country_blacklist' => [
'mx',
'ae',
'ca'
],
'captcha' => true,
'robots_block' => true
]
]),
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>",
"Content-Type: application/json"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://api.brightdata.com/webarchive/search"
payload := strings.NewReader("{\n \"filters\": {\n \"max_age\": \"Duration\",\n \"min_date\": \"YYYY-MM-DD\",\n \"max_date\": \"YYYY-MM-DD\",\n \"domain_whitelist\": [\n \"example.com\"\n ],\n \"domain_blacklist\": [\n \"example.com\"\n ],\n \"domain_regex_whitelist\": [\n \".*example..*\"\n ],\n \"domain_regex_blacklist\": [\n \".*example..*\"\n ],\n \"domain_like_whitelist\": [\n \"%.example.%\",\n \"example%\"\n ],\n \"domain_like_blacklist\": [\n \"%.example.ca\"\n ],\n \"category_whitelist\": [\n \"Motor Vehicles\"\n ],\n \"category_blacklist\": [\n \"Motor Vehicles\"\n ],\n \"url_regex_whitelist\": [\n \".*/products/.*\"\n ],\n \"url_regex_blacklist\": [\n \".*/products/.*\"\n ],\n \"url_like_whitelist\": [\n \"%/products/%\",\n \"%/search%\"\n ],\n \"url_like_blacklist\": [\n \"%/review/%\"\n ],\n \"language_whitelist\": [\n \"eng\"\n ],\n \"language_blacklist\": [\n \"eng\"\n ],\n \"ip_country_whitelist\": [\n \"us\",\n \"ie\",\n \"in\"\n ],\n \"ip_country_blacklist\": [\n \"mx\",\n \"ae\",\n \"ca\"\n ],\n \"captcha\": true,\n \"robots_block\": true\n }\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("Authorization", "Bearer <token>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://api.brightdata.com/webarchive/search")
.header("Authorization", "Bearer <token>")
.header("Content-Type", "application/json")
.body("{\n \"filters\": {\n \"max_age\": \"Duration\",\n \"min_date\": \"YYYY-MM-DD\",\n \"max_date\": \"YYYY-MM-DD\",\n \"domain_whitelist\": [\n \"example.com\"\n ],\n \"domain_blacklist\": [\n \"example.com\"\n ],\n \"domain_regex_whitelist\": [\n \".*example..*\"\n ],\n \"domain_regex_blacklist\": [\n \".*example..*\"\n ],\n \"domain_like_whitelist\": [\n \"%.example.%\",\n \"example%\"\n ],\n \"domain_like_blacklist\": [\n \"%.example.ca\"\n ],\n \"category_whitelist\": [\n \"Motor Vehicles\"\n ],\n \"category_blacklist\": [\n \"Motor Vehicles\"\n ],\n \"url_regex_whitelist\": [\n \".*/products/.*\"\n ],\n \"url_regex_blacklist\": [\n \".*/products/.*\"\n ],\n \"url_like_whitelist\": [\n \"%/products/%\",\n \"%/search%\"\n ],\n \"url_like_blacklist\": [\n \"%/review/%\"\n ],\n \"language_whitelist\": [\n \"eng\"\n ],\n \"language_blacklist\": [\n \"eng\"\n ],\n \"ip_country_whitelist\": [\n \"us\",\n \"ie\",\n \"in\"\n ],\n \"ip_country_blacklist\": [\n \"mx\",\n \"ae\",\n \"ca\"\n ],\n \"captcha\": true,\n \"robots_block\": true\n }\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://api.brightdata.com/webarchive/search")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n \"filters\": {\n \"max_age\": \"Duration\",\n \"min_date\": \"YYYY-MM-DD\",\n \"max_date\": \"YYYY-MM-DD\",\n \"domain_whitelist\": [\n \"example.com\"\n ],\n \"domain_blacklist\": [\n \"example.com\"\n ],\n \"domain_regex_whitelist\": [\n \".*example..*\"\n ],\n \"domain_regex_blacklist\": [\n \".*example..*\"\n ],\n \"domain_like_whitelist\": [\n \"%.example.%\",\n \"example%\"\n ],\n \"domain_like_blacklist\": [\n \"%.example.ca\"\n ],\n \"category_whitelist\": [\n \"Motor Vehicles\"\n ],\n \"category_blacklist\": [\n \"Motor Vehicles\"\n ],\n \"url_regex_whitelist\": [\n \".*/products/.*\"\n ],\n \"url_regex_blacklist\": [\n \".*/products/.*\"\n ],\n \"url_like_whitelist\": [\n \"%/products/%\",\n \"%/search%\"\n ],\n \"url_like_blacklist\": [\n \"%/review/%\"\n ],\n \"language_whitelist\": [\n \"eng\"\n ],\n \"language_blacklist\": [\n \"eng\"\n ],\n \"ip_country_whitelist\": [\n \"us\",\n \"ie\",\n \"in\"\n ],\n \"ip_country_blacklist\": [\n \"mx\",\n \"ae\",\n \"ca\"\n ],\n \"captcha\": true,\n \"robots_block\": true\n }\n}"
response = http.request(request)
puts response.read_body{
"search_id": "ucd_abc123xyz"
}Mandatory:
Either use
Either use
max_age OR a combination of min_date + max_dateIf the search takes longer than 30 seconds, the response returns only a
search_id and you should poll the status asynchronously. If the search completes within 30 seconds, the response returns the full search result object (same as GET /webarchive/search/<search_id>).You can run up to 100 searches per day without triggering a dump.
Once you trigger a dump, that search no longer count against your limit.
LIKE vs Regex Filters
LIKE vs Regex Filters
- Use LIKE filters (
domain_like_*,url_like_*) for simple pattern matching with%(any sequence) and_(single character). - LIKE patterns are case-insensitive and often faster than regex for simple prefix/suffix matching like
%.comoramazon%. - Use regex filters (
domain_regex_*,url_regex_*) for complex patterns requiring full regex syntax. LIKE patterns use backslash escaping:\%for literal%,\_for literal_.
Authorizations
Use your Bright Data API Key as a Bearer token in the Authorization header.
How to authenticate:
- Obtain your API Key from the Bright Data account settings at https://brightdata.com/cp/setting/users
- Include the API Key in the Authorization header of your requests
- Format:
Authorization: Bearer YOUR_API_KEY
Example:
Authorization: Bearer b5648e1096c6442f60a6c4bbbe73f8d2234d3d8324554bd6a7ec8f3f251f07df
Learn how to get your Bright Data API key: https://docs.brightdata.com/api-reference/authentication
Body
application/json
The filters used for this search (echoed back)
Show child attributes
Show child attributes
Response
Search initiated successfully
- Async (Still Running)
- Completed within 30s
Returned if search is async
Example:
"ucd_abc123xyz"
Was this page helpful?
⌘I