Deliver a snapshot to Amazon S3, Azure Blob Storage or a Webhook
curl --request POST \
--url https://api.brightdata.com/webarchive/dump \
--header 'Authorization: Bearer <token>' \
--header 'Content-Type: application/json' \
--data '
{
"search_id": "<string>",
"delivery": {
"strategy": "s3",
"settings": {
"bucket": "<string>",
"assume_role": {
"role_arn": "<string>"
},
"prefix": "<string>"
}
},
"max_entries": 123
}
'import requests
url = "https://api.brightdata.com/webarchive/dump"
payload = {
"search_id": "<string>",
"delivery": {
"strategy": "s3",
"settings": {
"bucket": "<string>",
"assume_role": { "role_arn": "<string>" },
"prefix": "<string>"
}
},
"max_entries": 123
}
headers = {
"Authorization": "Bearer <token>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
body: JSON.stringify({
search_id: '<string>',
delivery: {
strategy: 's3',
settings: {bucket: '<string>', assume_role: {role_arn: '<string>'}, prefix: '<string>'}
},
max_entries: 123
})
};
fetch('https://api.brightdata.com/webarchive/dump', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.brightdata.com/webarchive/dump",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'search_id' => '<string>',
'delivery' => [
'strategy' => 's3',
'settings' => [
'bucket' => '<string>',
'assume_role' => [
'role_arn' => '<string>'
],
'prefix' => '<string>'
]
],
'max_entries' => 123
]),
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>",
"Content-Type: application/json"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://api.brightdata.com/webarchive/dump"
payload := strings.NewReader("{\n \"search_id\": \"<string>\",\n \"delivery\": {\n \"strategy\": \"s3\",\n \"settings\": {\n \"bucket\": \"<string>\",\n \"assume_role\": {\n \"role_arn\": \"<string>\"\n },\n \"prefix\": \"<string>\"\n }\n },\n \"max_entries\": 123\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("Authorization", "Bearer <token>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://api.brightdata.com/webarchive/dump")
.header("Authorization", "Bearer <token>")
.header("Content-Type", "application/json")
.body("{\n \"search_id\": \"<string>\",\n \"delivery\": {\n \"strategy\": \"s3\",\n \"settings\": {\n \"bucket\": \"<string>\",\n \"assume_role\": {\n \"role_arn\": \"<string>\"\n },\n \"prefix\": \"<string>\"\n }\n },\n \"max_entries\": 123\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://api.brightdata.com/webarchive/dump")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n \"search_id\": \"<string>\",\n \"delivery\": {\n \"strategy\": \"s3\",\n \"settings\": {\n \"bucket\": \"<string>\",\n \"assume_role\": {\n \"role_arn\": \"<string>\"\n },\n \"prefix\": \"<string>\"\n }\n },\n \"max_entries\": 123\n}"
response = http.request(request)
puts response.read_body{
"dump_id": "ucd_abc123-1234567890"
}{
"error": "Request validation failed",
"error_code": "validation",
"details": [
{
"message": "\"search_id\" is required",
"path": [
"search_id"
],
"type": "any.required"
}
]
}网络归档 API
将快照传送到 S3、Azure Blob 或 GCP
将 Bright Data Archive API 快照传送到 Amazon S3、Azure Blob Storage、Google Cloud Storage 或 webhook。POST /webarchive/dump 返回 dump_id。
POST
/
webarchive
/
dump
Deliver a snapshot to Amazon S3, Azure Blob Storage or a Webhook
curl --request POST \
--url https://api.brightdata.com/webarchive/dump \
--header 'Authorization: Bearer <token>' \
--header 'Content-Type: application/json' \
--data '
{
"search_id": "<string>",
"delivery": {
"strategy": "s3",
"settings": {
"bucket": "<string>",
"assume_role": {
"role_arn": "<string>"
},
"prefix": "<string>"
}
},
"max_entries": 123
}
'import requests
url = "https://api.brightdata.com/webarchive/dump"
payload = {
"search_id": "<string>",
"delivery": {
"strategy": "s3",
"settings": {
"bucket": "<string>",
"assume_role": { "role_arn": "<string>" },
"prefix": "<string>"
}
},
"max_entries": 123
}
headers = {
"Authorization": "Bearer <token>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
body: JSON.stringify({
search_id: '<string>',
delivery: {
strategy: 's3',
settings: {bucket: '<string>', assume_role: {role_arn: '<string>'}, prefix: '<string>'}
},
max_entries: 123
})
};
fetch('https://api.brightdata.com/webarchive/dump', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.brightdata.com/webarchive/dump",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'search_id' => '<string>',
'delivery' => [
'strategy' => 's3',
'settings' => [
'bucket' => '<string>',
'assume_role' => [
'role_arn' => '<string>'
],
'prefix' => '<string>'
]
],
'max_entries' => 123
]),
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>",
"Content-Type: application/json"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://api.brightdata.com/webarchive/dump"
payload := strings.NewReader("{\n \"search_id\": \"<string>\",\n \"delivery\": {\n \"strategy\": \"s3\",\n \"settings\": {\n \"bucket\": \"<string>\",\n \"assume_role\": {\n \"role_arn\": \"<string>\"\n },\n \"prefix\": \"<string>\"\n }\n },\n \"max_entries\": 123\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("Authorization", "Bearer <token>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://api.brightdata.com/webarchive/dump")
.header("Authorization", "Bearer <token>")
.header("Content-Type", "application/json")
.body("{\n \"search_id\": \"<string>\",\n \"delivery\": {\n \"strategy\": \"s3\",\n \"settings\": {\n \"bucket\": \"<string>\",\n \"assume_role\": {\n \"role_arn\": \"<string>\"\n },\n \"prefix\": \"<string>\"\n }\n },\n \"max_entries\": 123\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://api.brightdata.com/webarchive/dump")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n \"search_id\": \"<string>\",\n \"delivery\": {\n \"strategy\": \"s3\",\n \"settings\": {\n \"bucket\": \"<string>\",\n \"assume_role\": {\n \"role_arn\": \"<string>\"\n },\n \"prefix\": \"<string>\"\n }\n },\n \"max_entries\": 123\n}"
response = http.request(request)
puts response.read_body{
"dump_id": "ucd_abc123-1234567890"
}{
"error": "Request validation failed",
"error_code": "validation",
"details": [
{
"message": "\"search_id\" is required",
"path": [
"search_id"
],
"type": "any.required"
}
]
}POST /webarchive/dump 将已完成搜索的快照传送到 Amazon S3、Azure Blob Storage、Google Cloud Storage 或 webhook,并返回 dump_id。
要使用 Google Cloud Storage 传递,请创建一个存储桶并提供所需的 GCP 传递设置。
webhook 传递策略不适合大型数据转储,除非您在自己的基础设施上托管 webhook。第三方检查工具(如 webhook.site)施加了严格的请求体大小限制,将无法接收可能达到 1 GB 大小的有效负载。对于大型传递,请改用 Amazon S3、Azure Blob Storage 或 Google Cloud Storage。
常见数据转储参数:
search_id(必需):来自已完成搜索的搜索 IDmax_entries(可选):限制要包含在数据转储中的文件数量delivery(必需):传递配置(S3、Azure、GCP 或 webhook)
如果您运行的是 linux/macos 机器,可以使用此页面上的代码模拟 Bright Data 的传递 webhook 之一。
什么情况会返回 400
当请求体未通过校验时,POST /webarchive/dump 返回 HTTP 400。响应包含 error 摘要和列出每个校验失败字段的 details 数组。常见原因:
- 缺少
search_id。 - 缺少
delivery。 delivery.settings与所选的delivery.strategy不匹配。每种策略有各自的必需设置:Amazon S3 需要bucket和assume_role,Google Cloud Storage 需要bucket,Azure Blob Storage 需要container和credentials,webhook 需要url。
details[].path 中指出的字段后重新发送请求。授权
Use your Bright Data API Key as a Bearer token in the Authorization header.
How to authenticate:
- Obtain your API Key from the Bright Data account settings at https://brightdata.com/cp/setting/users
- Include the API Key in the Authorization header of your requests
- Format:
Authorization: Bearer YOUR_API_KEY
Example:
Authorization: Bearer b5648e1096c6442f60a6c4bbbe73f8d2234d3d8324554bd6a7ec8f3f251f07df
Learn how to get your Bright Data API key: https://docs.brightdata.com/api-reference/authentication
请求体
application/json
响应
Dump created successfully
ID of the created dump
示例:
"ucd_abc123-1234567890"
此页面对您有帮助吗?