LLM友好的文档参考

# 保存到项目规则目录
curl -o .cursor/rules/brightdata.md https://docs.brightdata.com/llms.txt
# 或者用于 Windsurf：
curl -o .windsurf/rules/brightdata.md https://docs.brightdata.com/llms.txt

import httpx
from langchain.text_splitter import MarkdownTextSplitter

# 获取完整文档
response = httpx.get("https://docs.brightdata.com/llms-full.txt")
docs_content = response.text

# 分割成块
splitter = MarkdownTextSplitter(chunk_size=1000, chunk_overlap=100)
chunks = splitter.create_documents([docs_content])

# 添加到你的向量存储
vectorstore.add_documents(chunks)

import httpx

# 获取索引
llms_txt = httpx.get("https://docs.brightdata.com/llms.txt").text

messages = [
    {
        "role": "system",
        "content": f"""You are a helpful assistant with expertise in Bright Data's web data APIs.

Here is the full index of available documentation:
{llms_txt}

Use this to understand which products and APIs are available, then fetch
specific pages when you need full details on a product."""
    },
    {"role": "user", "content": "How do I scrape Amazon product pages?"}
]