Bright Data MCP Server

BRIGHT_DATA_API_TOKEN=your_bright_data_api_token
BRIGHT_DATA_BROWSER_AUTH=your_bright_data_browser_auth

from mcp import StdioServerParameters
from crewai_tools import MCPServerAdapter
import os

server_params = StdioServerParameters(
    command="pnpm",
    args=["dlx", "@brightdata/mcp"],
    env={
        "API_TOKEN": os.environ["BRIGHT_DATA_API_TOKEN"],
        "BROWSER_AUTH": os.environ["BRIGHT_DATA_BROWSER_AUTH"]
    },
)

# Initialize within agent workflow
async def run_research_crew(topic: str):
    with MCPServerAdapter(server_params) as tools:
        # Tools are now available for agents
        web_scraping_tools = [tool for tool in tools if tool.name in ["scrape_as_markdown"]]
        search_tools = [tool for tool in tools if tool.name in ["search_engine"]]

# Used by Link Collector Agent
link_collector_task = Task(
    description="""Using the search query provided, collect relevant links using the search engine tool.
    
    Follow these steps:
    1. Use the search_engine tool with parameters:
       - engine: "google"
       - query: the provided search query
    2. Select 10 of the most relevant and authoritative links
    3. Focus on high-quality sources
    """,
    agent=web_scraping_link_collector,
    tools=web_scraping_link_collector_tools
)

# Used by Web Scraper Agent
web_scraping_task = Task(
    description="""Extract raw content from the URL:
    
    1. Use scrape_as_markdown to capture ALL raw text
    2. Return the raw text as a string
    3. Preserve ALL text exactly as it appears
    """,
    agent=web_scraper,
    tools=web_scraping_tools
)

import asyncio

# Execute multiple scraping tasks in parallel
web_scraping_tasks = []
for link in links:
    web_scraping_tasks.append(
        web_scraping_crew.kickoff_async(inputs={
            "url": link.url,
            "current_time": current_time,
        })
    )

# Gather results concurrently
web_scraping_results = await asyncio.gather(*web_scraping_tasks)

# Process scraped data
for link, result in zip(links, web_scraping_results):
    scraped_data.append({
        "url": link.url,
        "page_title": link.title,
        "content": result.raw
    })

# Automatic retry configuration
web_scraping_task = Task(
    description=TOPIC_RESEARCH_TASK_CONFIGS["web_scraping"]["description"],
    expected_output=TOPIC_RESEARCH_TASK_CONFIGS["web_scraping"]["expected_output"],
    agent=web_scraper,
    max_retries=5  # Automatic retry on failure
)

from loguru import logger

logger.info(f"Running web scraping crew for {len(links)} links")

# Crew logs to file
web_scraping_crew = Crew(
    agents=[web_scraper],
    tasks=[web_scraping_task],
    verbose=True,
    output_log_file=f"logs/web_scraping_crew_{current_time}.log"
)

web_scraping_crew = Crew(
    agents=[web_scraper],
    tasks=[web_scraping_task],
    max_rpm=20  # Maximum requests per minute
)