Deep Research with AI Agents

# Planning Crew - Generates search strategy
web_scraping_planner = Agent(
    role="Strategic Research Planner",
    goal="Generate optimal search queries",
    llm=llm,
    output_pydantic=WebScrapingPlannerTaskResult
)

# Link Collection - Executed in parallel for all queries
web_scraping_link_collector = Agent(
    role="Web Research Specialist",
    goal="Collect high-quality relevant sources",
    tools=[search_engine_tool],
    llm=llm,
    output_pydantic=WebScrapingLinkCollectorTaskResult
)

# Content Extraction - Parallel scraping of all URLs
web_scraper = Agent(
    role="Content Extraction Specialist",
    goal="Extract complete content from URLs",
    tools=[scrape_as_markdown_tool],
    llm=llm,
    max_iter=50
)

# Execute all link collection tasks in parallel
link_collector_tasks = [
    web_scraping_link_collector_crew.kickoff_async(inputs={
        "topic": topic,
        "search_query": query,
        "current_time": current_time,
    })
    for query in search_queries
]
link_collector_results = await asyncio.gather(*link_collector_tasks)

# Execute all web scraping tasks in parallel
web_scraping_tasks = [
    web_scraping_crew.kickoff_async(inputs={
        "topic": topic,
        "url": link.url,
        "current_time": current_time,
    })
    for link in links
]
web_scraping_results = await asyncio.gather(*web_scraping_tasks)