Multi-Source Research

// Source types
type Source = {
  type: "URL" | "TEXT" | "FILE";
  value: string;           // URL or text content
  filename?: string;       // For file uploads
  filePath?: string;       // R2 storage path
}

// Backend processing
const sourcesToSave = sources.map(source => {
  if (source.type === "URL") {
    return {
      sourceType: "URL",
      sourceUrl: source.value,
    };
  } else if (source.type === "FILE") {
    return {
      sourceType: "UPLOAD",
      sourceUrl: source.value,
      filePath: source.filePath,
      filename: source.filename,
    };
  } else {
    return {
      sourceType: "MANUAL",
      content: source.value,
    };
  }
});

async def run_sources_research_crew(sources: List[ResearchSource]):
    # Extract URLs from sources
    links = [WebLink(url=source.source_url, title=source.source_url)
             for source in sources
             if source.source_type == "URL"]
    
    # Scrape all URLs in parallel
    web_scraping_tasks = [
        web_scraping_crew.kickoff_async(inputs={
            "url": link.url,
            "current_time": current_time,
        })
        for link in links
    ]
    web_scraping_results = await asyncio.gather(*web_scraping_tasks)
    
    # Get textual content from MANUAL sources
    textual_content = ""
    for source in sources:
        if source.source_type == "MANUAL":
            textual_content += f"\n---\n- {source.source_content}\n---\n"
    
    # Convert uploaded files to markdown
    if any(source.source_type == "UPLOAD" for source in sources):
        markdown_files = await markdown_converter.convert_urls_to_markdown(
            [source.source_url for source in sources if source.source_type == "UPLOAD"]
        )

# Convert uploaded files to markdown
markdown_files = await markdown_converter.convert_urls_to_markdown(
    [source.source_url for source in sources if source.source_type == "UPLOAD"]
)
for url, markdown_content in markdown_files.items():
    file_content += f"\n---\n- File: {url}\n---\n{markdown_content}\n---\n"
    file_data.append({
        "file_name": url,
        "content": markdown_content
    })