Adding Data Sources

export async function fetchXPosts(
  query: string,
  token: string,
  options: ScrapeDoOptions = {}
): Promise<ScrapeDoResult> {
  const targetUrl = `https://x.com/search?q=${encodeURIComponent(
    query
  )}&src=typed_query&f=live`;
  
  const apiUrl = buildApiUrl(token, targetUrl, {
    render: true,           // Enable JavaScript rendering
    waitUntil: 'networkidle0', // Wait until network is idle
    ...options,
  });
  
  const res = await fetch(apiUrl);
  const html = await res.text();
  const posts = parseXHtml(html, query);
  
  return { posts, source: 'X via Scrape.do', status: 'success' };
}

export async function fetchRedditPosts(
  query: string,
  token: string,
  options: ScrapeDoOptions = {}
): Promise<ScrapeDoResult> {
  const targetUrl = `https://www.reddit.com/search.json?q=${encodeURIComponent(
    query
  )}&sort=new&limit=25`;
  
  const apiUrl = buildApiUrl(token, targetUrl, {
    render: false, // JSON endpoint, no JS rendering needed
    ...options,
  });
  
  const res = await fetch(apiUrl);
  const text = await res.text();
  const data = JSON.parse(text);
  const posts = parseRedditJson(data, query);
  
  return { posts, source: 'Reddit via Scrape.do', status: 'success' };
}

async function fetchYouTubeComments(
  query: string
): Promise<{ comments: string[]; count: number }> {
  const comments: string[] = [];
  
  // Step 1: Search for top 5 relevant videos
  const searchUrl = `https://www.googleapis.com/youtube/v3/search?part=id,snippet&q=${encodeURIComponent(
    query
  )}&type=video&order=relevance&maxResults=5&key=${YOUTUBE_KEY}`;
  
  const searchRes = await fetch(searchUrl);
  const searchData = await searchRes.json();
  const videoIds = searchData.items.map(item => item.id.videoId);
  
  // Step 2: Fetch comments from top 3 videos
  for (const videoId of videoIds.slice(0, 3)) {
    const commentsUrl = `https://www.googleapis.com/youtube/v3/commentThreads?part=snippet&videoId=${videoId}&order=relevance&maxResults=25&key=${YOUTUBE_KEY}`;
    
    const cRes = await fetch(commentsUrl);
    const cData = await cRes.json();
    
    for (const item of cData.items || []) {
      const text = item.snippet?.topLevelComment?.snippet?.textDisplay || '';
      if (text.length > 5 && text.length < 500) {
        comments.push(text);
      }
    }
  }
  
  return { comments, count: comments.length };
}

async function fetchNewsHeadlines(query: string): Promise<string[]> {
  const rssUrl = `https://news.google.com/rss/search?q=${encodeURIComponent(
    query
  )}&hl=en&gl=US&ceid=US:en`;
  
  const proxyUrl = `https://api.scrape.do?token=${SCRAPE_TOKEN}&url=${encodeURIComponent(
    rssUrl
  )}`;
  
  const res = await fetch(proxyUrl);
  const xml = await res.text();
  
  // Parse XML for <item><title> elements
  const items = xml.match(/<item>[\s\S]*?<\/item>/gi) || [];
  const headlines: string[] = [];
  
  for (const item of items) {
    const m = item.match(/<title><!\[CDATA\[([\s\S]*?)\]\]><\/title>/);
    if (m?.[1]) {
      const clean = m[1].replace(/<[^>]+>/g, '').trim();
      if (clean.length > 15 && clean.length < 250) {
        headlines.push(clean);
      }
    }
  }
  
  return headlines.slice(0, 10); // Top 10 headlines
}

// Tier 1: Gemini
if (geminiKey) {
  try {
    // Stream from Gemini 2.0 Flash
    return await streamGemini(prompt);
  } catch (e) {
    console.warn('Gemini failed:', e);
  }
}

// Tier 2: Groq
if (groqKey) {
  try {
    // Stream from Groq Llama 3.3 70B
    return await streamGroq(prompt);
  } catch (e) {
    console.warn('Groq failed:', e);
  }
}

// Tier 3: Local (guaranteed, no API needed)
return buildLocalSummary(analysis);

Frontend (Browser)
  ↓
Supabase Edge Function (Server-Side)
  ↓
Scrape.do / YouTube / Gemini / Groq
  ↓
Return results to frontend

const [ytResult, headlinesResult, scrapeResult] = await Promise.allSettled([
  fetchYouTubeComments(topic.title),
  fetchNewsHeadlines(topic.title),
  fetchAllScrapeDoSources(topic.title, SCRAPE_TOKEN, ['x', 'reddit']),
]);