Reddit Data Source

const redditUrl = `https://www.reddit.com/search.json?q=${encodeURIComponent(topic.query)}&sort=new&limit=25`;

const redditResult = await fetch(
  buildScrapeDoUrl(SCRAPE_DO_TOKEN, redditUrl, { render: false })
);

if (redditResult.ok) {
  const data = await redditResult.json();
  const redditPosts = parseRedditJson(data, topic.query);
  posts.push(...redditPosts);
}

const redditUrl = `https://www.reddit.com/search/?q=${encodeURIComponent(topic.query)}&sort=new`;
const scrapeUrl = buildScrapeDoUrl(SCRAPE_DO_TOKEN, redditUrl);

const res = await fetch(scrapeUrl);
if (res.ok) {
  const html = await res.text();
  
  // Check for bot detection
  if (html.includes("Are you a human?") || html.length < 5000) {
    scrapeStatus = "blocked";
  } else {
    const sentences = parseRedditHtml(html);
    posts = sentences.map((text, i) => ({
      id: `reddit_scrape_${topic_id}_${i}`,
      text,
      author: "reddit_user",
      created_at: new Date().toISOString(),
    }));
  }
}

function parseRedditJson(data: unknown, query: string): ScrapedPost[] {
  const posts: ScrapedPost[] = [];
  const record = data as Record<string, unknown>;
  const dataNode = record?.data as Record<string, unknown> | undefined;
  const children = (dataNode?.children as Array<Record<string, unknown>>) ?? [];

  for (const child of children) {
    const post = child?.data as Record<string, unknown> | undefined;
    if (!post) continue;
    
    const title = (post.title as string) ?? "";
    const selftext = (post.selftext as string) ?? "";
    const combined = [title, selftext].filter(Boolean).join(". ");
    const text = decodeEntities(combined.substring(0, 500));
    
    if (text.length > 10) {
      posts.push({
        id: `reddit_${post.id ?? posts.length}`,
        text,
        author: `u/${(post.author as string) ?? "redditor"}`,
        platform: "reddit",
        url: (post.url as string) ?? `https://www.reddit.com/search/?q=${encodeURIComponent(query)}`,
        postedAt: post.created_utc
          ? new Date((post.created_utc as number) * 1000).toISOString()
          : new Date().toISOString(),
      });
    }
  }

  return posts;
}

function parseRedditHtml(html: string): string[] {
  const results: string[] = [];

  // Strategy 1: shreddit-post web-component attribute
  const shredditRe = /post-title="([^"]{20,300})"/gi;
  let match: RegExpExecArray | null;

  while ((match = shredditRe.exec(html)) !== null) {
    const title = decodeHtmlEntities(match[1]).trim();
    if (!results.includes(title)) results.push(title);
    if (results.length >= 20) break;
  }

  // Strategy 2: h3 headings (classic Reddit fallback)
  if (results.length < 3) {
    const h3Re = /<h3[^>]*>([\s\S]{20,300}?)<\/h3>/gi;
    while ((match = h3Re.exec(html)) !== null) {
      const text = stripHtml(match[1]).trim();
      if (text.length >= 20 && !results.includes(text)) results.push(text);
      if (results.length >= 20) break;
    }
  }

  // Strategy 3: paragraph snippets (post-body previews)
  if (results.length < 5) {
    const pRe = /<p[^>]*>([\s\S]{30,300}?)<\/p>/gi;
    while ((match = pRe.exec(html)) !== null) {
      const text = stripHtml(match[1]).trim();
      if (text.length >= 30 && !results.includes(text)) results.push(text);
      if (results.length >= 20) break;
    }
  }

  return results.slice(0, 20);
}

if (res.ok) {
  const html = await res.text();
  
  // Check for CAPTCHA or empty page
  if (html.includes("Are you a human?") || 
      (!html.toLowerCase().includes("reddit") && html.length < 5000)) {
    scrapeStatus = "blocked";
    console.warn("Scrape.do/Reddit: bot-check detected");
  } else {
    const sentences = parseRedditHtml(html);
    scrapeStatus = sentences.length > 0 ? "ok" : "blocked";
  }
}

let inserted = 0;
for (const post of posts) {
  const { error } = await supabase.from("posts").upsert(
    {
      topic_id,
      platform: "reddit",
      external_id: post.id,
      author: `@${post.author}`,  // Normalize to @username format
      content: post.text,
      posted_at: post.created_at,
    },
    { onConflict: "platform,external_id" }
  );
  if (!error) inserted++;
}

{
  "success": true,
  "fetched": 20,
  "inserted": 18,
  "info": "Scrape.do (Reddit)",
  "scrape_status": "ok"
}

// In supabase/functions/fetch-twitter/index.ts
const [xResult, redditResult] = await Promise.allSettled([
  fetch(buildScrapeDoUrl(token, xUrl, { render: true })),
  fetch(buildScrapeDoUrl(token, redditUrl, { render: false }))
]);

// Both sources are merged
posts.push(...xPosts, ...redditPosts);