Web Scraper

export const webScraper = ai.defineTool(
  {
    name: 'webScraper',
    description: 'Fetches the full content of a given URL using Firecrawl. Use this to read the content of an article or webpage.',
    inputSchema: WebScraperInputSchema,
    outputSchema: WebScraperOutputSchema,
  },
  async (input) => { ... }
)

export async function batchScrapeParallel(
  urls: string[]
): Promise<{ url: string; content: string; source: string }[]>

const WebScraperInputSchema = z.object({
  url: z.string().url().describe('The URL of the webpage to scrape.'),
});

const WebScraperOutputSchema = z.string().describe('The extracted textual content of the webpage.');

const urls = [
  'https://example.com/article1',
  'https://example.com/article2',
  'https://example.com/article3',
];

const results = await batchScrapeParallel(urls);

console.log(`Scraped ${results.length}/${urls.length} URLs successfully`);

results.forEach(result => {
  console.log(`${result.url}: ${result.content.length} chars`);
});

import { webScraper } from '@/ai/tools/web-scraper';

const content = await webScraper({ 
  url: 'https://example.com/article' 
});

console.log(content); // Markdown text

import { scrapeWithFirecrawl } from '@/ai/tools/web-scraper';

const content = await scrapeWithFirecrawl('https://example.com/article');

if (content) {
  console.log(`Scraped ${content.length} characters`);
} else {
  console.log('Scrape failed');
}

import { batchScrapeParallel } from '@/ai/tools/web-scraper';

const urls = [
  'https://bbc.com/news/article1',
  'https://reuters.com/article2',
  'https://nytimes.com/article3',
];

const results = await batchScrapeParallel(urls);

import { ai } from '@/ai/genkit';
import { webScraper } from '@/ai/tools/web-scraper';

const response = await ai.generate({
  prompt: 'Summarize the article at https://example.com/article',
  tools: [webScraper],
});

FIRECRAWL_API_KEY=your_api_key_here