fetch_url Tool

interface FetchUrlToolOptions {
  url: string;
  followRedirects?: boolean;
  scrapeMode?: ScrapeMode;
  headers?: Record<string, string>;
}

enum ScrapeMode {
  Fetch = "fetch",
  Playwright = "playwright",
  Auto = "auto"
}

{
  "name": "fetch_url",
  "arguments": {
    "url": "https://react.dev/reference/react/useState"
  }
}

{
  "name": "fetch_url",
  "arguments": {
    "url": "https://docs.private-company.com/api",
    "headers": {
      "Authorization": "Bearer token123"
    }
  }
}

{
  "name": "fetch_url",
  "arguments": {
    "url": "https://example.com/old-page",
    "followRedirects": false
  }
}

{
  "name": "fetch_url",
  "arguments": {
    "url": "https://spa-app.com/docs",
    "scrapeMode": "playwright"
  }
}

# useState

`useState` is a React Hook that lets you add a state variable to your component.

```js
const [state, setState] = useState(initialState);

import { useState } from 'react';

function MyComponent() {
  const [age, setAge] = useState(28);
  // ...
}

## MCP Output

When called through MCP (see `src/mcp/mcpServer.ts:458`), the response is the raw Markdown content.

## Error Cases

### Invalid URL

Throws `ValidationError`:

```json
{
  "error": "Invalid URL: not-a-url. Must be an HTTP/HTTPS URL or a file:// URL."
}

{
  "error": "Unable to fetch or process the URL \"https://example.com/404\". Please verify the URL is correct and accessible."
}

{
  "error": "Processing resulted in empty content for https://example.com/empty"
}

{
  "url": "file:///home/user/docs/readme.md"
}

Fetch the content from https://react.dev/reference/react/useState and show it to me.

import { Client } from '@modelcontextprotocol/sdk/client/index.js';

const result = await client.callTool({
  name: 'fetch_url',
  arguments: {
    url: 'https://react.dev/reference/react/useState',
    followRedirects: true
  }
});

console.log(result);

async execute(options: FetchUrlToolOptions): Promise<string> {
  const { url, scrapeMode = ScrapeMode.Auto, headers } = options;
  
  // Validate URL
  if (!this.fetcher.canFetch(url)) {
    throw new ValidationError(
      `Invalid URL: ${url}. Must be an HTTP/HTTPS URL or a file:// URL.`,
      this.constructor.name,
    );
  }
  
  try {
    logger.info(`📡 Fetching ${url}...`);
    
    // Fetch content
    const rawContent: RawContent = await this.fetcher.fetch(url, {
      followRedirects: options.followRedirects ?? true,
      maxRetries: 3,
      headers,
    });
    
    logger.info("🔄 Processing content...");
    
    // Process content through pipelines
    let processed: Awaited<PipelineResult> | undefined;
    for (const pipeline of this.pipelines) {
      if (pipeline.canProcess(rawContent.mimeType, rawContent.content)) {
        processed = await pipeline.process(rawContent, {...}, this.fetcher);
        break;
      }
    }
    
    if (!processed) {
      // Fallback: return raw content with charset detection
      const resolvedCharset = resolveCharset(
        rawContent.charset,
        rawContent.content,
        rawContent.mimeType,
      );
      return convertToString(rawContent.content, resolvedCharset);
    }
    
    if (!processed.textContent?.trim()) {
      throw new ToolError(
        `Processing resulted in empty content for ${url}`,
        this.constructor.name,
      );
    }
    
    logger.info(`✅ Successfully processed ${url}`);
    return processed.textContent;
  } catch (error) {
    // Error handling
    if (error instanceof ToolError) {
      throw error;
    }
    throw new ToolError(
      `Unable to fetch or process the URL "${url}". Please verify the URL is correct and accessible.`,
      this.constructor.name,
    );
  } finally {
    // Cleanup pipelines and fetcher
    await Promise.allSettled([
      ...this.pipelines.map((pipeline) => pipeline.close()),
      this.fetcher.close(),
    ]);
  }
}

const content = await fetchUrl({
  url: "https://react.dev/reference/react/useState"
});

try {
  await fetchUrl({ url: "https://docs.example.com" });
  // URL is accessible, start scraping
  await scrapeDocs({ url: "https://docs.example.com", ... });
} catch (error) {
  console.error("URL not accessible:", error);
}

const content = await fetchUrl({
  url: "https://internal.company.com/docs",
  headers: {
    "Authorization": "Bearer token123"
  }
});

const content = await fetchUrl({
  url: "file:///home/user/docs/guide.html"
});