Document Loaders

// PDF loader configuration
{
  pdfFile: "FILE-STORAGE::document.pdf",
  usage: "perPage", // or "perFile"
  legacyBuild: false,
  metadata: {
    source: "user-manual",
    version: "2.0"
  },
  omitMetadataKeys: "pdf.version, pdf.info"
}

// From Pdf.ts
const loader = new PDFLoader(new Blob([buffer]), {
  splitPages: usage === 'perFile' ? false : true,
  pdfjs: () => legacyBuild 
    ? import('pdfjs-dist/legacy/build/pdf.js')
    : import('pdf-parse/lib/pdf.js/v1.10.100/build/pdf.js')
})

let docs = await loader.load()
if (textSplitter) {
  docs = await textSplitter.splitDocuments(docs)
}

// Cheerio configuration
{
  url: "https://docs.flowiseai.com",
  relativeLinksMethod: "webCrawl",
  limit: 10,
  selector: ".main-content", // CSS selector
  metadata: {
    type: "documentation"
  }
}

// From Cheerio.ts
const loader = new CheerioWebBaseLoader(url, {
  selector: '.main-content' // Optional CSS selector
})

let docs = await loader.load()
if (textSplitter) {
  docs = await textSplitter.splitDocuments(docs)
}

// Puppeteer configuration
{
  url: "https://example.com",
  waitForSelector: ".content-loaded",
  relativeLinksMethod: "webCrawl",
  limit: 20,
  metadata: {
    scraped_at: "2024-01-15"
  }
}

// Notion Page loader
{
  notionPageId: "abc123...",
  notionApiKey: "secret_...",
  metadata: {
    workspace: "engineering"
  }
}

// Notion Database loader
{
  notionDatabaseId: "def456...",
  notionApiKey: "secret_..."
}

// S3 File loader
{
  s3Bucket: "my-documents",
  s3Key: "docs/manual.pdf",
  s3Region: "us-east-1",
  // Optional: Use IAM role or provide credentials
}

// S3 Directory loader
{
  s3Bucket: "my-documents",
  s3Prefix: "docs/",
  s3Region: "us-east-1"
}

// Google Drive loader
{
  googleDriveId: "abc123...", // File or folder ID
  recursive: true, // Load subfolders
  metadata: {
    source: "google-drive"
  }
}

// Excel loader configuration
{
  excelFile: "FILE-STORAGE::data.xlsx",
  sheetName: "Sheet1", // Optional, loads all if not specified
  metadata: {
    source: "financial-data"
  }
}

// CSV loader configuration
{
  csvFile: "FILE-STORAGE::data.csv",
  columnMapping: {
    "text": "description",
    "metadata.category": "type"
  },
  metadata: {
    format: "csv"
  }
}

// JSON loader configuration
{
  jsonFile: "FILE-STORAGE::config.json",
  pointers: ["/data/items"], // JSONPath pointers
  metadata: {
    type: "config"
  }
}

// Folder loader configuration
{
  folderPath: "/path/to/documents",
  recursive: true,
  fileTypes: [".pdf", ".txt", ".md"],
  metadata: {
    batch: "import-2024"
  }
}

// API loader configuration
{
  apiUrl: "https://api.example.com/data",
  method: "GET",
  headers: {
    "Authorization": "Bearer token",
    "Content-Type": "application/json"
  },
  body: JSON.stringify({ query: "documents" }),
  textPath: "data.content" // Extract text from response
}

// FireCrawl configuration
{
  url: "https://example.com",
  apiKey: "fc-...",
  mode: "scrape", // or "crawl"
  limit: 100,
  metadata: {
    crawled_by: "firecrawl"
  }
}

// GitHub loader configuration
{
  repoLink: "https://github.com/FlowiseAI/Flowise",
  branch: "main",
  accessToken: "ghp_...", // Optional for private repos
  recursive: true,
  ignorePaths: ["node_modules", ".git"]
}

// Confluence loader
{
  confluenceUrl: "https://company.atlassian.net",
  spaceKey: "ENG",
  username: "[email protected]",
  accessToken: "...",
  limit: 25
}

// Airtable loader
{
  baseId: "app...",
  tableName: "Documents",
  apiKey: "key...",
  view: "All Records"
}

// Connect a Text Splitter node
{
  textSplitter: textSplitterNode,
  // Documents automatically split after loading
}

{
  metadata: {
    source: "documentation",
    version: "2.0",
    department: "engineering",
    indexed_at: "2024-01-15"
  }
}

{
  omitMetadataKeys: "pdf.version, loc.lines",
  // Or omit all except custom:
  omitMetadataKeys: "*"
}

{
  pdfFile: "FILE-STORAGE::[\"doc1.pdf\",\"doc2.pdf\",\"doc3.pdf\"]"
}

// Output as Document objects (default)
output: "document"

// Output as concatenated text
output: "text"

{
  url: "https://docs.example.com",
  relativeLinksMethod: "webCrawl", // or "scrapeXMLSitemap"
  limit: 50 // Max pages to crawl (0 = unlimited)
}

// Unstructured configuration
{
  file: "FILE-STORAGE::document.pdf",
  apiKey: "...",
  strategy: "hi_res", // or "fast", "ocr_only"
  // Supports: PDF, DOCX, PPTX, images, and more
}

// For large PDFs, use perFile mode
{
  usage: "perFile",
  // Then split with text splitter
}

// Use appropriate scraper:
// - Cheerio: Fast, static content
// - Puppeteer: JavaScript-heavy sites
// - Playwright: Modern complex sites
// - FireCrawl: AI-powered, best quality

// Limit crawled pages
{
  limit: 10, // Don't crawl entire site
  timeout: 30000 // 30 second timeout
}

// Invalid URLs or files are skipped
// Check logs for details:
// "Error loading document: ..."

// Try legacy build for problematic PDFs
{
  legacyBuild: true
}

// Use Puppeteer for JavaScript sites
// Add wait selector:
{
  waitForSelector: ".content-loaded"
}

Error: FILE-STORAGE:: file not found

{
  timeout: 60000 // Increase timeout
}

// Split large documents
{
  textSplitter: textSplitterNode,
  chunkSize: 1000,
  chunkOverlap: 200
}