import { rss } from '@deepagents/retrieval/connectors';

import { rss } from '@deepagents/retrieval/connectors';
import { ingest, fastembed, SqliteStore } from '@deepagents/retrieval';
import Database from 'better-sqlite3';

const db = new Database('./vectors.db');
const store = new SqliteStore(db, 384);
const embedder = fastembed();

// Ingest RSS feed
await ingest({
  connector: rss('https://hnrss.org/frontpage'),
  store,
  embedder,
});

function rss(feedUrl: string, options?: {
  maxItems?: number;          // Max items to ingest (default: 50)
  fetchFullArticles?: boolean; // Extract full article content (default: false)
}): Connector

const connector = rss('https://hnrss.org/frontpage');

const connector = rss('https://example.com/feed', {
  maxItems: 10, // Only ingest latest 10 items
});

const connector = rss('https://blog.example.com/feed', {
  fetchFullArticles: true, // Extract full article text
});

{
  title: string;
  description: string;
  link: string;
  language: string;
  lastBuildDate: string;
  items: Array<{
    title: string;
    description: string;    // Summary or snippet
    link: string;
    pubDate: string;
    author: string;
    categories: string[];
    guid: string;
    contentEncoded: string; // Full content (if available)
  }>;
}

Title: {title}
Author: {author}
Published: {pubDate}
Categories: {categories}
Link: {link}
Content:
{content}

Summary: {title} - {description}

RSS Feed: {feed.title}
Description: {feed.description}
Website: {feed.link}
Language: {feed.language}
Last Updated: {lastBuildDate}
Total Items: {count}

This feed provides: {description}

const connector = rss('https://blog.example.com/feed', {
  maxItems: 5,
  fetchFullArticles: true,
});

await ingest({ connector, store, embedder });

// If extraction fails, falls back to RSS content
console.warn(`Failed to fetch article content from ${url}:`, error.message);
return ''; // Empty string fallback

const connector = rss('https://example.com/feed');
console.log(connector.sourceId);
// "rss:https://example.com/feed"

connector.instructions = `
You answer questions about articles and content from the RSS feed: ${feedUrl}.
Always cite the article title and link when referencing specific content.
The feed contains recent articles, blog posts, and news items.
When referencing content, include the publication date and author when available.
${fetchFullArticles ? 'Full article content has been extracted...' : 'Content includes RSS summaries...'}
`;

import { rss } from '@deepagents/retrieval/connectors';
import { similaritySearch } from '@deepagents/retrieval';

const connector = rss('https://hnrss.org/frontpage', {
  maxItems: 20,
});

await ingest({ connector, store, embedder });

const results = await similaritySearch(
  'What are the latest AI developments?',
  { connector, store, embedder }
);

console.log(results[0].content);

const connector = rss('https://blog.example.com/feed', {
  maxItems: 10,
  fetchFullArticles: true, // Extract complete articles
});

await ingest({ connector, store, embedder });

const feeds = [
  rss('https://hnrss.org/frontpage'),
  rss('https://news.ycombinator.com/rss'),
  rss('https://blog.example.com/feed'),
];

for (const connector of feeds) {
  await ingest({ connector, store, embedder });
  console.log(`Ingested: ${connector.sourceId}`);
}

const feeds = [
  rss('https://techcrunch.com/feed/'),
  rss('https://theverge.com/rss/index.xml'),
];

// Ingest all feeds
for (const connector of feeds) {
  await ingest({ connector, store, embedder });
}

// Search across all
const allResults = [];
for (const connector of feeds) {
  const results = await similaritySearch('AI news', {
    connector,
    store,
    embedder,
  });
  allResults.push(...results);
}

allResults.sort((a, b) => b.similarity - a.similarity);
console.log(`Found ${allResults.length} results across all feeds`);

metadata: {
  title: 'Article Title',
  author: 'Author Name',
  pubDate: '2024-01-01T12:00:00Z',
  categories: ['tech', 'AI'],
  link: 'https://example.com/article',
}

const results = await similaritySearch('query', config);
results.forEach(r => {
  console.log(`Title: ${r.metadata?.title}`);
  console.log(`Author: ${r.metadata?.author}`);
  console.log(`Link: ${r.metadata?.link}`);
});

const connector = rss('https://example.com/feed', {
  maxItems: 50,
  fetchFullArticles: false, // Fast
});

const connector = rss('https://example.com/feed', {
  maxItems: 10, // Reduce items for faster ingestion
  fetchFullArticles: true, // Slower
});

signal: AbortSignal.timeout(10000) // 10 seconds

'User-Agent': 'Mozilla/5.0 (compatible; RSS-RAG-Bot/1.0)'

try {
  await ingest({
    connector: rss('https://invalid-feed.com/feed'),
    store,
    embedder,
  });
} catch (error) {
  console.error('RSS parsing failed:', error.message);
}

// Article extraction failure
console.warn(`Failed to fetch article: ${error.message}`);
// Falls back to RSS content

const connector = rss('https://news.example.com/feed', {
  maxItems: 20,
  // Re-ingest after 1 hour
});

// Manual expiry in ingestion
await ingest({
  connector,
  store,
  embedder,
});

import { local } from '@deepagents/retrieval/connectors';

// Note: RSS connector doesn't support ingestWhen directly
// Use manual expiry logic or re-ingest periodically

if (fullContent.length < 200) {
  throw new Error('Extracted content too short');
}

rss('https://example.com/feed', {
  maxItems: 10,
  fetchFullArticles: true,
})

rss('https://example.com/feed', {
  maxItems: 50,
  fetchFullArticles: false,
})

// Every hour
setInterval(async () => {
  await ingest({ connector, store, embedder });
}, 60 * 60 * 1000);

const recent = results.filter(r => {
  const pubDate = new Date(r.metadata?.pubDate);
  return pubDate > oneDayAgo;
});