Agentic RAG with Web Search

from qdrant_client import QdrantClient
from qdrant_client.models import PointStruct, Distance, VectorParams
import os

# Initialize Qdrant client
qdrant = QdrantClient(
    url=os.getenv("QDRANT_URL"),
    api_key=os.getenv("QDRANT_API_KEY")
)

collection_name = "rag_with_web_search"

# Create collection with vector configuration
qdrant.create_collection(
    collection_name=collection_name,
    vectors_config=VectorParams(
        size=3072,              # OpenAI text-embedding-3-large dimensions
        distance=Distance.COSINE  # Cosine similarity metric
    )
)

import pdfplumber
from openai import OpenAI
import uuid

client = OpenAI(api_key=os.getenv("OPENAI_API_KEY"))

def extract_text_from_pdf(pdf_path):
    """Extract text from PDF using pdfplumber."""
    text = []
    with pdfplumber.open(pdf_path) as pdf:
        for page in pdf.pages:
            page_text = page.extract_text()
            if page_text:
                text.append(page_text.strip())
    return text

def get_openai_embedding(text):
    """Generate OpenAI embeddings."""
    response = client.embeddings.create(
        input=text,
        model="text-embedding-3-large"  # 3072 dimensions
    )
    return response.data[0].embedding

def load_pdf_to_qdrant(pdf_path):
    """Load PDF into Qdrant vector database."""
    # Extract text chunks
    text_chunks = extract_text_from_pdf(pdf_path)
    
    # Create collection if it doesn't exist
    if qdrant.collection_exists(collection_name):
        qdrant.delete_collection(collection_name)
    
    qdrant.create_collection(
        collection_name=collection_name,
        vectors_config=VectorParams(size=3072, distance=Distance.COSINE)
    )
    
    # Generate embeddings and store
    points = []
    for chunk in text_chunks:
        embedding = get_openai_embedding(chunk)
        points.append(PointStruct(
            id=str(uuid.uuid4()),
            vector=embedding,
            payload={"text": chunk}  # Store original text as metadata
        ))
    
    qdrant.upsert(collection_name=collection_name, points=points)

from crewai import Agent, Task, Crew, Process
from crewai_tools import EXASearchTool, QdrantVectorSearchTool

# Initialize tools
search_tool = EXASearchTool()
qdrant_tool = QdrantVectorSearchTool(
    qdrant_url=os.getenv("QDRANT_URL"),
    qdrant_api_key=os.getenv("QDRANT_API_KEY"),
    collection_name=collection_name,
    limit=3,              # Return top 3 results
    score_threshold=0.35  # Minimum similarity score
)

# Database search agent
db_search_agent = Agent(
    role="Senior Semantic Search Agent",
    goal="Find and analyze documents based on semantic search",
    backstory="""You are an expert research assistant who can find relevant 
    information using semantic search in a Qdrant database.""",
    tools=[qdrant_tool],
    verbose=True
)

# Web search agent
search_agent = Agent(
    role="Senior Search Agent",
    goal="Search for relevant information using web search",
    backstory="""You are an expert search assistant who can find relevant
    information using the EXA search tool.""",
    tools=[search_tool],
    verbose=True
)

# Answer synthesis agent
answer_agent = Agent(
    role="Senior Answer Assistant",
    goal="Generate answers based on context provided",
    backstory="""You are an expert answer assistant who can generate 
    comprehensive answers based on multiple sources.""",
    verbose=True
)

from crewai import Task

# Database search task
db_search_task = Task(
    description="""Search for relevant documents about the {query}.
    Your final answer should include:
    - The relevant information found
    - The similarity scores of the results
    - The metadata of the relevant documents""",
    expected_output="A list of relevant documents with similarity scores and metadata.",
    agent=db_search_agent,
    tools=[qdrant_tool]
)

# Web search task
search_task = Task(
    description="""Search for relevant information about the {query} using web search.""",
    expected_output="Search results with relevant context and ranking.",
    agent=search_agent,
    tools=[search_tool]
)

# Answer generation task
answer_task = Task(
    description="""Given the context from database and web search,
    generate a comprehensive answer to: {query}
    
    Format your response with:
    - Summary of findings
    - Key results from both sources
    - Actionable insights
    - References with links
    """,
    expected_output="A comprehensive, well-formatted markdown answer.",
    agent=answer_agent
)

from crewai import Crew, Process

# Create crew with sequential process
crew = Crew(
    agents=[db_search_agent, search_agent, answer_agent],
    tasks=[db_search_task, search_task, answer_task],
    process=Process.sequential,  # Run tasks in order
    verbose=True
)

# Run the crew
result = crew.kickoff(inputs={"query": "What is quantum computing?"})
print(result)

import agentops
import os

AGENTOPS_API_KEY = os.getenv("AGENTOPS_API_KEY")
agentops.init(
    api_key=AGENTOPS_API_KEY,
    default_tags=['crewai', 'rag']
)

import streamlit as st
import tempfile
import os

st.title("🤖 Agentic RAG with Web Search")

# Sidebar: PDF upload and API keys
with st.sidebar:
    st.header("Configuration")
    
    qdrant_api_key = st.text_input("Qdrant API Key", type="password")
    exa_api_key = st.text_input("Exa API Key", type="password")
    
    st.divider()
    
    st.header("Upload PDF")
    uploaded_file = st.file_uploader("Choose a PDF", type=["pdf"])
    
    if uploaded_file:
        # Save to temp file
        with tempfile.NamedTemporaryFile(delete=False, suffix=".pdf") as tmp_file:
            tmp_file.write(uploaded_file.getbuffer())
            temp_path = tmp_file.name
        
        with st.spinner("Loading PDF into Qdrant..."):
            try:
                load_pdf_to_qdrant(temp_path)
                st.success("✅ PDF loaded successfully!")
                st.session_state.pdf_loaded = True
            except Exception as e:
                st.error(f"Error: {str(e)}")
        
        # Clean up temp file
        os.unlink(temp_path)

# Chat interface
if "messages" not in st.session_state:
    st.session_state.messages = []

for message in st.session_state.messages:
    with st.chat_message(message["role"]):
        st.markdown(message["content"])

if prompt := st.chat_input("Ask a question about your document..."):
    if not st.session_state.get('pdf_loaded', False):
        st.warning("Please upload a PDF first.")
    else:
        # Add user message
        st.session_state.messages.append({"role": "user", "content": prompt})
        with st.chat_message("user"):
            st.markdown(prompt)
        
        # Run crew
        with st.chat_message("assistant"):
            with st.spinner("Thinking..."):
                result = crew.kickoff(inputs={"query": prompt})
                st.markdown(result)
                st.session_state.messages.append({"role": "assistant", "content": result})

git clone https://github.com/Arindam200/awesome-ai-apps.git
cd rag_apps/agentic_rag_with_web_search
uv sync

OPENAI_API_KEY=your_openai_api_key
QDRANT_API_KEY=your_qdrant_api_key
QDRANT_URL=your_qdrant_cluster_url
EXA_API_KEY=your_exa_api_key
AGENTOPS_API_KEY=your_agentops_api_key

streamlit run main.py