Web Scraping AI Agent

import streamlit as st
from scrapegraphai.graphs import SmartScraperGraph

st.title("Web Scraping AI Agent 🕵️‍♂️")
st.caption("Scrape websites using OpenAI API")

# Get API key
openai_api_key = st.text_input("OpenAI API Key", type="password")

if openai_api_key:
    # Select model
    model = st.radio("Select the model", ["gpt-4o", "gpt-5"], index=0)
    
    # Configure scraper
    graph_config = {
        "llm": {
            "api_key": openai_api_key,
            "model": model,
        },
    }
    
    # Get URL and prompt
    url = st.text_input("Enter the URL of the website")
    user_prompt = st.text_input("What do you want to extract?")
    
    # Create scraper
    smart_scraper_graph = SmartScraperGraph(
        prompt=user_prompt,
        source=url,
        config=graph_config
    )
    
    # Scrape
    if st.button("Scrape"):
        result = smart_scraper_graph.run()
        st.write(result)

from scrapegraph_py import Client

# Initialize client
client = Client(api_key="your-sgai-api-key")

# Extract structured data
response = client.smartscraper(
    website_url="https://example.com/products",
    user_prompt="Extract all product names, prices, and availability"
)

print(response)
# Output: {"products": [{"name": "...", "price": "...", "available": true}, ...]}

# AI-powered web search with structured results
response = client.searchscraper(
    user_prompt="Find the top 5 AI news websites",
    num_results=5
)

for result in response["results"]:
    print(f"{result['title']}: {result['url']}")

# Crawl multiple pages
request_id = client.smartcrawler(
    url="https://docs.example.com",
    user_prompt="Extract all API endpoints and their descriptions",
    max_pages=50
)

# Check progress
status = client.smartcrawler_progress(request_id)
print(f"Progress: {status['progress']}%")

# Get results when complete
if status['status'] == 'completed':
    results = client.smartcrawler_result(request_id)
    print(results)

# Convert webpage to markdown
response = client.markdownify(
    website_url="https://example.com/article"
)

print(response["markdown"])
# Output: Clean markdown version of the webpage