OmniScraperGraph

OmniScraperGraph(
    prompt: str,              # Natural language description including text and image requirements
    source: str,              # URL or path to local HTML file
    config: dict,             # Configuration dictionary
    schema: Optional[BaseModel] = None  # Pydantic schema for structured output
)

import os
import json
from dotenv import load_dotenv
from scrapegraphai.graphs import OmniScraperGraph
from scrapegraphai.utils import prettify_exec_info

load_dotenv()

openai_key = os.getenv("OPENAI_API_KEY")

graph_config = {
    "llm": {
        "api_key": openai_key,
        "model": "openai/gpt-4o",  # Must be a vision-capable model
    },
    "max_images": 5,
    "verbose": True,
}

# Create the OmniScraperGraph instance
omni_scraper = OmniScraperGraph(
    prompt="List me all the attractions in Chioggia and describe their pictures.",
    source="https://en.wikipedia.org/wiki/Chioggia",
    config=graph_config,
)

# Run the graph
result = omni_scraper.run()
print(json.dumps(result, indent=4))

# Get execution info
graph_exec_info = omni_scraper.get_execution_info()
print(prettify_exec_info(graph_exec_info))

graph_config = {
    "llm": {
        "api_key": os.getenv("OPENAI_API_KEY"),
        "model": "openai/gpt-4o",  # Recommended
    },
    "max_images": 5,
}

graph_config = {
    "llm": {
        "api_key": openai_key,
        "model": "openai/gpt-4o",
    },
    "max_images": 10,  # Process up to 10 images
}

omni_scraper = OmniScraperGraph(
    prompt="Describe the product images and their features",
    source="https://example.com/product",
    config=graph_config,
)

from pydantic import BaseModel, Field
from typing import List

class ImageDescription(BaseModel):
    url: str = Field(description="Image URL")
    description: str = Field(description="AI-generated image description")
    alt_text: str = Field(description="Image alt text if available")

class Attraction(BaseModel):
    name: str = Field(description="Attraction name")
    description: str = Field(description="Text description")
    images: List[ImageDescription] = Field(description="Associated images")

class AttractionList(BaseModel):
    attractions: List[Attraction]

graph_config = {
    "llm": {
        "model": "openai/gpt-4o",
        "api_key": os.getenv("OPENAI_API_KEY"),
    },
    "max_images": 8,
}

omni_scraper = OmniScraperGraph(
    prompt="Extract all attractions with their descriptions and images",
    source="https://tourism-site.com/attractions",
    config=graph_config,
    schema=AttractionList
)

result = omni_scraper.run()
for attraction in result['attractions']:
    print(f"{attraction['name']}: {len(attraction['images'])} images")

from pydantic import BaseModel
from typing import List

class ProductImage(BaseModel):
    description: str
    features_visible: List[str]

class Product(BaseModel):
    name: str
    price: float
    text_description: str
    images: List[ProductImage]
    visual_features: List[str]

graph_config = {
    "llm": {
        "model": "openai/gpt-4o",
        "api_key": os.getenv("OPENAI_API_KEY"),
    },
    "max_images": 6,
}

omni_scraper = OmniScraperGraph(
    prompt="Extract product information including name, price, description, and analyze all product images to identify visible features",
    source="https://shop.example.com/product/12345",
    config=graph_config,
    schema=Product
)

result = omni_scraper.run()
print(f"Product: {result['name']}")
print(f"Visual features found: {', '.join(result['visual_features'])}")

from pydantic import BaseModel
from typing import List

class PropertyImage(BaseModel):
    room_type: str
    description: str
    condition: str

class Property(BaseModel):
    address: str
    price: int
    bedrooms: int
    bathrooms: int
    description: str
    image_analysis: List[PropertyImage]

graph_config = {
    "llm": {
        "model": "openai/gpt-4o",
        "api_key": os.getenv("OPENAI_API_KEY"),
    },
    "max_images": 10,
}

omni_scraper = OmniScraperGraph(
    prompt="Extract property details and analyze images to identify rooms, their condition, and notable features",
    source="https://realestate.com/listing/12345",
    config=graph_config,
    schema=Property
)

result = omni_scraper.run()

from pydantic import BaseModel
from typing import List

class DishImage(BaseModel):
    description: str
    visual_appeal: str
    ingredients_visible: List[str]

class Dish(BaseModel):
    name: str
    price: float
    description: str
    image_analysis: DishImage

class Menu(BaseModel):
    restaurant_name: str
    dishes: List[Dish]

omni_scraper = OmniScraperGraph(
    prompt="Extract menu items with prices and analyze food images to describe appearance and visible ingredients",
    source="https://restaurant.com/menu",
    config=graph_config,
    schema=Menu
)

result = omni_scraper.run()

from pydantic import BaseModel
from typing import List

class Artwork(BaseModel):
    title: str
    artist: str
    year: str
    description: str
    visual_analysis: str
    style: str
    colors: List[str]
    subject_matter: str

class Exhibition(BaseModel):
    title: str
    artworks: List[Artwork]

graph_config = {
    "llm": {
        "model": "openai/gpt-4o",
        "api_key": os.getenv("OPENAI_API_KEY"),
    },
    "max_images": 15,
}

omni_scraper = OmniScraperGraph(
    prompt="Extract artwork information and provide detailed visual analysis of each piece including style, colors, and subject matter",
    source="https://museum.com/exhibition",
    config=graph_config,
    schema=Exhibition
)

result = omni_scraper.run()

result = omni_scraper.run()
# Returns: Dictionary with both text extraction and image descriptions
# or schema-validated object if schema provided

# Example structure:
# {
#     "title": "Product Name",
#     "description": "Text description from page",
#     "images": [
#         {
#             "url": "https://...",
#             "description": "AI-generated description of what's in the image"
#         }
#     ]
# }

try:
    result = omni_scraper.run()
    if result:
        print("Scraping successful!")
        print(f"Extracted data: {result}")
    else:
        print("No data extracted")
except Exception as e:
    print(f"Error during omni-scraping: {e}")

omni_scraper = OmniScraperGraph(
    prompt="Analyze all images and text in this local page",
    source="/path/to/local/page.html",
    config=graph_config,
)

result = omni_scraper.run()

# Good prompt for image analysis
prompt = """
Extract product information including:
- Product name and price from text
- Visual features visible in images (color, material, design)
- Condition assessment from photos
- Any text visible in product images
"""

# Not as effective
prompt = "Extract product information"