ScriptCreatorGraph

ScriptCreatorGraph(
    prompt: str,              # Description of what data to extract
    source: str,              # URL or path to local HTML file
    config: dict,             # Configuration dictionary (must include 'library')
    schema: Optional[BaseModel] = None  # Pydantic schema for structured output
)

result = script_creator.run()

# Example output:
'''
from bs4 import BeautifulSoup

def extract_data(html: str) -> dict:
    """
    Extract news articles with their descriptions from the HTML.
    
    Args:
        html: Raw HTML string of the page
    
    Returns:
        Dictionary containing extracted news articles
    """
    soup = BeautifulSoup(html, 'html.parser')
    articles = []
    
    # Find all article elements
    for article in soup.find_all('article', class_='post'):
        title = article.find('h2').text.strip()
        description = article.find('p', class_='description').text.strip()
        
        articles.append({
            'title': title,
            'description': description
        })
    
    return {'articles': articles}
'''

import requests
from scrapegraphai.graphs import ScriptCreatorGraph

# Generate the script
script_creator = ScriptCreatorGraph(
    prompt="Extract all product names and prices",
    source="https://example.com/products",
    config=graph_config,
)

generated_code = script_creator.run()

# Save to file
with open("my_scraper.py", "w") as f:
    f.write(generated_code)

# Later, import and use it
from my_scraper import extract_data

html = requests.get("https://example.com/products").text
data = extract_data(html)
print(data)

from pydantic import BaseModel, Field
from typing import List

class Article(BaseModel):
    title: str = Field(description="Article title")
    description: str = Field(description="Article description")
    author: str = Field(description="Author name")
    date: str = Field(description="Publication date")

class ArticleList(BaseModel):
    articles: List[Article]

graph_config = {
    "llm": {"model": "openai/gpt-4o", "api_key": os.getenv("OPENAI_API_KEY")},
    "library": "beautifulsoup",
}

script_creator = ScriptCreatorGraph(
    prompt="Extract all articles with their metadata",
    source="https://blog.example.com",
    config=graph_config,
    schema=ArticleList
)

code = script_creator.run()
# Generated code will match the schema structure

script_creator = ScriptCreatorGraph(
    prompt="Extract all table data",
    source="/path/to/local/page.html",
    config=graph_config,
)

code = script_creator.run()

from pydantic import BaseModel
from typing import List, Optional

class Product(BaseModel):
    name: str
    price: float
    description: Optional[str]
    rating: Optional[float]
    image_url: Optional[str]

class ProductList(BaseModel):
    products: List[Product]

graph_config = {
    "llm": {
        "api_key": os.getenv("OPENAI_API_KEY"),
        "model": "openai/gpt-4o",
    },
    "library": "beautifulsoup",
    "verbose": True,
}

script_creator = ScriptCreatorGraph(
    prompt="Extract all products with name, price, description, rating, and image URL",
    source="https://example-shop.com/products",
    config=graph_config,
    schema=ProductList
)

scraper_code = script_creator.run()

# Save the generated scraper
with open("product_scraper.py", "w") as f:
    f.write(scraper_code)

print("Scraper generated successfully!")

import requests
from scrapegraphai.graphs import ScriptCreatorGraph

# Generate script
script_creator = ScriptCreatorGraph(
    prompt="Extract article titles and summaries",
    source="https://news.example.com",
    config=graph_config,
)

code = script_creator.run()

# Execute the generated code
exec(code)

# Test the function
html = requests.get("https://news.example.com").text
try:
    result = extract_data(html)
    print("Generated scraper works!")
    print(f"Extracted {len(result)} items")
except Exception as e:
    print(f"Error in generated code: {e}")

try:
    code = script_creator.run()
    if code and "def extract_data" in code:
        print("Script generated successfully!")
        # Save or use the code
        with open("scraper.py", "w") as f:
            f.write(code)
    else:
        print("Failed to generate valid code")
except Exception as e:
    print(f"Error during code generation: {e}")