Browser Automation Examples

import verifiers as vf
from verifiers.envs.integrations.browser_env import BrowserEnv
from datasets import Dataset

DOM_SYSTEM_PROMPT = """You are a browser automation agent using Stagehand's AI-driven tools.

Available tools:
- navigate(url): Navigate to a URL
- observe(instruction): Find possible actions matching the instruction
- act(instruction): Execute an action described in natural language
- extract(instruction, schema_json): Extract structured data from the page

Use natural language to describe what you want to do. Stagehand will intelligently
find elements and execute actions without needing CSS selectors or coordinates.

Complete the given task efficiently."""

def create_example_dataset() -> Dataset:
    return Dataset.from_dict({
        "question": [
            "What does the headline say on the primeintellect.ai homepage?"
        ],
        "answer": ["The Open Superintelligence Stack"],
        "start_url": ["https://primeintellect.ai"],
        "task_id": ["dom-example-0"],
    })

def load_environment(
    project_id: str,
    max_turns: int = 10,
    judge_model: str = "gpt-4o-mini",
    system_prompt: str = DOM_SYSTEM_PROMPT,
    browserbase_api_key_var: str = "BROWSERBASE_API_KEY",
    stagehand_model: str = "openai/gpt-4o-mini",
    model_api_key_var: str = "MODEL_API_KEY",
    proxy_model_to_stagehand: bool = False,
    **kwargs,
) -> vf.Environment:
    import os

    # Check required env vars
    missing = []
    if not os.getenv(browserbase_api_key_var):
        missing.append(browserbase_api_key_var)
    if not os.getenv(model_api_key_var):
        missing.append(model_api_key_var)

    if missing:
        raise ValueError(
            f"Missing required environment variables: {', '.join(missing)}"
        )

    dataset = create_example_dataset()

    # Create judge rubric
    rubric = vf.JudgeRubric(
        judge_model=judge_model,
        judge_prompt=JUDGE_PROMPT,
    )
    rubric.add_reward_func(judge_answer, weight=1.0)

    return BrowserEnv(
        mode="dom",
        dataset=dataset,
        rubric=rubric,
        max_turns=max_turns,
        system_prompt=system_prompt,
        project_id=project_id,
        browserbase_api_key_var=browserbase_api_key_var,
        stagehand_model=stagehand_model,
        model_api_key_var=model_api_key_var,
        proxy_model_to_stagehand=proxy_model_to_stagehand,
        **kwargs,
    )

from typing import Literal
import verifiers as vf
from verifiers.envs.integrations.browser_env import BrowserEnv
from datasets import Dataset

CUA_SYSTEM_PROMPT = """You are a browser automation agent. You can control a web browser using the provided tools.

Available tools:
- click(x, y, button): Click at coordinates
- double_click(x, y): Double-click at coordinates
- type_text(text): Type text into focused element
- keypress(keys): Press keyboard keys
- scroll(x, y, scroll_x, scroll_y): Scroll at position
- goto(url): Navigate to URL
- back(): Go back in history
- forward(): Go forward in history
- wait(time_ms): Wait for specified milliseconds
- screenshot(): Capture current page state

After each action, you will receive a screenshot showing the current page state.
Analyze the screenshot to determine your next action.

Complete the given task efficiently using the minimum number of actions necessary."""

def load_environment(
    max_turns: int = 15,
    judge_model: str = "gpt-4o-mini",
    system_prompt: str = CUA_SYSTEM_PROMPT,
    # CUA mode configuration
    use_sandbox: bool = True,
    server_url: str = "http://localhost:3000",
    # Browserbase configuration
    browserbase_api_key: str | None = None,
    browserbase_project_id: str | None = None,
    env: Literal["LOCAL", "BROWSERBASE"] = "BROWSERBASE",
    # Pre-built image (fastest startup)
    use_prebuilt_image: bool = True,
    prebuilt_image: str = "deepdream19/cua-server:latest",
    **kwargs,
) -> vf.Environment:
    dataset = create_example_dataset()

    rubric = vf.JudgeRubric(
        judge_model=judge_model,
        judge_prompt=JUDGE_PROMPT,
    )
    rubric.add_reward_func(judge_answer, weight=1.0)

    return BrowserEnv(
        mode="cua",
        dataset=dataset,
        rubric=rubric,
        max_turns=max_turns,
        system_prompt=system_prompt,
        use_sandbox=use_sandbox,
        server_url=server_url,
        env=env,
        browserbase_api_key=browserbase_api_key,
        browserbase_project_id=browserbase_project_id,
        use_prebuilt_image=use_prebuilt_image,
        prebuilt_image=prebuilt_image,
        **kwargs,
    )