V3Evaluator Class

import { V3Evaluator } from "@browserbasehq/stagehand";

const evaluator = new V3Evaluator(
  stagehand,
  modelName?,
  modelClientOptions?
);

const result = await evaluator.ask(options);

const results = await evaluator.batchAsk(options);

import { Stagehand, V3Evaluator } from "@browserbasehq/stagehand";

const stagehand = new Stagehand({ env: "LOCAL" });
await stagehand.init();

const page = await stagehand.context.newPage();
await page.goto("https://example.com");

// Perform actions
await stagehand.act("click the login button");
await stagehand.act("type '[email protected]' in email field");
await stagehand.act("type 'password123' in password field");
await stagehand.act("click submit");

// Evaluate result
const evaluator = new V3Evaluator(stagehand);
const result = await evaluator.ask({
  question: "Is the user successfully logged in?",
  screenshot: true,
});

if (result.evaluation === "YES") {
  console.log("Login successful!");
  console.log("Reasoning:", result.reasoning);
} else {
  console.log("Login failed.");
  console.log("Reasoning:", result.reasoning);
}

await stagehand.close();

const evaluator = new V3Evaluator(stagehand);

const result = await evaluator.ask({
  question: "What is the page title?",
  answer: "Welcome to Example Site",
  screenshot: true,
});

if (result.evaluation === "YES") {
  console.log("Title matches expected value");
} else {
  console.log("Title does not match:", result.reasoning);
}

const agent = stagehand.agent();
const agentResult = await agent.execute(
  "Book a flight from NYC to LA"
);

// Format agent actions for evaluation
const agentReasoning = agentResult.actions
  .map((a, i) => `${i + 1}. ${a.type}: ${a.reasoning || a.action}`)
  .join("\n");

const evaluator = new V3Evaluator(stagehand);
const result = await evaluator.ask({
  question: "Did the agent successfully book a flight?",
  agentReasoning,
  screenshot: true,
});

console.log("Success:", result.evaluation === "YES");
console.log("Reasoning:", result.reasoning);

const evaluator = new V3Evaluator(stagehand);

const results = await evaluator.batchAsk({
  questions: [
    { question: "Is the search bar visible?" },
    { question: "Is the user logged in?" },
    { question: "Are there any error messages?" },
    { 
      question: "What is the product price?",
      answer: "$29.99"
    },
  ],
  screenshot: true,
});

results.forEach((result, i) => {
  console.log(`\nQuestion ${i + 1}:`);
  console.log(`Evaluation: ${result.evaluation}`);
  console.log(`Reasoning: ${result.reasoning}`);
});

// Use a different model for evaluation
const evaluator = new V3Evaluator(
  stagehand,
  "anthropic/claude-3-5-sonnet-latest",
  {
    apiKey: process.env.ANTHROPIC_API_KEY,
    temperature: 0.3,
  }
);

const result = await evaluator.ask({
  question: "Is the checkout process complete?",
});

const evaluator = new V3Evaluator(stagehand);

// Evaluate based on agent reasoning only
const result = await evaluator.ask({
  question: "Did the task complete successfully?",
  screenshot: false,
  agentReasoning: `
    Step 1: Navigated to homepage
    Step 2: Clicked login button
    Step 3: Filled username and password
    Step 4: Submitted form
    Step 5: Redirected to dashboard
  `,
});

import fs from "fs";

const screenshot1 = fs.readFileSync("./before.png");
const screenshot2 = fs.readFileSync("./after.png");

const evaluator = new V3Evaluator(stagehand);

const result = await evaluator.ask({
  question: "Did the page content change as expected?",
  screenshot: [screenshot1, screenshot2],
});

import { test, expect } from "@playwright/test";
import { Stagehand, V3Evaluator } from "@browserbasehq/stagehand";

test("user can complete checkout", async () => {
  const stagehand = new Stagehand({ env: "LOCAL" });
  await stagehand.init();
  
  const page = await stagehand.context.newPage();
  await page.goto("https://example-shop.com");
  
  // Perform checkout steps
  await stagehand.act("add first item to cart");
  await stagehand.act("go to checkout");
  await stagehand.act("fill in shipping information");
  await stagehand.act("submit order");
  
  // Evaluate success
  const evaluator = new V3Evaluator(stagehand);
  const result = await evaluator.ask({
    question: "Was the order successfully placed?",
    screenshot: true,
  });
  
  expect(result.evaluation).toBe("YES");
  
  await stagehand.close();
});

const evaluator = new V3Evaluator(stagehand);

const result = await evaluator.ask({
  question: "Is the dashboard showing the correct data?",
  systemPrompt: `You are an expert QA engineer evaluating UI tests.
    Be strict about validation and look for any inconsistencies.
    Check that all expected elements are present and correct.
    Today's date is ${new Date().toLocaleDateString()}`,
  screenshot: true,
});