pip install --upgrade google-cloud-aiplatform[evaluation]

import vertexai
from vertexai.evaluation import EvalTask
from vertexai.generative_models import GenerativeModel
from vertexai.preview.evaluation import notebook_utils
import pandas as pd

PROJECT_ID = "your-project-id"
LOCATION = "us-central1"

vertexai.init(project=PROJECT_ID, location=LOCATION)

instruction = "Summarize the following article: "

contexts = [
    "To make classic spaghetti carbonara, start by bringing salted water to a boil. Cook pancetta in olive oil until crispy. Whisk eggs and Parmesan cheese. Toss cooked pasta with the egg mixture and pasta water to create a creamy sauce.",
    "Preparing perfect risotto requires patience. Heat butter, add chopped onions and garlic, cook until soft. Add Arborio rice and toast. Add white wine, then gradually add hot broth while stirring until creamy.",
    "For flavorful grilled steak, season ribeye generously with salt and pepper. Preheat grill to high heat. Grill for 4-5 minutes per side for medium-rare. Let rest before slicing.",
    "Creating homemade tomato soup starts with heating olive oil. Sauté onions and garlic until fragrant. Add chopped tomatoes, broth, and basil. Simmer for 20-30 minutes. Puree until smooth and season.",
    "To bake chocolate cake, cream butter and sugar until fluffy. Beat in eggs one at a time. Alternate adding dry ingredients and buttermilk. Bake at 350°F for 25-30 minutes."
]

references = [
    "Making spaghetti carbonara involves boiling pasta, crisping pancetta, whisking eggs and Parmesan, and tossing everything together.",
    "Preparing risotto entails sautéing aromatics, toasting rice, adding wine and broth gradually, and stirring until creamy.",
    "Grilling steak involves seasoning generously, preheating the grill, cooking to desired doneness, and resting before slicing.",
    "Creating tomato soup includes sautéing aromatics, simmering with tomatoes and broth, pureeing, and seasoning.",
    "Baking chocolate cake requires creaming butter and sugar, beating in eggs, alternating dry ingredients with buttermilk, and baking."
]

eval_dataset = pd.DataFrame({
    "prompt": [instruction + ctx for ctx in contexts],
    "reference": references
})

metrics = [
    # Reference-based metrics
    "rouge_l_sum",
    "bleu",
    
    # Model-based metrics
    "fluency",
    "coherence",
    "safety",
    "groundedness",
    "verbosity",
    "text_quality",
    "summarization_quality"
]

notebook_utils.display_radar_plot(
    results,
    metrics=[
        "fluency",
        "coherence",
        "safety",
        "groundedness",
        "verbosity",
        "text_quality",
        "summarization_quality"
    ]
)

notebook_utils.display_bar_plot(
    results,
    metrics=["rouge_l_sum", "bleu"]
)

models_to_compare = [
    ("gemini-2.0-flash", GenerativeModel("gemini-2.0-flash")),
    ("gemini-1.5-flash", GenerativeModel("gemini-1.5-flash")),
    ("gemini-1.5-pro", GenerativeModel("gemini-1.5-pro"))
]

all_results = []

for name, model in models_to_compare:
    result = eval_task.evaluate(
        model=model,
        experiment_run_name=f"eval-{name}"
    )
    all_results.append((name, result))
    
# Compare all models
notebook_utils.display_radar_plot(
    all_results,
    metrics=["coherence", "fluency", "safety", "text_quality"]
)

configurations = [
    {"name": "low-temp", "temperature": 0.2},
    {"name": "medium-temp", "temperature": 0.5},
    {"name": "high-temp", "temperature": 0.9}
]

config_results = []

for config in configurations:
    model = GenerativeModel(
        "gemini-2.0-flash",
        generation_config={"temperature": config["temperature"]}
    )
    
    result = eval_task.evaluate(
        model=model,
        experiment_run_name=f"eval-{config['name']}"
    )
    
    config_results.append((config["name"], result))

notebook_utils.display_bar_plot(
    config_results,
    metrics=["coherence", "fluency"]
)

prompt_styles = [
    ("direct", "Summarize: {context}"),
    ("detailed", "Please provide a concise summary of: {context}"),
    ("structured", "Create a summary with key points from: {context}")
]

for style_name, template in prompt_styles:
    style_dataset = pd.DataFrame({
        "prompt": [template.format(context=ctx) for ctx in contexts],
        "reference": references
    })
    
    eval_task = EvalTask(
        dataset=style_dataset,
        metrics=metrics,
        experiment=f"prompt-style-{style_name}"
    )
    
    result = eval_task.evaluate(model=gemini_model)

# Healthcare example
healthcare_dataset = pd.DataFrame({
    "prompt": [
        "Explain type 2 diabetes to a patient",
        "What are symptoms of hypertension?"
    ],
    "reference": [
        "Type 2 diabetes affects how your body processes blood sugar...",
        "Hypertension symptoms include headaches, shortness of breath..."
    ]
})

eval_task = EvalTask(
    dataset=healthcare_dataset,
    metrics=["coherence", "safety", "text_quality"],
    experiment="healthcare-comparison"
)

# Compare models on domain data
results = []
for name, model in models_to_compare:
    result = eval_task.evaluate(model=model)
    results.append((name, result))

eval_task = EvalTask(
    dataset=eval_dataset,
    metrics=metrics,
    experiment="production-model-evaluation"
)

# Weekly evaluation runs
result_week1 = eval_task.evaluate(
    model=model,
    experiment_run_name="2024-03-week1"
)

result_week2 = eval_task.evaluate(
    model=model,
    experiment_run_name="2024-03-week2"
)

# Compare across time
eval_task.display_runs()