Cost Management

# model_config.py
def get_benchmark_models() -> list:
    return [
        # All free via OpenRouter!
        BAMLModel.OPENROUTER_DEVSTRAL,
        BAMLModel.OPENROUTER_MIMO_V2_FLASH,
        BAMLModel.OPENROUTER_NEMOTRON_NANO,
        BAMLModel.OPENROUTER_LLAMA_33_70B,
    ]

def get_benchmark_models() -> list:
    return [
        # Free baseline
        BAMLModel.OPENROUTER_DEVSTRAL,
        # Cost-effective
        BAMLModel.DEEPSEEK_CHAT,
        BAMLModel.GEMINI_25_FLASH,
        # Premium comparison
        BAMLModel.GPT5_MINI,
    ]

# benchmark.py
GAMES_PER_COMBINATION = 1  # Quick validation
# vs
GAMES_PER_COMBINATION = 5  # Production benchmark

test_combinations = [
    # Compare GPT-5 vs Claude head-to-head
    (BAMLModel.GPT5_MINI, BAMLModel.GPT5_MINI,
     BAMLModel.CLAUDE_SONNET_45, BAMLModel.CLAUDE_SONNET_45),
    # Test mixed teams
    (BAMLModel.GPT5_MINI, BAMLModel.DEEPSEEK_CHAT,
     BAMLModel.CLAUDE_HAIKU_45, BAMLModel.GEMINI_25_FLASH),
]

total_games = len(test_combinations) * GAMES_PER_COMBINATION

# Instead of expensive models
BAMLModel.GPT5          # ~$0.10/game
BAMLModel.CLAUDE_OPUS_41  # ~$0.30/game

# Use affordable alternatives
BAMLModel.GPT5_MINI     # ~$0.02/game  
BAMLModel.CLAUDE_HAIKU_45  # ~$0.01/game

result = runner.run()

# Access per-model statistics
for model_key, metrics in result.model_performance.items():
    games = metrics['games_played']
    turns = metrics['turns_played']
    print(f"{model_key}: {games} games, {turns} turns")

import json
from config import LLMConfig

# Load results
with open('benchmark_results/benchmark_20260305.json') as f:
    data = json.load(f)

# Calculate costs
costs = LLMConfig.MODEL_COSTS
total_cost = 0

for game in data['games']:
    for role in ['blue_hint_giver_model', 'blue_guesser_model',
                 'red_hint_giver_model', 'red_guesser_model']:
        model = game[role]
        # Estimate based on turns and model pricing
        # (Actual calculation would need token counts)
        
print(f"Estimated total cost: ${total_cost:.2f}")

# Regular pricing
GPT5: $1.75/1M input, $14/1M output

# Cached input (repeated prompts)
GPT5: $0.175/1M input, $14/1M output

# Cache the Codenames rules and board state
# Reduces costs for multi-turn games

from config import LLMConfig

def estimate_benchmark_cost(result_file):
    with open(result_file) as f:
        data = json.load(f)
    
    costs = LLMConfig.MODEL_COSTS
    total = 0
    
    for model_key, metrics in data['model_performance'].items():
        model_name = metrics['model']
        games = metrics['games_played']
        
        # Estimate tokens (simplified)
        est_input = games * 4000
        est_output = games * 750
        
        if model_name in costs:
            pricing = costs[model_name]
            cost = (est_input/1000 * pricing['input'] + 
                   est_output/1000 * pricing['output'])
            total += cost
            
            print(f"{model_key}: ${cost:.2f}")
    
    print(f"\nTotal estimated: ${total:.2f}")

estimate_benchmark_cost('benchmark_results/benchmark_20260305.json')