Evaluation Tasks

from tasks.arc import ARC

# Load the task
task = ARC(subset="ARC-Easy", split="test")

# Get an example
conversation = task[0]
# {
#   "messages": [
#     {"role": "user", "content": "Question: ...\n\nA) ...\nB) ...\nC) ...\nD) ..."},
#     {"role": "assistant", "content": "A"}
#   ],
#   "letters": ["A", "B", "C", "D"]
# }

# Evaluate
is_correct = task.evaluate(conversation, "A")

def evaluate(self, conversation, assistant_response):
    assistant_message = conversation['messages'][-1]['content']
    return assistant_response == assistant_message

from tasks.mmlu import MMLU

# Load all subjects
task = MMLU(subset="all", split="test")

# Get an example
conversation = task[0]
# {
#   "messages": [...],
#   "subject": "college_biology",
#   "letters": ("A", "B", "C", "D")
# }

Question:
Weng earns $12 an hour for babysitting. Yesterday, she just did 50 minutes of babysitting. How much did she earn?

Answer:
Weng earns 12/60 = $<<12/60=0.2>>0.2 per minute.
Working 50 minutes, she earned 0.2 x 50 = $<<0.2*50=10>>10.
#### 10

from tasks.gsm8k import GSM8K

task = GSM8K(subset="main", split="test")

# Conversations include tool calls
conversation = task[0]
# {
#   "messages": [
#     {"role": "user", "content": "Weng earns $12..."},
#     {"role": "assistant", "content": [
#       {"type": "text", "text": "Weng earns 12/60 = $"},
#       {"type": "python", "text": "12/60"},
#       {"type": "python_output", "text": "0.2"},
#       {"type": "text", "text": "0.2 per minute...\n#### 10"}
#     ]}
#   ]
# }

def evaluate(self, conversation, assistant_response):
    # Extract ground truth after #### marker
    ref_num = extract_answer(ground_truth)  # e.g., "10"
    # Extract prediction after #### marker
    pred_num = extract_answer(assistant_response)
    # Compare
    return int(pred_num == ref_num)

from tasks.humaneval import HumanEval

task = HumanEval()

conversation = task[0]
# {
#   "messages": [
#     {"role": "user", "content": "def add(a, b):\n    ..."},
#     {"role": "assistant", "content": "def add(a, b):\n    return a + b"}
#   ],
#   "entry_point": "add",
#   "test": "def check(candidate): ..."
# }

def evaluate(self, conversation, completion):
    # Extract imports from prompt
    imports = extract_imports(conversation['messages'][0]['content'])
    # Extract code from completion (handles markdown blocks)
    code = extract_program(completion)
    # Build full program
    program = (
        imports + "\n\n" +
        code + "\n\n" +
        conversation['test'] + "\n" +
        f"check({conversation['entry_point']})"
    )
    # Execute and check if it passes
    result = execute_code(program)
    return result.success

User: How many r are in strawberry?

Assistant:
We are asked to find the number 'r' in the word 'strawberry'. Let me try a manual approach first.

First spell the word out:
strawberry:s,t,r,a,w,b,e,r,r,y

Then count the occurrences of 'r':
1:s
2:t
3:r hit! count=1
4:a
5:w
6:b
7:e
8:r hit! count=2
9:r hit! count=3
10:y

This gives us 3.

Let me double check this using Python:

<<'strawberry'.count('r')=3>>

Python gives us 3.

My final answer is:

#### 3

from tasks.spellingbee import SpellingBee

task = SpellingBee(size=256, split="test")

# Examples are procedurally generated
conversation = task[0]
# User messages have variations (30+ templates, multilingual)
# Assistant responses include manual counting + Python verification

from tasks.smoltalk import SmolTalk

task = SmolTalk(split="train")

conversation = task[0]
# {
#   "messages": [
#     {"role": "system", "content": "..."},  # Optional
#     {"role": "user", "content": "..."},
#     {"role": "assistant", "content": "..."},
#     {"role": "user", "content": "..."},
#     {"role": "assistant", "content": "..."},
#     # ... alternating user/assistant
#   ]
# }

python -m scripts.chat_eval \
  --source sft \
  --task-name ARC-Easy

python -m scripts.chat_eval \
  --source sft \
  --task-name "ARC-Easy|MMLU|GSM8K"

python -m scripts.chat_eval --source sft

torchrun --nproc_per_node=8 -m scripts.chat_eval \
  --source sft \
  --task-name "ARC-Easy|ARC-Challenge|MMLU|GSM8K|HumanEval"

python -m scripts.chat_eval \
  --source sft \
  --task-name ARC-Easy \
  --max-problems 100

# Center each task against its random baseline
for task_name, acc in results.items():
    baseline_acc = baseline_accuracies[task_name]
    centered_acc = (acc - baseline_acc) / (1.0 - baseline_acc)
    centered_mean += centered_acc

chatcore_metric = centered_mean / len(results)

class Task:
    @property
    def eval_type(self):
        """Return 'categorical' or 'generative'"""
        raise NotImplementedError
    
    def num_examples(self):
        """Total number of examples"""
        raise NotImplementedError
    
    def get_example(self, index):
        """Get example at index as a conversation dict"""
        raise NotImplementedError
    
    def evaluate(self, conversation, assistant_response):
        """Return True/False or 0/1 for correctness"""
        raise NotImplementedError
    
    def __len__(self):
        return self.num_examples()
    
    def __getitem__(self, index):
        return self.get_example(index)