METEOR Scoring

from nltk.translate.meteor_score import meteor_score

# Example usage
reference = ["The FDA approved COVID-19 vaccine for children aged 5 to 11".split()]
candidate = "FDA approves COVID vaccine for kids ages 5-11".split()

score = meteor_score(reference, candidate)
print(f"METEOR Score: {score:.4f}")  # Range: 0.0 to 1.0

Reference: The FDA approved COVID-19 vaccine for children aged 5 to 11
Candidate: FDA approves COVID vaccine for kids ages 5-11
           ^^^ ^^^^^^^ ^^^^^ ^^^^^^^ ^^^ ^^^^ ^^ ^^ ^^
Exact:     FDA         COVID vaccine            5    11

Reference: approved
Candidate: approves
Stem:      approv* → MATCH

Reference: children
Candidate: kids
WordNet:   children ≈ kids → MATCH

Reference: aged 5 to 11
Candidate: ages 5-11
Paraphrase: Similar phrasing → MATCH

# Count matched words
matches = count_aligned_words()

# Precision: What portion of candidate words matched?
precision = matches / len(candidate_words)

# Recall: What portion of reference words matched?
recall = matches / len(reference_words)

Reference: "The FDA approved COVID-19 vaccine for children aged 5 to 11" (11 words)
Candidate: "FDA approves COVID vaccine for kids ages 5-11" (9 words)
Matches: 9 (including stems/synonyms)

Precision = 9 / 9 = 1.00 (all candidate words matched)
Recall = 9 / 11 = 0.82 (missed "The" and "to")

# METEOR weights recall more heavily
alpha = 0.9  # Standard value

# Harmonic mean
if precision > 0 and recall > 0:
    f_mean = (10 * precision * recall) / (recall + alpha * precision)
else:
    f_mean = 0

Good alignment (continuous):
Reference: The cat sat on the mat
Candidate: cat sat on mat
           ^^^^^^^^^^^^^^^^^ (1 chunk)

Bad alignment (fragmented):
Reference: The cat sat on the mat  
Candidate: mat on sat cat
           ^^^ ^^ ^^^ ^^^ (4 chunks)

# Count alignment chunks
chunks = count_contiguous_alignment_chunks()

# Calculate penalty
fragmentation = chunks / matches
penalty = 0.5 * (fragmentation ** 3)

# Final score
meteor_score = f_mean * (1 - penalty)

# From /home/daytona/workspace/source/claim_norm.py:45-47

METEOR_SCORE = start_extraction(
    MODEL, 
    PROMPT_STYLE, 
    DEV_DATA[0:1], 
    REFINE_ITERATIONS, 
    CROSS_REFINE_MODEL
)

print(f"\nAverage METEOR Score: {METEOR_SCORE}")

import pandas as pd
from nltk.translate.meteor_score import meteor_score

# Load dataset with reference claims
dev_data = pd.read_csv("data/dev.csv")

meteor_scores = []
for idx, row in dev_data.iterrows():
    original_post = row['original_post']
    reference_claim = row['normalized_claim']
    
    # Generate candidate claim
    candidate_claim = model.normalize(original_post)
    
    # Calculate METEOR
    score = meteor_score(
        [reference_claim.split()],
        candidate_claim.split()
    )
    
    meteor_scores.append(score)

# Report results
average_meteor = sum(meteor_scores) / len(meteor_scores)
print(f"Average METEOR Score: {average_meteor:.4f}")

Reference: "COVID-19 vaccines are safe and effective"
Candidate: "COVID-19 vaccines pose serious health risks"

METEOR: ~0.65 (high surface similarity)
Semantic: Opposite meaning!

Post: "Elon Musk's company SpaceX launched 60 satellites yesterday"

Valid Claim 1: "SpaceX launched 60 satellites"
Valid Claim 2: "Elon Musk's company launched 60 satellites"
Valid Claim 3: "60 SpaceX satellites were launched"

METEOR only compares against one reference!

Reference: "The FDA approved Pfizer vaccine" (5 words)

Candidate A: "FDA approved Pfizer vaccine" (4 words)
METEOR: 0.95 (high - very similar)

Candidate B: "The FDA regulatory agency approved the Pfizer 
               COVID-19 vaccine" (9 words)
METEOR: 0.72 (lower - added words penalized)

Reference: "COVID-19"
Candidate: "coronavirus"

WordNet: No synonym relationship
METEOR: 0.0 (no match)

Human judgment: Clearly related!

BREAKING: The World Health Organization just announced that they 
are investigating a new variant of concern that has emerged in 
South Africa!!! 😱 This could change everything! #COVID #Variant

"WHO is investigating a new COVID-19 variant in South Africa"

"The global health authority is examining an emerging strain 
of the coronavirus disease discovered in a southern African nation"

"The WHO is investigating a new COVID-19 variant in South Africa"

post_id,normalized_claim
001,"WHO is investigating a new COVID-19 variant in South Africa"
002,"FDA approved Pfizer vaccine for children aged 5 to 11"
003,"Studies show coffee consumption linked to reduced cancer risk"
...

# Simplified version of competition scoring

import pandas as pd
from nltk.translate.meteor_score import meteor_score

def evaluate_submission(predictions_file, references_file):
    """
    Evaluate competition submission using METEOR.
    """
    predictions = pd.read_csv(predictions_file)
    references = pd.read_csv(references_file)
    
    scores = []
    for pred_row, ref_row in zip(predictions.iterrows(), 
                                   references.iterrows()):
        pred_claim = pred_row[1]['normalized_claim']
        ref_claim = ref_row[1]['normalized_claim']
        
        score = meteor_score(
            [ref_claim.split()],
            pred_claim.split()
        )
        scores.append(score)
    
    return {
        'average_meteor': sum(scores) / len(scores),
        'median_meteor': sorted(scores)[len(scores)//2],
        'scores_per_post': scores
    }

results = evaluate_submission(
    'predictions.csv',
    'references.csv'
)

print(f"Average METEOR: {results['average_meteor']:.4f}")

def evaluate_claim_quality(
    original_post: str,
    normalized_claim: str,
    reference_claim: Optional[str] = None
) -> Dict[str, Any]:
    """
    Comprehensive evaluation using both METEOR and G-Eval.
    """
    results = {}
    
    # G-Eval: Nuanced quality assessment
    geval_metric = GEval(
        criteria=STATIC_EVAL_SPECS.criteria,
        evaluation_steps=STATIC_EVAL_SPECS.evaluation_steps,
        model=eval_model,
        threshold=0.5
    )
    
    test_case = LLMTestCase(
        input=original_post,
        actual_output=normalized_claim
    )
    
    geval_metric.measure(test_case)
    results['geval_score'] = geval_metric.score
    results['geval_feedback'] = geval_metric.reason
    
    # METEOR: Reference-based similarity
    if reference_claim:
        meteor = meteor_score(
            [reference_claim.split()],
            normalized_claim.split()
        )
        results['meteor_score'] = meteor
    
    # Combined decision
    results['overall_quality'] = 'HIGH' if (
        results['geval_score'] >= 0.7 and 
        results.get('meteor_score', 1.0) >= 0.6
    ) else 'NEEDS_REFINEMENT'
    
    return results

# Installation
pip install nltk

# Download required data
python -c "import nltk; nltk.download('wordnet'); nltk.download('omw-1.4')"

from nltk.translate.meteor_score import meteor_score
from nltk.translate.meteor_score import single_meteor_score

# Single reference
score = meteor_score(
    [reference.split()],  # List of references (usually 1)
    candidate.split()      # Candidate claim
)

# Multiple references (rare in competition)
scores = [
    meteor_score([ref.split()], candidate.split())
    for ref in references
]
best_score = max(scores)

# Batch processing for speed
import multiprocessing as mp

def compute_meteor(args):
    reference, candidate = args
    return meteor_score([reference.split()], candidate.split())

# Parallel computation
with mp.Pool(processes=8) as pool:
    scores = pool.map(
        compute_meteor,
        zip(references, candidates)
    )

average_meteor = sum(scores) / len(scores)