G-Eval Framework

criteria = """Evaluate the normalized claim against the following:
- Verifiability: Can this be fact-checked with reliable sources?
- Self-Containment: Is it understandable without extra context?
- Clarity: Is the language clear and unambiguous?
- Conciseness: Is it brief while preserving meaning?
- Factual Consistency: Does it accurately represent the source?
"""

evaluation_steps = [
    "Check if the claim contains verifiable factual assertions",
    "Verify the claim is self-contained without additional context",
    "Assess if the claim is written in clear language",
    "Confirm the claim is concise yet comprehensive",
    "Ensure factual consistency with source material"
]

from deepeval.test_case import LLMTestCaseParams

evaluation_params = [
    LLMTestCaseParams.INPUT,          # Original post
    LLMTestCaseParams.ACTUAL_OUTPUT   # Normalized claim
]

from deepeval.metrics import GEval
from deepeval.test_case import LLMTestCase, LLMTestCaseParams
from deepeval.models import GPTModel, AnthropicModel, GeminiModel

# Create evaluation model
eval_model = GPTModel(
    model="gpt-4o",
    _openai_api_key="your-api-key"
)

# Define G-Eval metric
metric = GEval(
    name="Claim Quality Assessment",
    criteria=STATIC_EVAL_SPECS.criteria,
    evaluation_steps=STATIC_EVAL_SPECS.evaluation_steps,
    evaluation_params=[LLMTestCaseParams.INPUT, LLMTestCaseParams.ACTUAL_OUTPUT],
    model=eval_model,
    threshold=0.5
)

# Create test case
test_case = LLMTestCase(
    input="Original social media post text...",
    actual_output="Normalized claim text..."
)

# Run evaluation
metric.measure(test_case)

print(f"Score: {metric.score}")      # 0.0 to 1.0
print(f"Reason: {metric.reason}")    # Explanatory feedback

# From /api/types/evals.py:25-50

STATIC_EVAL_SPECS = StaticEvaluation(
    criteria="""Evaluate the normalized claim against the following 
    criteria: Verifiability and Self-Containment, Claim Centrality 
    and Extraction Quality, Conciseness and Clarity, Check-Worthiness 
    Alignment, and Factual Consistency""",
    
    evaluation_steps=[
        # Verifiability and Self-Containment
        "Check if the claim contains verifiable factual assertions 
         that can be independently checked",
        "Check if the claim is self-contained without requiring 
         additional context from the original post",
        
        # Claim Centrality and Extraction Quality
        "Check if the normalized claim captures the central assertion 
         from the source text while removing extraneous information",
        "Check if the claim represents the core factual assertion 
         that requires fact-checking, not peripheral details",
        
        # Conciseness and Clarity
        "Check if the claim is presented in a straightforward, 
         concise manner that fact-checkers can easily process",
        "Check if the claim is significantly shorter than source 
         posts while preserving essential meaning",
        
        # Check-Worthiness Alignment
        "Check if the normalized claim meets check-worthiness 
         standards for fact-verification",
        "Check if the claim has general public interest, potential 
         for harm, and likelihood of being false",
        
        # Factual Consistency
        "Check if the normalized claim is factually consistent 
         with the source material without hallucinations or distortions",
        "Check if the claim accurately reflects the original 
         assertion without introducing new information",
    ]
)

# Default threshold
default_threshold = 0.5

# Refinement loop continues until:
if metric.score >= threshold:
    # Claim passes - no refinement needed
    return final_claim
else:
    # Continue refinement
    refined_claim = refine(claim, feedback)

{
  "refine_threshold": 0.8,  // Higher threshold = stricter quality
  "refine_max_iters": 3     // Maximum refinement attempts
}

eval_result = metric.measure(test_case)

# Example feedback
feedback = eval_result.reason
# "The claim is verifiable (8/10) and self-contained (7/10), 
#  but could be more concise (5/10). The phrase 'according to 
#  multiple sources' is vague and should specify the sources. 
#  Clarity could improve by removing hedging language."

OMG!! 😱 Did you know that eating chocolate 🍫 can 
actually help you lose weight?!? Scientists at Oxford 
found that dark chocolate boosts metabolism by 20%!! 
I'm never dieting again lol 😂 #chocolate #health #science

"Oxford scientists found dark chocolate boosts metabolism by 20%"

"Chocolate may have health benefits"

from deepeval.metrics import GEval

# Custom metric for scientific claims
scientific_accuracy = GEval(
    name="Scientific Accuracy Assessment",
    criteria="""Evaluate whether the claim accurately 
    represents scientific findings without overstatement""",
    evaluation_steps=[
        "Check if the claim distinguishes correlation from causation",
        "Verify the claim doesn't overstate confidence (e.g., 'proves' vs 'suggests')",
        "Assess if the claim acknowledges study limitations",
        "Confirm the claim doesn't generalize beyond the study scope"
    ],
    evaluation_params=[INPUT, ACTUAL_OUTPUT],
    model=eval_model,
    threshold=0.7
)

metrics = [
    verifiability_metric,
    check_worthiness_metric,
    clarity_metric,
    scientific_accuracy_metric
]

for metric in metrics:
    metric.measure(test_case)
    print(f"{metric.name}: {metric.score}")

# Calculate weighted average
weights = [0.3, 0.3, 0.2, 0.2]
overall_score = sum(m.score * w for m, w in zip(metrics, weights))

def refine_with_feedback(
    original_post: str,
    current_claim: str,
    eval_feedback: str
) -> str:
    """
    Use G-Eval feedback to refine the claim.
    """
    refine_prompt = f"""
    ## Original Post
    {original_post}
    
    ## Current Normalized Claim
    {current_claim}
    
    ## Feedback from Quality Assessment
    {eval_feedback}
    
    ## Task
    Refine the normalized claim based on the feedback to improve:
    - Verifiability and self-containment
    - Centrality and extraction quality  
    - Conciseness and clarity
    - Check-worthiness alignment
    - Factual consistency
    
    Return only the refined claim, nothing else.
    """
    
    refined = model.generate(refine_prompt)
    return refined

refinement_history = []
for i in range(max_iters):
    score = evaluate(current_claim)
    
    refinement_history.append({
        "iteration": i,
        "score": score,
        "claim": current_claim
    })
    
    # Check for convergence
    if score >= threshold:
        break
    
    # Check if improvement is minimal
    if i > 0 and abs(score - refinement_history[i-1]["score"]) < 0.05:
        break  # Diminishing returns
    
    current_claim = refine(current_claim, feedback)

# From /api/_utils/deepeval_model.py

class DeepEvalModel:
    def getEvalModel(self):
        if self.api_provider == 'OPENAI':
            return GPTModel(
                model=self.model,
                _openai_api_key=self.api_key
            )
        elif self.api_provider == 'ANTHROPIC':
            return AnthropicModel(
                model=self.model,
                _anthropic_api_key=self.api_key
            )
        elif self.api_provider == 'GEMINI':
            return GeminiModel(
                model=self.model,
                api_key=self.api_key
            )
        elif self.api_provider == 'XAI':
            return GrokModel(
                model=self.model,
                api_key=self.api_key
            )

# Reduce variance with lower temperature
eval_model = GPTModel(
    model="gpt-4o",
    temperature=0.1,  # Lower = more consistent
    _openai_api_key=api_key
)