Evaluator-Optimizer Workflow

import asyncio
from fast_agent import FastAgent

fast = FastAgent("Evaluator-Optimizer")

# Define generator agent
@fast.agent(
    name="generator",
    instruction="""You are a career coach specializing in cover letter writing.
    You are tasked with generating a compelling cover letter given the job posting,
    candidate details, and company information. Tailor the response to the company 
    and job requirements.""",
    servers=["fetch"],
    model="gpt-5-nano.low",
    use_history=True,
)

# Define evaluator agent
@fast.agent(
    name="evaluator",
    instruction="""Evaluate the following response based on the criteria below:
    1. Clarity: Is the language clear, concise, and grammatically correct?
    2. Specificity: Does the response include relevant and concrete details?
    3. Relevance: Does the response align with the prompt?
    4. Tone and Style: Is the tone professional and appropriate?
    5. Persuasiveness: Does the response effectively highlight value?
    6. Grammar and Mechanics: Are there spelling or grammatical issues?
    7. Feedback Alignment: Has the response addressed previous feedback?

    For each criterion:
    - Provide a rating (EXCELLENT, GOOD, FAIR, or POOR)
    - Offer specific feedback or suggestions for improvement

    Summarize your evaluation with overall quality rating and specific feedback.""",
    model="o3-mini.medium",
)

# Define the evaluator-optimizer workflow
@fast.evaluator_optimizer(
    name="cover_letter_writer",
    generator="generator",
    evaluator="evaluator",
    min_rating="EXCELLENT",
    max_refinements=3,
)
async def main() -> None:
    async with fast.run() as agent:
        job_posting = (
            "Software Engineer at LastMile AI. Responsibilities include developing AI systems, "
            "collaborating with cross-functional teams, and enhancing scalability. Skills required: "
            "Python, distributed systems, and machine learning."
        )
        candidate_details = (
            "Alex Johnson, 3 years in machine learning, contributor to open-source AI projects, "
            "proficient in Python and TensorFlow. Motivated by building scalable AI systems."
        )
        company_information = (
            "Look up from the LastMile AI About page: https://lastmileai.dev/about"
        )

        await agent.cover_letter_writer.send(
            f"Write a cover letter for the following job posting: {job_posting}\n\n"
            f"Candidate Details: {candidate_details}\n\n"
            f"Company information: {company_information}",
        )

if __name__ == "__main__":
    asyncio.run(main())

class QualityRating(str, Enum):
    POOR = "POOR"          # Major improvements needed
    FAIR = "FAIR"          # Several improvements needed
    GOOD = "GOOD"          # Minor improvements possible
    EXCELLENT = "EXCELLENT" # No improvements needed

class EvaluationResult(BaseModel):
    rating: QualityRating
    feedback: str
    needs_improvement: bool
    focus_areas: List[str]

{
  "rating": "GOOD",
  "feedback": "Cover letter demonstrates strong technical alignment but could better highlight specific achievements and company research.",
  "needs_improvement": true,
  "focus_areas": [
    "Add quantifiable achievements from past projects",
    "Reference specific LastMile AI products or initiatives",
    "Strengthen the closing call-to-action"
  ]
}

@fast.agent(
    name="generator",
    instruction="Your instruction here",
    use_history=True,  # Generator sees conversation context
)

You are tasked with improving your previous response.
This is iteration 2 of the refinement process.

<fastagent:feedback>
  <rating>GOOD</rating>
  <details>Strong technical content but needs more specific examples</details>
  <focus-areas>
    * Add quantifiable achievements
    * Reference specific company initiatives
  </focus-areas>
</fastagent:feedback>

Create an improved version...

@fast.agent(
    "researcher",
    instruction="""Research a topic thoroughly using available sources.
    Produce a comprehensive, well-cited report with clear sections.""",
    servers=["fetch"],
    model="sonnet",
    use_history=True,
)
@fast.agent(
    "research_evaluator",
    instruction="""Evaluate research reports on:
    1. Depth of research and source quality
    2. Accuracy and factual correctness
    3. Organization and structure
    4. Citation quality and completeness
    5. Clarity and readability
    
    Provide detailed, actionable feedback for improvement.""",
    model="sonnet",
)
@fast.evaluator_optimizer(
    name="research_assistant",
    generator="researcher",
    evaluator="research_evaluator",
    min_rating="EXCELLENT",
    max_refinements=5,
)
async def main() -> None:
    async with fast.run() as agent:
        await agent.research_assistant.send(
            "Produce a comprehensive report on the environmental impact of cryptocurrency mining"
        )

@fast.agent(
    "code_generator",
    instruction="""Generate clean, well-documented Python code that solves the given problem.
    Follow PEP 8 style guidelines and include docstrings.""",
    servers=["filesystem"],
    use_history=True,
)
@fast.agent(
    "code_reviewer",
    instruction="""Review code for:
    1. Correctness and bug-free implementation
    2. Code style and PEP 8 compliance
    3. Documentation quality
    4. Performance and efficiency
    5. Error handling
    6. Test coverage
    
    Provide specific line-level feedback.""",
    model="sonnet",
)
@fast.evaluator_optimizer(
    name="code_improver",
    generator="code_generator",
    evaluator="code_reviewer",
    min_rating="GOOD",
    max_refinements=4,
)
async def main() -> None:
    async with fast.run() as agent:
        await agent.code_improver.send(
            "Create a Python class for managing a connection pool with automatic retry logic"
        )

@fast.agent(
    "copywriter",
    instruction="""Write compelling marketing copy that engages the target audience,
    highlights key benefits, and includes a strong call-to-action.""",
    use_history=True,
)
@fast.agent(
    "copy_critic",
    instruction="""Evaluate marketing copy on:
    1. Attention-grabbing headline
    2. Clear value proposition
    3. Emotional resonance
    4. Call-to-action effectiveness
    5. Brand voice alignment
    6. Grammar and readability
    
    Focus on conversion optimization.""",
)
@fast.evaluator_optimizer(
    name="copy_optimizer",
    generator="copywriter",
    evaluator="copy_critic",
    min_rating="EXCELLENT",
    max_refinements=3,
)
async def main() -> None:
    async with fast.run() as agent:
        await agent.copy_optimizer.send(
            "Write landing page copy for a new AI-powered project management tool"
        )

CUSTOM_REFINEMENT = """
You are an academic writing specialist.
Each refinement should strengthen:
1. Thesis clarity and argumentation
2. Evidence quality and citation accuracy
3. Academic tone and formality
4. Logical flow between paragraphs
5. Critical analysis depth
"""

@fast.evaluator_optimizer(
    name="academic_writer",
    generator="writer",
    evaluator="evaluator",
    refinement_instruction=CUSTOM_REFINEMENT,
    min_rating="EXCELLENT",
    max_refinements=4,
)

result = await agent.my_optimizer.send("Generate content...")

# Access refinement history
history = agent.my_optimizer.refinement_history
for iteration in history:
    print(f"Attempt {iteration['attempt']}")
    print(f"Rating: {iteration['evaluation']['rating']}")
    print(f"Feedback: {iteration['evaluation']['feedback']}")