Technical Architecture

flask==3.0.0
gunicorn==21.2.0          # Production WSGI server
gevent==24.2.1           # Async support for streaming
requests==2.31.0         # HTTP client for OpenRouter API
python-dotenv==1.0.0     # Environment variable management

def _convert_system_messages(messages: list, model: str) -> list:
    """
    Convert system role messages to user messages for models that don't support the system role.
    Merges the system prompt into the first user message.
    """
    needs_conversion = any(model.startswith(prefix) for prefix in MODELS_WITHOUT_SYSTEM_ROLE)
    if not needs_conversion:
        return messages

    converted = []
    system_content = ""
    for msg in messages:
        if msg["role"] == "system":
            system_content += msg["content"] + "\n"
        elif msg["role"] == "user" and system_content:
            converted.append({
                "role": "user",
                "content": f"[Instructions: {system_content.strip()}]\n\n{msg['content']}"
            })
            system_content = ""
        else:
            converted.append(msg)
    return converted

# Rate limit handling configuration
MAX_RETRIES = 3
RETRY_DELAYS = [2, 5, 10]  # Exponential backoff delays in seconds

# Fallback models when primary model is rate-limited (in order of preference)
FALLBACK_MODELS = [
    "google/gemma-3-27b-it:free",
    "google/gemma-3-4b-it:free",
    "meta-llama/llama-3.3-70b-instruct:free",
    "meta-llama/llama-3.1-405b-instruct:free",
]

for model_index, current_model in enumerate(models_to_try):
    app.logger.info(f"Trying model: {current_model} (attempt {model_index + 1}/{len(models_to_try)})")
    
    # Retry loop for current model
    for retry in range(MAX_RETRIES):
        response, error_info = _make_api_request(messages, current_model)
        
        if response is not None:
            # Success! Return the response
            data = response.json()
            content = data["choices"][0]["message"]["content"]
            return (content, False)
        
        # Handle error with exponential backoff
        if error_info['is_rate_limit']:
            if retry < MAX_RETRIES - 1:
                delay = RETRY_DELAYS[min(retry, len(RETRY_DELAYS) - 1)]
                time.sleep(delay)
            else:
                # Try next model
                break

@app.route('/api/chat/stream', methods=['POST'])
def api_chat_stream():
    """
    Handle streaming chat messages using Server-Sent Events.
    Returns: SSE stream with content chunks
    """
    # Build messages and get figure data
    messages = [{"role": "system", "content": system_prompt}]
    messages.extend(history[-MAX_HISTORY:])
    messages.append({"role": "user", "content": user_message})
    
    # Return streaming response
    return Response(
        stream_llm(messages, model),
        mimetype='text/event-stream',
        headers={
            'Cache-Control': 'no-cache',
            'Connection': 'keep-alive',
            'X-Accel-Buffering': 'no'
        }
    )

MAX_HISTORY = 20  # Maximum number of messages to keep in history

FIGURE_PROMPT_TEMPLATE = """You are {name}, {title}, who lived from {birth_year} to {death_year}.

PERSONALITY & SPEAKING STYLE:
{personality}

KNOWN BELIEFS & VALUES:
{beliefs}

HISTORICAL CONTEXT:
- You have knowledge of events up to {death_year}
- You do NOT know about anything that happened after your death
- When asked about modern concepts, react with genuine curiosity appropriate to your era

ROLEPLAY RULES:
- Stay in character at all times
- Use speech patterns and vocabulary appropriate to your era and background
- Reference your real historical experiences, works, and relationships
- Express your documented opinions and beliefs authentically
- If asked about something after your time, express confusion or ask the user to explain

CONVERSATION STYLE:
- Be engaging and conversational, not lecturing
- Ask the user questions back to create dialogue
- Show your personality through your responses
- Keep responses to 2-3 paragraphs maximum unless asked for detail
"""

curl -X POST http://localhost:5000/api/chat/stream \
  -H "Content-Type: application/json" \
  -d '{
    "figure_id": "einstein",
    "message": "What is your theory of relativity?",
    "history": [],
    "model": "google/gemma-3-12b-it:free"
  }'

gunicorn --worker-class gevent --workers 2 --bind 0.0.0.0:$PORT app:app