LLM Enhancement

def validate_llms_txt(content: str, original_urls: list[str]) -> tuple[bool, list[str]]:
    errors = []
    
    # Extract URLs from enhanced content
    enhanced_urls = extract_urls(content)
    
    # Check that all original URLs are preserved
    missing_urls = set(original_urls) - set(enhanced_urls)
    if missing_urls:
        errors.append(f"Missing {len(missing_urls)} URLs from original")
    
    # Check for unexpected new URLs
    extra_urls = set(enhanced_urls) - set(original_urls)
    if extra_urls:
        errors.append(f"Added {len(extra_urls)} URLs not in original")
    
    # Ensure content is not empty
    if not content.strip():
        errors.append("Enhanced content is empty")
    
    # Ensure reasonable length
    if len(content) < 100:
        errors.append("Enhanced content too short")
    
    return (len(errors) == 0, errors)

def truncate_descriptions(content: str, max_length: int = 800) -> str:
    """
    Truncate blockquote descriptions that exceed max_length.
    Truncates at sentence boundaries for clean cuts.
    """
    lines = content.split('\n')
    result = []
    
    in_blockquote = False
    blockquote_buffer = []
    
    for line in lines:
        if line.startswith('>'):
            in_blockquote = True
            blockquote_buffer.append(line)
        else:
            if in_blockquote:
                # End of blockquote - process buffer
                full_text = '\n'.join(blockquote_buffer)
                if len(full_text) > max_length:
                    # Truncate at sentence boundary
                    truncated = full_text[:max_length]
                    last_period = truncated.rfind('.')
                    if last_period > max_length * 0.7:
                        truncated = truncated[:last_period + 1]
                    result.append(truncated + '...')
                else:
                    result.extend(blockquote_buffer)
                blockquote_buffer = []
                in_blockquote = False
            result.append(line)
    
    return '\n'.join(result)

# Enable LLM enhancement feature
LLM_ENHANCEMENT_ENABLED=true

# OpenRouter API key (required if enabled)
OPENROUTER_API_KEY=sk-or-v1-...

# Model selection
OPENROUTER_MODEL=x-ai/grok-2-1212

# Timeout and retries
LLM_TIMEOUT_SECONDS=60
LLM_MAX_RETRIES=2

# Temperature (0.0 = deterministic, 1.0 = creative)
LLM_TEMPERATURE=0.3

@dataclass
class ProcessingResult:
    success: bool
    output: str
    error: str | None = None
    stats: dict | None = None

    @classmethod
    def success_result(cls, output: str, stats: dict) -> "ProcessingResult":
        return cls(success=True, output=output, error=None, stats=stats)

    @classmethod
    def failure_result(cls, original_output: str, error: str) -> "ProcessingResult":
        return cls(success=False, output=original_output, error=error, stats=None)