Translation Algorithms

async def translate_page_by_page(
    content: DocumentContent,
    config: TranslationConfig,
    translator: ModelInterface,
    progress: Progress | None = None,
    checkpoint_manager: CheckpointManager | None = None,
) -> TranslationResponse:
    """Translate a document page by page."""
    translated_pages_by_num: dict[int, str] = {}
    
    # Translate pages - iterate by actual page number (1-indexed)
    for page_num, page in enumerate(content.pages, start=1):
        request = TranslationRequest(
            source_lang=config.source_lang,
            target_lang=config.target_lang,
            content=page,
            context=None,  # No context for page-by-page
            content_type=content.content_type,
            model=config.model,
        )
        
        response = await translator.translate(request)
        translated_pages_by_num[page_num] = response.text

tinbox translate --to de --algorithm page document.pdf

def create_windows(
    text: str,
    window_size: int,
    overlap_size: int,
) -> list[str]:
    """Create overlapping windows from text."""
    windows = []
    start = 0
    
    while start < len(text):
        end = min(start + window_size, len(text))
        window = text[start:end]
        windows.append(window)
        
        if end == len(text):
            break
            
        # Move start position with overlap
        start = end - min(overlap_size, end - start)
    
    return windows

def merge_chunks(chunks: list[str], overlap_size: int) -> str:
    """Merge translated chunks, handling overlaps."""
    result = chunks[0]
    for current_chunk in chunks[1:]:
        # Try to find the overlap region
        for overlap_len in range(
            min(len(result), len(current_chunk), overlap_size), 0, -1
        ):
            if result[-overlap_len:] == current_chunk[:overlap_len]:
                result += "\n\n" + current_chunk[overlap_len:]
                break

tinbox translate --to de --algorithm sliding-window \
  --window-size 1500 \
  --overlap-size 150 \
  document.txt

def smart_text_split(
    text: str, 
    target_size: int, 
    custom_split_token: str | None = None
) -> list[str]:
    """Split text at natural boundaries or custom tokens."""
    if custom_split_token:
        return [chunk for chunk in text.split(custom_split_token) if chunk]
    
    # Try to find natural split points
    chunk_text = text[current_pos:end_pos]
    
    # Priority 1: Paragraph breaks
    paragraph_matches = list(re.finditer(r"\n\n", chunk_text))
    if paragraph_matches:
        best_split_pos = paragraph_matches[-1].end()
    # Priority 2: Sentence endings...

def build_translation_context_info(
    source_lang: str,
    target_lang: str,
    previous_chunk: str | None = None,
    previous_translation: str | None = None,
    next_chunk: str | None = None,
) -> str | None:
    """Build context information for translation consistency."""
    context_parts = []
    
    if previous_chunk and previous_translation:
        context_parts.append(f"[PREVIOUS_CHUNK]\n{previous_chunk}\n[/PREVIOUS_CHUNK]")
        context_parts.append(
            f"[PREVIOUS_CHUNK_TRANSLATION]\n{previous_translation}\n[/PREVIOUS_CHUNK_TRANSLATION]"
        )
    
    if next_chunk:
        context_parts.append(f"[NEXT_CHUNK]\n{next_chunk}\n[/NEXT_CHUNK]")
    
    if context_parts:
        context_parts.append(
            "Use this context to maintain consistency in terminology and style."
        )
        return "\n\n".join(context_parts)
    
    return None

# Standard context-aware translation
tinbox translate --to de --algorithm context-aware document.txt

# With custom split token
tinbox translate --to de --algorithm context-aware \
  --custom-split-token "---" \
  document.txt

# With custom context size
tinbox translate --to de --algorithm context-aware \
  --context-size 3000 \
  document.txt

tinbox translate --to de \
  --checkpoint-dir ./checkpoints \
  --checkpoint-frequency 5 \
  document.pdf