Processing Pipeline

# From src/engine/article_processor.py:70
def check_relevance(
    self,
    article_content: str,
    article_id: str,
) -> PhaseOutcome:
    """Check whether an article is relevant to the configured domain."""
    try:
        if self.model_type == "ollama":
            result = ollama_check_relevance(
                text=article_content,
                model=self.specific_model,
                domain=self.domain,
            )
        else:
            result = gemini_check_relevance(
                text=article_content,
                model=self.specific_model,
                domain=self.domain,
            )
        
        # Normalize result to boolean
        is_relevant = bool(result.is_relevant)
        reason = getattr(result, "reason", "")
        
        return PhaseOutcome.ok(
            "relevance",
            value=is_relevant,
            meta={"reason": reason},
        )
    except Exception as e:
        # On error, fail open (assume relevant)
        return PhaseOutcome.fail(
            "relevance",
            error=e,
            fallback=False,
            context={"article_id": article_id},
        )

# From src/engine/article_processor.py:305
if max_workers > 1:
    from concurrent.futures import ThreadPoolExecutor
    
    with ThreadPoolExecutor(
        max_workers=min(max_workers, len(entity_types))
    ) as pool:
        futures = {
            et: pool.submit(
                self.extract_single_entity_type,
                et,
                article_content,
                article_id,
            )
            for et in entity_types
        }
        # Iterate in stable order (not completion order)
        for et in entity_types:
            outcome = futures[et].result()
            entities[et] = outcome.value or []
            extraction_outcomes[et] = outcome.to_metadata_dict()
else:
    # Serial extraction (max_workers=1)
    for et in entity_types:
        outcome = self.extract_single_entity_type(
            et, article_content, article_id
        )
        entities[et] = outcome.value or []
        extraction_outcomes[et] = outcome.to_metadata_dict()

# From src/engine/article_processor.py:175
# Attempt 1
raw_dicts = self._run_extraction(extractor, article_content)
cleaned, qc_report = run_extraction_qc(
    entity_type=entity_type,
    entities=raw_dicts,
    domain=self.domain,
)

# Conditional retry
if _should_retry_extraction(qc_report.flags):
    logger.info(f"Retrying {entity_type} extraction (triggers: {qc_report.flags})")
    
    hint = _build_repair_hint(entity_type, qc_report.flags)
    raw_dicts_v2 = self._run_extraction(
        extractor, article_content, repair_hint=hint
    )
    cleaned_v2, qc_report_v2 = run_extraction_qc(
        entity_type=entity_type,
        entities=raw_dicts_v2,
        domain=self.domain,
    )
    
    # Pick the better result (higher output_count wins)
    use_v2 = qc_report_v2.output_count > qc_report.output_count
    if use_v2:
        cleaned, qc_report = cleaned_v2, qc_report_v2

# From src/engine/mergers.py (conceptual)
def _build_evidence_text(
    entity_name: str,
    article_text: str,
    max_chars: int = 1500,
    window_chars: int = 240,
    max_windows: int = 3,
) -> str:
    """
    Build evidence text from article mentions with context windows.
    
    Example output:
    "John Doe, detainee, described in source as: 'John Doe was 
    transferred to Guantanamo in 2002 after being captured in 
    Afghanistan...' [240 chars] '...Doe's lawyer filed a habeas 
    corpus petition...' [240 chars]"
    """

dedup:
  name_variants:
    organizations:
      equivalence_groups:
        - ["Department of Defense", "Defense Department", "DoD", "Pentagon"]
        - ["American Civil Liberties Union", "ACLU"]
        - ["Joint Task Force Guantanamo", "JTF-GTMO", "JTF GTMO"]
    locations:
      equivalence_groups:
        - ["Guantanamo Bay", "Guantanamo", "GTMO", "Naval Station Guantanamo Bay"]
        - ["United States", "U.S.", "US"]

class VersionedProfile(BaseModel):
    """Container for versioned profile history."""
    
    current_version: int = 1
    versions: List[ProfileVersion] = Field(default_factory=list)
    
    def add_version(
        self, 
        profile_data: Dict, 
        trigger_article_id: Optional[str] = None
    ) -> ProfileVersion:
        """Add a new version to the history."""
        new_version = ProfileVersion(
            version_number=len(self.versions) + 1,
            profile_data=copy.deepcopy(profile_data),
            trigger_article_id=trigger_article_id,
        )
        self.versions.append(new_version)
        self.current_version = new_version.version_number
        return new_version

Processing complete

Articles read: 100
Articles processed: 87
Articles skipped (relevance): 8
Articles skipped (already processed): 5

Final entity counts:
• People: 342
• Organizations: 156
• Locations: 89
• Events: 203

Reflection statistics:
• Total reflection attempts: 42
• Average reflection attempts per article: 0.48

Grounding complete: 234 verified, 301 unchanged, 256 no citations