Document Metadata Extraction

import fenic as fc
from pydantic import BaseModel, Field
from typing import List, Literal, Optional

config = fc.SessionConfig(
    app_name="document_extraction",
    semantic=fc.SemanticConfig(
        language_models={
            "mini": fc.OpenAILanguageModel(
                model_name="gpt-4o-mini",
                rpm=500,
                tpm=200_000,
            )
        }
    ),
)

session = fc.Session.get_or_create(config)

# Define metadata extraction schema
class DocumentMetadata(BaseModel):
    """Pydantic model for document metadata extraction."""
    title: str = Field(
        description="The main title or subject of the document"
    )
    document_type: Literal[
        "research paper",
        "product announcement",
        "meeting notes",
        "news article",
        "technical documentation",
        "other"
    ] = Field(
        description="Type of document"
    )
    date: str = Field(
        description="Any date mentioned in the document (publication date, meeting date, etc.)"
    )
    keywords: List[str] = Field(
        description="List of key topics, technologies, or important terms mentioned in the document"
    )
    summary: str = Field(
        description="Brief one-sentence summary of the document's main purpose or content"
    )

documents_data = [
    {
        "id": "doc_001",
        "text": "Neural Networks for Climate Prediction: A Comprehensive Study. Published March 15, 2024. This research presents a novel deep learning approach for predicting climate patterns using multi-layered neural networks. Our methodology combines satellite imagery data with ground-based sensor readings to achieve 94% accuracy in temperature forecasting. Keywords: machine learning, climate modeling, neural networks, environmental science."
    },
    {
        "id": "doc_002",
        "text": "Introducing CloudSync Pro - Next-Generation File Synchronization. Release Date: January 8, 2024. CloudSync Pro revolutionizes how teams collaborate with real-time file synchronization across unlimited devices. Features include end-to-end encryption, automatic conflict resolution, and integration with over 50 productivity tools. Pricing starts at $12/month per user."
    },
    # ... more documents
]

docs_df = session.create_dataframe(documents_data)

# Extract metadata using Pydantic model
pydantic_extracted_df = docs_df.select(
    "id",
    fc.semantic.extract("text", DocumentMetadata).alias("metadata")
)

# Flatten the extracted metadata into separate columns
pydantic_results = pydantic_extracted_df.select(
    "id",
    pydantic_extracted_df.metadata.title.alias("title"),
    pydantic_extracted_df.metadata.document_type.alias("document_type"),
    pydantic_extracted_df.metadata.date.alias("date"),
    pydantic_extracted_df.metadata.keywords.alias("keywords"),
    pydantic_extracted_df.metadata.summary.alias("summary")
)

print("Extraction Results:")
pydantic_results.show()

class Author(BaseModel):
    name: str = Field(description="Author's full name")
    affiliation: str = Field(description="Author's institution or company")

class ResearchPaper(BaseModel):
    title: str = Field(description="Paper title")
    authors: List[Author] = Field(description="List of paper authors")
    abstract: str = Field(description="Paper abstract")
    keywords: List[str] = Field(description="Research keywords")

class DocumentMetadata(BaseModel):
    title: str = Field(description="Document title")
    subtitle: Optional[str] = Field(
        default=None,
        description="Document subtitle if present"
    )
    publication_date: Optional[str] = Field(
        default=None,
        description="Publication date if mentioned"
    )

from typing import Literal

class Article(BaseModel):
    sentiment: Literal["positive", "negative", "neutral"] = Field(
        description="Overall sentiment of the article"
    )
    urgency: Literal["low", "medium", "high", "critical"] = Field(
        description="Urgency level based on content"
    )

# Set your API key
export OPENAI_API_KEY="your-api-key"

# Run the extraction
python document_extraction.py

# Good - specific and clear
keywords: List[str] = Field(
    description="Technical terms, product names, and key concepts mentioned in the text"
)

# Bad - vague
keywords: List[str] = Field(description="Keywords")

# Use Literal when values come from a known set
document_type: Literal["email", "report", "memo"] = Field(...)

# Use str when values are open-ended
title: str = Field(...)

date: str = Field(
    description="Date in format YYYY-MM-DD (e.g., 2024-03-15)"
)

author: Optional[str] = Field(
    default=None,
    description="Author name if mentioned in the document"
)