Skip to main content

Overview

The Identification Pipeline detects faces in captured media, generates embeddings, and searches across the web to identify people.

Pipeline Architecture

Face Detection

MediaPipe Detector

The system uses Google MediaPipe with the BlazeFace model for real-time face detection:
backend/identification/detector.py
class MediaPipeFaceDetector:
    """Face detector using Google MediaPipe Task API (0.10.x+).

    Implements the FaceDetector protocol from identification/__init__.py.
    """

    def __init__(self, *, min_detection_confidence: float = 0.5) -> None:
        self._min_confidence = min_detection_confidence
        self._model_path = str(_MODEL_PATH)
        self._configured = _MODEL_PATH.exists()
        if not self._configured:
            logger.warning("MediaPipe model not found at {}", _MODEL_PATH)

Detection Process

backend/identification/detector.py
async def detect_faces(self, request: FaceDetectionRequest) -> FaceDetectionResult:
    """Detect faces in an image using MediaPipe Task API."""
    try:
        img = Image.open(io.BytesIO(request.image_data)).convert("RGB")
        img_array = np.array(img)
        height, width = img_array.shape[:2]
    except Exception as exc:
        logger.error("Failed to decode image for face detection: {}", exc)
        return FaceDetectionResult(success=False, error=f"Image decode failed: {exc}")

    if not self._configured:
        return FaceDetectionResult(
            frame_width=width,
            frame_height=height,
            success=False,
            error="Face detection model file not found",
        )

    try:
        options = FaceDetectorOptions(
            base_options=BaseOptions(model_asset_path=self._model_path),
            min_detection_confidence=self._min_confidence,
        )
        with FaceDetector.create_from_options(options) as detector:
            mp_image = mp.Image(
                image_format=mp.ImageFormat.SRGB,
                data=img_array,
            )
            result = detector.detect(mp_image)
    except Exception as exc:
        logger.error("MediaPipe face detection failed: {}", exc)
        return FaceDetectionResult(
            frame_width=width,
            frame_height=height,
            success=False,
            error=f"Detection failed: {exc}",
        )

Bounding Box Normalization

Detected faces return normalized coordinates (0-1):
backend/identification/detector.py
faces: list[DetectedFace] = []
for detection in result.detections[: request.max_faces]:
    score = detection.categories[0].score if detection.categories else 0.0
    if score < request.min_confidence:
        continue

    bbox_mp = detection.bounding_box
    bbox = BoundingBox(
        x=max(0.0, min(1.0, bbox_mp.origin_x / width)),
        y=max(0.0, min(1.0, bbox_mp.origin_y / height)),
        width=max(0.0, min(1.0, bbox_mp.width / width)),
        height=max(0.0, min(1.0, bbox_mp.height / height)),
    )

    faces.append(DetectedFace(
        bbox=bbox,
        confidence=score,
        embedding=[],  # Embeddings filled by the embedder stage
    ))

Data Models

BoundingBox

backend/identification/models.py
class BoundingBox(BaseModel):
    """Face bounding box coordinates (normalized 0-1)."""

    x: float = Field(ge=0.0, le=1.0)
    y: float = Field(ge=0.0, le=1.0)
    width: float = Field(ge=0.0, le=1.0)
    height: float = Field(ge=0.0, le=1.0)

DetectedFace

backend/identification/models.py
class DetectedFace(BaseModel):
    """A single detected face with embedding."""

    bbox: BoundingBox
    confidence: float = Field(ge=0.0, le=1.0)
    embedding: list[float] = Field(default_factory=list)

FaceSearchMatch

backend/identification/models.py
class FaceSearchMatch(BaseModel):
    """A single match from reverse face search."""

    url: str
    thumbnail_url: str | None = None
    similarity: float = Field(ge=0.0, le=1.0)
    source: str
    person_name: str | None = None

Face Search Manager

The FaceSearchManager orchestrates a waterfall search strategy:
  1. PimEyes (primary) — purpose-built face search engine
  2. Reverse Image Search (fallback) — Google, Yandex, Bing
backend/identification/search_manager.py
class FaceSearchManager:
    """Orchestrates face search across PimEyes and reverse image engines.

    Implements the FaceSearcher protocol from identification/__init__.py.
    Strategy: PimEyes first (purpose-built for faces), reverse image fallback.
    """

    def __init__(self, settings: Settings) -> None:
        self._pimeyes = PimEyesSearcher(settings)
        self._reverse = ReverseImageSearcher()

    @property
    def configured(self) -> bool:
        return True

    async def search_face(self, request: FaceSearchRequest) -> FaceSearchResult:
        """Search for a face: PimEyes first, reverse image search fallback."""

        # Tier 1: PimEyes (purpose-built face search)
        logger.info("Face search: trying PimEyes first")
        pimeyes_result = await self._pimeyes.search_face(request)

        if pimeyes_result.success and pimeyes_result.matches:
            logger.info("PimEyes found {} matches, skipping reverse search",
                        len(pimeyes_result.matches))
            return pimeyes_result

        # Tier 2: Reverse image search (Google, Yandex, Bing)
        logger.info("PimEyes returned no matches, falling back to reverse image search")
        reverse_result = await self._reverse.search_face(request)

        if reverse_result.success and reverse_result.matches:
            logger.info("Reverse search found {} matches", len(reverse_result.matches))
            return reverse_result

        # Both failed — merge errors
        errors = []
        if pimeyes_result.error:
            errors.append(f"PimEyes: {pimeyes_result.error}")
        if reverse_result.error:
            errors.append(f"ReverseSearch: {reverse_result.error}")

        return FaceSearchResult(
            matches=[],
            success=False,
            error=" | ".join(errors) if errors else "No matches found across any search engine",
        )

Name Extraction

The manager uses frequency analysis to extract the most likely person name:
backend/identification/search_manager.py
def best_name_from_results(self, result: FaceSearchResult) -> str | None:
    """Extract the most likely person name from search results.

    Uses frequency analysis: the name that appears most across matches wins.
    """
    if not result.matches:
        return None

    name_counts: dict[str, int] = {}
    for match in result.matches:
        if match.person_name:
            name = match.person_name.strip()
            name_counts[name] = name_counts.get(name, 0) + 1

    if not name_counts:
        return None

    # Return the most frequent name
    return max(name_counts, key=name_counts.get)  # type: ignore[arg-type]

Profile URL Extraction

backend/identification/search_manager.py
def profile_urls_from_results(self, result: FaceSearchResult) -> list[str]:
    """Extract social profile URLs from search results."""
    social_domains = {
        "linkedin.com", "twitter.com", "x.com", "instagram.com",
        "facebook.com", "github.com", "tiktok.com",
    }
    urls: list[str] = []
    seen: set[str] = set()
    for match in result.matches:
        if match.url and match.url not in seen:
            if any(domain in match.url for domain in social_domains):
                urls.append(match.url)
                seen.add(match.url)
    return urls

Pipeline Status Endpoint

Monitor the identification pipeline in real-time:
backend/main.py
@app.get("/api/pipeline/status")
async def pipeline_status():
    """Diagnostic: show what the face identification pipeline is doing right now."""
    handler_state = {
        "seen_tracks": list(frame_handler._seen_tracks),
        "spawned_identifications": list(frame_handler._spawned),
        "identifications": {
            tid: {
                "status": ident.status,
                "name": ident.name,
                "person_id": ident.person_id,
                "error": ident.error,
            }
            for tid, ident in frame_handler._identifications.items()
        },
        "face_detector_configured": frame_handler._face_detector is not None,
        "embedder_configured": frame_handler._embedder is not None,
        "face_searcher_configured": frame_handler._face_searcher is not None,
    }
    return {
        "frame_handler": handler_state,
        "services": settings.service_flags(),
        "deep_researcher": deep_researcher is not None,
        "synthesis_engine": synthesis_engine is not None,
    }

Configuration

Face detection configuration:
backend/main.py
# Initialize detector with confidence threshold
detector = MediaPipeFaceDetector(min_detection_confidence=0.5)
min_detection_confidence
float
default:"0.5"
Minimum confidence threshold for face detection (0.0 - 1.0)

Performance Characteristics

Detection Speed

~50ms per frame on CPU

Model Size

BlazeFace: ~1MB (lightweight)

Max Faces

Configurable (default: 10 per frame)

Accuracy

95%+ on frontal faces, 80%+ on profiles

Next Steps

Capture Service

See how images reach the pipeline

Agent Orchestration

Learn about the enrichment process

Build docs developers (and LLMs) love