Multimodal Search

const response = await fetch('/api/search', {
  method: 'POST',
  headers: {
    'Content-Type': 'application/json',
  },
  body: JSON.stringify({
    query: 'What type of flower is this?',
    mode: 'default',
    user_images: [
      {
        data: 'base64_encoded_image_data_here',
        mimeType: 'image/jpeg'
      }
    ]
  })
});

async function searchWithImages(query: string, files: File[]) {
  // Convert files to base64
  const imagePromises = files.map(file => {
    return new Promise((resolve, reject) => {
      const reader = new FileReader();
      reader.onload = (e) => {
        const base64 = e.target?.result as string;
        // Remove data URL prefix
        const data = base64.split(',')[1];
        resolve({
          data,
          mimeType: file.type
        });
      };
      reader.onerror = reject;
      reader.readAsDataURL(file);
    });
  });

  const user_images = await Promise.all(imagePromises);

  const response = await fetch('/api/search', {
    method: 'POST',
    headers: { 'Content-Type': 'application/json' },
    body: JSON.stringify({
      query,
      mode: 'default',
      user_images
    })
  });

  return response.json();
}

import { useImageStore } from '@/store/imageStore';

export function useImageUpload() {
  const { addImage } = useImageStore();

  const handleImageUpload = async (files: File[]) => {
    // Maximum 4 images
    if (files.length > 4) {
      throw new Error('Maximum 4 images allowed');
    }

    // Convert to base64
    for (const file of files) {
      const reader = new FileReader();
      reader.onload = (e) => {
        const base64 = e.target?.result as string;
        addImage({
          id: Math.random().toString(36),
          data: base64.split(',')[1],
          mimeType: file.type
        });
      };
      reader.readAsDataURL(file);
    }
  };

  return { handleImageUpload };
}

if (user_images && user_images.length > 0) {
  chat = model.startChat({
    tools: [{ google_search: {} }],
    history: [
      {
        role: "user",
        parts: [
          ...user_images.map((img: UserImage) => ({
            inlineData: {
              data: img.data,
              mimeType: img.mimeType
            }
          })),
          { text: 'Use uploaded images to search for information' }
        ]
      }
    ]
  });
}