Voice Agent

POST /api/voice-agent

const response = await fetch('/api/voice-agent', {
  method: 'POST',
  headers: {
    'Content-Type': 'application/json',
  },
  body: JSON.stringify({
    model: 'gpt-realtime-mini',
    voice: 'ash'
  })
});

const session = await response.json();
console.log(session);

{
  "id": "sess_abc123xyz",
  "model": "gpt-realtime-mini",
  "expires_at": 1234567890,
  "client_secret": {
    "value": "wss://api.openai.com/v1/realtime?model=gpt-realtime-mini",
    "expires_at": 1234567890
  },
  "modalities": ["text", "audio"],
  "instructions": "You are a voice AI assistant...",
  "voice": "ash",
  "input_audio_format": "pcm16",
  "output_audio_format": "pcm16",
  "input_audio_transcription": {
    "model": "whisper-1"
  },
  "turn_detection": {
    "type": "server_vad"
  },
  "tools": [...]
}

const ws = new WebSocket(session.client_secret.value);

ws.onopen = () => {
  console.log('Connected to voice agent');
  
  // Send audio or text messages
  ws.send(JSON.stringify({
    type: 'conversation.item.create',
    item: {
      type: 'message',
      role: 'user',
      content: [{
        type: 'input_text',
        text: 'Hello, who am I?'
      }]
    }
  }));
};

ws.onmessage = (event) => {
  const data = JSON.parse(event.data);
  console.log('Received:', data);
  
  // Handle different event types:
  // - conversation.item.created
  // - response.audio.delta (audio chunks)
  // - response.done
  // - etc.
};

ws.onerror = (error) => {
  console.error('WebSocket error:', error);
};

ws.onclose = () => {
  console.log('Disconnected from voice agent');
};

// Send PCM16 audio data
const audioData = new Int16Array(buffer); // Your audio buffer
const base64Audio = btoa(String.fromCharCode(...new Uint8Array(audioData.buffer)));

ws.send(JSON.stringify({
  type: 'input_audio_buffer.append',
  audio: base64Audio
}));

// Commit the audio buffer when user stops speaking
ws.send(JSON.stringify({
  type: 'input_audio_buffer.commit'
}));