import "dotenv/config";
import { VoiceAgent } from "voice-agent-ai";
import { tool } from "ai";
import { z } from "zod";
import { openai } from "@ai-sdk/openai";
import { writeFile } from "fs/promises";
// 1. Define Tools using standard AI SDK
const weatherTool = tool({
description: "Get the weather in a location",
inputSchema: z.object({
location: z.string().describe("The location to get the weather for"),
}),
execute: async ({ location }) => ({
location,
temperature: 72 + Math.floor(Math.random() * 21) - 10,
conditions: ["sunny", "cloudy", "rainy", "partly cloudy"][Math.floor(Math.random() * 4)],
}),
});
const timeTool = tool({
description: "Get the current time",
inputSchema: z.object({}),
execute: async () => ({
time: new Date().toLocaleTimeString(),
timezone: Intl.DateTimeFormat().resolvedOptions().timeZone,
}),
});
// 2. Initialize Agent with full voice support
const agent = new VoiceAgent({
// Chat model for text generation
model: openai("gpt-4o"),
// Transcription model for speech-to-text
transcriptionModel: openai.transcription("whisper-1"),
// Speech model for text-to-speech
speechModel: openai.speech("gpt-4o-mini-tts"),
// System instructions
instructions: `You are a helpful voice assistant.
Keep responses concise and conversational since they will be spoken aloud.
Use tools when needed to provide accurate information.`,
// TTS voice configuration
voice: "alloy", // Options: alloy, echo, fable, onyx, nova, shimmer
speechInstructions: "Speak in a friendly, natural conversational tone.",
outputFormat: "mp3",
// Streaming speech tuning
streamingSpeech: {
minChunkSize: 40,
maxChunkSize: 180,
parallelGeneration: true,
maxParallelRequests: 2,
},
// WebSocket endpoint (optional)
endpoint: process.env.VOICE_WS_ENDPOINT,
// Tools
tools: {
getWeather: weatherTool,
getTime: timeTool,
},
});
// 3. Handle Events
// Connection events
agent.on("connected", () => console.log("✓ Connected to WebSocket"));
agent.on("disconnected", () => console.log("✗ Disconnected from WebSocket"));
// Transcription events (when audio is converted to text)
agent.on("transcription", ({ text, language }: { text: string; language?: string }) => {
console.log(`[Transcription] (${language || "unknown"}): ${text}`);
});
// Text events (user input and assistant responses)
agent.on("text", (msg: { role: string; text: string }) => {
const prefix = msg.role === "user" ? "👤 User" : "🤖 Assistant";
console.log(`${prefix}: ${msg.text}`);
});
// Streaming text delta events (real-time text chunks)
agent.on("chunk:text_delta", ({ text }: { text: string }) => {
process.stdout.write(text);
});
// Tool events (stream-level)
agent.on("chunk:tool_call", ({ toolName, input }: { toolName: string; input: unknown }) => {
console.log(`\n[Tool] Calling ${toolName}...`, input ? JSON.stringify(input) : "");
});
agent.on("tool_result", ({ name, result }: { name: string; result: unknown }) => {
console.log(`[Tool] ${name} result:`, JSON.stringify(result));
});
// Speech events
agent.on("speech_start", ({ streaming }: { streaming: boolean }) => {
console.log(`[TTS] Speech started (streaming=${streaming})`);
});
agent.on("speech_complete", () => {
console.log("[TTS] Speech generation complete");
});
agent.on("speech_chunk_queued", ({ id, text }: { id: number; text: string }) => {
console.log(`[TTS] Queued chunk #${id}: ${text.substring(0, 40)}...`);
});
// Streaming audio chunk events
agent.on(
"audio_chunk",
async ({ chunkId, format, uint8Array }: { chunkId: number; format: string; uint8Array: Uint8Array }) => {
console.log(`[Audio] Chunk #${chunkId} (${uint8Array.length} bytes, ${format})`);
await writeFile(`output_chunk_${chunkId}.${format}`, Buffer.from(uint8Array));
},
);
// Full audio event (non-streaming fallback)
agent.on("audio", async (audio: { data: string; format: string; uint8Array: Uint8Array }) => {
console.log(`[Audio] Full response audio (${audio.uint8Array.length} bytes, ${audio.format})`);
await writeFile(`output_full.${audio.format}`, Buffer.from(audio.uint8Array));
});
// Speech interruption (barge-in)
agent.on("speech_interrupted", ({ reason }: { reason: string }) => {
console.log(`[TTS] Speech interrupted: ${reason}`);
});
// Audio received event
agent.on("audio_received", ({ size }: { size: number }) => {
console.log(`[Audio] Received ${(size / 1024).toFixed(1)} KB of audio input`);
});
// Warning events
agent.on("warning", (msg: string) => {
console.warn(`[Warning] ${msg}`);
});
// Error handling
agent.on("error", (error: Error) => {
console.error("[Error]", error.message);
});
// 4. Main execution
(async () => {
console.log("\n=== Voice Agent Demo ===");
console.log("Testing text-only mode (no WebSocket required)\n");
try {
// Test 1: Simple text query with streaming
console.log("--- Test 1: Text Query ---");
await agent.sendText("What's the weather in San Francisco?");
console.log("\n");
// Show conversation history
console.log("--- Conversation History ---");
const history = agent.getHistory();
console.log(`Total messages: ${history.length}`);
// Optional: Connect to WebSocket for real-time voice
if (process.env.VOICE_WS_ENDPOINT) {
console.log("\n--- Connecting to WebSocket ---");
await agent.connect(process.env.VOICE_WS_ENDPOINT);
console.log("Agent connected. Listening for audio input...");
// Keep the process running to receive WebSocket messages
// In a real app, you would stream microphone audio here
}
} catch (error) {
console.error("Agent run failed:", error);
process.exit(1);
}
})();
