VoiceAgent

wss.on("connection", (socket) => {
  const agent = new VoiceAgent({ model, ... });
  agent.handleSocket(socket);
  agent.on("disconnected", () => agent.destroy());
});

new VoiceAgent(options: VoiceAgentOptions)

import { VoiceAgent } from "@dimension-labs/voice-agent-ai";
import { openai } from "@ai-sdk/openai";

const agent = new VoiceAgent({
  model: openai("gpt-4o"),
  transcriptionModel: openai.transcription("whisper-1"),
  speechModel: openai.speech("tts-1"),
  instructions: "You are a helpful assistant.",
  voice: "alloy",
  streamingSpeech: {
    minChunkSize: 30,
    maxChunkSize: 150,
  },
  history: {
    maxMessages: 50,
  },
});

async connect(url?: string): Promise<void>

await agent.connect("ws://localhost:8080");

handleSocket(socket: WebSocket): void

import { WebSocketServer } from "ws";

const wss = new WebSocketServer({ port: 8080 });

wss.on("connection", (socket) => {
  const agent = new VoiceAgent({ model, ... });
  agent.handleSocket(socket);
});

disconnect(): void

destroy(): void

async sendText(text: string): Promise<string>

const response = await agent.sendText("What's the weather today?");
console.log(response);

async sendAudio(audioData: string): Promise<void>

const base64Audio = "...";
await agent.sendAudio(base64Audio);

async sendAudioBuffer(audioBuffer: Buffer | Uint8Array): Promise<void>

const audioBuffer = fs.readFileSync("audio.wav");
await agent.sendAudioBuffer(audioBuffer);

async transcribeAudio(audioData: Buffer | Uint8Array): Promise<string>

const transcript = await agent.transcribeAudio(audioBuffer);
console.log("User said:", transcript);

async generateAndSendSpeechFull(text: string): Promise<void>

await agent.generateAndSendSpeechFull("Hello, how can I help you?");

async generateSpeechFromText(text: string, abortSignal?: AbortSignal): Promise<Uint8Array>

const audioData = await agent.generateSpeechFromText("Hello world");
// Process or save the audio data
fs.writeFileSync("output.mp3", audioData);

interruptSpeech(reason?: string): void

agent.interruptSpeech("user_speaking");

interruptCurrentResponse(reason?: string): void

agent.interruptCurrentResponse("user_speaking");

startListening(): void

agent.startListening();
agent.on("listening", () => {
  console.log("Agent is now listening for voice input");
});

stopListening(): void

agent.stopListening();
agent.on("stopped", () => {
  console.log("Agent stopped listening");
});

clearHistory(): void

agent.clearHistory();

getHistory(): ModelMessage[]

const history = agent.getHistory();
console.log(`Conversation has ${history.length} messages`);

setHistory(history: ModelMessage[]): void

// Save history
const history = agent.getHistory();
saveToDatabase(userId, history);

// Restore later
const savedHistory = loadFromDatabase(userId);
agent.setHistory(savedHistory);

registerTools(tools: Record<string, Tool>): void

import { tool } from "ai";
import { z } from "zod";

agent.registerTools({
  getWeather: tool({
    description: "Get the weather for a location",
    parameters: z.object({
      location: z.string(),
    }),
    execute: async ({ location }) => {
      // Fetch weather data
      return { temperature: 72, condition: "sunny" };
    },
  }),
});

get connected(): boolean

if (agent.connected) {
  console.log("Agent is connected");
}

get processing(): boolean

if (agent.processing) {
  console.log("Agent is thinking...");
}

get speaking(): boolean

if (agent.speaking) {
  console.log("Agent is speaking...");
}

get pendingSpeechChunks(): number

console.log(`${agent.pendingSpeechChunks} chunks in queue`);

get destroyed(): boolean

if (agent.destroyed) {
  console.log("Agent has been destroyed");
}

agent.on("connected", () => {
  console.log("Agent connected");
});

agent.on("disconnected", () => {
  console.log("Agent disconnected");
  agent.destroy();
});

agent.on("error", (error: Error) => {
  console.error("Agent error:", error);
});

agent.on("warning", (message: string) => {
  console.warn("Warning:", message);
});

agent.on("listening", () => {
  console.log("Agent started listening for voice input");
});

agent.on("stopped", () => {
  console.log("Agent stopped listening");
});

agent.on("text", ({ role, text }) => {
  console.log(`${role}: ${text}`);
});

agent.on("text_delta", ({ delta }) => {
  process.stdout.write(delta);
});

agent.on("tool_call", ({ name, args, toolCallId }) => {
  console.log(`Tool called: ${name}`, args);
});

agent.on("tool_result", ({ name, result, toolCallId }) => {
  console.log(`Tool result from ${name}:`, result);
});

agent.on("speech_start", () => {
  console.log("Started speaking");
});

agent.on("speech_complete", () => {
  console.log("Finished speaking");
});

agent.on("speech_interrupted", ({ reason }) => {
  console.log(`Speech interrupted: ${reason}`);
});

agent.on("speech_chunk_queued", ({ chunkId, text }) => {
  console.log(`Queued chunk ${chunkId}: ${text}`);
});

agent.on("audio_chunk", ({ chunkId, audio }) => {
  console.log(`Generated audio chunk ${chunkId}`);
});

agent.on("audio", ({ audio, format }) => {
  console.log(`Audio ready: ${audio.length} bytes, format: ${format}`);
});

agent.on("transcription", ({ text, duration }) => {
  console.log(`Transcribed: ${text}`);
});

agent.on("audio_received", ({ size, format }) => {
  console.log(`Received audio: ${size} bytes, format: ${format}`);
});

agent.on("history_cleared", () => {
  console.log("History cleared");
});

agent.on("history_trimmed", ({ removedCount, reason }) => {
  console.log(`Trimmed ${removedCount} messages: ${reason}`);
});