VideoAgent

wss.on("connection", (socket) => {
  const agent = new VideoAgent({ model, ... });
  agent.handleSocket(socket);
  agent.on("disconnected", () => agent.destroy());
});

new VideoAgent(options: VideoAgentOptions)

import { VideoAgent } from "@dimension-labs/voice-agent-ai";
import { openai } from "@ai-sdk/openai";

const agent = new VideoAgent({
  model: openai("gpt-4o"), // Vision-enabled model
  transcriptionModel: openai.transcription("whisper-1"),
  speechModel: openai.speech("tts-1"),
  instructions: "You are a helpful assistant that can see and hear.",
  voice: "alloy",
  maxContextFrames: 15,
  maxFrameInputSize: 5 * 1024 * 1024, // 5 MB
});

async connect(url?: string): Promise<void>

await agent.connect("ws://localhost:8080");

handleSocket(socket: WebSocket): void

import { WebSocketServer } from "ws";

const wss = new WebSocketServer({ port: 8080 });

wss.on("connection", (socket) => {
  const agent = new VideoAgent({ model, ... });
  agent.handleSocket(socket);
});

disconnect(): void

destroy(): void

async sendText(text: string): Promise<string>

const response = await agent.sendText("What do you see in this frame?");
console.log(response);

async sendAudio(audioData: string): Promise<void>

const base64Audio = "...";
await agent.sendAudio(base64Audio);

async sendAudioBuffer(audioBuffer: Buffer | Uint8Array): Promise<void>

const audioBuffer = fs.readFileSync("audio.wav");
await agent.sendAudioBuffer(audioBuffer);

async sendFrame(
  frameData: string,
  query?: string,
  options?: { width?: number; height?: number; format?: string }
): Promise<string>

const frameData = canvas.toDataURL("image/webp").split(",")[1];
const response = await agent.sendFrame(
  frameData,
  "What objects do you see?",
  { width: 1280, height: 720, format: "webp" }
);

requestFrameCapture(reason: FrameTriggerReason): void

agent.requestFrameCapture("user_request");

async transcribeAudio(audioData: Buffer | Uint8Array): Promise<string>

const transcript = await agent.transcribeAudio(audioBuffer);

async generateSpeechFromText(
  text: string,
  abortSignal?: AbortSignal
): Promise<Uint8Array>

const audio = await agent.generateSpeechFromText("Hello there");

interruptSpeech(reason?: string): void

agent.interruptSpeech("user_speaking");

interruptCurrentResponse(reason?: string): void

agent.interruptCurrentResponse("user_speaking");

clearHistory(): void

agent.clearHistory();

getHistory(): ModelMessage[]

const history = agent.getHistory();

setHistory(history: ModelMessage[]): void

const savedHistory = loadFromDatabase(userId);
agent.setHistory(savedHistory);

getFrameContext(): FrameContext[]

const frames = agent.getFrameContext();
console.log(`${frames.length} frames in context`);

getSessionId(): string

const sessionId = agent.getSessionId();

getConfig(): VideoAgentConfig

const config = agent.getConfig();
console.log(`Max frames: ${config.maxContextFrames}`);

updateConfig(config: Partial<VideoAgentConfig>): void

agent.updateConfig({ maxContextFrames: 20 });

registerTools(tools: Record<string, Tool>): void

import { tool } from "ai";
import { z } from "zod";

agent.registerTools({
  identifyObject: tool({
    description: "Identify an object in the current frame",
    parameters: z.object({
      objectName: z.string(),
    }),
    execute: async ({ objectName }) => {
      return { identified: true, confidence: 0.95 };
    },
  }),
});

get connected(): boolean

get processing(): boolean

get speaking(): boolean

get pendingSpeechChunks(): number

get destroyed(): boolean

get currentFrameSequence(): number

get hasVisualContext(): boolean

if (agent.hasVisualContext) {
  console.log("Agent can see");
}

agent.on("frame_received", (data) => {
  console.log(`Frame ${data.sequence} received:`, data.dimensions);
  console.log(`Size: ${data.size} bytes, reason: ${data.triggerReason}`);
});

agent.on("frame_requested", ({ reason }) => {
  console.log(`Frame requested: ${reason}`);
});

agent.on("client_ready", (capabilities) => {
  console.log("Client ready with capabilities:", capabilities);
});

agent.on("config_changed", (config) => {
  console.log("Config updated:", config);
});

agent.on("text", ({ role, text, hasImage }) => {
  if (hasImage) {
    console.log(`${role} (with image): ${text}`);
  }
});

interface VideoAgentConfig {
  maxContextFrames: number;
}

interface VideoFrame {
  type: "video_frame";
  sessionId: string;
  sequence: number;
  timestamp: number;
  triggerReason: FrameTriggerReason;
  previousFrameRef?: string;
  image: {
    data: string;        // Base64-encoded image
    format: string;      // Image format (e.g., "webp")
    width: number;       // Frame width in pixels
    height: number;      // Frame height in pixels
  };
}

interface FrameContext {
  sequence: number;
  timestamp: number;
  triggerReason: FrameTriggerReason;
  frameHash: string;
  description?: string;
}

type FrameTriggerReason = "scene_change" | "user_request" | "timer" | "initial";

import { VideoAgent } from "@dimension-labs/voice-agent-ai";
import { openai } from "@ai-sdk/openai";
import { WebSocketServer } from "ws";

const wss = new WebSocketServer({ port: 8080 });

wss.on("connection", (socket) => {
  const agent = new VideoAgent({
    model: openai("gpt-4o"),
    transcriptionModel: openai.transcription("whisper-1"),
    speechModel: openai.speech("tts-1"),
    instructions: "You are a helpful assistant that can see and hear.",
    voice: "alloy",
    maxContextFrames: 15,
  });

  agent.handleSocket(socket);

  // Listen to events
  agent.on("frame_received", (data) => {
    console.log(`Frame ${data.sequence}: ${data.size} bytes`);
  });

  agent.on("text", ({ role, text, hasImage }) => {
    console.log(`${role}${hasImage ? " (with frame)" : ""}: ${text}`);
  });

  agent.on("disconnected", () => {
    console.log("Client disconnected");
    agent.destroy();
  });

  agent.on("error", (error) => {
    console.error("Agent error:", error);
  });
});

console.log("Video agent server listening on port 8080");