VoiceAgent/example/demo.ts

import "dotenv/config";
import { VoiceAgent } from "../src";
import { tool } from "ai";
import { z } from "zod";
import { openai } from "@ai-sdk/openai";
import { writeFile } from "fs/promises";

// 1. Define Tools using standard AI SDK
const weatherTool = tool({
    description: "Get the weather in a location",
    inputSchema: z.object({
        location: z.string().describe("The location to get the weather for"),
    }),
    execute: async ({ location }) => ({
        location,
        temperature: 72 + Math.floor(Math.random() * 21) - 10,
        conditions: ["sunny", "cloudy", "rainy", "partly cloudy"][Math.floor(Math.random() * 4)],
    }),
});

const timeTool = tool({
    description: "Get the current time",
    inputSchema: z.object({}),
    execute: async () => ({
        time: new Date().toLocaleTimeString(),
        timezone: Intl.DateTimeFormat().resolvedOptions().timeZone,
    }),
});

// 2. Initialize Agent with full voice support
const agent = new VoiceAgent({
    // Chat model for text generation
    model: openai("gpt-4o"),
    // Transcription model for speech-to-text
    transcriptionModel: openai.transcription("whisper-1"),
    // Speech model for text-to-speech
    speechModel: openai.speech("gpt-4o-mini-tts"),
    // System instructions
    instructions: `You are a helpful voice assistant. 
Keep responses concise and conversational since they will be spoken aloud.
Use tools when needed to provide accurate information.`,
    // TTS voice configuration
    voice: "alloy", // Options: alloy, echo, fable, onyx, nova, shimmer
    speechInstructions: "Speak in a friendly, natural conversational tone.",
    outputFormat: "mp3",
    // Streaming speech tuning
    streamingSpeech: {
        minChunkSize: 40,
        maxChunkSize: 180,
        parallelGeneration: true,
        maxParallelRequests: 2,
    },
    // WebSocket endpoint
    endpoint: process.env.VOICE_WS_ENDPOINT,
    // Tools
    tools: {
        getWeather: weatherTool,
        getTime: timeTool,
    },
});

// 3. Handle Events

// Connection events
agent.on("connected", () => console.log("✓ Connected to WebSocket"));
agent.on("disconnected", () => console.log("✗ Disconnected from WebSocket"));

// Transcription events (when audio is converted to text)
agent.on("transcription", ({ text, language }: { text: string; language?: string }) => {
    console.log(`[Transcription] (${language || "unknown"}): ${text}`);
});

// Text events (user input and assistant responses)
agent.on("text", (msg: { role: string; text: string }) => {
    const prefix = msg.role === "user" ? "👤 User" : "🤖 Assistant";
    console.log(`${prefix}: ${msg.text}`);
});

// Streaming text delta events (real-time text chunks)
agent.on("chunk:text_delta", ({ text }: { text: string }) => {
    process.stdout.write(text);
});

// Tool events (stream-level)
agent.on("chunk:tool_call", ({ toolName, input }: { toolName: string; input: unknown }) => {
    console.log(`\n[Tool] Calling ${toolName}...`, input ? JSON.stringify(input) : "");
});

agent.on("tool_result", ({ name, result }: { name: string; result: unknown }) => {
    console.log(`[Tool] ${name} result:`, JSON.stringify(result));
});

// Speech events
agent.on("speech_start", ({ streaming }: { streaming: boolean }) => {
    console.log(`[TTS] Speech started (streaming=${streaming})`);
});

agent.on("speech_complete", () => {
    console.log("[TTS] Speech generation complete");
});

agent.on("speech_chunk_queued", ({ id, text }: { id: number; text: string }) => {
    console.log(`[TTS] Queued chunk #${id}: ${text.substring(0, 40)}...`);
});

// Streaming audio chunk events
agent.on(
    "audio_chunk",
    async ({ chunkId, format, uint8Array }: { chunkId: number; format: string; uint8Array: Uint8Array }) => {
        console.log(`[Audio] Chunk #${chunkId} (${uint8Array.length} bytes, ${format})`);
        await writeFile(`output_chunk_${chunkId}.${format}`, Buffer.from(uint8Array));
    },
);

// Full audio event (non-streaming fallback via generateAndSendSpeechFull)
agent.on("audio", async (audio: { data: string; format: string; uint8Array: Uint8Array }) => {
    console.log(`[Audio] Full response audio (${audio.uint8Array.length} bytes, ${audio.format})`);
    await writeFile(`output_full.${audio.format}`, Buffer.from(audio.uint8Array));
});

// Speech interruption (barge-in)
agent.on("speech_interrupted", ({ reason }: { reason: string }) => {
    console.log(`[TTS] Speech interrupted: ${reason}`);
});

// Transcription event (when server-side Whisper is used)
agent.on("transcription", ({ text, language }: { text: string; language?: string }) => {
    console.log(`[STT] Transcription (${language || "unknown"}): ${text}`);
});

// Audio received event
agent.on("audio_received", ({ size }: { size: number }) => {
    console.log(`[Audio] Received ${(size / 1024).toFixed(1)} KB of audio input`);
});

// Warning events
agent.on("warning", (msg: string) => {
    console.warn(`[Warning] ${msg}`);
});

// Error handling
agent.on("error", (error: Error) => {
    console.error("[Error]", error.message);
});

// 4. Main execution
(async () => {
    console.log("\n=== Voice Agent Demo ===");
    console.log("Testing text-only mode (no WebSocket required)\n");

    try {
        // Test 1: Simple text query with streaming
        console.log("--- Test 1: Text Query ---");
        await agent.sendText("What's the weather in San Francisco?");
        console.log("\n");
        // Show conversation history
        console.log("--- Conversation History ---");
        const history = agent.getHistory();
        console.log(`Total messages: ${history.length}`);

        // Optional: Connect to WebSocket for real-time voice
        if (process.env.VOICE_WS_ENDPOINT) {
            console.log("\n--- Connecting to WebSocket ---");
            await agent.connect(process.env.VOICE_WS_ENDPOINT);
            console.log("Agent connected. Listening for audio input...");

            // Keep the process running to receive WebSocket messages
            // In a real app, you would stream microphone audio here
        }
    } catch (error) {
        console.error("Agent run failed:", error);
        process.exit(1);
    }
})();

// Example: How to send audio in a real application
// ---------------------------------------------
// import { readFile } from "fs/promises";
//
// // Option 1: Send base64 encoded audio
// const audioBase64 = (await readFile("recording.mp3")).toString("base64");
// await agent.sendAudio(audioBase64);
//
// // Option 2: Send raw audio buffer
// const audioBuffer = await readFile("recording.mp3");
// await agent.sendAudioBuffer(audioBuffer);
//
// // Option 3: Transcribe audio directly
// const transcribedText = await agent.transcribeAudio(audioBuffer);
// console.log("Transcribed:", transcribedText);