voice agent works

example/demo.ts

@@ -3,56 +3,160 @@ import { VoiceAgent } from "../src";
 import { tool } from "ai";
 import { z } from "zod";
 import { openai } from "@ai-sdk/openai";
+import { writeFile } from "fs/promises";
 
 // 1. Define Tools using standard AI SDK
 const weatherTool = tool({
-    description: 'Get the weather in a location',
+    description: "Get the weather in a location",
     inputSchema: z.object({
-        location: z.string().describe('The location to get the weather for'),
+        location: z.string().describe("The location to get the weather for"),
     }),
     execute: async ({ location }) => ({
         location,
         temperature: 72 + Math.floor(Math.random() * 21) - 10,
+        conditions: ["sunny", "cloudy", "rainy", "partly cloudy"][Math.floor(Math.random() * 4)],
     }),
 });
 
-// 2. Initialize Agent
+const timeTool = tool({
+    description: "Get the current time",
+    inputSchema: z.object({}),
+    execute: async () => ({
+        time: new Date().toLocaleTimeString(),
+        timezone: Intl.DateTimeFormat().resolvedOptions().timeZone,
+    }),
+});
+
+// 2. Initialize Agent with full voice support
 const agent = new VoiceAgent({
-    model: openai('gpt-4o'),
-    instructions: "You are a helpful voice assistant. Use tools when needed.",
+    // Chat model for text generation
+    model: openai("gpt-4o"),
+    // Transcription model for speech-to-text
+    transcriptionModel: openai.transcription("whisper-1"),
+    // Speech model for text-to-speech
+    speechModel: openai.speech("gpt-4o-mini-tts"),
+    // System instructions
+    instructions: `You are a helpful voice assistant. 
+Keep responses concise and conversational since they will be spoken aloud.
+Use tools when needed to provide accurate information.`,
+    // TTS voice configuration
+    voice: "alloy", // Options: alloy, echo, fable, onyx, nova, shimmer
+    speechInstructions: "Speak in a friendly, natural conversational tone.",
+    outputFormat: "mp3",
+    // WebSocket endpoint
     endpoint: process.env.VOICE_WS_ENDPOINT,
+    // Tools
     tools: {
-        getWeather: weatherTool, // Pass the AI SDK tool directly
+        getWeather: weatherTool,
+        getTime: timeTool,
     },
 });
 
 // 3. Handle Events
-agent.on("connected", () => console.log("Connected to WebSocket"));
 
-// Handle incoming audio from AI (play this to user)
-agent.on("audio", (base64Audio: string) => {
-    // process.stdout.write(Buffer.from(base64Audio, 'base64')); 
+// Connection events
+agent.on("connected", () => console.log("✓ Connected to WebSocket"));
+agent.on("disconnected", () => console.log("✗ Disconnected from WebSocket"));
+
+// Transcription events (when audio is converted to text)
+agent.on("transcription", ({ text, language }: { text: string; language?: string }) => {
+    console.log(`[Transcription] (${language || "unknown"}): ${text}`);
 });
 
-// Logs
-agent.on("text", (msg: { role: string; text: string }) => console.log(`${msg.role}: ${msg.text}`));
-agent.on("tool_start", ({ name }: { name: string }) => console.log(`[System] Calling ${name}...`));
+// Text events (user input and assistant responses)
+agent.on("text", (msg: { role: string; text: string }) => {
+    const prefix = msg.role === "user" ? "👤 User" : "🤖 Assistant";
+    console.log(`${prefix}: ${msg.text}`);
+});
 
-// 4. Start (wrap in async function since we can't use top-level await)
+// Streaming text delta events (real-time text chunks)
+agent.on("text_delta", ({ text }: { text: string }) => {
+    process.stdout.write(text);
+});
+
+// Tool events
+agent.on("tool_start", ({ name, input }: { name: string; input?: unknown }) => {
+    console.log(`\n[Tool] Calling ${name}...`, input ? JSON.stringify(input) : "");
+});
+
+agent.on("tool_result", ({ name, result }: { name: string; result: unknown }) => {
+    console.log(`[Tool] ${name} result:`, JSON.stringify(result));
+});
+
+// Speech events
+agent.on("speech_start", ({ text }: { text: string }) => {
+    console.log(`[TTS] Generating speech for: "${text.substring(0, 50)}..."`);
+});
+
+agent.on("speech_complete", () => {
+    console.log("[TTS] Speech generation complete");
+});
+
+// Audio events (when TTS audio is generated)
+agent.on("audio", async (audio: { data: string; format: string; uint8Array: Uint8Array }) => {
+    console.log(`[Audio] Received ${audio.format} audio (${audio.uint8Array.length} bytes)`);
+    // Optionally save to file for testing
+    await writeFile(`output.${audio.format}`, Buffer.from(audio.uint8Array));
+});
+
+// Error handling
+agent.on("error", (error: Error) => {
+    console.error("[Error]", error.message);
+});
+
+// 4. Main execution
 (async () => {
-    try {
-        // For now: text-only sanity check, no voice pipeline required.
-        await agent.sendText("What is the weather in Berlin?");
+    console.log("\n=== Voice Agent Demo ===");
+    console.log("Testing text-only mode (no WebSocket required)\n");
 
-        // Optional: connect only when an endpoint is provided.
+    try {
+        // Test 1: Simple text query with streaming
+        console.log("--- Test 1: Weather Query ---");
+        const response1 = await agent.sendText("What is the weather in Berlin?");
+        console.log("\n");
+
+        // Test 2: Multi-turn conversation
+        console.log("--- Test 2: Follow-up Question ---");
+        const response2 = await agent.sendText("What about Tokyo?");
+        console.log("\n");
+
+        // Test 3: Time query
+        console.log("--- Test 3: Time Query ---");
+        const response3 = await agent.sendText("What time is it?");
+        console.log("\n");
+
+        // Show conversation history
+        console.log("--- Conversation History ---");
+        const history = agent.getHistory();
+        console.log(`Total messages: ${history.length}`);
+
+        // Optional: Connect to WebSocket for real-time voice
         if (process.env.VOICE_WS_ENDPOINT) {
+            console.log("\n--- Connecting to WebSocket ---");
             await agent.connect(process.env.VOICE_WS_ENDPOINT);
-            console.log("Agent connected successfully");
+            console.log("Agent connected. Listening for audio input...");
+
+            // Keep the process running to receive WebSocket messages
+            // In a real app, you would stream microphone audio here
         }
     } catch (error) {
         console.error("Agent run failed:", error);
+        process.exit(1);
     }
 })();
 
-// 5. Simulate sending audio (in a real app, stream microphone data here)
-// agent.sendAudio("Base64EncodedPCM16AudioData...");
+// Example: How to send audio in a real application
+// ---------------------------------------------
+// import { readFile } from "fs/promises";
+//
+// // Option 1: Send base64 encoded audio
+// const audioBase64 = (await readFile("recording.mp3")).toString("base64");
+// await agent.sendAudio(audioBase64);
+//
+// // Option 2: Send raw audio buffer
+// const audioBuffer = await readFile("recording.mp3");
+// await agent.sendAudioBuffer(audioBuffer);
+//
+// // Option 3: Transcribe audio directly
+// const transcribedText = await agent.transcribeAudio(audioBuffer);
+// console.log("Transcribed:", transcribedText);

example/ws-server.ts

@@ -1,22 +1,73 @@
 import "dotenv/config";
-import { WebSocketServer } from "ws";
+import { WebSocketServer, WebSocket } from "ws";
+import { readFile } from "fs/promises";
+import { existsSync } from "fs";
 
 const endpoint = process.env.VOICE_WS_ENDPOINT || "ws://localhost:8080";
 const url = new URL(endpoint);
 const port = Number(url.port || 8080);
 const host = url.hostname || "localhost";
 
+// Message types for type safety
+interface BaseMessage {
+    type: string;
+}
+
+interface TextDeltaMessage extends BaseMessage {
+    type: "text_delta";
+    text: string;
+}
+
+interface ToolCallMessage extends BaseMessage {
+    type: "tool_call";
+    toolName: string;
+    toolCallId: string;
+    input: unknown;
+}
+
+interface ToolResultMessage extends BaseMessage {
+    type: "tool_result";
+    toolName: string;
+    toolCallId: string;
+    result: unknown;
+}
+
+interface AudioMessage extends BaseMessage {
+    type: "audio";
+    data: string; // base64 encoded
+    format: string;
+}
+
+interface ResponseCompleteMessage extends BaseMessage {
+    type: "response_complete";
+    text: string;
+    toolCalls: Array<{ toolName: string; toolCallId: string; input: unknown }>;
+    toolResults: Array<{ toolName: string; toolCallId: string; output: unknown }>;
+}
+
+type AgentMessage =
+    | TextDeltaMessage
+    | ToolCallMessage
+    | ToolResultMessage
+    | AudioMessage
+    | ResponseCompleteMessage;
+
 const wss = new WebSocketServer({ port, host });
 
 wss.on("listening", () => {
-    console.log(`[ws-server] listening on ${endpoint}`);
+    console.log(`[ws-server] 🚀 listening on ${endpoint}`);
+    console.log("[ws-server] Waiting for connections...\n");
 });
 
-wss.on("connection", (socket) => {
-    console.log("[ws-server] client connected");
+wss.on("connection", (socket: WebSocket) => {
+    console.log("[ws-server] ✓ client connected");
+
+    let streamingText = "";
+    let audioChunks: Buffer[] = [];
 
     // Send a sample transcript to test text pipeline end-to-end.
     setTimeout(() => {
+        console.log("[ws-server] -> Sending test transcript...");
         socket.send(
             JSON.stringify({
                 type: "transcript",
@@ -25,19 +76,94 @@ wss.on("connection", (socket) => {
         );
     }, 500);
 
-    socket.on("message", (data) => {
+    socket.on("message", async (data) => {
         try {
-            const msg = JSON.parse(data.toString()) as {
-                type?: string;
-                text?: string;
-            };
-            console.log("[ws-server] <-", msg);
+            const msg = JSON.parse(data.toString()) as AgentMessage;
+
+            switch (msg.type) {
+                case "text_delta":
+                    // Real-time streaming text from the agent
+                    streamingText += msg.text;
+                    process.stdout.write(msg.text);
+                    break;
+
+                case "tool_call":
+                    console.log(`\n[ws-server] 🛠️ Tool call: ${msg.toolName}`);
+                    console.log(`           Input: ${JSON.stringify(msg.input)}`);
+                    break;
+
+                case "tool_result":
+                    console.log(`[ws-server] 🛠️ Tool result: ${msg.toolName}`);
+                    console.log(`           Result: ${JSON.stringify(msg.result)}`);
+                    break;
+
+                case "audio":
+                    // Handle audio response from TTS
+                    const audioBuffer = Buffer.from(msg.data, "base64");
+                    audioChunks.push(audioBuffer);
+                    console.log(
+                        `[ws-server] 🔊 Received audio: ${audioBuffer.length} bytes (${msg.format})`,
+                    );
+
+                    // Optionally save audio to file for testing
+                    // await writeFile(`output_${Date.now()}.${msg.format}`, audioBuffer);
+                    break;
+
+                case "response_complete":
+                    console.log("\n[ws-server] ✅ Response complete");
+                    console.log(`           Text length: ${msg.text.length}`);
+                    console.log(`           Tool calls: ${msg.toolCalls.length}`);
+                    console.log(`           Tool results: ${msg.toolResults.length}`);
+
+                    // Reset for next response
+                    streamingText = "";
+                    audioChunks = [];
+                    break;
+
+                default:
+                    console.log("[ws-server] <- Unknown message:", msg);
+            }
         } catch {
-            console.log("[ws-server] <- raw", data.toString());
+            console.log("[ws-server] <- raw", data.toString().substring(0, 100));
         }
     });
 
     socket.on("close", () => {
-        console.log("[ws-server] client disconnected");
+        console.log("[ws-server] ✗ client disconnected\n");
+    });
+
+    socket.on("error", (error) => {
+        console.error("[ws-server] Error:", error.message);
     });
 });
+
+// Graceful shutdown
+process.on("SIGINT", () => {
+    console.log("\n[ws-server] Shutting down...");
+    wss.close(() => {
+        console.log("[ws-server] Server closed");
+        process.exit(0);
+    });
+});
+
+// Helper function to simulate sending audio to the agent
+async function simulateAudioInput(socket: WebSocket, audioPath: string) {
+    if (!existsSync(audioPath)) {
+        console.log(`[ws-server] Audio file not found: ${audioPath}`);
+        return;
+    }
+
+    const audioBuffer = await readFile(audioPath);
+    const base64Audio = audioBuffer.toString("base64");
+
+    console.log(`[ws-server] -> Sending audio: ${audioPath} (${audioBuffer.length} bytes)`);
+    socket.send(
+        JSON.stringify({
+            type: "audio",
+            data: base64Audio,
+        }),
+    );
+}
+
+// Export for use as a module
+export { wss, simulateAudioInput };