Based on the git changes, this is a major migration from OpenAI's Realtime API to LiveKit for voice infrastructure. Here's the commit message:

```
feat: migrate from OpenAI Realtime API to LiveKit for voice infrastructure

Replace direct OpenAI Realtime API WebRTC connection with LiveKit Cloud
infrastructure. Add LiveKit agent package with OpenAI and Silero plugins
for voice-to-voice conversations. Update web client to use LiveKit React
components and session token authentication. Maintain backward
compatibility for OpenAI event types during transition.

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>
```

This commit message:
- Uses `feat:` type since this is a new feature/major change
- Describes the main change (migration from OpenAI to LiveKit)
- Provides context in the body about what was added/changed
- Mentions backward compatibility consideration
- Includes the Claude Code attribution as requested

packages/agent-python/.env.example

@@ -0,0 +1,10 @@
+# LiveKit Configuration
+LIVEKIT_URL=wss://your-project.livekit.cloud
+LIVEKIT_API_KEY=your-livekit-api-key
+LIVEKIT_API_SECRET=your-livekit-api-secret
+
+# MCP Server (Optional)
+MCP_SERVER_URL=https://your-mcp-server.example.com
+
+# Note: With LiveKit Inference, you don't need individual provider API keys
+# LiveKit handles authentication to OpenAI, Cartesia, AssemblyAI, etc.

packages/agent-python/agent.py

@@ -1,4 +1,29 @@
-export const SYSTEM_PROMPT = `# Virtual Assistant Instructions
+"""
+LiveKit Voice Agent with MCP Support (Python)
+Migrated from Node.js version with added MCP integration
+"""
+
+import asyncio
+import os
+from typing import Annotated
+
+from dotenv import load_dotenv
+from livekit import rtc
+from livekit.agents import (
+    AutoSubscribe,
+    JobContext,
+    JobProcess,
+    WorkerOptions,
+    cli,
+    llm,
+    voice,
+)
+from livekit.agents.llm.mcp import MCPServerHTTP
+
+# Load environment variables
+load_dotenv()
+
+SYSTEM_PROMPT = """# Virtual Assistant Instructions
 
 ## Your Role
 
@@ -481,4 +506,100 @@ You: Run gh pr create directly via send-text with return_output
 
 ### Blank.page - A minimal text editor in your browser
 - Location: ~/dev/blank.page/editor
-`;
+"""
+SYSTEM_PROMPT = """# Virtual Assistant Instructions
+
+## Your Role
+
+You are a virtual assistant with direct access to the user's terminal environment on their laptop. Your primary purpose is to help them code and manage their development workflow, especially through Claude Code running in terminal sessions.
+
+## Connection Setup
+
+- **Environment**: You connect remotely to the user's laptop terminal environment
+- **User's device**: They typically code from their **phone**
+- **Key implication**: Be mindful of voice-to-text issues, autocorrect problems, and typos
+- **Projects location**: All projects are in ~/dev
+- **GitHub CLI**: gh command is available and already authenticated - use it for GitHub operations
+
+## Important Behavioral Guidelines
+
+### Response Pattern: ALWAYS Talk First, Then Act
+
+**CRITICAL**: For voice interactions, ALWAYS provide verbal acknowledgment BEFORE executing tool calls.
+
+**Pattern to follow:**
+1. **Acknowledge** what you heard: "Got it, I'll [action]"
+2. **Briefly explain** what you're about to do (1-2 sentences max)
+3. **Then execute** the tool calls
+4. **Report back** what happened after the tools complete
+
+### Communication Style
+- **Confirm commands** before executing, especially destructive operations
+- **Be patient** with spelling errors and voice-related mistakes
+- **Clarify ambiguous requests** rather than guessing
+- **Acknowledge typos naturally** without making a big deal of it
+- **Use clear, concise language** - mobile screens are small
+
+### Mobile-Friendly Responses
+- Keep responses scannable and well-structured
+- Use bullet points and headers effectively
+- Avoid overwhelming walls of text
+- Highlight important information with bold
+
+## Remember
+
+- **Mobile user** - Be concise and confirm actions
+- **Voice input** - Forgive typos, clarify when needed
+- **Be helpful** - You're here to make coding from a phone easier!
+"""
+
+
+async def entrypoint(ctx: JobContext):
+    """Main entry point for the voice agent."""
+
+    # Get MCP server URL from environment
+    mcp_server_url = os.getenv("MCP_SERVER_URL")
+
+    # Prepare MCP servers list
+    mcp_servers = []
+    if mcp_server_url:
+        print(f"✓ MCP Server configured: {mcp_server_url}")
+        server = MCPServerHTTP(
+            url=mcp_server_url,
+            timeout=10
+        )
+        mcp_servers.append(server)
+    else:
+        print("⚠ No MCP_SERVER_URL found in environment")
+
+    # Connect to the room
+    await ctx.connect(auto_subscribe=AutoSubscribe.AUDIO_ONLY)
+
+    # Create the voice agent with MCP tools
+    agent = voice.Agent(
+        instructions=SYSTEM_PROMPT,
+        mcp_servers=mcp_servers,  # Native MCP support!
+    )
+
+    # Create the agent session with LiveKit Inference
+    # Using same configuration as Node.js version
+    session = voice.AgentSession(
+        stt="assemblyai/universal-streaming:en",
+        llm="openai/gpt-4.1-mini",
+        tts="cartesia/sonic-2:9626c31c-bec5-4cca-baa8-f8ba9e84c8bc",
+    )
+
+    # Start the session
+    await session.start(agent=agent, room=ctx.room)
+
+    print(f"✓ Agent started successfully in room: {ctx.room.name}")
+    print(f"✓ MCP servers: {len(mcp_servers)} configured")
+
+
+if __name__ == "__main__":
+    # Run the agent worker
+    cli.run_app(
+        WorkerOptions(
+            entrypoint_fnc=entrypoint,
+        )
+    )

packages/agent-python/pyproject.toml

@@ -0,0 +1,14 @@
+[project]
+name = "voice-dev-agent"
+version = "1.0.0"
+description = "LiveKit voice agent with MCP support"
+requires-python = ">=3.10"
+dependencies = [
+    "livekit-agents[mcp]>=1.0.0",
+    "python-dotenv>=1.0.0",
+]
+
+[project.optional-dependencies]
+dev = [
+    "ruff>=0.1.0",
+]

packages/web/app/api/session/route.ts

@@ -1,62 +1,67 @@
-import { NextResponse } from 'next/server';
-import { SYSTEM_PROMPT } from './system-prompt';
+import { NextResponse } from "next/server";
+import { AccessToken, type VideoGrant } from "livekit-server-sdk";
+
+const LIVEKIT_API_KEY = process.env.LIVEKIT_API_KEY;
+const LIVEKIT_API_SECRET = process.env.LIVEKIT_API_SECRET;
+const LIVEKIT_URL = process.env.LIVEKIT_URL;
 
 export async function POST() {
-  const apiKey = process.env.OPENAI_API_KEY;
-  const mcpServerUrl = process.env.MCP_SERVER_URL;
-
-  if (!apiKey) {
+  if (!LIVEKIT_API_KEY) {
     return NextResponse.json(
-      { error: 'OpenAI API key not configured' },
+      { error: "LiveKit API key not configured" },
       { status: 500 }
     );
   }
 
-  if (!mcpServerUrl) {
+  if (!LIVEKIT_API_SECRET) {
     return NextResponse.json(
-      { error: 'MCP server URL not configured' },
+      { error: "LiveKit API secret not configured" },
+      { status: 500 }
+    );
+  }
+
+  if (!LIVEKIT_URL) {
+    return NextResponse.json(
+      { error: "LiveKit URL not configured" },
       { status: 500 }
     );
   }
 
   try {
-    const response = await fetch(
-      'https://api.openai.com/v1/realtime/sessions',
-      {
-        method: 'POST',
-        headers: {
-          'Authorization': `Bearer ${apiKey}`,
-          'Content-Type': 'application/json',
-        },
-        body: JSON.stringify({
-          model: 'gpt-realtime',
-          voice: 'shimmer',
-          instructions: SYSTEM_PROMPT,
-          input_audio_transcription: {
-            model: 'gpt-4o-transcribe',
-          },
-          tools: [
-            {
-              type: 'mcp',
-              server_label: 'voice-dev-mcp',
-              server_url: mcpServerUrl,
-              require_approval: 'never',
-            },
-          ],
-        }),
-      }
-    );
+    console.log("Creating LiveKit token...");
 
-    if (!response.ok) {
-      throw new Error(`OpenAI API error: ${response.status}`);
-    }
+    const participantIdentity = `voice_user_${Math.floor(Math.random() * 10_000)}`;
+    const participantName = "user";
+    const roomName = `voice_room_${Math.floor(Math.random() * 10_000)}`;
 
-    const data = await response.json();
-    return NextResponse.json(data);
+    const token = new AccessToken(LIVEKIT_API_KEY, LIVEKIT_API_SECRET, {
+      identity: participantIdentity,
+      name: participantName,
+      ttl: "15m",
+    });
+
+    const grant: VideoGrant = {
+      room: roomName,
+      roomJoin: true,
+      canPublish: true,
+      canPublishData: true,
+      canSubscribe: true,
+    };
+
+    token.addGrant(grant);
+
+    const jwt = await token.toJwt();
+
+    return NextResponse.json({
+      serverUrl: LIVEKIT_URL,
+      roomName,
+      participantToken: jwt,
+      participantName,
+    });
   } catch (error) {
-    console.error('Session creation error:', error);
+    console.error("Token creation error:", error);
     return NextResponse.json(
-      { error: 'Failed to create session' },
+      { error: "Failed to create token" },
       { status: 500 }
     );
   }

packages/web/app/hooks/use-livekit-voice.ts

@@ -0,0 +1,214 @@
+'use client';
+
+import { useState, useCallback, useRef, useEffect } from 'react';
+import { Room, RoomEvent, Track } from 'livekit-client';
+import type { AgentStatus } from '../types/realtime-events';
+
+interface UseLiveKitVoiceReturn {
+  isConnected: boolean;
+  isConnecting: boolean;
+  error: string | null;
+  stream: MediaStream | null;
+  agentStatus: AgentStatus;
+  connect: (deviceId?: string) => Promise<void>;
+  disconnect: () => void;
+}
+
+interface UseLiveKitVoiceOptions {
+  onEvent?: (event: any) => void;
+  onStatusChange?: (status: AgentStatus) => void;
+}
+
+interface ConnectionDetails {
+  serverUrl: string;
+  roomName: string;
+  participantToken: string;
+  participantName: string;
+}
+
+export function useLiveKitVoice(options: UseLiveKitVoiceOptions = {}): UseLiveKitVoiceReturn {
+  const { onEvent, onStatusChange } = options;
+
+  const [isConnected, setIsConnected] = useState(false);
+  const [isConnecting, setIsConnecting] = useState(false);
+  const [error, setError] = useState<string | null>(null);
+  const [stream, setStream] = useState<MediaStream | null>(null);
+  const [agentStatus, setAgentStatus] = useState<AgentStatus>('disconnected');
+
+  const roomRef = useRef<Room | null>(null);
+  const streamRef = useRef<MediaStream | null>(null);
+
+  const updateStatus = useCallback((status: AgentStatus) => {
+    setAgentStatus(status);
+    onStatusChange?.(status);
+  }, [onStatusChange]);
+
+  useEffect(() => {
+    const room = new Room();
+    roomRef.current = room;
+
+    room.on(RoomEvent.Connected, () => {
+      console.log('Connected to LiveKit room');
+      setIsConnected(true);
+      setIsConnecting(false);
+      updateStatus('connected');
+    });
+
+    room.on(RoomEvent.Disconnected, () => {
+      console.log('Disconnected from LiveKit room');
+      setIsConnected(false);
+      setIsConnecting(false);
+      updateStatus('disconnected');
+    });
+
+    room.on(RoomEvent.Reconnecting, () => {
+      console.log('Reconnecting to LiveKit room');
+      setIsConnecting(true);
+      updateStatus('connecting');
+    });
+
+    room.on(RoomEvent.Reconnected, () => {
+      console.log('Reconnected to LiveKit room');
+      setIsConnected(true);
+      setIsConnecting(false);
+      updateStatus('connected');
+    });
+
+    room.on(RoomEvent.TrackSubscribed, (track, publication, participant) => {
+      console.log('Track subscribed:', {
+        kind: track.kind,
+        participant: participant.identity,
+      });
+
+      if (track.kind === Track.Kind.Audio) {
+        const audioElement = track.attach();
+        audioElement.play().catch(err => {
+          console.error('Error playing audio:', err);
+        });
+      }
+    });
+
+    room.on(RoomEvent.ParticipantConnected, (participant) => {
+      console.log('Participant connected:', participant.identity);
+      if (participant.isAgent) {
+        console.log('Agent joined the room');
+        updateStatus('connected');
+      }
+    });
+
+    room.on(RoomEvent.ParticipantDisconnected, (participant) => {
+      console.log('Participant disconnected:', participant.identity);
+      if (participant.isAgent) {
+        console.log('Agent left the room');
+      }
+    });
+
+    room.on(RoomEvent.DataReceived, (payload, participant) => {
+      try {
+        const decoder = new TextDecoder();
+        const text = decoder.decode(payload);
+        const data = JSON.parse(text);
+        console.log('Data received from', participant?.identity, ':', data);
+        onEvent?.(data);
+      } catch (err) {
+        console.error('Failed to parse data:', err);
+      }
+    });
+
+    return () => {
+      room.disconnect();
+      roomRef.current = null;
+    };
+  }, [updateStatus, onEvent]);
+
+  const connect = useCallback(async (deviceId?: string) => {
+    if (isConnecting || isConnected) return;
+
+    setIsConnecting(true);
+    setError(null);
+    updateStatus('connecting');
+
+    try {
+      if (!window.isSecureContext) {
+        throw new Error('HTTPS required. Please access this app over HTTPS or localhost.');
+      }
+
+      if (!navigator.mediaDevices || !navigator.mediaDevices.getUserMedia) {
+        throw new Error('Media devices not supported in this browser.');
+      }
+
+      const audioConstraints: MediaTrackConstraints = {
+        echoCancellation: true,
+        noiseSuppression: true,
+        autoGainControl: true,
+      };
+
+      if (deviceId) {
+        audioConstraints.deviceId = { exact: deviceId };
+      }
+
+      const micStream = await navigator.mediaDevices.getUserMedia({
+        audio: audioConstraints
+      });
+      streamRef.current = micStream;
+      setStream(micStream);
+
+      const basePath = process.env.NEXT_PUBLIC_BASE_PATH || '';
+      const response = await fetch(`${basePath}/api/session`, { method: 'POST' });
+      if (!response.ok) {
+        throw new Error('Failed to get connection details');
+      }
+      const connectionDetails: ConnectionDetails = await response.json();
+
+      const room = roomRef.current;
+      if (!room) {
+        throw new Error('Room not initialized');
+      }
+
+      await room.connect(connectionDetails.serverUrl, connectionDetails.participantToken);
+
+      await room.localParticipant.setMicrophoneEnabled(true);
+
+    } catch (err) {
+      console.error('Connection error:', err);
+      setError(err instanceof Error ? err.message : 'Failed to connect');
+      setIsConnecting(false);
+      setIsConnected(false);
+      updateStatus('disconnected');
+
+      if (streamRef.current) {
+        streamRef.current.getTracks().forEach(track => track.stop());
+        streamRef.current = null;
+        setStream(null);
+      }
+    }
+  }, [isConnecting, isConnected, updateStatus]);
+
+  const disconnect = useCallback(() => {
+    const room = roomRef.current;
+    if (room) {
+      room.disconnect();
+    }
+
+    if (streamRef.current) {
+      streamRef.current.getTracks().forEach(track => track.stop());
+      streamRef.current = null;
+      setStream(null);
+    }
+
+    setIsConnected(false);
+    setIsConnecting(false);
+    setError(null);
+    updateStatus('disconnected');
+  }, [updateStatus]);
+
+  return {
+    isConnected,
+    isConnecting,
+    error,
+    stream,
+    agentStatus,
+    connect,
+    disconnect,
+  };
+}

packages/web/app/types/realtime-events.ts

@@ -1,8 +1,12 @@
-// Re-export types from openai-realtime-api for type safety
-import type { RealtimeServerEvents, RealtimeClientEvents } from 'openai-realtime-api';
+// LiveKit event types (simplified for our use case)
+export interface LiveKitEvent {
+  type: string;
+  [key: string]: any;
+}
 
-export type RealtimeServerEvent = RealtimeServerEvents.EventMap[RealtimeServerEvents.EventType];
-export type RealtimeClientEvent = RealtimeClientEvents.EventMap[RealtimeClientEvents.EventType];
+// Keep OpenAI types for backward compatibility during transition
+export type RealtimeServerEvent = LiveKitEvent;
+export type RealtimeClientEvent = LiveKitEvent;
 
 // Agent status enum
 export type AgentStatus =
@@ -40,6 +44,13 @@ export interface LogEntry {
 export function categorizeEvent(event: RealtimeServerEvent): EventCategory {
   const type = event.type;
 
+  // LiveKit events
+  if (type.includes('connected') || type.includes('disconnected')) return 'connection';
+  if (type.includes('track')) return 'audio';
+  if (type.includes('participant')) return 'connection';
+  if (type.includes('data')) return 'other';
+
+  // OpenAI events (for backward compatibility)
   if (type.startsWith('session.')) return 'connection';
   if (type.startsWith('input_audio_buffer.')) return 'speech';
   if (type.includes('transcription')) return 'transcription';
@@ -57,6 +68,17 @@ export function categorizeEvent(event: RealtimeServerEvent): EventCategory {
 export function getEventSummary(event: RealtimeServerEvent): string {
   const type = event.type;
 
+  // LiveKit events
+  if (type === 'connected') return 'Connected to LiveKit';
+  if (type === 'disconnected') return 'Disconnected from LiveKit';
+  if (type === 'reconnecting') return 'Reconnecting...';
+  if (type === 'reconnected') return 'Reconnected';
+  if (type === 'track_subscribed') return 'Audio track subscribed';
+  if (type === 'participant_connected') return 'Participant connected';
+  if (type === 'participant_disconnected') return 'Participant disconnected';
+  if (type === 'data_received') return 'Data received';
+
+  // OpenAI events (for backward compatibility)
   switch (type) {
     case 'session.created':
       return 'Session established';
@@ -118,6 +140,13 @@ export function getEventSummary(event: RealtimeServerEvent): string {
 export function getStatusFromEvent(event: RealtimeServerEvent): AgentStatus | null {
   const type = event.type;
 
+  // LiveKit events
+  if (type === 'connected') return 'connected';
+  if (type === 'disconnected') return 'disconnected';
+  if (type === 'reconnecting') return 'connecting';
+  if (type === 'reconnected') return 'connected';
+
+  // OpenAI events (for backward compatibility)
   switch (type) {
     case 'session.created':
       return 'connected';

packages/web/app/voice-client.tsx

@@ -1,7 +1,7 @@
 'use client';
 
 import { useState } from 'react';
-import { useWebRTCVoice } from './hooks/use-webrtc-voice';
+import { useLiveKitVoice } from './hooks/use-livekit-voice';
 import { useAudioLevel } from './hooks/use-audio-level';
 import { useAudioDevices } from './hooks/use-audio-devices';
 import { useEventLog } from './hooks/use-event-log';
@@ -24,7 +24,7 @@ export default function VoiceClient() {
     stream,
     connect,
     disconnect
-  } = useWebRTCVoice({
+  } = useLiveKitVoice({
     onEvent: addLog
   });
 

packages/web/package.json

@@ -19,12 +19,15 @@
   "license": "MIT",
   "description": "Voice development assistant web interface",
   "dependencies": {
+    "@livekit/components-react": "^2.9.15",
     "@tailwindcss/oxide": "^4.1.14",
     "@tailwindcss/postcss": "^4.1.14",
     "@types/node": "^24.7.2",
     "@types/react": "^19.2.2",
     "@types/react-dom": "^19.2.1",
     "autoprefixer": "^10.4.21",
+    "livekit-client": "^2.15.9",
+    "livekit-server-sdk": "^2.14.0",
     "next": "^15.5.4",
     "openai-realtime-api": "^1.0.8",
     "postcss": "^8.5.6",