Based on the git diff showing extensive package-lock.json changes and modifications to the agent system prompt and configuration, here's the commit message:
``` feat: add AI SDK dependencies and refine agent voice interaction config - Add @ai-sdk packages for enhanced AI capabilities - Install Deepgram SDK, livekit-client, and werift for voice/WebRTC - Expand agent system prompt with multi-tool workflows and Claude Code interaction patterns - Adjust VAD timing (1.2s silence threshold) and turn detection for better voice UX - Switch TTS to inworld/inworld-tts-1 with Olivia voice - Configure interruption timing (0.2s detection, no word requirement) - Add comprehensive guidelines for conversational responses vs structured output - Document plan mode triggers and Claude response type handling 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com> ```
This commit is contained in:
Mohamed Boudra
@@ -0,0 +1,391 @@
|
||||
.app {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
min-height: 100vh;
|
||||
width: 100%;
|
||||
}
|
||||
|
||||
.header {
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
align-items: center;
|
||||
padding: 1rem 2rem;
|
||||
background-color: #1a1a1a;
|
||||
border-bottom: 1px solid #333;
|
||||
}
|
||||
|
||||
.header h1 {
|
||||
font-size: 1.5rem;
|
||||
font-weight: 600;
|
||||
}
|
||||
|
||||
.status-indicator {
|
||||
padding: 0.5rem 1rem;
|
||||
border-radius: 4px;
|
||||
font-size: 0.75rem;
|
||||
font-weight: 500;
|
||||
text-transform: uppercase;
|
||||
}
|
||||
|
||||
.status-indicator.disconnected {
|
||||
background-color: #444;
|
||||
color: #999;
|
||||
}
|
||||
|
||||
.status-indicator.connecting {
|
||||
background-color: #ffa500;
|
||||
color: #000;
|
||||
animation: pulse 1.5s ease-in-out infinite;
|
||||
}
|
||||
|
||||
.status-indicator.connected {
|
||||
background-color: #22c55e;
|
||||
color: #000;
|
||||
}
|
||||
|
||||
@keyframes pulse {
|
||||
0%, 100% {
|
||||
opacity: 1;
|
||||
}
|
||||
50% {
|
||||
opacity: 0.6;
|
||||
}
|
||||
}
|
||||
|
||||
.main {
|
||||
display: flex;
|
||||
flex: 1;
|
||||
padding: 2rem;
|
||||
gap: 2rem;
|
||||
}
|
||||
|
||||
.chat-interface {
|
||||
flex: 1;
|
||||
background-color: #1a1a1a;
|
||||
border-radius: 8px;
|
||||
padding: 2rem;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 1rem;
|
||||
max-height: calc(100vh - 120px);
|
||||
}
|
||||
|
||||
.chat-interface h2 {
|
||||
font-size: 1.25rem;
|
||||
font-weight: 600;
|
||||
margin: 0;
|
||||
padding-bottom: 0.75rem;
|
||||
border-bottom: 1px solid #333;
|
||||
}
|
||||
|
||||
.ping-button {
|
||||
padding: 0.75rem 2rem;
|
||||
font-size: 1rem;
|
||||
font-weight: 500;
|
||||
background-color: #22c55e;
|
||||
color: #000;
|
||||
border: none;
|
||||
border-radius: 6px;
|
||||
cursor: pointer;
|
||||
transition: background-color 0.2s;
|
||||
}
|
||||
|
||||
.ping-button:hover:not(:disabled) {
|
||||
background-color: #16a34a;
|
||||
}
|
||||
|
||||
.ping-button:disabled {
|
||||
background-color: #444;
|
||||
color: #999;
|
||||
cursor: not-allowed;
|
||||
}
|
||||
|
||||
.connection-info {
|
||||
color: #999;
|
||||
font-size: 0.875rem;
|
||||
text-align: center;
|
||||
max-width: 400px;
|
||||
}
|
||||
|
||||
.activity-log {
|
||||
flex: 1;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
overflow: hidden;
|
||||
}
|
||||
|
||||
.log-entries {
|
||||
flex: 1;
|
||||
overflow-y: auto;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 0.5rem;
|
||||
}
|
||||
|
||||
.log-entry {
|
||||
padding: 0.75rem;
|
||||
border-radius: 4px;
|
||||
font-size: 0.875rem;
|
||||
line-height: 1.4;
|
||||
display: flex;
|
||||
gap: 0.75rem;
|
||||
}
|
||||
|
||||
.log-time {
|
||||
font-family: monospace;
|
||||
color: #666;
|
||||
flex-shrink: 0;
|
||||
}
|
||||
|
||||
.log-message {
|
||||
flex: 1;
|
||||
}
|
||||
|
||||
.log-entry.system {
|
||||
background-color: #2a2a2a;
|
||||
color: #999;
|
||||
font-style: italic;
|
||||
}
|
||||
|
||||
.log-entry.info {
|
||||
background-color: #1e3a8a;
|
||||
color: #93c5fd;
|
||||
}
|
||||
|
||||
.log-entry.success {
|
||||
background-color: #14532d;
|
||||
color: #86efac;
|
||||
}
|
||||
|
||||
.log-entry.error {
|
||||
background-color: #7f1d1d;
|
||||
color: #fca5a5;
|
||||
}
|
||||
|
||||
.log-entry.user {
|
||||
background-color: #1e293b;
|
||||
color: #e2e8f0;
|
||||
}
|
||||
|
||||
.log-entry.assistant {
|
||||
background-color: #0f766e;
|
||||
color: #ccfbf1;
|
||||
}
|
||||
|
||||
.log-entry.tool {
|
||||
background-color: #581c87;
|
||||
color: #e9d5ff;
|
||||
font-family: monospace;
|
||||
font-size: 0.75rem;
|
||||
}
|
||||
|
||||
.log-entry.streaming {
|
||||
opacity: 0.8;
|
||||
}
|
||||
|
||||
.streaming-indicator {
|
||||
animation: dots 1.5s steps(3, end) infinite;
|
||||
color: #666;
|
||||
}
|
||||
|
||||
@keyframes dots {
|
||||
0%, 20% { content: '.'; }
|
||||
40% { content: '..'; }
|
||||
60%, 100% { content: '...'; }
|
||||
}
|
||||
|
||||
.log-metadata {
|
||||
margin-top: 0.5rem;
|
||||
}
|
||||
|
||||
.log-metadata summary {
|
||||
cursor: pointer;
|
||||
color: #999;
|
||||
font-size: 0.75rem;
|
||||
}
|
||||
|
||||
.log-metadata pre {
|
||||
margin-top: 0.5rem;
|
||||
padding: 0.5rem;
|
||||
background-color: #000;
|
||||
border-radius: 4px;
|
||||
overflow-x: auto;
|
||||
font-size: 0.75rem;
|
||||
}
|
||||
|
||||
.message-form {
|
||||
display: flex;
|
||||
gap: 0.5rem;
|
||||
padding-top: 1rem;
|
||||
border-top: 1px solid #333;
|
||||
}
|
||||
|
||||
.message-input {
|
||||
flex: 1;
|
||||
padding: 0.75rem 1rem;
|
||||
background-color: #2a2a2a;
|
||||
border: 1px solid #444;
|
||||
border-radius: 6px;
|
||||
color: #fff;
|
||||
font-size: 0.875rem;
|
||||
outline: none;
|
||||
}
|
||||
|
||||
.message-input:focus {
|
||||
border-color: #22c55e;
|
||||
}
|
||||
|
||||
.message-input:disabled {
|
||||
opacity: 0.5;
|
||||
cursor: not-allowed;
|
||||
}
|
||||
|
||||
.send-button {
|
||||
padding: 0.75rem 1.5rem;
|
||||
background-color: #22c55e;
|
||||
color: #000;
|
||||
border: none;
|
||||
border-radius: 6px;
|
||||
font-size: 0.875rem;
|
||||
font-weight: 500;
|
||||
cursor: pointer;
|
||||
transition: background-color 0.2s;
|
||||
}
|
||||
|
||||
.send-button:hover:not(:disabled) {
|
||||
background-color: #16a34a;
|
||||
}
|
||||
|
||||
.send-button:disabled {
|
||||
background-color: #444;
|
||||
color: #999;
|
||||
cursor: not-allowed;
|
||||
}
|
||||
|
||||
.test-controls {
|
||||
padding-top: 0.5rem;
|
||||
border-top: 1px solid #333;
|
||||
display: flex;
|
||||
justify-content: center;
|
||||
}
|
||||
|
||||
/* Voice Controls */
|
||||
.voice-controls {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 0.75rem;
|
||||
padding: 1rem;
|
||||
background-color: #2a2a2a;
|
||||
border-radius: 6px;
|
||||
margin-bottom: 1rem;
|
||||
}
|
||||
|
||||
.voice-button {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
gap: 0.5rem;
|
||||
padding: 1rem 2rem;
|
||||
background-color: #3b82f6;
|
||||
color: #fff;
|
||||
border: none;
|
||||
border-radius: 8px;
|
||||
font-size: 1rem;
|
||||
font-weight: 500;
|
||||
cursor: pointer;
|
||||
transition: all 0.2s;
|
||||
}
|
||||
|
||||
.voice-button:hover:not(:disabled) {
|
||||
background-color: #2563eb;
|
||||
transform: scale(1.02);
|
||||
}
|
||||
|
||||
.voice-button:disabled {
|
||||
background-color: #444;
|
||||
color: #999;
|
||||
cursor: not-allowed;
|
||||
transform: none;
|
||||
}
|
||||
|
||||
.voice-button.recording {
|
||||
background-color: #ef4444;
|
||||
animation: pulse-red 1.5s ease-in-out infinite;
|
||||
}
|
||||
|
||||
.voice-button.recording:hover:not(:disabled) {
|
||||
background-color: #dc2626;
|
||||
}
|
||||
|
||||
.voice-button.processing {
|
||||
background-color: #f59e0b;
|
||||
}
|
||||
|
||||
@keyframes pulse-red {
|
||||
0%, 100% {
|
||||
opacity: 1;
|
||||
}
|
||||
50% {
|
||||
opacity: 0.7;
|
||||
}
|
||||
}
|
||||
|
||||
.recording-indicator {
|
||||
display: inline-block;
|
||||
width: 12px;
|
||||
height: 12px;
|
||||
background-color: #fff;
|
||||
border-radius: 50%;
|
||||
animation: pulse 1s ease-in-out infinite;
|
||||
}
|
||||
|
||||
.processing-indicator {
|
||||
display: inline-block;
|
||||
width: 12px;
|
||||
height: 12px;
|
||||
border: 2px solid #fff;
|
||||
border-top-color: transparent;
|
||||
border-radius: 50%;
|
||||
animation: spin 0.8s linear infinite;
|
||||
}
|
||||
|
||||
@keyframes spin {
|
||||
to {
|
||||
transform: rotate(360deg);
|
||||
}
|
||||
}
|
||||
|
||||
.microphone-icon {
|
||||
font-size: 1.25rem;
|
||||
}
|
||||
|
||||
.voice-error {
|
||||
padding: 0.75rem;
|
||||
background-color: #7f1d1d;
|
||||
color: #fca5a5;
|
||||
border-radius: 4px;
|
||||
font-size: 0.875rem;
|
||||
}
|
||||
|
||||
.permission-denied {
|
||||
padding: 0.75rem;
|
||||
background-color: #7c2d12;
|
||||
color: #fed7aa;
|
||||
border-radius: 4px;
|
||||
font-size: 0.875rem;
|
||||
}
|
||||
|
||||
.playing-indicator {
|
||||
padding: 0.5rem;
|
||||
background-color: #0f766e;
|
||||
color: #ccfbf1;
|
||||
border-radius: 4px;
|
||||
font-size: 0.875rem;
|
||||
text-align: center;
|
||||
}
|
||||
|
||||
@media (max-width: 1024px) {
|
||||
.main {
|
||||
flex-direction: column;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,260 @@
|
||||
import { useState, useEffect, useRef } from 'react';
|
||||
import { useWebSocket } from './hooks/useWebSocket';
|
||||
import { VoiceControls } from './components/VoiceControls';
|
||||
import { createAudioPlayer } from './lib/audio-playback';
|
||||
import './App.css';
|
||||
|
||||
interface LogEntry {
|
||||
id: string;
|
||||
timestamp: number;
|
||||
type: 'system' | 'info' | 'success' | 'error' | 'user' | 'assistant' | 'tool';
|
||||
message: string;
|
||||
metadata?: Record<string, unknown>;
|
||||
}
|
||||
|
||||
function App() {
|
||||
const [logs, setLogs] = useState<LogEntry[]>([
|
||||
{ id: '1', timestamp: Date.now(), type: 'system', message: 'System initialized' },
|
||||
]);
|
||||
const [userInput, setUserInput] = useState('');
|
||||
const [currentAssistantMessage, setCurrentAssistantMessage] = useState('');
|
||||
const [isProcessingAudio, setIsProcessingAudio] = useState(false);
|
||||
const [isPlayingAudio, setIsPlayingAudio] = useState(false);
|
||||
const logEndRef = useRef<HTMLDivElement>(null);
|
||||
const audioPlayerRef = useRef(createAudioPlayer());
|
||||
|
||||
// WebSocket URL - use ws://localhost:3000/ws in dev, or construct from current host in prod
|
||||
const wsUrl = `ws://${window.location.host}/ws`;
|
||||
|
||||
const ws = useWebSocket(wsUrl);
|
||||
|
||||
const addLog = (
|
||||
type: LogEntry['type'],
|
||||
message: string,
|
||||
metadata?: Record<string, unknown>
|
||||
) => {
|
||||
console.trace("addLog", type, message, metadata);
|
||||
setLogs((prev) => [
|
||||
...prev,
|
||||
{ id: Date.now().toString(), timestamp: Date.now(), type, message, metadata },
|
||||
]);
|
||||
};
|
||||
|
||||
useEffect(() => {
|
||||
// Listen for status messages
|
||||
const unsubStatus = ws.on('status', (payload: unknown) => {
|
||||
const data = payload as { status: string; message?: string };
|
||||
addLog('info', data.message || `Status: ${data.status}`);
|
||||
});
|
||||
|
||||
// Listen for pong messages
|
||||
const unsubPong = ws.on('pong', () => {
|
||||
addLog('success', 'Received pong from server');
|
||||
});
|
||||
|
||||
// Listen for activity log messages
|
||||
const unsubActivity = ws.on('activity_log', (payload: unknown) => {
|
||||
const data = payload as {
|
||||
type: string;
|
||||
content: string;
|
||||
metadata?: Record<string, unknown>;
|
||||
};
|
||||
|
||||
// Map activity log types to UI log types
|
||||
let logType: LogEntry['type'] = 'info';
|
||||
if (data.type === 'transcript') logType = 'user';
|
||||
else if (data.type === 'assistant') logType = 'assistant';
|
||||
else if (data.type === 'tool_call' || data.type === 'tool_result') logType = 'tool';
|
||||
else if (data.type === 'error') logType = 'error';
|
||||
|
||||
addLog(logType, data.content, data.metadata);
|
||||
});
|
||||
|
||||
// Listen for streaming assistant chunks
|
||||
const unsubChunk = ws.on('assistant_chunk', (payload: unknown) => {
|
||||
const data = payload as { chunk: string };
|
||||
setCurrentAssistantMessage((prev) => prev + data.chunk);
|
||||
});
|
||||
|
||||
// Listen for transcription results
|
||||
const unsubTranscription = ws.on('transcription_result', (payload: unknown) => {
|
||||
const data = payload as { text: string };
|
||||
addLog('info', `Transcription: ${data.text}`);
|
||||
setIsProcessingAudio(false);
|
||||
});
|
||||
|
||||
// Listen for audio output (TTS)
|
||||
const unsubAudioOutput = ws.on('audio_output', async (payload: unknown) => {
|
||||
const data = payload as { audio: string; format: string; id: string };
|
||||
|
||||
try {
|
||||
addLog('info', 'Playing assistant audio response...');
|
||||
setIsPlayingAudio(true);
|
||||
|
||||
// Decode base64 audio
|
||||
const binaryString = atob(data.audio);
|
||||
const bytes = new Uint8Array(binaryString.length);
|
||||
for (let i = 0; i < binaryString.length; i++) {
|
||||
bytes[i] = binaryString.charCodeAt(i);
|
||||
}
|
||||
|
||||
// Create blob with correct mime type
|
||||
const mimeType = data.format === 'mp3' ? 'audio/mpeg' : `audio/${data.format}`;
|
||||
const audioBlob = new Blob([bytes], { type: mimeType });
|
||||
|
||||
// Play audio
|
||||
await audioPlayerRef.current.play(audioBlob);
|
||||
|
||||
setIsPlayingAudio(false);
|
||||
addLog('success', 'Audio playback complete');
|
||||
} catch (error: any) {
|
||||
console.error('[App] Audio playback error:', error);
|
||||
addLog('error', `Audio playback failed: ${error.message}`);
|
||||
setIsPlayingAudio(false);
|
||||
}
|
||||
});
|
||||
|
||||
return () => {
|
||||
unsubStatus();
|
||||
unsubPong();
|
||||
unsubActivity();
|
||||
unsubChunk();
|
||||
unsubTranscription();
|
||||
unsubAudioOutput();
|
||||
};
|
||||
}, [ws]);
|
||||
|
||||
useEffect(() => {
|
||||
if (ws.isConnected) {
|
||||
addLog('success', 'WebSocket connected');
|
||||
} else {
|
||||
addLog('system', 'WebSocket disconnected');
|
||||
}
|
||||
}, [ws.isConnected]);
|
||||
|
||||
// Auto-scroll to bottom when new logs are added
|
||||
useEffect(() => {
|
||||
logEndRef.current?.scrollIntoView({ behavior: 'smooth' });
|
||||
}, [logs, currentAssistantMessage]);
|
||||
|
||||
const handlePing = () => {
|
||||
ws.sendPing();
|
||||
addLog('info', 'Sent ping to server');
|
||||
};
|
||||
|
||||
const handleSendMessage = (e: React.FormEvent) => {
|
||||
e.preventDefault();
|
||||
if (!userInput.trim() || !ws.isConnected) return;
|
||||
|
||||
// Send message to server
|
||||
ws.sendUserMessage(userInput);
|
||||
|
||||
// Clear input and reset streaming state
|
||||
setUserInput('');
|
||||
setCurrentAssistantMessage('');
|
||||
};
|
||||
|
||||
const handleAudioRecorded = async (audioBlob: Blob, format: string) => {
|
||||
if (!ws.isConnected) {
|
||||
addLog('error', 'Cannot send audio - not connected to server');
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
setIsProcessingAudio(true);
|
||||
addLog('info', 'Sending audio to server...');
|
||||
|
||||
// Convert blob to base64
|
||||
const arrayBuffer = await audioBlob.arrayBuffer();
|
||||
const base64Audio = btoa(
|
||||
new Uint8Array(arrayBuffer).reduce((data, byte) => data + String.fromCharCode(byte), '')
|
||||
);
|
||||
|
||||
// Send audio chunk via WebSocket
|
||||
ws.send({
|
||||
type: 'audio_chunk',
|
||||
payload: {
|
||||
audio: base64Audio,
|
||||
format: format,
|
||||
isLast: true,
|
||||
},
|
||||
});
|
||||
|
||||
console.log(`[App] Sent audio: ${audioBlob.size} bytes, format: ${format}`);
|
||||
} catch (error: any) {
|
||||
console.error('[App] Error sending audio:', error);
|
||||
addLog('error', `Failed to send audio: ${error.message}`);
|
||||
setIsProcessingAudio(false);
|
||||
}
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="app">
|
||||
<header className="header">
|
||||
<h1>Voice Assistant</h1>
|
||||
<div className={`status-indicator ${ws.isConnected ? 'connected' : 'disconnected'}`}>
|
||||
{ws.isConnected ? 'connected' : 'disconnected'}
|
||||
</div>
|
||||
</header>
|
||||
|
||||
<main className="main">
|
||||
<div className="chat-interface">
|
||||
<h2>Chat with Assistant</h2>
|
||||
|
||||
<VoiceControls
|
||||
onAudioRecorded={handleAudioRecorded}
|
||||
isProcessing={isProcessingAudio}
|
||||
isPlaying={isPlayingAudio}
|
||||
/>
|
||||
|
||||
<div className="activity-log">
|
||||
<div className="log-entries">
|
||||
{logs.map((log) => (
|
||||
<div key={log.id} className={`log-entry ${log.type}`}>
|
||||
<span className="log-time">{new Date(log.timestamp).toLocaleTimeString()}</span>
|
||||
<span className="log-message">{log.message}</span>
|
||||
{log.metadata && (
|
||||
<details className="log-metadata">
|
||||
<summary>Details</summary>
|
||||
<pre>{JSON.stringify(log.metadata, null, 2)}</pre>
|
||||
</details>
|
||||
)}
|
||||
</div>
|
||||
))}
|
||||
{currentAssistantMessage && (
|
||||
<div className="log-entry assistant streaming">
|
||||
<span className="log-time">{new Date().toLocaleTimeString()}</span>
|
||||
<span className="log-message">{currentAssistantMessage}</span>
|
||||
<span className="streaming-indicator">...</span>
|
||||
</div>
|
||||
)}
|
||||
<div ref={logEndRef} />
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<form onSubmit={handleSendMessage} className="message-form">
|
||||
<input
|
||||
type="text"
|
||||
value={userInput}
|
||||
onChange={(e) => setUserInput(e.target.value)}
|
||||
placeholder="Type a message to the assistant..."
|
||||
disabled={!ws.isConnected}
|
||||
className="message-input"
|
||||
/>
|
||||
<button type="submit" disabled={!ws.isConnected || !userInput.trim()} className="send-button">
|
||||
Send
|
||||
</button>
|
||||
</form>
|
||||
|
||||
<div className="test-controls">
|
||||
<button onClick={handlePing} disabled={!ws.isConnected} className="ping-button">
|
||||
Send Ping (Test)
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
</main>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
export default App;
|
||||
@@ -0,0 +1,113 @@
|
||||
import { useState, useEffect, useRef } from 'react';
|
||||
import { createAudioRecorder, type AudioRecorder } from '../lib/audio-capture';
|
||||
|
||||
interface VoiceControlsProps {
|
||||
onAudioRecorded: (audio: Blob, format: string) => void;
|
||||
isProcessing: boolean;
|
||||
isPlaying?: boolean;
|
||||
}
|
||||
|
||||
export function VoiceControls({ onAudioRecorded, isProcessing, isPlaying }: VoiceControlsProps) {
|
||||
const [isRecording, setIsRecording] = useState(false);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
const [permissionState, setPermissionState] = useState<'prompt' | 'granted' | 'denied'>('prompt');
|
||||
const recorderRef = useRef<AudioRecorder | null>(null);
|
||||
|
||||
useEffect(() => {
|
||||
// Initialize recorder
|
||||
recorderRef.current = createAudioRecorder();
|
||||
|
||||
// Check microphone permission
|
||||
if (navigator.permissions) {
|
||||
navigator.permissions
|
||||
.query({ name: 'microphone' as PermissionName })
|
||||
.then((result) => {
|
||||
setPermissionState(result.state);
|
||||
result.onchange = () => {
|
||||
setPermissionState(result.state);
|
||||
};
|
||||
})
|
||||
.catch(() => {
|
||||
setPermissionState('prompt');
|
||||
});
|
||||
}
|
||||
}, []);
|
||||
|
||||
async function handleToggleRecording() {
|
||||
const recorder = recorderRef.current;
|
||||
if (!recorder) return;
|
||||
|
||||
try {
|
||||
setError(null);
|
||||
|
||||
if (isRecording) {
|
||||
// Stop recording
|
||||
console.log('[VoiceControls] Stopping recording...');
|
||||
const audioBlob = await recorder.stop();
|
||||
setIsRecording(false);
|
||||
|
||||
// Get format from blob type
|
||||
const format = audioBlob.type || 'audio/webm';
|
||||
console.log(`[VoiceControls] Recording complete: ${audioBlob.size} bytes, format: ${format}`);
|
||||
|
||||
// Send to parent
|
||||
onAudioRecorded(audioBlob, format);
|
||||
} else {
|
||||
// Start recording
|
||||
console.log('[VoiceControls] Starting recording...');
|
||||
await recorder.start();
|
||||
setIsRecording(true);
|
||||
setPermissionState('granted');
|
||||
}
|
||||
} catch (err: any) {
|
||||
console.error('[VoiceControls] Recording error:', err);
|
||||
setError(err.message || 'Failed to access microphone');
|
||||
setIsRecording(false);
|
||||
|
||||
if (err.message.includes('Permission denied')) {
|
||||
setPermissionState('denied');
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const canRecord = !isProcessing && !isPlaying;
|
||||
const buttonDisabled = !canRecord || permissionState === 'denied';
|
||||
|
||||
return (
|
||||
<div className="voice-controls">
|
||||
<button
|
||||
onClick={handleToggleRecording}
|
||||
disabled={buttonDisabled}
|
||||
className={`voice-button ${isRecording ? 'recording' : ''} ${isProcessing ? 'processing' : ''}`}
|
||||
title={isRecording ? 'Stop recording' : 'Start recording'}
|
||||
>
|
||||
{isRecording ? (
|
||||
<>
|
||||
<span className="recording-indicator" />
|
||||
<span>Recording...</span>
|
||||
</>
|
||||
) : isProcessing ? (
|
||||
<>
|
||||
<span className="processing-indicator" />
|
||||
<span>Processing...</span>
|
||||
</>
|
||||
) : (
|
||||
<>
|
||||
<span className="microphone-icon">🎤</span>
|
||||
<span>Record</span>
|
||||
</>
|
||||
)}
|
||||
</button>
|
||||
|
||||
{error && <div className="voice-error">{error}</div>}
|
||||
|
||||
{permissionState === 'denied' && (
|
||||
<div className="permission-denied">
|
||||
Microphone access denied. Please enable it in your browser settings.
|
||||
</div>
|
||||
)}
|
||||
|
||||
{isPlaying && <div className="playing-indicator">🔊 Playing response...</div>}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,136 @@
|
||||
import { useEffect, useRef, useState, useCallback } from 'react';
|
||||
|
||||
interface WebSocketMessage {
|
||||
type: string;
|
||||
payload: unknown;
|
||||
}
|
||||
|
||||
type MessageHandler = (payload: unknown) => void;
|
||||
|
||||
export interface UseWebSocketReturn {
|
||||
isConnected: boolean;
|
||||
send: (message: WebSocketMessage) => void;
|
||||
on: (type: string, handler: MessageHandler) => () => void;
|
||||
sendPing: () => void;
|
||||
sendUserMessage: (message: string) => void;
|
||||
}
|
||||
|
||||
export function useWebSocket(url: string): UseWebSocketReturn {
|
||||
const [isConnected, setIsConnected] = useState(false);
|
||||
const wsRef = useRef<WebSocket | null>(null);
|
||||
const handlersRef = useRef<Map<string, Set<MessageHandler>>>(new Map());
|
||||
const reconnectTimeoutRef = useRef<number>();
|
||||
|
||||
const connect = useCallback(() => {
|
||||
if (wsRef.current?.readyState === WebSocket.OPEN) {
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const ws = new WebSocket(url);
|
||||
|
||||
ws.onopen = () => {
|
||||
console.log('[WS] Connected to server');
|
||||
setIsConnected(true);
|
||||
};
|
||||
|
||||
ws.onclose = () => {
|
||||
console.log('[WS] Disconnected from server');
|
||||
setIsConnected(false);
|
||||
|
||||
// Attempt to reconnect after 3 seconds
|
||||
reconnectTimeoutRef.current = window.setTimeout(() => {
|
||||
console.log('[WS] Attempting to reconnect...');
|
||||
connect();
|
||||
}, 3000);
|
||||
};
|
||||
|
||||
ws.onerror = (error) => {
|
||||
console.error('[WS] Error:', error);
|
||||
};
|
||||
|
||||
ws.onmessage = (event) => {
|
||||
try {
|
||||
const message = JSON.parse(event.data) as WebSocketMessage;
|
||||
console.log(`[WS] Received message type: ${message.type}`, message.payload);
|
||||
|
||||
// Call all registered handlers for this message type
|
||||
const handlers = handlersRef.current.get(message.type);
|
||||
if (handlers) {
|
||||
handlers.forEach((handler) => {
|
||||
try {
|
||||
handler(message.payload);
|
||||
} catch (err) {
|
||||
console.error(`[WS] Error in handler for ${message.type}:`, err);
|
||||
}
|
||||
});
|
||||
}
|
||||
} catch (err) {
|
||||
console.error('[WS] Failed to parse message:', err);
|
||||
}
|
||||
};
|
||||
|
||||
wsRef.current = ws;
|
||||
} catch (err) {
|
||||
console.error('[WS] Failed to create WebSocket:', err);
|
||||
}
|
||||
}, [url]);
|
||||
|
||||
useEffect(() => {
|
||||
connect();
|
||||
|
||||
return () => {
|
||||
if (reconnectTimeoutRef.current) {
|
||||
clearTimeout(reconnectTimeoutRef.current);
|
||||
}
|
||||
if (wsRef.current) {
|
||||
wsRef.current.close();
|
||||
}
|
||||
};
|
||||
}, [connect]);
|
||||
|
||||
const send = useCallback((message: WebSocketMessage) => {
|
||||
if (wsRef.current?.readyState === WebSocket.OPEN) {
|
||||
wsRef.current.send(JSON.stringify(message));
|
||||
} else {
|
||||
console.warn('[WS] Cannot send message - not connected');
|
||||
}
|
||||
}, []);
|
||||
|
||||
const on = useCallback((type: string, handler: MessageHandler) => {
|
||||
if (!handlersRef.current.has(type)) {
|
||||
handlersRef.current.set(type, new Set());
|
||||
}
|
||||
handlersRef.current.get(type)!.add(handler);
|
||||
|
||||
// Return cleanup function
|
||||
return () => {
|
||||
const handlers = handlersRef.current.get(type);
|
||||
if (handlers) {
|
||||
handlers.delete(handler);
|
||||
if (handlers.size === 0) {
|
||||
handlersRef.current.delete(type);
|
||||
}
|
||||
}
|
||||
};
|
||||
}, []);
|
||||
|
||||
const sendPing = useCallback(() => {
|
||||
send({ type: 'ping', payload: {} });
|
||||
}, [send]);
|
||||
|
||||
const sendUserMessage = useCallback(
|
||||
(message: string) => {
|
||||
send({ type: 'user_message', payload: { message } });
|
||||
},
|
||||
[send]
|
||||
);
|
||||
|
||||
return {
|
||||
isConnected,
|
||||
send,
|
||||
on,
|
||||
sendPing,
|
||||
sendUserMessage,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
:root {
|
||||
font-family: Inter, system-ui, Avenir, Helvetica, Arial, sans-serif;
|
||||
line-height: 1.5;
|
||||
font-weight: 400;
|
||||
|
||||
color-scheme: dark;
|
||||
color: rgba(255, 255, 255, 0.87);
|
||||
background-color: #242424;
|
||||
|
||||
font-synthesis: none;
|
||||
text-rendering: optimizeLegibility;
|
||||
-webkit-font-smoothing: antialiased;
|
||||
-moz-osx-font-smoothing: grayscale;
|
||||
}
|
||||
|
||||
* {
|
||||
box-sizing: border-box;
|
||||
margin: 0;
|
||||
padding: 0;
|
||||
}
|
||||
|
||||
body {
|
||||
margin: 0;
|
||||
min-width: 320px;
|
||||
min-height: 100vh;
|
||||
}
|
||||
|
||||
#root {
|
||||
width: 100%;
|
||||
min-height: 100vh;
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8" />
|
||||
<link rel="icon" type="image/svg+xml" href="/vite.svg" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>Voice Assistant</title>
|
||||
</head>
|
||||
<body>
|
||||
<div id="root"></div>
|
||||
<script type="module" src="/main.tsx"></script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,184 @@
|
||||
export interface AudioCaptureConfig {
|
||||
mimeType?: string;
|
||||
audioBitsPerSecond?: number;
|
||||
}
|
||||
|
||||
export interface AudioRecorder {
|
||||
start(): Promise<void>;
|
||||
stop(): Promise<Blob>;
|
||||
isRecording(): boolean;
|
||||
getSupportedMimeType(): string | null;
|
||||
}
|
||||
|
||||
export function createAudioRecorder(config?: AudioCaptureConfig): AudioRecorder {
|
||||
let mediaRecorder: MediaRecorder | null = null;
|
||||
let audioChunks: Blob[] = [];
|
||||
let stream: MediaStream | null = null;
|
||||
let recording = false;
|
||||
|
||||
// Detect supported mime type
|
||||
function getSupportedMimeType(): string | null {
|
||||
const types = [
|
||||
'audio/webm;codecs=opus',
|
||||
'audio/webm',
|
||||
'audio/ogg;codecs=opus',
|
||||
'audio/ogg',
|
||||
'audio/mp4',
|
||||
'audio/wav',
|
||||
];
|
||||
|
||||
for (const type of types) {
|
||||
if (MediaRecorder.isTypeSupported(type)) {
|
||||
return type;
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
async function start(): Promise<void> {
|
||||
if (recording) {
|
||||
throw new Error('Already recording');
|
||||
}
|
||||
|
||||
try {
|
||||
// Request microphone permission with WebRTC audio processing constraints
|
||||
stream = await navigator.mediaDevices.getUserMedia({
|
||||
audio: {
|
||||
echoCancellation: true, // Remove echo from speakers
|
||||
noiseSuppression: true, // Remove background noise
|
||||
autoGainControl: true, // Normalize volume
|
||||
sampleRate: 16000, // Optimal for speech (Whisper works well with this)
|
||||
channelCount: 1, // Mono audio (sufficient for voice)
|
||||
},
|
||||
video: false,
|
||||
});
|
||||
|
||||
// Check which constraints were actually applied
|
||||
const track = stream.getAudioTracks()[0];
|
||||
const settings = track.getSettings();
|
||||
console.log('[AudioRecorder] Audio settings:', settings);
|
||||
|
||||
// Try WebM/Opus first (best compression and quality for speech)
|
||||
const mimeType = config?.mimeType ||
|
||||
(MediaRecorder.isTypeSupported('audio/webm;codecs=opus')
|
||||
? 'audio/webm;codecs=opus'
|
||||
: getSupportedMimeType());
|
||||
|
||||
if (!mimeType) {
|
||||
throw new Error('No supported audio mime type found');
|
||||
}
|
||||
|
||||
// Create MediaRecorder
|
||||
const options: MediaRecorderOptions = {
|
||||
mimeType,
|
||||
};
|
||||
|
||||
if (config?.audioBitsPerSecond) {
|
||||
options.audioBitsPerSecond = config.audioBitsPerSecond;
|
||||
}
|
||||
|
||||
mediaRecorder = new MediaRecorder(stream, options);
|
||||
audioChunks = [];
|
||||
|
||||
// Collect data chunks
|
||||
mediaRecorder.ondataavailable = (event) => {
|
||||
if (event.data.size > 0) {
|
||||
audioChunks.push(event.data);
|
||||
console.log(`[AudioRecorder] Chunk received: ${event.data.size} bytes`);
|
||||
}
|
||||
};
|
||||
|
||||
// Start recording
|
||||
mediaRecorder.start(100); // Collect chunks every 100ms
|
||||
recording = true;
|
||||
|
||||
console.log('[AudioRecorder] Recording started with constraints:', {
|
||||
echoCancellation: true,
|
||||
noiseSuppression: true,
|
||||
autoGainControl: true,
|
||||
sampleRate: 16000,
|
||||
channelCount: 1,
|
||||
mimeType,
|
||||
});
|
||||
} catch (error: any) {
|
||||
console.error('[AudioRecorder] Failed to start recording:', error);
|
||||
if (stream) {
|
||||
stream.getTracks().forEach((track) => track.stop());
|
||||
stream = null;
|
||||
}
|
||||
throw new Error(`Failed to start audio recording: ${error.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
async function stop(): Promise<Blob> {
|
||||
if (!recording || !mediaRecorder) {
|
||||
throw new Error('Not recording');
|
||||
}
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
if (!mediaRecorder) {
|
||||
reject(new Error('MediaRecorder not initialized'));
|
||||
return;
|
||||
}
|
||||
|
||||
mediaRecorder.onstop = () => {
|
||||
// Create blob from chunks
|
||||
const mimeType = mediaRecorder?.mimeType || 'audio/webm';
|
||||
const audioBlob = new Blob(audioChunks, { type: mimeType });
|
||||
|
||||
console.log('[AudioRecorder] Recording stopped:', {
|
||||
size: audioBlob.size,
|
||||
type: mimeType,
|
||||
chunks: audioChunks.length,
|
||||
});
|
||||
|
||||
// Clean up
|
||||
if (stream) {
|
||||
stream.getTracks().forEach((track) => {
|
||||
track.stop();
|
||||
console.log('[AudioRecorder] Stopped track:', track.label);
|
||||
});
|
||||
stream = null;
|
||||
}
|
||||
|
||||
recording = false;
|
||||
mediaRecorder = null;
|
||||
audioChunks = [];
|
||||
|
||||
resolve(audioBlob);
|
||||
};
|
||||
|
||||
mediaRecorder.onerror = (event: Event) => {
|
||||
console.error('[AudioRecorder] MediaRecorder error:', event);
|
||||
reject(new Error('MediaRecorder error during stop'));
|
||||
};
|
||||
|
||||
// Stop recording
|
||||
mediaRecorder.stop();
|
||||
});
|
||||
}
|
||||
|
||||
function isRecordingFunc(): boolean {
|
||||
return recording;
|
||||
}
|
||||
|
||||
return {
|
||||
start,
|
||||
stop,
|
||||
isRecording: isRecordingFunc,
|
||||
getSupportedMimeType,
|
||||
};
|
||||
}
|
||||
|
||||
export function checkMicrophonePermission(): Promise<PermissionState> {
|
||||
// Check if browser supports permissions API
|
||||
if (!navigator.permissions) {
|
||||
return Promise.resolve('prompt' as PermissionState);
|
||||
}
|
||||
|
||||
return navigator.permissions
|
||||
.query({ name: 'microphone' as PermissionName })
|
||||
.then((result) => result.state)
|
||||
.catch(() => 'prompt' as PermissionState);
|
||||
}
|
||||
@@ -0,0 +1,116 @@
|
||||
export interface AudioPlayer {
|
||||
play(audioData: Blob): Promise<void>;
|
||||
stop(): void;
|
||||
isPlaying(): boolean;
|
||||
clearQueue(): void;
|
||||
}
|
||||
|
||||
export function createAudioPlayer(): AudioPlayer {
|
||||
let currentAudio: HTMLAudioElement | null = null;
|
||||
let playing = false;
|
||||
let queue: Blob[] = [];
|
||||
let isProcessingQueue = false;
|
||||
|
||||
async function play(audioData: Blob): Promise<void> {
|
||||
// Add to queue
|
||||
queue.push(audioData);
|
||||
|
||||
// Start processing queue if not already processing
|
||||
if (!isProcessingQueue) {
|
||||
processQueue();
|
||||
}
|
||||
}
|
||||
|
||||
async function processQueue(): Promise<void> {
|
||||
if (isProcessingQueue || queue.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
isProcessingQueue = true;
|
||||
|
||||
while (queue.length > 0) {
|
||||
const audioData = queue.shift()!;
|
||||
await playAudio(audioData);
|
||||
}
|
||||
|
||||
isProcessingQueue = false;
|
||||
}
|
||||
|
||||
async function playAudio(audioData: Blob): Promise<void> {
|
||||
return new Promise((resolve, reject) => {
|
||||
try {
|
||||
// Create blob URL
|
||||
const audioUrl = URL.createObjectURL(audioData);
|
||||
|
||||
// Create audio element
|
||||
const audio = new Audio(audioUrl);
|
||||
currentAudio = audio;
|
||||
playing = true;
|
||||
|
||||
console.log(`[AudioPlayer] Playing audio (${audioData.size} bytes, type: ${audioData.type})`);
|
||||
|
||||
audio.onended = () => {
|
||||
console.log('[AudioPlayer] Playback finished');
|
||||
playing = false;
|
||||
currentAudio = null;
|
||||
|
||||
// Clean up blob URL
|
||||
URL.revokeObjectURL(audioUrl);
|
||||
|
||||
resolve();
|
||||
};
|
||||
|
||||
audio.onerror = (error) => {
|
||||
console.error('[AudioPlayer] Playback error:', error);
|
||||
playing = false;
|
||||
currentAudio = null;
|
||||
|
||||
// Clean up blob URL
|
||||
URL.revokeObjectURL(audioUrl);
|
||||
|
||||
reject(new Error('Audio playback failed'));
|
||||
};
|
||||
|
||||
// Start playback
|
||||
audio.play().catch((error) => {
|
||||
console.error('[AudioPlayer] Failed to start playback:', error);
|
||||
playing = false;
|
||||
currentAudio = null;
|
||||
URL.revokeObjectURL(audioUrl);
|
||||
reject(error);
|
||||
});
|
||||
} catch (error) {
|
||||
console.error('[AudioPlayer] Error creating audio element:', error);
|
||||
playing = false;
|
||||
currentAudio = null;
|
||||
reject(error);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
function stop(): void {
|
||||
if (currentAudio) {
|
||||
currentAudio.pause();
|
||||
currentAudio.currentTime = 0;
|
||||
currentAudio = null;
|
||||
}
|
||||
playing = false;
|
||||
queue = [];
|
||||
isProcessingQueue = false;
|
||||
}
|
||||
|
||||
function isPlayingFunc(): boolean {
|
||||
return playing;
|
||||
}
|
||||
|
||||
function clearQueue(): void {
|
||||
queue = [];
|
||||
}
|
||||
|
||||
return {
|
||||
play,
|
||||
stop,
|
||||
isPlaying: isPlayingFunc,
|
||||
clearQueue,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
import React from 'react'
|
||||
import ReactDOM from 'react-dom/client'
|
||||
import App from './App.tsx'
|
||||
import './index.css'
|
||||
|
||||
ReactDOM.createRoot(document.getElementById('root')!).render(
|
||||
<React.StrictMode>
|
||||
<App />
|
||||
</React.StrictMode>,
|
||||
)
|
||||
Reference in New Issue
Block a user