Based on the git diff showing extensive package-lock.json changes and modifications to the agent system prompt and configuration, here's the commit message:
``` feat: add AI SDK dependencies and refine agent voice interaction config - Add @ai-sdk packages for enhanced AI capabilities - Install Deepgram SDK, livekit-client, and werift for voice/WebRTC - Expand agent system prompt with multi-tool workflows and Claude Code interaction patterns - Adjust VAD timing (1.2s silence threshold) and turn detection for better voice UX - Switch TTS to inworld/inworld-tts-1 with Olivia voice - Configure interruption timing (0.2s detection, no word requirement) - Add comprehensive guidelines for conversational responses vs structured output - Document plan mode triggers and Claude response type handling 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com> ```
This commit is contained in:
Mohamed Boudra
@@ -0,0 +1,184 @@
|
||||
export interface AudioCaptureConfig {
|
||||
mimeType?: string;
|
||||
audioBitsPerSecond?: number;
|
||||
}
|
||||
|
||||
export interface AudioRecorder {
|
||||
start(): Promise<void>;
|
||||
stop(): Promise<Blob>;
|
||||
isRecording(): boolean;
|
||||
getSupportedMimeType(): string | null;
|
||||
}
|
||||
|
||||
export function createAudioRecorder(config?: AudioCaptureConfig): AudioRecorder {
|
||||
let mediaRecorder: MediaRecorder | null = null;
|
||||
let audioChunks: Blob[] = [];
|
||||
let stream: MediaStream | null = null;
|
||||
let recording = false;
|
||||
|
||||
// Detect supported mime type
|
||||
function getSupportedMimeType(): string | null {
|
||||
const types = [
|
||||
'audio/webm;codecs=opus',
|
||||
'audio/webm',
|
||||
'audio/ogg;codecs=opus',
|
||||
'audio/ogg',
|
||||
'audio/mp4',
|
||||
'audio/wav',
|
||||
];
|
||||
|
||||
for (const type of types) {
|
||||
if (MediaRecorder.isTypeSupported(type)) {
|
||||
return type;
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
async function start(): Promise<void> {
|
||||
if (recording) {
|
||||
throw new Error('Already recording');
|
||||
}
|
||||
|
||||
try {
|
||||
// Request microphone permission with WebRTC audio processing constraints
|
||||
stream = await navigator.mediaDevices.getUserMedia({
|
||||
audio: {
|
||||
echoCancellation: true, // Remove echo from speakers
|
||||
noiseSuppression: true, // Remove background noise
|
||||
autoGainControl: true, // Normalize volume
|
||||
sampleRate: 16000, // Optimal for speech (Whisper works well with this)
|
||||
channelCount: 1, // Mono audio (sufficient for voice)
|
||||
},
|
||||
video: false,
|
||||
});
|
||||
|
||||
// Check which constraints were actually applied
|
||||
const track = stream.getAudioTracks()[0];
|
||||
const settings = track.getSettings();
|
||||
console.log('[AudioRecorder] Audio settings:', settings);
|
||||
|
||||
// Try WebM/Opus first (best compression and quality for speech)
|
||||
const mimeType = config?.mimeType ||
|
||||
(MediaRecorder.isTypeSupported('audio/webm;codecs=opus')
|
||||
? 'audio/webm;codecs=opus'
|
||||
: getSupportedMimeType());
|
||||
|
||||
if (!mimeType) {
|
||||
throw new Error('No supported audio mime type found');
|
||||
}
|
||||
|
||||
// Create MediaRecorder
|
||||
const options: MediaRecorderOptions = {
|
||||
mimeType,
|
||||
};
|
||||
|
||||
if (config?.audioBitsPerSecond) {
|
||||
options.audioBitsPerSecond = config.audioBitsPerSecond;
|
||||
}
|
||||
|
||||
mediaRecorder = new MediaRecorder(stream, options);
|
||||
audioChunks = [];
|
||||
|
||||
// Collect data chunks
|
||||
mediaRecorder.ondataavailable = (event) => {
|
||||
if (event.data.size > 0) {
|
||||
audioChunks.push(event.data);
|
||||
console.log(`[AudioRecorder] Chunk received: ${event.data.size} bytes`);
|
||||
}
|
||||
};
|
||||
|
||||
// Start recording
|
||||
mediaRecorder.start(100); // Collect chunks every 100ms
|
||||
recording = true;
|
||||
|
||||
console.log('[AudioRecorder] Recording started with constraints:', {
|
||||
echoCancellation: true,
|
||||
noiseSuppression: true,
|
||||
autoGainControl: true,
|
||||
sampleRate: 16000,
|
||||
channelCount: 1,
|
||||
mimeType,
|
||||
});
|
||||
} catch (error: any) {
|
||||
console.error('[AudioRecorder] Failed to start recording:', error);
|
||||
if (stream) {
|
||||
stream.getTracks().forEach((track) => track.stop());
|
||||
stream = null;
|
||||
}
|
||||
throw new Error(`Failed to start audio recording: ${error.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
async function stop(): Promise<Blob> {
|
||||
if (!recording || !mediaRecorder) {
|
||||
throw new Error('Not recording');
|
||||
}
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
if (!mediaRecorder) {
|
||||
reject(new Error('MediaRecorder not initialized'));
|
||||
return;
|
||||
}
|
||||
|
||||
mediaRecorder.onstop = () => {
|
||||
// Create blob from chunks
|
||||
const mimeType = mediaRecorder?.mimeType || 'audio/webm';
|
||||
const audioBlob = new Blob(audioChunks, { type: mimeType });
|
||||
|
||||
console.log('[AudioRecorder] Recording stopped:', {
|
||||
size: audioBlob.size,
|
||||
type: mimeType,
|
||||
chunks: audioChunks.length,
|
||||
});
|
||||
|
||||
// Clean up
|
||||
if (stream) {
|
||||
stream.getTracks().forEach((track) => {
|
||||
track.stop();
|
||||
console.log('[AudioRecorder] Stopped track:', track.label);
|
||||
});
|
||||
stream = null;
|
||||
}
|
||||
|
||||
recording = false;
|
||||
mediaRecorder = null;
|
||||
audioChunks = [];
|
||||
|
||||
resolve(audioBlob);
|
||||
};
|
||||
|
||||
mediaRecorder.onerror = (event: Event) => {
|
||||
console.error('[AudioRecorder] MediaRecorder error:', event);
|
||||
reject(new Error('MediaRecorder error during stop'));
|
||||
};
|
||||
|
||||
// Stop recording
|
||||
mediaRecorder.stop();
|
||||
});
|
||||
}
|
||||
|
||||
function isRecordingFunc(): boolean {
|
||||
return recording;
|
||||
}
|
||||
|
||||
return {
|
||||
start,
|
||||
stop,
|
||||
isRecording: isRecordingFunc,
|
||||
getSupportedMimeType,
|
||||
};
|
||||
}
|
||||
|
||||
export function checkMicrophonePermission(): Promise<PermissionState> {
|
||||
// Check if browser supports permissions API
|
||||
if (!navigator.permissions) {
|
||||
return Promise.resolve('prompt' as PermissionState);
|
||||
}
|
||||
|
||||
return navigator.permissions
|
||||
.query({ name: 'microphone' as PermissionName })
|
||||
.then((result) => result.state)
|
||||
.catch(() => 'prompt' as PermissionState);
|
||||
}
|
||||
@@ -0,0 +1,116 @@
|
||||
export interface AudioPlayer {
|
||||
play(audioData: Blob): Promise<void>;
|
||||
stop(): void;
|
||||
isPlaying(): boolean;
|
||||
clearQueue(): void;
|
||||
}
|
||||
|
||||
export function createAudioPlayer(): AudioPlayer {
|
||||
let currentAudio: HTMLAudioElement | null = null;
|
||||
let playing = false;
|
||||
let queue: Blob[] = [];
|
||||
let isProcessingQueue = false;
|
||||
|
||||
async function play(audioData: Blob): Promise<void> {
|
||||
// Add to queue
|
||||
queue.push(audioData);
|
||||
|
||||
// Start processing queue if not already processing
|
||||
if (!isProcessingQueue) {
|
||||
processQueue();
|
||||
}
|
||||
}
|
||||
|
||||
async function processQueue(): Promise<void> {
|
||||
if (isProcessingQueue || queue.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
isProcessingQueue = true;
|
||||
|
||||
while (queue.length > 0) {
|
||||
const audioData = queue.shift()!;
|
||||
await playAudio(audioData);
|
||||
}
|
||||
|
||||
isProcessingQueue = false;
|
||||
}
|
||||
|
||||
async function playAudio(audioData: Blob): Promise<void> {
|
||||
return new Promise((resolve, reject) => {
|
||||
try {
|
||||
// Create blob URL
|
||||
const audioUrl = URL.createObjectURL(audioData);
|
||||
|
||||
// Create audio element
|
||||
const audio = new Audio(audioUrl);
|
||||
currentAudio = audio;
|
||||
playing = true;
|
||||
|
||||
console.log(`[AudioPlayer] Playing audio (${audioData.size} bytes, type: ${audioData.type})`);
|
||||
|
||||
audio.onended = () => {
|
||||
console.log('[AudioPlayer] Playback finished');
|
||||
playing = false;
|
||||
currentAudio = null;
|
||||
|
||||
// Clean up blob URL
|
||||
URL.revokeObjectURL(audioUrl);
|
||||
|
||||
resolve();
|
||||
};
|
||||
|
||||
audio.onerror = (error) => {
|
||||
console.error('[AudioPlayer] Playback error:', error);
|
||||
playing = false;
|
||||
currentAudio = null;
|
||||
|
||||
// Clean up blob URL
|
||||
URL.revokeObjectURL(audioUrl);
|
||||
|
||||
reject(new Error('Audio playback failed'));
|
||||
};
|
||||
|
||||
// Start playback
|
||||
audio.play().catch((error) => {
|
||||
console.error('[AudioPlayer] Failed to start playback:', error);
|
||||
playing = false;
|
||||
currentAudio = null;
|
||||
URL.revokeObjectURL(audioUrl);
|
||||
reject(error);
|
||||
});
|
||||
} catch (error) {
|
||||
console.error('[AudioPlayer] Error creating audio element:', error);
|
||||
playing = false;
|
||||
currentAudio = null;
|
||||
reject(error);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
function stop(): void {
|
||||
if (currentAudio) {
|
||||
currentAudio.pause();
|
||||
currentAudio.currentTime = 0;
|
||||
currentAudio = null;
|
||||
}
|
||||
playing = false;
|
||||
queue = [];
|
||||
isProcessingQueue = false;
|
||||
}
|
||||
|
||||
function isPlayingFunc(): boolean {
|
||||
return playing;
|
||||
}
|
||||
|
||||
function clearQueue(): void {
|
||||
queue = [];
|
||||
}
|
||||
|
||||
return {
|
||||
play,
|
||||
stop,
|
||||
isPlaying: isPlayingFunc,
|
||||
clearQueue,
|
||||
};
|
||||
}
|
||||
Reference in New Issue
Block a user