Initial monorepo setup with mcp-server and web packages
This commit is contained in:
Mohamed Boudra
@@ -0,0 +1,80 @@
|
||||
'use client';
|
||||
|
||||
import { useState, useEffect } from 'react';
|
||||
|
||||
interface AudioDevice {
|
||||
deviceId: string;
|
||||
label: string;
|
||||
}
|
||||
|
||||
function isSecureContext(): boolean {
|
||||
return typeof window !== 'undefined' && window.isSecureContext;
|
||||
}
|
||||
|
||||
function hasMediaDevices(): boolean {
|
||||
return typeof navigator !== 'undefined' && 'mediaDevices' in navigator && !!navigator.mediaDevices;
|
||||
}
|
||||
|
||||
export function useAudioDevices() {
|
||||
const [devices, setDevices] = useState<AudioDevice[]>([]);
|
||||
const [selectedDeviceId, setSelectedDeviceId] = useState<string>('');
|
||||
const [isLoading, setIsLoading] = useState(true);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
|
||||
useEffect(() => {
|
||||
async function loadDevices() {
|
||||
if (!isSecureContext()) {
|
||||
setError('HTTPS required. Please access this app over HTTPS or localhost.');
|
||||
setIsLoading(false);
|
||||
return;
|
||||
}
|
||||
|
||||
if (!hasMediaDevices()) {
|
||||
setError('Media devices not supported in this browser.');
|
||||
setIsLoading(false);
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
await navigator.mediaDevices.getUserMedia({ audio: true });
|
||||
|
||||
const deviceList = await navigator.mediaDevices.enumerateDevices();
|
||||
const audioInputs = deviceList
|
||||
.filter(device => device.kind === 'audioinput')
|
||||
.map(device => ({
|
||||
deviceId: device.deviceId,
|
||||
label: device.label || `Microphone ${device.deviceId.slice(0, 8)}`,
|
||||
}));
|
||||
|
||||
setDevices(audioInputs);
|
||||
|
||||
if (audioInputs.length > 0 && !selectedDeviceId) {
|
||||
setSelectedDeviceId(audioInputs[0].deviceId);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Error loading audio devices:', error);
|
||||
setError('Failed to access microphone. Please grant permission.');
|
||||
} finally {
|
||||
setIsLoading(false);
|
||||
}
|
||||
}
|
||||
|
||||
loadDevices();
|
||||
|
||||
if (hasMediaDevices()) {
|
||||
navigator.mediaDevices.addEventListener('devicechange', loadDevices);
|
||||
|
||||
return () => {
|
||||
navigator.mediaDevices.removeEventListener('devicechange', loadDevices);
|
||||
};
|
||||
}
|
||||
}, [selectedDeviceId]);
|
||||
|
||||
return {
|
||||
devices,
|
||||
selectedDeviceId,
|
||||
setSelectedDeviceId,
|
||||
isLoading,
|
||||
error,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
'use client';
|
||||
|
||||
import { useState, useEffect } from 'react';
|
||||
|
||||
export function useAudioLevel(stream: MediaStream | null): number {
|
||||
const [volume, setVolume] = useState(0);
|
||||
|
||||
useEffect(() => {
|
||||
if (!stream) {
|
||||
setVolume(0);
|
||||
return;
|
||||
}
|
||||
|
||||
const audioContext = new AudioContext();
|
||||
const analyser = audioContext.createAnalyser();
|
||||
analyser.fftSize = 2048;
|
||||
|
||||
const source = audioContext.createMediaStreamSource(stream);
|
||||
source.connect(analyser);
|
||||
|
||||
const dataArray = new Uint8Array(analyser.frequencyBinCount);
|
||||
let animationId: number;
|
||||
|
||||
function updateVolume() {
|
||||
analyser.getByteTimeDomainData(dataArray);
|
||||
|
||||
const rms = Math.sqrt(
|
||||
dataArray.reduce((sum, val) => sum + val * val, 0) / dataArray.length
|
||||
);
|
||||
|
||||
const normalized = (rms / 255) * 100;
|
||||
setVolume(Math.min(100, normalized * 1.5));
|
||||
|
||||
animationId = requestAnimationFrame(updateVolume);
|
||||
}
|
||||
|
||||
updateVolume();
|
||||
|
||||
return () => {
|
||||
cancelAnimationFrame(animationId);
|
||||
source.disconnect();
|
||||
audioContext.close();
|
||||
};
|
||||
}, [stream]);
|
||||
|
||||
return volume;
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
'use client';
|
||||
|
||||
import { useState, useCallback } from 'react';
|
||||
import type {
|
||||
RealtimeServerEvent,
|
||||
AgentStatus,
|
||||
LogEntry,
|
||||
EventCategory
|
||||
} from '../types/realtime-events';
|
||||
import {
|
||||
categorizeEvent,
|
||||
getEventSummary,
|
||||
getStatusFromEvent
|
||||
} from '../types/realtime-events';
|
||||
|
||||
const MAX_LOG_ENTRIES = 100;
|
||||
|
||||
interface UseEventLogReturn {
|
||||
logs: LogEntry[];
|
||||
agentStatus: AgentStatus;
|
||||
addLog: (event: RealtimeServerEvent) => void;
|
||||
setAgentStatus: (status: AgentStatus) => void;
|
||||
clearLogs: () => void;
|
||||
}
|
||||
|
||||
export function useEventLog(): UseEventLogReturn {
|
||||
const [logs, setLogs] = useState<LogEntry[]>([]);
|
||||
const [agentStatus, setAgentStatus] = useState<AgentStatus>('disconnected');
|
||||
|
||||
const addLog = useCallback((event: RealtimeServerEvent) => {
|
||||
const entry: LogEntry = {
|
||||
id: `${Date.now()}-${Math.random().toString(36).substring(7)}`,
|
||||
timestamp: Date.now(),
|
||||
category: categorizeEvent(event),
|
||||
eventType: event.type,
|
||||
event,
|
||||
summary: getEventSummary(event)
|
||||
};
|
||||
|
||||
setLogs(prevLogs => {
|
||||
const newLogs = [...prevLogs, entry];
|
||||
// Keep only the last MAX_LOG_ENTRIES
|
||||
if (newLogs.length > MAX_LOG_ENTRIES) {
|
||||
return newLogs.slice(-MAX_LOG_ENTRIES);
|
||||
}
|
||||
return newLogs;
|
||||
});
|
||||
|
||||
// Auto-update agent status based on event
|
||||
const newStatus = getStatusFromEvent(event);
|
||||
if (newStatus) {
|
||||
setAgentStatus(newStatus);
|
||||
}
|
||||
}, []);
|
||||
|
||||
const clearLogs = useCallback(() => {
|
||||
setLogs([]);
|
||||
}, []);
|
||||
|
||||
return {
|
||||
logs,
|
||||
agentStatus,
|
||||
addLog,
|
||||
setAgentStatus,
|
||||
clearLogs
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,202 @@
|
||||
'use client';
|
||||
|
||||
import { useState, useCallback, useRef } from 'react';
|
||||
import type { RealtimeServerEvent, AgentStatus } from '../types/realtime-events';
|
||||
|
||||
interface UseWebRTCVoiceReturn {
|
||||
isConnected: boolean;
|
||||
isConnecting: boolean;
|
||||
error: string | null;
|
||||
stream: MediaStream | null;
|
||||
agentStatus: AgentStatus;
|
||||
connect: (deviceId?: string) => Promise<void>;
|
||||
disconnect: () => void;
|
||||
}
|
||||
|
||||
interface UseWebRTCVoiceOptions {
|
||||
onEvent?: (event: RealtimeServerEvent) => void;
|
||||
onStatusChange?: (status: AgentStatus) => void;
|
||||
}
|
||||
|
||||
export function useWebRTCVoice(options: UseWebRTCVoiceOptions = {}): UseWebRTCVoiceReturn {
|
||||
const { onEvent, onStatusChange } = options;
|
||||
|
||||
const [isConnected, setIsConnected] = useState(false);
|
||||
const [isConnecting, setIsConnecting] = useState(false);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
const [stream, setStream] = useState<MediaStream | null>(null);
|
||||
const [agentStatus, setAgentStatus] = useState<AgentStatus>('disconnected');
|
||||
|
||||
const peerConnectionRef = useRef<RTCPeerConnection | null>(null);
|
||||
const dataChannelRef = useRef<RTCDataChannel | null>(null);
|
||||
const audioElementRef = useRef<HTMLAudioElement | null>(null);
|
||||
const streamRef = useRef<MediaStream | null>(null);
|
||||
|
||||
const connect = useCallback(async (deviceId?: string) => {
|
||||
if (isConnecting || isConnected) return;
|
||||
|
||||
setIsConnecting(true);
|
||||
setError(null);
|
||||
|
||||
try {
|
||||
if (!window.isSecureContext) {
|
||||
throw new Error('HTTPS required. Please access this app over HTTPS or localhost.');
|
||||
}
|
||||
|
||||
if (!navigator.mediaDevices || !navigator.mediaDevices.getUserMedia) {
|
||||
throw new Error('Media devices not supported in this browser.');
|
||||
}
|
||||
|
||||
const audioConstraints: MediaTrackConstraints = {
|
||||
echoCancellation: true,
|
||||
noiseSuppression: true,
|
||||
autoGainControl: true,
|
||||
};
|
||||
|
||||
if (deviceId) {
|
||||
audioConstraints.deviceId = { exact: deviceId };
|
||||
}
|
||||
|
||||
const micStream = await navigator.mediaDevices.getUserMedia({
|
||||
audio: audioConstraints
|
||||
});
|
||||
streamRef.current = micStream;
|
||||
setStream(micStream);
|
||||
|
||||
const basePath = process.env.NEXT_PUBLIC_BASE_PATH || '';
|
||||
const response = await fetch(`${basePath}/api/session`, { method: 'POST' });
|
||||
if (!response.ok) {
|
||||
throw new Error('Failed to get session token');
|
||||
}
|
||||
const { client_secret } = await response.json();
|
||||
|
||||
const pc = new RTCPeerConnection();
|
||||
peerConnectionRef.current = pc;
|
||||
|
||||
micStream.getTracks().forEach(track => pc.addTrack(track, micStream));
|
||||
|
||||
const dc = pc.createDataChannel('oai-events');
|
||||
dataChannelRef.current = dc;
|
||||
|
||||
// Listen for data channel messages (realtime events from server)
|
||||
dc.onmessage = (event) => {
|
||||
try {
|
||||
const data = JSON.parse(event.data) as RealtimeServerEvent;
|
||||
onEvent?.(data);
|
||||
} catch (err) {
|
||||
console.error('Failed to parse data channel message:', err);
|
||||
}
|
||||
};
|
||||
|
||||
dc.onopen = () => {
|
||||
console.log('Data channel opened');
|
||||
};
|
||||
|
||||
dc.onclose = () => {
|
||||
console.log('Data channel closed');
|
||||
};
|
||||
|
||||
pc.ontrack = (event) => {
|
||||
const audioElement = new Audio();
|
||||
audioElement.autoplay = true;
|
||||
audioElement.srcObject = event.streams[0];
|
||||
audioElementRef.current = audioElement;
|
||||
audioElement.play().catch(err => {
|
||||
console.error('Error playing audio:', err);
|
||||
});
|
||||
};
|
||||
|
||||
pc.onconnectionstatechange = () => {
|
||||
if (pc.connectionState === 'connected') {
|
||||
setIsConnected(true);
|
||||
setIsConnecting(false);
|
||||
setAgentStatus('connected');
|
||||
onStatusChange?.('connected');
|
||||
} else if (pc.connectionState === 'failed' || pc.connectionState === 'disconnected') {
|
||||
setError('Connection failed');
|
||||
setIsConnected(false);
|
||||
setIsConnecting(false);
|
||||
setAgentStatus('disconnected');
|
||||
onStatusChange?.('disconnected');
|
||||
}
|
||||
};
|
||||
|
||||
const offer = await pc.createOffer();
|
||||
await pc.setLocalDescription(offer);
|
||||
|
||||
const sdpResponse = await fetch(
|
||||
`https://api.openai.com/v1/realtime?model=gpt-4o-realtime-preview-2024-12-17`,
|
||||
{
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Authorization': `Bearer ${client_secret.value}`,
|
||||
'Content-Type': 'application/sdp',
|
||||
},
|
||||
body: offer.sdp,
|
||||
}
|
||||
);
|
||||
|
||||
if (!sdpResponse.ok) {
|
||||
throw new Error('Failed to connect to OpenAI');
|
||||
}
|
||||
|
||||
const answerSdp = await sdpResponse.text();
|
||||
const answer: RTCSessionDescriptionInit = {
|
||||
type: 'answer',
|
||||
sdp: answerSdp,
|
||||
};
|
||||
await pc.setRemoteDescription(answer);
|
||||
|
||||
} catch (err) {
|
||||
console.error('Connection error:', err);
|
||||
setError(err instanceof Error ? err.message : 'Failed to connect');
|
||||
setIsConnecting(false);
|
||||
setIsConnected(false);
|
||||
|
||||
if (streamRef.current) {
|
||||
streamRef.current.getTracks().forEach(track => track.stop());
|
||||
streamRef.current = null;
|
||||
setStream(null);
|
||||
}
|
||||
}
|
||||
}, [isConnecting, isConnected]);
|
||||
|
||||
const disconnect = useCallback(() => {
|
||||
if (dataChannelRef.current) {
|
||||
dataChannelRef.current.close();
|
||||
dataChannelRef.current = null;
|
||||
}
|
||||
|
||||
if (peerConnectionRef.current) {
|
||||
peerConnectionRef.current.close();
|
||||
peerConnectionRef.current = null;
|
||||
}
|
||||
|
||||
if (streamRef.current) {
|
||||
streamRef.current.getTracks().forEach(track => track.stop());
|
||||
streamRef.current = null;
|
||||
setStream(null);
|
||||
}
|
||||
|
||||
if (audioElementRef.current) {
|
||||
audioElementRef.current.pause();
|
||||
audioElementRef.current.srcObject = null;
|
||||
audioElementRef.current = null;
|
||||
}
|
||||
|
||||
setIsConnected(false);
|
||||
setIsConnecting(false);
|
||||
setError(null);
|
||||
setAgentStatus('disconnected');
|
||||
}, []);
|
||||
|
||||
return {
|
||||
isConnected,
|
||||
isConnecting,
|
||||
error,
|
||||
stream,
|
||||
agentStatus,
|
||||
connect,
|
||||
disconnect,
|
||||
};
|
||||
}
|
||||
Reference in New Issue
Block a user