jb/paseo
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: restructure voice assistant server architecture

Browse Source 
- Introduce a new Session class to encapsulate conversation state and message processing, replacing the previous ClientSession structure.
- Implement STTManager and TTSManager for dedicated handling of speech-to-text and text-to-speech functionalities.
- Refactor WebSocket server to utilize the new session management, improving message routing and state handling.
- Remove deprecated orchestrator functions and streamline message schemas using Zod for validation.
- Enhance audio processing with improved buffering and interruption handling, ensuring seamless user interactions.

This refactor enhances maintainability and prepares the codebase for future feature expansions.
This commit is contained in:
Mohamed Boudra
2025-10-20 15:28:28 +02:00
parent 8365fb889b
commit af67235997
10 changed files with 1114 additions and 937 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/voice-assistant/src/ui/App.tsx
+42 -45
View File
@@ -105,31 +105,23 @@ function App() {
useEffect(() => {
// Listen for status messages
const unsubStatus = ws.on("status", (payload: unknown) => {
const data = payload as { status: string; message?: string };
addLog("info", data.message || `Status: ${data.status}`);
});
// Listen for pong messages
const unsubPong = ws.on("pong", () => {
addLog("success", "Received pong from server");
const unsubStatus = ws.on("status", (message) => {
if (message.type !== 'status') return;
const msg = 'message' in message.payload ? String(message.payload.message) : undefined;
addLog("info", msg || `Status: ${message.payload.status}`);
});
// Listen for activity log messages
const unsubActivity = ws.on("activity_log", (payload: unknown) => {
const data = payload as {
id: string;
type: string;
content: string;
metadata?: Record<string, unknown>;
};
const unsubActivity = ws.on("activity_log", (message) => {
if (message.type !== 'activity_log') return;
const data = message.payload;
// Handle tool calls
if (data.type === "tool_call") {
if (data.type === "tool_call" && data.metadata) {
const { toolCallId, toolName, arguments: args } = data.metadata as {
toolCallId: string;
toolName: string;
arguments: any;
arguments: unknown;
};
setLogs((prev) => [
@@ -146,10 +138,10 @@ function App() {
return;
}
if (data.type === "tool_result") {
if (data.type === "tool_result" && data.metadata) {
const { toolCallId, result } = data.metadata as {
toolCallId: string;
result: any;
result: unknown;
};
setLogs((prev) =>
@@ -163,10 +155,10 @@ function App() {
}
// Handle tool errors - update tool call status to failed
if (data.type === "error" && data.metadata?.toolCallId) {
if (data.type === "error" && data.metadata && 'toolCallId' in data.metadata) {
const { toolCallId, error } = data.metadata as {
toolCallId: string;
error: any;
error: unknown;
};
setLogs((prev) =>
@@ -194,24 +186,22 @@ function App() {
});
// Listen for streaming assistant chunks
const unsubChunk = ws.on("assistant_chunk", (payload: unknown) => {
const data = payload as { chunk: string };
setCurrentAssistantMessage((prev) => prev + data.chunk);
const unsubChunk = ws.on("assistant_chunk", (message) => {
if (message.type !== 'assistant_chunk') return;
setCurrentAssistantMessage((prev) => prev + message.payload.chunk);
});
// Listen for transcription results
const unsubTranscription = ws.on(
"transcription_result",
(_payload: unknown) => {
// Note: Transcription is already broadcast as activity_log with type "transcript"
// No need to log it again here to avoid duplication
setIsProcessingAudio(false);
}
);
const unsubTranscription = ws.on("transcription_result", () => {
// Note: Transcription is already broadcast as activity_log with type "transcript"
// No need to log it again here to avoid duplication
setIsProcessingAudio(false);
});
// Listen for artifacts
const unsubArtifact = ws.on("artifact", (payload: unknown) => {
const artifact = payload as Artifact;
const unsubArtifact = ws.on("artifact", (message) => {
if (message.type !== 'artifact') return;
const artifact = message.payload;
// Add to artifacts map
setArtifacts((prev) => {
@@ -238,8 +228,9 @@ function App() {
});
// Listen for audio output (TTS)
const unsubAudioOutput = ws.on("audio_output", async (payload: unknown) => {
const data = payload as { audio: string; format: string; id: string };
const unsubAudioOutput = ws.on("audio_output", async (message) => {
if (message.type !== 'audio_output') return;
const data = message.payload;
try {
setIsPlayingAudio(true);
@@ -261,8 +252,11 @@ function App() {
// Send confirmation back to server
ws.send({
type: "audio_played",
payload: { id: data.id },
type: "session",
message: {
type: "audio_played",
id: data.id,
},
});
setIsPlayingAudio(false);
@@ -275,7 +269,6 @@ function App() {
return () => {
unsubStatus();
unsubPong();
unsubActivity();
unsubChunk();
unsubTranscription();
@@ -337,8 +330,9 @@ function App() {
);
ws.send({
type: "audio_chunk",
payload: {
type: "session",
message: {
type: "audio_chunk",
audio: base64Audio,
format: format,
isLast: true,
@@ -438,8 +432,10 @@ function App() {
// Send abort request to server immediately
ws.send({
type: "abort_request",
payload: {},
type: "session",
message: {
type: "abort_request",
},
});
},
onSpeechEnd: async (audioData: Float32Array) => {
@@ -462,8 +458,9 @@ function App() {
);
ws.send({
type: "audio_chunk",
payload: {
type: "session",
message: {
type: "audio_chunk",
audio: base64Audio,
format: audioBlob.type,
isLast: true,
packages/voice-assistant/src/ui/hooks/useWebSocket.ts
+41 -26
View File
@@ -1,16 +1,14 @@
import { useEffect, useRef, useState, useCallback } from 'react';
interface WebSocketMessage {
type: string;
payload: unknown;
}
type MessageHandler = (payload: unknown) => void;
import type {
WSInboundMessage,
WSOutboundMessage,
SessionOutboundMessage
} from '../../server/messages.js';
export interface UseWebSocketReturn {
isConnected: boolean;
send: (message: WebSocketMessage) => void;
on: (type: string, handler: MessageHandler) => () => void;
send: (message: WSInboundMessage) => void;
on: (type: SessionOutboundMessage['type'], handler: (message: SessionOutboundMessage) => void) => () => void;
sendPing: () => void;
sendUserMessage: (message: string) => void;
}
@@ -18,7 +16,7 @@ export interface UseWebSocketReturn {
export function useWebSocket(url: string): UseWebSocketReturn {
const [isConnected, setIsConnected] = useState(false);
const wsRef = useRef<WebSocket | null>(null);
const handlersRef = useRef<Map<string, Set<MessageHandler>>>(new Map());
const handlersRef = useRef<Map<SessionOutboundMessage['type'], Set<(message: SessionOutboundMessage) => void>>>(new Map());
const reconnectTimeoutRef = useRef<number>();
const connect = useCallback(() => {
@@ -51,19 +49,27 @@ export function useWebSocket(url: string): UseWebSocketReturn {
ws.onmessage = (event) => {
try {
const message = JSON.parse(event.data) as WebSocketMessage;
console.log(`[WS] Received message type: ${message.type}`, message.payload);
const wsMessage: WSOutboundMessage = JSON.parse(event.data);
// Call all registered handlers for this message type
const handlers = handlersRef.current.get(message.type);
if (handlers) {
handlers.forEach((handler) => {
try {
handler(message.payload);
} catch (err) {
console.error(`[WS] Error in handler for ${message.type}:`, err);
}
});
// Only session messages trigger handlers
if (wsMessage.type === 'session') {
const sessionMessage = wsMessage.message;
console.log(`[WS] Received session message type: ${sessionMessage.type}`);
// Call all registered handlers for this message type
const handlers = handlersRef.current.get(sessionMessage.type);
if (handlers) {
handlers.forEach((handler) => {
try {
handler(sessionMessage);
} catch (err) {
console.error(`[WS] Error in handler for ${sessionMessage.type}:`, err);
}
});
}
} else {
// pong - just log
console.log(`[WS] Received ${wsMessage.type}`);
}
} catch (err) {
console.error('[WS] Failed to parse message:', err);
@@ -89,7 +95,7 @@ export function useWebSocket(url: string): UseWebSocketReturn {
};
}, [connect]);
const send = useCallback((message: WebSocketMessage) => {
const send = useCallback((message: WSInboundMessage) => {
if (wsRef.current?.readyState === WebSocket.OPEN) {
wsRef.current.send(JSON.stringify(message));
} else {
@@ -97,7 +103,10 @@ export function useWebSocket(url: string): UseWebSocketReturn {
}
}, []);
const on = useCallback((type: string, handler: MessageHandler) => {
const on = useCallback((
type: SessionOutboundMessage['type'],
handler: (message: SessionOutboundMessage) => void
) => {
if (!handlersRef.current.has(type)) {
handlersRef.current.set(type, new Set());
}
@@ -116,12 +125,18 @@ export function useWebSocket(url: string): UseWebSocketReturn {
}, []);
const sendPing = useCallback(() => {
send({ type: 'ping', payload: {} });
send({ type: 'ping' });
}, [send]);
const sendUserMessage = useCallback(
(message: string) => {
send({ type: 'user_message', payload: { message } });
send({
type: 'session',
message: {
type: 'user_text',
text: message,
},
});
},
[send]
);