refactor: restructure voice assistant server architecture

- Introduce a new Session class to encapsulate conversation state and message processing, replacing the previous ClientSession structure. - Implement STTManager and TTSManager for dedicated handling of speech-to-text and text-to-speech functionalities. - Refactor WebSocket server to utilize the new session management, improving message routing and state handling. - Remove deprecated orchestrator functions and streamline message schemas using Zod for validation. - Enhance audio processing with improved buffering and interruption handling, ensuring seamless user interactions. This refactor enhances maintainability and prepares the codebase for future feature expansions.
2025-10-20 15:28:28 +02:00
parent 8365fb889b
commit af67235997
10 changed files with 1114 additions and 937 deletions
@@ -0,0 +1,202 @@
+import { z } from "zod";
+
+// ============================================================================
+// Session Inbound Messages (Session receives these)
+// ============================================================================
+
+export const UserTextMessageSchema = z.object({
+  type: z.literal("user_text"),
+  text: z.string(),
+});
+
+export const AudioChunkMessageSchema = z.object({
+  type: z.literal("audio_chunk"),
+  audio: z.string(), // base64 encoded
+  format: z.string(),
+  isLast: z.boolean(),
+});
+
+export const AbortRequestMessageSchema = z.object({
+  type: z.literal("abort_request"),
+});
+
+export const AudioPlayedMessageSchema = z.object({
+  type: z.literal("audio_played"),
+  id: z.string(),
+});
+
+export const SessionInboundMessageSchema = z.discriminatedUnion("type", [
+  UserTextMessageSchema,
+  AudioChunkMessageSchema,
+  AbortRequestMessageSchema,
+  AudioPlayedMessageSchema,
+]);
+
+export type SessionInboundMessage = z.infer<typeof SessionInboundMessageSchema>;
+
+// ============================================================================
+// Session Outbound Messages (Session emits these)
+// ============================================================================
+
+export const ActivityLogPayloadSchema = z.object({
+  id: z.string(),
+  timestamp: z.date(),
+  type: z.enum([
+    "transcript",
+    "assistant",
+    "tool_call",
+    "tool_result",
+    "error",
+    "system",
+  ]),
+  content: z.string(),
+  metadata: z.record(z.unknown()).optional(),
+});
+
+export const ActivityLogMessageSchema = z.object({
+  type: z.literal("activity_log"),
+  payload: ActivityLogPayloadSchema,
+});
+
+export const AssistantChunkMessageSchema = z.object({
+  type: z.literal("assistant_chunk"),
+  payload: z.object({
+    chunk: z.string(),
+  }),
+});
+
+export const AudioOutputMessageSchema = z.object({
+  type: z.literal("audio_output"),
+  payload: z.object({
+    audio: z.string(), // base64 encoded
+    format: z.string(),
+    id: z.string(),
+  }),
+});
+
+export const TranscriptionResultMessageSchema = z.object({
+  type: z.literal("transcription_result"),
+  payload: z.object({
+    text: z.string(),
+    language: z.string().optional(),
+    duration: z.number().optional(),
+  }),
+});
+
+export const StatusMessageSchema = z.object({
+  type: z.literal("status"),
+  payload: z
+    .object({
+      status: z.string(),
+    })
+    .passthrough(), // Allow additional fields
+});
+
+export const ArtifactMessageSchema = z.object({
+  type: z.literal("artifact"),
+  payload: z.object({
+    type: z.enum(["markdown", "diff", "image", "code"]),
+    id: z.string(),
+    title: z.string(),
+    content: z.string(),
+    isBase64: z.boolean(),
+  }),
+});
+
+export const SessionOutboundMessageSchema = z.discriminatedUnion("type", [
+  ActivityLogMessageSchema,
+  AssistantChunkMessageSchema,
+  AudioOutputMessageSchema,
+  TranscriptionResultMessageSchema,
+  StatusMessageSchema,
+  ArtifactMessageSchema,
+]);
+
+export type SessionOutboundMessage = z.infer<
+  typeof SessionOutboundMessageSchema
+>;
+
+// Type exports for individual message types
+export type ActivityLogMessage = z.infer<typeof ActivityLogMessageSchema>;
+export type AssistantChunkMessage = z.infer<typeof AssistantChunkMessageSchema>;
+export type AudioOutputMessage = z.infer<typeof AudioOutputMessageSchema>;
+export type TranscriptionResultMessage = z.infer<typeof TranscriptionResultMessageSchema>;
+export type StatusMessage = z.infer<typeof StatusMessageSchema>;
+export type ArtifactMessage = z.infer<typeof ArtifactMessageSchema>;
+
+// Type exports for payload types
+export type ActivityLogPayload = z.infer<typeof ActivityLogPayloadSchema>;
+
+// ============================================================================
+// WebSocket Level Messages (wraps session messages)
+// ============================================================================
+
+// WebSocket-only messages (not session messages)
+export const WSPingMessageSchema = z.object({
+  type: z.literal("ping"),
+});
+
+export const WSPongMessageSchema = z.object({
+  type: z.literal("pong"),
+});
+
+export const WSRecordingStateMessageSchema = z.object({
+  type: z.literal("recording_state"),
+  isRecording: z.boolean(),
+});
+
+// Wrapped session message
+export const WSSessionInboundSchema = z.object({
+  type: z.literal("session"),
+  message: SessionInboundMessageSchema,
+});
+
+export const WSSessionOutboundSchema = z.object({
+  type: z.literal("session"),
+  message: SessionOutboundMessageSchema,
+});
+
+// Complete WebSocket message schemas
+export const WSInboundMessageSchema = z.discriminatedUnion("type", [
+  WSPingMessageSchema,
+  WSRecordingStateMessageSchema,
+  WSSessionInboundSchema,
+]);
+
+export const WSOutboundMessageSchema = z.discriminatedUnion("type", [
+  WSPongMessageSchema,
+  WSSessionOutboundSchema,
+]);
+
+export type WSInboundMessage = z.infer<typeof WSInboundMessageSchema>;
+export type WSOutboundMessage = z.infer<typeof WSOutboundMessageSchema>;
+
+// ============================================================================
+// Helper functions for message conversion
+// ============================================================================
+
+/**
+ * Extract session message from WebSocket message
+ * Returns null if message should be handled at WS level only
+ */
+export function extractSessionMessage(
+  wsMsg: WSInboundMessage
+): SessionInboundMessage | null {
+  if (wsMsg.type === "session") {
+    return wsMsg.message;
+  }
+  // Ping and recording_state are WS-level only
+  return null;
+}
+
+/**
+ * Wrap session message in WebSocket envelope
+ */
+export function wrapSessionMessage(
+  sessionMsg: SessionOutboundMessage
+): WSOutboundMessage {
+  return {
+    type: "session",
+    message: sessionMsg,
+  };
+}