jb/paseo
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: rename realtime to voice throughout codebase

Browse Source
This commit is contained in:
Mohamed Boudra
2026-02-03 23:12:17 +07:00
parent 4c5651fddc
commit d500b25632
16 changed files with 526 additions and 581 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/server/src/client/daemon-client-v2.ts
+2 -2
View File
@@ -1132,12 +1132,12 @@ export class DaemonClientV2 {
this.sendSessionMessage({ type: "set_voice_conversation", enabled, voiceConversationId });
}
async sendRealtimeAudioChunk(
async sendVoiceAudioChunk(
audio: string,
format: string,
isLast: boolean
): Promise<void> {
this.sendSessionMessage({ type: "realtime_audio_chunk", audio, format, isLast });
this.sendSessionMessage({ type: "voice_audio_chunk", audio, format, isLast });
}
startDictationStream(dictationId: string, format: string): Promise<void> {
packages/server/src/server/agent/tts-manager.ts
+2 -2
View File
@@ -32,7 +32,7 @@ export class TTSManager {
text: string,
emitMessage: (msg: SessionOutboundMessage) => void,
abortSignal: AbortSignal,
isRealtimeMode: boolean
isVoiceMode: boolean
): Promise<void> {
if (!this.tts) {
throw new Error("TTS not configured");
@@ -117,7 +117,7 @@ export class TTSManager {
isLastChunk: next.done,
audio: chunkBuffer.toString("base64"),
format,
isRealtimeMode,
isVoiceMode,
},
});
packages/server/src/server/agent/wait-for-agent-tracker.ts
+1 -1
View File
@@ -4,7 +4,7 @@ export type WaitForAgentCanceler = (agentId: string, reason?: string) => boolean
/**
* Tracks long-running wait_for_agent tool calls so they can be cancelled
* explicitly (e.g., when a realtime barge-in aborts the current turn).
* explicitly (e.g., when a voice barge-in aborts the current turn).
*/
export class WaitForAgentTracker {
private waiters = new Map<string, Set<(reason?: string) => void>>();
packages/server/src/server/daemon-client-v2.e2e.test.ts
+3 -3
View File
@@ -544,7 +544,7 @@ describe("daemon client v2 E2E", () => {
);
test(
"realtime mode buffers audio until isLast and emits transcription_result",
"voice mode buffers audio until isLast and emits transcription_result",
async () => {
requireEnv("OPENAI_API_KEY");
@@ -613,7 +613,7 @@ describe("daemon client v2 E2E", () => {
const chunkBytes = 3200; // 100ms @ 16kHz mono PCM16
const firstChunk = pcm16.subarray(0, Math.min(chunkBytes, pcm16.length));
await ctx.client.sendRealtimeAudioChunk(firstChunk.toString("base64"), format, false);
await ctx.client.sendVoiceAudioChunk(firstChunk.toString("base64"), format, false);
await earlyTranscription
.then(() => {
throw new Error("Expected no transcription_result before isLast=true");
@@ -623,7 +623,7 @@ describe("daemon client v2 E2E", () => {
for (let offset = chunkBytes; offset < pcm16.length; offset += chunkBytes) {
const chunk = pcm16.subarray(offset, Math.min(pcm16.length, offset + chunkBytes));
const isLast = offset + chunkBytes >= pcm16.length;
await ctx.client.sendRealtimeAudioChunk(chunk.toString("base64"), format, isLast);
await ctx.client.sendVoiceAudioChunk(chunk.toString("base64"), format, isLast);
}
const outcome = await Promise.race([
packages/server/src/server/session.ts
+52 -25
View File
@@ -260,8 +260,8 @@ export class Session {
private processingPhase: ProcessingPhase = "idle";
private currentStreamPromise: Promise<void> | null = null;
// Realtime mode state
private isRealtimeMode = false;
// Voice mode state
private isVoiceMode = false;
private speechInProgress = false;
private voiceConversationId: string | null = null;
@@ -369,6 +369,28 @@ export class Session {
this.sessionLogger.info("Session created");
}
private escapeXmlText(value: string): string {
return value
.replace(/&/g, "&amp;")
.replace(/</g, "&lt;")
.replace(/>/g, "&gt;");
}
private escapeXmlAttribute(value: string): string {
return this.escapeXmlText(value)
.replace(/"/g, "&quot;")
.replace(/'/g, "&apos;");
}
private formatVoiceTranscriptionForLLM(text: string): string {
const trimmed = text.trim();
const focusedAgentId = this.clientActivity?.focusedAgentId ?? null;
const focusedAttr = focusedAgentId
? ` focused-agent-id="${this.escapeXmlAttribute(focusedAgentId)}"`
: "";
return `<voice-transcription${focusedAttr}>${this.escapeXmlText(trimmed)}</voice-transcription>`;
}
/**
* Get the client's current activity state
*/
@@ -735,7 +757,7 @@ export class Session {
await this.handleUserText(msg.text);
break;
case "realtime_audio_chunk":
case "voice_audio_chunk":
await this.handleAudioChunk(msg);
break;
@@ -1214,7 +1236,7 @@ export class Session {
}
/**
* Handle realtime mode toggle
* Handle voice mode toggle
*/
private async handleSetVoiceConversation(
enabled: boolean,
@@ -1226,7 +1248,7 @@ export class Session {
return;
}
this.isRealtimeMode = true;
this.isVoiceMode = true;
this.voiceConversationId = voiceConversationId;
const loaded = await this.voiceConversationStore.load(
@@ -1242,7 +1264,7 @@ export class Session {
return;
}
this.isRealtimeMode = false;
this.isVoiceMode = false;
const idToPersist = this.voiceConversationId;
if (idToPersist) {
try {
@@ -3875,8 +3897,8 @@ export class Session {
// Add to conversation
this.messages.push({ role: "user", content: text });
// Process through LLM (TTS enabled in realtime mode for voice conversations)
this.currentStreamPromise = this.processWithLLM(this.isRealtimeMode);
// Process through LLM (TTS enabled in voice mode for voice conversations)
this.currentStreamPromise = this.processWithLLM(this.isVoiceMode);
await this.currentStreamPromise;
}
@@ -3884,9 +3906,9 @@ export class Session {
* Handle audio chunk for buffering and transcription
*/
private async handleAudioChunk(
msg: Extract<SessionInboundMessage, { type: "realtime_audio_chunk" }>
msg: Extract<SessionInboundMessage, { type: "voice_audio_chunk" }>
): Promise<void> {
await this.handleRealtimeSpeechStart();
await this.handleVoiceSpeechStart();
const chunkBuffer = Buffer.from(msg.audio, "base64");
const chunkFormat = msg.format || "audio/wav";
@@ -3932,19 +3954,19 @@ export class Session {
`Buffered audio chunk (${chunkBuffer.length} bytes, chunks: ${this.audioBuffer.chunks.length}${this.audioBuffer.isPCM ? `, PCM bytes: ${this.audioBuffer.totalPCMBytes}` : ""})`
);
// In realtime mode, only process audio when the user has finished speaking (isLast = true)
// In voice mode, only process audio when the user has finished speaking (isLast = true)
// This prevents partial transcriptions from being sent to the LLM
if (this.isRealtimeMode) {
if (this.isVoiceMode) {
if (!msg.isLast) {
this.sessionLogger.debug("Realtime mode: buffering audio, waiting for speech end");
this.sessionLogger.debug("Voice mode: buffering audio, waiting for speech end");
return;
}
this.sessionLogger.debug("Realtime mode: speech ended, processing complete audio");
this.sessionLogger.debug("Voice mode: speech ended, processing complete audio");
}
// In non-realtime mode, use streaming threshold to process chunks
// In non-voice mode, use streaming threshold to process chunks
const reachedStreamingThreshold =
!this.isRealtimeMode &&
!this.isVoiceMode &&
this.audioBuffer.isPCM &&
this.audioBuffer.totalPCMBytes >= MIN_STREAMING_SEGMENT_BYTES;
@@ -4073,7 +4095,7 @@ export class Session {
const requestId = uuidv4();
const result = await this.sttManager.transcribe(audio, format, {
requestId,
label: this.isRealtimeMode ? "realtime" : "buffered",
label: this.isVoiceMode ? "voice" : "buffered",
});
const transcriptText = result.text.trim();
@@ -4143,12 +4165,17 @@ export class Session {
});
// Add to conversation
this.messages.push({ role: "user", content: result.text });
this.messages.push({
role: "user",
content: this.isVoiceMode
? this.formatVoiceTranscriptionForLLM(result.text)
: result.text,
});
// Set phase to LLM and process (TTS enabled in realtime mode for voice conversations)
// Set phase to LLM and process (TTS enabled in voice mode for voice conversations)
this.clearSpeechInProgress("transcription complete");
this.setPhase("llm");
this.currentStreamPromise = this.processWithLLM(this.isRealtimeMode);
this.currentStreamPromise = this.processWithLLM(this.isVoiceMode);
await this.currentStreamPromise;
this.setPhase("idle");
} catch (error: any) {
@@ -4180,7 +4207,7 @@ export class Session {
if (textBuffer.length > 0) {
// TTS handling (capture mode at generation time for drift protection)
if (enableTTS && !this.speechInProgress) {
const modeAtGeneration = this.isRealtimeMode;
const modeAtGeneration = this.isVoiceMode;
pendingTTS = this.ttsManager.generateAndWaitForPlayback(
textBuffer,
(msg) => this.emit(msg),
@@ -4572,7 +4599,7 @@ export class Session {
/**
* Mark speech detection start and abort any active playback/LLM
*/
private async handleRealtimeSpeechStart(): Promise<void> {
private async handleVoiceSpeechStart(): Promise<void> {
if (this.speechInProgress) {
return;
}
@@ -4582,13 +4609,13 @@ export class Session {
const hadActiveStream = Boolean(this.currentStreamPromise);
this.speechInProgress = true;
this.sessionLogger.debug("Realtime speech chunk detected – aborting playback and LLM");
this.ttsManager.cancelPendingPlaybacks("realtime speech detected");
this.sessionLogger.debug("Voice speech detected – aborting playback and LLM");
this.ttsManager.cancelPendingPlaybacks("voice speech detected");
if (this.pendingAudioSegments.length > 0) {
this.sessionLogger.debug(
{ segmentCount: this.pendingAudioSegments.length },
`Dropping ${this.pendingAudioSegments.length} buffered audio segment(s) due to realtime speech`
`Dropping ${this.pendingAudioSegments.length} buffered audio segment(s) due to voice speech`
);
this.pendingAudioSegments = [];
}
packages/server/src/shared/messages.ts
+5 -5
View File
@@ -288,8 +288,8 @@ export const UserTextMessageSchema = z.object({
text: z.string(),
});
export const RealtimeAudioChunkMessageSchema = z.object({
type: z.literal("realtime_audio_chunk"),
export const VoiceAudioChunkMessageSchema = z.object({
type: z.literal("voice_audio_chunk"),
audio: z.string(), // base64 encoded
format: z.string(),
isLast: z.boolean(),
@@ -804,7 +804,7 @@ export const KillTerminalRequestSchema = z.object({
export const SessionInboundMessageSchema = z.discriminatedUnion("type", [
UserTextMessageSchema,
RealtimeAudioChunkMessageSchema,
VoiceAudioChunkMessageSchema,
AbortRequestMessageSchema,
AudioPlayedMessageSchema,
FetchAgentsRequestMessageSchema,
@@ -900,7 +900,7 @@ export const AudioOutputMessageSchema = z.object({
audio: z.string(), // base64 encoded
format: z.string(),
id: z.string(),
isRealtimeMode: z.boolean(), // Mode when audio was generated (for drift protection)
isVoiceMode: z.boolean(), // Mode when audio was generated (for drift protection)
groupId: z.string().optional(), // Logical utterance id
chunkIndex: z.number().int().nonnegative().optional(),
isLastChunk: z.boolean().optional(),
@@ -1643,7 +1643,7 @@ export type ActivityLogPayload = z.infer<typeof ActivityLogPayloadSchema>;
// Type exports for inbound message types
export type UserTextMessage = z.infer<typeof UserTextMessageSchema>;
export type RealtimeAudioChunkMessage = z.infer<typeof RealtimeAudioChunkMessageSchema>;
export type VoiceAudioChunkMessage = z.infer<typeof VoiceAudioChunkMessageSchema>;
export type FetchAgentsRequestMessage = z.infer<typeof FetchAgentsRequestMessageSchema>;
export type FetchAgentRequestMessage = z.infer<typeof FetchAgentRequestMessageSchema>;
export type SendAgentMessageRequest = z.infer<typeof SendAgentMessageRequestSchema>;