423 lines
13 KiB
TypeScript
423 lines
13 KiB
TypeScript
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
|
import type { DaemonServerInfo } from "@/stores/session-store";
|
|
import type { AudioEngine } from "@/voice/audio-engine-types";
|
|
import {
|
|
createVoiceRuntime,
|
|
type VoiceRuntime,
|
|
type VoiceSessionAdapter,
|
|
} from "@/voice/voice-runtime";
|
|
import { REALTIME_VOICE_VAD_CONFIG } from "@/voice/realtime-voice-config";
|
|
|
|
function createAudioEngineMock(): AudioEngine {
|
|
return {
|
|
initialize: vi.fn().mockResolvedValue(undefined),
|
|
destroy: vi.fn().mockResolvedValue(undefined),
|
|
startCapture: vi.fn().mockResolvedValue(undefined),
|
|
stopCapture: vi.fn().mockResolvedValue(undefined),
|
|
toggleMute: vi.fn().mockReturnValue(true),
|
|
isMuted: vi.fn().mockReturnValue(false),
|
|
play: vi.fn().mockResolvedValue(0.1),
|
|
stop: vi.fn(),
|
|
clearQueue: vi.fn(),
|
|
isPlaying: vi.fn().mockReturnValue(false),
|
|
};
|
|
}
|
|
|
|
function createSessionAdapter(serverId = "server-1"): VoiceSessionAdapter {
|
|
return {
|
|
serverId,
|
|
setVoiceMode: vi.fn().mockResolvedValue(undefined),
|
|
sendVoiceAudioChunk: vi.fn().mockResolvedValue(undefined),
|
|
audioPlayed: vi.fn().mockResolvedValue(undefined),
|
|
abortRequest: vi.fn().mockResolvedValue(undefined),
|
|
setAssistantAudioPlaying: vi.fn(),
|
|
};
|
|
}
|
|
|
|
function createAudioPayload(args: {
|
|
id: string;
|
|
groupId: string;
|
|
chunkIndex: number;
|
|
isLastChunk: boolean;
|
|
isVoiceMode?: boolean;
|
|
}) {
|
|
return {
|
|
id: args.id,
|
|
groupId: args.groupId,
|
|
chunkIndex: args.chunkIndex,
|
|
isLastChunk: args.isLastChunk,
|
|
isVoiceMode: args.isVoiceMode ?? true,
|
|
format: "pcm",
|
|
audio: Buffer.from(`chunk-${args.chunkIndex}`).toString("base64"),
|
|
} as const;
|
|
}
|
|
|
|
function createServerInfo(): DaemonServerInfo {
|
|
return {
|
|
serverId: "server-1",
|
|
hostname: "host",
|
|
version: "1.0.0",
|
|
capabilities: {
|
|
voice: {
|
|
dictation: { enabled: true, reason: "" },
|
|
voice: { enabled: true, reason: "" },
|
|
},
|
|
},
|
|
};
|
|
}
|
|
|
|
function createRuntime(options?: {
|
|
engine?: AudioEngine;
|
|
getServerInfo?: (serverId: string) => DaemonServerInfo | null;
|
|
}) {
|
|
const engine = options?.engine ?? createAudioEngineMock();
|
|
const runtime = createVoiceRuntime({
|
|
engine,
|
|
getServerInfo: options?.getServerInfo ?? (() => createServerInfo()),
|
|
activateKeepAwake: vi.fn().mockResolvedValue(undefined),
|
|
deactivateKeepAwake: vi.fn().mockResolvedValue(undefined),
|
|
});
|
|
|
|
return { runtime, engine };
|
|
}
|
|
|
|
describe("voice runtime", () => {
|
|
beforeEach(() => {
|
|
vi.useFakeTimers();
|
|
});
|
|
|
|
afterEach(() => {
|
|
vi.useRealTimers();
|
|
});
|
|
|
|
it("starts voice when adapter is ready", async () => {
|
|
const adapter = createSessionAdapter();
|
|
const { runtime, engine } = createRuntime();
|
|
runtime.registerSession(adapter);
|
|
|
|
await runtime.startVoice("server-1", "agent-1");
|
|
|
|
expect(engine.initialize).toHaveBeenCalled();
|
|
expect(adapter.setVoiceMode).toHaveBeenCalledWith(true, "agent-1");
|
|
expect(engine.startCapture).toHaveBeenCalled();
|
|
expect(runtime.getSnapshot()).toMatchObject({
|
|
phase: "listening",
|
|
isVoiceMode: true,
|
|
activeServerId: "server-1",
|
|
activeAgentId: "agent-1",
|
|
});
|
|
});
|
|
|
|
it("streams continuous PCM chunks and waits after receiving a transcript", async () => {
|
|
const adapter = createSessionAdapter();
|
|
const { runtime } = createRuntime();
|
|
runtime.registerSession(adapter);
|
|
|
|
await runtime.startVoice("server-1", "agent-1");
|
|
runtime.handleCapturePcm(new Uint8Array(32000));
|
|
expect(runtime.getSnapshot().phase).toBe("listening");
|
|
|
|
expect(adapter.sendVoiceAudioChunk).toHaveBeenLastCalledWith(
|
|
expect.any(String),
|
|
"audio/pcm;rate=16000;bits=16",
|
|
);
|
|
|
|
runtime.onTranscriptionResult("server-1", "hello");
|
|
expect(runtime.getSnapshot().phase).toBe("waiting");
|
|
});
|
|
|
|
it("moves from waiting to playing on the first assistant audio", async () => {
|
|
const adapter = createSessionAdapter();
|
|
const { runtime, engine } = createRuntime();
|
|
runtime.registerSession(adapter);
|
|
|
|
await runtime.startVoice("server-1", "agent-1");
|
|
runtime.onAssistantAudioStarted("server-1");
|
|
|
|
expect(runtime.getSnapshot().phase).toBe("playing");
|
|
expect(adapter.setAssistantAudioPlaying).toHaveBeenCalledWith(true);
|
|
});
|
|
|
|
it("starts playback on the first chunk before isLastChunk arrives", async () => {
|
|
const adapter = createSessionAdapter();
|
|
let resolvePlay!: (duration: number) => void;
|
|
const engine = createAudioEngineMock();
|
|
vi.mocked(engine.play).mockImplementation(
|
|
() =>
|
|
new Promise<number>((resolve) => {
|
|
resolvePlay = resolve;
|
|
}),
|
|
);
|
|
const { runtime } = createRuntime({ engine });
|
|
runtime.registerSession(adapter);
|
|
|
|
await runtime.startVoice("server-1", "agent-1");
|
|
runtime.onTurnEvent("server-1", "agent-1", "turn_started");
|
|
|
|
runtime.handleAudioOutput(
|
|
"server-1",
|
|
createAudioPayload({
|
|
id: "chunk-0",
|
|
groupId: "group-1",
|
|
chunkIndex: 0,
|
|
isLastChunk: false,
|
|
}),
|
|
);
|
|
|
|
await vi.waitFor(() => {
|
|
expect(engine.play).toHaveBeenCalledTimes(1);
|
|
expect(runtime.getSnapshot().phase).toBe("playing");
|
|
expect(adapter.audioPlayed).not.toHaveBeenCalled();
|
|
});
|
|
|
|
runtime.handleAudioOutput(
|
|
"server-1",
|
|
createAudioPayload({
|
|
id: "chunk-1",
|
|
groupId: "group-1",
|
|
chunkIndex: 1,
|
|
isLastChunk: true,
|
|
}),
|
|
);
|
|
|
|
expect(engine.play).toHaveBeenCalledTimes(1);
|
|
resolvePlay(0.1);
|
|
|
|
await vi.waitFor(() => {
|
|
expect(adapter.audioPlayed).toHaveBeenCalledWith("chunk-0");
|
|
expect(engine.play).toHaveBeenCalledTimes(2);
|
|
});
|
|
});
|
|
|
|
it("acknowledges chunks only after they are consumed and finishes after the final chunk", async () => {
|
|
const adapter = createSessionAdapter();
|
|
const playResolvers: Array<(duration: number) => void> = [];
|
|
const engine = createAudioEngineMock();
|
|
vi.mocked(engine.play).mockImplementation(
|
|
() =>
|
|
new Promise<number>((resolve) => {
|
|
playResolvers.push(resolve);
|
|
}),
|
|
);
|
|
const { runtime } = createRuntime({ engine });
|
|
runtime.registerSession(adapter);
|
|
|
|
await runtime.startVoice("server-1", "agent-1");
|
|
runtime.onTurnEvent("server-1", "agent-1", "turn_started");
|
|
|
|
runtime.handleAudioOutput(
|
|
"server-1",
|
|
createAudioPayload({
|
|
id: "chunk-0",
|
|
groupId: "group-1",
|
|
chunkIndex: 0,
|
|
isLastChunk: false,
|
|
}),
|
|
);
|
|
runtime.handleAudioOutput(
|
|
"server-1",
|
|
createAudioPayload({
|
|
id: "chunk-1",
|
|
groupId: "group-1",
|
|
chunkIndex: 1,
|
|
isLastChunk: true,
|
|
}),
|
|
);
|
|
|
|
await vi.waitFor(() => {
|
|
expect(engine.play).toHaveBeenCalledTimes(1);
|
|
});
|
|
expect(adapter.audioPlayed).not.toHaveBeenCalled();
|
|
|
|
playResolvers.shift()!(0.1);
|
|
await vi.waitFor(() => {
|
|
expect(adapter.audioPlayed).toHaveBeenCalledWith("chunk-0");
|
|
expect(engine.play).toHaveBeenCalledTimes(2);
|
|
});
|
|
|
|
playResolvers.shift()!(0.1);
|
|
await vi.waitFor(() => {
|
|
expect(adapter.audioPlayed).toHaveBeenCalledWith("chunk-1");
|
|
expect(runtime.getSnapshot().phase).toBe("waiting");
|
|
});
|
|
});
|
|
|
|
it("returns to waiting after assistant playback when the turn is still active", async () => {
|
|
const adapter = createSessionAdapter();
|
|
const { runtime, engine } = createRuntime();
|
|
runtime.registerSession(adapter);
|
|
|
|
await runtime.startVoice("server-1", "agent-1");
|
|
runtime.onTurnEvent("server-1", "agent-1", "turn_started");
|
|
runtime.onAssistantAudioStarted("server-1");
|
|
runtime.onAssistantAudioFinished("server-1");
|
|
|
|
expect(runtime.getSnapshot().phase).toBe("waiting");
|
|
expect(engine.play).toHaveBeenCalled();
|
|
});
|
|
|
|
it("starts the thinking tone when an agent turn begins before playback", async () => {
|
|
const adapter = createSessionAdapter();
|
|
const { runtime, engine } = createRuntime();
|
|
runtime.registerSession(adapter);
|
|
|
|
await runtime.startVoice("server-1", "agent-1");
|
|
runtime.onTurnEvent("server-1", "agent-1", "turn_started");
|
|
|
|
expect(runtime.getSnapshot().phase).toBe("waiting");
|
|
expect(engine.play).toHaveBeenCalled();
|
|
});
|
|
|
|
it("does not restart the thinking tone from local detection jitter while waiting", async () => {
|
|
const adapter = createSessionAdapter();
|
|
let resolvePlay!: (duration: number) => void;
|
|
const engine = createAudioEngineMock();
|
|
vi.mocked(engine.play).mockImplementation(
|
|
() =>
|
|
new Promise<number>((resolve) => {
|
|
resolvePlay = resolve;
|
|
}),
|
|
);
|
|
const { runtime } = createRuntime({ engine });
|
|
runtime.registerSession(adapter);
|
|
|
|
await runtime.startVoice("server-1", "agent-1");
|
|
runtime.onTurnEvent("server-1", "agent-1", "turn_started");
|
|
|
|
expect(runtime.getSnapshot().phase).toBe("waiting");
|
|
expect(engine.play).toHaveBeenCalledTimes(1);
|
|
|
|
runtime.handleCaptureVolume(REALTIME_VOICE_VAD_CONFIG.volumeThreshold + 0.05);
|
|
runtime.handleCaptureVolume(0);
|
|
|
|
expect(engine.stop).not.toHaveBeenCalled();
|
|
expect(engine.clearQueue).not.toHaveBeenCalled();
|
|
expect(engine.play).toHaveBeenCalledTimes(1);
|
|
|
|
runtime.onServerSpeechStateChanged("server-1", true);
|
|
|
|
expect(engine.stop).toHaveBeenCalledTimes(1);
|
|
expect(engine.clearQueue).toHaveBeenCalledTimes(1);
|
|
|
|
resolvePlay(0.1);
|
|
});
|
|
|
|
it("returns to listening after assistant playback once the turn is complete", async () => {
|
|
const adapter = createSessionAdapter();
|
|
const { runtime, engine } = createRuntime();
|
|
runtime.registerSession(adapter);
|
|
|
|
await runtime.startVoice("server-1", "agent-1");
|
|
runtime.onTurnEvent("server-1", "agent-1", "turn_started");
|
|
runtime.onAssistantAudioStarted("server-1");
|
|
runtime.onTurnEvent("server-1", "agent-1", "turn_completed");
|
|
runtime.onAssistantAudioFinished("server-1");
|
|
|
|
expect(runtime.getSnapshot().phase).toBe("listening");
|
|
expect(engine.play).toHaveBeenCalled();
|
|
});
|
|
|
|
it("keeps local volume alone non-authoritative for playback interruption", async () => {
|
|
const adapter = createSessionAdapter();
|
|
const { runtime, engine } = createRuntime();
|
|
runtime.registerSession(adapter);
|
|
|
|
await runtime.startVoice("server-1", "agent-1");
|
|
runtime.onTurnEvent("server-1", "agent-1", "turn_started");
|
|
runtime.onAssistantAudioStarted("server-1");
|
|
|
|
runtime.handleCaptureVolume(0.5);
|
|
expect(runtime.getTelemetrySnapshot().isSpeaking).toBe(false);
|
|
expect(adapter.abortRequest).not.toHaveBeenCalled();
|
|
expect(engine.stop).not.toHaveBeenCalled();
|
|
});
|
|
|
|
it("keeps the meter white state driven by server speech detection", async () => {
|
|
const adapter = createSessionAdapter();
|
|
const { runtime } = createRuntime();
|
|
runtime.registerSession(adapter);
|
|
|
|
await runtime.startVoice("server-1", "agent-1");
|
|
runtime.handleCaptureVolume(0.7);
|
|
|
|
expect(runtime.getTelemetrySnapshot()).toMatchObject({
|
|
volume: expect.any(Number),
|
|
isSpeaking: false,
|
|
});
|
|
|
|
runtime.onServerSpeechStateChanged("server-1", true);
|
|
expect(runtime.getTelemetrySnapshot().isSpeaking).toBe(true);
|
|
|
|
runtime.onServerSpeechStateChanged("server-1", false);
|
|
expect(runtime.getTelemetrySnapshot().isSpeaking).toBe(false);
|
|
});
|
|
|
|
it("stops assistant playback immediately when server speech starts", async () => {
|
|
const adapter = createSessionAdapter();
|
|
const { runtime, engine } = createRuntime();
|
|
runtime.registerSession(adapter);
|
|
|
|
await runtime.startVoice("server-1", "agent-1");
|
|
runtime.onAssistantAudioStarted("server-1");
|
|
|
|
runtime.onServerSpeechStateChanged("server-1", true);
|
|
|
|
expect(engine.stop).toHaveBeenCalled();
|
|
expect(engine.clearQueue).toHaveBeenCalled();
|
|
expect(adapter.setAssistantAudioPlaying).toHaveBeenCalledWith(false);
|
|
expect(runtime.getTelemetrySnapshot().isSpeaking).toBe(true);
|
|
});
|
|
|
|
it("authoritatively stops and suppresses later voice audio", async () => {
|
|
const adapter = createSessionAdapter();
|
|
const { runtime, engine } = createRuntime();
|
|
runtime.registerSession(adapter);
|
|
|
|
await runtime.startVoice("server-1", "agent-1");
|
|
await runtime.stopVoice();
|
|
|
|
expect(adapter.setVoiceMode).toHaveBeenLastCalledWith(false);
|
|
expect(engine.stopCapture).toHaveBeenCalled();
|
|
expect(runtime.getSnapshot().phase).toBe("disabled");
|
|
expect(runtime.shouldPlayVoiceAudio("server-1")).toBe(false);
|
|
});
|
|
|
|
it("returns an explicit not-ready error when the adapter is missing", async () => {
|
|
const { runtime } = createRuntime();
|
|
await expect(runtime.startVoice("server-1", "agent-1")).rejects.toThrow(
|
|
"Voice runtime is not ready for host server-1",
|
|
);
|
|
});
|
|
|
|
it("resyncs voice mode after connection recovers", async () => {
|
|
const adapter = createSessionAdapter();
|
|
const { runtime } = createRuntime();
|
|
runtime.registerSession(adapter);
|
|
|
|
await runtime.startVoice("server-1", "agent-1");
|
|
vi.mocked(adapter.setVoiceMode).mockClear();
|
|
|
|
runtime.updateSessionConnection("server-1", false);
|
|
runtime.updateSessionConnection("server-1", true);
|
|
await Promise.resolve();
|
|
|
|
expect(adapter.setVoiceMode).toHaveBeenCalledWith(true, "agent-1");
|
|
});
|
|
|
|
it("does not emit when the snapshot is unchanged", async () => {
|
|
const adapter = createSessionAdapter();
|
|
const { runtime } = createRuntime();
|
|
runtime.registerSession(adapter);
|
|
await runtime.startVoice("server-1", "agent-1");
|
|
|
|
const listener = vi.fn();
|
|
const unsubscribe = runtime.subscribe(listener);
|
|
|
|
runtime.handleCaptureVolume(0);
|
|
runtime.handleCaptureVolume(0);
|
|
|
|
expect(listener).not.toHaveBeenCalled();
|
|
unsubscribe();
|
|
});
|
|
});
|