refactor(server): extract voice mode subsystem into VoiceSession (#1640)
* refactor(server): extract voice mode subsystem into VoiceSession session.ts was a 10.5k-line god object and the most-churned file in the repo. ~1,100 of those lines were an entire voice/audio subsystem — the STT/TTS/dictation managers, the barge-in audio-buffering state machine, voice-turn orchestration, and the MCP voice bridge — interleaved field by field and method by method with workspace/git/agent/provider concerns. Move the whole subsystem into a new VoiceSession deep module. Session now holds one `voiceSession` field, constructs it once, and delegates the voice/dictation/abort message types, the permission auto-allow gate, and cleanup to it. VoiceSession owns all 18 voice fields and ~25 methods and reaches back into the agent run only through a narrow VoiceSessionHost seam (emit, loadAgent, reloadAgentSession, sendSpokenInput, interruptAgentIfRunning, hasActiveAgentRun). Behavior is preserved (verbatim method moves); deleting voice-session.ts now removes the feature mechanically. The voice unit tests drive the VoiceSession boundary instead of reaching into Session internals, so future voice tests can construct a VoiceSession with a fake host. session.ts: 10,468 -> 9,272 lines. * refactor(server): move VoiceSession into voice/ and test it at the boundary Address Greptile review on #1640: - Move voice-session.ts into the existing voice/ subdirectory, alongside voice-turn-controller.ts, instead of adding another peer to the 30+ file server/ directory. The directory now carries the domain. - Add voice/voice-session.test.ts beside the module: it constructs a VoiceSession with a fake VoiceSessionHost and drives it through the public API (handleSetVoiceMode, then turn-detection/STT events), asserting on the host seam (sendSpokenInput) and emitted messages. No Session, no private field/method access. - Remove the voice tests from session.test.ts that reached through Session's private voiceSession field into VoiceSession internals. Same three behaviors are covered (streaming final -> agent, finalization timeout empty path, low-confidence filtering); driving handleSetVoiceMode additionally exercises the enable path the old tests bypassed.
This commit is contained in:
@@ -1,5 +1,4 @@
|
|||||||
import { execSync } from "child_process";
|
import { execSync } from "child_process";
|
||||||
import { EventEmitter } from "events";
|
|
||||||
import { mkdtempSync, realpathSync, rmSync, symlinkSync, writeFileSync } from "fs";
|
import { mkdtempSync, realpathSync, rmSync, symlinkSync, writeFileSync } from "fs";
|
||||||
import { homedir, tmpdir } from "os";
|
import { homedir, tmpdir } from "os";
|
||||||
import { join, resolve as resolvePath } from "path";
|
import { join, resolve as resolvePath } from "path";
|
||||||
@@ -17,16 +16,6 @@ import { StructuredAgentFallbackError } from "./agent/agent-response-loop.js";
|
|||||||
import type { StoredAgentRecord } from "./agent/agent-storage.js";
|
import type { StoredAgentRecord } from "./agent/agent-storage.js";
|
||||||
import type { ProviderSnapshotManager } from "./agent/provider-snapshot-manager.js";
|
import type { ProviderSnapshotManager } from "./agent/provider-snapshot-manager.js";
|
||||||
import type { SessionOptions } from "./session.js";
|
import type { SessionOptions } from "./session.js";
|
||||||
import type {
|
|
||||||
SpeechToTextProvider,
|
|
||||||
StreamingTranscriptionCommittedEvent,
|
|
||||||
StreamingTranscriptionEvent,
|
|
||||||
StreamingTranscriptionSession,
|
|
||||||
} from "./speech/speech-provider.js";
|
|
||||||
import type {
|
|
||||||
TurnDetectionProvider,
|
|
||||||
TurnDetectionSession,
|
|
||||||
} from "./speech/turn-detection-provider.js";
|
|
||||||
import {
|
import {
|
||||||
asSessionInternals as asSessionInternalsHelper,
|
asSessionInternals as asSessionInternalsHelper,
|
||||||
asAgentManager,
|
asAgentManager,
|
||||||
@@ -50,8 +39,6 @@ import type {
|
|||||||
} from "../services/github-service.js";
|
} from "../services/github-service.js";
|
||||||
|
|
||||||
interface SessionHandlerInternals {
|
interface SessionHandlerInternals {
|
||||||
startVoiceTurnController(): Promise<void>;
|
|
||||||
stopVoiceTurnController(): Promise<void>;
|
|
||||||
handleSendAgentMessage(
|
handleSendAgentMessage(
|
||||||
agentId: string,
|
agentId: string,
|
||||||
text: string,
|
text: string,
|
||||||
@@ -82,9 +69,6 @@ interface SessionHandlerInternals {
|
|||||||
handleStashPopRequest(params: unknown): Promise<unknown>;
|
handleStashPopRequest(params: unknown): Promise<unknown>;
|
||||||
createPaseoWorktree(params: unknown): Promise<unknown>;
|
createPaseoWorktree(params: unknown): Promise<unknown>;
|
||||||
handleStartWorkspaceScriptRequest(params: unknown): Promise<unknown>;
|
handleStartWorkspaceScriptRequest(params: unknown): Promise<unknown>;
|
||||||
sttManager: {
|
|
||||||
transcribe(audio: Buffer, format: string): Promise<unknown>;
|
|
||||||
};
|
|
||||||
}
|
}
|
||||||
|
|
||||||
function asSessionInternals(session: Session): SessionHandlerInternals {
|
function asSessionInternals(session: Session): SessionHandlerInternals {
|
||||||
@@ -313,198 +297,6 @@ function createSessionForTest(options: SessionForTestOptions = {}): Session {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
class FakeVoiceTurnDetectionSession extends EventEmitter implements TurnDetectionSession {
|
|
||||||
public readonly requiredSampleRate = 16000;
|
|
||||||
|
|
||||||
async connect(): Promise<void> {}
|
|
||||||
|
|
||||||
appendPcm16(_chunk: Buffer): void {}
|
|
||||||
|
|
||||||
flush(): void {}
|
|
||||||
reset(): void {}
|
|
||||||
close(): void {}
|
|
||||||
}
|
|
||||||
|
|
||||||
class FakeVoiceSttSession extends EventEmitter implements StreamingTranscriptionSession {
|
|
||||||
public readonly requiredSampleRate = 16000;
|
|
||||||
public commitCount = 0;
|
|
||||||
|
|
||||||
async connect(): Promise<void> {}
|
|
||||||
|
|
||||||
appendPcm16(_pcm16le: Buffer): void {}
|
|
||||||
|
|
||||||
commit(): void {
|
|
||||||
this.commitCount += 1;
|
|
||||||
}
|
|
||||||
|
|
||||||
clear(): void {}
|
|
||||||
close(): void {}
|
|
||||||
|
|
||||||
emitCommitted(event: StreamingTranscriptionCommittedEvent): void {
|
|
||||||
this.emit("committed", event);
|
|
||||||
}
|
|
||||||
|
|
||||||
emitTranscript(event: StreamingTranscriptionEvent): void {
|
|
||||||
this.emit("transcript", event);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
function createVoiceSessionHarness() {
|
|
||||||
const messages: unknown[] = [];
|
|
||||||
const detector = new FakeVoiceTurnDetectionSession();
|
|
||||||
const sttSession = new FakeVoiceSttSession();
|
|
||||||
const sttProvider: SpeechToTextProvider = {
|
|
||||||
id: "local",
|
|
||||||
createSession: vi.fn(() => sttSession),
|
|
||||||
};
|
|
||||||
const turnDetection: TurnDetectionProvider = {
|
|
||||||
id: "local",
|
|
||||||
createSession: vi.fn(() => detector),
|
|
||||||
};
|
|
||||||
const session = createSessionForTest({
|
|
||||||
messages,
|
|
||||||
stt: sttProvider,
|
|
||||||
voice: { turnDetection },
|
|
||||||
});
|
|
||||||
Object.assign(session, {
|
|
||||||
isVoiceMode: true,
|
|
||||||
voiceModeAgentId: "11111111-1111-4111-8111-111111111111",
|
|
||||||
});
|
|
||||||
const internals = asSessionInternals(session);
|
|
||||||
const sendAgentMessage = vi
|
|
||||||
.spyOn(internals, "handleSendAgentMessage")
|
|
||||||
.mockResolvedValue({ ok: true });
|
|
||||||
const transcribe = vi.spyOn(asSessionInternals(session).sttManager, "transcribe");
|
|
||||||
|
|
||||||
return {
|
|
||||||
session,
|
|
||||||
internals,
|
|
||||||
messages,
|
|
||||||
detector,
|
|
||||||
sttSession,
|
|
||||||
sendAgentMessage,
|
|
||||||
transcribe,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
async function settleVoiceSession(): Promise<void> {
|
|
||||||
await Promise.resolve();
|
|
||||||
await Promise.resolve();
|
|
||||||
await Promise.resolve();
|
|
||||||
}
|
|
||||||
|
|
||||||
describe("session voice mode streaming transcription", () => {
|
|
||||||
test("submits the streaming final transcript to the agent without batch transcribe", async () => {
|
|
||||||
const harness = createVoiceSessionHarness();
|
|
||||||
|
|
||||||
await harness.internals.startVoiceTurnController();
|
|
||||||
harness.detector.emit("speech_started");
|
|
||||||
await settleVoiceSession();
|
|
||||||
harness.detector.emit("speech_stopped");
|
|
||||||
await settleVoiceSession();
|
|
||||||
harness.sttSession.emitCommitted({ segmentId: "segment-1", previousSegmentId: null });
|
|
||||||
harness.sttSession.emitTranscript({
|
|
||||||
segmentId: "segment-1",
|
|
||||||
transcript: "ship the streaming final",
|
|
||||||
isFinal: true,
|
|
||||||
language: "en",
|
|
||||||
avgLogprob: -0.1,
|
|
||||||
isLowConfidence: false,
|
|
||||||
});
|
|
||||||
await settleVoiceSession();
|
|
||||||
|
|
||||||
expect(harness.sttSession.commitCount).toBe(1);
|
|
||||||
expect(harness.transcribe).not.toHaveBeenCalled();
|
|
||||||
expect(harness.sendAgentMessage).toHaveBeenCalledWith(
|
|
||||||
"11111111-1111-4111-8111-111111111111",
|
|
||||||
"ship the streaming final",
|
|
||||||
undefined,
|
|
||||||
undefined,
|
|
||||||
undefined,
|
|
||||||
undefined,
|
|
||||||
{ spokenInput: true },
|
|
||||||
);
|
|
||||||
expect(harness.messages).toContainEqual(
|
|
||||||
expect.objectContaining({
|
|
||||||
type: "transcription_result",
|
|
||||||
payload: expect.objectContaining({
|
|
||||||
text: "ship the streaming final",
|
|
||||||
language: "en",
|
|
||||||
avgLogprob: -0.1,
|
|
||||||
}),
|
|
||||||
}),
|
|
||||||
);
|
|
||||||
|
|
||||||
await harness.internals.stopVoiceTurnController();
|
|
||||||
});
|
|
||||||
|
|
||||||
test("uses the finalization timeout empty transcript path without agent submission", async () => {
|
|
||||||
vi.useFakeTimers();
|
|
||||||
try {
|
|
||||||
const harness = createVoiceSessionHarness();
|
|
||||||
|
|
||||||
await harness.internals.startVoiceTurnController();
|
|
||||||
harness.detector.emit("speech_started");
|
|
||||||
await settleVoiceSession();
|
|
||||||
harness.detector.emit("speech_stopped");
|
|
||||||
await settleVoiceSession();
|
|
||||||
harness.sttSession.emitCommitted({ segmentId: "segment-1", previousSegmentId: null });
|
|
||||||
|
|
||||||
await vi.advanceTimersByTimeAsync(10_000);
|
|
||||||
await settleVoiceSession();
|
|
||||||
|
|
||||||
expect(harness.transcribe).not.toHaveBeenCalled();
|
|
||||||
expect(harness.sendAgentMessage).not.toHaveBeenCalled();
|
|
||||||
expect(harness.messages).toContainEqual(
|
|
||||||
expect.objectContaining({
|
|
||||||
type: "transcription_result",
|
|
||||||
payload: expect.objectContaining({
|
|
||||||
text: "",
|
|
||||||
}),
|
|
||||||
}),
|
|
||||||
);
|
|
||||||
|
|
||||||
await harness.internals.stopVoiceTurnController();
|
|
||||||
} finally {
|
|
||||||
vi.useRealTimers();
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
test("filters low-confidence streaming finals without agent submission", async () => {
|
|
||||||
const harness = createVoiceSessionHarness();
|
|
||||||
|
|
||||||
await harness.internals.startVoiceTurnController();
|
|
||||||
harness.detector.emit("speech_started");
|
|
||||||
await settleVoiceSession();
|
|
||||||
harness.detector.emit("speech_stopped");
|
|
||||||
await settleVoiceSession();
|
|
||||||
harness.sttSession.emitCommitted({ segmentId: "segment-1", previousSegmentId: null });
|
|
||||||
harness.sttSession.emitTranscript({
|
|
||||||
segmentId: "segment-1",
|
|
||||||
transcript: "background noise",
|
|
||||||
isFinal: true,
|
|
||||||
avgLogprob: -2.5,
|
|
||||||
isLowConfidence: true,
|
|
||||||
});
|
|
||||||
await settleVoiceSession();
|
|
||||||
|
|
||||||
expect(harness.transcribe).not.toHaveBeenCalled();
|
|
||||||
expect(harness.sendAgentMessage).not.toHaveBeenCalled();
|
|
||||||
expect(harness.messages).toContainEqual(
|
|
||||||
expect.objectContaining({
|
|
||||||
type: "transcription_result",
|
|
||||||
payload: expect.objectContaining({
|
|
||||||
text: "",
|
|
||||||
avgLogprob: -2.5,
|
|
||||||
isLowConfidence: true,
|
|
||||||
}),
|
|
||||||
}),
|
|
||||||
);
|
|
||||||
|
|
||||||
await harness.internals.stopVoiceTurnController();
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
describe("file explorer binary responses", () => {
|
describe("file explorer binary responses", () => {
|
||||||
const tempDirs: string[] = [];
|
const tempDirs: string[] = [];
|
||||||
|
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,211 @@
|
|||||||
|
import { EventEmitter } from "node:events";
|
||||||
|
import pino from "pino";
|
||||||
|
import { describe, expect, test, vi } from "vitest";
|
||||||
|
|
||||||
|
import { VoiceSession, type VoiceSessionHost } from "./voice-session.js";
|
||||||
|
import type { ManagedAgent } from "../agent/agent-manager.js";
|
||||||
|
import type { SessionOutboundMessage } from "../messages.js";
|
||||||
|
import type {
|
||||||
|
SpeechToTextProvider,
|
||||||
|
StreamingTranscriptionCommittedEvent,
|
||||||
|
StreamingTranscriptionEvent,
|
||||||
|
StreamingTranscriptionSession,
|
||||||
|
} from "../speech/speech-provider.js";
|
||||||
|
import type {
|
||||||
|
TurnDetectionProvider,
|
||||||
|
TurnDetectionSession,
|
||||||
|
} from "../speech/turn-detection-provider.js";
|
||||||
|
|
||||||
|
const VOICE_AGENT_ID = "11111111-1111-4111-8111-111111111111";
|
||||||
|
|
||||||
|
class FakeVoiceTurnDetectionSession extends EventEmitter implements TurnDetectionSession {
|
||||||
|
public readonly requiredSampleRate = 16000;
|
||||||
|
|
||||||
|
async connect(): Promise<void> {}
|
||||||
|
|
||||||
|
appendPcm16(_chunk: Buffer): void {}
|
||||||
|
|
||||||
|
flush(): void {}
|
||||||
|
reset(): void {}
|
||||||
|
close(): void {}
|
||||||
|
}
|
||||||
|
|
||||||
|
class FakeVoiceSttSession extends EventEmitter implements StreamingTranscriptionSession {
|
||||||
|
public readonly requiredSampleRate = 16000;
|
||||||
|
public commitCount = 0;
|
||||||
|
|
||||||
|
async connect(): Promise<void> {}
|
||||||
|
|
||||||
|
appendPcm16(_pcm16le: Buffer): void {}
|
||||||
|
|
||||||
|
commit(): void {
|
||||||
|
this.commitCount += 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
clear(): void {}
|
||||||
|
close(): void {}
|
||||||
|
|
||||||
|
emitCommitted(event: StreamingTranscriptionCommittedEvent): void {
|
||||||
|
this.emit("committed", event);
|
||||||
|
}
|
||||||
|
|
||||||
|
emitTranscript(event: StreamingTranscriptionEvent): void {
|
||||||
|
this.emit("transcript", event);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
interface FakeVoiceHost extends VoiceSessionHost {
|
||||||
|
readonly emitted: SessionOutboundMessage[];
|
||||||
|
readonly spokenInput: Array<{ agentId: string; text: string }>;
|
||||||
|
}
|
||||||
|
|
||||||
|
function createFakeHost(): FakeVoiceHost {
|
||||||
|
const emitted: SessionOutboundMessage[] = [];
|
||||||
|
const spokenInput: Array<{ agentId: string; text: string }> = [];
|
||||||
|
return {
|
||||||
|
emitted,
|
||||||
|
spokenInput,
|
||||||
|
emit: (msg) => {
|
||||||
|
emitted.push(msg);
|
||||||
|
},
|
||||||
|
loadAgent: async (agentId) =>
|
||||||
|
({ id: agentId, config: { systemPrompt: undefined } }) as unknown as ManagedAgent,
|
||||||
|
reloadAgentSession: async (agentId) => ({ id: agentId }) as unknown as ManagedAgent,
|
||||||
|
sendSpokenInput: async (agentId, text) => {
|
||||||
|
spokenInput.push({ agentId, text });
|
||||||
|
},
|
||||||
|
interruptAgentIfRunning: async () => {},
|
||||||
|
hasActiveAgentRun: () => false,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function createVoiceSession() {
|
||||||
|
const detector = new FakeVoiceTurnDetectionSession();
|
||||||
|
const sttSession = new FakeVoiceSttSession();
|
||||||
|
const stt: SpeechToTextProvider = {
|
||||||
|
id: "local",
|
||||||
|
createSession: vi.fn(() => sttSession),
|
||||||
|
};
|
||||||
|
const turnDetection: TurnDetectionProvider = {
|
||||||
|
id: "local",
|
||||||
|
createSession: vi.fn(() => detector),
|
||||||
|
};
|
||||||
|
const host = createFakeHost();
|
||||||
|
const voiceSession = new VoiceSession({
|
||||||
|
host,
|
||||||
|
logger: pino({ level: "silent" }),
|
||||||
|
sessionId: "voice-session-test",
|
||||||
|
sttLanguage: "en",
|
||||||
|
tts: null,
|
||||||
|
stt,
|
||||||
|
voice: { turnDetection },
|
||||||
|
});
|
||||||
|
return { voiceSession, detector, sttSession, host };
|
||||||
|
}
|
||||||
|
|
||||||
|
async function settle(): Promise<void> {
|
||||||
|
await Promise.resolve();
|
||||||
|
await Promise.resolve();
|
||||||
|
await Promise.resolve();
|
||||||
|
}
|
||||||
|
|
||||||
|
describe("VoiceSession streaming transcription", () => {
|
||||||
|
test("delivers the streaming final transcript to the agent exactly once", async () => {
|
||||||
|
const { voiceSession, detector, sttSession, host } = createVoiceSession();
|
||||||
|
|
||||||
|
await voiceSession.handleSetVoiceMode(true, VOICE_AGENT_ID);
|
||||||
|
detector.emit("speech_started");
|
||||||
|
await settle();
|
||||||
|
detector.emit("speech_stopped");
|
||||||
|
await settle();
|
||||||
|
sttSession.emitCommitted({ segmentId: "segment-1", previousSegmentId: null });
|
||||||
|
sttSession.emitTranscript({
|
||||||
|
segmentId: "segment-1",
|
||||||
|
transcript: "ship the streaming final",
|
||||||
|
isFinal: true,
|
||||||
|
language: "en",
|
||||||
|
avgLogprob: -0.1,
|
||||||
|
isLowConfidence: false,
|
||||||
|
});
|
||||||
|
await settle();
|
||||||
|
|
||||||
|
expect(sttSession.commitCount).toBe(1);
|
||||||
|
expect(host.spokenInput).toEqual([
|
||||||
|
{ agentId: VOICE_AGENT_ID, text: "ship the streaming final" },
|
||||||
|
]);
|
||||||
|
expect(host.emitted).toContainEqual(
|
||||||
|
expect.objectContaining({
|
||||||
|
type: "transcription_result",
|
||||||
|
payload: expect.objectContaining({
|
||||||
|
text: "ship the streaming final",
|
||||||
|
language: "en",
|
||||||
|
avgLogprob: -0.1,
|
||||||
|
}),
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
await voiceSession.cleanup();
|
||||||
|
});
|
||||||
|
|
||||||
|
test("emits an empty transcript on finalization timeout without submitting to the agent", async () => {
|
||||||
|
vi.useFakeTimers();
|
||||||
|
try {
|
||||||
|
const { voiceSession, detector, sttSession, host } = createVoiceSession();
|
||||||
|
|
||||||
|
await voiceSession.handleSetVoiceMode(true, VOICE_AGENT_ID);
|
||||||
|
detector.emit("speech_started");
|
||||||
|
await settle();
|
||||||
|
detector.emit("speech_stopped");
|
||||||
|
await settle();
|
||||||
|
sttSession.emitCommitted({ segmentId: "segment-1", previousSegmentId: null });
|
||||||
|
|
||||||
|
await vi.advanceTimersByTimeAsync(10_000);
|
||||||
|
await settle();
|
||||||
|
|
||||||
|
expect(host.spokenInput).toEqual([]);
|
||||||
|
expect(host.emitted).toContainEqual(
|
||||||
|
expect.objectContaining({
|
||||||
|
type: "transcription_result",
|
||||||
|
payload: expect.objectContaining({ text: "" }),
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
await voiceSession.cleanup();
|
||||||
|
} finally {
|
||||||
|
vi.useRealTimers();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test("filters a low-confidence streaming final without submitting to the agent", async () => {
|
||||||
|
const { voiceSession, detector, sttSession, host } = createVoiceSession();
|
||||||
|
|
||||||
|
await voiceSession.handleSetVoiceMode(true, VOICE_AGENT_ID);
|
||||||
|
detector.emit("speech_started");
|
||||||
|
await settle();
|
||||||
|
detector.emit("speech_stopped");
|
||||||
|
await settle();
|
||||||
|
sttSession.emitCommitted({ segmentId: "segment-1", previousSegmentId: null });
|
||||||
|
sttSession.emitTranscript({
|
||||||
|
segmentId: "segment-1",
|
||||||
|
transcript: "background noise",
|
||||||
|
isFinal: true,
|
||||||
|
avgLogprob: -2.5,
|
||||||
|
isLowConfidence: true,
|
||||||
|
});
|
||||||
|
await settle();
|
||||||
|
|
||||||
|
expect(host.spokenInput).toEqual([]);
|
||||||
|
expect(host.emitted).toContainEqual(
|
||||||
|
expect.objectContaining({
|
||||||
|
type: "transcription_result",
|
||||||
|
payload: expect.objectContaining({
|
||||||
|
text: "",
|
||||||
|
avgLogprob: -2.5,
|
||||||
|
isLowConfidence: true,
|
||||||
|
}),
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
await voiceSession.cleanup();
|
||||||
|
});
|
||||||
|
});
|
||||||
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user