refactor(server): extract SpeechService and defer init until after listen

Speech runtime was leaking 8+ individual resolvers into bootstrap and
blocking server startup for ~3s with synchronous Sherpa native model
loading. Refactor into a self-contained SpeechService that owns its
full lifecycle (init, download, monitor, cleanup) and defer start()
until after httpServer.listen() so native loading doesn't block
accepting connections. Server startup drops from ~3.2s to ~0.5s.

Also removes unused client-triggered speech model download/list path
(CLI commands, session handlers, message types) — the service manages
its own models.

packages/cli/src/commands/speech/download.ts

@@ -1,63 +0,0 @@
-import type { Command } from "commander";
-import type { CommandError, CommandOptions, ListResult, OutputSchema } from "../../output/index.js";
-import { connectToDaemon } from "../../utils/client.js";
-
-interface SpeechDownloadRow {
-  modelId: string;
-  status: "downloaded";
-}
-
-const speechDownloadSchema: OutputSchema<SpeechDownloadRow> = {
-  idField: "modelId",
-  columns: [
-    { header: "MODEL", field: "modelId", width: 36 },
-    { header: "STATUS", field: "status", width: 12, color: () => "green" },
-  ],
-};
-
-export type SpeechDownloadResult = ListResult<SpeechDownloadRow>;
-
-export interface SpeechDownloadOptions extends CommandOptions {
-  host?: string;
-  model?: string[];
-}
-
-export async function runSpeechDownloadCommand(
-  options: SpeechDownloadOptions,
-  _command: Command,
-): Promise<SpeechDownloadResult> {
-  const client = await connectToDaemon({ host: options.host });
-  try {
-    const response = await client.downloadSpeechModels({
-      modelIds: options.model && options.model.length > 0 ? options.model : undefined,
-    });
-    if (response.error) {
-      const commandError: CommandError = {
-        code: "SPEECH_MODELS_DOWNLOAD_FAILED",
-        message: response.error,
-      };
-      throw commandError;
-    }
-
-    return {
-      type: "list",
-      data: response.downloadedModelIds.map((modelId) => ({
-        modelId,
-        status: "downloaded" as const,
-      })),
-      schema: speechDownloadSchema,
-    };
-  } catch (error) {
-    if (typeof error === "object" && error && "code" in error && "message" in error) {
-      throw error;
-    }
-    const message = error instanceof Error ? error.message : String(error);
-    const commandError: CommandError = {
-      code: "SPEECH_MODELS_DOWNLOAD_FAILED",
-      message: `Failed to download speech models: ${message}`,
-    };
-    throw commandError;
-  } finally {
-    await client.close().catch(() => {});
-  }
-}

packages/cli/src/commands/speech/index.ts

@@ -1,22 +1,5 @@
 import { Command } from "commander";
-import { withOutput } from "../../output/index.js";
-import { runSpeechModelsCommand } from "./models.js";
-import { runSpeechDownloadCommand } from "./download.js";
-import { addJsonAndDaemonHostOptions, collectMultiple } from "../../utils/command-options.js";
 
 export function createSpeechCommand(): Command {
-  const speech = new Command("speech").description("Manage local speech models");
+  return new Command("speech").description("Speech commands");
-
-  addJsonAndDaemonHostOptions(
-    speech.command("models").description("List local speech model download status"),
-  ).action(withOutput(runSpeechModelsCommand));
-
-  addJsonAndDaemonHostOptions(
-    speech
-      .command("download")
-      .description("Download local speech models")
-      .option("--model <id>", "Model ID to download (repeatable)", collectMultiple, []),
-  ).action(withOutput(runSpeechDownloadCommand));
-
-  return speech;
 }

packages/cli/src/commands/speech/models.ts

@@ -1,67 +0,0 @@
-import type { Command } from "commander";
-import type { CommandError, CommandOptions, ListResult, OutputSchema } from "../../output/index.js";
-import { connectToDaemon } from "../../utils/client.js";
-
-interface SpeechModelListItem {
-  id: string;
-  kind: string;
-  status: "downloaded" | "missing";
-  modelDir: string;
-  missingFiles: string;
-}
-
-const speechModelsSchema: OutputSchema<SpeechModelListItem> = {
-  idField: "id",
-  columns: [
-    { header: "MODEL", field: "id", width: 36 },
-    { header: "KIND", field: "kind", width: 12 },
-    {
-      header: "STATUS",
-      field: "status",
-      width: 12,
-      color: (value) => (value === "downloaded" ? "green" : "yellow"),
-    },
-    { header: "MODEL DIR", field: "modelDir", width: 44 },
-    { header: "MISSING FILES", field: "missingFiles", width: 40 },
-  ],
-};
-
-export type SpeechModelsResult = ListResult<SpeechModelListItem>;
-
-export interface SpeechModelsOptions extends CommandOptions {
-  host?: string;
-}
-
-export async function runSpeechModelsCommand(
-  options: SpeechModelsOptions,
-  _command: Command,
-): Promise<SpeechModelsResult> {
-  const client = await connectToDaemon({ host: options.host });
-  try {
-    const response = await client.listSpeechModels();
-    const rows: SpeechModelListItem[] = response.models
-      .slice()
-      .sort((a, b) => a.kind.localeCompare(b.kind) || a.id.localeCompare(b.id))
-      .map((model) => ({
-        id: model.id,
-        kind: model.kind,
-        status: model.isDownloaded ? "downloaded" : "missing",
-        modelDir: model.modelDir,
-        missingFiles: model.missingFiles?.join(", ") ?? "",
-      }));
-    return {
-      type: "list",
-      data: rows,
-      schema: speechModelsSchema,
-    };
-  } catch (error) {
-    const message = error instanceof Error ? error.message : String(error);
-    const commandError: CommandError = {
-      code: "SPEECH_MODELS_LIST_FAILED",
-      message: `Failed to list speech models: ${message}`,
-    };
-    throw commandError;
-  } finally {
-    await client.close().catch(() => {});
-  }
-}

packages/server/src/client/daemon-client.ts

@@ -39,8 +39,6 @@ import type {
   ListCommandsResponse,
   ListProviderModelsResponseMessage,
   ListAvailableProvidersResponse,
-  SpeechModelsListResponse,
-  SpeechModelsDownloadResponse,
   ListTerminalsResponse,
   CreateTerminalResponse,
   SubscribeTerminalResponse,
@@ -217,8 +215,6 @@ type FileExplorerPayload = FileExplorerResponse["payload"];
 type FileDownloadTokenPayload = FileDownloadTokenResponse["payload"];
 type ListProviderModelsPayload = ListProviderModelsResponseMessage["payload"];
 type ListAvailableProvidersPayload = ListAvailableProvidersResponse["payload"];
-type SpeechModelsListPayload = SpeechModelsListResponse["payload"];
-type SpeechModelsDownloadPayload = SpeechModelsDownloadResponse["payload"];
 type ListCommandsPayload = ListCommandsResponse["payload"];
 type ListCommandsDraftConfig = Pick<
   AgentSessionConfig,
@@ -2487,32 +2483,6 @@ export class DaemonClient {
     });
   }
 
-  async listSpeechModels(requestId?: string): Promise<SpeechModelsListPayload> {
-    return this.sendCorrelatedSessionRequest({
-      requestId,
-      message: {
-        type: "speech_models_list_request",
-      },
-      responseType: "speech_models_list_response",
-      timeout: 30000,
-    });
-  }
-
-  async downloadSpeechModels(options?: {
-    modelIds?: string[];
-    requestId?: string;
-  }): Promise<SpeechModelsDownloadPayload> {
-    return this.sendCorrelatedSessionRequest({
-      requestId: options?.requestId,
-      message: {
-        type: "speech_models_download_request",
-        modelIds: options?.modelIds,
-      },
-      responseType: "speech_models_download_response",
-      timeout: 30 * 60 * 1000,
-    });
-  }
-
   async listCommands(agentId: string, requestId?: string): Promise<ListCommandsPayload>;
   async listCommands(agentId: string, options?: ListCommandsOptions): Promise<ListCommandsPayload>;
   async listCommands(

packages/server/src/server/bootstrap.ts

@@ -91,7 +91,7 @@ import { DownloadTokenStore } from "./file-download/token-store.js";
 import type { OpenAiSpeechProviderConfig } from "./speech/providers/openai/config.js";
 import type { LocalSpeechProviderConfig } from "./speech/providers/local/config.js";
 import type { RequestedSpeechProviders } from "./speech/speech-types.js";
-import { initializeSpeechRuntime } from "./speech/speech-runtime.js";
+import { createSpeechService } from "./speech/speech-runtime.js";
 import { AgentManager } from "./agent/agent-manager.js";
 import { AgentStorage } from "./agent/agent-storage.js";
 import { attachAgentStoragePersistence } from "./persistence-hooks.js";
@@ -426,7 +426,6 @@ export async function createPaseoDaemon(
     logger.info({ elapsed: elapsed() }, "Preparing voice and MCP runtime");
     let wsServer: VoiceAssistantWebSocketServer | null = null;
     let voiceMcpBridgeManager: VoiceMcpSocketBridgeManager | null = null;
-    let unsubscribeSpeechReadiness: (() => void) | null = null;
 
     // Create in-memory transport for Session's Agent MCP client (voice assistant tools)
     const createInMemoryAgentMcpTransport = async (): Promise<InMemoryTransport> => {
@@ -589,21 +588,12 @@ export async function createPaseoDaemon(
         });
       },
     });
-    const {
+    const speechService = createSpeechService({
-      resolveVoiceTurnDetection,
-      resolveVoiceStt,
-      resolveVoiceTts,
-      resolveDictationStt,
-      getSpeechReadiness,
-      subscribeSpeechReadiness,
-      cleanup: cleanupSpeechRuntime,
-      localModelConfig,
-    } = await initializeSpeechRuntime({
       logger,
       openaiConfig: config.openai,
       speechConfig: config.speech,
     });
-    logger.info({ elapsed: elapsed() }, "Speech runtime initialized");
+    logger.info({ elapsed: elapsed() }, "Speech service created");
 
     wsServer = new VoiceAssistantWebSocketServer(
       httpServer,
@@ -615,7 +605,7 @@ export async function createPaseoDaemon(
       config.paseoHome,
       createInMemoryAgentMcpTransport,
       { allowedOrigins, allowedHosts: config.allowedHosts },
-      { turnDetection: resolveVoiceTurnDetection, stt: resolveVoiceStt, tts: resolveVoiceTts },
+      speechService,
       terminalManager,
       {
         voiceAgentMcpStdio: {
@@ -633,9 +623,6 @@ export async function createPaseoDaemon(
       },
       {
         finalTimeoutMs: config.dictationFinalTimeoutMs,
-        stt: resolveDictationStt,
-        localModels: localModelConfig ?? undefined,
-        getSpeechReadiness,
       },
       config.agentProviderSettings,
       daemonVersion,
@@ -653,9 +640,6 @@ export async function createPaseoDaemon(
       scheduleService,
       checkoutDiffManager,
     );
-    unsubscribeSpeechReadiness = subscribeSpeechReadiness((snapshot) => {
-      wsServer?.publishSpeechReadiness(snapshot);
-    });
 
     logger.info({ elapsed: elapsed() }, "Bootstrap complete, ready to start listening");
 
@@ -741,6 +725,10 @@ export async function createPaseoDaemon(
           httpServer.listen(listenTarget.path);
         }
       });
+
+      // Start speech service after listening so synchronous Sherpa native
+      // model loading doesn't block the server from accepting connections.
+      speechService.start();
     };
 
     const stop = async () => {
@@ -752,9 +740,7 @@ export async function createPaseoDaemon(
         runtimeSettings: config.agentProviderSettings,
       });
       terminalManager.killAll();
-      unsubscribeSpeechReadiness?.();
+      speechService.stop();
-      unsubscribeSpeechReadiness = null;
-      cleanupSpeechRuntime();
       await scheduleService.stop().catch(() => undefined);
       await relayTransport?.stop().catch(() => undefined);
       if (wsServer) {

packages/server/src/server/session.ts

@@ -170,12 +170,7 @@ import {
   toCheckoutError,
 } from "./checkout-git-utils.js";
 import { CheckoutDiffManager } from "./checkout-diff-manager.js";
-import {
+import type { LocalSpeechModelId } from "./speech/providers/local/models.js";
-  ensureLocalSpeechModels,
-  getLocalSpeechModelDir,
-  listLocalSpeechModels,
-  type LocalSpeechModelId,
-} from "./speech/providers/local/models.js";
 import { toResolver, type Resolvable } from "./speech/provider-resolver.js";
 import type { SpeechReadinessSnapshot, SpeechReadinessState } from "./speech/speech-runtime.js";
 import type pino from "pino";
@@ -401,10 +396,6 @@ export type SessionOptions = {
   dictation?: {
     finalTimeoutMs?: number;
     stt?: Resolvable<SpeechToTextProvider | null>;
-    localModels?: {
-      modelsDir: string;
-      defaultModelIds: LocalSpeechModelId[];
-    };
     getSpeechReadiness?: () => SpeechReadinessSnapshot;
   };
   agentProviderRuntimeSettings?: AgentProviderRuntimeSettingsMap;
@@ -596,8 +587,6 @@ export class Session {
   private readonly checkoutDiffSubscriptions = new Map<string, () => void>();
   private readonly workspaceGitWatchTargets = new Map<string, WorkspaceGitWatchTarget>();
   private readonly voiceAgentMcpStdio: VoiceMcpStdioConfig | null;
-  private readonly localSpeechModelsDir: string;
-  private readonly defaultLocalSpeechModelIds: LocalSpeechModelId[];
   private readonly registerVoiceSpeakHandler?: (
     agentId: string,
     handler: VoiceSpeakHandler,
@@ -669,15 +658,6 @@ export class Session {
     }
     this.voiceAgentMcpStdio = voice?.voiceAgentMcpStdio ?? null;
     this.resolveVoiceTurnDetection = toResolver(voice?.turnDetection ?? null);
-    const configuredModelsDir = dictation?.localModels?.modelsDir?.trim();
-    this.localSpeechModelsDir =
-      configuredModelsDir && configuredModelsDir.length > 0
-        ? configuredModelsDir
-        : join(this.paseoHome, "models", "local-speech");
-    this.defaultLocalSpeechModelIds =
-      dictation?.localModels?.defaultModelIds && dictation.localModels.defaultModelIds.length > 0
-        ? [...new Set(dictation.localModels.defaultModelIds)]
-        : ["parakeet-tdt-0.6b-v2-int8", "kokoro-en-v0_19"];
     this.registerVoiceSpeakHandler = voiceBridge?.registerVoiceSpeakHandler;
     this.unregisterVoiceSpeakHandler = voiceBridge?.unregisterVoiceSpeakHandler;
     this.registerVoiceCallerContext = voiceBridge?.registerVoiceCallerContext;
@@ -1665,14 +1645,6 @@ export class Session {
           await this.handleListAvailableProvidersRequest(msg);
           break;
 
-        case "speech_models_list_request":
-          await this.handleSpeechModelsListRequest(msg);
-          break;
-
-        case "speech_models_download_request":
-          await this.handleSpeechModelsDownloadRequest(msg);
-          break;
-
         case "clear_agent_attention":
           await this.handleClearAgentAttention(msg.agentId);
           break;
@@ -3062,104 +3034,6 @@ export class Session {
     }
   }
 
-  private async handleSpeechModelsListRequest(
-    msg: Extract<SessionInboundMessage, { type: "speech_models_list_request" }>,
-  ): Promise<void> {
-    const modelsDir = this.localSpeechModelsDir;
-
-    const models = await Promise.all(
-      listLocalSpeechModels().map(async (model) => {
-        const modelDir = getLocalSpeechModelDir(modelsDir, model.id);
-        const missingFiles: string[] = [];
-        for (const rel of model.requiredFiles) {
-          const filePath = join(modelDir, rel);
-          try {
-            const fileStat = await stat(filePath);
-            if (fileStat.isDirectory()) {
-              continue;
-            }
-            if (!fileStat.isFile() || fileStat.size <= 0) {
-              missingFiles.push(rel);
-            }
-          } catch {
-            missingFiles.push(rel);
-          }
-        }
-
-        return {
-          id: model.id,
-          kind: model.kind,
-          description: model.description,
-          modelDir,
-          isDownloaded: missingFiles.length === 0,
-          ...(missingFiles.length > 0 ? { missingFiles } : {}),
-        };
-      }),
-    );
-
-    this.emit({
-      type: "speech_models_list_response",
-      payload: {
-        modelsDir,
-        models,
-        requestId: msg.requestId,
-      },
-    });
-  }
-
-  private async handleSpeechModelsDownloadRequest(
-    msg: Extract<SessionInboundMessage, { type: "speech_models_download_request" }>,
-  ): Promise<void> {
-    const modelsDir = this.localSpeechModelsDir;
-
-    const modelIdsRaw =
-      msg.modelIds && msg.modelIds.length > 0 ? msg.modelIds : this.defaultLocalSpeechModelIds;
-
-    const allModelIds = new Set(listLocalSpeechModels().map((m) => m.id));
-    const invalid = modelIdsRaw.filter((id) => !allModelIds.has(id as LocalSpeechModelId));
-    if (invalid.length > 0) {
-      this.emit({
-        type: "speech_models_download_response",
-        payload: {
-          modelsDir,
-          downloadedModelIds: [],
-          error: `Unknown speech model id(s): ${invalid.join(", ")}`,
-          requestId: msg.requestId,
-        },
-      });
-      return;
-    }
-
-    const modelIds = modelIdsRaw as LocalSpeechModelId[];
-    try {
-      await ensureLocalSpeechModels({
-        modelsDir,
-        modelIds,
-        logger: this.sessionLogger,
-      });
-      this.emit({
-        type: "speech_models_download_response",
-        payload: {
-          modelsDir,
-          downloadedModelIds: modelIds,
-          error: null,
-          requestId: msg.requestId,
-        },
-      });
-    } catch (error) {
-      this.sessionLogger.error({ err: error, modelIds }, "Failed to download speech models");
-      this.emit({
-        type: "speech_models_download_response",
-        payload: {
-          modelsDir,
-          downloadedModelIds: [],
-          error: error instanceof Error ? error.message : String(error),
-          requestId: msg.requestId,
-        },
-      });
-    }
-  }
-
   private normalizeGitOptions(
     gitOptions?: GitSetupOptions,
     legacyWorktreeName?: string,

packages/server/src/server/speech/speech-runtime.test.ts

@@ -5,7 +5,7 @@ import type { PaseoSpeechConfig } from "../bootstrap.js";
 import type { InitializedLocalSpeech } from "./providers/local/runtime.js";
 import type { SpeechToTextProvider, TextToSpeechProvider } from "./speech-provider.js";
 import type { TurnDetectionProvider } from "./turn-detection-provider.js";
-import { initializeSpeechRuntime } from "./speech-runtime.js";
+import { createSpeechService } from "./speech-runtime.js";
 
 const { initializeLocalSpeechServicesMock } = vi.hoisted(() => ({
   initializeLocalSpeechServicesMock: vi.fn<(args: unknown) => Promise<InitializedLocalSpeech>>(),
@@ -70,7 +70,7 @@ function createSpeechConfig(providers: PaseoSpeechConfig["providers"]): PaseoSpe
   return { providers };
 }
 
-describe("initializeSpeechRuntime readiness", () => {
+describe("createSpeechService readiness", () => {
   beforeEach(() => {
     initializeLocalSpeechServicesMock.mockReset();
   });
@@ -92,7 +92,7 @@ describe("initializeSpeechRuntime readiness", () => {
       cleanup: () => {},
     });
 
-    const runtime = await initializeSpeechRuntime({
+    const runtime = createSpeechService({
       logger: pino({ level: "silent" }),
       speechConfig: createSpeechConfig({
         dictationStt: { provider: "local", enabled: true, explicit: true },
@@ -101,14 +101,16 @@ describe("initializeSpeechRuntime readiness", () => {
         voiceTts: { provider: "local", enabled: false, explicit: true },
       }),
     });
+    runtime.start();
+    await runtime.ready;
 
-    const readiness = runtime.getSpeechReadiness();
+    const readiness = runtime.getReadiness();
     expect(readiness.dictation.available).toBe(true);
     expect(readiness.realtimeVoice.reasonCode).toBe("disabled");
     expect(readiness.voiceFeature.available).toBe(true);
     expect(readiness.voiceFeature.reasonCode).toBe("ready");
 
-    runtime.cleanup();
+    runtime.stop();
   });
 
   it("keeps voice feature available when only realtime voice is enabled and ready", async () => {
@@ -130,7 +132,7 @@ describe("initializeSpeechRuntime readiness", () => {
       cleanup: () => {},
     });
 
-    const runtime = await initializeSpeechRuntime({
+    const runtime = createSpeechService({
       logger: pino({ level: "silent" }),
       speechConfig: createSpeechConfig({
         dictationStt: { provider: "local", enabled: false, explicit: true },
@@ -139,13 +141,15 @@ describe("initializeSpeechRuntime readiness", () => {
         voiceTts: { provider: "local", enabled: true, explicit: true },
       }),
     });
+    runtime.start();
+    await runtime.ready;
 
-    const readiness = runtime.getSpeechReadiness();
+    const readiness = runtime.getReadiness();
     expect(readiness.realtimeVoice.available).toBe(true);
     expect(readiness.dictation.reasonCode).toBe("disabled");
     expect(readiness.voiceFeature.available).toBe(true);
     expect(readiness.voiceFeature.reasonCode).toBe("ready");
 
-    runtime.cleanup();
+    runtime.stop();
   });
 });

packages/server/src/server/speech/speech-runtime.ts

@@ -336,25 +336,23 @@ function resolveEffectiveProviderIds(params: {
   };
 }
 
-export type InitializedSpeechRuntime = {
+export type SpeechService = {
-  resolveVoiceTurnDetection: () => TurnDetectionProvider | null;
+  resolveStt: () => SpeechToTextProvider | null;
-  resolveVoiceStt: () => SpeechToTextProvider | null;
+  resolveTts: () => TextToSpeechProvider | null;
-  resolveVoiceTts: () => TextToSpeechProvider | null;
+  resolveTurnDetection: () => TurnDetectionProvider | null;
   resolveDictationStt: () => SpeechToTextProvider | null;
-  getSpeechReadiness: () => SpeechReadinessSnapshot;
+  getReadiness: () => SpeechReadinessSnapshot;
-  subscribeSpeechReadiness: (listener: (snapshot: SpeechReadinessSnapshot) => void) => () => void;
+  onReadinessChange: (listener: (snapshot: SpeechReadinessSnapshot) => void) => () => void;
-  cleanup: () => void;
+  start: () => void;
-  localModelConfig: {
+  stop: () => void;
-    modelsDir: string;
+  ready: Promise<void>;
-    defaultModelIds: LocalSpeechModelId[];
-  } | null;
 };
 
-export async function initializeSpeechRuntime(params: {
+export function createSpeechService(params: {
   logger: Logger;
   openaiConfig?: PaseoOpenAIConfig;
   speechConfig?: PaseoSpeechConfig;
-}): Promise<InitializedSpeechRuntime> {
+}): SpeechService {
   const logger = params.logger.child({ module: "speech-runtime" });
   const speechConfig = params.speechConfig ?? null;
   const openaiConfig = params.openaiConfig;
@@ -397,6 +395,14 @@ export async function initializeSpeechRuntime(params: {
   const readinessListeners = new Set<(snapshot: SpeechReadinessSnapshot) => void>();
   let lastReadinessFingerprint: string | null = null;
   let lastPublishedReadinessSnapshot: SpeechReadinessSnapshot | null = null;
+  let started = false;
+  let readySettled = false;
+  let resolveReady!: () => void;
+  let rejectReady!: (error: unknown) => void;
+  const ready = new Promise<void>((resolve, reject) => {
+    resolveReady = resolve;
+    rejectReady = reject;
+  });
 
   const computeReadinessSnapshot = (): SpeechReadinessSnapshot => {
     const realtimeVoice = buildRealtimeVoiceReadiness({
@@ -658,16 +664,36 @@ export async function initializeSpeechRuntime(params: {
     }
   };
 
-  await runReconcile();
+  const start = (): void => {
-  const snapshot = computeReadinessSnapshot();
+    if (started || stopped) {
-  if (snapshot.voiceFeature.enabled && !snapshot.voiceFeature.available) {
+      return;
-    if (missingLocalModelIds.length > 0) {
-      startBackgroundDownload();
     }
-    scheduleMonitor();
+    started = true;
-  }
+    void (async () => {
+      try {
+        await runReconcile();
+        const snapshot = computeReadinessSnapshot();
+        if (snapshot.voiceFeature.enabled && !snapshot.voiceFeature.available) {
+          if (missingLocalModelIds.length > 0) {
+            startBackgroundDownload();
+          }
+          scheduleMonitor();
+        }
+        if (!readySettled) {
+          readySettled = true;
+          resolveReady();
+        }
+      } catch (error) {
+        if (!readySettled) {
+          readySettled = true;
+          rejectReady(error);
+        }
+        logger.error({ err: error }, "Speech runtime failed during initial reconcile");
+      }
+    })();
+  };
 
-  const cleanup = (): void => {
+  const stop = (): void => {
     stopped = true;
     if (monitorTimeout) {
       clearTimeout(monitorTimeout);
@@ -677,13 +703,14 @@ export async function initializeSpeechRuntime(params: {
   };
 
   return {
-    resolveVoiceTurnDetection: () => turnDetectionService,
+    resolveTurnDetection: () => turnDetectionService,
-    resolveVoiceStt: () => sttService,
+    resolveStt: () => sttService,
-    resolveVoiceTts: () => ttsService,
+    resolveTts: () => ttsService,
     resolveDictationStt: () => dictationSttService,
-    getSpeechReadiness: () => lastPublishedReadinessSnapshot ?? computeReadinessSnapshot(),
+    getReadiness: () => lastPublishedReadinessSnapshot ?? computeReadinessSnapshot(),
-    subscribeSpeechReadiness,
+    onReadinessChange: subscribeSpeechReadiness,
-    cleanup,
+    start,
-    localModelConfig,
+    stop,
+    ready,
   };
 }

packages/server/src/server/websocket-server.ts

@@ -33,11 +33,7 @@ import type { AgentProvider } from "./agent/agent-sdk-types.js";
 import type { AgentProviderRuntimeSettingsMap } from "./agent/provider-launch-config.js";
 import { PushTokenStore } from "./push/token-store.js";
 import { PushService } from "./push/push-service.js";
-import type { SpeechToTextProvider, TextToSpeechProvider } from "./speech/speech-provider.js";
+import type { SpeechReadinessSnapshot, SpeechService } from "./speech/speech-runtime.js";
-import type { TurnDetectionProvider } from "./speech/turn-detection-provider.js";
-import type { Resolvable } from "./speech/provider-resolver.js";
-import type { SpeechReadinessSnapshot } from "./speech/speech-runtime.js";
-import type { LocalSpeechModelId } from "./speech/providers/local/models.js";
 import type { VoiceCallerContext, VoiceMcpStdioConfig, VoiceSpeakHandler } from "./voice-types.js";
 import {
   computeShouldNotifyClient,
@@ -243,18 +239,10 @@ export class VoiceAssistantWebSocketServer {
   private readonly pushTokenStore: PushTokenStore;
   private readonly pushService: PushService;
   private readonly createAgentMcpTransport: AgentMcpTransportFactory;
-  private readonly stt: Resolvable<SpeechToTextProvider | null>;
+  private readonly speech: SpeechService | null;
-  private readonly tts: Resolvable<TextToSpeechProvider | null>;
-  private readonly turnDetection: Resolvable<TurnDetectionProvider | null>;
   private readonly terminalManager: TerminalManager | null;
   private readonly dictation: {
     finalTimeoutMs?: number;
-    stt?: Resolvable<SpeechToTextProvider | null>;
-    localModels?: {
-      modelsDir: string;
-      defaultModelIds: LocalSpeechModelId[];
-    };
-    getSpeechReadiness?: () => SpeechReadinessSnapshot;
   } | null;
   private readonly voice: {
     voiceAgentMcpStdio?: VoiceMcpStdioConfig | null;
@@ -288,6 +276,7 @@ export class VoiceAssistantWebSocketServer {
   private readonly inboundSessionRequestCounts = new Map<string, number>();
   private readonly requestLatencies = new Map<string, number[]>();
   private runtimeMetricsInterval: ReturnType<typeof setInterval> | null = null;
+  private unsubscribeSpeechReadiness: (() => void) | null = null;
 
   constructor(
     server: HTTPServer,
@@ -299,11 +288,7 @@ export class VoiceAssistantWebSocketServer {
     paseoHome: string,
     createAgentMcpTransport: AgentMcpTransportFactory,
     wsConfig: WebSocketServerConfig,
-    speech?: {
+    speech?: SpeechService | null,
-      turnDetection: Resolvable<TurnDetectionProvider | null>;
-      stt: Resolvable<SpeechToTextProvider | null>;
-      tts: Resolvable<TextToSpeechProvider | null>;
-    },
     terminalManager?: TerminalManager | null,
     voice?: {
       voiceAgentMcpStdio?: VoiceMcpStdioConfig | null;
@@ -312,12 +297,6 @@ export class VoiceAssistantWebSocketServer {
     },
     dictation?: {
       finalTimeoutMs?: number;
-      stt?: Resolvable<SpeechToTextProvider | null>;
-      localModels?: {
-        modelsDir: string;
-        defaultModelIds: LocalSpeechModelId[];
-      };
-      getSpeechReadiness?: () => SpeechReadinessSnapshot;
     },
     agentProviderRuntimeSettings?: AgentProviderRuntimeSettingsMap,
     daemonVersion?: string,
@@ -358,17 +337,18 @@ export class VoiceAssistantWebSocketServer {
     this.downloadTokenStore = downloadTokenStore;
     this.paseoHome = paseoHome;
     this.createAgentMcpTransport = createAgentMcpTransport;
-    this.turnDetection = speech?.turnDetection ?? null;
+    this.speech = speech ?? null;
-    this.stt = speech?.stt ?? null;
-    this.tts = speech?.tts ?? null;
     this.terminalManager = terminalManager ?? null;
     this.voice = voice ?? null;
     this.dictation = dictation ?? null;
     this.agentProviderRuntimeSettings = agentProviderRuntimeSettings;
     this.onLifecycleIntent = onLifecycleIntent ?? null;
     this.serverCapabilities = buildServerCapabilities({
-      readiness: this.dictation?.getSpeechReadiness?.() ?? null,
+      readiness: this.speech?.getReadiness() ?? null,
     });
+    this.unsubscribeSpeechReadiness = this.speech?.onReadinessChange((snapshot) => {
+      this.publishSpeechReadiness(snapshot);
+    }) ?? null;
 
     const pushLogger = this.logger.child({ module: "push" });
     this.pushTokenStore = new PushTokenStore(pushLogger, join(paseoHome, "push-tokens.json"));
@@ -457,6 +437,8 @@ export class VoiceAssistantWebSocketServer {
   }
 
   public async close(): Promise<void> {
+    this.unsubscribeSpeechReadiness?.();
+    this.unsubscribeSpeechReadiness = null;
     if (this.runtimeMetricsInterval) {
       clearInterval(this.runtimeMetricsInterval);
       this.runtimeMetricsInterval = null;
@@ -658,12 +640,12 @@ export class VoiceAssistantWebSocketServer {
       scheduleService: this.scheduleService,
       checkoutDiffManager: this.checkoutDiffManager,
       createAgentMcpTransport: this.createAgentMcpTransport,
-      stt: this.stt,
+      stt: () => this.speech?.resolveStt() ?? null,
-      tts: this.tts,
+      tts: () => this.speech?.resolveTts() ?? null,
       terminalManager: this.terminalManager,
       voice: {
         ...(this.voice ?? {}),
-        turnDetection: this.turnDetection,
+        turnDetection: () => this.speech?.resolveTurnDetection() ?? null,
       },
       voiceBridge: {
         registerVoiceSpeakHandler: (agentId, handler) => {
@@ -681,7 +663,14 @@ export class VoiceAssistantWebSocketServer {
         ensureVoiceMcpSocketForAgent: this.voice?.ensureVoiceMcpSocketForAgent,
         removeVoiceMcpSocketForAgent: this.voice?.removeVoiceMcpSocketForAgent,
       },
-      dictation: this.dictation ?? undefined,
+      dictation:
+        this.dictation || this.speech
+          ? {
+              finalTimeoutMs: this.dictation?.finalTimeoutMs,
+              stt: () => this.speech?.resolveDictationStt() ?? null,
+              getSpeechReadiness: () => this.speech!.getReadiness(),
+            }
+          : undefined,
       agentProviderRuntimeSettings: this.agentProviderRuntimeSettings,
     });
 

packages/server/src/shared/messages.ts

@@ -726,17 +726,6 @@ export const ListAvailableProvidersRequestMessageSchema = z.object({
   requestId: z.string(),
 });
 
-export const SpeechModelsListRequestSchema = z.object({
-  type: z.literal("speech_models_list_request"),
-  requestId: z.string(),
-});
-
-export const SpeechModelsDownloadRequestSchema = z.object({
-  type: z.literal("speech_models_download_request"),
-  modelIds: z.array(z.string()).optional(),
-  requestId: z.string(),
-});
-
 export const ResumeAgentRequestMessageSchema = z.object({
   type: z.literal("resume_agent_request"),
   handle: AgentPersistenceHandleSchema,
@@ -1203,8 +1192,6 @@ export const SessionInboundMessageSchema = z.discriminatedUnion("type", [
   CreateAgentRequestMessageSchema,
   ListProviderModelsRequestMessageSchema,
   ListAvailableProvidersRequestMessageSchema,
-  SpeechModelsListRequestSchema,
-  SpeechModelsDownloadRequestSchema,
   ResumeAgentRequestMessageSchema,
   RefreshAgentRequestMessageSchema,
   CancelAgentRequestMessageSchema,
@@ -2104,34 +2091,6 @@ export const ListAvailableProvidersResponseSchema = z.object({
   }),
 });
 
-export const SpeechModelsListResponseSchema = z.object({
-  type: z.literal("speech_models_list_response"),
-  payload: z.object({
-    modelsDir: z.string(),
-    models: z.array(
-      z.object({
-        id: z.string(),
-        kind: z.string(),
-        description: z.string(),
-        modelDir: z.string(),
-        isDownloaded: z.boolean(),
-        missingFiles: z.array(z.string()).optional(),
-      }),
-    ),
-    requestId: z.string(),
-  }),
-});
-
-export const SpeechModelsDownloadResponseSchema = z.object({
-  type: z.literal("speech_models_download_response"),
-  payload: z.object({
-    modelsDir: z.string(),
-    downloadedModelIds: z.array(z.string()),
-    error: z.string().nullable(),
-    requestId: z.string(),
-  }),
-});
-
 const AgentSlashCommandSchema = z.object({
   name: z.string(),
   description: z.string(),
@@ -2311,8 +2270,6 @@ export const SessionOutboundMessageSchema = z.discriminatedUnion("type", [
   FileDownloadTokenResponseSchema,
   ListProviderModelsResponseMessageSchema,
   ListAvailableProvidersResponseSchema,
-  SpeechModelsListResponseSchema,
-  SpeechModelsDownloadResponseSchema,
   ListCommandsResponseSchema,
   ListTerminalsResponseSchema,
   TerminalsChangedSchema,
@@ -2381,8 +2338,6 @@ export type ListProviderModelsResponseMessage = z.infer<
   typeof ListProviderModelsResponseMessageSchema
 >;
 export type ListAvailableProvidersResponse = z.infer<typeof ListAvailableProvidersResponseSchema>;
-export type SpeechModelsListResponse = z.infer<typeof SpeechModelsListResponseSchema>;
-export type SpeechModelsDownloadResponse = z.infer<typeof SpeechModelsDownloadResponseSchema>;
 export type ChatCreateResponse = z.infer<typeof ChatCreateResponseSchema>;
 export type ChatListResponse = z.infer<typeof ChatListResponseSchema>;
 export type ChatInspectResponse = z.infer<typeof ChatInspectResponseSchema>;
@@ -2443,8 +2398,6 @@ export type LoopListRequest = z.infer<typeof LoopListRequestSchema>;
 export type LoopInspectRequest = z.infer<typeof LoopInspectRequestSchema>;
 export type LoopLogsRequest = z.infer<typeof LoopLogsRequestSchema>;
 export type LoopStopRequest = z.infer<typeof LoopStopRequestSchema>;
-export type SpeechModelsListRequestMessage = z.infer<typeof SpeechModelsListRequestSchema>;
-export type SpeechModelsDownloadRequestMessage = z.infer<typeof SpeechModelsDownloadRequestSchema>;
 export type ResumeAgentRequestMessage = z.infer<typeof ResumeAgentRequestMessageSchema>;
 export type DeleteAgentRequestMessage = z.infer<typeof DeleteAgentRequestMessageSchema>;
 export type UpdateAgentRequestMessage = z.infer<typeof UpdateAgentRequestMessageSchema>;