Stream dictation via OpenAI Realtime and simplify retry UI

packages/app/src/components/dictation-controls.tsx

@@ -18,7 +18,6 @@ interface DictationControlsProps {
   onRetry?: () => void;
   onDiscard?: () => void;
   disabled?: boolean;
-  retryStatusText?: string;
 }
 
 function formatDuration(seconds: number): string {
@@ -42,13 +41,11 @@ export function DictationControls({
   onRetry,
   onDiscard,
   disabled = false,
-  retryStatusText,
 }: DictationControlsProps) {
   const { theme } = useUnistyles();
-  const isRetrying = status === "retrying";
   const isFailed = status === "failed";
-  const showActiveState = isRecording || isProcessing || isRetrying || isFailed;
+  const showActiveState = isRecording || isProcessing || isFailed;
-  const actionsDisabled = isProcessing || isRetrying;
+  const actionsDisabled = isProcessing;
   const handleCancel = isFailed && onDiscard ? onDiscard : onCancel;
 
   if (!showActiveState) {
@@ -123,13 +120,6 @@ export function DictationControls({
           </>
         )}
       </View>
-      {retryStatusText && (isRetrying || isFailed) && (
-        <Text
-          style={[styles.statusLabel, { color: theme.colors.mutedForeground }]}
-        >
-          {retryStatusText}
-        </Text>
-      )}
     </View>
   );
 }
@@ -149,12 +139,11 @@ export function DictationOverlay({
   onAcceptAndSend,
   onRetry,
   onDiscard,
-}: Omit<DictationControlsProps, "onStart" | "disabled" | "retryStatusText">) {
+}: Omit<DictationControlsProps, "onStart" | "disabled">) {
   const { theme } = useUnistyles();
-  const isRetrying = status === "retrying";
   const isFailed = status === "failed";
-  const showActiveState = isRecording || isProcessing || isRetrying || isFailed;
+  const showActiveState = isRecording || isProcessing || isFailed;
-  const actionsDisabled = isProcessing || isRetrying;
+  const actionsDisabled = isProcessing;
   const handleCancel = isFailed && onDiscard ? onDiscard : onCancel;
 
   if (!showActiveState) {

packages/app/src/components/message-input.tsx

@@ -210,7 +210,7 @@ export const MessageInput = forwardRef<MessageInputRef, MessageInputProps>(
   });
 
   useEffect(() => {
-    if (isDictating || isDictationProcessing || dictationStatus === "retrying") {
+    if (isDictating || isDictationProcessing) {
       return;
     }
     sendAfterTranscriptRef.current = false;
@@ -246,7 +246,6 @@ export const MessageInput = forwardRef<MessageInputRef, MessageInputProps>(
     const showOverlay =
       isDictating ||
       isDictationProcessing ||
-      dictationStatus === "retrying" ||
       dictationStatus === "failed";
     overlayTransition.value = withTiming(showOverlay ? 1 : 0, {
       duration: 200,

packages/app/src/components/volume-meter.tsx

@@ -88,7 +88,7 @@ export function VolumeMeter({ volume, isMuted = false, isDetecting = false, isSp
       return;
     }
 
-    if (volume > 0.01) {
+    if (volume > 0.001) {
       // Active volume - animate heights based on volume
       const target1 = MIN_HEIGHT + (MAX_HEIGHT * volume * 1.2);
       const target2 = MIN_HEIGHT + (MAX_HEIGHT * volume * 1.05);
@@ -136,7 +136,7 @@ export function VolumeMeter({ volume, isMuted = false, isDetecting = false, isSp
     const baseOpacity = isMuted ? 0.3 : isActive ? 0.9 : 0.5;
     const volumeBoost = isMuted ? 0 : volume * 0.3;
     return {
-      height: line1Height.value * (isMuted || volume > 0.01 ? 1 : line1Pulse.value),
+      height: line1Height.value * (isMuted || volume > 0.001 ? 1 : line1Pulse.value),
       opacity: baseOpacity + volumeBoost,
     };
   });
@@ -146,7 +146,7 @@ export function VolumeMeter({ volume, isMuted = false, isDetecting = false, isSp
     const baseOpacity = isMuted ? 0.3 : isActive ? 0.9 : 0.5;
     const volumeBoost = isMuted ? 0 : volume * 0.3;
     return {
-      height: line2Height.value * (isMuted || volume > 0.01 ? 1 : line2Pulse.value),
+      height: line2Height.value * (isMuted || volume > 0.001 ? 1 : line2Pulse.value),
       opacity: baseOpacity + volumeBoost,
     };
   });
@@ -156,7 +156,7 @@ export function VolumeMeter({ volume, isMuted = false, isDetecting = false, isSp
     const baseOpacity = isMuted ? 0.3 : isActive ? 0.9 : 0.5;
     const volumeBoost = isMuted ? 0 : volume * 0.3;
     return {
-      height: line3Height.value * (isMuted || volume > 0.01 ? 1 : line3Pulse.value),
+      height: line3Height.value * (isMuted || volume > 0.001 ? 1 : line3Pulse.value),
       opacity: baseOpacity + volumeBoost,
     };
   });

packages/app/src/dictation/dictation-stream-sender.test.ts

@@ -0,0 +1,126 @@
+import { describe, expect, it } from "vitest";
+
+import { DictationStreamSender } from "@/dictation/dictation-stream-sender";
+
+type FakeFinish = { dictationId: string; finalSeq: number };
+type FakeStart = { dictationId: string; format: string };
+type FakeChunk = { dictationId: string; seq: number; audio: string; format: string };
+
+class FakeDaemonClient {
+  isConnected = true;
+  starts: FakeStart[] = [];
+  chunks: FakeChunk[] = [];
+  finishes: FakeFinish[] = [];
+  cancels: string[] = [];
+
+  async startDictationStream(dictationId: string, format: string): Promise<void> {
+    this.starts.push({ dictationId, format });
+  }
+
+  sendDictationStreamChunk(dictationId: string, seq: number, audio: string, format: string): void {
+    this.chunks.push({ dictationId, seq, audio, format });
+  }
+
+  async finishDictationStream(dictationId: string, finalSeq: number): Promise<{ dictationId: string; text: string }> {
+    this.finishes.push({ dictationId, finalSeq });
+    return { dictationId, text: "ok" };
+  }
+
+  cancelDictationStream(dictationId: string): void {
+    this.cancels.push(dictationId);
+  }
+}
+
+const tick = async () => {
+  await Promise.resolve();
+  await Promise.resolve();
+};
+
+describe("DictationStreamSender", () => {
+  it("enqueues segments and sends them after stream start", async () => {
+    const client = new FakeDaemonClient();
+    const ids = ["d1"];
+    const sender = new DictationStreamSender({
+      client: client as any,
+      format: "audio/pcm;rate=16000;bits=16",
+      createDictationId: () => ids.shift() ?? "dX",
+    });
+
+    sender.enqueueSegment("seg0");
+    sender.enqueueSegment("seg1");
+
+    await tick();
+
+    expect(client.starts).toEqual([{ dictationId: "d1", format: "audio/pcm;rate=16000;bits=16" }]);
+    expect(client.chunks).toEqual([
+      { dictationId: "d1", seq: 0, audio: "seg0", format: "audio/pcm;rate=16000;bits=16" },
+      { dictationId: "d1", seq: 1, audio: "seg1", format: "audio/pcm;rate=16000;bits=16" },
+    ]);
+  });
+
+  it("restarts stream and resends from seq=0 on reconnect", async () => {
+    const client = new FakeDaemonClient();
+    const ids = ["d1", "d2"];
+    const sender = new DictationStreamSender({
+      client: client as any,
+      format: "audio/pcm;rate=16000;bits=16",
+      createDictationId: () => ids.shift() ?? "dX",
+    });
+
+    sender.enqueueSegment("seg0");
+    sender.enqueueSegment("seg1");
+    await tick();
+
+    await sender.restartStream("reconnect");
+
+    expect(client.starts.map((s) => s.dictationId)).toEqual(["d1", "d2"]);
+    const d2Chunks = client.chunks.filter((c) => c.dictationId === "d2");
+    expect(d2Chunks.map((c) => [c.seq, c.audio])).toEqual([
+      [0, "seg0"],
+      [1, "seg1"],
+    ]);
+  });
+
+  it("finish flushes all queued segments and sends finish with finalSeq", async () => {
+    const client = new FakeDaemonClient();
+    const ids = ["d1"];
+    const sender = new DictationStreamSender({
+      client: client as any,
+      format: "audio/pcm;rate=16000;bits=16",
+      createDictationId: () => ids.shift() ?? "dX",
+    });
+
+    sender.enqueueSegment("seg0");
+    sender.enqueueSegment("seg1");
+
+    const finalSeq = sender.getFinalSeq();
+    const result = await sender.finish(finalSeq);
+
+    expect(result.text).toBe("ok");
+    expect(client.chunks.map((c) => c.seq)).toEqual([0, 1]);
+    expect(client.finishes).toEqual([{ dictationId: "d1", finalSeq: 1 }]);
+  });
+
+  it("keeps segments while disconnected and sends them after restart when reconnected", async () => {
+    const client = new FakeDaemonClient();
+    client.isConnected = false;
+    const ids = ["d1"];
+    const sender = new DictationStreamSender({
+      client: client as any,
+      format: "audio/pcm;rate=16000;bits=16",
+      createDictationId: () => ids.shift() ?? "dX",
+    });
+
+    sender.enqueueSegment("seg0");
+    sender.enqueueSegment("seg1");
+
+    expect(client.starts).toHaveLength(0);
+    expect(client.chunks).toHaveLength(0);
+
+    client.isConnected = true;
+    await sender.restartStream("reconnect");
+
+    expect(client.chunks.map((c) => c.seq)).toEqual([0, 1]);
+  });
+});
+

packages/app/src/dictation/dictation-stream-sender.ts

@@ -0,0 +1,189 @@
+import { generateMessageId } from "@/types/stream";
+import type { DaemonClientV2 } from "@server/client/daemon-client-v2";
+
+export type DictationStreamSenderParams = {
+  client: DaemonClientV2 | null;
+  format: string;
+  createDictationId?: () => string;
+};
+
+type DictationFinishResult = { dictationId: string; text: string };
+
+/**
+ * Small, non-React state machine for dictation streaming.
+ *
+ * Responsibilities:
+ * - Maintain an ordered buffer of base64 PCM segments
+ * - Start/restart a dictation stream (dictationId)
+ * - Send missing segments (seq) when connected
+ * - Finish/cancel the stream
+ *
+ * This class intentionally keeps sending synchronous (no internal async mutex),
+ * so enqueues can't "miss" a flush due to in-flight await/coalescing bugs.
+ */
+export class DictationStreamSender {
+  private client: DaemonClientV2 | null;
+  private readonly format: string;
+  private readonly createDictationId: () => string;
+
+  private dictationId: string | null = null;
+  private sendSeq = 0;
+  private segments: string[] = [];
+  private streamReady = false;
+
+  private startGeneration = 0;
+  private startPromise: Promise<void> | null = null;
+
+  constructor(params: DictationStreamSenderParams) {
+    this.client = params.client;
+    this.format = params.format;
+    this.createDictationId = params.createDictationId ?? generateMessageId;
+  }
+
+  setClient(client: DaemonClientV2 | null): void {
+    this.client = client;
+  }
+
+  getDictationId(): string | null {
+    return this.dictationId;
+  }
+
+  getSegmentCount(): number {
+    return this.segments.length;
+  }
+
+  getFinalSeq(): number {
+    return this.segments.length - 1;
+  }
+
+  hasSegments(): boolean {
+    return this.segments.length > 0;
+  }
+
+  clearAll(): void {
+    this.dictationId = null;
+    this.sendSeq = 0;
+    this.segments = [];
+    this.streamReady = false;
+    this.startPromise = null;
+    this.startGeneration += 1;
+  }
+
+  resetStreamForReplay(): void {
+    this.dictationId = null;
+    this.sendSeq = 0;
+    this.streamReady = false;
+    this.startPromise = null;
+    this.startGeneration += 1;
+  }
+
+  enqueueSegment(base64Pcm: string): void {
+    this.segments.push(base64Pcm);
+
+    const client = this.client;
+    if (!client?.isConnected) {
+      return;
+    }
+
+    if (!this.dictationId) {
+      void this.restartStream("enqueue");
+      return;
+    }
+
+    this.flush();
+  }
+
+  flush(): number {
+    const client = this.client;
+    const dictationId = this.dictationId;
+    if (!client?.isConnected || !dictationId || !this.streamReady) {
+      return 0;
+    }
+
+    let sent = 0;
+    while (this.sendSeq < this.segments.length) {
+      const seq = this.sendSeq;
+      const audio = this.segments[seq]!;
+      client.sendDictationStreamChunk(dictationId, seq, audio, this.format);
+      this.sendSeq = seq + 1;
+      sent += 1;
+    }
+    return sent;
+  }
+
+  async restartStream(reason: string): Promise<void> {
+    const client = this.client;
+    if (!client?.isConnected) {
+      return;
+    }
+
+    this.startGeneration += 1;
+    const generation = this.startGeneration;
+
+    const dictationId = this.createDictationId();
+    this.dictationId = dictationId;
+    this.sendSeq = 0;
+    this.streamReady = false;
+
+    const start = (async () => {
+      await client.startDictationStream(dictationId, this.format);
+      if (this.startGeneration !== generation) {
+        return;
+      }
+      if (this.dictationId !== dictationId) {
+        return;
+      }
+      this.streamReady = true;
+      this.flush();
+    })().catch((error) => {
+      // If starting failed, keep the segments for retry but clear the stream so finish can error cleanly.
+      if (this.startGeneration === generation && this.dictationId === dictationId) {
+        this.dictationId = null;
+        this.streamReady = false;
+      }
+      throw error;
+    }).finally(() => {
+      if (this.startPromise === start) {
+        this.startPromise = null;
+      }
+    });
+
+    this.startPromise = start;
+    await start;
+    void reason;
+  }
+
+  async finish(finalSeq: number): Promise<DictationFinishResult> {
+    const client = this.client;
+    if (!client) {
+      throw new Error("Daemon client unavailable");
+    }
+    if (!client.isConnected) {
+      throw new Error("Daemon client is disconnected");
+    }
+
+    if (!this.dictationId) {
+      await this.restartStream("finalize");
+    }
+    if (this.startPromise) {
+      await this.startPromise;
+    }
+
+    const dictationId = this.dictationId;
+    if (!dictationId || !this.streamReady) {
+      throw new Error("Failed to start dictation stream");
+    }
+
+    this.flush();
+    return client.finishDictationStream(dictationId, finalSeq);
+  }
+
+  cancel(): void {
+    const client = this.client;
+    const dictationId = this.dictationId;
+    if (client?.isConnected && dictationId) {
+      client.cancelDictationStream(dictationId);
+    }
+    this.resetStreamForReplay();
+  }
+}

packages/app/src/hooks/use-agent-form-state.ts

@@ -251,6 +251,10 @@ export function useAgentFormState(
     model: "",
     workingDir: "",
   }));
+  const formStateRef = useRef(formState);
+  useEffect(() => {
+    formStateRef.current = formState;
+  }, [formState]);
 
   // Track if we've done initial resolution (to avoid flickering)
   const hasResolvedRef = useRef(false);
@@ -299,16 +303,16 @@ export function useAgentFormState(
       preferences,
       availableModels,
       userModified,
-      formState
+      formStateRef.current
     );
 
     // Only update if something changed
     if (
-      resolved.serverId !== formState.serverId ||
+      resolved.serverId !== formStateRef.current.serverId ||
-      resolved.provider !== formState.provider ||
+      resolved.provider !== formStateRef.current.provider ||
-      resolved.modeId !== formState.modeId ||
+      resolved.modeId !== formStateRef.current.modeId ||
-      resolved.model !== formState.model ||
+      resolved.model !== formStateRef.current.model ||
-      resolved.workingDir !== formState.workingDir
+      resolved.workingDir !== formStateRef.current.workingDir
     ) {
       setFormState(resolved);
     }
@@ -322,7 +326,6 @@ export function useAgentFormState(
     preferences,
     availableModels,
     userModified,
-    formState,
   ]);
 
   // Persist inferred serverId so reloads keep the selection (e.g. URL serverId or first-time load).

packages/app/src/hooks/use-dictation-audio-source.d.ts

@@ -0,0 +1,2 @@
+export * from "./use-dictation-audio-source.native";
+

packages/app/src/hooks/use-dictation-audio-source.native.ts

@@ -0,0 +1,44 @@
+import { useCallback, useEffect, useRef } from "react";
+
+import { useSpeechmaticsAudio } from "@/hooks/use-speechmatics-audio";
+
+import type { DictationAudioSource, DictationAudioSourceConfig } from "./use-dictation-audio-source.types";
+
+export function useDictationAudioSource(config: DictationAudioSourceConfig): DictationAudioSource {
+  const onPcmSegmentRef = useRef(config.onPcmSegment);
+  const onErrorRef = useRef(config.onError);
+
+  useEffect(() => {
+    onPcmSegmentRef.current = config.onPcmSegment;
+    onErrorRef.current = config.onError;
+  }, [config.onPcmSegment, config.onError]);
+
+  const speechmatics = useSpeechmaticsAudio({
+    enableContinuousStreaming: true,
+    onAudioSegment: ({ audioData }) => {
+      onPcmSegmentRef.current(audioData);
+    },
+    onError: (err) => {
+      onErrorRef.current?.(err);
+    },
+    volumeThreshold: 0.3,
+    silenceDuration: 2000,
+    speechConfirmationDuration: 300,
+    detectionGracePeriod: 200,
+  });
+
+  const start = useCallback(async () => {
+    await speechmatics.start();
+  }, [speechmatics]);
+
+  const stop = useCallback(async () => {
+    await speechmatics.stop();
+  }, [speechmatics]);
+
+  return {
+    start,
+    stop,
+    volume: speechmatics.volume,
+  };
+}
+

packages/app/src/hooks/use-dictation-audio-source.types.ts

@@ -0,0 +1,11 @@
+export type DictationAudioSourceConfig = {
+  onPcmSegment: (pcm16Base64: string) => void;
+  onError?: (error: Error) => void;
+};
+
+export type DictationAudioSource = {
+  start: () => Promise<void>;
+  stop: () => Promise<void>;
+  volume: number;
+};
+

packages/app/src/hooks/use-dictation-audio-source.web.ts

@@ -0,0 +1,247 @@
+import { useCallback, useEffect, useMemo, useRef, useState } from "react";
+
+import type { DictationAudioSource, DictationAudioSourceConfig } from "./use-dictation-audio-source.types";
+
+const getAudioContextCtor = (): (typeof AudioContext) | null => {
+  if (typeof window === "undefined") {
+    return null;
+  }
+  const ctor =
+    (window as typeof window & { webkitAudioContext?: typeof AudioContext }).AudioContext ||
+    (window as typeof window & { webkitAudioContext?: typeof AudioContext }).webkitAudioContext;
+  return ctor ?? null;
+};
+
+const floatToInt16 = (sample: number): number => {
+  const clamped = Math.max(-1, Math.min(1, sample));
+  return clamped < 0 ? Math.round(clamped * 0x8000) : Math.round(clamped * 0x7fff);
+};
+
+const resampleToPcm16 = (input: Float32Array, inputRate: number, outputRate: number): Int16Array => {
+  if (input.length === 0) {
+    return new Int16Array(0);
+  }
+  if (inputRate === outputRate) {
+    const out = new Int16Array(input.length);
+    for (let i = 0; i < input.length; i++) {
+      out[i] = floatToInt16(input[i]);
+    }
+    return out;
+  }
+
+  const ratio = inputRate / outputRate;
+  const outputLength = Math.max(1, Math.round(input.length / ratio));
+  const out = new Int16Array(outputLength);
+  for (let i = 0; i < outputLength; i++) {
+    const sourceIndex = i * ratio;
+    const i0 = Math.floor(sourceIndex);
+    const i1 = Math.min(input.length - 1, i0 + 1);
+    const frac = sourceIndex - i0;
+    const sample = input[i0] * (1 - frac) + input[i1] * frac;
+    out[i] = floatToInt16(sample);
+  }
+  return out;
+};
+
+const concatInt16 = (a: Int16Array, b: Int16Array): Int16Array => {
+  if (a.length === 0) {
+    return b;
+  }
+  if (b.length === 0) {
+    return a;
+  }
+  const out = new Int16Array(a.length + b.length);
+  out.set(a, 0);
+  out.set(b, a.length);
+  return out;
+};
+
+const int16ToBase64 = (pcm: Int16Array): string => {
+  const bytes = new Uint8Array(pcm.buffer, pcm.byteOffset, pcm.byteLength);
+  let binary = "";
+  for (let i = 0; i < bytes.length; i++) {
+    binary += String.fromCharCode(bytes[i]);
+  }
+  return btoa(binary);
+};
+
+export function useDictationAudioSource(config: DictationAudioSourceConfig): DictationAudioSource {
+  const [volume, setVolume] = useState(0);
+
+  const onPcmSegmentRef = useRef(config.onPcmSegment);
+  const onErrorRef = useRef(config.onError);
+
+  useEffect(() => {
+    onPcmSegmentRef.current = config.onPcmSegment;
+    onErrorRef.current = config.onError;
+  }, [config.onPcmSegment, config.onError]);
+
+  const refs = useRef<{
+    stream: MediaStream | null;
+    context: AudioContext | null;
+    source: MediaStreamAudioSourceNode | null;
+    processor: ScriptProcessorNode | null;
+    gain: GainNode | null;
+    pending: Int16Array;
+    started: boolean;
+  }>({ stream: null, context: null, source: null, processor: null, gain: null, pending: new Int16Array(0), started: false });
+
+  const start = useCallback(async () => {
+    const missingNavigator =
+      typeof navigator === "undefined" ||
+      !navigator.mediaDevices ||
+      typeof navigator.mediaDevices.getUserMedia !== "function";
+
+    const secureContext =
+      typeof window !== "undefined" && typeof window.isSecureContext === "boolean"
+        ? window.isSecureContext
+        : true;
+    const currentOrigin =
+      typeof window !== "undefined" && window.location ? window.location.origin : "unknown";
+
+    if (missingNavigator) {
+      throw new Error("Microphone capture is not supported in this environment");
+    }
+    if (!secureContext) {
+      throw new Error(`Microphone access requires HTTPS or localhost. Current origin: ${currentOrigin}`);
+    }
+
+    const AudioContextCtor = getAudioContextCtor();
+    if (!AudioContextCtor) {
+      throw new Error("AudioContext unavailable");
+    }
+
+    const stream = await navigator.mediaDevices.getUserMedia({
+      audio: {
+        channelCount: 1,
+        noiseSuppression: true,
+        echoCancellation: true,
+        autoGainControl: true,
+      },
+    });
+
+    const context = new AudioContextCtor();
+    const source = context.createMediaStreamSource(stream);
+    const processor = context.createScriptProcessor(4096, 1, 1);
+    const gain = context.createGain();
+    gain.gain.value = 0;
+
+    const outputRate = 16000;
+    const chunkSamples = outputRate; // ~1s
+
+    refs.current.started = true;
+
+    processor.onaudioprocess = (event) => {
+      if (!refs.current.started) {
+        return;
+      }
+      const input = event.inputBuffer.getChannelData(0);
+
+      let sumSquares = 0;
+      for (let i = 0; i < input.length; i++) {
+        const sample = input[i];
+        sumSquares += sample * sample;
+      }
+      const rms = Math.sqrt(sumSquares / Math.max(1, input.length));
+      const normalized = Math.min(1, Math.max(0, rms * 2));
+      setVolume(normalized);
+
+      const next = resampleToPcm16(input, context.sampleRate, outputRate);
+      refs.current.pending = concatInt16(refs.current.pending, next);
+
+      while (refs.current.pending.length >= chunkSamples) {
+        const chunk = refs.current.pending.slice(0, chunkSamples);
+        refs.current.pending = refs.current.pending.slice(chunkSamples);
+        onPcmSegmentRef.current(int16ToBase64(chunk));
+      }
+    };
+
+    source.connect(processor);
+    processor.connect(gain);
+    gain.connect(context.destination);
+
+    refs.current = { stream, context, source, processor, gain, pending: new Int16Array(0), started: true };
+  }, []);
+
+  const stop = useCallback(async () => {
+    refs.current.started = false;
+    setVolume(0);
+
+    const { processor, source, gain, context, stream, pending } = refs.current;
+
+    if (processor) {
+      try {
+        processor.onaudioprocess = null;
+      } catch {
+        // no-op
+      }
+      try {
+        processor.disconnect();
+      } catch {
+        // no-op
+      }
+    }
+    if (source) {
+      try {
+        source.disconnect();
+      } catch {
+        // no-op
+      }
+    }
+    if (gain) {
+      try {
+        gain.disconnect();
+      } catch {
+        // no-op
+      }
+    }
+    if (stream) {
+      stream.getTracks().forEach((track) => {
+        try {
+          track.stop();
+        } catch {
+          // no-op
+        }
+      });
+    }
+    if (context) {
+      try {
+        await context.close();
+      } catch {
+        // no-op
+      }
+    }
+
+    if (pending.length > 0) {
+      onPcmSegmentRef.current(int16ToBase64(pending));
+    }
+
+    refs.current = { stream: null, context: null, source: null, processor: null, gain: null, pending: new Int16Array(0), started: false };
+  }, []);
+
+  useEffect(() => {
+    return () => {
+      void stop().catch((err) => {
+        onErrorRef.current?.(err instanceof Error ? err : new Error(String(err)));
+      });
+    };
+  }, [stop]);
+
+  return useMemo(
+    () => ({
+      start: async () => {
+        try {
+          await start();
+        } catch (err) {
+          const normalized = err instanceof Error ? err : new Error(String(err));
+          onErrorRef.current?.(normalized);
+          throw normalized;
+        }
+      },
+      stop,
+      volume,
+    }),
+    [start, stop, volume]
+  );
+}
+

packages/app/src/hooks/use-dictation.d.ts

@@ -0,0 +1 @@
+export * from "./use-dictation";

packages/app/src/hooks/use-dictation.shared.ts

@@ -0,0 +1,40 @@
+export type DictationStatus = "idle" | "recording" | "uploading" | "failed";
+
+export type UseDictationOptions = {
+  client: import("@server/client/daemon-client-v2").DaemonClientV2 | null;
+  onTranscript: (text: string, meta: { requestId: string }) => void;
+  onError?: (error: Error) => void;
+  onPermanentFailure?: (error: Error, context: { requestId: string }) => void;
+  canStart?: () => boolean;
+  canConfirm?: () => boolean;
+  autoStopWhenHidden?: { isVisible: boolean };
+  enableDuration?: boolean;
+};
+
+export type UseDictationResult = {
+  isRecording: boolean;
+  isProcessing: boolean;
+  volume: number;
+  duration: number;
+  error: string | null;
+  status: DictationStatus;
+  startDictation: () => Promise<void>;
+  cancelDictation: () => Promise<void>;
+  confirmDictation: () => Promise<void>;
+  retryFailedDictation: () => Promise<void>;
+  discardFailedDictation: () => void;
+  reset: () => void;
+};
+
+export const DURATION_TICK_MS = 1000;
+export const PCM_DICTATION_FORMAT = "audio/pcm;rate=16000;bits=16";
+
+export const toError = (error: unknown): Error => {
+  if (error instanceof Error) {
+    return error;
+  }
+  if (typeof error === "string" && error.trim().length > 0) {
+    return new Error(error);
+  }
+  return new Error("An unexpected error occurred while handling dictation.");
+};

packages/app/src/hooks/use-dictation.ts

@@ -1,160 +1,23 @@
 import { useCallback, useEffect, useRef, useState } from "react";
-import { useMutation } from "@tanstack/react-query";
 
-import { useAudioRecorder } from "@/hooks/use-audio-recorder";
+import { DictationStreamSender } from "@/dictation/dictation-stream-sender";
-import type { DaemonClientV2 } from "@server/client/daemon-client-v2";
+import { useDictationAudioSource } from "@/hooks/use-dictation-audio-source";
-import type { TranscriptionResultMessage } from "@server/shared/messages";
 import { generateMessageId } from "@/types/stream";
 import { AttemptGuard } from "@/utils/attempt-guard";
-
+import {
-export type DictationStatus = "idle" | "recording" | "uploading" | "retrying" | "failed";
+  DURATION_TICK_MS,
-
+  PCM_DICTATION_FORMAT,
-type DictationRetryReason = "dispatch" | "timeout" | "response" | "disconnected";
+  toError,
-
+  type DictationStatus,
-export type DictationRetryInfo = {
+  type UseDictationOptions,
-  attempt: number;
+  type UseDictationResult,
-  maxAttempts: number;
+} from "./use-dictation.shared";
-  reason: DictationRetryReason;
-  errorMessage: string;
-  nextRetryMs: number;
-};
-
-export type FailedDictationRecording = {
-  requestId: string;
-  durationSeconds: number;
-  sizeBytes: number;
-  format: string;
-  recordedAt: number;
-  errorMessage: string;
-};
-
-export type DictationOutcome =
-  | { type: "success"; requestId: string; timestamp: number }
-  | { type: "failure"; requestId: string; errorMessage: string; timestamp: number };
-
-export type UseDictationOptions = {
-  client: DaemonClientV2 | null;
-  onTranscript: (text: string, meta: { requestId: string }) => void;
-  onError?: (error: Error) => void;
-  onRetryAttempt?: (info: DictationRetryInfo) => void;
-  onPermanentFailure?: (error: Error, context: { requestId: string }) => void;
-  canStart?: () => boolean;
-  canConfirm?: () => boolean;
-  autoStopWhenHidden?: { isVisible: boolean };
-  enableDuration?: boolean;
-};
-
-export type UseDictationResult = {
-  isRecording: boolean;
-  isProcessing: boolean;
-  volume: number;
-  duration: number;
-  pendingRequestId: string | null;
-  error: string | null;
-  status: DictationStatus;
-  retryAttempt: number;
-  maxRetryAttempts: number;
-  retryInfo: DictationRetryInfo | null;
-  failedRecording: FailedDictationRecording | null;
-  lastOutcome: DictationOutcome | null;
-  startDictation: () => Promise<void>;
-  cancelDictation: () => Promise<void>;
-  confirmDictation: () => Promise<void>;
-  retryFailedDictation: () => Promise<void>;
-  discardFailedDictation: () => void;
-  reset: () => void;
-};
-
-const DURATION_TICK_MS = 1000;
-const MAX_AUTO_RETRY_ATTEMPTS = 5;
-const RETRY_BASE_DELAY_MS = 2000;
-const RETRY_MAX_DELAY_MS = 12000;
-const RETRY_BACKOFF_FACTOR = 1.8;
-const RETRY_JITTER_MS = 400;
-const TRANSCRIPTION_TIMEOUT_MS = 120000;
-
-const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
-
-const computeRetryDelayMs = (attempt: number): number => {
-  const exponential = RETRY_BASE_DELAY_MS * RETRY_BACKOFF_FACTOR ** Math.max(0, attempt - 1);
-  const capped = Math.min(RETRY_MAX_DELAY_MS, exponential);
-  if (RETRY_JITTER_MS <= 0) {
-    return capped;
-  }
-  return capped + Math.floor(Math.random() * RETRY_JITTER_MS);
-};
-
-const isTimeoutError = (error: Error): boolean =>
-  /timed out|timeout/i.test(error.message);
-
-class DictationAttemptError extends Error {
-  public readonly reason: DictationRetryReason;
-
-  constructor(reason: DictationRetryReason, error: Error) {
-    super(error.message);
-    this.name = "DictationAttemptError";
-    this.reason = reason;
-    (this as Error & { cause?: Error }).cause = error;
-  }
-}
-
-const toError = (error: unknown): Error => {
-  if (error instanceof Error) {
-    return error;
-  }
-  if (typeof error === "string" && error.trim().length > 0) {
-    return new Error(error);
-  }
-  return new Error("An unexpected error occurred while handling dictation.");
-};
-
-type CapturedAudioPayload = {
-  blob: Blob;
-  format: string;
-  sizeBytes: number;
-  durationSeconds: number;
-  recordedAt: number;
-};
-
-type TranscriptionPayload = TranscriptionResultMessage["payload"];
-
-const blobToBase64 = async (blob: Blob): Promise<string> => {
-  const arrayBuffer = await blob.arrayBuffer();
-  const bytes = new Uint8Array(arrayBuffer);
-  let binary = "";
-  for (let i = 0; i < bytes.length; i++) {
-    binary += String.fromCharCode(bytes[i]);
-  }
-  return btoa(binary);
-};
-
-const deriveFormatFromMime = (mimeType?: string): string => {
-  if (!mimeType || mimeType.length === 0) {
-    return "webm";
-  }
-  const slashIndex = mimeType.indexOf("/");
-  let formatPart = slashIndex >= 0 ? mimeType.slice(slashIndex + 1) : mimeType;
-  const semicolonIndex = formatPart.indexOf(";");
-  if (semicolonIndex >= 0) {
-    formatPart = formatPart.slice(0, semicolonIndex);
-  }
-  return formatPart.trim().length > 0 ? formatPart.trim() : "webm";
-};
-
-const buildCapturedAudioPayload = (blob: Blob, durationSeconds: number): CapturedAudioPayload => ({
-  blob,
-  format: deriveFormatFromMime(blob.type),
-  sizeBytes: typeof blob.size === "number" ? blob.size : 0,
-  durationSeconds,
-  recordedAt: Date.now(),
-});
 
 export function useDictation(options: UseDictationOptions): UseDictationResult {
   const {
     client,
     onTranscript,
     onError,
-    onRetryAttempt,
     onPermanentFailure,
     canStart,
     canConfirm,
@@ -164,127 +27,9 @@ export function useDictation(options: UseDictationOptions): UseDictationResult {
 
   const [isRecording, setIsRecording] = useState(false);
   const [isProcessing, setIsProcessing] = useState(false);
-  const [volume, setVolume] = useState(0);
   const [duration, setDuration] = useState(0);
-  const [pendingRequestId, setPendingRequestId] = useState<string | null>(null);
   const [error, setError] = useState<string | null>(null);
   const [status, setStatus] = useState<DictationStatus>("idle");
-  const [retryAttempt, setRetryAttempt] = useState(0);
-  const [retryInfo, setRetryInfo] = useState<DictationRetryInfo | null>(null);
-  const [failedRecording, setFailedRecording] = useState<FailedDictationRecording | null>(null);
-  const [lastOutcome, setLastOutcome] = useState<DictationOutcome | null>(null);
-  const maxRetryAttempts = MAX_AUTO_RETRY_ATTEMPTS;
-
-  const transcriptionMutation = useMutation({
-    mutationFn: async (): Promise<TranscriptionPayload> => {
-      const capturedAudio = pendingAudioRef.current;
-      if (!capturedAudio) {
-        throw new Error("No recorded audio available for transcription");
-      }
-
-      setRetryAttempt(1);
-      setRetryInfo(null);
-
-      let attempt = 1;
-
-      while (attempt <= MAX_AUTO_RETRY_ATTEMPTS) {
-        try {
-          if (!client) {
-            throw new DictationAttemptError(
-              "disconnected",
-              new Error("Daemon client unavailable")
-            );
-          }
-          if (!client.isConnected) {
-            throw new DictationAttemptError(
-              "disconnected",
-              new Error("Daemon client is disconnected")
-            );
-          }
-
-          console.info("[useDictation] sending transcription request", {
-            attempt,
-            size: capturedAudio.sizeBytes,
-            durationSeconds: capturedAudio.durationSeconds,
-          });
-
-          setStatus("uploading");
-          setRetryAttempt(attempt);
-
-          try {
-            const base64Audio = await blobToBase64(capturedAudio.blob);
-            return await client.transcribeAudio({
-              audio: base64Audio,
-              format: capturedAudio.format,
-              timeout: TRANSCRIPTION_TIMEOUT_MS,
-            });
-          } catch (error) {
-            const normalized = toError(error);
-            const reason: DictationRetryReason = isTimeoutError(normalized)
-              ? "timeout"
-              : "response";
-            throw new DictationAttemptError(reason, normalized);
-          }
-        } catch (error) {
-          const attemptError =
-            error instanceof DictationAttemptError
-              ? error
-              : new DictationAttemptError("response", toError(error));
-
-          if (attempt >= MAX_AUTO_RETRY_ATTEMPTS) {
-            throw attemptError;
-          }
-
-          const nextAttempt = attempt + 1;
-          const delayMs = computeRetryDelayMs(nextAttempt);
-          const info: DictationRetryInfo = {
-            attempt: nextAttempt,
-            maxAttempts: MAX_AUTO_RETRY_ATTEMPTS,
-            reason: attemptError.reason,
-            errorMessage: attemptError.message,
-            nextRetryMs: delayMs,
-          };
-
-          setStatus("retrying");
-          setRetryAttempt(nextAttempt);
-          setRetryInfo(info);
-          onRetryAttemptRef.current?.(info);
-          console.warn("[useDictation] retry scheduled", {
-            attempt: nextAttempt,
-            maxAttempts: MAX_AUTO_RETRY_ATTEMPTS,
-            reason: attemptError.reason,
-            error: attemptError.message,
-            nextRetryMs: delayMs,
-          });
-
-          await sleep(delayMs);
-          attempt = nextAttempt;
-        }
-      }
-
-      throw new Error("Failed to complete transcription");
-    },
-  });
-  const {
-    mutateAsync: runTranscription,
-    reset: resetTranscriptionMutation,
-  } = transcriptionMutation;
-
-  const pendingAudioRef = useRef<CapturedAudioPayload | null>(null);
-  const handleAudioLevel = useCallback((level: number) => {
-    setVolume(level);
-  }, []);
-
-  const durationRef = useRef(0);
-  useEffect(() => {
-    durationRef.current = duration;
-  }, [duration]);
-
-  const recorder = useAudioRecorder({ onAudioLevel: handleAudioLevel });
-  const recorderRef = useRef(recorder);
-  useEffect(() => {
-    recorderRef.current = recorder;
-  }, [recorder]);
 
   const onTranscriptRef = useRef(onTranscript);
   useEffect(() => {
@@ -296,11 +41,6 @@ export function useDictation(options: UseDictationOptions): UseDictationResult {
     onErrorRef.current = onError;
   }, [onError]);
 
-  const onRetryAttemptRef = useRef(onRetryAttempt);
-  useEffect(() => {
-    onRetryAttemptRef.current = onRetryAttempt;
-  }, [onRetryAttempt]);
-
   const onPermanentFailureRef = useRef(onPermanentFailure);
   useEffect(() => {
     onPermanentFailureRef.current = onPermanentFailure;
@@ -316,10 +56,27 @@ export function useDictation(options: UseDictationOptions): UseDictationResult {
     isProcessingRef.current = isProcessing;
   }, [isProcessing]);
 
-  const pendingRequestIdRef = useRef<string | null>(null);
+  // duration is used for UI only; no need to mirror into a ref.
-  const activeStopPromiseRef = useRef<Promise<Blob> | null>(null);
+
   const durationIntervalRef = useRef<ReturnType<typeof setInterval> | null>(null);
   const attemptGuardRef = useRef(new AttemptGuard());
+  const actionGateRef = useRef<{ starting: boolean; confirming: boolean; cancelling: boolean }>({
+    starting: false,
+    confirming: false,
+    cancelling: false,
+  });
+
+  const senderRef = useRef<DictationStreamSender | null>(null);
+  if (!senderRef.current) {
+    senderRef.current = new DictationStreamSender({
+      client,
+      format: PCM_DICTATION_FORMAT,
+      createDictationId: generateMessageId,
+    });
+  }
+  useEffect(() => {
+    senderRef.current?.setClient(client);
+  }, [client]);
 
   const stopDurationTracking = useCallback(() => {
     if (durationIntervalRef.current) {
@@ -330,7 +87,6 @@ export function useDictation(options: UseDictationOptions): UseDictationResult {
 
   const startDurationTracking = useCallback(() => {
     if (!enableDuration) {
-      console.log("[useDictation] startDictation blocked by canStart()");
       return;
     }
     if (durationIntervalRef.current) {
@@ -351,6 +107,9 @@ export function useDictation(options: UseDictationOptions): UseDictationResult {
   const reportError = useCallback(
     (err: unknown, context?: string) => {
       const normalized = toError(err);
+      if (normalized.name === "AttemptCancelledError") {
+        return;
+      }
       if (context) {
         console.error(`[useDictation] ${context}`, normalized);
       } else {
@@ -362,136 +121,114 @@ export function useDictation(options: UseDictationOptions): UseDictationResult {
     [setError]
   );
 
-  const stopRecorder = useCallback(async (): Promise<Blob> => {
+  const clearStreamingState = useCallback(() => {
-    if (activeStopPromiseRef.current) {
+    senderRef.current?.clearAll();
-      return activeStopPromiseRef.current;
-    }
-    const recorderInstance = recorderRef.current;
-    if (!recorderInstance) {
-      throw new Error("Recorder unavailable");
-    }
-    const stopPromise = (async () => {
-      try {
-        return await recorderInstance.stop();
-      } finally {
-        activeStopPromiseRef.current = null;
-      }
-    })();
-    activeStopPromiseRef.current = stopPromise;
-    return stopPromise;
   }, []);
 
-  const transmitDictation = useCallback(
+  const startNewStream = useCallback(
-    async () => runTranscription(),
+    async (reason: string) => {
-    [runTranscription]
+      await senderRef.current?.restartStream(reason);
+    },
+    []
   );
 
-  const handleTranscriptionSuccess = useCallback(
+  const ensureFinalTranscript = useCallback(
-    (transcription: TranscriptionPayload) => {
+    async (finalSeq: number): Promise<string> => {
-      const requestId = transcription.requestId;
+      const result = await senderRef.current!.finish(finalSeq);
-      pendingRequestIdRef.current = null;
+      return result.text;
-      setPendingRequestId(null);
+    },
+    []
+  );
+
+  useEffect(() => {
+    if (!client) {
+      return;
+    }
+    return client.subscribeConnectionStatus((next) => {
+      if (next.status !== "connected") {
+        return;
+      }
+      if (!isRecordingRef.current) {
+        return;
+      }
+      void startNewStream("reconnect");
+    });
+  }, [client, startNewStream]);
+
+  const audio = useDictationAudioSource({
+    onPcmSegment: (audioData) => {
+      senderRef.current?.enqueueSegment(audioData);
+    },
+    onError: (err) => {
+      onErrorRef.current?.(err);
+    },
+  });
+  const audioStopRef = useRef(audio.stop);
+  useEffect(() => {
+    audioStopRef.current = audio.stop;
+  }, [audio.stop]);
+
+  const handleStreamingTranscriptionSuccess = useCallback(
+    (text: string, requestId: string) => {
       setIsProcessing(false);
       setDuration(0);
       setStatus("idle");
-      setRetryAttempt(0);
-      setRetryInfo(null);
-      setFailedRecording(null);
-      pendingAudioRef.current = null;
-      setLastOutcome({ type: "success", requestId, timestamp: Date.now() });
 
-      const transcriptText = transcription.text?.trim();
+      clearStreamingState();
+
+      const transcriptText = text.trim();
       if (!transcriptText) {
         return;
       }
-
-      console.log("[useDictation] transcription_result received", {
-        requestId,
-        textLength: transcriptText.length,
-      });
       onTranscriptRef.current?.(transcriptText, { requestId });
     },
-    [onTranscriptRef]
+    [clearStreamingState]
   );
 
   const handleDictationFailure = useCallback(
     (failure: unknown) => {
       const normalized = toError(failure);
       const failureId = generateMessageId();
-      pendingRequestIdRef.current = null;
-      setPendingRequestId(null);
       setIsProcessing(false);
       isRecordingRef.current = false;
       setIsRecording(false);
-      setVolume(0);
-      setRetryInfo(null);
 
-      const capturedAudio = pendingAudioRef.current;
+      if (senderRef.current?.hasSegments()) {
-      if (capturedAudio) {
         setStatus("failed");
-        setFailedRecording({
-          requestId: failureId,
-          durationSeconds: capturedAudio.durationSeconds,
-          sizeBytes: capturedAudio.sizeBytes,
-          format: capturedAudio.format,
-          recordedAt: capturedAudio.recordedAt,
-          errorMessage: normalized.message,
-        });
         onPermanentFailureRef.current?.(normalized, { requestId: failureId });
       } else {
         setStatus("idle");
       }
 
-      setRetryAttempt(0);
-      setLastOutcome({
-        type: "failure",
-        requestId: failureId,
-        errorMessage: normalized.message,
-        timestamp: Date.now(),
-      });
       reportError(normalized, "Failed to complete dictation");
     },
-    [onPermanentFailureRef, reportError]
+    [reportError]
   );
 
   const startDictation = useCallback(async () => {
-    console.log("[useDictation] startDictation requested", {
+    if (actionGateRef.current.starting || actionGateRef.current.confirming || actionGateRef.current.cancelling) {
-      isRecording: isRecordingRef.current,
+      return;
-      isProcessing,
+    }
-    });
+    if (isRecordingRef.current || isProcessingRef.current) {
-    if (isRecordingRef.current || isProcessing) {
-      console.log("[useDictation] startDictation aborted: already recording/processing", {
-        isRecording: isRecordingRef.current,
-        isProcessing,
-      });
       return;
     }
     const startAllowed = canStart ? canStart() : true;
     if (!startAllowed) {
-      console.log("[useDictation] startDictation blocked by canStart()");
       return;
     }
 
+    actionGateRef.current.starting = true;
     setError(null);
-    setVolume(0);
     setDuration(0);
     setIsProcessing(false);
     setStatus("recording");
-    setRetryAttempt(0);
+    clearStreamingState();
-    setRetryInfo(null);
-    setFailedRecording(null);
-    pendingAudioRef.current = null;
-    setLastOutcome(null);
-    pendingRequestIdRef.current = null;
-    setPendingRequestId(null);
 
     try {
-      const recorderInstance = recorderRef.current;
+      await audio.start();
-      if (!recorderInstance) {
+      if (client?.isConnected) {
-        throw new Error("Recorder unavailable");
+        void startNewStream("start");
       }
-      await recorderInstance.start();
-      console.log("[useDictation] recorder.start succeeded");
       isRecordingRef.current = true;
       setIsRecording(true);
       if (enableDuration) {
@@ -502,160 +239,155 @@ export function useDictation(options: UseDictationOptions): UseDictationResult {
       isRecordingRef.current = false;
       setIsRecording(false);
       reportError(err, "Failed to start dictation");
+    } finally {
+      actionGateRef.current.starting = false;
     }
   }, [
+    audio,
     canStart,
+    clearStreamingState,
+    client,
     enableDuration,
     isProcessing,
     reportError,
     startDurationTracking,
+    startNewStream,
     stopDurationTracking,
   ]);
 
   const cancelDictation = useCallback(async () => {
-    console.log("[useDictation] cancelDictation requested", {
-      isRecording: isRecordingRef.current,
-      hasActiveStop: Boolean(activeStopPromiseRef.current),
-    });
     attemptGuardRef.current.cancel();
-    if (!isRecordingRef.current && !activeStopPromiseRef.current) {
+    if (actionGateRef.current.cancelling) {
-      console.log("[useDictation] cancelDictation ignored: nothing to cancel");
       return;
     }
+    if (!isRecordingRef.current && !isProcessingRef.current) {
+      return;
+    }
+    actionGateRef.current.cancelling = true;
     stopDurationTracking();
     setDuration(0);
     setError(null);
 
     try {
-      await stopRecorder();
+      try {
+        senderRef.current?.cancel();
+      } catch {
+        // no-op
+      }
+      await audio.stop();
     } catch (err) {
       reportError(err, "Failed to cancel dictation");
     } finally {
       isRecordingRef.current = false;
       setIsRecording(false);
       setIsProcessing(false);
-      setVolume(0);
+      isProcessingRef.current = false;
       setStatus("idle");
-      setRetryAttempt(0);
+      clearStreamingState();
-      setRetryInfo(null);
+      actionGateRef.current.cancelling = false;
-      pendingAudioRef.current = null;
-      setFailedRecording(null);
-      setLastOutcome(null);
     }
-  }, [reportError, stopDurationTracking, stopRecorder]);
+  }, [audio, clearStreamingState, client, reportError, stopDurationTracking]);
 
   const confirmDictation = useCallback(async () => {
-    console.log("[useDictation] confirmDictation requested", {
+    if (actionGateRef.current.confirming) {
-      isRecording: isRecordingRef.current,
+      return;
-      isProcessing,
+    }
-    });
+    if (!isRecordingRef.current || isProcessingRef.current) {
-    if (!isRecordingRef.current || isProcessing) {
-      console.log("[useDictation] confirmDictation ignored: recording flag", {
-        isRecording: isRecordingRef.current,
-        isProcessing,
-      });
       return;
     }
     const confirmAllowed = canConfirm ? canConfirm() : true;
     if (!confirmAllowed) {
-      console.log("[useDictation] confirmDictation blocked by canConfirm()");
       return;
     }
 
+    actionGateRef.current.confirming = true;
     setError(null);
     stopDurationTracking();
     setIsProcessing(true);
-    setRetryInfo(null);
+    isProcessingRef.current = true;
-    setRetryAttempt(0);
-    setLastOutcome(null);
 
     const attemptId = attemptGuardRef.current.next();
 
     try {
-      const audioData = await stopRecorder();
+      await audio.stop();
       attemptGuardRef.current.assertCurrent(attemptId);
-      const recordedDurationSeconds = durationRef.current;
+
-      pendingAudioRef.current = buildCapturedAudioPayload(audioData, recordedDurationSeconds);
       setStatus("uploading");
       isRecordingRef.current = false;
       setIsRecording(false);
-      setVolume(0);
 
-      const transcription = await transmitDictation();
+      const finalSeq = senderRef.current?.getFinalSeq() ?? -1;
+      if (finalSeq < 0) {
+        handleStreamingTranscriptionSuccess("", generateMessageId());
+        return;
+      }
+
+      const transcriptText = await ensureFinalTranscript(finalSeq);
       attemptGuardRef.current.assertCurrent(attemptId);
-      handleTranscriptionSuccess(transcription);
+      handleStreamingTranscriptionSuccess(transcriptText, generateMessageId());
     } catch (err) {
-      resetTranscriptionMutation();
+      if (err instanceof Error && err.name === "AttemptCancelledError") {
+        return;
+      }
       handleDictationFailure(err);
+    } finally {
+      actionGateRef.current.confirming = false;
     }
   }, [
+    audio,
     canConfirm,
     isProcessing,
     handleDictationFailure,
-    handleTranscriptionSuccess,
+    handleStreamingTranscriptionSuccess,
-    resetTranscriptionMutation,
     stopDurationTracking,
-    stopRecorder,
+    ensureFinalTranscript,
-    transmitDictation,
   ]);
 
   const retryFailedDictation = useCallback(async () => {
-    if (!pendingAudioRef.current) {
+    if (!senderRef.current?.hasSegments()) {
       return;
     }
     setError(null);
-    setRetryInfo(null);
-    setRetryAttempt(0);
     setStatus("uploading");
     setIsProcessing(true);
-    setLastOutcome(null);
+    isProcessingRef.current = true;
 
     try {
-      const transcription = await transmitDictation();
+      if (!client?.isConnected) {
-      handleTranscriptionSuccess(transcription);
+        throw new Error("Daemon client is disconnected");
+      }
+      senderRef.current.resetStreamForReplay();
+      const finalSeq = senderRef.current.getFinalSeq();
+      const text = await ensureFinalTranscript(finalSeq);
+      handleStreamingTranscriptionSuccess(text, generateMessageId());
     } catch (err) {
-      resetTranscriptionMutation();
+      if (err instanceof Error && err.name === "AttemptCancelledError") {
+        return;
+      }
       handleDictationFailure(err);
     }
-  }, [
+  }, [client, ensureFinalTranscript, handleDictationFailure, handleStreamingTranscriptionSuccess]);
-    handleDictationFailure,
-    handleTranscriptionSuccess,
-    resetTranscriptionMutation,
-    transmitDictation,
-  ]);
 
   const discardFailedDictation = useCallback(() => {
-    pendingAudioRef.current = null;
-    pendingRequestIdRef.current = null;
-    setPendingRequestId(null);
     setIsProcessing(false);
+    isProcessingRef.current = false;
     setDuration(0);
-    setFailedRecording(null);
     setStatus("idle");
-    setRetryAttempt(0);
-    setRetryInfo(null);
     setError(null);
-    setLastOutcome(null);
+    clearStreamingState();
-  }, []);
+  }, [clearStreamingState]);
 
   const reset = useCallback(() => {
-    pendingRequestIdRef.current = null;
-    setPendingRequestId(null);
     setIsRecording(false);
     isRecordingRef.current = false;
     setIsProcessing(false);
+    isProcessingRef.current = false;
     stopDurationTracking();
     setDuration(0);
-    setVolume(0);
     setError(null);
     setStatus("idle");
-    setRetryAttempt(0);
+    clearStreamingState();
-    setRetryInfo(null);
+  }, [clearStreamingState, stopDurationTracking]);
-    setFailedRecording(null);
-    pendingAudioRef.current = null;
-    setLastOutcome(null);
-    resetTranscriptionMutation();
-  }, [resetTranscriptionMutation, stopDurationTracking]);
 
   const cancelRef = useRef<(() => void) | null>(null);
   useEffect(() => {
@@ -668,8 +400,7 @@ export function useDictation(options: UseDictationOptions): UseDictationResult {
     typeof autoStopWhenHidden?.isVisible === "boolean" ? autoStopWhenHidden.isVisible : null
   );
   useEffect(() => {
-    const nextVisible =
+    const nextVisible = typeof autoStopWhenHidden?.isVisible === "boolean" ? autoStopWhenHidden.isVisible : null;
-      typeof autoStopWhenHidden?.isVisible === "boolean" ? autoStopWhenHidden.isVisible : null;
     const prevVisible = visibilityRef.current;
     visibilityRef.current = nextVisible;
 
@@ -685,48 +416,28 @@ export function useDictation(options: UseDictationOptions): UseDictationResult {
         stopDurationTracking();
         setDuration(0);
         setIsProcessing(false);
-        setVolume(0);
         setError(null);
         setStatus("idle");
-        setRetryAttempt(0);
+        clearStreamingState();
-        setRetryInfo(null);
-        pendingAudioRef.current = null;
-        setFailedRecording(null);
-        setLastOutcome(null);
       }
     }
-  }, [autoStopWhenHidden?.isVisible, stopDurationTracking]);
+  }, [autoStopWhenHidden?.isVisible, clearStreamingState, stopDurationTracking]);
-
 
   useEffect(() => {
     return () => {
       attemptGuardRef.current.cancel();
       stopDurationTracking();
-      const activeStop = activeStopPromiseRef.current;
+      void audioStopRef.current().catch(() => undefined);
-      if (activeStop) {
-        void activeStop.catch(() => undefined);
-        return;
-      }
-      const recorderInstance = recorderRef.current;
-      if (recorderInstance?.isRecording?.()) {
-        void recorderInstance.stop().catch(() => undefined);
-      }
     };
   }, [stopDurationTracking]);
 
   return {
     isRecording,
     isProcessing,
-    volume,
+    volume: audio.volume,
     duration,
-    pendingRequestId,
     error,
     status,
-    retryAttempt,
-    maxRetryAttempts,
-    retryInfo,
-    failedRecording,
-    lastOutcome,
     startDictation,
     cancelDictation,
     confirmDictation,
@@ -735,3 +446,9 @@ export function useDictation(options: UseDictationOptions): UseDictationResult {
     reset,
   };
 }
+
+export type {
+  DictationStatus,
+  UseDictationOptions,
+  UseDictationResult,
+} from "./use-dictation.shared";

packages/app/src/hooks/use-speechmatics-audio.native.ts

@@ -1,4 +1,5 @@
 import { useState, useEffect, useRef, useCallback } from "react";
+import { Buffer } from "buffer";
 import {
   initialize,
   useMicrophonePermissions,
@@ -14,6 +15,8 @@ export interface SpeechmaticsAudioConfig {
   onSpeechStart?: () => void;
   onSpeechEnd?: () => void;
   onError?: (error: Error) => void;
+  /** When true, stream microphone PCM continuously without VAD gating. */
+  enableContinuousStreaming?: boolean;
   volumeThreshold: number; // Volume threshold for speech detection (0-1)
   silenceDuration: number; // ms of silence before ending segment
   speechConfirmationDuration: number; // ms of sustained speech before confirming
@@ -33,11 +36,17 @@ export interface SpeechmaticsAudio {
 }
 
 function uint8ArrayToBase64(bytes: Uint8Array): string {
-  let binary = "";
+  // NOTE: This is performance-sensitive during continuous streaming.
-  for (let i = 0; i < bytes.length; i++) {
+  // Buffer-backed base64 is significantly faster than manual string building.
-    binary += String.fromCharCode(bytes[i]);
+  try {
+    return Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength).toString("base64");
+  } catch {
+    let binary = "";
+    for (let i = 0; i < bytes.length; i++) {
+      binary += String.fromCharCode(bytes[i]);
+    }
+    return btoa(binary);
   }
-  return btoa(binary);
 }
 
 function concatenateUint8Arrays(arrays: Uint8Array[]): Uint8Array {
@@ -67,6 +76,8 @@ export function useSpeechmaticsAudio(
   const [isMuted, setIsMuted] = useState(false);
   const [segmentDuration, setSegmentDuration] = useState(0);
 
+  const enableContinuousStreaming = config.enableContinuousStreaming === true;
+
   const audioBufferRef = useRef<Uint8Array[]>([]);
   const silenceStartRef = useRef<number | null>(null);
   const isSpeakingRef = useRef(false);
@@ -136,6 +147,15 @@ export function useSpeechmaticsAudio(
 
         const pcmData: Uint8Array = event.data;
 
+        if (enableContinuousStreaming) {
+          audioBufferRef.current.push(pcmData);
+          bufferedBytesRef.current += pcmData.length;
+          if (bufferedBytesRef.current >= MIN_CHUNK_BYTES) {
+            flushBufferedAudio(false);
+          }
+          return;
+        }
+
         // Buffer the audio chunk if we're detecting or speaking
         // Start buffering from first spike to capture beginning of speech
         if (speechDetectionStartRef.current !== null || isSpeakingRef.current) {
@@ -150,7 +170,7 @@ export function useSpeechmaticsAudio(
           }
         }
       },
-      [isActive, isMuted, flushBufferedAudio, MIN_CHUNK_BYTES]
+      [enableContinuousStreaming, isActive, isMuted, flushBufferedAudio, MIN_CHUNK_BYTES]
     )
   );
 
@@ -165,6 +185,7 @@ export function useSpeechmaticsAudio(
         setVolume(volumeLevel);
 
         if (isMuted) return;
+        if (enableContinuousStreaming) return;
 
         const speechDetected = volumeLevel > VOLUME_THRESHOLD;
 
@@ -270,6 +291,7 @@ export function useSpeechmaticsAudio(
         }
       },
       [
+        enableContinuousStreaming,
         isActive,
         isMuted,
         VOLUME_THRESHOLD,
@@ -341,6 +363,10 @@ export function useSpeechmaticsAudio(
       toggleRecording(false);
     }
 
+    if (enableContinuousStreaming) {
+      flushBufferedAudio(true);
+    }
+
     // Tear down audio session
     if (audioInitialized) {
       tearDown();

packages/app/src/hooks/use-speechmatics-audio.web.ts

@@ -5,6 +5,7 @@ export interface SpeechmaticsAudioConfig {
   onSpeechStart?: () => void;
   onSpeechEnd?: () => void;
   onError?: (error: Error) => void;
+  enableContinuousStreaming?: boolean;
   volumeThreshold: number;
   silenceDuration: number;
   speechConfirmationDuration: number;

packages/server/src/client/daemon-client-v2.ts

@@ -42,7 +42,6 @@ import type {
   SendAgentMessage,
   SessionInboundMessage,
   SessionOutboundMessage,
-  TranscriptionResultMessage,
 } from "../shared/messages.js";
 import type {
   AgentPermissionRequest,
@@ -150,12 +149,6 @@ export type DaemonClientV2Config = {
 
 export type SendMessageOptions = Pick<SendAgentMessage, "messageId" | "images">;
 
-export type TranscribeAudioOptions = {
-  audio: string; // base64 encoded
-  format: string;
-  timeout?: number;
-};
-
 type AgentConfigOverrides = Partial<Omit<AgentSessionConfig, "provider" | "cwd">>;
 
 export type CreateAgentRequestOptions = {
@@ -188,7 +181,6 @@ type FileDownloadTokenPayload = FileDownloadTokenResponse["payload"];
 type ListProviderModelsPayload = ListProviderModelsResponseMessage["payload"];
 type ListCommandsPayload = ListCommandsResponse["payload"];
 type ExecuteCommandPayload = ExecuteCommandResponse["payload"];
-type TranscriptionResultPayload = TranscriptionResultMessage["payload"];
 type AgentPermissionResolvedPayload = AgentPermissionResolvedMessage["payload"];
 type ListTerminalsPayload = ListTerminalsResponse["payload"];
 type CreateTerminalPayload = CreateTerminalResponse["payload"];
@@ -472,7 +464,26 @@ export class DaemonClientV2 {
       throw new Error("Transport not connected");
     }
     const payload = SessionInboundMessageSchema.parse(message);
-    this.transport.send(JSON.stringify({ type: "session", message: payload }));
+    try {
+      this.transport.send(JSON.stringify({ type: "session", message: payload }));
+    } catch (error) {
+      if (this.config.suppressSendErrors) {
+        return;
+      }
+      throw error instanceof Error ? error : new Error(String(error));
+    }
+  }
+
+  private sendSessionMessageStrict(message: SessionInboundMessage): void {
+    if (!this.transport || this.connectionState.status !== "connected") {
+      throw new Error("Transport not connected");
+    }
+    const payload = SessionInboundMessageSchema.parse(message);
+    try {
+      this.transport.send(JSON.stringify({ type: "session", message: payload }));
+    } catch (error) {
+      throw error instanceof Error ? error : new Error(String(error));
+    }
   }
 
   sendUserMessage(text: string): void {
@@ -839,37 +850,6 @@ export class DaemonClientV2 {
     await this.sendAgentMessage(agentId, text, options);
   }
 
-  async transcribeAudio(
-    options: TranscribeAudioOptions
-  ): Promise<TranscriptionResultPayload> {
-    const requestId = this.createRequestId();
-    const timeout = options.timeout ?? 120000;
-
-    const responsePromise = this.waitFor(
-      (msg) => {
-        if (msg.type !== "transcription_result") {
-          return null;
-        }
-        if (msg.payload.requestId !== requestId) {
-          return null;
-        }
-        return msg.payload;
-      },
-      timeout,
-      { skipQueue: true }
-    );
-
-    const message = SessionInboundMessageSchema.parse({
-      type: "transcribe_audio_request",
-      audio: options.audio,
-      format: options.format,
-      requestId,
-    });
-    this.sendSessionMessage(message);
-
-    return responsePromise;
-  }
-
   async cancelAgent(agentId: string): Promise<void> {
     this.sendSessionMessage({ type: "cancel_agent_request", agentId });
   }
@@ -929,6 +909,87 @@ export class DaemonClientV2 {
     this.sendSessionMessage({ type: "realtime_audio_chunk", audio, format, isLast });
   }
 
+  startDictationStream(dictationId: string, format: string): Promise<void> {
+    const ackPromise = this.waitFor(
+      (msg) => {
+        if (msg.type !== "dictation_stream_ack") {
+          return null;
+        }
+        if (msg.payload.dictationId !== dictationId) {
+          return null;
+        }
+        if (msg.payload.ackSeq !== -1) {
+          return null;
+        }
+        return msg.payload;
+      },
+      30000,
+      { skipQueue: true }
+    ).then(() => undefined);
+
+    const errorPromise = this.waitFor(
+      (msg) => {
+        if (msg.type !== "dictation_stream_error") {
+          return null;
+        }
+        if (msg.payload.dictationId !== dictationId) {
+          return null;
+        }
+        return msg.payload;
+      },
+      30000,
+      { skipQueue: true }
+    ).then((payload) => {
+      throw new Error(payload.error);
+    });
+
+    this.sendSessionMessageStrict({ type: "dictation_stream_start", dictationId, format });
+    return Promise.race([ackPromise, errorPromise]);
+  }
+
+  sendDictationStreamChunk(dictationId: string, seq: number, audio: string, format: string): void {
+    this.sendSessionMessageStrict({ type: "dictation_stream_chunk", dictationId, seq, audio, format });
+  }
+
+  finishDictationStream(dictationId: string, finalSeq: number): Promise<{ dictationId: string; text: string }> {
+    const finalPromise = this.waitFor(
+      (msg) => {
+        if (msg.type !== "dictation_stream_final") {
+          return null;
+        }
+        if (msg.payload.dictationId !== dictationId) {
+          return null;
+        }
+        return msg.payload;
+      },
+      120000,
+      { skipQueue: true }
+    );
+
+    const errorPromise = this.waitFor(
+      (msg) => {
+        if (msg.type !== "dictation_stream_error") {
+          return null;
+        }
+        if (msg.payload.dictationId !== dictationId) {
+          return null;
+        }
+        return msg.payload;
+      },
+      120000,
+      { skipQueue: true }
+    ).then((payload) => {
+      throw new Error(payload.error);
+    });
+
+    this.sendSessionMessageStrict({ type: "dictation_stream_finish", dictationId, finalSeq });
+    return Promise.race([finalPromise, errorPromise]);
+  }
+
+  cancelDictationStream(dictationId: string): void {
+    this.sendSessionMessageStrict({ type: "dictation_stream_cancel", dictationId });
+  }
+
   async abortRequest(): Promise<void> {
     this.sendSessionMessage({ type: "abort_request" });
   }
@@ -2097,7 +2158,12 @@ function createWebSocketTransportFactory(
   return ({ url, headers }) => {
     const ws = factory(url, { headers });
     return {
-      send: (data) => ws.send(data),
+      send: (data) => {
+        if (typeof ws.readyState === "number" && ws.readyState !== 1) {
+          throw new Error(`WebSocket not open (readyState=${ws.readyState})`);
+        }
+        ws.send(data);
+      },
       close: (code?: number, reason?: string) => ws.close(code, reason),
       onOpen: (handler) => bindWsHandler(ws, "open", handler),
       onClose: (handler) => bindWsHandler(ws, "close", handler),

packages/server/src/server/agent/openai-realtime-transcription.ts

@@ -0,0 +1,207 @@
+import type pino from "pino";
+import WebSocket from "ws";
+import { EventEmitter } from "node:events";
+
+type OpenAIClientEvent =
+  | {
+      type: "session.update";
+      session: {
+        type: "transcription";
+        audio: {
+          input: {
+            format: { type: "audio/pcm"; rate: 24000 };
+            transcription: {
+              model: string;
+              language?: string;
+              prompt?: string;
+            };
+            turn_detection: null;
+          };
+        };
+      };
+    }
+  | { type: "input_audio_buffer.append"; audio: string }
+  | { type: "input_audio_buffer.commit" }
+  | { type: "input_audio_buffer.clear" };
+
+type OpenAIServerEvent =
+  | { type: "session.created" | "session.updated" }
+  | {
+      type: "input_audio_buffer.committed";
+      item_id: string;
+      previous_item_id: string | null;
+    }
+  | {
+      type: "conversation.item.input_audio_transcription.completed";
+      item_id: string;
+      transcript: string;
+    }
+  | { type: "error"; error?: { message?: string } };
+
+export class OpenAIRealtimeTranscriptionSession extends EventEmitter {
+  private readonly apiKey: string;
+  private readonly logger: pino.Logger;
+  private readonly transcriptionModel: string;
+  private readonly language?: string;
+
+  private ws: WebSocket | null = null;
+  private ready: Promise<void> | null = null;
+  private closing = false;
+
+  constructor(params: {
+    apiKey: string;
+    logger: pino.Logger;
+    transcriptionModel: string;
+    language?: string;
+  }) {
+    super();
+    this.apiKey = params.apiKey;
+    this.logger = params.logger.child({ provider: "openai", component: "realtime-transcription" });
+    this.transcriptionModel = params.transcriptionModel;
+    this.language = params.language;
+  }
+
+  public async connect(): Promise<void> {
+    if (this.ready) {
+      return this.ready;
+    }
+
+    this.closing = false;
+    this.ready = new Promise<void>((resolve, reject) => {
+      const url = "wss://api.openai.com/v1/realtime?intent=transcription";
+      const ws = new WebSocket(url, {
+        headers: {
+          Authorization: `Bearer ${this.apiKey}`,
+        },
+      });
+      this.ws = ws;
+
+      let resolved = false;
+
+      const fail = (error: Error) => {
+        if (resolved) {
+          this.emit("error", error);
+          return;
+        }
+        resolved = true;
+        reject(error);
+      };
+
+      ws.on("open", () => {
+        this.logger.debug("OpenAI realtime transcription websocket connected");
+        const update: OpenAIClientEvent = {
+          type: "session.update",
+          session: {
+            type: "transcription",
+            audio: {
+              input: {
+                format: { type: "audio/pcm", rate: 24000 },
+                transcription: {
+                  model: this.transcriptionModel,
+                  ...(this.language ? { language: this.language } : {}),
+                },
+                // We commit periodically ourselves; no server-side VAD for dictation.
+                turn_detection: null,
+              },
+            },
+          },
+        };
+        ws.send(JSON.stringify(update));
+      });
+
+      ws.on("message", (data) => {
+        const text = typeof data === "string" ? data : data.toString("utf-8");
+        let parsed: unknown;
+        try {
+          parsed = JSON.parse(text);
+        } catch {
+          return;
+        }
+
+        const event = parsed as OpenAIServerEvent;
+        if (
+          event.type === "session.created" ||
+          event.type === "session.updated"
+        ) {
+          if (!resolved) {
+            resolved = true;
+            resolve();
+          }
+          return;
+        }
+
+        if (event.type === "input_audio_buffer.committed") {
+          this.emit("committed", {
+            itemId: event.item_id,
+            previousItemId: event.previous_item_id,
+          });
+          return;
+        }
+
+        if (event.type === "conversation.item.input_audio_transcription.completed") {
+          this.emit("transcript", { itemId: event.item_id, transcript: event.transcript });
+          return;
+        }
+
+        if (event.type === "error") {
+          const message = event.error?.message ?? "OpenAI realtime error";
+          fail(new Error(message));
+        }
+      });
+
+      ws.on("error", (err) => {
+        fail(err instanceof Error ? err : new Error(String(err)));
+      });
+
+      ws.on("close", () => {
+        this.logger.debug("OpenAI realtime websocket closed");
+        if (this.closing) {
+          return;
+        }
+        if (!resolved) {
+          fail(new Error("OpenAI realtime websocket closed before ready"));
+          return;
+        }
+        fail(new Error("OpenAI realtime websocket closed"));
+      });
+    });
+
+    return this.ready;
+  }
+
+  public appendPcm16Base64(base64Audio: string): void {
+    if (!this.ws || this.ws.readyState !== WebSocket.OPEN) {
+      throw new Error("OpenAI realtime websocket not connected");
+    }
+    const event: OpenAIClientEvent = { type: "input_audio_buffer.append", audio: base64Audio };
+    this.ws.send(JSON.stringify(event));
+  }
+
+  public commit(): void {
+    if (!this.ws || this.ws.readyState !== WebSocket.OPEN) {
+      throw new Error("OpenAI realtime websocket not connected");
+    }
+    const event: OpenAIClientEvent = { type: "input_audio_buffer.commit" };
+    this.ws.send(JSON.stringify(event));
+  }
+
+  public clear(): void {
+    if (!this.ws || this.ws.readyState !== WebSocket.OPEN) {
+      return;
+    }
+    const event: OpenAIClientEvent = { type: "input_audio_buffer.clear" };
+    this.ws.send(JSON.stringify(event));
+  }
+
+  public close(): void {
+    try {
+      this.closing = true;
+      this.ws?.close();
+    } catch {
+      // no-op
+    } finally {
+      this.ws = null;
+      this.ready = null;
+    }
+  }
+}

packages/server/src/server/agent/pcm16-resampler.ts

@@ -0,0 +1,83 @@
+export class Pcm16MonoResampler {
+  private readonly inputRate: number;
+  private readonly outputRate: number;
+  private readonly step: number;
+  private pos: number;
+  private carrySample: number | null;
+
+  constructor(params: { inputRate: number; outputRate: number }) {
+    this.inputRate = params.inputRate;
+    this.outputRate = params.outputRate;
+    this.step = this.inputRate / this.outputRate;
+    this.pos = 0;
+    this.carrySample = null;
+  }
+
+  public reset(): void {
+    this.pos = 0;
+    this.carrySample = null;
+  }
+
+  public processChunk(pcm16le: Buffer): Buffer {
+    if (pcm16le.length === 0) {
+      return Buffer.alloc(0);
+    }
+    if (pcm16le.length % 2 !== 0) {
+      throw new Error(`PCM16 chunk byteLength must be even, got ${pcm16le.length}`);
+    }
+
+    const srcChunk = new Int16Array(
+      pcm16le.buffer,
+      pcm16le.byteOffset,
+      pcm16le.byteLength / 2
+    );
+
+    const hasCarry = this.carrySample !== null;
+    const srcLen = srcChunk.length + (hasCarry ? 1 : 0);
+    if (srcLen < 2) {
+      this.carrySample = srcChunk.length ? srcChunk[srcChunk.length - 1] : this.carrySample;
+      return Buffer.alloc(0);
+    }
+
+    const src = new Float32Array(srcLen);
+    let offset = 0;
+    if (hasCarry) {
+      src[0] = (this.carrySample as number) / 32768;
+      offset = 1;
+    }
+    for (let i = 0; i < srcChunk.length; i += 1) {
+      src[offset + i] = srcChunk[i] / 32768;
+    }
+
+    const out: number[] = [];
+    const maxPos = src.length - 1;
+
+    while (this.pos < maxPos) {
+      const i = Math.floor(this.pos);
+      const frac = this.pos - i;
+      const s0 = src[i]!;
+      const s1 = src[i + 1]!;
+      const sample = s0 + (s1 - s0) * frac;
+      const clamped = Math.max(-1, Math.min(1, sample));
+      const int16 = Math.round(clamped * 32767);
+      out.push(int16);
+      this.pos += this.step;
+    }
+
+    // Keep the last input sample as carry for the next chunk.
+    const lastInput = srcChunk[srcChunk.length - 1]!;
+    this.carrySample = lastInput;
+
+    // Shift position so next chunk (which will include carry sample) continues smoothly.
+    const shift = src.length - 1;
+    this.pos = this.pos - shift;
+    if (this.pos < 0) {
+      // Guard against floating point drift.
+      this.pos = 0;
+    }
+
+    const outArr = Int16Array.from(out);
+    return Buffer.from(outArr.buffer, outArr.byteOffset, outArr.byteLength);
+  }
+}
+

packages/server/src/server/daemon-client-v2.e2e.test.ts

@@ -23,6 +23,9 @@ if (envPath) {
   dotenv.config({ path: envPath });
 }
 
+// Make dictation streaming commit frequently in tests so we exercise multi-commit assembly logic.
+process.env.OPENAI_REALTIME_DICTATION_COMMIT_MS ??= "1000";
+
 function tmpCwd(): string {
   return mkdtempSync(path.join(tmpdir(), "daemon-client-v2-"));
 }
@@ -630,6 +633,216 @@ describe("daemon client v2 E2E", () => {
     180000
   );
 
+  test(
+    "streams dictation PCM and returns final transcript via OpenAI Realtime transcription",
+    async () => {
+      if (process.env.PASEO_E2E_DICTATION_REALTIME !== "1") {
+        // Requires OpenAI realtime transcription and is inherently network-flaky.
+        return;
+      }
+      if (!process.env.OPENAI_API_KEY) {
+        return;
+      }
+
+      const fixturePath = path.resolve(
+        process.cwd(),
+        "..",
+        "app",
+        "e2e",
+        "fixtures",
+        "recording.wav"
+      );
+      const wav = await import("node:fs/promises").then((fs) => fs.readFile(fixturePath));
+
+      const parsePcm16MonoWav = (buffer: Buffer): Buffer => {
+        if (buffer.toString("ascii", 0, 4) !== "RIFF" || buffer.toString("ascii", 8, 12) !== "WAVE") {
+          throw new Error("Invalid WAV header");
+        }
+        let offset = 12;
+        let fmt: { audioFormat: number; channels: number; sampleRate: number; bitsPerSample: number } | null = null;
+        let dataChunk: Buffer | null = null;
+
+        while (offset + 8 <= buffer.length) {
+          const id = buffer.toString("ascii", offset, offset + 4);
+          const size = buffer.readUInt32LE(offset + 4);
+          const payloadStart = offset + 8;
+          const payloadEnd = payloadStart + size;
+          if (payloadEnd > buffer.length) {
+            break;
+          }
+
+          if (id === "fmt ") {
+            const audioFormat = buffer.readUInt16LE(payloadStart);
+            const channels = buffer.readUInt16LE(payloadStart + 2);
+            const sampleRate = buffer.readUInt32LE(payloadStart + 4);
+            const bitsPerSample = buffer.readUInt16LE(payloadStart + 14);
+            fmt = { audioFormat, channels, sampleRate, bitsPerSample };
+          } else if (id === "data") {
+            dataChunk = buffer.subarray(payloadStart, payloadEnd);
+          }
+
+          offset = payloadEnd + (size % 2);
+        }
+
+        if (!fmt || !dataChunk) {
+          throw new Error("Missing WAV fmt/data chunks");
+        }
+        if (fmt.audioFormat !== 1) {
+          throw new Error(`Unsupported WAV encoding (audioFormat=${fmt.audioFormat})`);
+        }
+        if (fmt.channels !== 1 || fmt.sampleRate !== 16000 || fmt.bitsPerSample !== 16) {
+          throw new Error(
+            `Unexpected WAV format: channels=${fmt.channels} rate=${fmt.sampleRate} bits=${fmt.bitsPerSample}`
+          );
+        }
+        if (dataChunk.length % 2 !== 0) {
+          throw new Error("WAV PCM16 data length must be even");
+        }
+        return dataChunk;
+      };
+
+      const pcm16 = parsePcm16MonoWav(wav);
+      const dictationId = `dict-${Date.now()}`;
+      const format = "audio/pcm;rate=16000;bits=16";
+
+      await ctx.client.startDictationStream(dictationId, format);
+
+      const chunkBytes = 3200; // ~100ms @ 16kHz mono PCM16 (1600 samples * 2 bytes)
+      let seq = 0;
+      for (let offset = 0; offset < pcm16.length; offset += chunkBytes) {
+        const chunk = pcm16.subarray(offset, Math.min(pcm16.length, offset + chunkBytes));
+        ctx.client.sendDictationStreamChunk(dictationId, seq, chunk.toString("base64"), format);
+        seq += 1;
+      }
+
+      const finalSeq = seq - 1;
+      const result = await ctx.client.finishDictationStream(dictationId, finalSeq);
+
+      expect(result.dictationId).toBe(dictationId);
+      expect(result.text.toLowerCase()).toContain("voice note");
+    },
+    180000
+  );
+
+  test(
+    "does not commit an empty OpenAI realtime audio buffer on finish",
+    async () => {
+      if (process.env.PASEO_E2E_DICTATION_REALTIME !== "1") {
+        return;
+      }
+      if (!process.env.OPENAI_API_KEY) {
+        return;
+      }
+
+      const fixturePath = path.resolve(
+        process.cwd(),
+        "..",
+        "app",
+        "e2e",
+        "fixtures",
+        "recording.wav"
+      );
+      const wav = await import("node:fs/promises").then((fs) => fs.readFile(fixturePath));
+
+      const parsePcm16MonoWav = (buffer: Buffer): Buffer => {
+        if (buffer.toString("ascii", 0, 4) !== "RIFF" || buffer.toString("ascii", 8, 12) !== "WAVE") {
+          throw new Error("Invalid WAV header");
+        }
+        let offset = 12;
+        let fmt: { audioFormat: number; channels: number; sampleRate: number; bitsPerSample: number } | null = null;
+        let dataChunk: Buffer | null = null;
+
+        while (offset + 8 <= buffer.length) {
+          const id = buffer.toString("ascii", offset, offset + 4);
+          const size = buffer.readUInt32LE(offset + 4);
+          const payloadStart = offset + 8;
+          const payloadEnd = payloadStart + size;
+          if (payloadEnd > buffer.length) {
+            break;
+          }
+
+          if (id === "fmt ") {
+            const audioFormat = buffer.readUInt16LE(payloadStart);
+            const channels = buffer.readUInt16LE(payloadStart + 2);
+            const sampleRate = buffer.readUInt32LE(payloadStart + 4);
+            const bitsPerSample = buffer.readUInt16LE(payloadStart + 14);
+            fmt = { audioFormat, channels, sampleRate, bitsPerSample };
+          } else if (id === "data") {
+            dataChunk = buffer.subarray(payloadStart, payloadEnd);
+          }
+
+          offset = payloadEnd + (size % 2);
+        }
+
+        if (!fmt || !dataChunk) {
+          throw new Error("Missing WAV fmt/data chunks");
+        }
+        if (fmt.audioFormat !== 1) {
+          throw new Error(`Unsupported WAV encoding (audioFormat=${fmt.audioFormat})`);
+        }
+        if (fmt.channels !== 1 || fmt.sampleRate !== 16000 || fmt.bitsPerSample !== 16) {
+          throw new Error(
+            `Unexpected WAV format: channels=${fmt.channels} rate=${fmt.sampleRate} bits=${fmt.bitsPerSample}`
+          );
+        }
+        if (dataChunk.length % 2 !== 0) {
+          throw new Error("WAV PCM16 data length must be even");
+        }
+        return dataChunk;
+      };
+
+      const pcm16 = parsePcm16MonoWav(wav);
+      const dictationId = `dict-empty-commit-${Date.now()}`;
+      const format = "audio/pcm;rate=16000;bits=16";
+
+      await ctx.client.startDictationStream(dictationId, format);
+
+      // Send exactly 10x 100ms chunks. With OPENAI_REALTIME_DICTATION_COMMIT_MS=1000 in this test file,
+      // the server will auto-commit at the 1s boundary. Finishing immediately after that should not
+      // attempt an additional empty commit.
+      const chunkBytes = 3200; // 100ms @ 16kHz mono PCM16
+      const targetChunks = 10;
+      let seq = 0;
+      for (let i = 0; i < targetChunks; i += 1) {
+        const start = i * chunkBytes;
+        const end = Math.min(pcm16.length, start + chunkBytes);
+        const chunk = pcm16.subarray(start, end);
+        ctx.client.sendDictationStreamChunk(dictationId, seq, chunk.toString("base64"), format);
+        seq += 1;
+      }
+
+      const finalSeq = seq - 1;
+      const result = await ctx.client.finishDictationStream(dictationId, finalSeq);
+
+      expect(result.dictationId).toBe(dictationId);
+      expect(typeof result.text).toBe("string");
+    },
+    180000
+  );
+
+  test(
+    "fails fast if dictation finishes without sending required chunks",
+    async () => {
+      if (process.env.PASEO_E2E_DICTATION_REALTIME !== "1") {
+        return;
+      }
+      if (!process.env.OPENAI_API_KEY) {
+        return;
+      }
+
+      const dictationId = `dict-missing-chunks-${Date.now()}`;
+      const format = "audio/pcm;rate=16000;bits=16";
+
+      await ctx.client.startDictationStream(dictationId, format);
+
+      // Claim that we sent chunk 0, but actually send no chunks.
+      await expect(ctx.client.finishDictationStream(dictationId, 0)).rejects.toThrow(
+        /no audio chunks were received/i
+      );
+    },
+    180000
+  );
+
   test(
     "supports git and file operations",
     async () => {

packages/server/src/server/session.ts

@@ -35,6 +35,8 @@ import { TTSManager } from "./agent/tts-manager.js";
 import { STTManager } from "./agent/stt-manager.js";
 import type { OpenAISTT } from "./agent/stt-openai.js";
 import type { OpenAITTS } from "./agent/tts-openai.js";
+import { OpenAIRealtimeTranscriptionSession } from "./agent/openai-realtime-transcription.js";
+import { Pcm16MonoResampler } from "./agent/pcm16-resampler.js";
 import type { VoiceConversationStore } from "./voice-conversation-store.js";
 import {
   buildConfigOverrides,
@@ -135,6 +137,21 @@ const MIN_STREAMING_SEGMENT_DURATION_MS = 1000;
 const MIN_STREAMING_SEGMENT_BYTES = Math.round(
   PCM_BYTES_PER_MS * MIN_STREAMING_SEGMENT_DURATION_MS
 );
+
+const DICTATION_PCM_INPUT_RATE = 16000;
+const DICTATION_PCM_OUTPUT_RATE = 24000;
+const DICTATION_COMMIT_INTERVAL_MS = Number.parseInt(
+  process.env.OPENAI_REALTIME_DICTATION_COMMIT_MS ?? "5000",
+  10
+);
+const DICTATION_COMMIT_TARGET_BYTES = Math.round(
+  (DICTATION_PCM_OUTPUT_RATE * 2 * DICTATION_COMMIT_INTERVAL_MS) / 1000
+);
+const DICTATION_MIN_COMMIT_MS = 100;
+const DICTATION_MIN_COMMIT_BYTES = Math.round(
+  (DICTATION_PCM_OUTPUT_RATE * 2 * DICTATION_MIN_COMMIT_MS) / 1000
+);
+const DICTATION_FINAL_TIMEOUT_MS = 120000;
 const SAFE_GIT_REF_PATTERN = /^[A-Za-z0-9._\/-]+$/;
 
 /**
@@ -249,6 +266,27 @@ export class Session {
   private speechInProgress = false;
   private voiceConversationId: string | null = null;
 
+  private dictationStreams = new Map<
+    string,
+    {
+      dictationId: string;
+      inputFormat: string;
+      openai: OpenAIRealtimeTranscriptionSession;
+      resampler: Pcm16MonoResampler;
+      receivedChunks: Map<number, Buffer>;
+      nextSeqToForward: number;
+      ackSeq: number;
+      bytesSinceCommit: number;
+      expectedCommits: number;
+      committedCount: number;
+      committedItemIds: string[];
+      transcriptsByItemId: Map<string, string>;
+      finishRequested: boolean;
+      finalSeq: number | null;
+      finalTimeout: ReturnType<typeof setTimeout> | null;
+    }
+  >();
+
   // Audio buffering for interruption handling
   private pendingAudioSegments: Array<{ audio: Buffer; format: string }> = [];
   private bufferTimeout: NodeJS.Timeout | null = null;
@@ -735,8 +773,20 @@ export class Session {
           );
           break;
 
-        case "transcribe_audio_request":
+        case "dictation_stream_start":
-          await this.handleTranscribeAudio(msg);
+          await this.handleDictationStreamStart(msg);
+          break;
+
+        case "dictation_stream_chunk":
+          await this.handleDictationStreamChunk(msg);
+          break;
+
+        case "dictation_stream_finish":
+          await this.handleDictationStreamFinish(msg);
+          break;
+
+        case "dictation_stream_cancel":
+          await this.handleDictationStreamCancel(msg);
           break;
 
         case "create_agent_request":
@@ -1177,56 +1227,279 @@ export class Session {
     this.startAgentStream(agentId, prompt);
   }
 
-  /**
+  private emitDictationAck(dictationId: string, ackSeq: number): void {
-   * Handle audio transcription request
+    this.emit({
-   */
+      type: "dictation_stream_ack",
-  private async handleTranscribeAudio(
+      payload: { dictationId, ackSeq },
-    msg: Extract<SessionInboundMessage, { type: "transcribe_audio_request" }>
+    });
-  ): Promise<void> {
+  }
-    const { audio, format, requestId } = msg;
 
-    const audioBuffer = Buffer.from(audio, "base64");
+  private failDictationStream(dictationId: string, error: string, retryable: boolean): void {
+    this.emit({
+      type: "dictation_stream_error",
+      payload: { dictationId, error, retryable },
+    });
+  }
 
-    this.sessionLogger.debug({ requestId }, "Transcribing audio");
+  private cleanupDictationStream(dictationId: string): void {
-
+    const state = this.dictationStreams.get(dictationId) ?? null;
-    try {
+    if (!state) {
-      const result = await this.sttManager.transcribe(audioBuffer, format, {
+      return;
-        requestId,
-        label: "dictation",
-      });
-
-      this.sessionLogger.info(
-        { requestId, textLength: result.text.length },
-        "Transcription complete"
-      );
-
-      this.emit({
-        type: "transcription_result",
-        payload: {
-          text: result.text,
-          language: result.language,
-          duration: result.duration,
-          requestId,
-          avgLogprob: result.avgLogprob,
-          isLowConfidence: result.isLowConfidence,
-          byteLength: result.byteLength,
-          format: result.format,
-          debugRecordingPath: result.debugRecordingPath,
-        },
-      });
-    } catch (error: any) {
-      this.sessionLogger.error({ err: error, requestId }, "Transcription failed");
-      this.emit({
-        type: "activity_log",
-        payload: {
-          id: uuidv4(),
-          timestamp: new Date(),
-          type: "error",
-          content: `Transcription failed: ${error.message}`,
-        },
-      });
-      throw error;
     }
+    if (state.finalTimeout) {
+      clearTimeout(state.finalTimeout);
+    }
+    try {
+      state.openai.close();
+    } catch {
+      // no-op
+    }
+    this.dictationStreams.delete(dictationId);
+  }
+
+  private maybeFinalizeDictationStream(dictationId: string): void {
+    const state = this.dictationStreams.get(dictationId);
+    if (!state) {
+      return;
+    }
+
+    if (!state.finishRequested || state.finalSeq === null) {
+      return;
+    }
+    if (state.ackSeq < state.finalSeq) {
+      return;
+    }
+    if (state.committedCount < state.expectedCommits) {
+      return;
+    }
+
+    const allTranscriptsReady = state.committedItemIds.every((itemId) =>
+      state.transcriptsByItemId.has(itemId)
+    );
+    if (!allTranscriptsReady) {
+      return;
+    }
+
+    const orderedText = state.committedItemIds
+      .map((itemId) => state.transcriptsByItemId.get(itemId) ?? "")
+      .join(" ")
+      .trim();
+
+    this.emit({
+      type: "dictation_stream_final",
+      payload: { dictationId, text: orderedText },
+    });
+
+    this.cleanupDictationStream(dictationId);
+  }
+
+  private async handleDictationStreamStart(
+    msg: Extract<SessionInboundMessage, { type: "dictation_stream_start" }>
+  ): Promise<void> {
+    const dictationId = msg.dictationId;
+    this.cleanupDictationStream(dictationId);
+
+    const apiKey = process.env.OPENAI_API_KEY;
+    if (!apiKey) {
+      this.failDictationStream(dictationId, "OPENAI_API_KEY not set", false);
+      return;
+    }
+
+    const transcriptionModel =
+      process.env.OPENAI_REALTIME_TRANSCRIPTION_MODEL ?? "gpt-4o-transcribe";
+
+    const openai = new OpenAIRealtimeTranscriptionSession({
+      apiKey,
+      logger: this.sessionLogger.child({ dictationId }),
+      transcriptionModel,
+      language: "en",
+    });
+
+    openai.on("committed", ({ itemId }: { itemId: string }) => {
+      const state = this.dictationStreams.get(dictationId);
+      if (!state) {
+        return;
+      }
+      state.committedCount += 1;
+      state.committedItemIds.push(itemId);
+      this.maybeFinalizeDictationStream(dictationId);
+    });
+
+    openai.on("transcript", ({ itemId, transcript }: { itemId: string; transcript: string }) => {
+      const state = this.dictationStreams.get(dictationId);
+      if (!state) {
+        return;
+      }
+      state.transcriptsByItemId.set(itemId, transcript);
+      this.maybeFinalizeDictationStream(dictationId);
+    });
+
+    openai.on("error", (err) => {
+      const error = err instanceof Error ? err.message : String(err);
+      this.failDictationStream(dictationId, error, true);
+      this.cleanupDictationStream(dictationId);
+    });
+
+    await openai.connect();
+
+    this.dictationStreams.set(dictationId, {
+      dictationId,
+      inputFormat: msg.format,
+      openai,
+      resampler: new Pcm16MonoResampler({
+        inputRate: DICTATION_PCM_INPUT_RATE,
+        outputRate: DICTATION_PCM_OUTPUT_RATE,
+      }),
+      receivedChunks: new Map(),
+      nextSeqToForward: 0,
+      ackSeq: -1,
+      bytesSinceCommit: 0,
+      expectedCommits: 0,
+      committedCount: 0,
+      committedItemIds: [],
+      transcriptsByItemId: new Map(),
+      finishRequested: false,
+      finalSeq: null,
+      finalTimeout: null,
+    });
+
+    this.emitDictationAck(dictationId, -1);
+  }
+
+  private async handleDictationStreamChunk(
+    msg: Extract<SessionInboundMessage, { type: "dictation_stream_chunk" }>
+  ): Promise<void> {
+    const state = this.dictationStreams.get(msg.dictationId);
+    if (!state) {
+      this.failDictationStream(msg.dictationId, "Dictation stream not started", true);
+      return;
+    }
+
+    if (msg.format !== state.inputFormat) {
+      this.failDictationStream(
+        msg.dictationId,
+        `Mismatched dictation stream format: ${msg.format}`,
+        false
+      );
+      return;
+    }
+
+    if (msg.seq < state.nextSeqToForward) {
+      this.emitDictationAck(msg.dictationId, state.ackSeq);
+      return;
+    }
+
+    if (!state.receivedChunks.has(msg.seq)) {
+      state.receivedChunks.set(msg.seq, Buffer.from(msg.audio, "base64"));
+    }
+
+    while (state.receivedChunks.has(state.nextSeqToForward)) {
+      const seq = state.nextSeqToForward;
+      const pcm16 = state.receivedChunks.get(seq)!;
+      state.receivedChunks.delete(seq);
+
+      const resampled = state.resampler.processChunk(pcm16);
+      if (resampled.length > 0) {
+        state.openai.appendPcm16Base64(resampled.toString("base64"));
+        state.bytesSinceCommit += resampled.length;
+      }
+
+      state.nextSeqToForward += 1;
+      state.ackSeq = state.nextSeqToForward - 1;
+
+      if (state.bytesSinceCommit >= DICTATION_COMMIT_TARGET_BYTES) {
+        state.expectedCommits += 1;
+        state.openai.commit();
+        state.bytesSinceCommit = 0;
+      }
+    }
+
+    this.emitDictationAck(msg.dictationId, state.ackSeq);
+    this.maybeFinalizeDictationStream(msg.dictationId);
+  }
+
+  private async handleDictationStreamFinish(
+    msg: Extract<SessionInboundMessage, { type: "dictation_stream_finish" }>
+  ): Promise<void> {
+    const state = this.dictationStreams.get(msg.dictationId);
+    if (!state) {
+      this.failDictationStream(msg.dictationId, "Dictation stream not started", true);
+      return;
+    }
+
+    state.finishRequested = true;
+    state.finalSeq = msg.finalSeq;
+
+    // If the client claims it sent audio (finalSeq >= 0) but we haven't received any chunks at all,
+    // we should fail fast instead of hanging until DICTATION_FINAL_TIMEOUT_MS.
+    if (msg.finalSeq >= 0 && state.ackSeq < 0 && state.nextSeqToForward === 0 && state.receivedChunks.size === 0) {
+      this.sessionLogger.debug(
+        {
+          dictationId: msg.dictationId,
+          finalSeq: msg.finalSeq,
+          ackSeq: state.ackSeq,
+          nextSeqToForward: state.nextSeqToForward,
+          receivedChunks: state.receivedChunks.size,
+          expectedCommits: state.expectedCommits,
+          committedCount: state.committedCount,
+        },
+        "Dictation finish: no chunks received (failing fast)"
+      );
+      this.failDictationStream(
+        msg.dictationId,
+        `Dictation finished (finalSeq=${msg.finalSeq}) but no audio chunks were received`,
+        true
+      );
+      this.cleanupDictationStream(msg.dictationId);
+      return;
+    }
+
+    // Commit any remaining uncommitted audio.
+    // IMPORTANT: OpenAI requires at least ~100ms of audio in the buffer for a commit.
+    // Also, committing an empty buffer throws "buffer too small ... 0.00ms".
+    if (state.bytesSinceCommit > 0) {
+      this.sessionLogger.debug(
+        { dictationId: msg.dictationId, bytesSinceCommit: state.bytesSinceCommit },
+        "Dictation finish: committing pending audio"
+      );
+      if (state.bytesSinceCommit < DICTATION_MIN_COMMIT_BYTES) {
+        const padBytes = DICTATION_MIN_COMMIT_BYTES - state.bytesSinceCommit;
+        this.sessionLogger.debug(
+          { dictationId: msg.dictationId, padBytes },
+          "Dictation finish: padding to minimum commit size"
+        );
+        state.openai.appendPcm16Base64(Buffer.alloc(padBytes).toString("base64"));
+        state.bytesSinceCommit += padBytes;
+      }
+      state.expectedCommits += 1;
+      state.openai.commit();
+      state.bytesSinceCommit = 0;
+    } else {
+      this.sessionLogger.debug(
+        { dictationId: msg.dictationId },
+        "Dictation finish: no pending audio to commit"
+      );
+    }
+
+    if (state.finalTimeout) {
+      clearTimeout(state.finalTimeout);
+    }
+    state.finalTimeout = setTimeout(() => {
+      this.failDictationStream(
+        msg.dictationId,
+        "Timed out waiting for final transcription",
+        true
+      );
+      this.cleanupDictationStream(msg.dictationId);
+    }, DICTATION_FINAL_TIMEOUT_MS);
+
+    this.maybeFinalizeDictationStream(msg.dictationId);
+  }
+
+  private async handleDictationStreamCancel(
+    msg: Extract<SessionInboundMessage, { type: "dictation_stream_cancel" }>
+  ): Promise<void> {
+    this.cleanupDictationStream(msg.dictationId);
   }
 
   /**
@@ -4209,6 +4482,10 @@ export class Session {
     this.ttsManager.cleanup();
     this.sttManager.cleanup();
 
+    for (const dictationId of this.dictationStreams.keys()) {
+      this.cleanupDictationStream(dictationId);
+    }
+
     // Close MCP clients
     if (this.agentMcpClient) {
       try {

packages/server/src/server/websocket-server.ts

@@ -57,7 +57,13 @@ export class VoiceAssistantWebSocketServer {
       path: "/ws",
       verifyClient: ({ req }, callback) => {
         const origin = req.headers.origin;
-        if (!origin || allowedOrigins.has(origin)) {
+        const requestHost = typeof req.headers.host === "string" ? req.headers.host : null;
+        const sameOrigin =
+          !!origin &&
+          !!requestHost &&
+          (origin === `http://${requestHost}` || origin === `https://${requestHost}`);
+
+        if (!origin || allowedOrigins.has(origin) || sameOrigin) {
           callback(true);
         } else {
           this.logger.warn({ origin }, "Rejected connection from origin");

packages/server/src/shared/messages.ts

@@ -329,11 +329,33 @@ export const SendAgentMessageSchema = z.object({
   })).optional(),
 });
 
-export const TranscribeAudioRequestSchema = z.object({
+// ============================================================================
-  type: z.literal("transcribe_audio_request"),
+// Dictation Streaming (lossless, resumable)
-  audio: z.string(), // base64 encoded
+// ============================================================================
-  format: z.string(),
+
-  requestId: z.string(),
+export const DictationStreamStartMessageSchema = z.object({
+  type: z.literal("dictation_stream_start"),
+  dictationId: z.string(),
+  format: z.string(), // e.g. "audio/pcm;rate=16000;bits=16"
+});
+
+export const DictationStreamChunkMessageSchema = z.object({
+  type: z.literal("dictation_stream_chunk"),
+  dictationId: z.string(),
+  seq: z.number().int().nonnegative(),
+  audio: z.string(), // base64 encoded chunk
+  format: z.string(), // e.g. "audio/pcm;rate=16000;bits=16"
+});
+
+export const DictationStreamFinishMessageSchema = z.object({
+  type: z.literal("dictation_stream_finish"),
+  dictationId: z.string(),
+  finalSeq: z.number().int().nonnegative(),
+});
+
+export const DictationStreamCancelMessageSchema = z.object({
+  type: z.literal("dictation_stream_cancel"),
+  dictationId: z.string(),
 });
 
 const GitSetupOptionsSchema = z.object({
@@ -683,7 +705,10 @@ export const SessionInboundMessageSchema = z.discriminatedUnion("type", [
   DeleteAgentRequestMessageSchema,
   SetVoiceConversationMessageSchema,
   SendAgentMessageSchema,
-  TranscribeAudioRequestSchema,
+  DictationStreamStartMessageSchema,
+  DictationStreamChunkMessageSchema,
+  DictationStreamFinishMessageSchema,
+  DictationStreamCancelMessageSchema,
   CreateAgentRequestMessageSchema,
   ListProviderModelsRequestMessageSchema,
   ResumeAgentRequestMessageSchema,
@@ -780,6 +805,31 @@ export const TranscriptionResultMessageSchema = z.object({
   }),
 });
 
+export const DictationStreamAckMessageSchema = z.object({
+  type: z.literal("dictation_stream_ack"),
+  payload: z.object({
+    dictationId: z.string(),
+    ackSeq: z.number().int(),
+  }),
+});
+
+export const DictationStreamFinalMessageSchema = z.object({
+  type: z.literal("dictation_stream_final"),
+  payload: z.object({
+    dictationId: z.string(),
+    text: z.string(),
+  }),
+});
+
+export const DictationStreamErrorMessageSchema = z.object({
+  type: z.literal("dictation_stream_error"),
+  payload: z.object({
+    dictationId: z.string(),
+    error: z.string(),
+    retryable: z.boolean(),
+  }),
+});
+
 export const StatusMessageSchema = z.object({
   type: z.literal("status"),
   payload: z
@@ -1244,6 +1294,9 @@ export const SessionOutboundMessageSchema = z.discriminatedUnion("type", [
   AssistantChunkMessageSchema,
   AudioOutputMessageSchema,
   TranscriptionResultMessageSchema,
+  DictationStreamAckMessageSchema,
+  DictationStreamFinalMessageSchema,
+  DictationStreamErrorMessageSchema,
   StatusMessageSchema,
   InitializeAgentResponseMessageSchema,
   ArtifactMessageSchema,
@@ -1323,7 +1376,10 @@ export type ActivityLogPayload = z.infer<typeof ActivityLogPayloadSchema>;
 export type UserTextMessage = z.infer<typeof UserTextMessageSchema>;
 export type RealtimeAudioChunkMessage = z.infer<typeof RealtimeAudioChunkMessageSchema>;
 export type SendAgentMessage = z.infer<typeof SendAgentMessageSchema>;
-export type TranscribeAudioRequest = z.infer<typeof TranscribeAudioRequestSchema>;
+export type DictationStreamStartMessage = z.infer<typeof DictationStreamStartMessageSchema>;
+export type DictationStreamChunkMessage = z.infer<typeof DictationStreamChunkMessageSchema>;
+export type DictationStreamFinishMessage = z.infer<typeof DictationStreamFinishMessageSchema>;
+export type DictationStreamCancelMessage = z.infer<typeof DictationStreamCancelMessageSchema>;
 export type CreateAgentRequestMessage = z.infer<typeof CreateAgentRequestMessageSchema>;
 export type ListProviderModelsRequestMessage = z.infer<
   typeof ListProviderModelsRequestMessageSchema