Add streaming ASR partial transcripts and reset hosts to env default

- Stream partial transcripts to the client during dictation for live feedback
- Use OpenAI semantic VAD instead of manual commit intervals for better word boundaries
- Append silence tail on finish to flush final utterance through VAD
- Skip committing silence-only audio tails
- Reset hosts to env-defined daemon when env var is present, overriding stored state
This commit is contained in:
Mohamed Boudra
2026-01-24 19:01:14 +07:00
parent 9b82c02fd2
commit b498c9ea81
8 changed files with 290 additions and 79 deletions
@@ -3,6 +3,7 @@ export type DictationStatus = "idle" | "recording" | "uploading" | "failed";
export type UseDictationOptions = {
client: import("@server/client/daemon-client-v2").DaemonClientV2 | null;
onTranscript: (text: string, meta: { requestId: string }) => void;
onPartialTranscript?: (text: string, meta: { requestId: string }) => void;
onError?: (error: Error) => void;
onPermanentFailure?: (error: Error, context: { requestId: string }) => void;
canStart?: () => boolean;
@@ -14,6 +15,7 @@ export type UseDictationOptions = {
export type UseDictationResult = {
isRecording: boolean;
isProcessing: boolean;
partialTranscript: string;
volume: number;
duration: number;
error: string | null;
+32
View File
@@ -17,6 +17,7 @@ export function useDictation(options: UseDictationOptions): UseDictationResult {
const {
client,
onTranscript,
onPartialTranscript,
onError,
onPermanentFailure,
canStart,
@@ -27,6 +28,7 @@ export function useDictation(options: UseDictationOptions): UseDictationResult {
const [isRecording, setIsRecording] = useState(false);
const [isProcessing, setIsProcessing] = useState(false);
const [partialTranscript, setPartialTranscript] = useState("");
const [duration, setDuration] = useState(0);
const [error, setError] = useState<string | null>(null);
const [status, setStatus] = useState<DictationStatus>("idle");
@@ -36,6 +38,11 @@ export function useDictation(options: UseDictationOptions): UseDictationResult {
onTranscriptRef.current = onTranscript;
}, [onTranscript]);
const onPartialTranscriptRef = useRef(onPartialTranscript);
useEffect(() => {
onPartialTranscriptRef.current = onPartialTranscript;
}, [onPartialTranscript]);
const onErrorRef = useRef(onError);
useEffect(() => {
onErrorRef.current = onError;
@@ -123,6 +130,7 @@ export function useDictation(options: UseDictationOptions): UseDictationResult {
const clearStreamingState = useCallback(() => {
senderRef.current?.clearAll();
setPartialTranscript("");
}, []);
const startNewStream = useCallback(
@@ -155,6 +163,27 @@ export function useDictation(options: UseDictationOptions): UseDictationResult {
});
}, [client, startNewStream]);
useEffect(() => {
if (!client) {
return;
}
return client.on("dictation_stream_partial", (message) => {
if (message.type !== "dictation_stream_partial") {
return;
}
const activeDictationId = senderRef.current?.getDictationId();
if (!activeDictationId) {
return;
}
if (message.payload.dictationId !== activeDictationId) {
return;
}
const next = message.payload.text ?? "";
setPartialTranscript(next);
onPartialTranscriptRef.current?.(next, { requestId: generateMessageId() });
});
}, [client]);
const audio = useDictationAudioSource({
onPcmSegment: (audioData) => {
senderRef.current?.enqueueSegment(audioData);
@@ -171,6 +200,7 @@ export function useDictation(options: UseDictationOptions): UseDictationResult {
const handleStreamingTranscriptionSuccess = useCallback(
(text: string, requestId: string) => {
setIsProcessing(false);
setPartialTranscript("");
setDuration(0);
setStatus("idle");
@@ -219,6 +249,7 @@ export function useDictation(options: UseDictationOptions): UseDictationResult {
actionGateRef.current.starting = true;
setError(null);
setPartialTranscript("");
setDuration(0);
setIsProcessing(false);
setStatus("recording");
@@ -434,6 +465,7 @@ export function useDictation(options: UseDictationOptions): UseDictationResult {
return {
isRecording,
isProcessing,
partialTranscript,
volume: audio.volume,
duration,
error,