Add streaming ASR partial transcripts and reset hosts to env default

- Stream partial transcripts to the client during dictation for live feedback
- Use OpenAI semantic VAD instead of manual commit intervals for better word boundaries
- Append silence tail on finish to flush final utterance through VAD
- Skip committing silence-only audio tails
- Reset hosts to env-defined daemon when env var is present, overriding stored state
This commit is contained in:
Mohamed Boudra
2026-01-24 19:01:14 +07:00
parent 9b82c02fd2
commit b498c9ea81
8 changed files with 290 additions and 79 deletions
@@ -8,6 +8,7 @@ import type { DictationStatus } from "@/hooks/use-dictation";
interface DictationControlsProps {
volume: number;
duration: number;
transcript?: string;
isRecording: boolean;
isProcessing: boolean;
status: DictationStatus;
@@ -131,6 +132,7 @@ export function DictationControls({
export function DictationOverlay({
volume,
duration,
transcript,
isRecording,
isProcessing,
status,
@@ -169,21 +171,34 @@ export function DictationOverlay({
</Pressable>
<View style={overlayStyles.centerContainer}>
<VolumeMeter
volume={volume}
isMuted={false}
isDetecting
isSpeaking={false}
orientation="horizontal"
/>
<Text
style={[
overlayStyles.timerText,
{ color: theme.colors.palette.white },
]}
>
{formatDuration(duration)}
</Text>
<View style={overlayStyles.meterRow}>
<VolumeMeter
volume={volume}
isMuted={false}
isDetecting
isSpeaking={false}
orientation="horizontal"
/>
<Text
style={[
overlayStyles.timerText,
{ color: theme.colors.palette.white },
]}
>
{formatDuration(duration)}
</Text>
</View>
{!!transcript && (
<Text
numberOfLines={2}
style={[
overlayStyles.transcriptText,
{ color: theme.colors.palette.white },
]}
>
{transcript}
</Text>
)}
</View>
<View style={overlayStyles.actionButtonsContainer}>
@@ -334,6 +349,12 @@ const overlayStyles = StyleSheet.create((theme) => ({
},
centerContainer: {
flex: 1,
flexDirection: "column",
alignItems: "center",
justifyContent: "center",
gap: theme.spacing[2],
},
meterRow: {
flexDirection: "row",
alignItems: "center",
justifyContent: "center",
@@ -344,6 +365,13 @@ const overlayStyles = StyleSheet.create((theme) => ({
fontWeight: theme.fontWeight.semibold,
fontVariant: ["tabular-nums"],
},
transcriptText: {
fontSize: theme.fontSize.sm,
fontWeight: theme.fontWeight.normal,
textAlign: "center",
paddingHorizontal: theme.spacing[2],
opacity: 0.9,
},
actionButtonsContainer: {
flexDirection: "row",
alignItems: "center",
@@ -191,6 +191,7 @@ export const MessageInput = forwardRef<MessageInputRef, MessageInputProps>(
const {
isRecording: isDictating,
isProcessing: isDictationProcessing,
partialTranscript: dictationPartialTranscript,
volume: dictationVolume,
duration: dictationDuration,
status: dictationStatus,
@@ -591,6 +592,7 @@ export const MessageInput = forwardRef<MessageInputRef, MessageInputProps>(
<DictationOverlay
volume={dictationVolume}
duration={dictationDuration}
transcript={dictationPartialTranscript}
isRecording={isDictating}
isProcessing={isDictationProcessing}
status={dictationStatus}
@@ -329,6 +329,14 @@ function migrateLegacyToHostProfile(legacy: LegacyDaemonProfile): HostProfile {
async function loadDaemonRegistryFromStorage(): Promise<HostProfile[]> {
try {
// When env vars define a default daemon, always reset to that daemon only.
// This ensures the app uses the configured daemon regardless of stored state.
const envDefaults = parseEnvDaemonDefaults();
if (envDefaults.length > 0) {
await AsyncStorage.setItem(REGISTRY_STORAGE_KEY, JSON.stringify(envDefaults));
return envDefaults;
}
const stored = await AsyncStorage.getItem(REGISTRY_STORAGE_KEY);
if (stored) {
const parsed = JSON.parse(stored) as unknown;
@@ -390,12 +398,6 @@ async function loadDaemonRegistryFromStorage(): Promise<HostProfile[]> {
}
}
const envDefaults = parseEnvDaemonDefaults();
if (envDefaults.length > 0) {
await AsyncStorage.setItem(REGISTRY_STORAGE_KEY, JSON.stringify(envDefaults));
return envDefaults;
}
// No implicit localhost fallback: a fresh install starts with zero hosts.
return [];
} catch (error) {
@@ -3,6 +3,7 @@ export type DictationStatus = "idle" | "recording" | "uploading" | "failed";
export type UseDictationOptions = {
client: import("@server/client/daemon-client-v2").DaemonClientV2 | null;
onTranscript: (text: string, meta: { requestId: string }) => void;
onPartialTranscript?: (text: string, meta: { requestId: string }) => void;
onError?: (error: Error) => void;
onPermanentFailure?: (error: Error, context: { requestId: string }) => void;
canStart?: () => boolean;
@@ -14,6 +15,7 @@ export type UseDictationOptions = {
export type UseDictationResult = {
isRecording: boolean;
isProcessing: boolean;
partialTranscript: string;
volume: number;
duration: number;
error: string | null;
+32
View File
@@ -17,6 +17,7 @@ export function useDictation(options: UseDictationOptions): UseDictationResult {
const {
client,
onTranscript,
onPartialTranscript,
onError,
onPermanentFailure,
canStart,
@@ -27,6 +28,7 @@ export function useDictation(options: UseDictationOptions): UseDictationResult {
const [isRecording, setIsRecording] = useState(false);
const [isProcessing, setIsProcessing] = useState(false);
const [partialTranscript, setPartialTranscript] = useState("");
const [duration, setDuration] = useState(0);
const [error, setError] = useState<string | null>(null);
const [status, setStatus] = useState<DictationStatus>("idle");
@@ -36,6 +38,11 @@ export function useDictation(options: UseDictationOptions): UseDictationResult {
onTranscriptRef.current = onTranscript;
}, [onTranscript]);
const onPartialTranscriptRef = useRef(onPartialTranscript);
useEffect(() => {
onPartialTranscriptRef.current = onPartialTranscript;
}, [onPartialTranscript]);
const onErrorRef = useRef(onError);
useEffect(() => {
onErrorRef.current = onError;
@@ -123,6 +130,7 @@ export function useDictation(options: UseDictationOptions): UseDictationResult {
const clearStreamingState = useCallback(() => {
senderRef.current?.clearAll();
setPartialTranscript("");
}, []);
const startNewStream = useCallback(
@@ -155,6 +163,27 @@ export function useDictation(options: UseDictationOptions): UseDictationResult {
});
}, [client, startNewStream]);
useEffect(() => {
if (!client) {
return;
}
return client.on("dictation_stream_partial", (message) => {
if (message.type !== "dictation_stream_partial") {
return;
}
const activeDictationId = senderRef.current?.getDictationId();
if (!activeDictationId) {
return;
}
if (message.payload.dictationId !== activeDictationId) {
return;
}
const next = message.payload.text ?? "";
setPartialTranscript(next);
onPartialTranscriptRef.current?.(next, { requestId: generateMessageId() });
});
}, [client]);
const audio = useDictationAudioSource({
onPcmSegment: (audioData) => {
senderRef.current?.enqueueSegment(audioData);
@@ -171,6 +200,7 @@ export function useDictation(options: UseDictationOptions): UseDictationResult {
const handleStreamingTranscriptionSuccess = useCallback(
(text: string, requestId: string) => {
setIsProcessing(false);
setPartialTranscript("");
setDuration(0);
setStatus("idle");
@@ -219,6 +249,7 @@ export function useDictation(options: UseDictationOptions): UseDictationResult {
actionGateRef.current.starting = true;
setError(null);
setPartialTranscript("");
setDuration(0);
setIsProcessing(false);
setStatus("recording");
@@ -434,6 +465,7 @@ export function useDictation(options: UseDictationOptions): UseDictationResult {
return {
isRecording,
isProcessing,
partialTranscript,
volume: audio.volume,
duration,
error,