Add streaming ASR partial transcripts and reset hosts to env default
- Stream partial transcripts to the client during dictation for live feedback - Use OpenAI semantic VAD instead of manual commit intervals for better word boundaries - Append silence tail on finish to flush final utterance through VAD - Skip committing silence-only audio tails - Reset hosts to env-defined daemon when env var is present, overriding stored state
This commit is contained in:
@@ -8,6 +8,7 @@ import type { DictationStatus } from "@/hooks/use-dictation";
|
||||
interface DictationControlsProps {
|
||||
volume: number;
|
||||
duration: number;
|
||||
transcript?: string;
|
||||
isRecording: boolean;
|
||||
isProcessing: boolean;
|
||||
status: DictationStatus;
|
||||
@@ -131,6 +132,7 @@ export function DictationControls({
|
||||
export function DictationOverlay({
|
||||
volume,
|
||||
duration,
|
||||
transcript,
|
||||
isRecording,
|
||||
isProcessing,
|
||||
status,
|
||||
@@ -169,21 +171,34 @@ export function DictationOverlay({
|
||||
</Pressable>
|
||||
|
||||
<View style={overlayStyles.centerContainer}>
|
||||
<VolumeMeter
|
||||
volume={volume}
|
||||
isMuted={false}
|
||||
isDetecting
|
||||
isSpeaking={false}
|
||||
orientation="horizontal"
|
||||
/>
|
||||
<Text
|
||||
style={[
|
||||
overlayStyles.timerText,
|
||||
{ color: theme.colors.palette.white },
|
||||
]}
|
||||
>
|
||||
{formatDuration(duration)}
|
||||
</Text>
|
||||
<View style={overlayStyles.meterRow}>
|
||||
<VolumeMeter
|
||||
volume={volume}
|
||||
isMuted={false}
|
||||
isDetecting
|
||||
isSpeaking={false}
|
||||
orientation="horizontal"
|
||||
/>
|
||||
<Text
|
||||
style={[
|
||||
overlayStyles.timerText,
|
||||
{ color: theme.colors.palette.white },
|
||||
]}
|
||||
>
|
||||
{formatDuration(duration)}
|
||||
</Text>
|
||||
</View>
|
||||
{!!transcript && (
|
||||
<Text
|
||||
numberOfLines={2}
|
||||
style={[
|
||||
overlayStyles.transcriptText,
|
||||
{ color: theme.colors.palette.white },
|
||||
]}
|
||||
>
|
||||
{transcript}
|
||||
</Text>
|
||||
)}
|
||||
</View>
|
||||
|
||||
<View style={overlayStyles.actionButtonsContainer}>
|
||||
@@ -334,6 +349,12 @@ const overlayStyles = StyleSheet.create((theme) => ({
|
||||
},
|
||||
centerContainer: {
|
||||
flex: 1,
|
||||
flexDirection: "column",
|
||||
alignItems: "center",
|
||||
justifyContent: "center",
|
||||
gap: theme.spacing[2],
|
||||
},
|
||||
meterRow: {
|
||||
flexDirection: "row",
|
||||
alignItems: "center",
|
||||
justifyContent: "center",
|
||||
@@ -344,6 +365,13 @@ const overlayStyles = StyleSheet.create((theme) => ({
|
||||
fontWeight: theme.fontWeight.semibold,
|
||||
fontVariant: ["tabular-nums"],
|
||||
},
|
||||
transcriptText: {
|
||||
fontSize: theme.fontSize.sm,
|
||||
fontWeight: theme.fontWeight.normal,
|
||||
textAlign: "center",
|
||||
paddingHorizontal: theme.spacing[2],
|
||||
opacity: 0.9,
|
||||
},
|
||||
actionButtonsContainer: {
|
||||
flexDirection: "row",
|
||||
alignItems: "center",
|
||||
|
||||
@@ -191,6 +191,7 @@ export const MessageInput = forwardRef<MessageInputRef, MessageInputProps>(
|
||||
const {
|
||||
isRecording: isDictating,
|
||||
isProcessing: isDictationProcessing,
|
||||
partialTranscript: dictationPartialTranscript,
|
||||
volume: dictationVolume,
|
||||
duration: dictationDuration,
|
||||
status: dictationStatus,
|
||||
@@ -591,6 +592,7 @@ export const MessageInput = forwardRef<MessageInputRef, MessageInputProps>(
|
||||
<DictationOverlay
|
||||
volume={dictationVolume}
|
||||
duration={dictationDuration}
|
||||
transcript={dictationPartialTranscript}
|
||||
isRecording={isDictating}
|
||||
isProcessing={isDictationProcessing}
|
||||
status={dictationStatus}
|
||||
|
||||
@@ -329,6 +329,14 @@ function migrateLegacyToHostProfile(legacy: LegacyDaemonProfile): HostProfile {
|
||||
|
||||
async function loadDaemonRegistryFromStorage(): Promise<HostProfile[]> {
|
||||
try {
|
||||
// When env vars define a default daemon, always reset to that daemon only.
|
||||
// This ensures the app uses the configured daemon regardless of stored state.
|
||||
const envDefaults = parseEnvDaemonDefaults();
|
||||
if (envDefaults.length > 0) {
|
||||
await AsyncStorage.setItem(REGISTRY_STORAGE_KEY, JSON.stringify(envDefaults));
|
||||
return envDefaults;
|
||||
}
|
||||
|
||||
const stored = await AsyncStorage.getItem(REGISTRY_STORAGE_KEY);
|
||||
if (stored) {
|
||||
const parsed = JSON.parse(stored) as unknown;
|
||||
@@ -390,12 +398,6 @@ async function loadDaemonRegistryFromStorage(): Promise<HostProfile[]> {
|
||||
}
|
||||
}
|
||||
|
||||
const envDefaults = parseEnvDaemonDefaults();
|
||||
if (envDefaults.length > 0) {
|
||||
await AsyncStorage.setItem(REGISTRY_STORAGE_KEY, JSON.stringify(envDefaults));
|
||||
return envDefaults;
|
||||
}
|
||||
|
||||
// No implicit localhost fallback: a fresh install starts with zero hosts.
|
||||
return [];
|
||||
} catch (error) {
|
||||
|
||||
@@ -3,6 +3,7 @@ export type DictationStatus = "idle" | "recording" | "uploading" | "failed";
|
||||
export type UseDictationOptions = {
|
||||
client: import("@server/client/daemon-client-v2").DaemonClientV2 | null;
|
||||
onTranscript: (text: string, meta: { requestId: string }) => void;
|
||||
onPartialTranscript?: (text: string, meta: { requestId: string }) => void;
|
||||
onError?: (error: Error) => void;
|
||||
onPermanentFailure?: (error: Error, context: { requestId: string }) => void;
|
||||
canStart?: () => boolean;
|
||||
@@ -14,6 +15,7 @@ export type UseDictationOptions = {
|
||||
export type UseDictationResult = {
|
||||
isRecording: boolean;
|
||||
isProcessing: boolean;
|
||||
partialTranscript: string;
|
||||
volume: number;
|
||||
duration: number;
|
||||
error: string | null;
|
||||
|
||||
@@ -17,6 +17,7 @@ export function useDictation(options: UseDictationOptions): UseDictationResult {
|
||||
const {
|
||||
client,
|
||||
onTranscript,
|
||||
onPartialTranscript,
|
||||
onError,
|
||||
onPermanentFailure,
|
||||
canStart,
|
||||
@@ -27,6 +28,7 @@ export function useDictation(options: UseDictationOptions): UseDictationResult {
|
||||
|
||||
const [isRecording, setIsRecording] = useState(false);
|
||||
const [isProcessing, setIsProcessing] = useState(false);
|
||||
const [partialTranscript, setPartialTranscript] = useState("");
|
||||
const [duration, setDuration] = useState(0);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
const [status, setStatus] = useState<DictationStatus>("idle");
|
||||
@@ -36,6 +38,11 @@ export function useDictation(options: UseDictationOptions): UseDictationResult {
|
||||
onTranscriptRef.current = onTranscript;
|
||||
}, [onTranscript]);
|
||||
|
||||
const onPartialTranscriptRef = useRef(onPartialTranscript);
|
||||
useEffect(() => {
|
||||
onPartialTranscriptRef.current = onPartialTranscript;
|
||||
}, [onPartialTranscript]);
|
||||
|
||||
const onErrorRef = useRef(onError);
|
||||
useEffect(() => {
|
||||
onErrorRef.current = onError;
|
||||
@@ -123,6 +130,7 @@ export function useDictation(options: UseDictationOptions): UseDictationResult {
|
||||
|
||||
const clearStreamingState = useCallback(() => {
|
||||
senderRef.current?.clearAll();
|
||||
setPartialTranscript("");
|
||||
}, []);
|
||||
|
||||
const startNewStream = useCallback(
|
||||
@@ -155,6 +163,27 @@ export function useDictation(options: UseDictationOptions): UseDictationResult {
|
||||
});
|
||||
}, [client, startNewStream]);
|
||||
|
||||
useEffect(() => {
|
||||
if (!client) {
|
||||
return;
|
||||
}
|
||||
return client.on("dictation_stream_partial", (message) => {
|
||||
if (message.type !== "dictation_stream_partial") {
|
||||
return;
|
||||
}
|
||||
const activeDictationId = senderRef.current?.getDictationId();
|
||||
if (!activeDictationId) {
|
||||
return;
|
||||
}
|
||||
if (message.payload.dictationId !== activeDictationId) {
|
||||
return;
|
||||
}
|
||||
const next = message.payload.text ?? "";
|
||||
setPartialTranscript(next);
|
||||
onPartialTranscriptRef.current?.(next, { requestId: generateMessageId() });
|
||||
});
|
||||
}, [client]);
|
||||
|
||||
const audio = useDictationAudioSource({
|
||||
onPcmSegment: (audioData) => {
|
||||
senderRef.current?.enqueueSegment(audioData);
|
||||
@@ -171,6 +200,7 @@ export function useDictation(options: UseDictationOptions): UseDictationResult {
|
||||
const handleStreamingTranscriptionSuccess = useCallback(
|
||||
(text: string, requestId: string) => {
|
||||
setIsProcessing(false);
|
||||
setPartialTranscript("");
|
||||
setDuration(0);
|
||||
setStatus("idle");
|
||||
|
||||
@@ -219,6 +249,7 @@ export function useDictation(options: UseDictationOptions): UseDictationResult {
|
||||
|
||||
actionGateRef.current.starting = true;
|
||||
setError(null);
|
||||
setPartialTranscript("");
|
||||
setDuration(0);
|
||||
setIsProcessing(false);
|
||||
setStatus("recording");
|
||||
@@ -434,6 +465,7 @@ export function useDictation(options: UseDictationOptions): UseDictationResult {
|
||||
return {
|
||||
isRecording,
|
||||
isProcessing,
|
||||
partialTranscript,
|
||||
volume: audio.volume,
|
||||
duration,
|
||||
error,
|
||||
|
||||
Reference in New Issue
Block a user