feat(app): move realtime voice controls into input overlay
This commit is contained in:
@@ -53,8 +53,6 @@ import { ToolCallSheetProvider } from "./tool-call-sheet";
|
|||||||
import { createMarkdownStyles } from "@/styles/markdown-styles";
|
import { createMarkdownStyles } from "@/styles/markdown-styles";
|
||||||
import { MAX_CONTENT_WIDTH } from "@/constants/layout";
|
import { MAX_CONTENT_WIDTH } from "@/constants/layout";
|
||||||
import { isPerfLoggingEnabled, measurePayload, perfLog } from "@/utils/perf";
|
import { isPerfLoggingEnabled, measurePayload, perfLog } from "@/utils/perf";
|
||||||
import { VoiceCompactIndicator } from "./voice-compact-indicator";
|
|
||||||
import { useVoice } from "@/contexts/voice-context";
|
|
||||||
|
|
||||||
const isUserMessageItem = (item?: StreamItem) => item?.kind === "user_message";
|
const isUserMessageItem = (item?: StreamItem) => item?.kind === "user_message";
|
||||||
const isToolSequenceItem = (item?: StreamItem) =>
|
const isToolSequenceItem = (item?: StreamItem) =>
|
||||||
@@ -97,7 +95,6 @@ export function AgentStreamView({
|
|||||||
const flatListRef = useRef<FlatList<StreamItem>>(null);
|
const flatListRef = useRef<FlatList<StreamItem>>(null);
|
||||||
const { theme } = useUnistyles();
|
const { theme } = useUnistyles();
|
||||||
const insets = useSafeAreaInsets();
|
const insets = useSafeAreaInsets();
|
||||||
const { isVoiceModeForAgent } = useVoice();
|
|
||||||
const [isNearBottom, setIsNearBottom] = useState(true);
|
const [isNearBottom, setIsNearBottom] = useState(true);
|
||||||
const hasScrolledInitially = useRef(false);
|
const hasScrolledInitially = useRef(false);
|
||||||
const hasAutoScrolledOnce = useRef(false);
|
const hasAutoScrolledOnce = useRef(false);
|
||||||
@@ -109,7 +106,6 @@ export function AgentStreamView({
|
|||||||
|
|
||||||
// Get serverId (fallback to agent's serverId if not provided)
|
// Get serverId (fallback to agent's serverId if not provided)
|
||||||
const resolvedServerId = serverId ?? agent.serverId ?? "";
|
const resolvedServerId = serverId ?? agent.serverId ?? "";
|
||||||
const isVoiceMode = isVoiceModeForAgent(resolvedServerId, agentId);
|
|
||||||
|
|
||||||
const client = useSessionStore(
|
const client = useSessionStore(
|
||||||
(state) => state.sessions[resolvedServerId]?.client ?? null
|
(state) => state.sessions[resolvedServerId]?.client ?? null
|
||||||
@@ -516,7 +512,7 @@ export function AgentStreamView({
|
|||||||
}, [agentId, pendingPermissionItems.length, streamHead, streamItems]);
|
}, [agentId, pendingPermissionItems.length, streamHead, streamItems]);
|
||||||
|
|
||||||
const showWorkingIndicator = agent.status === "running";
|
const showWorkingIndicator = agent.status === "running";
|
||||||
const showBottomBar = showWorkingIndicator || isVoiceMode;
|
const showBottomBar = showWorkingIndicator;
|
||||||
|
|
||||||
const listHeaderComponent = useMemo(() => {
|
const listHeaderComponent = useMemo(() => {
|
||||||
const hasPermissions = pendingPermissionItems.length > 0;
|
const hasPermissions = pendingPermissionItems.length > 0;
|
||||||
@@ -563,14 +559,7 @@ export function AgentStreamView({
|
|||||||
})
|
})
|
||||||
: null}
|
: null}
|
||||||
|
|
||||||
{showBottomBar ? (
|
{showBottomBar ? <View style={stylesheet.bottomBarWrapper}>{leftContent}</View> : null}
|
||||||
<View style={stylesheet.bottomBarWrapper}>
|
|
||||||
<View style={stylesheet.bottomBarLeft}>{leftContent}</View>
|
|
||||||
<View style={stylesheet.bottomBarRight}>
|
|
||||||
{isVoiceMode ? <VoiceCompactIndicator /> : null}
|
|
||||||
</View>
|
|
||||||
</View>
|
|
||||||
) : null}
|
|
||||||
</View>
|
</View>
|
||||||
</View>
|
</View>
|
||||||
);
|
);
|
||||||
@@ -582,7 +571,6 @@ export function AgentStreamView({
|
|||||||
renderStreamItemContent,
|
renderStreamItemContent,
|
||||||
tightGap,
|
tightGap,
|
||||||
showBottomBar,
|
showBottomBar,
|
||||||
isVoiceMode,
|
|
||||||
]);
|
]);
|
||||||
|
|
||||||
const flatListExtraData = useMemo(
|
const flatListExtraData = useMemo(
|
||||||
@@ -590,13 +578,11 @@ export function AgentStreamView({
|
|||||||
pendingPermissionCount: pendingPermissionItems.length,
|
pendingPermissionCount: pendingPermissionItems.length,
|
||||||
showWorkingIndicator,
|
showWorkingIndicator,
|
||||||
showBottomBar,
|
showBottomBar,
|
||||||
isVoiceMode,
|
|
||||||
}),
|
}),
|
||||||
[
|
[
|
||||||
pendingPermissionItems.length,
|
pendingPermissionItems.length,
|
||||||
showWorkingIndicator,
|
showWorkingIndicator,
|
||||||
showBottomBar,
|
showBottomBar,
|
||||||
isVoiceMode,
|
|
||||||
]
|
]
|
||||||
);
|
);
|
||||||
|
|
||||||
@@ -1252,21 +1238,13 @@ const stylesheet = StyleSheet.create((theme) => ({
|
|||||||
bottomBarWrapper: {
|
bottomBarWrapper: {
|
||||||
flexDirection: "row",
|
flexDirection: "row",
|
||||||
alignItems: "center",
|
alignItems: "center",
|
||||||
justifyContent: "space-between",
|
justifyContent: "flex-start",
|
||||||
paddingLeft: 3,
|
paddingLeft: 3,
|
||||||
paddingRight: 3,
|
paddingRight: 3,
|
||||||
paddingTop: theme.spacing[3],
|
paddingTop: theme.spacing[3],
|
||||||
paddingBottom: theme.spacing[2],
|
paddingBottom: theme.spacing[2],
|
||||||
gap: theme.spacing[2],
|
gap: theme.spacing[2],
|
||||||
},
|
},
|
||||||
bottomBarLeft: {
|
|
||||||
flex: 1,
|
|
||||||
alignItems: "flex-start",
|
|
||||||
},
|
|
||||||
bottomBarRight: {
|
|
||||||
flexShrink: 0,
|
|
||||||
alignItems: "flex-end",
|
|
||||||
},
|
|
||||||
workingIndicatorBubble: {
|
workingIndicatorBubble: {
|
||||||
flexDirection: "row",
|
flexDirection: "row",
|
||||||
alignItems: "center",
|
alignItems: "center",
|
||||||
|
|||||||
@@ -27,6 +27,7 @@ import Animated, {
|
|||||||
} from "react-native-reanimated";
|
} from "react-native-reanimated";
|
||||||
import { useDictation } from "@/hooks/use-dictation";
|
import { useDictation } from "@/hooks/use-dictation";
|
||||||
import { DictationOverlay } from "./dictation-controls";
|
import { DictationOverlay } from "./dictation-controls";
|
||||||
|
import { RealtimeVoiceOverlay } from "./realtime-voice-overlay";
|
||||||
import type { DaemonClient } from "@server/client/daemon-client";
|
import type { DaemonClient } from "@server/client/daemon-client";
|
||||||
import { usePanelStore } from "@/stores/panel-store";
|
import { usePanelStore } from "@/stores/panel-store";
|
||||||
import { useVoiceOptional } from "@/contexts/voice-context";
|
import { useVoiceOptional } from "@/contexts/voice-context";
|
||||||
@@ -250,6 +251,10 @@ export const MessageInput = forwardRef<MessageInputRef, MessageInputProps>(
|
|||||||
!!voiceServerId &&
|
!!voiceServerId &&
|
||||||
!!voiceAgentId &&
|
!!voiceAgentId &&
|
||||||
voice.isVoiceModeForAgent(voiceServerId, voiceAgentId);
|
voice.isVoiceModeForAgent(voiceServerId, voiceAgentId);
|
||||||
|
const showDictationOverlay =
|
||||||
|
isDictating || isDictationProcessing || dictationStatus === "failed";
|
||||||
|
const showRealtimeOverlay = isRealtimeVoiceForCurrentAgent;
|
||||||
|
const showOverlay = showDictationOverlay || showRealtimeOverlay;
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
if (isDictating || isDictationProcessing) {
|
if (isDictating || isDictationProcessing) {
|
||||||
@@ -302,14 +307,10 @@ export const MessageInput = forwardRef<MessageInputRef, MessageInputProps>(
|
|||||||
|
|
||||||
// Animate overlay
|
// Animate overlay
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
const showOverlay =
|
|
||||||
isDictating ||
|
|
||||||
isDictationProcessing ||
|
|
||||||
dictationStatus === "failed";
|
|
||||||
overlayTransition.value = withTiming(showOverlay ? 1 : 0, {
|
overlayTransition.value = withTiming(showOverlay ? 1 : 0, {
|
||||||
duration: 200,
|
duration: 200,
|
||||||
});
|
});
|
||||||
}, [isDictating, isDictationProcessing, dictationStatus, overlayTransition]);
|
}, [overlayTransition, showOverlay]);
|
||||||
|
|
||||||
const overlayAnimatedStyle = useAnimatedStyle(() => ({
|
const overlayAnimatedStyle = useAnimatedStyle(() => ({
|
||||||
opacity: overlayTransition.value,
|
opacity: overlayTransition.value,
|
||||||
@@ -361,6 +362,25 @@ export const MessageInput = forwardRef<MessageInputRef, MessageInputProps>(
|
|||||||
discardFailedDictation();
|
discardFailedDictation();
|
||||||
}, [discardFailedDictation]);
|
}, [discardFailedDictation]);
|
||||||
|
|
||||||
|
const handleStopRealtimeVoice = useCallback(async () => {
|
||||||
|
if (!voice || !isRealtimeVoiceForCurrentAgent) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const tasks: Promise<unknown>[] = [];
|
||||||
|
if (isAgentRunning && client && voiceAgentId) {
|
||||||
|
tasks.push(client.cancelAgent(voiceAgentId));
|
||||||
|
}
|
||||||
|
tasks.push(voice.stopVoice());
|
||||||
|
|
||||||
|
const results = await Promise.allSettled(tasks);
|
||||||
|
results.forEach((result) => {
|
||||||
|
if (result.status === "rejected") {
|
||||||
|
console.error("[MessageInput] Failed to stop realtime voice", result.reason);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}, [client, isAgentRunning, isRealtimeVoiceForCurrentAgent, voice, voiceAgentId]);
|
||||||
|
|
||||||
const handleSendMessage = useCallback(() => {
|
const handleSendMessage = useCallback(() => {
|
||||||
const trimmed = value.trim();
|
const trimmed = value.trim();
|
||||||
if (!trimmed && images.length === 0) return;
|
if (!trimmed && images.length === 0) return;
|
||||||
@@ -602,7 +622,9 @@ export const MessageInput = forwardRef<MessageInputRef, MessageInputProps>(
|
|||||||
multiline
|
multiline
|
||||||
scrollEnabled={IS_WEB ? inputHeight >= MAX_INPUT_HEIGHT : true}
|
scrollEnabled={IS_WEB ? inputHeight >= MAX_INPUT_HEIGHT : true}
|
||||||
onContentSizeChange={handleContentSizeChange}
|
onContentSizeChange={handleContentSizeChange}
|
||||||
editable={!isDictating && isConnected && !disabled}
|
editable={
|
||||||
|
!isDictating && !isRealtimeVoiceForCurrentAgent && isConnected && !disabled
|
||||||
|
}
|
||||||
onKeyPress={
|
onKeyPress={
|
||||||
shouldHandleDesktopSubmit ? handleDesktopKeyPress : undefined
|
shouldHandleDesktopSubmit ? handleDesktopKeyPress : undefined
|
||||||
}
|
}
|
||||||
@@ -705,27 +727,41 @@ export const MessageInput = forwardRef<MessageInputRef, MessageInputProps>(
|
|||||||
|
|
||||||
{/* Dictation overlay */}
|
{/* Dictation overlay */}
|
||||||
<Animated.View style={[styles.overlayContainer, overlayAnimatedStyle]}>
|
<Animated.View style={[styles.overlayContainer, overlayAnimatedStyle]}>
|
||||||
<DictationOverlay
|
{showDictationOverlay ? (
|
||||||
volume={dictationVolume}
|
<DictationOverlay
|
||||||
duration={dictationDuration}
|
volume={dictationVolume}
|
||||||
isRecording={isDictating}
|
duration={dictationDuration}
|
||||||
isProcessing={isDictationProcessing}
|
isRecording={isDictating}
|
||||||
status={dictationStatus}
|
isProcessing={isDictationProcessing}
|
||||||
errorText={dictationStatus === "failed" ? dictationError ?? undefined : undefined}
|
status={dictationStatus}
|
||||||
onCancel={handleCancelRecording}
|
errorText={dictationStatus === "failed" ? dictationError ?? undefined : undefined}
|
||||||
onAccept={handleAcceptRecording}
|
onCancel={handleCancelRecording}
|
||||||
onAcceptAndSend={handleAcceptAndSendRecording}
|
onAccept={handleAcceptRecording}
|
||||||
onRetry={
|
onAcceptAndSend={handleAcceptAndSendRecording}
|
||||||
dictationStatus === "failed"
|
onRetry={
|
||||||
? handleRetryFailedRecording
|
dictationStatus === "failed"
|
||||||
: undefined
|
? handleRetryFailedRecording
|
||||||
}
|
: undefined
|
||||||
onDiscard={
|
}
|
||||||
dictationStatus === "failed"
|
onDiscard={
|
||||||
? handleDiscardFailedRecording
|
dictationStatus === "failed"
|
||||||
: undefined
|
? handleDiscardFailedRecording
|
||||||
}
|
: undefined
|
||||||
/>
|
}
|
||||||
|
/>
|
||||||
|
) : showRealtimeOverlay && voice ? (
|
||||||
|
<RealtimeVoiceOverlay
|
||||||
|
volume={voice.volume}
|
||||||
|
isMuted={voice.isMuted}
|
||||||
|
isDetecting={voice.isDetecting}
|
||||||
|
isSpeaking={voice.isSpeaking}
|
||||||
|
isSwitching={voice.isVoiceSwitching}
|
||||||
|
onToggleMute={voice.toggleMute}
|
||||||
|
onStop={() => {
|
||||||
|
void handleStopRealtimeVoice();
|
||||||
|
}}
|
||||||
|
/>
|
||||||
|
) : null}
|
||||||
</Animated.View>
|
</Animated.View>
|
||||||
</View>
|
</View>
|
||||||
);
|
);
|
||||||
|
|||||||
@@ -0,0 +1,138 @@
|
|||||||
|
import { ActivityIndicator, Pressable, View } from "react-native";
|
||||||
|
import { StyleSheet, useUnistyles } from "react-native-unistyles";
|
||||||
|
import { Mic, MicOff, Square } from "lucide-react-native";
|
||||||
|
import { FOOTER_HEIGHT } from "@/constants/layout";
|
||||||
|
import { VolumeMeter } from "./volume-meter";
|
||||||
|
|
||||||
|
interface RealtimeVoiceOverlayProps {
|
||||||
|
volume: number;
|
||||||
|
isMuted: boolean;
|
||||||
|
isDetecting: boolean;
|
||||||
|
isSpeaking: boolean;
|
||||||
|
isSwitching: boolean;
|
||||||
|
onToggleMute: () => void;
|
||||||
|
onStop: () => void;
|
||||||
|
}
|
||||||
|
|
||||||
|
const OVERLAY_BUTTON_SIZE = 44;
|
||||||
|
const OVERLAY_VERTICAL_PADDING = (FOOTER_HEIGHT - OVERLAY_BUTTON_SIZE) / 2;
|
||||||
|
|
||||||
|
export function RealtimeVoiceOverlay({
|
||||||
|
volume,
|
||||||
|
isMuted,
|
||||||
|
isDetecting,
|
||||||
|
isSpeaking,
|
||||||
|
isSwitching,
|
||||||
|
onToggleMute,
|
||||||
|
onStop,
|
||||||
|
}: RealtimeVoiceOverlayProps) {
|
||||||
|
const { theme } = useUnistyles();
|
||||||
|
|
||||||
|
return (
|
||||||
|
<View style={styles.container}>
|
||||||
|
<View style={styles.meterContainer}>
|
||||||
|
<VolumeMeter
|
||||||
|
volume={volume}
|
||||||
|
isMuted={isMuted}
|
||||||
|
isDetecting={isDetecting}
|
||||||
|
isSpeaking={isSpeaking}
|
||||||
|
orientation="horizontal"
|
||||||
|
/>
|
||||||
|
</View>
|
||||||
|
|
||||||
|
<View style={styles.actionsContainer}>
|
||||||
|
<Pressable
|
||||||
|
onPress={onToggleMute}
|
||||||
|
disabled={isSwitching}
|
||||||
|
accessibilityRole="button"
|
||||||
|
accessibilityLabel={isMuted ? "Unmute realtime voice" : "Mute realtime voice"}
|
||||||
|
style={[
|
||||||
|
styles.actionButton,
|
||||||
|
styles.muteButton,
|
||||||
|
isMuted ? styles.muteButtonMuted : undefined,
|
||||||
|
isSwitching ? styles.buttonDisabled : undefined,
|
||||||
|
]}
|
||||||
|
>
|
||||||
|
{isMuted ? (
|
||||||
|
<MicOff size={20} color={theme.colors.palette.white} strokeWidth={2.5} />
|
||||||
|
) : (
|
||||||
|
<Mic size={20} color={theme.colors.foreground} strokeWidth={2.5} />
|
||||||
|
)}
|
||||||
|
</Pressable>
|
||||||
|
|
||||||
|
<Pressable
|
||||||
|
onPress={onStop}
|
||||||
|
disabled={isSwitching}
|
||||||
|
accessibilityRole="button"
|
||||||
|
accessibilityLabel="Stop realtime voice and interrupt turn"
|
||||||
|
style={[
|
||||||
|
styles.actionButton,
|
||||||
|
styles.stopButton,
|
||||||
|
isSwitching ? styles.buttonDisabled : undefined,
|
||||||
|
]}
|
||||||
|
>
|
||||||
|
{isSwitching ? (
|
||||||
|
<ActivityIndicator size="small" color={theme.colors.palette.white} />
|
||||||
|
) : (
|
||||||
|
<Square
|
||||||
|
size={20}
|
||||||
|
color={theme.colors.palette.white}
|
||||||
|
fill={theme.colors.palette.white}
|
||||||
|
strokeWidth={2.5}
|
||||||
|
/>
|
||||||
|
)}
|
||||||
|
</Pressable>
|
||||||
|
</View>
|
||||||
|
</View>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
const styles = StyleSheet.create((theme) => ({
|
||||||
|
container: {
|
||||||
|
flexDirection: "row",
|
||||||
|
alignItems: "center",
|
||||||
|
width: "100%",
|
||||||
|
height: FOOTER_HEIGHT,
|
||||||
|
borderRadius: theme.borderRadius["2xl"],
|
||||||
|
justifyContent: "space-between",
|
||||||
|
paddingHorizontal: theme.spacing[4],
|
||||||
|
paddingVertical: OVERLAY_VERTICAL_PADDING,
|
||||||
|
backgroundColor: theme.colors.surface1,
|
||||||
|
borderWidth: theme.borderWidth[1],
|
||||||
|
borderColor: theme.colors.border,
|
||||||
|
},
|
||||||
|
meterContainer: {
|
||||||
|
flex: 1,
|
||||||
|
alignItems: "center",
|
||||||
|
justifyContent: "center",
|
||||||
|
},
|
||||||
|
actionsContainer: {
|
||||||
|
flexDirection: "row",
|
||||||
|
alignItems: "center",
|
||||||
|
gap: theme.spacing[2],
|
||||||
|
},
|
||||||
|
actionButton: {
|
||||||
|
width: OVERLAY_BUTTON_SIZE,
|
||||||
|
height: OVERLAY_BUTTON_SIZE,
|
||||||
|
borderRadius: theme.borderRadius.full,
|
||||||
|
alignItems: "center",
|
||||||
|
justifyContent: "center",
|
||||||
|
},
|
||||||
|
muteButton: {
|
||||||
|
backgroundColor: theme.colors.surface0,
|
||||||
|
borderWidth: theme.borderWidth[1],
|
||||||
|
borderColor: theme.colors.border,
|
||||||
|
},
|
||||||
|
muteButtonMuted: {
|
||||||
|
backgroundColor: theme.colors.palette.red[600],
|
||||||
|
borderColor: theme.colors.palette.red[800],
|
||||||
|
},
|
||||||
|
stopButton: {
|
||||||
|
backgroundColor: theme.colors.palette.red[600],
|
||||||
|
borderWidth: theme.borderWidth[1],
|
||||||
|
borderColor: theme.colors.palette.red[800],
|
||||||
|
},
|
||||||
|
buttonDisabled: {
|
||||||
|
opacity: 0.5,
|
||||||
|
},
|
||||||
|
}));
|
||||||
Reference in New Issue
Block a user