```
refactor: reset message IDs on new turn and add activity log handlers - Reset assistant message and thought IDs when starting new agent turn - Add comprehensive activity log handlers in session context - Handle tool calls, results, and errors with proper state updates - Add transcription and assistant chunk streaming handlers - Improve audio playback with proper error handling and confirmation - Use generateMessageId() for consistent ID generation - Filter out transcription activity logs from chat - Stop audio playback when real speech is detected ```
This commit is contained in:
@@ -20,6 +20,7 @@ import { theme as defaultTheme } from "@/styles/theme";
|
|||||||
import { useRecentPaths } from "@/hooks/use-recent-paths";
|
import { useRecentPaths } from "@/hooks/use-recent-paths";
|
||||||
import { useSession } from "@/contexts/session-context";
|
import { useSession } from "@/contexts/session-context";
|
||||||
import { useRouter } from "expo-router";
|
import { useRouter } from "expo-router";
|
||||||
|
import { generateMessageId } from "@/types/stream";
|
||||||
|
|
||||||
interface CreateAgentModalProps {
|
interface CreateAgentModalProps {
|
||||||
isVisible: boolean;
|
isVisible: boolean;
|
||||||
@@ -163,7 +164,7 @@ export function CreateAgentModal({
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Generate request ID
|
// Generate request ID
|
||||||
const requestId = crypto.randomUUID();
|
const requestId = generateMessageId();
|
||||||
|
|
||||||
setIsLoading(true);
|
setIsLoading(true);
|
||||||
setPendingRequestId(requestId);
|
setPendingRequestId(requestId);
|
||||||
|
|||||||
@@ -260,49 +260,205 @@ export function SessionProvider({ children, serverUrl }: SessionProviderProps) {
|
|||||||
});
|
});
|
||||||
|
|
||||||
// Audio output
|
// Audio output
|
||||||
const unsubAudioOutput = ws.on("audio_output", (message) => {
|
const unsubAudioOutput = ws.on("audio_output", async (message) => {
|
||||||
if (message.type !== "audio_output") return;
|
if (message.type !== "audio_output") return;
|
||||||
const { audio, format, id } = message.payload;
|
const data = message.payload;
|
||||||
|
|
||||||
console.log("[Session] Received audio output:", id, "format:", format);
|
try {
|
||||||
|
setIsPlayingAudio(true);
|
||||||
|
|
||||||
// Create Blob-like object for React Native
|
// Create blob-like object with correct mime type (React Native compatible)
|
||||||
// React Native doesn't support creating Blobs from binary data
|
const mimeType =
|
||||||
const audioBlob = {
|
data.format === "mp3" ? "audio/mpeg" : `audio/${data.format}`;
|
||||||
type: format,
|
const base64Audio = data.audio;
|
||||||
size: audio.length,
|
|
||||||
arrayBuffer: async () => {
|
// Create a Blob-like object that works in React Native
|
||||||
// Convert base64 to ArrayBuffer
|
const audioBlob = {
|
||||||
const binaryString = atob(audio);
|
type: mimeType,
|
||||||
const bytes = new Uint8Array(binaryString.length);
|
size: Math.ceil((base64Audio.length * 3) / 4), // Approximate size from base64
|
||||||
for (let i = 0; i < binaryString.length; i++) {
|
arrayBuffer: async () => {
|
||||||
bytes[i] = binaryString.charCodeAt(i);
|
// Convert base64 to ArrayBuffer
|
||||||
}
|
const binaryString = atob(base64Audio);
|
||||||
return bytes.buffer;
|
const bytes = new Uint8Array(binaryString.length);
|
||||||
|
for (let i = 0; i < binaryString.length; i++) {
|
||||||
|
bytes[i] = binaryString.charCodeAt(i);
|
||||||
|
}
|
||||||
|
return bytes.buffer;
|
||||||
|
},
|
||||||
|
} as Blob;
|
||||||
|
|
||||||
|
// Play audio
|
||||||
|
await audioPlayer.play(audioBlob);
|
||||||
|
|
||||||
|
// Send confirmation back to server
|
||||||
|
const confirmMessage: WSInboundMessage = {
|
||||||
|
type: "session",
|
||||||
|
message: {
|
||||||
|
type: "audio_played",
|
||||||
|
id: data.id,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
ws.send(confirmMessage);
|
||||||
|
|
||||||
|
setIsPlayingAudio(false);
|
||||||
|
} catch (error: any) {
|
||||||
|
console.error("[Session] Audio playback error:", error);
|
||||||
|
setIsPlayingAudio(false);
|
||||||
|
|
||||||
|
// Still send confirmation even on error to prevent server from waiting
|
||||||
|
const confirmMessage: WSInboundMessage = {
|
||||||
|
type: "session",
|
||||||
|
message: {
|
||||||
|
type: "audio_played",
|
||||||
|
id: data.id,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
ws.send(confirmMessage);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
// Activity log handler
|
||||||
|
const unsubActivity = ws.on("activity_log", (message) => {
|
||||||
|
if (message.type !== "activity_log") return;
|
||||||
|
const data = message.payload;
|
||||||
|
|
||||||
|
// Filter out transcription activity logs
|
||||||
|
if (data.type === "system" && data.content.includes("Transcribing")) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Handle tool calls
|
||||||
|
if (data.type === "tool_call" && data.metadata) {
|
||||||
|
const {
|
||||||
|
toolCallId,
|
||||||
|
toolName,
|
||||||
|
arguments: args,
|
||||||
|
} = data.metadata as {
|
||||||
|
toolCallId: string;
|
||||||
|
toolName: string;
|
||||||
|
arguments: unknown;
|
||||||
|
};
|
||||||
|
|
||||||
|
setMessages((prev) => [
|
||||||
|
...prev,
|
||||||
|
{
|
||||||
|
type: "tool_call",
|
||||||
|
id: toolCallId,
|
||||||
|
timestamp: Date.now(),
|
||||||
|
toolName,
|
||||||
|
args,
|
||||||
|
status: "executing",
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Handle tool results
|
||||||
|
if (data.type === "tool_result" && data.metadata) {
|
||||||
|
const { toolCallId, result } = data.metadata as {
|
||||||
|
toolCallId: string;
|
||||||
|
result: unknown;
|
||||||
|
};
|
||||||
|
|
||||||
|
setMessages((prev) =>
|
||||||
|
prev.map((msg) =>
|
||||||
|
msg.type === "tool_call" && msg.id === toolCallId
|
||||||
|
? { ...msg, result, status: "completed" as const }
|
||||||
|
: msg
|
||||||
|
)
|
||||||
|
);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Handle tool errors
|
||||||
|
if (
|
||||||
|
data.type === "error" &&
|
||||||
|
data.metadata &&
|
||||||
|
"toolCallId" in data.metadata
|
||||||
|
) {
|
||||||
|
const { toolCallId, error } = data.metadata as {
|
||||||
|
toolCallId: string;
|
||||||
|
error: unknown;
|
||||||
|
};
|
||||||
|
|
||||||
|
setMessages((prev) =>
|
||||||
|
prev.map((msg) =>
|
||||||
|
msg.type === "tool_call" && msg.id === toolCallId
|
||||||
|
? { ...msg, error, status: "failed" as const }
|
||||||
|
: msg
|
||||||
|
)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Map activity types to message types
|
||||||
|
let activityType: "system" | "info" | "success" | "error" = "info";
|
||||||
|
if (data.type === "error") activityType = "error";
|
||||||
|
|
||||||
|
// Add user transcripts as user messages
|
||||||
|
if (data.type === "transcript") {
|
||||||
|
setMessages((prev) => [
|
||||||
|
...prev,
|
||||||
|
{
|
||||||
|
type: "user",
|
||||||
|
id: generateMessageId(),
|
||||||
|
timestamp: Date.now(),
|
||||||
|
message: data.content,
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Add assistant messages
|
||||||
|
if (data.type === "assistant") {
|
||||||
|
setMessages((prev) => [
|
||||||
|
...prev,
|
||||||
|
{
|
||||||
|
type: "assistant",
|
||||||
|
id: generateMessageId(),
|
||||||
|
timestamp: Date.now(),
|
||||||
|
message: data.content,
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
setCurrentAssistantMessage("");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Add activity log for other types
|
||||||
|
setMessages((prev) => [
|
||||||
|
...prev,
|
||||||
|
{
|
||||||
|
type: "activity",
|
||||||
|
id: generateMessageId(),
|
||||||
|
timestamp: Date.now(),
|
||||||
|
activityType,
|
||||||
|
message: data.content,
|
||||||
|
metadata: data.metadata,
|
||||||
},
|
},
|
||||||
} as Blob;
|
]);
|
||||||
|
});
|
||||||
|
|
||||||
// Play audio
|
// Assistant chunk handler (streaming)
|
||||||
setIsPlayingAudio(true);
|
const unsubChunk = ws.on("assistant_chunk", (message) => {
|
||||||
audioPlayer.play(audioBlob)
|
if (message.type !== "assistant_chunk") return;
|
||||||
.then(() => {
|
setCurrentAssistantMessage((prev) => prev + message.payload.chunk);
|
||||||
console.log("[Session] Audio playback complete:", id);
|
});
|
||||||
setIsPlayingAudio(false);
|
|
||||||
|
|
||||||
// Send confirmation to server
|
// Transcription result handler
|
||||||
const msg: WSInboundMessage = {
|
const unsubTranscription = ws.on("transcription_result", (message) => {
|
||||||
type: "session",
|
if (message.type !== "transcription_result") return;
|
||||||
message: {
|
|
||||||
type: "audio_played",
|
const transcriptText = message.payload.text.trim();
|
||||||
id,
|
|
||||||
},
|
if (!transcriptText) {
|
||||||
};
|
// Empty transcription - false positive, let playback continue
|
||||||
ws.send(msg);
|
console.log("[Session] Empty transcription (false positive) - ignoring");
|
||||||
})
|
} else {
|
||||||
.catch((error) => {
|
// Has content - real speech detected, stop playback
|
||||||
console.error("[Session] Audio playback failed:", error);
|
console.log("[Session] Transcription received - stopping playback");
|
||||||
setIsPlayingAudio(false);
|
audioPlayer.stop();
|
||||||
});
|
setIsPlayingAudio(false);
|
||||||
|
setCurrentAssistantMessage("");
|
||||||
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
return () => {
|
return () => {
|
||||||
@@ -312,6 +468,9 @@ export function SessionProvider({ children, serverUrl }: SessionProviderProps) {
|
|||||||
unsubAgentUpdate();
|
unsubAgentUpdate();
|
||||||
unsubPermissionRequest();
|
unsubPermissionRequest();
|
||||||
unsubAudioOutput();
|
unsubAudioOutput();
|
||||||
|
unsubActivity();
|
||||||
|
unsubChunk();
|
||||||
|
unsubTranscription();
|
||||||
};
|
};
|
||||||
}, [ws, audioPlayer, setIsPlayingAudio]);
|
}, [ws, audioPlayer, setIsPlayingAudio]);
|
||||||
|
|
||||||
|
|||||||
@@ -268,6 +268,10 @@ export class AgentManager {
|
|||||||
throw new Error(`Agent ${agentId} is ${agent.status}`);
|
throw new Error(`Agent ${agentId} is ${agent.status}`);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Reset message IDs for new turn - ensures each prompt gets fresh responses
|
||||||
|
agent.currentAssistantMessageId = null;
|
||||||
|
agent.currentThoughtId = null;
|
||||||
|
|
||||||
// Set session mode if specified
|
// Set session mode if specified
|
||||||
if (options?.sessionMode) {
|
if (options?.sessionMode) {
|
||||||
await this.setSessionMode(agentId, options.sessionMode);
|
await this.setSessionMode(agentId, options.sessionMode);
|
||||||
|
|||||||
Reference in New Issue
Block a user