feat: add Playwright MCP integration and TTS improvements
- Integrate Playwright MCP for browser automation capabilities - Add TTS manager for improved audio synthesis management - Add ToolCallCard UI component for better tool call visualization - Upgrade Vite to v7.1.10 for improved build performance - Add lucide-react for icon components - Add dotenv for environment variable management - Update agent prompt and orchestrator for enhanced interactions - Improve audio playback and voice controls in UI 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Mohamed Boudra
@@ -1,30 +1,71 @@
|
||||
.app {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
min-height: 100vh;
|
||||
height: 100svh;
|
||||
width: 100%;
|
||||
overflow: hidden;
|
||||
}
|
||||
|
||||
.header {
|
||||
display: flex;
|
||||
display: none;
|
||||
justify-content: space-between;
|
||||
align-items: center;
|
||||
padding: 1rem 2rem;
|
||||
padding: 0.5rem 1rem;
|
||||
background-color: #1a1a1a;
|
||||
border-bottom: 1px solid #333;
|
||||
flex-shrink: 0;
|
||||
min-height: 0;
|
||||
overflow: hidden;
|
||||
max-width: 100%;
|
||||
min-width: 0;
|
||||
}
|
||||
|
||||
.header h1 {
|
||||
font-size: 1.5rem;
|
||||
font-size: 1rem;
|
||||
font-weight: 600;
|
||||
margin: 0;
|
||||
white-space: nowrap;
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
min-width: 0;
|
||||
flex: 1;
|
||||
}
|
||||
|
||||
.header-status {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 0.5rem;
|
||||
flex-shrink: 0;
|
||||
min-width: 0;
|
||||
}
|
||||
|
||||
.audio-playing-indicator {
|
||||
width: 8px;
|
||||
height: 8px;
|
||||
background-color: #0f766e;
|
||||
border-radius: 50%;
|
||||
animation: pulse-audio 1.5s ease-in-out infinite;
|
||||
}
|
||||
|
||||
@keyframes pulse-audio {
|
||||
0%, 100% {
|
||||
opacity: 1;
|
||||
transform: scale(1);
|
||||
}
|
||||
50% {
|
||||
opacity: 0.5;
|
||||
transform: scale(1.2);
|
||||
}
|
||||
}
|
||||
|
||||
.status-indicator {
|
||||
padding: 0.5rem 1rem;
|
||||
padding: 0.25rem 0.75rem;
|
||||
border-radius: 4px;
|
||||
font-size: 0.75rem;
|
||||
font-weight: 500;
|
||||
text-transform: uppercase;
|
||||
flex-shrink: 0;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.status-indicator.disconnected {
|
||||
@@ -55,20 +96,24 @@
|
||||
.main {
|
||||
display: flex;
|
||||
flex: 1;
|
||||
gap: 2rem;
|
||||
max-height: 100vh;
|
||||
overflow: hidden;
|
||||
gap: 0.5rem;
|
||||
overflow: hidden;
|
||||
padding: 0.5rem;
|
||||
min-height: 0;
|
||||
max-width: 100%;
|
||||
}
|
||||
|
||||
.chat-interface {
|
||||
flex: 1;
|
||||
background-color: #1a1a1a;
|
||||
border-radius: 8px;
|
||||
padding: 2rem;
|
||||
padding: 1rem;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 1rem;
|
||||
height: 100%;
|
||||
gap: 0.5rem;
|
||||
min-height: 0;
|
||||
max-width: 100%;
|
||||
overflow: hidden;
|
||||
}
|
||||
|
||||
.chat-interface h2 {
|
||||
@@ -113,23 +158,31 @@ overflow: hidden;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
overflow: hidden;
|
||||
min-height: 0;
|
||||
max-width: 100%;
|
||||
}
|
||||
|
||||
.log-entries {
|
||||
flex: 1;
|
||||
overflow-y: auto;
|
||||
overflow-x: hidden;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 0.5rem;
|
||||
gap: 0.25rem;
|
||||
min-height: 0;
|
||||
max-width: 100%;
|
||||
}
|
||||
|
||||
.log-entry {
|
||||
padding: 0.75rem;
|
||||
padding: 0.5rem;
|
||||
border-radius: 4px;
|
||||
font-size: 0.875rem;
|
||||
line-height: 1.4;
|
||||
display: flex;
|
||||
gap: 0.75rem;
|
||||
gap: 0.5rem;
|
||||
flex-shrink: 0;
|
||||
max-width: 100%;
|
||||
overflow: hidden;
|
||||
}
|
||||
|
||||
.log-time {
|
||||
@@ -140,6 +193,9 @@ overflow: hidden;
|
||||
|
||||
.log-message {
|
||||
flex: 1;
|
||||
min-width: 0;
|
||||
word-wrap: break-word;
|
||||
overflow-wrap: break-word;
|
||||
}
|
||||
|
||||
.log-entry.system {
|
||||
@@ -197,6 +253,8 @@ overflow: hidden;
|
||||
|
||||
.log-metadata {
|
||||
margin-top: 0.5rem;
|
||||
max-width: 100%;
|
||||
overflow: hidden;
|
||||
}
|
||||
|
||||
.log-metadata summary {
|
||||
@@ -212,24 +270,40 @@ overflow: hidden;
|
||||
border-radius: 4px;
|
||||
overflow-x: auto;
|
||||
font-size: 0.75rem;
|
||||
max-width: 100%;
|
||||
word-wrap: break-word;
|
||||
overflow-wrap: break-word;
|
||||
white-space: pre-wrap;
|
||||
}
|
||||
|
||||
.message-form {
|
||||
display: flex;
|
||||
align-items: flex-end;
|
||||
gap: 0.5rem;
|
||||
padding-top: 1rem;
|
||||
padding-top: 0.5rem;
|
||||
border-top: 1px solid #333;
|
||||
flex-shrink: 0;
|
||||
min-width: 0;
|
||||
max-width: 100%;
|
||||
overflow: hidden;
|
||||
}
|
||||
|
||||
.message-input {
|
||||
flex: 1;
|
||||
padding: 0.75rem 1rem;
|
||||
min-width: 0;
|
||||
padding: 0.5rem 0.75rem;
|
||||
background-color: #2a2a2a;
|
||||
border: 1px solid #444;
|
||||
border-radius: 6px;
|
||||
color: #fff;
|
||||
font-size: 0.875rem;
|
||||
font-family: inherit;
|
||||
outline: none;
|
||||
resize: none;
|
||||
overflow-y: auto;
|
||||
max-height: 200px;
|
||||
line-height: 1.5;
|
||||
min-height: 26px;
|
||||
}
|
||||
|
||||
.message-input:focus {
|
||||
@@ -242,27 +316,74 @@ overflow: hidden;
|
||||
}
|
||||
|
||||
.send-button {
|
||||
padding: 0.75rem 1.5rem;
|
||||
background-color: #22c55e;
|
||||
color: #000;
|
||||
width: 44px;
|
||||
height: 44px;
|
||||
min-width: 44px;
|
||||
min-height: 44px;
|
||||
padding: 0;
|
||||
background-color: #3b82f6;
|
||||
color: #fff;
|
||||
border: none;
|
||||
border-radius: 6px;
|
||||
font-size: 0.875rem;
|
||||
font-weight: 500;
|
||||
border-radius: 8px;
|
||||
cursor: pointer;
|
||||
transition: background-color 0.2s;
|
||||
transition: all 0.2s;
|
||||
flex-shrink: 0;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
}
|
||||
|
||||
.send-button:hover:not(:disabled) {
|
||||
.send-button.has-text {
|
||||
background-color: #22c55e;
|
||||
color: #000;
|
||||
}
|
||||
|
||||
.send-button.has-text:hover:not(:disabled) {
|
||||
background-color: #16a34a;
|
||||
}
|
||||
|
||||
.send-button:not(.has-text):hover:not(:disabled) {
|
||||
background-color: #2563eb;
|
||||
}
|
||||
|
||||
.send-button.recording {
|
||||
background-color: #ef4444;
|
||||
animation: pulse-red 1.5s ease-in-out infinite;
|
||||
}
|
||||
|
||||
.send-button.recording:hover:not(:disabled) {
|
||||
background-color: #dc2626;
|
||||
}
|
||||
|
||||
.send-button.processing {
|
||||
background-color: #f59e0b;
|
||||
}
|
||||
|
||||
.send-button:disabled {
|
||||
background-color: #444;
|
||||
color: #999;
|
||||
cursor: not-allowed;
|
||||
}
|
||||
|
||||
.send-button .recording-indicator {
|
||||
display: inline-block;
|
||||
width: 12px;
|
||||
height: 12px;
|
||||
background-color: #fff;
|
||||
border-radius: 50%;
|
||||
animation: pulse 1s ease-in-out infinite;
|
||||
}
|
||||
|
||||
.send-button .processing-indicator {
|
||||
display: inline-block;
|
||||
width: 12px;
|
||||
height: 12px;
|
||||
border: 2px solid #fff;
|
||||
border-top-color: transparent;
|
||||
border-radius: 50%;
|
||||
animation: spin 0.8s linear infinite;
|
||||
}
|
||||
|
||||
.test-controls {
|
||||
padding-top: 0.5rem;
|
||||
border-top: 1px solid #333;
|
||||
@@ -274,11 +395,14 @@ overflow: hidden;
|
||||
.voice-controls {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 0.75rem;
|
||||
padding: 1rem;
|
||||
gap: 0.5rem;
|
||||
padding: 0.75rem;
|
||||
background-color: #2a2a2a;
|
||||
border-radius: 6px;
|
||||
margin-bottom: 1rem;
|
||||
flex-shrink: 0;
|
||||
min-width: 0;
|
||||
max-width: 100%;
|
||||
overflow: hidden;
|
||||
}
|
||||
|
||||
.voice-button {
|
||||
@@ -286,15 +410,17 @@ overflow: hidden;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
gap: 0.5rem;
|
||||
padding: 1rem 2rem;
|
||||
padding: 0.75rem 1.5rem;
|
||||
background-color: #3b82f6;
|
||||
color: #fff;
|
||||
border: none;
|
||||
border-radius: 8px;
|
||||
font-size: 1rem;
|
||||
font-size: 0.875rem;
|
||||
font-weight: 500;
|
||||
cursor: pointer;
|
||||
transition: all 0.2s;
|
||||
max-width: 100%;
|
||||
overflow: hidden;
|
||||
}
|
||||
|
||||
.voice-button:hover:not(:disabled) {
|
||||
@@ -376,13 +502,138 @@ overflow: hidden;
|
||||
font-size: 0.875rem;
|
||||
}
|
||||
|
||||
.playing-indicator {
|
||||
padding: 0.5rem;
|
||||
background-color: #0f766e;
|
||||
color: #ccfbf1;
|
||||
border-radius: 4px;
|
||||
/* Tool Call Card Styles */
|
||||
.tool-call-card {
|
||||
background-color: #2a2a2a;
|
||||
border-radius: 6px;
|
||||
overflow: hidden;
|
||||
margin: 0.25rem 0;
|
||||
border: 1px solid #444;
|
||||
flex-shrink: 0;
|
||||
}
|
||||
|
||||
.tool-call-card[data-status="executing"] {
|
||||
border-color: #f59e0b;
|
||||
}
|
||||
|
||||
.tool-call-card[data-status="completed"] {
|
||||
border-color: #22c55e;
|
||||
}
|
||||
|
||||
.tool-call-header {
|
||||
width: 100%;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 0.5rem;
|
||||
padding: 0.75rem;
|
||||
background: none;
|
||||
border: none;
|
||||
color: #e2e8f0;
|
||||
cursor: pointer;
|
||||
font-family: inherit;
|
||||
font-size: 0.875rem;
|
||||
text-align: center;
|
||||
text-align: left;
|
||||
transition: background-color 0.2s;
|
||||
}
|
||||
|
||||
.tool-call-header:hover {
|
||||
background-color: #333;
|
||||
}
|
||||
|
||||
.tool-call-icon {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
color: #999;
|
||||
flex-shrink: 0;
|
||||
}
|
||||
|
||||
.tool-call-name {
|
||||
flex: 1;
|
||||
font-family: monospace;
|
||||
font-weight: 500;
|
||||
color: #e2e8f0;
|
||||
}
|
||||
|
||||
.tool-call-status {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 0.375rem;
|
||||
padding: 0.25rem 0.5rem;
|
||||
border-radius: 4px;
|
||||
font-size: 0.75rem;
|
||||
flex-shrink: 0;
|
||||
}
|
||||
|
||||
.tool-call-card[data-status="executing"] .tool-call-status {
|
||||
background-color: rgba(245, 158, 11, 0.2);
|
||||
color: #fbbf24;
|
||||
}
|
||||
|
||||
.tool-call-card[data-status="completed"] .tool-call-status {
|
||||
background-color: rgba(34, 197, 94, 0.2);
|
||||
color: #86efac;
|
||||
}
|
||||
|
||||
.status-icon {
|
||||
flex-shrink: 0;
|
||||
}
|
||||
|
||||
.status-icon.spinning {
|
||||
animation: spin 1s linear infinite;
|
||||
}
|
||||
|
||||
.status-text {
|
||||
font-weight: 500;
|
||||
text-transform: capitalize;
|
||||
}
|
||||
|
||||
.tool-call-details {
|
||||
padding: 0 0.75rem 0.75rem 0.75rem;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 0.75rem;
|
||||
animation: expand 0.2s ease-out;
|
||||
}
|
||||
|
||||
@keyframes expand {
|
||||
from {
|
||||
opacity: 0;
|
||||
max-height: 0;
|
||||
}
|
||||
to {
|
||||
opacity: 1;
|
||||
max-height: 1000px;
|
||||
}
|
||||
}
|
||||
|
||||
.tool-call-section {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 0.375rem;
|
||||
}
|
||||
|
||||
.section-header {
|
||||
font-size: 0.75rem;
|
||||
font-weight: 600;
|
||||
color: #999;
|
||||
text-transform: uppercase;
|
||||
letter-spacing: 0.025em;
|
||||
}
|
||||
|
||||
.section-content {
|
||||
padding: 0.5rem;
|
||||
background-color: #1a1a1a;
|
||||
border-radius: 4px;
|
||||
font-family: monospace;
|
||||
font-size: 0.75rem;
|
||||
color: #e2e8f0;
|
||||
overflow-x: auto;
|
||||
border: 1px solid #444;
|
||||
margin: 0;
|
||||
white-space: pre-wrap;
|
||||
word-wrap: break-word;
|
||||
overflow-wrap: break-word;
|
||||
}
|
||||
|
||||
@media (max-width: 1024px) {
|
||||
@@ -390,3 +641,28 @@ overflow: hidden;
|
||||
flex-direction: column;
|
||||
}
|
||||
}
|
||||
|
||||
@media (max-width: 768px) {
|
||||
.main {
|
||||
padding: 0.25rem;
|
||||
gap: 0.25rem;
|
||||
}
|
||||
|
||||
.chat-interface {
|
||||
padding: 0.5rem;
|
||||
}
|
||||
|
||||
.header {
|
||||
padding: 0.5rem 0.5rem;
|
||||
gap: 0.5rem;
|
||||
}
|
||||
|
||||
.header h1 {
|
||||
font-size: 0.875rem;
|
||||
}
|
||||
|
||||
.status-indicator {
|
||||
padding: 0.25rem 0.5rem;
|
||||
font-size: 0.7rem;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,16 +1,28 @@
|
||||
import { useState, useEffect, useRef } from "react";
|
||||
import { Mic, Send } from "lucide-react";
|
||||
import { useWebSocket } from "./hooks/useWebSocket";
|
||||
import { VoiceControls } from "./components/VoiceControls";
|
||||
import { createAudioPlayer } from "./lib/audio-playback";
|
||||
import { createAudioRecorder, type AudioRecorder } from "./lib/audio-capture";
|
||||
import { ToolCallCard } from "./components/ToolCallCard";
|
||||
import "./App.css";
|
||||
|
||||
interface LogEntry {
|
||||
id: string;
|
||||
timestamp: number;
|
||||
type: "system" | "info" | "success" | "error" | "user" | "assistant" | "tool";
|
||||
message: string;
|
||||
metadata?: Record<string, unknown>;
|
||||
}
|
||||
type LogEntry =
|
||||
| {
|
||||
type: "system" | "info" | "success" | "error" | "user" | "assistant";
|
||||
id: string;
|
||||
timestamp: number;
|
||||
message: string;
|
||||
metadata?: Record<string, unknown>;
|
||||
}
|
||||
| {
|
||||
type: "tool_call";
|
||||
id: string;
|
||||
timestamp: number;
|
||||
toolName: string;
|
||||
args: any;
|
||||
result?: any;
|
||||
status: "executing" | "completed";
|
||||
};
|
||||
|
||||
function App() {
|
||||
const [logs, setLogs] = useState<LogEntry[]>([
|
||||
@@ -25,8 +37,11 @@ function App() {
|
||||
const [currentAssistantMessage, setCurrentAssistantMessage] = useState("");
|
||||
const [isProcessingAudio, setIsProcessingAudio] = useState(false);
|
||||
const [isPlayingAudio, setIsPlayingAudio] = useState(false);
|
||||
const [isRecording, setIsRecording] = useState(false);
|
||||
const logEndRef = useRef<HTMLDivElement>(null);
|
||||
const audioPlayerRef = useRef(createAudioPlayer());
|
||||
const recorderRef = useRef<AudioRecorder | null>(null);
|
||||
const textareaRef = useRef<HTMLTextAreaElement>(null);
|
||||
|
||||
// WebSocket URL - use ws://localhost:3000/ws in dev, or construct from current host in prod
|
||||
const wsUrl = `${window.location.protocol === "https:" ? "wss" : "ws"}://${
|
||||
@@ -35,8 +50,20 @@ function App() {
|
||||
|
||||
const ws = useWebSocket(wsUrl);
|
||||
|
||||
useEffect(() => {
|
||||
recorderRef.current = createAudioRecorder();
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
if (textareaRef.current) {
|
||||
textareaRef.current.style.height = 'auto';
|
||||
const newHeight = Math.max(textareaRef.current.scrollHeight, 44);
|
||||
textareaRef.current.style.height = `${newHeight}px`;
|
||||
}
|
||||
}, [userInput]);
|
||||
|
||||
const addLog = (
|
||||
type: LogEntry["type"],
|
||||
type: "system" | "info" | "success" | "error" | "user" | "assistant",
|
||||
message: string,
|
||||
metadata?: Record<string, unknown>
|
||||
) => {
|
||||
@@ -67,22 +94,59 @@ function App() {
|
||||
// Listen for activity log messages
|
||||
const unsubActivity = ws.on("activity_log", (payload: unknown) => {
|
||||
const data = payload as {
|
||||
id: string;
|
||||
type: string;
|
||||
content: string;
|
||||
metadata?: Record<string, unknown>;
|
||||
};
|
||||
|
||||
// Handle tool calls
|
||||
if (data.type === "tool_call") {
|
||||
const { toolCallId, toolName, arguments: args } = data.metadata as {
|
||||
toolCallId: string;
|
||||
toolName: string;
|
||||
arguments: any;
|
||||
};
|
||||
|
||||
setLogs((prev) => [
|
||||
...prev,
|
||||
{
|
||||
type: "tool_call",
|
||||
id: toolCallId,
|
||||
timestamp: Date.now(),
|
||||
toolName,
|
||||
args,
|
||||
status: "executing",
|
||||
},
|
||||
]);
|
||||
return;
|
||||
}
|
||||
|
||||
if (data.type === "tool_result") {
|
||||
const { toolCallId, result } = data.metadata as {
|
||||
toolCallId: string;
|
||||
result: any;
|
||||
};
|
||||
|
||||
setLogs((prev) =>
|
||||
prev.map((log) =>
|
||||
log.type === "tool_call" && log.id === toolCallId
|
||||
? { ...log, result, status: "completed" as const }
|
||||
: log
|
||||
)
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
// Map activity log types to UI log types
|
||||
let logType: LogEntry["type"] = "info";
|
||||
let logType: "system" | "info" | "success" | "error" | "user" | "assistant" = "info";
|
||||
if (data.type === "transcript") logType = "user";
|
||||
else if (data.type === "assistant") logType = "assistant";
|
||||
else if (data.type === "tool_call" || data.type === "tool_result")
|
||||
logType = "tool";
|
||||
else if (data.type === "error") logType = "error";
|
||||
|
||||
addLog(logType, data.content, data.metadata);
|
||||
|
||||
// Clear streaming state when complete assistant message arrives
|
||||
// Clear streaming state when assistant segment is received
|
||||
if (data.type === "assistant") {
|
||||
setCurrentAssistantMessage("");
|
||||
}
|
||||
@@ -97,7 +161,7 @@ function App() {
|
||||
// Listen for transcription results
|
||||
const unsubTranscription = ws.on(
|
||||
"transcription_result",
|
||||
(payload: unknown) => {
|
||||
(_payload: unknown) => {
|
||||
// Note: Transcription is already broadcast as activity_log with type "transcript"
|
||||
// No need to log it again here to avoid duplication
|
||||
setIsProcessingAudio(false);
|
||||
@@ -109,7 +173,6 @@ function App() {
|
||||
const data = payload as { audio: string; format: string; id: string };
|
||||
|
||||
try {
|
||||
addLog("info", "Playing assistant audio response...");
|
||||
setIsPlayingAudio(true);
|
||||
|
||||
// Decode base64 audio
|
||||
@@ -127,8 +190,13 @@ function App() {
|
||||
// Play audio
|
||||
await audioPlayerRef.current.play(audioBlob);
|
||||
|
||||
// Send confirmation back to server
|
||||
ws.send({
|
||||
type: "audio_played",
|
||||
payload: { id: data.id },
|
||||
});
|
||||
|
||||
setIsPlayingAudio(false);
|
||||
addLog("success", "Audio playback complete");
|
||||
} catch (error: any) {
|
||||
console.error("[App] Audio playback error:", error);
|
||||
addLog("error", `Audio playback failed: ${error.message}`);
|
||||
@@ -159,13 +227,7 @@ function App() {
|
||||
logEndRef.current?.scrollIntoView({ behavior: "smooth" });
|
||||
}, [logs, currentAssistantMessage]);
|
||||
|
||||
const handlePing = () => {
|
||||
ws.sendPing();
|
||||
addLog("info", "Sent ping to server");
|
||||
};
|
||||
|
||||
const handleSendMessage = (e: React.FormEvent) => {
|
||||
e.preventDefault();
|
||||
const handleSendMessage = () => {
|
||||
if (!userInput.trim() || !ws.isConnected) return;
|
||||
|
||||
// Send message to server
|
||||
@@ -176,42 +238,74 @@ function App() {
|
||||
setCurrentAssistantMessage("");
|
||||
};
|
||||
|
||||
const handleAudioRecorded = async (audioBlob: Blob, format: string) => {
|
||||
if (!ws.isConnected) {
|
||||
addLog("error", "Cannot send audio - not connected to server");
|
||||
return;
|
||||
}
|
||||
const handleToggleRecording = async () => {
|
||||
const recorder = recorderRef.current;
|
||||
if (!recorder || !ws.isConnected) return;
|
||||
|
||||
try {
|
||||
setIsProcessingAudio(true);
|
||||
addLog("info", "Sending audio to server...");
|
||||
if (isRecording) {
|
||||
console.log('[App] Stopping recording...');
|
||||
const audioBlob = await recorder.stop();
|
||||
setIsRecording(false);
|
||||
|
||||
// Convert blob to base64
|
||||
const arrayBuffer = await audioBlob.arrayBuffer();
|
||||
const base64Audio = btoa(
|
||||
new Uint8Array(arrayBuffer).reduce(
|
||||
(data, byte) => data + String.fromCharCode(byte),
|
||||
""
|
||||
)
|
||||
);
|
||||
const format = audioBlob.type || 'audio/webm';
|
||||
console.log(`[App] Recording complete: ${audioBlob.size} bytes, format: ${format}`);
|
||||
|
||||
// Send audio chunk via WebSocket
|
||||
ws.send({
|
||||
type: "audio_chunk",
|
||||
payload: {
|
||||
audio: base64Audio,
|
||||
format: format,
|
||||
isLast: true,
|
||||
},
|
||||
});
|
||||
setIsProcessingAudio(true);
|
||||
addLog("info", "Sending audio to server...");
|
||||
|
||||
console.log(
|
||||
`[App] Sent audio: ${audioBlob.size} bytes, format: ${format}`
|
||||
);
|
||||
const arrayBuffer = await audioBlob.arrayBuffer();
|
||||
const base64Audio = btoa(
|
||||
new Uint8Array(arrayBuffer).reduce(
|
||||
(data, byte) => data + String.fromCharCode(byte),
|
||||
""
|
||||
)
|
||||
);
|
||||
|
||||
ws.send({
|
||||
type: "audio_chunk",
|
||||
payload: {
|
||||
audio: base64Audio,
|
||||
format: format,
|
||||
isLast: true,
|
||||
},
|
||||
});
|
||||
|
||||
console.log(`[App] Sent audio: ${audioBlob.size} bytes, format: ${format}`);
|
||||
} else {
|
||||
console.log('[App] Starting recording...');
|
||||
await recorder.start();
|
||||
setIsRecording(true);
|
||||
}
|
||||
} catch (error: any) {
|
||||
console.error("[App] Error sending audio:", error);
|
||||
addLog("error", `Failed to send audio: ${error.message}`);
|
||||
setIsProcessingAudio(false);
|
||||
console.error('[App] Recording error:', error);
|
||||
addLog("error", `Failed to record audio: ${error.message}`);
|
||||
setIsRecording(false);
|
||||
}
|
||||
};
|
||||
|
||||
const handleButtonClick = async (e: React.MouseEvent<HTMLButtonElement>) => {
|
||||
e.preventDefault();
|
||||
|
||||
if (userInput.trim()) {
|
||||
handleSendMessage();
|
||||
} else {
|
||||
await handleToggleRecording();
|
||||
}
|
||||
};
|
||||
|
||||
const handleKeyDown = (e: React.KeyboardEvent<HTMLTextAreaElement>) => {
|
||||
// Detect if device is desktop (not mobile/tablet)
|
||||
const isDesktop = window.matchMedia('(pointer: fine)').matches;
|
||||
|
||||
if (e.key === 'Enter') {
|
||||
if (isDesktop && !e.shiftKey && userInput.trim()) {
|
||||
// Desktop: plain Enter sends message
|
||||
e.preventDefault();
|
||||
handleSendMessage();
|
||||
}
|
||||
// Desktop: Shift+Enter creates new line (default behavior)
|
||||
// Mobile: Enter creates new line (default behavior)
|
||||
}
|
||||
};
|
||||
|
||||
@@ -219,12 +313,15 @@ function App() {
|
||||
<div className="app">
|
||||
<header className="header">
|
||||
<h1>Voice Assistant</h1>
|
||||
<div
|
||||
className={`status-indicator ${
|
||||
ws.isConnected ? "connected" : "disconnected"
|
||||
}`}
|
||||
>
|
||||
{ws.isConnected ? "connected" : "disconnected"}
|
||||
<div className="header-status">
|
||||
{isPlayingAudio && <div className="audio-playing-indicator" title="Playing audio" />}
|
||||
<div
|
||||
className={`status-indicator ${
|
||||
ws.isConnected ? "connected" : "disconnected"
|
||||
}`}
|
||||
>
|
||||
{ws.isConnected ? "connected" : "disconnected"}
|
||||
</div>
|
||||
</div>
|
||||
</header>
|
||||
|
||||
@@ -232,25 +329,29 @@ function App() {
|
||||
<div className="chat-interface">
|
||||
<div className="activity-log">
|
||||
<div className="log-entries">
|
||||
{logs.map((log) => (
|
||||
<div key={log.id} className={`log-entry ${log.type}`}>
|
||||
<span className="log-time">
|
||||
{new Date(log.timestamp).toLocaleTimeString()}
|
||||
</span>
|
||||
<span className="log-message">{log.message}</span>
|
||||
{log.metadata && (
|
||||
<details className="log-metadata">
|
||||
<summary>Details</summary>
|
||||
<pre>{JSON.stringify(log.metadata, null, 2)}</pre>
|
||||
</details>
|
||||
)}
|
||||
</div>
|
||||
))}
|
||||
{logs.map((log) =>
|
||||
log.type === "tool_call" ? (
|
||||
<ToolCallCard
|
||||
key={log.id}
|
||||
toolName={log.toolName}
|
||||
args={log.args}
|
||||
result={log.result}
|
||||
status={log.status}
|
||||
/>
|
||||
) : (
|
||||
<div key={log.id} className={`log-entry ${log.type}`}>
|
||||
<span className="log-message">{log.message}</span>
|
||||
{log.metadata && (
|
||||
<details className="log-metadata">
|
||||
<summary>Details</summary>
|
||||
<pre>{JSON.stringify(log.metadata, null, 2)}</pre>
|
||||
</details>
|
||||
)}
|
||||
</div>
|
||||
)
|
||||
)}
|
||||
{currentAssistantMessage && (
|
||||
<div className="log-entry assistant streaming">
|
||||
<span className="log-time">
|
||||
{new Date().toLocaleTimeString()}
|
||||
</span>
|
||||
<span className="log-message">{currentAssistantMessage}</span>
|
||||
<span className="streaming-indicator">...</span>
|
||||
</div>
|
||||
@@ -259,29 +360,34 @@ function App() {
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<form onSubmit={handleSendMessage} className="message-form">
|
||||
<input
|
||||
type="text"
|
||||
<div className="message-form">
|
||||
<textarea
|
||||
ref={textareaRef}
|
||||
value={userInput}
|
||||
onChange={(e) => setUserInput(e.target.value)}
|
||||
placeholder="Type a message to the assistant..."
|
||||
disabled={!ws.isConnected}
|
||||
onKeyDown={handleKeyDown}
|
||||
placeholder="Type a message or tap mic to talk..."
|
||||
disabled={!ws.isConnected || isRecording}
|
||||
className="message-input"
|
||||
rows={1}
|
||||
/>
|
||||
<button
|
||||
type="submit"
|
||||
disabled={!ws.isConnected || !userInput.trim()}
|
||||
className="send-button"
|
||||
type="button"
|
||||
onClick={handleButtonClick}
|
||||
disabled={!ws.isConnected || isProcessingAudio}
|
||||
className={`send-button ${isRecording ? 'recording' : ''} ${isProcessingAudio ? 'processing' : ''} ${userInput.trim() ? 'has-text' : ''}`}
|
||||
>
|
||||
Send
|
||||
{isRecording ? (
|
||||
<span className="recording-indicator" />
|
||||
) : isProcessingAudio ? (
|
||||
<span className="processing-indicator" />
|
||||
) : userInput.trim() ? (
|
||||
<Send size={20} />
|
||||
) : (
|
||||
<Mic size={20} />
|
||||
)}
|
||||
</button>
|
||||
</form>
|
||||
|
||||
<VoiceControls
|
||||
onAudioRecorded={handleAudioRecorded}
|
||||
isProcessing={isProcessingAudio}
|
||||
isPlaying={isPlayingAudio}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
</main>
|
||||
</div>
|
||||
|
||||
@@ -0,0 +1,61 @@
|
||||
import { useState } from "react";
|
||||
import { ChevronDown, ChevronRight, CheckCircle, Loader2 } from "lucide-react";
|
||||
|
||||
export interface ToolCallCardProps {
|
||||
toolName: string;
|
||||
args: any;
|
||||
result?: any;
|
||||
status: "executing" | "completed";
|
||||
}
|
||||
|
||||
export function ToolCallCard({
|
||||
toolName,
|
||||
args,
|
||||
result,
|
||||
status,
|
||||
}: ToolCallCardProps) {
|
||||
const [isExpanded, setIsExpanded] = useState(false);
|
||||
|
||||
return (
|
||||
<div className="tool-call-card" data-status={status}>
|
||||
<button
|
||||
className="tool-call-header"
|
||||
onClick={() => setIsExpanded(!isExpanded)}
|
||||
type="button"
|
||||
>
|
||||
<div className="tool-call-icon">
|
||||
{isExpanded ? (
|
||||
<ChevronDown size={16} />
|
||||
) : (
|
||||
<ChevronRight size={16} />
|
||||
)}
|
||||
</div>
|
||||
<span className="tool-call-name">{toolName}</span>
|
||||
<div className="tool-call-status">
|
||||
{status === "executing" ? (
|
||||
<Loader2 size={14} className="status-icon spinning" />
|
||||
) : (
|
||||
<CheckCircle size={14} className="status-icon" />
|
||||
)}
|
||||
<span className="status-text">{status}</span>
|
||||
</div>
|
||||
</button>
|
||||
|
||||
{isExpanded && (
|
||||
<div className="tool-call-details">
|
||||
<div className="tool-call-section">
|
||||
<div className="section-header">Arguments</div>
|
||||
<pre className="section-content">{JSON.stringify(args, null, 2)}</pre>
|
||||
</div>
|
||||
|
||||
{result !== undefined && (
|
||||
<div className="tool-call-section">
|
||||
<div className="section-header">Result</div>
|
||||
<pre className="section-content">{JSON.stringify(result, null, 2)}</pre>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -4,10 +4,9 @@ import { createAudioRecorder, type AudioRecorder } from '../lib/audio-capture';
|
||||
interface VoiceControlsProps {
|
||||
onAudioRecorded: (audio: Blob, format: string) => void;
|
||||
isProcessing: boolean;
|
||||
isPlaying?: boolean;
|
||||
}
|
||||
|
||||
export function VoiceControls({ onAudioRecorded, isProcessing, isPlaying }: VoiceControlsProps) {
|
||||
export function VoiceControls({ onAudioRecorded, isProcessing }: VoiceControlsProps) {
|
||||
const [isRecording, setIsRecording] = useState(false);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
const [permissionState, setPermissionState] = useState<'prompt' | 'granted' | 'denied'>('prompt');
|
||||
@@ -70,8 +69,7 @@ export function VoiceControls({ onAudioRecorded, isProcessing, isPlaying }: Voic
|
||||
}
|
||||
}
|
||||
|
||||
const canRecord = !isProcessing && !isPlaying;
|
||||
const buttonDisabled = !canRecord || permissionState === 'denied';
|
||||
const buttonDisabled = isProcessing || permissionState === 'denied';
|
||||
|
||||
return (
|
||||
<div className="voice-controls">
|
||||
@@ -106,8 +104,6 @@ export function VoiceControls({ onAudioRecorded, isProcessing, isPlaying }: Voic
|
||||
Microphone access denied. Please enable it in your browser settings.
|
||||
</div>
|
||||
)}
|
||||
|
||||
{isPlaying && <div className="playing-indicator">🔊 Playing response...</div>}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
@@ -13,6 +13,11 @@
|
||||
-moz-osx-font-smoothing: grayscale;
|
||||
}
|
||||
|
||||
html {
|
||||
overflow-x: hidden;
|
||||
max-width: 100vw;
|
||||
}
|
||||
|
||||
* {
|
||||
box-sizing: border-box;
|
||||
margin: 0;
|
||||
@@ -21,11 +26,7 @@
|
||||
|
||||
body {
|
||||
margin: 0;
|
||||
min-width: 320px;
|
||||
min-height: 100vh;
|
||||
}
|
||||
|
||||
#root {
|
||||
width: 100%;
|
||||
min-height: 100vh;
|
||||
padding: 0;
|
||||
overflow-x: hidden;
|
||||
overflow-y: hidden;
|
||||
}
|
||||
|
||||
@@ -1,24 +1,32 @@
|
||||
export interface AudioPlayer {
|
||||
play(audioData: Blob): Promise<void>;
|
||||
play(audioData: Blob): Promise<number>;
|
||||
stop(): void;
|
||||
isPlaying(): boolean;
|
||||
clearQueue(): void;
|
||||
}
|
||||
|
||||
interface QueuedAudio {
|
||||
audioData: Blob;
|
||||
resolve: (duration: number) => void;
|
||||
reject: (error: Error) => void;
|
||||
}
|
||||
|
||||
export function createAudioPlayer(): AudioPlayer {
|
||||
let currentAudio: HTMLAudioElement | null = null;
|
||||
let playing = false;
|
||||
let queue: Blob[] = [];
|
||||
let queue: QueuedAudio[] = [];
|
||||
let isProcessingQueue = false;
|
||||
|
||||
async function play(audioData: Blob): Promise<void> {
|
||||
// Add to queue
|
||||
queue.push(audioData);
|
||||
async function play(audioData: Blob): Promise<number> {
|
||||
return new Promise((resolve, reject) => {
|
||||
// Add to queue with its promise handlers
|
||||
queue.push({ audioData, resolve, reject });
|
||||
|
||||
// Start processing queue if not already processing
|
||||
if (!isProcessingQueue) {
|
||||
processQueue();
|
||||
}
|
||||
// Start processing queue if not already processing
|
||||
if (!isProcessingQueue) {
|
||||
processQueue();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
async function processQueue(): Promise<void> {
|
||||
@@ -29,14 +37,19 @@ export function createAudioPlayer(): AudioPlayer {
|
||||
isProcessingQueue = true;
|
||||
|
||||
while (queue.length > 0) {
|
||||
const audioData = queue.shift()!;
|
||||
await playAudio(audioData);
|
||||
const item = queue.shift()!;
|
||||
try {
|
||||
const duration = await playAudio(item.audioData);
|
||||
item.resolve(duration);
|
||||
} catch (error) {
|
||||
item.reject(error as Error);
|
||||
}
|
||||
}
|
||||
|
||||
isProcessingQueue = false;
|
||||
}
|
||||
|
||||
async function playAudio(audioData: Blob): Promise<void> {
|
||||
async function playAudio(audioData: Blob): Promise<number> {
|
||||
return new Promise((resolve, reject) => {
|
||||
try {
|
||||
// Create blob URL
|
||||
@@ -47,40 +60,45 @@ export function createAudioPlayer(): AudioPlayer {
|
||||
currentAudio = audio;
|
||||
playing = true;
|
||||
|
||||
console.log(`[AudioPlayer] Playing audio (${audioData.size} bytes, type: ${audioData.type})`);
|
||||
console.log(
|
||||
`[AudioPlayer] Playing audio (${audioData.size} bytes, type: ${audioData.type})`
|
||||
);
|
||||
|
||||
audio.onended = () => {
|
||||
console.log('[AudioPlayer] Playback finished');
|
||||
const duration = audio.duration;
|
||||
console.log(
|
||||
`[AudioPlayer] Playback finished (duration: ${duration}s)`
|
||||
);
|
||||
playing = false;
|
||||
currentAudio = null;
|
||||
|
||||
// Clean up blob URL
|
||||
URL.revokeObjectURL(audioUrl);
|
||||
|
||||
resolve();
|
||||
resolve(duration);
|
||||
};
|
||||
|
||||
audio.onerror = (error) => {
|
||||
console.error('[AudioPlayer] Playback error:', error);
|
||||
console.error("[AudioPlayer] Playback error:", error);
|
||||
playing = false;
|
||||
currentAudio = null;
|
||||
|
||||
// Clean up blob URL
|
||||
URL.revokeObjectURL(audioUrl);
|
||||
|
||||
reject(new Error('Audio playback failed'));
|
||||
reject(new Error("Audio playback failed"));
|
||||
};
|
||||
|
||||
// Start playback
|
||||
audio.play().catch((error) => {
|
||||
console.error('[AudioPlayer] Failed to start playback:', error);
|
||||
console.error("[AudioPlayer] Failed to start playback:", error);
|
||||
playing = false;
|
||||
currentAudio = null;
|
||||
URL.revokeObjectURL(audioUrl);
|
||||
reject(error);
|
||||
});
|
||||
} catch (error) {
|
||||
console.error('[AudioPlayer] Error creating audio element:', error);
|
||||
console.error("[AudioPlayer] Error creating audio element:", error);
|
||||
playing = false;
|
||||
currentAudio = null;
|
||||
reject(error);
|
||||
@@ -95,7 +113,13 @@ export function createAudioPlayer(): AudioPlayer {
|
||||
currentAudio = null;
|
||||
}
|
||||
playing = false;
|
||||
queue = [];
|
||||
|
||||
// Reject all pending promises in the queue
|
||||
while (queue.length > 0) {
|
||||
const item = queue.shift()!;
|
||||
item.reject(new Error("Playback stopped"));
|
||||
}
|
||||
|
||||
isProcessingQueue = false;
|
||||
}
|
||||
|
||||
@@ -104,7 +128,11 @@ export function createAudioPlayer(): AudioPlayer {
|
||||
}
|
||||
|
||||
function clearQueue(): void {
|
||||
queue = [];
|
||||
// Reject all pending promises in the queue
|
||||
while (queue.length > 0) {
|
||||
const item = queue.shift()!;
|
||||
item.reject(new Error("Queue cleared"));
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
|
||||
Reference in New Issue
Block a user