feat: add realtime voice mode, artifact presentation, and Playwright MCP integration
This commit is contained in:
Mohamed Boudra
@@ -236,6 +236,27 @@
|
||||
font-size: 0.75rem;
|
||||
}
|
||||
|
||||
.log-entry.artifact {
|
||||
background-color: #1e3a5f;
|
||||
color: #93c5fd;
|
||||
border: 1px solid #3b82f6;
|
||||
}
|
||||
|
||||
.log-entry.artifact.clickable {
|
||||
cursor: pointer;
|
||||
transition: all 0.2s ease;
|
||||
}
|
||||
|
||||
.log-entry.artifact.clickable:hover {
|
||||
background-color: #2563eb;
|
||||
color: #fff;
|
||||
transform: translateX(4px);
|
||||
}
|
||||
|
||||
.log-entry.artifact.clickable:active {
|
||||
transform: translateX(2px);
|
||||
}
|
||||
|
||||
.log-entry.streaming {
|
||||
opacity: 0.8;
|
||||
}
|
||||
@@ -384,6 +405,65 @@
|
||||
animation: spin 0.8s linear infinite;
|
||||
}
|
||||
|
||||
/* Realtime mode button styles */
|
||||
.send-button.realtime-button {
|
||||
background-color: #8b5cf6;
|
||||
}
|
||||
|
||||
.send-button.realtime-button:hover:not(:disabled) {
|
||||
background-color: #7c3aed;
|
||||
}
|
||||
|
||||
.send-button.realtime-button.active {
|
||||
background-color: #10b981;
|
||||
animation: pulse-realtime 2s ease-in-out infinite;
|
||||
}
|
||||
|
||||
.send-button.realtime-button.active:hover:not(:disabled) {
|
||||
background-color: #059669;
|
||||
}
|
||||
|
||||
.send-button.realtime-button.speech-detected {
|
||||
background-color: #ef4444;
|
||||
animation: pulse-speech 0.8s ease-in-out infinite;
|
||||
}
|
||||
|
||||
.send-button.realtime-button.loading {
|
||||
background-color: #f59e0b;
|
||||
}
|
||||
|
||||
.send-button .loading-indicator {
|
||||
display: inline-block;
|
||||
width: 12px;
|
||||
height: 12px;
|
||||
border: 2px solid #fff;
|
||||
border-top-color: transparent;
|
||||
border-radius: 50%;
|
||||
animation: spin 0.8s linear infinite;
|
||||
}
|
||||
|
||||
@keyframes pulse-realtime {
|
||||
0%, 100% {
|
||||
opacity: 1;
|
||||
transform: scale(1);
|
||||
}
|
||||
50% {
|
||||
opacity: 0.7;
|
||||
transform: scale(1.05);
|
||||
}
|
||||
}
|
||||
|
||||
@keyframes pulse-speech {
|
||||
0%, 100% {
|
||||
opacity: 1;
|
||||
box-shadow: 0 0 0 0 rgba(239, 68, 68, 0.7);
|
||||
}
|
||||
50% {
|
||||
opacity: 0.9;
|
||||
box-shadow: 0 0 0 8px rgba(239, 68, 68, 0);
|
||||
}
|
||||
}
|
||||
|
||||
.test-controls {
|
||||
padding-top: 0.5rem;
|
||||
border-top: 1px solid #333;
|
||||
|
||||
@@ -1,9 +1,11 @@
|
||||
import { useState, useEffect, useRef } from "react";
|
||||
import { Mic, Send } from "lucide-react";
|
||||
import { Mic, Send, Radio } from "lucide-react";
|
||||
import { useWebSocket } from "./hooks/useWebSocket";
|
||||
import { createAudioPlayer } from "./lib/audio-playback";
|
||||
import { createAudioRecorder, type AudioRecorder } from "./lib/audio-capture";
|
||||
import { createRealtimeVAD, float32ArrayToBlob, type RealtimeVAD } from "./lib/audio-realtime";
|
||||
import { ToolCallCard } from "./components/ToolCallCard";
|
||||
import { ArtifactDrawer, type Artifact } from "./components/ArtifactDrawer";
|
||||
import "./App.css";
|
||||
|
||||
type LogEntry =
|
||||
@@ -23,6 +25,14 @@ type LogEntry =
|
||||
result?: any;
|
||||
error?: any;
|
||||
status: "executing" | "completed" | "failed";
|
||||
}
|
||||
| {
|
||||
type: "artifact";
|
||||
id: string;
|
||||
timestamp: number;
|
||||
artifactId: string;
|
||||
artifactType: string;
|
||||
title: string;
|
||||
};
|
||||
|
||||
function App() {
|
||||
@@ -39,9 +49,15 @@ function App() {
|
||||
const [isProcessingAudio, setIsProcessingAudio] = useState(false);
|
||||
const [isPlayingAudio, setIsPlayingAudio] = useState(false);
|
||||
const [isRecording, setIsRecording] = useState(false);
|
||||
const [isRealtimeMode, setIsRealtimeMode] = useState(false);
|
||||
const [isSpeechDetected, setIsSpeechDetected] = useState(false);
|
||||
const [isVADLoading, setIsVADLoading] = useState(false);
|
||||
const [currentArtifact, setCurrentArtifact] = useState<Artifact | null>(null);
|
||||
const [artifacts, setArtifacts] = useState<Map<string, Artifact>>(new Map());
|
||||
const logEndRef = useRef<HTMLDivElement>(null);
|
||||
const audioPlayerRef = useRef(createAudioPlayer());
|
||||
const recorderRef = useRef<AudioRecorder | null>(null);
|
||||
const vadRef = useRef<RealtimeVAD | null>(null);
|
||||
const textareaRef = useRef<HTMLTextAreaElement>(null);
|
||||
|
||||
// WebSocket URL - use ws://localhost:3000/ws in dev, or construct from current host in prod
|
||||
@@ -53,6 +69,13 @@ function App() {
|
||||
|
||||
useEffect(() => {
|
||||
recorderRef.current = createAudioRecorder();
|
||||
|
||||
// Cleanup VAD on unmount
|
||||
return () => {
|
||||
if (vadRef.current) {
|
||||
vadRef.current.destroy();
|
||||
}
|
||||
};
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
@@ -186,6 +209,34 @@ function App() {
|
||||
}
|
||||
);
|
||||
|
||||
// Listen for artifacts
|
||||
const unsubArtifact = ws.on("artifact", (payload: unknown) => {
|
||||
const artifact = payload as Artifact;
|
||||
|
||||
// Add to artifacts map
|
||||
setArtifacts((prev) => {
|
||||
const newMap = new Map(prev);
|
||||
newMap.set(artifact.id, artifact);
|
||||
return newMap;
|
||||
});
|
||||
|
||||
// Add as clickable log entry
|
||||
setLogs((prev) => [
|
||||
...prev,
|
||||
{
|
||||
type: "artifact" as const,
|
||||
id: Date.now().toString(),
|
||||
timestamp: Date.now(),
|
||||
artifactId: artifact.id,
|
||||
artifactType: artifact.type,
|
||||
title: artifact.title,
|
||||
},
|
||||
]);
|
||||
|
||||
// Show drawer by default
|
||||
setCurrentArtifact(artifact);
|
||||
});
|
||||
|
||||
// Listen for audio output (TTS)
|
||||
const unsubAudioOutput = ws.on("audio_output", async (payload: unknown) => {
|
||||
const data = payload as { audio: string; format: string; id: string };
|
||||
@@ -228,6 +279,7 @@ function App() {
|
||||
unsubActivity();
|
||||
unsubChunk();
|
||||
unsubTranscription();
|
||||
unsubArtifact();
|
||||
unsubAudioOutput();
|
||||
};
|
||||
}, [ws]);
|
||||
@@ -336,6 +388,116 @@ function App() {
|
||||
}
|
||||
};
|
||||
|
||||
const handleCloseArtifact = () => {
|
||||
setCurrentArtifact(null);
|
||||
};
|
||||
|
||||
const handleOpenArtifact = (artifactId: string) => {
|
||||
const artifact = artifacts.get(artifactId);
|
||||
if (artifact) {
|
||||
setCurrentArtifact(artifact);
|
||||
}
|
||||
};
|
||||
|
||||
const handleToggleRealtimeMode = async () => {
|
||||
if (!ws.isConnected) return;
|
||||
|
||||
try {
|
||||
if (isRealtimeMode) {
|
||||
// Stop realtime mode
|
||||
if (vadRef.current) {
|
||||
vadRef.current.pause();
|
||||
}
|
||||
setIsRealtimeMode(false);
|
||||
setIsSpeechDetected(false);
|
||||
addLog("info", "Realtime mode stopped");
|
||||
} else {
|
||||
// Start realtime mode
|
||||
setIsVADLoading(true);
|
||||
addLog("info", "Starting realtime mode...");
|
||||
|
||||
// Create VAD instance if it doesn't exist
|
||||
if (!vadRef.current) {
|
||||
vadRef.current = createRealtimeVAD({
|
||||
onModelLoading: () => {
|
||||
console.log("[App] VAD model loading...");
|
||||
setIsVADLoading(true);
|
||||
},
|
||||
onModelLoaded: () => {
|
||||
console.log("[App] VAD model loaded");
|
||||
setIsVADLoading(false);
|
||||
},
|
||||
onSpeechStart: () => {
|
||||
console.log("[App] Speech detected!");
|
||||
setIsSpeechDetected(true);
|
||||
|
||||
// Interrupt playback when speech is detected
|
||||
audioPlayerRef.current.stop();
|
||||
setIsPlayingAudio(false);
|
||||
setCurrentAssistantMessage("");
|
||||
},
|
||||
onSpeechEnd: async (audioData: Float32Array) => {
|
||||
console.log("[App] Speech ended, processing...");
|
||||
setIsSpeechDetected(false);
|
||||
setIsProcessingAudio(true);
|
||||
|
||||
try {
|
||||
// Convert Float32Array to audio blob
|
||||
const audioBlob = float32ArrayToBlob(audioData, 16000);
|
||||
console.log(`[App] Converted to blob: ${audioBlob.size} bytes`);
|
||||
|
||||
// Send to server
|
||||
const arrayBuffer = await audioBlob.arrayBuffer();
|
||||
const base64Audio = btoa(
|
||||
new Uint8Array(arrayBuffer).reduce(
|
||||
(data, byte) => data + String.fromCharCode(byte),
|
||||
""
|
||||
)
|
||||
);
|
||||
|
||||
ws.send({
|
||||
type: "audio_chunk",
|
||||
payload: {
|
||||
audio: base64Audio,
|
||||
format: audioBlob.type,
|
||||
isLast: true,
|
||||
},
|
||||
});
|
||||
|
||||
console.log(`[App] Sent realtime audio to server`);
|
||||
} catch (error: any) {
|
||||
console.error("[App] Failed to process VAD audio:", error);
|
||||
addLog("error", `Failed to process audio: ${error.message}`);
|
||||
setIsProcessingAudio(false);
|
||||
}
|
||||
},
|
||||
onVADMisfire: () => {
|
||||
console.log("[App] VAD misfire");
|
||||
setIsSpeechDetected(false);
|
||||
},
|
||||
onError: (error: Error) => {
|
||||
console.error("[App] VAD error:", error);
|
||||
addLog("error", `VAD error: ${error.message}`);
|
||||
setIsVADLoading(false);
|
||||
setIsRealtimeMode(false);
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
// Start VAD
|
||||
await vadRef.current.start();
|
||||
setIsRealtimeMode(true);
|
||||
setIsVADLoading(false);
|
||||
addLog("success", "Realtime mode started - speak anytime!");
|
||||
}
|
||||
} catch (error: any) {
|
||||
console.error("[App] Realtime mode error:", error);
|
||||
addLog("error", `Failed to toggle realtime mode: ${error.message}`);
|
||||
setIsRealtimeMode(false);
|
||||
setIsVADLoading(false);
|
||||
}
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="app">
|
||||
<header className="header">
|
||||
@@ -356,17 +518,42 @@ function App() {
|
||||
<div className="chat-interface">
|
||||
<div className="activity-log">
|
||||
<div className="log-entries">
|
||||
{logs.map((log) =>
|
||||
log.type === "tool_call" ? (
|
||||
<ToolCallCard
|
||||
key={log.id}
|
||||
toolName={log.toolName}
|
||||
args={log.args}
|
||||
result={log.result}
|
||||
error={log.error}
|
||||
status={log.status}
|
||||
/>
|
||||
) : (
|
||||
{logs.map((log) => {
|
||||
if (log.type === "tool_call") {
|
||||
return (
|
||||
<ToolCallCard
|
||||
key={log.id}
|
||||
toolName={log.toolName}
|
||||
args={log.args}
|
||||
result={log.result}
|
||||
error={log.error}
|
||||
status={log.status}
|
||||
/>
|
||||
);
|
||||
}
|
||||
|
||||
if (log.type === "artifact") {
|
||||
return (
|
||||
<div
|
||||
key={log.id}
|
||||
className="log-entry artifact clickable"
|
||||
onClick={() => handleOpenArtifact(log.artifactId)}
|
||||
role="button"
|
||||
tabIndex={0}
|
||||
onKeyDown={(e) => {
|
||||
if (e.key === "Enter" || e.key === " ") {
|
||||
handleOpenArtifact(log.artifactId);
|
||||
}
|
||||
}}
|
||||
>
|
||||
<span className="log-message">
|
||||
📋 {log.artifactType}: {log.title}
|
||||
</span>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
return (
|
||||
<div key={log.id} className={`log-entry ${log.type}`}>
|
||||
<span className="log-message">{log.message}</span>
|
||||
{log.metadata && (
|
||||
@@ -376,8 +563,8 @@ function App() {
|
||||
</details>
|
||||
)}
|
||||
</div>
|
||||
)
|
||||
)}
|
||||
);
|
||||
})}
|
||||
{currentAssistantMessage && (
|
||||
<div className="log-entry assistant streaming">
|
||||
<span className="log-message">{currentAssistantMessage}</span>
|
||||
@@ -395,14 +582,27 @@ function App() {
|
||||
onChange={(e) => setUserInput(e.target.value)}
|
||||
onKeyDown={handleKeyDown}
|
||||
placeholder="Type a message or tap mic to talk..."
|
||||
disabled={!ws.isConnected || isRecording}
|
||||
disabled={!ws.isConnected || isRecording || isRealtimeMode}
|
||||
className="message-input"
|
||||
rows={1}
|
||||
/>
|
||||
<button
|
||||
type="button"
|
||||
onClick={handleToggleRealtimeMode}
|
||||
disabled={!ws.isConnected || isRecording || isVADLoading}
|
||||
className={`send-button realtime-button ${isRealtimeMode ? 'active' : ''} ${isSpeechDetected ? 'speech-detected' : ''} ${isVADLoading ? 'loading' : ''}`}
|
||||
title={isRealtimeMode ? "Stop realtime mode" : "Start realtime mode"}
|
||||
>
|
||||
{isVADLoading ? (
|
||||
<span className="loading-indicator" />
|
||||
) : (
|
||||
<Radio size={20} />
|
||||
)}
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
onClick={handleButtonClick}
|
||||
disabled={!ws.isConnected || isProcessingAudio}
|
||||
disabled={!ws.isConnected || isProcessingAudio || isRealtimeMode}
|
||||
className={`send-button ${isRecording ? 'recording' : ''} ${isProcessingAudio ? 'processing' : ''} ${userInput.trim() ? 'has-text' : ''}`}
|
||||
>
|
||||
{isRecording ? (
|
||||
@@ -418,6 +618,11 @@ function App() {
|
||||
</div>
|
||||
</div>
|
||||
</main>
|
||||
|
||||
<ArtifactDrawer
|
||||
artifact={currentArtifact}
|
||||
onClose={handleCloseArtifact}
|
||||
/>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,268 @@
|
||||
.artifact-drawer-overlay {
|
||||
position: fixed;
|
||||
top: 0;
|
||||
left: 0;
|
||||
right: 0;
|
||||
bottom: 0;
|
||||
background-color: rgba(0, 0, 0, 0.5);
|
||||
z-index: 1000;
|
||||
animation: fadeIn 0.2s ease-out;
|
||||
}
|
||||
|
||||
@keyframes fadeIn {
|
||||
from {
|
||||
opacity: 0;
|
||||
}
|
||||
to {
|
||||
opacity: 1;
|
||||
}
|
||||
}
|
||||
|
||||
.artifact-drawer {
|
||||
position: fixed;
|
||||
bottom: 0;
|
||||
left: 0;
|
||||
right: 0;
|
||||
max-height: 80vh;
|
||||
background: #1a1a1a;
|
||||
border-top-left-radius: 12px;
|
||||
border-top-right-radius: 12px;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
animation: slideUp 0.3s ease-out;
|
||||
box-shadow: 0 -4px 20px rgba(0, 0, 0, 0.3);
|
||||
}
|
||||
|
||||
@keyframes slideUp {
|
||||
from {
|
||||
transform: translateY(100%);
|
||||
}
|
||||
to {
|
||||
transform: translateY(0);
|
||||
}
|
||||
}
|
||||
|
||||
.artifact-drawer-header {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
padding: 20px 24px;
|
||||
border-bottom: 1px solid #333;
|
||||
flex-shrink: 0;
|
||||
}
|
||||
|
||||
.artifact-drawer-title {
|
||||
margin: 0;
|
||||
font-size: 1.25rem;
|
||||
font-weight: 600;
|
||||
color: #fff;
|
||||
flex: 1;
|
||||
}
|
||||
|
||||
.artifact-drawer-header-right {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 12px;
|
||||
}
|
||||
|
||||
.artifact-drawer-type-badge {
|
||||
padding: 4px 12px;
|
||||
background-color: #2563eb;
|
||||
color: #fff;
|
||||
border-radius: 12px;
|
||||
font-size: 0.75rem;
|
||||
font-weight: 500;
|
||||
text-transform: uppercase;
|
||||
letter-spacing: 0.5px;
|
||||
}
|
||||
|
||||
.artifact-drawer-close-button {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
width: 32px;
|
||||
height: 32px;
|
||||
padding: 0;
|
||||
background: transparent;
|
||||
border: none;
|
||||
color: #9ca3af;
|
||||
cursor: pointer;
|
||||
border-radius: 6px;
|
||||
transition: all 0.2s ease;
|
||||
}
|
||||
|
||||
.artifact-drawer-close-button:hover {
|
||||
background-color: #2a2a2a;
|
||||
color: #fff;
|
||||
}
|
||||
|
||||
.artifact-drawer-content {
|
||||
flex: 1;
|
||||
overflow-y: auto;
|
||||
padding: 24px;
|
||||
min-height: 200px;
|
||||
}
|
||||
|
||||
.artifact-plan {
|
||||
color: #e5e5e5;
|
||||
line-height: 1.7;
|
||||
}
|
||||
|
||||
/* Markdown styles */
|
||||
.artifact-plan h1 {
|
||||
font-size: 1.75rem;
|
||||
font-weight: 700;
|
||||
margin-top: 0;
|
||||
margin-bottom: 1rem;
|
||||
color: #fff;
|
||||
}
|
||||
|
||||
.artifact-plan h2 {
|
||||
font-size: 1.5rem;
|
||||
font-weight: 600;
|
||||
margin-top: 1.5rem;
|
||||
margin-bottom: 0.75rem;
|
||||
color: #fff;
|
||||
}
|
||||
|
||||
.artifact-plan h3 {
|
||||
font-size: 1.25rem;
|
||||
font-weight: 600;
|
||||
margin-top: 1.25rem;
|
||||
margin-bottom: 0.5rem;
|
||||
color: #fff;
|
||||
}
|
||||
|
||||
.artifact-plan h4,
|
||||
.artifact-plan h5,
|
||||
.artifact-plan h6 {
|
||||
font-size: 1rem;
|
||||
font-weight: 600;
|
||||
margin-top: 1rem;
|
||||
margin-bottom: 0.5rem;
|
||||
color: #fff;
|
||||
}
|
||||
|
||||
.artifact-plan p {
|
||||
margin-bottom: 1rem;
|
||||
}
|
||||
|
||||
.artifact-plan ul,
|
||||
.artifact-plan ol {
|
||||
margin-bottom: 1rem;
|
||||
padding-left: 1.5rem;
|
||||
}
|
||||
|
||||
.artifact-plan li {
|
||||
margin-bottom: 0.5rem;
|
||||
}
|
||||
|
||||
.artifact-plan code {
|
||||
background-color: #2a2a2a;
|
||||
padding: 2px 6px;
|
||||
border-radius: 4px;
|
||||
font-family: "Monaco", "Courier New", monospace;
|
||||
font-size: 0.9em;
|
||||
color: #fff;
|
||||
}
|
||||
|
||||
.artifact-plan pre {
|
||||
background-color: #2a2a2a;
|
||||
padding: 12px;
|
||||
border-radius: 6px;
|
||||
overflow-x: auto;
|
||||
margin-bottom: 1rem;
|
||||
}
|
||||
|
||||
.artifact-plan pre code {
|
||||
background-color: transparent;
|
||||
padding: 0;
|
||||
}
|
||||
|
||||
.artifact-plan blockquote {
|
||||
border-left: 4px solid #2563eb;
|
||||
padding-left: 1rem;
|
||||
margin-left: 0;
|
||||
margin-bottom: 1rem;
|
||||
color: #a3a3a3;
|
||||
font-style: italic;
|
||||
}
|
||||
|
||||
.artifact-plan a {
|
||||
color: #3b82f6;
|
||||
text-decoration: none;
|
||||
}
|
||||
|
||||
.artifact-plan a:hover {
|
||||
text-decoration: underline;
|
||||
}
|
||||
|
||||
/* Diff content styles */
|
||||
.artifact-diff {
|
||||
background-color: #2a2a2a;
|
||||
padding: 16px;
|
||||
border-radius: 6px;
|
||||
overflow-x: auto;
|
||||
color: #e5e5e5;
|
||||
font-family: "Monaco", "Courier New", monospace;
|
||||
font-size: 0.9em;
|
||||
line-height: 1.6;
|
||||
white-space: pre-wrap;
|
||||
word-break: break-word;
|
||||
}
|
||||
|
||||
.artifact-diff code {
|
||||
background-color: transparent;
|
||||
padding: 0;
|
||||
}
|
||||
|
||||
/* Code content styles */
|
||||
.artifact-code {
|
||||
background-color: #2a2a2a;
|
||||
padding: 16px;
|
||||
border-radius: 6px;
|
||||
overflow-x: auto;
|
||||
color: #e5e5e5;
|
||||
font-family: "Monaco", "Courier New", monospace;
|
||||
font-size: 0.9em;
|
||||
line-height: 1.6;
|
||||
}
|
||||
|
||||
.artifact-code code {
|
||||
background-color: transparent;
|
||||
padding: 0;
|
||||
}
|
||||
|
||||
/* Image content styles */
|
||||
.artifact-image {
|
||||
display: flex;
|
||||
justify-content: center;
|
||||
align-items: center;
|
||||
padding: 16px;
|
||||
}
|
||||
|
||||
.artifact-image img {
|
||||
max-width: 100%;
|
||||
max-height: 70vh;
|
||||
border-radius: 8px;
|
||||
box-shadow: 0 4px 12px rgba(0, 0, 0, 0.3);
|
||||
}
|
||||
|
||||
/* Mobile responsiveness */
|
||||
@media (max-width: 768px) {
|
||||
.artifact-drawer {
|
||||
max-height: 90vh;
|
||||
}
|
||||
|
||||
.artifact-drawer-header {
|
||||
padding: 16px 20px;
|
||||
}
|
||||
|
||||
.artifact-drawer-content {
|
||||
padding: 20px;
|
||||
}
|
||||
|
||||
.artifact-drawer-title {
|
||||
font-size: 1.1rem;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,91 @@
|
||||
import { useEffect, useRef } from "react";
|
||||
import ReactMarkdown from "react-markdown";
|
||||
import { X } from "lucide-react";
|
||||
import "./ArtifactDrawer.css";
|
||||
|
||||
export type Artifact = {
|
||||
type: "markdown" | "diff" | "image" | "code";
|
||||
id: string;
|
||||
title: string;
|
||||
content: string;
|
||||
isBase64: boolean;
|
||||
};
|
||||
|
||||
interface ArtifactDrawerProps {
|
||||
artifact: Artifact | null;
|
||||
onClose: () => void;
|
||||
}
|
||||
|
||||
export function ArtifactDrawer({ artifact, onClose }: ArtifactDrawerProps) {
|
||||
const drawerRef = useRef<HTMLDivElement>(null);
|
||||
|
||||
useEffect(() => {
|
||||
if (artifact) {
|
||||
// Prevent body scroll when drawer is open
|
||||
document.body.style.overflow = "hidden";
|
||||
} else {
|
||||
document.body.style.overflow = "";
|
||||
}
|
||||
|
||||
return () => {
|
||||
document.body.style.overflow = "";
|
||||
};
|
||||
}, [artifact]);
|
||||
|
||||
if (!artifact) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="artifact-drawer-overlay" onClick={onClose}>
|
||||
<div ref={drawerRef} className="artifact-drawer" onClick={(e) => e.stopPropagation()}>
|
||||
<div className="artifact-drawer-header">
|
||||
<h2 className="artifact-drawer-title">
|
||||
{artifact.title}
|
||||
</h2>
|
||||
<div className="artifact-drawer-header-right">
|
||||
<div className="artifact-drawer-type-badge">
|
||||
{artifact.type}
|
||||
</div>
|
||||
<button
|
||||
className="artifact-drawer-close-button"
|
||||
onClick={onClose}
|
||||
type="button"
|
||||
title="Close"
|
||||
>
|
||||
<X size={20} />
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="artifact-drawer-content">
|
||||
{artifact.type === "markdown" && (
|
||||
<div className="artifact-plan">
|
||||
<ReactMarkdown>
|
||||
{artifact.isBase64 ? atob(artifact.content) : artifact.content}
|
||||
</ReactMarkdown>
|
||||
</div>
|
||||
)}
|
||||
{artifact.type === "diff" && (
|
||||
<pre className="artifact-diff">
|
||||
<code>{artifact.isBase64 ? atob(artifact.content) : artifact.content}</code>
|
||||
</pre>
|
||||
)}
|
||||
{artifact.type === "code" && (
|
||||
<pre className="artifact-code">
|
||||
<code>{artifact.isBase64 ? atob(artifact.content) : artifact.content}</code>
|
||||
</pre>
|
||||
)}
|
||||
{artifact.type === "image" && (
|
||||
<div className="artifact-image">
|
||||
<img
|
||||
src={`data:image/png;base64,${artifact.content}`}
|
||||
alt={artifact.title}
|
||||
/>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,176 @@
|
||||
import { MicVAD } from "@ricky0123/vad-web";
|
||||
|
||||
export interface RealtimeVADConfig {
|
||||
onSpeechStart?: () => void;
|
||||
onSpeechEnd?: (audio: Float32Array) => void;
|
||||
onVADMisfire?: () => void;
|
||||
onModelLoading?: () => void;
|
||||
onModelLoaded?: () => void;
|
||||
onError?: (error: Error) => void;
|
||||
positiveSpeechThreshold?: number;
|
||||
negativeSpeechThreshold?: number;
|
||||
redemptionFrames?: number;
|
||||
frameSamples?: number;
|
||||
preSpeechPadFrames?: number;
|
||||
minSpeechFrames?: number;
|
||||
}
|
||||
|
||||
export interface RealtimeVAD {
|
||||
start(): Promise<void>;
|
||||
pause(): void;
|
||||
destroy(): void;
|
||||
isListening(): boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a realtime VAD (Voice Activity Detection) instance using @ricky0123/vad-web
|
||||
* Continuously listens for speech and invokes callbacks when speech is detected
|
||||
*/
|
||||
export function createRealtimeVAD(config: RealtimeVADConfig): RealtimeVAD {
|
||||
let vad: Awaited<ReturnType<typeof MicVAD.new>> | null = null;
|
||||
let isActive = false;
|
||||
|
||||
async function start(): Promise<void> {
|
||||
// If VAD already exists and is paused, just resume it
|
||||
if (vad && !isActive) {
|
||||
vad.start();
|
||||
isActive = true;
|
||||
console.log("[RealtimeVAD] VAD resumed");
|
||||
return;
|
||||
}
|
||||
|
||||
if (isActive) {
|
||||
throw new Error("VAD is already active");
|
||||
}
|
||||
|
||||
// Create new VAD instance
|
||||
try {
|
||||
config.onModelLoading?.();
|
||||
|
||||
vad = await MicVAD.new({
|
||||
onSpeechStart: () => {
|
||||
console.log("[RealtimeVAD] Speech start detected");
|
||||
config.onSpeechStart?.();
|
||||
},
|
||||
onSpeechEnd: (audio: Float32Array) => {
|
||||
console.log("[RealtimeVAD] Speech end detected", {
|
||||
samples: audio.length,
|
||||
durationSeconds: audio.length / 16000,
|
||||
});
|
||||
config.onSpeechEnd?.(audio);
|
||||
},
|
||||
onVADMisfire: () => {
|
||||
console.log("[RealtimeVAD] VAD misfire (false positive)");
|
||||
config.onVADMisfire?.();
|
||||
},
|
||||
positiveSpeechThreshold: config.positiveSpeechThreshold ?? 0.5,
|
||||
negativeSpeechThreshold: config.negativeSpeechThreshold ?? 0.35,
|
||||
redemptionMs: config.redemptionFrames ? config.redemptionFrames * 16 : 128,
|
||||
preSpeechPadMs: config.preSpeechPadFrames ? config.preSpeechPadFrames * 16 : 16,
|
||||
minSpeechMs: config.minSpeechFrames ? config.minSpeechFrames * 16 : 48,
|
||||
});
|
||||
|
||||
isActive = true;
|
||||
config.onModelLoaded?.();
|
||||
console.log("[RealtimeVAD] VAD created and started successfully");
|
||||
} catch (error) {
|
||||
console.error("[RealtimeVAD] Failed to start VAD:", error);
|
||||
const err =
|
||||
error instanceof Error ? error : new Error(String(error));
|
||||
config.onError?.(err);
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
function pause(): void {
|
||||
if (vad && isActive) {
|
||||
vad.pause();
|
||||
isActive = false;
|
||||
console.log("[RealtimeVAD] VAD paused");
|
||||
}
|
||||
}
|
||||
|
||||
function destroy(): void {
|
||||
if (vad) {
|
||||
vad.destroy();
|
||||
vad = null;
|
||||
isActive = false;
|
||||
console.log("[RealtimeVAD] VAD destroyed");
|
||||
}
|
||||
}
|
||||
|
||||
function isListening(): boolean {
|
||||
return isActive;
|
||||
}
|
||||
|
||||
return {
|
||||
start,
|
||||
pause,
|
||||
destroy,
|
||||
isListening,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert Float32Array PCM audio to audio blob
|
||||
* VAD returns Float32Array at 16kHz sample rate
|
||||
*/
|
||||
export function float32ArrayToBlob(
|
||||
audio: Float32Array,
|
||||
sampleRate: number = 16000
|
||||
): Blob {
|
||||
// Convert Float32Array to 16-bit PCM
|
||||
const pcm = new Int16Array(audio.length);
|
||||
for (let i = 0; i < audio.length; i++) {
|
||||
const s = Math.max(-1, Math.min(1, audio[i]));
|
||||
pcm[i] = s < 0 ? s * 0x8000 : s * 0x7fff;
|
||||
}
|
||||
|
||||
// Create WAV header
|
||||
const wavHeader = createWavHeader(pcm.length * 2, sampleRate);
|
||||
|
||||
// Combine header and data
|
||||
const wavBytes = new Uint8Array(wavHeader.length + pcm.length * 2);
|
||||
wavBytes.set(wavHeader, 0);
|
||||
wavBytes.set(new Uint8Array(pcm.buffer), wavHeader.length);
|
||||
|
||||
return new Blob([wavBytes], { type: "audio/wav" });
|
||||
}
|
||||
|
||||
/**
|
||||
* Create WAV file header
|
||||
*/
|
||||
function createWavHeader(dataLength: number, sampleRate: number): Uint8Array {
|
||||
const header = new ArrayBuffer(44);
|
||||
const view = new DataView(header);
|
||||
|
||||
// "RIFF" chunk descriptor
|
||||
writeString(view, 0, "RIFF");
|
||||
view.setUint32(4, 36 + dataLength, true);
|
||||
writeString(view, 8, "WAVE");
|
||||
|
||||
// "fmt " sub-chunk
|
||||
writeString(view, 12, "fmt ");
|
||||
view.setUint32(16, 16, true); // SubChunk1Size (16 for PCM)
|
||||
view.setUint16(20, 1, true); // AudioFormat (1 for PCM)
|
||||
view.setUint16(22, 1, true); // NumChannels (1 = mono)
|
||||
view.setUint32(24, sampleRate, true); // SampleRate
|
||||
view.setUint32(28, sampleRate * 2, true); // ByteRate
|
||||
view.setUint16(32, 2, true); // BlockAlign
|
||||
view.setUint16(34, 16, true); // BitsPerSample
|
||||
|
||||
// "data" sub-chunk
|
||||
writeString(view, 36, "data");
|
||||
view.setUint32(40, dataLength, true);
|
||||
|
||||
return new Uint8Array(header);
|
||||
}
|
||||
|
||||
/**
|
||||
* Write string to DataView
|
||||
*/
|
||||
function writeString(view: DataView, offset: number, str: string): void {
|
||||
for (let i = 0; i < str.length; i++) {
|
||||
view.setUint8(offset + i, str.charCodeAt(i));
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user