feat: add realtime voice mode, artifact presentation, and Playwright MCP integration
This commit is contained in:
Mohamed Boudra
@@ -11,6 +11,7 @@ import {
|
||||
killTerminal,
|
||||
} from "../daemon/terminal-manager.js";
|
||||
import invariant from "tiny-invariant";
|
||||
import { getPlaywrightTools } from "./playwright-mcp.js";
|
||||
|
||||
/**
|
||||
* Terminal tools using Vercel AI SDK tool() function
|
||||
@@ -208,8 +209,68 @@ export const terminalTools = {
|
||||
return { success: true };
|
||||
},
|
||||
}),
|
||||
|
||||
present_artifact: tool({
|
||||
description:
|
||||
"Present an artifact (plan, diff, screenshot, etc.) to the user for review. Use this when you need to show information that's hard to convey via TTS, such as markdown plans, code diffs, or visual content",
|
||||
inputSchema: z.object({
|
||||
type: z
|
||||
.enum(["markdown", "diff", "image", "code"])
|
||||
.describe("Type of artifact to present."),
|
||||
source: z.union([
|
||||
z.object({
|
||||
type: z.literal("file"),
|
||||
path: z.string(),
|
||||
}),
|
||||
z.object({
|
||||
type: z.literal("command_output"),
|
||||
command: z.string(),
|
||||
}),
|
||||
z.object({
|
||||
type: z.literal("text"),
|
||||
text: z.string(),
|
||||
}),
|
||||
]),
|
||||
title: z
|
||||
.string()
|
||||
.describe(
|
||||
"Title for the artifact (e.g., 'Implementation Plan', 'Refactoring Strategy', '/path/to/project/package.json')."
|
||||
),
|
||||
}),
|
||||
execute: async () => {
|
||||
// Artifact will be broadcast by orchestrator via onToolCall callback
|
||||
// We just return a simple acknowledgment here
|
||||
return {
|
||||
success: true,
|
||||
message: "Artifact presented to user.",
|
||||
};
|
||||
},
|
||||
}),
|
||||
};
|
||||
|
||||
/**
|
||||
* Cache Playwright MCP tools (lazy loaded on first use)
|
||||
*/
|
||||
let playwrightToolsCache: Record<string, any> | null = null;
|
||||
let playwrightToolsPromise: Promise<Record<string, any>> | null = null;
|
||||
|
||||
async function getCachedPlaywrightTools(): Promise<Record<string, any>> {
|
||||
if (playwrightToolsCache) {
|
||||
return playwrightToolsCache;
|
||||
}
|
||||
|
||||
if (playwrightToolsPromise) {
|
||||
return playwrightToolsPromise;
|
||||
}
|
||||
|
||||
playwrightToolsPromise = getPlaywrightTools().then((tools) => {
|
||||
playwrightToolsCache = tools;
|
||||
return tools;
|
||||
});
|
||||
|
||||
return playwrightToolsPromise;
|
||||
}
|
||||
|
||||
/**
|
||||
* Message interface for conversation
|
||||
*/
|
||||
@@ -252,11 +313,18 @@ export async function streamLLM(params: StreamLLMParams): Promise<string> {
|
||||
apiKey: process.env.OPENROUTER_API_KEY,
|
||||
});
|
||||
|
||||
// Merge terminal tools with Playwright MCP tools
|
||||
const pwTools = await getCachedPlaywrightTools();
|
||||
const allTools = {
|
||||
...terminalTools,
|
||||
...pwTools,
|
||||
};
|
||||
|
||||
const result = await streamText({
|
||||
model: openrouter("anthropic/claude-haiku-4.5"),
|
||||
system: params.systemPrompt,
|
||||
messages: params.messages,
|
||||
tools: terminalTools,
|
||||
tools: allTools,
|
||||
abortSignal: params.abortSignal,
|
||||
onChunk: async ({ chunk }) => {
|
||||
// console.log("onChunk", chunk);
|
||||
|
||||
@@ -1,8 +1,14 @@
|
||||
import { v4 as uuidv4 } from "uuid";
|
||||
import { readFile } from "fs/promises";
|
||||
import { exec } from "child_process";
|
||||
import { promisify } from "util";
|
||||
import { getSystemPrompt } from "./system-prompt.js";
|
||||
import { streamLLM, type Message } from "./llm-openai.js";
|
||||
import { generateTTSAndWaitForPlayback } from "./tts-manager.js";
|
||||
import type { VoiceAssistantWebSocketServer } from "../websocket-server.js";
|
||||
import type { ArtifactPayload } from "../types.js";
|
||||
|
||||
const execAsync = promisify(exec);
|
||||
|
||||
interface ConversationContext {
|
||||
id: string;
|
||||
@@ -109,6 +115,60 @@ export async function processUserMessage(params: {
|
||||
await pendingTTS;
|
||||
pendingTTS = null;
|
||||
}
|
||||
|
||||
// Handle present_artifact tool specially
|
||||
if (toolName === "present_artifact" && params.wsServer) {
|
||||
const artifactId = uuidv4();
|
||||
|
||||
// Resolve source to content
|
||||
let content: string;
|
||||
let isBase64 = false;
|
||||
try {
|
||||
if (args.source.type === "file") {
|
||||
const fileBuffer = await readFile(args.source.path);
|
||||
content = fileBuffer.toString("base64");
|
||||
isBase64 = true;
|
||||
} else if (args.source.type === "command_output") {
|
||||
const { stdout } = await execAsync(args.source.command, { encoding: 'buffer' });
|
||||
content = stdout.toString("base64");
|
||||
isBase64 = true;
|
||||
} else if (args.source.type === "text") {
|
||||
content = args.source.text;
|
||||
isBase64 = false;
|
||||
} else {
|
||||
content = "[Unknown source type]";
|
||||
isBase64 = false;
|
||||
}
|
||||
} catch (error) {
|
||||
console.error("Failed to resolve artifact source:", error);
|
||||
content = `[Error resolving source: ${error instanceof Error ? error.message : String(error)}]`;
|
||||
isBase64 = false;
|
||||
}
|
||||
|
||||
const artifact: ArtifactPayload = {
|
||||
type: args.type,
|
||||
id: artifactId,
|
||||
title: args.title,
|
||||
content,
|
||||
isBase64,
|
||||
};
|
||||
|
||||
// Broadcast artifact to client
|
||||
params.wsServer.broadcast({
|
||||
type: "artifact",
|
||||
payload: artifact,
|
||||
});
|
||||
|
||||
// Broadcast as activity log entry so it appears in the feed
|
||||
params.wsServer.broadcastActivityLog({
|
||||
id: artifactId,
|
||||
timestamp: new Date(),
|
||||
type: "system",
|
||||
content: `${args.type} artifact: ${args.title}`,
|
||||
metadata: { artifactId, artifactType: args.type },
|
||||
});
|
||||
}
|
||||
|
||||
// Broadcast tool call to WebSocket
|
||||
if (params.wsServer) {
|
||||
params.wsServer.broadcastActivityLog({
|
||||
@@ -240,3 +300,4 @@ export function getConversationStats(): {
|
||||
})),
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
import { experimental_createMCPClient } from "ai";
|
||||
import { StdioClientTransport } from "@modelcontextprotocol/sdk/client/stdio.js";
|
||||
|
||||
let playwrightMCPClient: Awaited<
|
||||
ReturnType<typeof experimental_createMCPClient>
|
||||
> | null = null;
|
||||
|
||||
/**
|
||||
* Get or create the Playwright MCP client (singleton)
|
||||
*/
|
||||
export async function getPlaywrightMCPClient() {
|
||||
if (playwrightMCPClient) {
|
||||
return playwrightMCPClient;
|
||||
}
|
||||
|
||||
const transport = new StdioClientTransport({
|
||||
command: "npx",
|
||||
args: ["@playwright/mcp", "--browser", "firefox"],
|
||||
});
|
||||
|
||||
playwrightMCPClient = await experimental_createMCPClient({
|
||||
transport,
|
||||
});
|
||||
|
||||
console.log("Playwright MCP client initialized with Firefox");
|
||||
|
||||
return playwrightMCPClient;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get Playwright MCP tools in AI SDK format
|
||||
*/
|
||||
export async function getPlaywrightTools() {
|
||||
try {
|
||||
const client = await getPlaywrightMCPClient();
|
||||
const tools = await client.tools();
|
||||
console.log(`Loaded ${Object.keys(tools).length} Playwright MCP tools`);
|
||||
return tools;
|
||||
} catch (error) {
|
||||
console.error("Failed to initialize Playwright MCP tools:", error);
|
||||
// Return empty object to allow app to continue without Playwright
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Cleanup function to close the MCP client
|
||||
*/
|
||||
export async function closePlaywrightMCP() {
|
||||
if (playwrightMCPClient) {
|
||||
await playwrightMCPClient.close();
|
||||
playwrightMCPClient = null;
|
||||
console.log("Playwright MCP client closed");
|
||||
}
|
||||
}
|
||||
@@ -261,6 +261,14 @@ async function main() {
|
||||
// Transcribe audio using OpenAI Whisper
|
||||
const result = await transcribeAudio(audio, format);
|
||||
|
||||
// Check if transcription is empty or only whitespace
|
||||
const transcriptText = result.text.trim();
|
||||
if (!transcriptText) {
|
||||
console.log("[STT] Transcription is empty or silence, skipping LLM processing");
|
||||
// Return empty string without broadcasting or processing
|
||||
return "";
|
||||
}
|
||||
|
||||
// Broadcast transcription result as activity log
|
||||
wsServer.broadcastActivityLog({
|
||||
id: uuidv4(),
|
||||
|
||||
@@ -15,7 +15,8 @@ export interface ActivityLogEntry {
|
||||
|
||||
export interface WebSocketMessage {
|
||||
type: 'activity_log' | 'status' | 'ping' | 'pong' | 'user_message' | 'assistant_chunk'
|
||||
| 'audio_chunk' | 'audio_output' | 'recording_state' | 'transcription_result' | 'audio_played';
|
||||
| 'audio_chunk' | 'audio_output' | 'recording_state' | 'transcription_result' | 'audio_played'
|
||||
| 'artifact';
|
||||
payload: unknown;
|
||||
}
|
||||
|
||||
@@ -44,3 +45,11 @@ export interface AudioOutputPayload {
|
||||
export interface AudioPlayedPayload {
|
||||
id: string; // unique ID of the audio that finished playing
|
||||
}
|
||||
|
||||
export interface ArtifactPayload {
|
||||
type: 'markdown' | 'diff' | 'image' | 'code';
|
||||
id: string;
|
||||
title: string;
|
||||
content: string; // resolved content (base64 for file/command sources, plain text for text source)
|
||||
isBase64: boolean; // true if content is base64 encoded (from file or command_output)
|
||||
}
|
||||
|
||||
@@ -236,6 +236,27 @@
|
||||
font-size: 0.75rem;
|
||||
}
|
||||
|
||||
.log-entry.artifact {
|
||||
background-color: #1e3a5f;
|
||||
color: #93c5fd;
|
||||
border: 1px solid #3b82f6;
|
||||
}
|
||||
|
||||
.log-entry.artifact.clickable {
|
||||
cursor: pointer;
|
||||
transition: all 0.2s ease;
|
||||
}
|
||||
|
||||
.log-entry.artifact.clickable:hover {
|
||||
background-color: #2563eb;
|
||||
color: #fff;
|
||||
transform: translateX(4px);
|
||||
}
|
||||
|
||||
.log-entry.artifact.clickable:active {
|
||||
transform: translateX(2px);
|
||||
}
|
||||
|
||||
.log-entry.streaming {
|
||||
opacity: 0.8;
|
||||
}
|
||||
@@ -384,6 +405,65 @@
|
||||
animation: spin 0.8s linear infinite;
|
||||
}
|
||||
|
||||
/* Realtime mode button styles */
|
||||
.send-button.realtime-button {
|
||||
background-color: #8b5cf6;
|
||||
}
|
||||
|
||||
.send-button.realtime-button:hover:not(:disabled) {
|
||||
background-color: #7c3aed;
|
||||
}
|
||||
|
||||
.send-button.realtime-button.active {
|
||||
background-color: #10b981;
|
||||
animation: pulse-realtime 2s ease-in-out infinite;
|
||||
}
|
||||
|
||||
.send-button.realtime-button.active:hover:not(:disabled) {
|
||||
background-color: #059669;
|
||||
}
|
||||
|
||||
.send-button.realtime-button.speech-detected {
|
||||
background-color: #ef4444;
|
||||
animation: pulse-speech 0.8s ease-in-out infinite;
|
||||
}
|
||||
|
||||
.send-button.realtime-button.loading {
|
||||
background-color: #f59e0b;
|
||||
}
|
||||
|
||||
.send-button .loading-indicator {
|
||||
display: inline-block;
|
||||
width: 12px;
|
||||
height: 12px;
|
||||
border: 2px solid #fff;
|
||||
border-top-color: transparent;
|
||||
border-radius: 50%;
|
||||
animation: spin 0.8s linear infinite;
|
||||
}
|
||||
|
||||
@keyframes pulse-realtime {
|
||||
0%, 100% {
|
||||
opacity: 1;
|
||||
transform: scale(1);
|
||||
}
|
||||
50% {
|
||||
opacity: 0.7;
|
||||
transform: scale(1.05);
|
||||
}
|
||||
}
|
||||
|
||||
@keyframes pulse-speech {
|
||||
0%, 100% {
|
||||
opacity: 1;
|
||||
box-shadow: 0 0 0 0 rgba(239, 68, 68, 0.7);
|
||||
}
|
||||
50% {
|
||||
opacity: 0.9;
|
||||
box-shadow: 0 0 0 8px rgba(239, 68, 68, 0);
|
||||
}
|
||||
}
|
||||
|
||||
.test-controls {
|
||||
padding-top: 0.5rem;
|
||||
border-top: 1px solid #333;
|
||||
|
||||
@@ -1,9 +1,11 @@
|
||||
import { useState, useEffect, useRef } from "react";
|
||||
import { Mic, Send } from "lucide-react";
|
||||
import { Mic, Send, Radio } from "lucide-react";
|
||||
import { useWebSocket } from "./hooks/useWebSocket";
|
||||
import { createAudioPlayer } from "./lib/audio-playback";
|
||||
import { createAudioRecorder, type AudioRecorder } from "./lib/audio-capture";
|
||||
import { createRealtimeVAD, float32ArrayToBlob, type RealtimeVAD } from "./lib/audio-realtime";
|
||||
import { ToolCallCard } from "./components/ToolCallCard";
|
||||
import { ArtifactDrawer, type Artifact } from "./components/ArtifactDrawer";
|
||||
import "./App.css";
|
||||
|
||||
type LogEntry =
|
||||
@@ -23,6 +25,14 @@ type LogEntry =
|
||||
result?: any;
|
||||
error?: any;
|
||||
status: "executing" | "completed" | "failed";
|
||||
}
|
||||
| {
|
||||
type: "artifact";
|
||||
id: string;
|
||||
timestamp: number;
|
||||
artifactId: string;
|
||||
artifactType: string;
|
||||
title: string;
|
||||
};
|
||||
|
||||
function App() {
|
||||
@@ -39,9 +49,15 @@ function App() {
|
||||
const [isProcessingAudio, setIsProcessingAudio] = useState(false);
|
||||
const [isPlayingAudio, setIsPlayingAudio] = useState(false);
|
||||
const [isRecording, setIsRecording] = useState(false);
|
||||
const [isRealtimeMode, setIsRealtimeMode] = useState(false);
|
||||
const [isSpeechDetected, setIsSpeechDetected] = useState(false);
|
||||
const [isVADLoading, setIsVADLoading] = useState(false);
|
||||
const [currentArtifact, setCurrentArtifact] = useState<Artifact | null>(null);
|
||||
const [artifacts, setArtifacts] = useState<Map<string, Artifact>>(new Map());
|
||||
const logEndRef = useRef<HTMLDivElement>(null);
|
||||
const audioPlayerRef = useRef(createAudioPlayer());
|
||||
const recorderRef = useRef<AudioRecorder | null>(null);
|
||||
const vadRef = useRef<RealtimeVAD | null>(null);
|
||||
const textareaRef = useRef<HTMLTextAreaElement>(null);
|
||||
|
||||
// WebSocket URL - use ws://localhost:3000/ws in dev, or construct from current host in prod
|
||||
@@ -53,6 +69,13 @@ function App() {
|
||||
|
||||
useEffect(() => {
|
||||
recorderRef.current = createAudioRecorder();
|
||||
|
||||
// Cleanup VAD on unmount
|
||||
return () => {
|
||||
if (vadRef.current) {
|
||||
vadRef.current.destroy();
|
||||
}
|
||||
};
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
@@ -186,6 +209,34 @@ function App() {
|
||||
}
|
||||
);
|
||||
|
||||
// Listen for artifacts
|
||||
const unsubArtifact = ws.on("artifact", (payload: unknown) => {
|
||||
const artifact = payload as Artifact;
|
||||
|
||||
// Add to artifacts map
|
||||
setArtifacts((prev) => {
|
||||
const newMap = new Map(prev);
|
||||
newMap.set(artifact.id, artifact);
|
||||
return newMap;
|
||||
});
|
||||
|
||||
// Add as clickable log entry
|
||||
setLogs((prev) => [
|
||||
...prev,
|
||||
{
|
||||
type: "artifact" as const,
|
||||
id: Date.now().toString(),
|
||||
timestamp: Date.now(),
|
||||
artifactId: artifact.id,
|
||||
artifactType: artifact.type,
|
||||
title: artifact.title,
|
||||
},
|
||||
]);
|
||||
|
||||
// Show drawer by default
|
||||
setCurrentArtifact(artifact);
|
||||
});
|
||||
|
||||
// Listen for audio output (TTS)
|
||||
const unsubAudioOutput = ws.on("audio_output", async (payload: unknown) => {
|
||||
const data = payload as { audio: string; format: string; id: string };
|
||||
@@ -228,6 +279,7 @@ function App() {
|
||||
unsubActivity();
|
||||
unsubChunk();
|
||||
unsubTranscription();
|
||||
unsubArtifact();
|
||||
unsubAudioOutput();
|
||||
};
|
||||
}, [ws]);
|
||||
@@ -336,6 +388,116 @@ function App() {
|
||||
}
|
||||
};
|
||||
|
||||
const handleCloseArtifact = () => {
|
||||
setCurrentArtifact(null);
|
||||
};
|
||||
|
||||
const handleOpenArtifact = (artifactId: string) => {
|
||||
const artifact = artifacts.get(artifactId);
|
||||
if (artifact) {
|
||||
setCurrentArtifact(artifact);
|
||||
}
|
||||
};
|
||||
|
||||
const handleToggleRealtimeMode = async () => {
|
||||
if (!ws.isConnected) return;
|
||||
|
||||
try {
|
||||
if (isRealtimeMode) {
|
||||
// Stop realtime mode
|
||||
if (vadRef.current) {
|
||||
vadRef.current.pause();
|
||||
}
|
||||
setIsRealtimeMode(false);
|
||||
setIsSpeechDetected(false);
|
||||
addLog("info", "Realtime mode stopped");
|
||||
} else {
|
||||
// Start realtime mode
|
||||
setIsVADLoading(true);
|
||||
addLog("info", "Starting realtime mode...");
|
||||
|
||||
// Create VAD instance if it doesn't exist
|
||||
if (!vadRef.current) {
|
||||
vadRef.current = createRealtimeVAD({
|
||||
onModelLoading: () => {
|
||||
console.log("[App] VAD model loading...");
|
||||
setIsVADLoading(true);
|
||||
},
|
||||
onModelLoaded: () => {
|
||||
console.log("[App] VAD model loaded");
|
||||
setIsVADLoading(false);
|
||||
},
|
||||
onSpeechStart: () => {
|
||||
console.log("[App] Speech detected!");
|
||||
setIsSpeechDetected(true);
|
||||
|
||||
// Interrupt playback when speech is detected
|
||||
audioPlayerRef.current.stop();
|
||||
setIsPlayingAudio(false);
|
||||
setCurrentAssistantMessage("");
|
||||
},
|
||||
onSpeechEnd: async (audioData: Float32Array) => {
|
||||
console.log("[App] Speech ended, processing...");
|
||||
setIsSpeechDetected(false);
|
||||
setIsProcessingAudio(true);
|
||||
|
||||
try {
|
||||
// Convert Float32Array to audio blob
|
||||
const audioBlob = float32ArrayToBlob(audioData, 16000);
|
||||
console.log(`[App] Converted to blob: ${audioBlob.size} bytes`);
|
||||
|
||||
// Send to server
|
||||
const arrayBuffer = await audioBlob.arrayBuffer();
|
||||
const base64Audio = btoa(
|
||||
new Uint8Array(arrayBuffer).reduce(
|
||||
(data, byte) => data + String.fromCharCode(byte),
|
||||
""
|
||||
)
|
||||
);
|
||||
|
||||
ws.send({
|
||||
type: "audio_chunk",
|
||||
payload: {
|
||||
audio: base64Audio,
|
||||
format: audioBlob.type,
|
||||
isLast: true,
|
||||
},
|
||||
});
|
||||
|
||||
console.log(`[App] Sent realtime audio to server`);
|
||||
} catch (error: any) {
|
||||
console.error("[App] Failed to process VAD audio:", error);
|
||||
addLog("error", `Failed to process audio: ${error.message}`);
|
||||
setIsProcessingAudio(false);
|
||||
}
|
||||
},
|
||||
onVADMisfire: () => {
|
||||
console.log("[App] VAD misfire");
|
||||
setIsSpeechDetected(false);
|
||||
},
|
||||
onError: (error: Error) => {
|
||||
console.error("[App] VAD error:", error);
|
||||
addLog("error", `VAD error: ${error.message}`);
|
||||
setIsVADLoading(false);
|
||||
setIsRealtimeMode(false);
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
// Start VAD
|
||||
await vadRef.current.start();
|
||||
setIsRealtimeMode(true);
|
||||
setIsVADLoading(false);
|
||||
addLog("success", "Realtime mode started - speak anytime!");
|
||||
}
|
||||
} catch (error: any) {
|
||||
console.error("[App] Realtime mode error:", error);
|
||||
addLog("error", `Failed to toggle realtime mode: ${error.message}`);
|
||||
setIsRealtimeMode(false);
|
||||
setIsVADLoading(false);
|
||||
}
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="app">
|
||||
<header className="header">
|
||||
@@ -356,17 +518,42 @@ function App() {
|
||||
<div className="chat-interface">
|
||||
<div className="activity-log">
|
||||
<div className="log-entries">
|
||||
{logs.map((log) =>
|
||||
log.type === "tool_call" ? (
|
||||
<ToolCallCard
|
||||
key={log.id}
|
||||
toolName={log.toolName}
|
||||
args={log.args}
|
||||
result={log.result}
|
||||
error={log.error}
|
||||
status={log.status}
|
||||
/>
|
||||
) : (
|
||||
{logs.map((log) => {
|
||||
if (log.type === "tool_call") {
|
||||
return (
|
||||
<ToolCallCard
|
||||
key={log.id}
|
||||
toolName={log.toolName}
|
||||
args={log.args}
|
||||
result={log.result}
|
||||
error={log.error}
|
||||
status={log.status}
|
||||
/>
|
||||
);
|
||||
}
|
||||
|
||||
if (log.type === "artifact") {
|
||||
return (
|
||||
<div
|
||||
key={log.id}
|
||||
className="log-entry artifact clickable"
|
||||
onClick={() => handleOpenArtifact(log.artifactId)}
|
||||
role="button"
|
||||
tabIndex={0}
|
||||
onKeyDown={(e) => {
|
||||
if (e.key === "Enter" || e.key === " ") {
|
||||
handleOpenArtifact(log.artifactId);
|
||||
}
|
||||
}}
|
||||
>
|
||||
<span className="log-message">
|
||||
📋 {log.artifactType}: {log.title}
|
||||
</span>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
return (
|
||||
<div key={log.id} className={`log-entry ${log.type}`}>
|
||||
<span className="log-message">{log.message}</span>
|
||||
{log.metadata && (
|
||||
@@ -376,8 +563,8 @@ function App() {
|
||||
</details>
|
||||
)}
|
||||
</div>
|
||||
)
|
||||
)}
|
||||
);
|
||||
})}
|
||||
{currentAssistantMessage && (
|
||||
<div className="log-entry assistant streaming">
|
||||
<span className="log-message">{currentAssistantMessage}</span>
|
||||
@@ -395,14 +582,27 @@ function App() {
|
||||
onChange={(e) => setUserInput(e.target.value)}
|
||||
onKeyDown={handleKeyDown}
|
||||
placeholder="Type a message or tap mic to talk..."
|
||||
disabled={!ws.isConnected || isRecording}
|
||||
disabled={!ws.isConnected || isRecording || isRealtimeMode}
|
||||
className="message-input"
|
||||
rows={1}
|
||||
/>
|
||||
<button
|
||||
type="button"
|
||||
onClick={handleToggleRealtimeMode}
|
||||
disabled={!ws.isConnected || isRecording || isVADLoading}
|
||||
className={`send-button realtime-button ${isRealtimeMode ? 'active' : ''} ${isSpeechDetected ? 'speech-detected' : ''} ${isVADLoading ? 'loading' : ''}`}
|
||||
title={isRealtimeMode ? "Stop realtime mode" : "Start realtime mode"}
|
||||
>
|
||||
{isVADLoading ? (
|
||||
<span className="loading-indicator" />
|
||||
) : (
|
||||
<Radio size={20} />
|
||||
)}
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
onClick={handleButtonClick}
|
||||
disabled={!ws.isConnected || isProcessingAudio}
|
||||
disabled={!ws.isConnected || isProcessingAudio || isRealtimeMode}
|
||||
className={`send-button ${isRecording ? 'recording' : ''} ${isProcessingAudio ? 'processing' : ''} ${userInput.trim() ? 'has-text' : ''}`}
|
||||
>
|
||||
{isRecording ? (
|
||||
@@ -418,6 +618,11 @@ function App() {
|
||||
</div>
|
||||
</div>
|
||||
</main>
|
||||
|
||||
<ArtifactDrawer
|
||||
artifact={currentArtifact}
|
||||
onClose={handleCloseArtifact}
|
||||
/>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,268 @@
|
||||
.artifact-drawer-overlay {
|
||||
position: fixed;
|
||||
top: 0;
|
||||
left: 0;
|
||||
right: 0;
|
||||
bottom: 0;
|
||||
background-color: rgba(0, 0, 0, 0.5);
|
||||
z-index: 1000;
|
||||
animation: fadeIn 0.2s ease-out;
|
||||
}
|
||||
|
||||
@keyframes fadeIn {
|
||||
from {
|
||||
opacity: 0;
|
||||
}
|
||||
to {
|
||||
opacity: 1;
|
||||
}
|
||||
}
|
||||
|
||||
.artifact-drawer {
|
||||
position: fixed;
|
||||
bottom: 0;
|
||||
left: 0;
|
||||
right: 0;
|
||||
max-height: 80vh;
|
||||
background: #1a1a1a;
|
||||
border-top-left-radius: 12px;
|
||||
border-top-right-radius: 12px;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
animation: slideUp 0.3s ease-out;
|
||||
box-shadow: 0 -4px 20px rgba(0, 0, 0, 0.3);
|
||||
}
|
||||
|
||||
@keyframes slideUp {
|
||||
from {
|
||||
transform: translateY(100%);
|
||||
}
|
||||
to {
|
||||
transform: translateY(0);
|
||||
}
|
||||
}
|
||||
|
||||
.artifact-drawer-header {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
padding: 20px 24px;
|
||||
border-bottom: 1px solid #333;
|
||||
flex-shrink: 0;
|
||||
}
|
||||
|
||||
.artifact-drawer-title {
|
||||
margin: 0;
|
||||
font-size: 1.25rem;
|
||||
font-weight: 600;
|
||||
color: #fff;
|
||||
flex: 1;
|
||||
}
|
||||
|
||||
.artifact-drawer-header-right {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 12px;
|
||||
}
|
||||
|
||||
.artifact-drawer-type-badge {
|
||||
padding: 4px 12px;
|
||||
background-color: #2563eb;
|
||||
color: #fff;
|
||||
border-radius: 12px;
|
||||
font-size: 0.75rem;
|
||||
font-weight: 500;
|
||||
text-transform: uppercase;
|
||||
letter-spacing: 0.5px;
|
||||
}
|
||||
|
||||
.artifact-drawer-close-button {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
width: 32px;
|
||||
height: 32px;
|
||||
padding: 0;
|
||||
background: transparent;
|
||||
border: none;
|
||||
color: #9ca3af;
|
||||
cursor: pointer;
|
||||
border-radius: 6px;
|
||||
transition: all 0.2s ease;
|
||||
}
|
||||
|
||||
.artifact-drawer-close-button:hover {
|
||||
background-color: #2a2a2a;
|
||||
color: #fff;
|
||||
}
|
||||
|
||||
.artifact-drawer-content {
|
||||
flex: 1;
|
||||
overflow-y: auto;
|
||||
padding: 24px;
|
||||
min-height: 200px;
|
||||
}
|
||||
|
||||
.artifact-plan {
|
||||
color: #e5e5e5;
|
||||
line-height: 1.7;
|
||||
}
|
||||
|
||||
/* Markdown styles */
|
||||
.artifact-plan h1 {
|
||||
font-size: 1.75rem;
|
||||
font-weight: 700;
|
||||
margin-top: 0;
|
||||
margin-bottom: 1rem;
|
||||
color: #fff;
|
||||
}
|
||||
|
||||
.artifact-plan h2 {
|
||||
font-size: 1.5rem;
|
||||
font-weight: 600;
|
||||
margin-top: 1.5rem;
|
||||
margin-bottom: 0.75rem;
|
||||
color: #fff;
|
||||
}
|
||||
|
||||
.artifact-plan h3 {
|
||||
font-size: 1.25rem;
|
||||
font-weight: 600;
|
||||
margin-top: 1.25rem;
|
||||
margin-bottom: 0.5rem;
|
||||
color: #fff;
|
||||
}
|
||||
|
||||
.artifact-plan h4,
|
||||
.artifact-plan h5,
|
||||
.artifact-plan h6 {
|
||||
font-size: 1rem;
|
||||
font-weight: 600;
|
||||
margin-top: 1rem;
|
||||
margin-bottom: 0.5rem;
|
||||
color: #fff;
|
||||
}
|
||||
|
||||
.artifact-plan p {
|
||||
margin-bottom: 1rem;
|
||||
}
|
||||
|
||||
.artifact-plan ul,
|
||||
.artifact-plan ol {
|
||||
margin-bottom: 1rem;
|
||||
padding-left: 1.5rem;
|
||||
}
|
||||
|
||||
.artifact-plan li {
|
||||
margin-bottom: 0.5rem;
|
||||
}
|
||||
|
||||
.artifact-plan code {
|
||||
background-color: #2a2a2a;
|
||||
padding: 2px 6px;
|
||||
border-radius: 4px;
|
||||
font-family: "Monaco", "Courier New", monospace;
|
||||
font-size: 0.9em;
|
||||
color: #fff;
|
||||
}
|
||||
|
||||
.artifact-plan pre {
|
||||
background-color: #2a2a2a;
|
||||
padding: 12px;
|
||||
border-radius: 6px;
|
||||
overflow-x: auto;
|
||||
margin-bottom: 1rem;
|
||||
}
|
||||
|
||||
.artifact-plan pre code {
|
||||
background-color: transparent;
|
||||
padding: 0;
|
||||
}
|
||||
|
||||
.artifact-plan blockquote {
|
||||
border-left: 4px solid #2563eb;
|
||||
padding-left: 1rem;
|
||||
margin-left: 0;
|
||||
margin-bottom: 1rem;
|
||||
color: #a3a3a3;
|
||||
font-style: italic;
|
||||
}
|
||||
|
||||
.artifact-plan a {
|
||||
color: #3b82f6;
|
||||
text-decoration: none;
|
||||
}
|
||||
|
||||
.artifact-plan a:hover {
|
||||
text-decoration: underline;
|
||||
}
|
||||
|
||||
/* Diff content styles */
|
||||
.artifact-diff {
|
||||
background-color: #2a2a2a;
|
||||
padding: 16px;
|
||||
border-radius: 6px;
|
||||
overflow-x: auto;
|
||||
color: #e5e5e5;
|
||||
font-family: "Monaco", "Courier New", monospace;
|
||||
font-size: 0.9em;
|
||||
line-height: 1.6;
|
||||
white-space: pre-wrap;
|
||||
word-break: break-word;
|
||||
}
|
||||
|
||||
.artifact-diff code {
|
||||
background-color: transparent;
|
||||
padding: 0;
|
||||
}
|
||||
|
||||
/* Code content styles */
|
||||
.artifact-code {
|
||||
background-color: #2a2a2a;
|
||||
padding: 16px;
|
||||
border-radius: 6px;
|
||||
overflow-x: auto;
|
||||
color: #e5e5e5;
|
||||
font-family: "Monaco", "Courier New", monospace;
|
||||
font-size: 0.9em;
|
||||
line-height: 1.6;
|
||||
}
|
||||
|
||||
.artifact-code code {
|
||||
background-color: transparent;
|
||||
padding: 0;
|
||||
}
|
||||
|
||||
/* Image content styles */
|
||||
.artifact-image {
|
||||
display: flex;
|
||||
justify-content: center;
|
||||
align-items: center;
|
||||
padding: 16px;
|
||||
}
|
||||
|
||||
.artifact-image img {
|
||||
max-width: 100%;
|
||||
max-height: 70vh;
|
||||
border-radius: 8px;
|
||||
box-shadow: 0 4px 12px rgba(0, 0, 0, 0.3);
|
||||
}
|
||||
|
||||
/* Mobile responsiveness */
|
||||
@media (max-width: 768px) {
|
||||
.artifact-drawer {
|
||||
max-height: 90vh;
|
||||
}
|
||||
|
||||
.artifact-drawer-header {
|
||||
padding: 16px 20px;
|
||||
}
|
||||
|
||||
.artifact-drawer-content {
|
||||
padding: 20px;
|
||||
}
|
||||
|
||||
.artifact-drawer-title {
|
||||
font-size: 1.1rem;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,91 @@
|
||||
import { useEffect, useRef } from "react";
|
||||
import ReactMarkdown from "react-markdown";
|
||||
import { X } from "lucide-react";
|
||||
import "./ArtifactDrawer.css";
|
||||
|
||||
export type Artifact = {
|
||||
type: "markdown" | "diff" | "image" | "code";
|
||||
id: string;
|
||||
title: string;
|
||||
content: string;
|
||||
isBase64: boolean;
|
||||
};
|
||||
|
||||
interface ArtifactDrawerProps {
|
||||
artifact: Artifact | null;
|
||||
onClose: () => void;
|
||||
}
|
||||
|
||||
export function ArtifactDrawer({ artifact, onClose }: ArtifactDrawerProps) {
|
||||
const drawerRef = useRef<HTMLDivElement>(null);
|
||||
|
||||
useEffect(() => {
|
||||
if (artifact) {
|
||||
// Prevent body scroll when drawer is open
|
||||
document.body.style.overflow = "hidden";
|
||||
} else {
|
||||
document.body.style.overflow = "";
|
||||
}
|
||||
|
||||
return () => {
|
||||
document.body.style.overflow = "";
|
||||
};
|
||||
}, [artifact]);
|
||||
|
||||
if (!artifact) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="artifact-drawer-overlay" onClick={onClose}>
|
||||
<div ref={drawerRef} className="artifact-drawer" onClick={(e) => e.stopPropagation()}>
|
||||
<div className="artifact-drawer-header">
|
||||
<h2 className="artifact-drawer-title">
|
||||
{artifact.title}
|
||||
</h2>
|
||||
<div className="artifact-drawer-header-right">
|
||||
<div className="artifact-drawer-type-badge">
|
||||
{artifact.type}
|
||||
</div>
|
||||
<button
|
||||
className="artifact-drawer-close-button"
|
||||
onClick={onClose}
|
||||
type="button"
|
||||
title="Close"
|
||||
>
|
||||
<X size={20} />
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="artifact-drawer-content">
|
||||
{artifact.type === "markdown" && (
|
||||
<div className="artifact-plan">
|
||||
<ReactMarkdown>
|
||||
{artifact.isBase64 ? atob(artifact.content) : artifact.content}
|
||||
</ReactMarkdown>
|
||||
</div>
|
||||
)}
|
||||
{artifact.type === "diff" && (
|
||||
<pre className="artifact-diff">
|
||||
<code>{artifact.isBase64 ? atob(artifact.content) : artifact.content}</code>
|
||||
</pre>
|
||||
)}
|
||||
{artifact.type === "code" && (
|
||||
<pre className="artifact-code">
|
||||
<code>{artifact.isBase64 ? atob(artifact.content) : artifact.content}</code>
|
||||
</pre>
|
||||
)}
|
||||
{artifact.type === "image" && (
|
||||
<div className="artifact-image">
|
||||
<img
|
||||
src={`data:image/png;base64,${artifact.content}`}
|
||||
alt={artifact.title}
|
||||
/>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,176 @@
|
||||
import { MicVAD } from "@ricky0123/vad-web";
|
||||
|
||||
export interface RealtimeVADConfig {
|
||||
onSpeechStart?: () => void;
|
||||
onSpeechEnd?: (audio: Float32Array) => void;
|
||||
onVADMisfire?: () => void;
|
||||
onModelLoading?: () => void;
|
||||
onModelLoaded?: () => void;
|
||||
onError?: (error: Error) => void;
|
||||
positiveSpeechThreshold?: number;
|
||||
negativeSpeechThreshold?: number;
|
||||
redemptionFrames?: number;
|
||||
frameSamples?: number;
|
||||
preSpeechPadFrames?: number;
|
||||
minSpeechFrames?: number;
|
||||
}
|
||||
|
||||
export interface RealtimeVAD {
|
||||
start(): Promise<void>;
|
||||
pause(): void;
|
||||
destroy(): void;
|
||||
isListening(): boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a realtime VAD (Voice Activity Detection) instance using @ricky0123/vad-web
|
||||
* Continuously listens for speech and invokes callbacks when speech is detected
|
||||
*/
|
||||
export function createRealtimeVAD(config: RealtimeVADConfig): RealtimeVAD {
|
||||
let vad: Awaited<ReturnType<typeof MicVAD.new>> | null = null;
|
||||
let isActive = false;
|
||||
|
||||
async function start(): Promise<void> {
|
||||
// If VAD already exists and is paused, just resume it
|
||||
if (vad && !isActive) {
|
||||
vad.start();
|
||||
isActive = true;
|
||||
console.log("[RealtimeVAD] VAD resumed");
|
||||
return;
|
||||
}
|
||||
|
||||
if (isActive) {
|
||||
throw new Error("VAD is already active");
|
||||
}
|
||||
|
||||
// Create new VAD instance
|
||||
try {
|
||||
config.onModelLoading?.();
|
||||
|
||||
vad = await MicVAD.new({
|
||||
onSpeechStart: () => {
|
||||
console.log("[RealtimeVAD] Speech start detected");
|
||||
config.onSpeechStart?.();
|
||||
},
|
||||
onSpeechEnd: (audio: Float32Array) => {
|
||||
console.log("[RealtimeVAD] Speech end detected", {
|
||||
samples: audio.length,
|
||||
durationSeconds: audio.length / 16000,
|
||||
});
|
||||
config.onSpeechEnd?.(audio);
|
||||
},
|
||||
onVADMisfire: () => {
|
||||
console.log("[RealtimeVAD] VAD misfire (false positive)");
|
||||
config.onVADMisfire?.();
|
||||
},
|
||||
positiveSpeechThreshold: config.positiveSpeechThreshold ?? 0.5,
|
||||
negativeSpeechThreshold: config.negativeSpeechThreshold ?? 0.35,
|
||||
redemptionMs: config.redemptionFrames ? config.redemptionFrames * 16 : 128,
|
||||
preSpeechPadMs: config.preSpeechPadFrames ? config.preSpeechPadFrames * 16 : 16,
|
||||
minSpeechMs: config.minSpeechFrames ? config.minSpeechFrames * 16 : 48,
|
||||
});
|
||||
|
||||
isActive = true;
|
||||
config.onModelLoaded?.();
|
||||
console.log("[RealtimeVAD] VAD created and started successfully");
|
||||
} catch (error) {
|
||||
console.error("[RealtimeVAD] Failed to start VAD:", error);
|
||||
const err =
|
||||
error instanceof Error ? error : new Error(String(error));
|
||||
config.onError?.(err);
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
function pause(): void {
|
||||
if (vad && isActive) {
|
||||
vad.pause();
|
||||
isActive = false;
|
||||
console.log("[RealtimeVAD] VAD paused");
|
||||
}
|
||||
}
|
||||
|
||||
function destroy(): void {
|
||||
if (vad) {
|
||||
vad.destroy();
|
||||
vad = null;
|
||||
isActive = false;
|
||||
console.log("[RealtimeVAD] VAD destroyed");
|
||||
}
|
||||
}
|
||||
|
||||
function isListening(): boolean {
|
||||
return isActive;
|
||||
}
|
||||
|
||||
return {
|
||||
start,
|
||||
pause,
|
||||
destroy,
|
||||
isListening,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert Float32Array PCM audio to audio blob
|
||||
* VAD returns Float32Array at 16kHz sample rate
|
||||
*/
|
||||
export function float32ArrayToBlob(
|
||||
audio: Float32Array,
|
||||
sampleRate: number = 16000
|
||||
): Blob {
|
||||
// Convert Float32Array to 16-bit PCM
|
||||
const pcm = new Int16Array(audio.length);
|
||||
for (let i = 0; i < audio.length; i++) {
|
||||
const s = Math.max(-1, Math.min(1, audio[i]));
|
||||
pcm[i] = s < 0 ? s * 0x8000 : s * 0x7fff;
|
||||
}
|
||||
|
||||
// Create WAV header
|
||||
const wavHeader = createWavHeader(pcm.length * 2, sampleRate);
|
||||
|
||||
// Combine header and data
|
||||
const wavBytes = new Uint8Array(wavHeader.length + pcm.length * 2);
|
||||
wavBytes.set(wavHeader, 0);
|
||||
wavBytes.set(new Uint8Array(pcm.buffer), wavHeader.length);
|
||||
|
||||
return new Blob([wavBytes], { type: "audio/wav" });
|
||||
}
|
||||
|
||||
/**
|
||||
* Create WAV file header
|
||||
*/
|
||||
function createWavHeader(dataLength: number, sampleRate: number): Uint8Array {
|
||||
const header = new ArrayBuffer(44);
|
||||
const view = new DataView(header);
|
||||
|
||||
// "RIFF" chunk descriptor
|
||||
writeString(view, 0, "RIFF");
|
||||
view.setUint32(4, 36 + dataLength, true);
|
||||
writeString(view, 8, "WAVE");
|
||||
|
||||
// "fmt " sub-chunk
|
||||
writeString(view, 12, "fmt ");
|
||||
view.setUint32(16, 16, true); // SubChunk1Size (16 for PCM)
|
||||
view.setUint16(20, 1, true); // AudioFormat (1 for PCM)
|
||||
view.setUint16(22, 1, true); // NumChannels (1 = mono)
|
||||
view.setUint32(24, sampleRate, true); // SampleRate
|
||||
view.setUint32(28, sampleRate * 2, true); // ByteRate
|
||||
view.setUint16(32, 2, true); // BlockAlign
|
||||
view.setUint16(34, 16, true); // BitsPerSample
|
||||
|
||||
// "data" sub-chunk
|
||||
writeString(view, 36, "data");
|
||||
view.setUint32(40, dataLength, true);
|
||||
|
||||
return new Uint8Array(header);
|
||||
}
|
||||
|
||||
/**
|
||||
* Write string to DataView
|
||||
*/
|
||||
function writeString(view: DataView, offset: number, str: string): void {
|
||||
for (let i = 0; i < str.length; i++) {
|
||||
view.setUint8(offset + i, str.charCodeAt(i));
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user