diff --git a/packages/voice-assistant/.claude/settings.local.json b/packages/voice-assistant/.claude/settings.local.json
index ac0af754f..ff0198ec9 100644
--- a/packages/voice-assistant/.claude/settings.local.json
+++ b/packages/voice-assistant/.claude/settings.local.json
@@ -3,7 +3,8 @@
"allow": [
"WebFetch(domain:vite.dev)",
"Read(//Users/moboudra/dev/voice-dev/packages/web/app/hooks/**)",
- "Read(//Users/moboudra/dev/voice-dev/packages/web/app/**)"
+ "Read(//Users/moboudra/dev/voice-dev/packages/web/app/**)",
+ "Read(//Users/moboudra/dev/voice-dev/**)"
],
"deny": [],
"ask": []
diff --git a/packages/voice-assistant/.playwright-mcp/after-fix-verification.png b/packages/voice-assistant/.playwright-mcp/after-fix-verification.png
new file mode 100644
index 000000000..891eb9e8a
Binary files /dev/null and b/packages/voice-assistant/.playwright-mcp/after-fix-verification.png differ
diff --git a/packages/voice-assistant/.playwright-mcp/iphone-se-initial.png b/packages/voice-assistant/.playwright-mcp/iphone-se-initial.png
new file mode 100644
index 000000000..cbea64985
Binary files /dev/null and b/packages/voice-assistant/.playwright-mcp/iphone-se-initial.png differ
diff --git a/packages/voice-assistant/.playwright-mcp/iphone-se-long-text.png b/packages/voice-assistant/.playwright-mcp/iphone-se-long-text.png
new file mode 100644
index 000000000..1433756d6
Binary files /dev/null and b/packages/voice-assistant/.playwright-mcp/iphone-se-long-text.png differ
diff --git a/packages/voice-assistant/.playwright-mcp/iphone-se-recording.png b/packages/voice-assistant/.playwright-mcp/iphone-se-recording.png
new file mode 100644
index 000000000..be0446d25
Binary files /dev/null and b/packages/voice-assistant/.playwright-mcp/iphone-se-recording.png differ
diff --git a/packages/voice-assistant/.playwright-mcp/mobile-layout-iphone12.png b/packages/voice-assistant/.playwright-mcp/mobile-layout-iphone12.png
new file mode 100644
index 000000000..b370ad199
Binary files /dev/null and b/packages/voice-assistant/.playwright-mcp/mobile-layout-iphone12.png differ
diff --git a/packages/voice-assistant/.playwright-mcp/mobile-long-text-input.png b/packages/voice-assistant/.playwright-mcp/mobile-long-text-input.png
new file mode 100644
index 000000000..1b888568b
Binary files /dev/null and b/packages/voice-assistant/.playwright-mcp/mobile-long-text-input.png differ
diff --git a/packages/voice-assistant/.playwright-mcp/mobile-recording-state.png b/packages/voice-assistant/.playwright-mcp/mobile-recording-state.png
new file mode 100644
index 000000000..9922f693d
Binary files /dev/null and b/packages/voice-assistant/.playwright-mcp/mobile-recording-state.png differ
diff --git a/packages/voice-assistant/.playwright-mcp/narrow-viewport-273-fixed.png b/packages/voice-assistant/.playwright-mcp/narrow-viewport-273-fixed.png
new file mode 100644
index 000000000..70ef52da1
Binary files /dev/null and b/packages/voice-assistant/.playwright-mcp/narrow-viewport-273-fixed.png differ
diff --git a/packages/voice-assistant/.playwright-mcp/narrow-viewport-273.png b/packages/voice-assistant/.playwright-mcp/narrow-viewport-273.png
new file mode 100644
index 000000000..70ef52da1
Binary files /dev/null and b/packages/voice-assistant/.playwright-mcp/narrow-viewport-273.png differ
diff --git a/packages/voice-assistant/agent-prompt.md b/packages/voice-assistant/agent-prompt.md
index 33ca329a9..cbc4f843c 100644
--- a/packages/voice-assistant/agent-prompt.md
+++ b/packages/voice-assistant/agent-prompt.md
@@ -269,7 +269,7 @@ You interact with the user's machine through **terminals** (isolated shell envir
- **create-terminal(name, workingDirectory, initialCommand?)** - Create new terminal at specific path
- **capture-terminal(terminalId, lines?, wait?)** - Get terminal output
- **send-text(terminalId, text, pressEnter?, return_output?)** - Type text/run commands
-- **send-keys(terminalId, keys, repeat?, return_output?)** - Send special keys (Escape, C-c, BTab, etc.)
+- **send-keys(terminalId, keys, repeat?, return_output?)** - Send special keys (Escape, C-c, BTab, Enter, etc.)
- **rename-terminal(terminalId, name)** - Rename a terminal
- **kill-terminal(terminalId)** - Close a terminal
@@ -291,12 +291,23 @@ You: create-terminal(name="faro", workingDirectory="~/dev/faro/main")
**Default only when no context:**
User: "Create a terminal"
-You: create-terminal(name="shell", workingDirectory="~") # Last resort!
+You: List terminals if you haven't already to see what are they working on
+You: create-terminal(name="shell", workingDirectory="WHATEVER THE CURRENT TERMINAL IS WORKING ON")
**With initial command:**
-User: "Create a terminal and run npm install"
+User: "run npm install in a new terminal"
You: create-terminal(name="install", workingDirectory="~/dev/project", initialCommand="npm install")
+**Launch a new Claude**
+User: "launch Claude to work on authentication"
+You: **make sure you have listed terminals to see what are they working on**
+You: create-terminal(name="authentication", workingDirectory="WHATEVER THE CURRENT TERMINAL IS WORKING ON", initialCommand="claude")
+
+**Launch a new Claude in plan mode with a prompt**
+User: "launch Claude in plan mode and ask it to to plan feature X"
+You: **make sure you have listed terminals to see what are they working on**
+You: create-terminal(name="authentication", workingDirectory="WHATEVER THE CURRENT TERMINAL IS WORKING ON", initialCommand="claude --permission-mode plan 'add feature X'")
+
### Terminal Context Tracking
**Keep track of:**
@@ -317,7 +328,7 @@ You: create-terminal(name="install", workingDirectory="~/dev/project", initialCo
### What is Claude Code?
Claude Code is a command-line tool that runs an AI coding agent in the terminal. The user launches it with:
-`claude --dangerously-skip-permissions`
+`claude`
### Vim Mode Input System
@@ -348,7 +359,7 @@ Claude Code cycles through **4 permission modes** with **shift+tab** (BTab):
**Starting Claude Code:**
1. create-terminal or use existing terminal
-2. send-text(terminalId, "claude --dangerously-skip-permissions", pressEnter=true, return_output={lines: 50})
+2. send-text(terminalId, "claude", pressEnter=true, return_output={lines: 50})
3. Wait for Claude Code interface to appear
**Asking Claude Code a question:**
@@ -379,7 +390,7 @@ User: "No"
You: create-terminal(
name="faro",
workingDirectory="~/dev/faro/main",
- initialCommand="claude --dangerously-skip-permissions"
+ initialCommand="claude"
)
You: "Claude launched in faro."
```
@@ -387,13 +398,13 @@ You: "Claude launched in faro."
**With plan mode:**
```
-initialCommand="claude --dangerously-skip-permissions --permission-mode plan"
+initialCommand="claude --permission-mode plan"
```
**With initial prompt:**
```
-initialCommand='claude --dangerously-skip-permissions "add dark mode toggle"'
+initialCommand='claude "add dark mode toggle"'
```
#### Pattern 2: Launch with Worktree
@@ -439,7 +450,7 @@ You: send-text(
Step 5: Launch Claude
You: send-text(
terminalName="fix-auth",
- text="claude --dangerously-skip-permissions",
+ text="claude",
pressEnter=true
)
@@ -466,16 +477,16 @@ You: "Claude launched in fix-auth worktree."
```bash
# Basic
-claude --dangerously-skip-permissions
+claude
# Plan mode
-claude --dangerously-skip-permissions --permission-mode plan
+claude --permission-mode plan
# With prompt
-claude --dangerously-skip-permissions "help me refactor the auth code"
+claude "help me refactor the auth code"
# Plan mode + prompt
-claude --dangerously-skip-permissions --permission-mode plan "add CSV export feature"
+claude --permission-mode plan "add CSV export feature"
```
## Git Worktree Utilities
diff --git a/packages/voice-assistant/src/server/agent/orchestrator.ts b/packages/voice-assistant/src/server/agent/orchestrator.ts
index f0e2f4c3e..ecdb1e653 100644
--- a/packages/voice-assistant/src/server/agent/orchestrator.ts
+++ b/packages/voice-assistant/src/server/agent/orchestrator.ts
@@ -133,6 +133,33 @@ export async function processUserMessage(params: {
});
}
},
+ onToolError: async (toolCallId, toolName, error) => {
+ // Broadcast tool error to WebSocket
+ if (params.wsServer) {
+ params.wsServer.broadcastActivityLog({
+ id: toolCallId,
+ timestamp: new Date(),
+ type: "error",
+ content: `Tool ${toolName} failed: ${
+ error instanceof Error ? error.message : String(error)
+ }`,
+ metadata: { toolCallId, toolName, error },
+ });
+ }
+ },
+ onError: async (error) => {
+ // Broadcast general stream error to WebSocket
+ if (params.wsServer) {
+ params.wsServer.broadcastActivityLog({
+ id: uuidv4(),
+ timestamp: new Date(),
+ type: "error",
+ content: `Stream error: ${
+ error instanceof Error ? error.message : String(error)
+ }`,
+ });
+ }
+ },
onFinish: async () => {
// Wait for any final pending TTS to complete
if (pendingTTS) {
diff --git a/packages/voice-assistant/src/server/daemon/tool-definitions.ts b/packages/voice-assistant/src/server/daemon/tool-definitions.ts
deleted file mode 100644
index 055bf23ba..000000000
--- a/packages/voice-assistant/src/server/daemon/tool-definitions.ts
+++ /dev/null
@@ -1,171 +0,0 @@
-/**
- * OpenAI function calling schemas for terminal operations
- * These tools are exposed to the LLM for terminal control
- */
-
-export const terminalTools = [
- {
- type: "function",
- function: {
- name: "list_terminals",
- description:
- "List all available terminals in the voice-dev session. Returns terminal ID, name, working directory, and current command for each terminal.",
- parameters: {
- type: "object",
- properties: {},
- required: [],
- },
- },
- },
- {
- type: "function",
- function: {
- name: "create_terminal",
- description:
- "Create a new terminal (tmux window) with a specific name and working directory. Optionally execute an initial command after creation. Terminal names must be unique.",
- parameters: {
- type: "object",
- properties: {
- name: {
- type: "string",
- description:
- "Unique name for the terminal. Should be descriptive of what the terminal is used for (e.g., 'web-dev', 'api-server', 'tests').",
- },
- workingDirectory: {
- type: "string",
- description:
- "Absolute path to the working directory for this terminal. Can use ~ for home directory.",
- },
- initialCommand: {
- type: "string",
- description:
- "Optional command to execute after creating the terminal (e.g., 'npm run dev', 'python -m venv venv').",
- },
- },
- required: ["name", "workingDirectory"],
- },
- },
- },
- {
- type: "function",
- function: {
- name: "capture_terminal",
- description:
- "Capture and return the output from a terminal. Returns the last N lines of terminal content. Useful for checking command results or monitoring running processes.",
- parameters: {
- type: "object",
- properties: {
- terminalId: {
- type: "string",
- description:
- "The terminal ID (window ID format: @123) to capture output from. Get this from list_terminals.",
- },
- lines: {
- type: "number",
- description:
- "Number of lines to capture from the terminal history. Defaults to 200.",
- },
- wait: {
- type: "number",
- description:
- "Optional milliseconds to wait before capturing output. Useful when you just sent a command and want to wait for it to complete.",
- },
- },
- required: ["terminalId"],
- },
- },
- },
- {
- type: "function",
- function: {
- name: "send_text",
- description:
- "Type text into a terminal. Can optionally press Enter to execute the text as a command. Can optionally return the output after execution. This is the primary way to run commands in terminals.",
- parameters: {
- type: "object",
- properties: {
- terminalId: {
- type: "string",
- description:
- "The terminal ID (window ID format: @123) to send text to. Get this from list_terminals.",
- },
- text: {
- type: "string",
- description:
- "The text to type into the terminal. Can be a command, input for a running program, or any text.",
- },
- pressEnter: {
- type: "boolean",
- description:
- "Whether to press Enter after typing the text. Set to true to execute commands. Defaults to false.",
- },
- return_output: {
- type: "object",
- description:
- "Optional. If provided, will wait for terminal activity to settle and return the output.",
- properties: {
- lines: {
- type: "number",
- description: "Number of lines to capture. Defaults to 200.",
- },
- waitForSettled: {
- type: "boolean",
- description:
- "Whether to wait for terminal activity to settle before returning output. Polls terminal output and waits 500ms after last change. Defaults to true.",
- },
- maxWait: {
- type: "number",
- description:
- "Maximum milliseconds to wait for activity to settle. Defaults to 120000 (2 minutes).",
- },
- },
- },
- },
- required: ["terminalId", "text"],
- },
- },
- },
- {
- type: "function",
- function: {
- name: "rename_terminal",
- description:
- "Rename an existing terminal. The new name must be unique among all terminals.",
- parameters: {
- type: "object",
- properties: {
- terminalId: {
- type: "string",
- description:
- "The terminal ID (window ID format: @123) to rename. Get this from list_terminals.",
- },
- newName: {
- type: "string",
- description:
- "The new unique name for the terminal. Should be descriptive of the terminal's purpose.",
- },
- },
- required: ["terminalId", "newName"],
- },
- },
- },
- {
- type: "function",
- function: {
- name: "kill_terminal",
- description:
- "Close and destroy a terminal. This will terminate any running processes in the terminal. Use with caution.",
- parameters: {
- type: "object",
- properties: {
- terminalId: {
- type: "string",
- description:
- "The terminal ID (window ID format: @123) to kill. Get this from list_terminals.",
- },
- },
- required: ["terminalId"],
- },
- },
- },
-] as const;
diff --git a/packages/voice-assistant/src/ui/App.css b/packages/voice-assistant/src/ui/App.css
index 70143500a..2013735b6 100644
--- a/packages/voice-assistant/src/ui/App.css
+++ b/packages/voice-assistant/src/ui/App.css
@@ -520,6 +520,10 @@
border-color: #22c55e;
}
+.tool-call-card[data-status="failed"] {
+ border-color: #ef4444;
+}
+
.tool-call-header {
width: 100%;
display: flex;
@@ -575,6 +579,11 @@
color: #86efac;
}
+.tool-call-card[data-status="failed"] .tool-call-status {
+ background-color: rgba(239, 68, 68, 0.2);
+ color: #fca5a5;
+}
+
.status-icon {
flex-shrink: 0;
}
diff --git a/packages/voice-assistant/src/ui/App.tsx b/packages/voice-assistant/src/ui/App.tsx
index 14c3cb3f2..c97bc7354 100644
--- a/packages/voice-assistant/src/ui/App.tsx
+++ b/packages/voice-assistant/src/ui/App.tsx
@@ -21,7 +21,8 @@ type LogEntry =
toolName: string;
args: any;
result?: any;
- status: "executing" | "completed";
+ error?: any;
+ status: "executing" | "completed" | "failed";
};
function App() {
@@ -138,6 +139,23 @@ function App() {
return;
}
+ // Handle tool errors - update tool call status to failed
+ if (data.type === "error" && data.metadata?.toolCallId) {
+ const { toolCallId, error } = data.metadata as {
+ toolCallId: string;
+ error: any;
+ };
+
+ setLogs((prev) =>
+ prev.map((log) =>
+ log.type === "tool_call" && log.id === toolCallId
+ ? { ...log, error, status: "failed" as const }
+ : log
+ )
+ );
+ // Don't return - still add the error message to the log
+ }
+
// Map activity log types to UI log types
let logType: "system" | "info" | "success" | "error" | "user" | "assistant" = "info";
if (data.type === "transcript") logType = "user";
@@ -336,6 +354,7 @@ function App() {
toolName={log.toolName}
args={log.args}
result={log.result}
+ error={log.error}
status={log.status}
/>
) : (
diff --git a/packages/voice-assistant/src/ui/components/ToolCallCard.tsx b/packages/voice-assistant/src/ui/components/ToolCallCard.tsx
index 63d3fa9ce..5bc75d4a7 100644
--- a/packages/voice-assistant/src/ui/components/ToolCallCard.tsx
+++ b/packages/voice-assistant/src/ui/components/ToolCallCard.tsx
@@ -1,17 +1,19 @@
import { useState } from "react";
-import { ChevronDown, ChevronRight, CheckCircle, Loader2 } from "lucide-react";
+import { ChevronDown, ChevronRight, CheckCircle, Loader2, XCircle } from "lucide-react";
export interface ToolCallCardProps {
toolName: string;
args: any;
result?: any;
- status: "executing" | "completed";
+ error?: any;
+ status: "executing" | "completed" | "failed";
}
export function ToolCallCard({
toolName,
args,
result,
+ error,
status,
}: ToolCallCardProps) {
const [isExpanded, setIsExpanded] = useState(false);
@@ -34,6 +36,8 @@ export function ToolCallCard({