Normalize Codex paseo_voice.speak MCP calls and wrap spoken input

Broaden the Codex speak tool name matcher to recognize paseo_voice.speak
MCP calls alongside the existing paseo.speak convention. Wrap voice
transcription input in <spoken-input> tags so agents can distinguish
spoken from typed messages.
This commit is contained in:
Mohamed Boudra
2026-04-07 18:57:03 +07:00
parent 76c6253ae0
commit 0defbc1dc3
4 changed files with 38 additions and 5 deletions
@@ -539,6 +539,26 @@ describe("codex tool-call mapper", () => {
}); });
}); });
it("normalizes codex paseo_voice.speak mcp calls and extracts spoken text", () => {
const item = mapCodexToolCallFromThreadItem({
type: "mcpToolCall",
id: "codex-speak-thread-2",
status: "completed",
server: "paseo_voice",
tool: "speak",
arguments: { text: "Voice response from Codex via paseo_voice." },
result: { ok: true },
});
expect(item).toBeTruthy();
expect(item?.name).toBe("speak");
expect(item?.detail).toEqual({
type: "unknown",
input: "Voice response from Codex via paseo_voice.",
output: null,
});
});
it("normalizes codex paseo speak rollout names and extracts spoken text", () => { it("normalizes codex paseo speak rollout names and extracts spoken text", () => {
const item = expectMapped( const item = expectMapped(
mapCodexRolloutToolCall({ mapCodexRolloutToolCall({
@@ -3,6 +3,7 @@ import { z } from "zod";
import type { ToolCallTimelineItem } from "../../agent-sdk-types.js"; import type { ToolCallTimelineItem } from "../../agent-sdk-types.js";
import { extractCodexShellOutput, truncateDiffText } from "../tool-call-mapper-utils.js"; import { extractCodexShellOutput, truncateDiffText } from "../tool-call-mapper-utils.js";
import { deriveCodexToolDetail, normalizeCodexFilePath } from "./tool-call-detail-parser.js"; import { deriveCodexToolDetail, normalizeCodexFilePath } from "./tool-call-detail-parser.js";
import { isSpeakToolName } from "../../tool-name-normalization.js";
type CodexMapperOptions = { cwd?: string | null }; type CodexMapperOptions = { cwd?: string | null };
@@ -67,7 +68,7 @@ const CodexEditToolNameSchema = z.union([
z.literal("apply_diff"), z.literal("apply_diff"),
]); ]);
const CodexSearchToolNameSchema = z.union([z.literal("search"), z.literal("web_search")]); const CodexSearchToolNameSchema = z.union([z.literal("search"), z.literal("web_search")]);
const CodexSpeakToolNameSchema = z.literal("paseo.speak"); const CodexSpeakToolNameSchema = z.string().min(1).refine((name) => isSpeakToolName(name.trim()));
const CodexToolKindSchema = z.enum([ const CodexToolKindSchema = z.enum([
"shell", "shell",
+12 -4
View File
@@ -133,6 +133,7 @@ import {
buildVoiceAgentMcpServerConfig, buildVoiceAgentMcpServerConfig,
buildVoiceModeSystemPrompt, buildVoiceModeSystemPrompt,
stripVoiceModeSystemPrompt, stripVoiceModeSystemPrompt,
wrapSpokenInput,
} from "./voice-config.js"; } from "./voice-config.js";
import { isVoicePermissionAllowed } from "./voice-permission-policy.js"; import { isVoicePermissionAllowed } from "./voice-permission-policy.js";
import { import {
@@ -2834,6 +2835,7 @@ export class Session {
messageId?: string, messageId?: string,
images?: Array<{ data: string; mimeType: string }>, images?: Array<{ data: string; mimeType: string }>,
runOptions?: AgentRunOptions, runOptions?: AgentRunOptions,
options?: { spokenInput?: boolean },
): Promise<void> { ): Promise<void> {
this.sessionLogger.info( this.sessionLogger.info(
{ agentId, textPreview: text.substring(0, 50), imageCount: images?.length ?? 0 }, { agentId, textPreview: text.substring(0, 50), imageCount: images?.length ?? 0 },
@@ -2861,7 +2863,8 @@ export class Session {
); );
} }
const prompt = this.buildAgentPrompt(text, images); const promptText = options?.spokenInput ? wrapSpokenInput(text) : text;
const prompt = this.buildAgentPrompt(promptText, images);
this.startAgentStream(agentId, prompt, runOptions); this.startAgentStream(agentId, prompt, runOptions);
} }
@@ -7019,9 +7022,14 @@ export class Session {
return; return;
} }
// Route voice utterances through the same send path as regular text input: await this.handleSendAgentMessage(
// interrupt-if-running, record message, then start a new stream. agentId,
await this.handleSendAgentMessage(agentId, result.text); result.text,
undefined,
undefined,
undefined,
{ spokenInput: true },
);
await this.flushPendingAudioSegments("transcription complete"); await this.flushPendingAudioSegments("transcription complete");
} }
@@ -53,6 +53,10 @@ export function buildVoiceModeSystemPrompt(existing: string | undefined, enabled
.join("\n\n"); .join("\n\n");
} }
export function wrapSpokenInput(text: string): string {
return `<spoken-input>\n${text}\n</spoken-input>`;
}
export function buildVoiceAgentMcpServerConfig(params: { export function buildVoiceAgentMcpServerConfig(params: {
command: string; command: string;
baseArgs: string[]; baseArgs: string[];