jb/paseo
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

Update files

Browse Source
This commit is contained in:
Mohamed Boudra
2026-02-11 11:04:49 +07:00
parent 002c09681d
commit 0d60588431
23 changed files with 1264 additions and 320 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/server/src/server/agent/agent-manager.test.ts
+88 -2
View File
@@ -150,7 +150,7 @@ describe("AgentManager", () => {
).rejects.toThrow("Working directory does not exist");
});
test("resumeAgent keeps metadata config and applies systemPrompt/mcpServers overrides", async () => {
test("resumeAgentFromPersistence keeps metadata config and applies systemPrompt/mcpServers overrides", async () => {
const workdir = mkdtempSync(join(tmpdir(), "agent-manager-resume-"));
const storagePath = join(workdir, "agents");
const storage = new AgentStorage(storagePath, logger);
@@ -211,7 +211,7 @@ describe("AgentManager", () => {
},
};
const resumed = await manager.resumeAgent(handle, {
const resumed = await manager.resumeAgentFromPersistence(handle, {
cwd: workdir,
systemPrompt: "new prompt",
mcpServers: {
@@ -243,6 +243,92 @@ describe("AgentManager", () => {
});
});
test("reloadAgentSession preserves timeline and does not force history replay", async () => {
const workdir = mkdtempSync(join(tmpdir(), "agent-manager-reload-"));
const storagePath = join(workdir, "agents");
const storage = new AgentStorage(storagePath, logger);
class HistoryProbeSession extends TestAgentSession {
constructor(
config: AgentSessionConfig,
private readonly historyText: string | null
) {
super(config);
}
async *streamHistory(): AsyncGenerator<AgentStreamEvent> {
if (!this.historyText) {
return;
}
yield {
type: "timeline",
provider: this.provider,
item: { type: "assistant_message", text: this.historyText },
};
}
}
class HistoryProbeClient implements AgentClient {
readonly provider = "codex" as const;
readonly capabilities = TEST_CAPABILITIES;
async isAvailable(): Promise<boolean> {
return true;
}
async createSession(config: AgentSessionConfig): Promise<AgentSession> {
return new HistoryProbeSession(config, null);
}
async resumeSession(
handle: AgentPersistenceHandle,
overrides?: Partial<AgentSessionConfig>
): Promise<AgentSession> {
const metadata = (handle.metadata ?? {}) as Partial<AgentSessionConfig>;
const merged: AgentSessionConfig = {
...metadata,
...overrides,
provider: "codex",
cwd: overrides?.cwd ?? metadata.cwd ?? process.cwd(),
};
return new HistoryProbeSession(merged, "history replay from provider");
}
}
const manager = new AgentManager({
clients: {
codex: new HistoryProbeClient(),
},
registry: storage,
logger,
idFactory: () => "00000000-0000-4000-8000-000000000113",
});
const snapshot = await manager.createAgent({
provider: "codex",
cwd: workdir,
});
await manager.appendTimelineItem(snapshot.id, {
type: "assistant_message",
text: "keep this timeline in memory",
});
await manager.hydrateTimelineFromProvider(snapshot.id);
const beforeReload = manager.getTimeline(snapshot.id);
expect(beforeReload).toHaveLength(1);
await manager.reloadAgentSession(snapshot.id, {
systemPrompt: "reloaded prompt",
});
const afterReload = manager.getTimeline(snapshot.id);
expect(afterReload).toEqual(beforeReload);
// If reload resets historyPrimed, this would replay provider history and append another item.
await manager.hydrateTimelineFromProvider(snapshot.id);
const afterHydrate = manager.getTimeline(snapshot.id);
expect(afterHydrate).toEqual(beforeReload);
});
test("createAgent fails when generated agent ID is not a UUID", async () => {
const workdir = mkdtempSync(join(tmpdir(), "agent-manager-test-"));
const storagePath = join(workdir, "agents");
packages/server/src/server/agent/agent-manager.ts
+40 -10
View File
@@ -371,7 +371,9 @@ export class AgentManager {
);
}
async resumeAgent(
// Reconstruct an agent from provider persistence. Callers should explicitly
// hydrate timeline history after resume.
async resumeAgentFromPersistence(
handle: AgentPersistenceHandle,
overrides?: Partial<AgentSessionConfig>,
agentId?: string,
@@ -384,7 +386,7 @@ export class AgentManager {
): Promise<ManagedAgent> {
const resolvedAgentId = validateAgentId(
agentId ?? this.idFactory(),
"resumeAgent"
"resumeAgentFromPersistence"
);
const metadata = (handle.metadata ?? {}) as Partial<AgentSessionConfig>;
const mergedConfig = {
@@ -407,11 +409,18 @@ export class AgentManager {
);
}
async refreshAgentFromPersistence(
// Hot-reload an active agent session with config overrides while preserving
// in-memory timeline state.
async reloadAgentSession(
agentId: string,
overrides?: Partial<AgentSessionConfig>
): Promise<ManagedAgent> {
const existing = this.requireAgent(agentId);
const preservedTimeline = [...existing.timeline];
const preservedHistoryPrimed = existing.historyPrimed;
const preservedLastUsage = existing.lastUsage;
const preservedLastError = existing.lastError;
const preservedAttention = existing.attention;
const handle = existing.persistence;
const provider = handle?.provider ?? existing.provider;
const client = this.requireClient(provider);
@@ -437,12 +446,17 @@ export class AgentManager {
);
}
// Preserve existing labels during refresh
// Preserve existing labels and timeline during reload.
return this.registerSession(session, normalizedConfig, agentId, {
labels: existing.labels,
createdAt: existing.createdAt,
updatedAt: existing.updatedAt,
lastUserMessageAt: existing.lastUserMessageAt,
timeline: preservedTimeline,
historyPrimed: preservedHistoryPrimed,
lastUsage: preservedLastUsage,
lastError: preservedLastError,
attention: preservedAttention,
});
}
@@ -844,9 +858,9 @@ export class AgentManager {
return iterator.done ? null : iterator.value;
}
async primeAgentHistory(agentId: string): Promise<void> {
async hydrateTimelineFromProvider(agentId: string): Promise<void> {
const agent = this.requireAgent(agentId);
await this.primeHistory(agent);
await this.hydrateTimeline(agent);
}
private getLastAssistantMessage(agentId: string): string | null {
@@ -1033,6 +1047,11 @@ export class AgentManager {
updatedAt?: Date;
lastUserMessageAt?: Date | null;
labels?: Record<string, string>;
timeline?: AgentTimelineItem[];
historyPrimed?: boolean;
lastUsage?: AgentUsage;
lastError?: string;
attention?: AttentionState;
}
): Promise<ManagedAgent> {
const resolvedAgentId = validateAgentId(agentId, "registerSession");
@@ -1056,11 +1075,22 @@ export class AgentManager {
currentModeId: null,
pendingPermissions: new Map(),
pendingRun: null,
timeline: [],
timeline: options?.timeline ? [...options.timeline] : [],
persistence: attachPersistenceCwd(session.describePersistence(), config.cwd),
historyPrimed: false,
historyPrimed: options?.historyPrimed ?? false,
lastUserMessageAt: options?.lastUserMessageAt ?? null,
attention: { requiresAttention: false },
lastUsage: options?.lastUsage,
lastError: options?.lastError,
attention:
options?.attention != null
? options.attention.requiresAttention
? {
requiresAttention: true,
attentionReason: options.attention.attentionReason,
attentionTimestamp: new Date(options.attention.attentionTimestamp),
}
: { requiresAttention: false }
: { requiresAttention: false },
internal: config.internal ?? false,
labels: options?.labels ?? {},
} as ActiveManagedAgent;
@@ -1144,7 +1174,7 @@ export class AgentManager {
}
}
private async primeHistory(agent: ActiveManagedAgent): Promise<void> {
private async hydrateTimeline(agent: ActiveManagedAgent): Promise<void> {
if (agent.historyPrimed) {
return;
}
packages/server/src/server/agent/agent-metadata-generator.test.ts
+2 -2
View File
@@ -82,7 +82,7 @@ function initGitRepo(repoDir: string): void {
modeId: "auto",
cwd: repoDir,
title: "Main Agent",
}, "metadata-title-agent");
}, "4e0a4508-e522-4fe9-8384-cf3bf889f16d");
await generateAndApplyAgentMetadata({
agentManager: manager,
@@ -122,7 +122,7 @@ function initGitRepo(repoDir: string): void {
modeId: "auto",
cwd: worktree.worktreePath,
title: "Worktree Agent",
}, "metadata-branch-agent");
}, "32bb765d-f637-44a2-9820-f2efd5261418");
await generateAndApplyAgentMetadata({
agentManager: manager,
packages/server/src/server/agent/providers/claude-agent.test.ts
+40
View File
@@ -825,6 +825,7 @@ async function startAgentMcpServer(): Promise<AgentMcpServerHandle> {
// Wait for history file to be written
const historyPaths = getClaudeHistoryPaths(cwd, handle!.sessionId);
expect(await waitForHistoryFile(historyPaths)).toBe(true);
expect(await waitForHistoryContains(historyPaths, secretWord)).toBe(true);
// Resume and verify context is preserved
const resumed = await client.resumeSession(handle!, { cwd });
@@ -1079,6 +1080,7 @@ async function startAgentMcpServer(): Promise<AgentMcpServerHandle> {
const historyPaths = getClaudeHistoryPaths(cwd, sessionId!);
expect(await waitForHistoryFile(historyPaths)).toBe(true);
expect(await waitForHistoryContains(historyPaths, "HYDRATION_PROOF_LINE_TWO")).toBe(true);
const resumed = await client.resumeSession(handle!, { cwd });
const hydrationUpdates: StreamHydrationUpdate[] = [];
@@ -1179,6 +1181,7 @@ async function startAgentMcpServer(): Promise<AgentMcpServerHandle> {
const historyPaths = getClaudeHistoryPaths(cwd, sessionId!);
expect(await waitForHistoryFile(historyPaths)).toBe(true);
expect(await waitForHistoryContains(historyPaths, promptMarker)).toBe(true);
const resumed = await client.resumeSession(handle!, { cwd });
const hydrationUpdates: StreamHydrationUpdate[] = [];
@@ -1524,6 +1527,43 @@ async function waitForHistoryFile(historyPaths: string | string[], timeoutMs = 1
return candidates.some((entry) => existsSync(entry));
}
async function waitForHistoryContains(
historyPaths: string | string[],
marker: string,
timeoutMs = 15_000
): Promise<boolean> {
const candidates = Array.isArray(historyPaths)
? Array.from(new Set(historyPaths))
: [historyPaths];
const deadline = Date.now() + timeoutMs;
const hasMarker = (): boolean => {
for (const historyPath of candidates) {
if (!existsSync(historyPath)) {
continue;
}
try {
const content = readFileSync(historyPath, "utf8");
if (content.includes(marker)) {
return true;
}
} catch {
// History file may still be mid-write; retry.
}
}
return false;
};
while (Date.now() < deadline) {
if (hasMarker()) {
return true;
}
await new Promise((resolve) => setTimeout(resolve, 250));
}
return hasMarker();
}
function rawContainsText(raw: unknown, text: string, depth = 0): boolean {
if (!raw || typeof text !== "string" || !text) {
return false;
packages/server/src/server/agent/providers/codex-app-server-agent.test.ts
+52 -31
View File
@@ -1214,11 +1214,15 @@ describe("Codex app-server provider (integration)", () => {
(item) => item.type === "assistant_message" || item.type === "user_message"
)
).toBe(true);
const response = await resumed.run(
`Respond with the exact token ${token} and stop.`
const historyIncludesToken = history.some(
(item) =>
(item.type === "assistant_message" || item.type === "user_message") &&
item.text.includes(token)
);
expect(response.finalText).toContain(token);
expect(historyIncludesToken).toBe(true);
const response = await resumed.run("Reply with CONTEXT_OK and stop.");
expect(response.finalText.toLowerCase()).toContain("context_ok");
const resumedHandle = resumed.describePersistence();
expect(resumedHandle?.sessionId).toBe(handle?.sessionId);
@@ -1327,34 +1331,51 @@ describe("Codex app-server provider (integration)", () => {
);
let failure: string | null = null;
for await (const event of events) {
if (event.type === "permission_requested" && event.request.name === "CodexFileChange") {
sawPermission = true;
captured = event.request;
await session.respondToPermission(event.request.id, { behavior: "allow" });
}
if (
event.type === "permission_resolved" &&
captured &&
event.requestId === captured.id &&
event.resolution.behavior === "allow"
) {
sawPermissionResolved = true;
}
if (event.type === "timeline" && event.item.type === "tool_call") {
timelineItems.push(event.item);
}
if (event.type === "turn_failed") {
failure = event.error;
break;
}
if (event.type === "turn_completed") {
break;
}
try {
await Promise.race([
(async () => {
for await (const event of events) {
if (event.type === "permission_requested" && event.request.name === "CodexFileChange") {
sawPermission = true;
captured = event.request;
await session.respondToPermission(event.request.id, { behavior: "allow" });
}
if (
event.type === "permission_resolved" &&
captured &&
event.requestId === captured.id &&
event.resolution.behavior === "allow"
) {
sawPermissionResolved = true;
}
if (event.type === "timeline" && event.item.type === "tool_call") {
timelineItems.push(event.item);
}
if (event.type === "turn_failed") {
failure = event.error;
break;
}
if (event.type === "turn_completed") {
break;
}
}
})(),
new Promise<never>((_, reject) =>
setTimeout(
() =>
reject(
new Error(
"Timed out waiting for Codex file approval flow to complete"
)
),
100_000
)
),
]);
} finally {
await session.close();
}
await session.close();
if (failure) {
throw new Error(failure);
}
@@ -1391,7 +1412,7 @@ describe("Codex app-server provider (integration)", () => {
cleanup();
rmSync(cwd, { recursive: true, force: true });
}
}, 60000);
}, 120000);
test.runIf(isCodexInstalled())("tool approval flow requests user input for app tools", async () => {
const cleanup = useTempCodexSessionDir();
packages/server/src/server/daemon-e2e/git-operations.e2e.test.ts
+5 -2
View File
@@ -424,7 +424,10 @@ describe("daemon E2E", () => {
);
expect(completed.callId).toBeTruthy();
expect(completed.output).toBeTruthy();
expect(completed.detail.type).toBe("unknown");
if (completed.detail.type === "unknown") {
expect(completed.detail.output).toBeTruthy();
}
expect(existsSync(path.join(agent.cwd, "setup-done.txt"))).toBe(true);
await ctx.client.deleteAgent(agent.id);
@@ -507,7 +510,7 @@ describe("daemon E2E", () => {
expect(existsSync(path.join(agent.cwd, "setup-start.txt"))).toBe(true);
const output = failed.output as any;
const output = failed.detail.type === "unknown" ? failed.detail.output as any : undefined;
const commands = output?.commands as any[] | undefined;
expect(Array.isArray(commands)).toBe(true);
expect(commands?.[0]?.exitCode).toBe(7);
packages/server/src/server/daemon-e2e/tool-calls.e2e.test.ts
+105 -21
View File
@@ -18,6 +18,78 @@ function tmpCwd(): string {
const CODEX_TEST_MODEL = "gpt-5.1-codex-mini";
const CODEX_TEST_THINKING_OPTION_ID = "low";
type ToolCallItem = Extract<AgentTimelineItem, { type: "tool_call" }>;
function asRecord(value: unknown): Record<string, unknown> | null {
if (typeof value !== "object" || value === null || Array.isArray(value)) {
return null;
}
return value as Record<string, unknown>;
}
function commandFromRawInput(value: unknown): string | null {
const record = asRecord(value);
if (!record) {
return null;
}
const commandValue = record.command ?? record.cmd;
if (typeof commandValue === "string" && commandValue.length > 0) {
return commandValue;
}
if (Array.isArray(commandValue)) {
const parts = commandValue.filter((entry): entry is string => typeof entry === "string");
if (parts.length > 0) {
return parts.join(" ");
}
}
return null;
}
function commandFromToolCallDetail(detail: ToolCallItem["detail"]): string | null {
if (detail.type === "shell") {
return detail.command;
}
if (detail.type === "unknown") {
return commandFromRawInput(detail.input);
}
return null;
}
function extractPathFromPayload(value: unknown): string | null {
const record = asRecord(value);
if (!record) {
return null;
}
const directPath = record.path ?? record.file_path ?? record.filePath;
if (typeof directPath === "string" && directPath.length > 0) {
return directPath;
}
if (Array.isArray(record.files)) {
for (const file of record.files) {
const fileRecord = asRecord(file);
const filePath = fileRecord?.path;
if (typeof filePath === "string" && filePath.length > 0) {
return filePath;
}
}
}
return null;
}
function pathFromToolCallDetail(detail: ToolCallItem["detail"]): string | null {
if (detail.type === "read" || detail.type === "edit" || detail.type === "write") {
return detail.filePath;
}
if (detail.type === "unknown") {
return extractPathFromPayload(detail.input) ?? extractPathFromPayload(detail.output);
}
return null;
}
describe("daemon E2E", () => {
let ctx: DaemonTestContext;
@@ -34,9 +106,9 @@ describe("daemon E2E", () => {
function extractToolCalls(
queue: SessionOutboundMessage[],
agentId: string
): AgentTimelineItem[] {
const byCallId = new Map<string, AgentTimelineItem>();
const noCallId: AgentTimelineItem[] = [];
): ToolCallItem[] {
const byCallId = new Map<string, ToolCallItem>();
const noCallId: ToolCallItem[] = [];
for (const m of queue) {
if (
@@ -58,9 +130,9 @@ describe("daemon E2E", () => {
}
// Helper to log tool call structure in a consistent format
function logToolCall(prefix: string, tc: AgentTimelineItem): void {
if (tc.type !== "tool_call") return;
function logToolCall(prefix: string, tc: ToolCallItem): void {
void prefix;
void tc;
}
test(
@@ -92,10 +164,15 @@ describe("daemon E2E", () => {
logToolCall("CLAUDE_READ", tc);
}
const readCall = toolCalls.find((tc) => tc.type === "tool_call" && tc.name === "Read");
const readCall = toolCalls.find((tc) => tc.name === "Read");
expect(readCall).toBeDefined();
expect(readCall?.name).toBe("Read");
expect(readCall?.input).toBeDefined();
expect(readCall?.detail).toBeDefined();
const readPath = readCall ? pathFromToolCallDetail(readCall.detail) : null;
expect(readPath).toBeTruthy();
if (readPath) {
expect(readPath).toContain("/etc/hosts");
}
await ctx.client.deleteAgent(agent.id);
collector.unsubscribe();
@@ -133,13 +210,15 @@ describe("daemon E2E", () => {
logToolCall("CLAUDE_BASH", tc);
}
const bashCall = toolCalls.find((tc) => tc.type === "tool_call" && tc.name === "Bash");
const bashCall = toolCalls.find((tc) => tc.name === "Bash");
expect(bashCall).toBeDefined();
expect(bashCall?.name).toBe("Bash");
expect(bashCall?.input).toBeDefined();
// Command text should be in input.command
const bashInput = bashCall?.input as { command?: string } | undefined;
expect(bashInput?.command).toContain("echo");
expect(bashCall?.detail).toBeDefined();
const command = bashCall ? commandFromToolCallDetail(bashCall.detail) : null;
expect(command).toBeTruthy();
if (command) {
expect(command).toContain("echo");
}
await ctx.client.deleteAgent(agent.id);
collector.unsubscribe();
@@ -179,9 +258,17 @@ describe("daemon E2E", () => {
logToolCall("CLAUDE_EDIT", tc);
}
const editCall = toolCalls.find((tc) => tc.type === "tool_call" && tc.name === "Edit");
const editCall = toolCalls.find((tc) => tc.name === "Edit");
expect(editCall).toBeDefined();
expect(editCall?.input).toBeDefined();
expect(editCall?.detail).toBeDefined();
const editPath = editCall ? pathFromToolCallDetail(editCall.detail) : null;
if (editPath) {
expect(editPath).toBe(testFile);
} else if (editCall?.detail.type === "unknown") {
expect(editCall.detail.input ?? editCall.detail.output).toBeTruthy();
} else {
expect(editCall?.detail.type).toBe("edit");
}
await ctx.client.deleteAgent(agent.id);
collector.unsubscribe();
@@ -219,15 +306,12 @@ describe("daemon E2E", () => {
logToolCall("CODEX_SHELL", tc);
}
const shellCalls = toolCalls.filter(
(tc) => tc.type === "tool_call" && tc.name === "shell"
);
const shellCalls = toolCalls.filter((tc) => tc.name === "shell");
expect(shellCalls.length).toBeGreaterThan(0);
const echoCall = shellCalls.find((tc) => {
const shellInput = tc.input as { command?: string } | undefined;
return typeof shellInput?.command === "string" &&
shellInput.command.includes("echo");
const command = commandFromToolCallDetail(tc.detail);
return typeof command === "string" && command.includes("echo");
});
expect(echoCall).toBeDefined();
packages/server/src/server/session.ts
+82 -156
View File
@@ -209,14 +209,7 @@ function deriveProjectGroupingName(projectKey: string): string {
return segments[segments.length - 1] || projectKey;
}
function escapeXmlText(value: string): string {
return value
.replace(/&/g, "&amp;")
.replace(/</g, "&lt;")
.replace(/>/g, "&gt;");
}
type ProcessingPhase = "idle" | "transcribing" | "llm";
type ProcessingPhase = "idle" | "transcribing";
type NormalizedGitOptions = {
baseBranch?: string;
@@ -392,7 +385,6 @@ export class Session {
// State machine
private abortController: AbortController;
private processingPhase: ProcessingPhase = "idle";
private currentStreamPromise: Promise<void> | null = null;
// Voice mode state
private isVoiceMode = false;
@@ -425,9 +417,6 @@ export class Session {
{ format: string; chunks: Buffer[] }
>();
// Conversation history
private turnIndex = 0;
// Per-session managers
private readonly ttsManager: TTSManager;
private readonly sttManager: STTManager;
@@ -519,7 +508,7 @@ export class Session {
this.defaultLocalSpeechModelIds =
dictation?.localModels?.defaultModelIds && dictation.localModels.defaultModelIds.length > 0
? [...new Set(dictation.localModels.defaultModelIds)]
: ["parakeet-tdt-0.6b-v2-int8", "pocket-tts-onnx-int8"];
: ["parakeet-tdt-0.6b-v2-int8", "kokoro-en-v0_19"];
this.registerVoiceSpeakHandler = voiceBridge?.registerVoiceSpeakHandler;
this.unregisterVoiceSpeakHandler = voiceBridge?.unregisterVoiceSpeakHandler;
this.registerVoiceCallerContext = voiceBridge?.registerVoiceCallerContext;
@@ -641,6 +630,19 @@ export class Session {
}
}
private hasActiveAgentRun(agentId: string | null): boolean {
if (!agentId) {
return false;
}
const snapshot = this.agentManager.getAgent(agentId);
if (!snapshot) {
return false;
}
return snapshot.lifecycle === "running" || Boolean(snapshot.pendingRun);
}
/**
* Start streaming an agent run and forward results via the websocket broadcast
*/
@@ -739,6 +741,29 @@ export class Session {
return;
}
if (
this.isVoiceMode &&
this.voiceModeAgentId === event.agentId &&
event.event.type === "permission_requested" &&
isVoicePermissionAllowed(event.event.request)
) {
const requestId = event.event.request.id;
void this.agentManager
.respondToPermission(event.agentId, requestId, {
behavior: "allow",
})
.catch((error) => {
this.sessionLogger.warn(
{
err: error,
agentId: event.agentId,
requestId,
},
"Failed to auto-allow speak tool permission in voice mode"
);
});
}
// Reduce bandwidth/CPU on mobile: only forward high-frequency agent stream events
// for the focused agent, with a short grace window while backgrounded.
//
@@ -874,7 +899,7 @@ export class Session {
const handle = toAgentPersistenceHandle(this.sessionLogger, record.persistence);
let snapshot: ManagedAgent;
if (handle) {
snapshot = await this.agentManager.resumeAgent(
snapshot = await this.agentManager.resumeAgentFromPersistence(
handle,
buildConfigOverrides(record),
agentId,
@@ -885,7 +910,7 @@ export class Session {
snapshot = await this.agentManager.createAgent(config, agentId, { labels: record.labels });
}
await this.agentManager.primeAgentHistory(agentId);
await this.agentManager.hydrateTimelineFromProvider(agentId);
return this.agentManager.getAgent(agentId) ?? snapshot;
})();
@@ -1592,7 +1617,7 @@ export class Session {
};
try {
const refreshed = await this.agentManager.refreshAgentFromPersistence(
const refreshed = await this.agentManager.reloadAgentSession(
agentId,
refreshOverrides
);
@@ -1630,7 +1655,7 @@ export class Session {
if (restoreAgentConfig && this.voiceModeBaseConfig) {
const baseConfig = this.voiceModeBaseConfig;
try {
await this.agentManager.refreshAgentFromPersistence(agentId, {
await this.agentManager.reloadAgentSession(agentId, {
systemPrompt: buildVoiceModeSystemPrompt(baseConfig.systemPrompt, false),
mcpServers: this.cloneMcpServers(baseConfig.mcpServers),
});
@@ -2110,11 +2135,11 @@ export class Session {
`Resuming agent ${handle.sessionId} (${handle.provider})`
);
try {
const snapshot = await this.agentManager.resumeAgent(
const snapshot = await this.agentManager.resumeAgentFromPersistence(
handle,
overrides
);
await this.agentManager.primeAgentHistory(snapshot.id);
await this.agentManager.hydrateTimelineFromProvider(snapshot.id);
await this.forwardAgentUpdate(snapshot);
const timelineSize = this.emitAgentTimelineSnapshot(snapshot);
if (requestId) {
@@ -2165,7 +2190,7 @@ export class Session {
if (existing) {
await this.interruptAgentIfRunning(agentId);
if (existing.persistence) {
snapshot = await this.agentManager.refreshAgentFromPersistence(
snapshot = await this.agentManager.reloadAgentSession(
agentId
);
} else {
@@ -2182,14 +2207,14 @@ export class Session {
`Agent ${agentId} cannot be refreshed because it lacks persistence`
);
}
snapshot = await this.agentManager.resumeAgent(
snapshot = await this.agentManager.resumeAgentFromPersistence(
handle,
buildConfigOverrides(record),
agentId,
extractTimestamps(record)
);
}
await this.agentManager.primeAgentHistory(agentId);
await this.agentManager.hydrateTimelineFromProvider(agentId);
await this.forwardAgentUpdate(snapshot);
const timelineSize = this.emitAgentTimelineSnapshot(snapshot);
if (requestId) {
@@ -5076,12 +5101,6 @@ export class Session {
// Has content - abort any in-progress stream now
this.createAbortController();
// Wait for aborted stream to finish cleanup (save partial response)
if (this.currentStreamPromise) {
this.sessionLogger.debug("Waiting for aborted stream to finish cleanup");
await this.currentStreamPromise;
}
if (result.debugRecordingPath) {
this.emit({
type: "activity_log",
@@ -5114,19 +5133,27 @@ export class Session {
});
this.clearSpeechInProgress("transcription complete");
this.setPhase("idle");
if (!this.isVoiceMode) {
this.sessionLogger.debug(
{ requestId: result.requestId },
"Skipping voice agent processing because voice mode is disabled"
);
this.setPhase("idle");
return;
}
this.setPhase("llm");
this.currentStreamPromise = this.processVoiceTurn(result.text);
await this.currentStreamPromise;
this.setPhase("idle");
const agentId = this.voiceModeAgentId;
if (!agentId) {
this.sessionLogger.warn(
{ requestId: result.requestId },
"Skipping voice agent processing because no agent is currently voice-enabled"
);
return;
}
// Route voice utterances through the same send path as regular text input:
// interrupt-if-running, record message, then start a new stream.
await this.handleSendAgentMessage(agentId, result.text);
}
private registerVoiceBridgeForAgent(agentId: string): void {
@@ -5168,109 +5195,6 @@ export class Session {
});
}
private buildVoiceAgentPrompt(userText: string): string {
const transcription = escapeXmlText(userText.trim());
return [
"<voice_transcription>",
transcription,
"</voice_transcription>",
].join("\n");
}
private async processWithVoiceAgent(
agentId: string,
userText: string
): Promise<void> {
await this.ensureAgentLoaded(agentId);
await this.interruptAgentIfRunning(agentId);
this.sessionLogger.info(
{
agentId,
userTextLength: userText.length,
userTextPreview: userText.slice(0, 160),
},
"Starting voice agent turn"
);
const prompt = this.buildVoiceAgentPrompt(userText);
this.agentManager.recordUserMessage(agentId, userText);
let sawSpeakToolCall = false;
const iterator = this.agentManager.streamAgent(agentId, prompt);
for await (const event of iterator) {
if (event.type === "turn_failed") {
throw new Error(event.error);
}
if (event.type === "timeline") {
if (event.item.type === "tool_call" && typeof event.item.name === "string") {
this.sessionLogger.info(
{
agentId,
toolName: event.item.name,
callId: event.item.callId,
},
"Voice agent timeline tool call"
);
if (event.item.name.toLowerCase().includes("speak")) {
sawSpeakToolCall = true;
}
}
}
if (event.type === "permission_requested") {
if (isVoicePermissionAllowed(event.request)) {
await this.agentManager.respondToPermission(agentId, event.request.id, {
behavior: "allow",
});
} else {
this.sessionLogger.info(
{ agentId, requestName: event.request.name, requestId: event.request.id },
"Voice assistant left non-speak permission request for user decision"
);
}
}
}
if (!sawSpeakToolCall) {
this.sessionLogger.warn(
{ agentId },
"Voice agent turn completed without speak tool call; no audio playback emitted"
);
} else {
this.sessionLogger.info(
{ agentId },
"Voice agent turn included speak tool call"
);
}
}
/**
* Process user message through LLM with streaming and tool execution
*/
private async processVoiceTurn(latestUserText?: string): Promise<void> {
try {
if (!this.isVoiceMode) {
this.sessionLogger.warn("Ignoring processVoiceTurn call while voice mode is disabled");
return;
}
const agentId = this.voiceModeAgentId;
if (!agentId) {
this.sessionLogger.warn(
"Ignoring processVoiceTurn call because no agent is currently voice-enabled"
);
return;
}
const normalized = (latestUserText ?? "").trim();
if (!normalized) {
return;
}
await this.processWithVoiceAgent(agentId, normalized);
} finally {
this.turnIndex++;
this.currentStreamPromise = null;
}
}
/**
* Handle abort request from client
*/
@@ -5280,29 +5204,32 @@ export class Session {
`Abort request, phase: ${this.processingPhase}`
);
if (this.processingPhase === "llm") {
// Already in LLM phase - abort and wait for cleanup
this.abortController.abort();
this.sessionLogger.debug("Aborted LLM processing");
this.abortController.abort();
this.ttsManager.cancelPendingPlaybacks("abort request");
// Wait for stream to finish saving partial response
if (this.currentStreamPromise) {
this.sessionLogger.debug("Waiting for stream cleanup after abort");
await this.currentStreamPromise;
// Voice abort should always interrupt active agent output immediately.
if (this.isVoiceMode && this.voiceModeAgentId) {
try {
await this.interruptAgentIfRunning(this.voiceModeAgentId);
} catch (error) {
this.sessionLogger.warn(
{ err: error, agentId: this.voiceModeAgentId },
"Failed to interrupt active voice-mode agent on abort"
);
}
}
// Reset phase to idle
this.setPhase("idle");
// Clear any pending segments and timeouts
this.pendingAudioSegments = [];
this.clearBufferTimeout();
} else if (this.processingPhase === "transcribing") {
if (this.processingPhase === "transcribing") {
// Still in STT phase - we'll buffer the next audio
this.sessionLogger.debug("Will buffer next audio (currently transcribing)");
// Phase stays as 'transcribing', handleAudioChunk will handle buffering
return;
}
// If idle, nothing to do
// Reset phase to idle and clear pending non-voice buffers.
this.setPhase("idle");
this.pendingAudioSegments = [];
this.clearBufferTimeout();
}
/**
@@ -5313,7 +5240,7 @@ export class Session {
}
/**
* Mark speech detection start and abort any active playback/LLM
* Mark speech detection start and abort any active playback/agent run.
*/
private async handleVoiceSpeechStart(): Promise<void> {
if (this.speechInProgress) {
@@ -5322,11 +5249,10 @@ export class Session {
const chunkReceivedAt = Date.now();
const phaseBeforeAbort = this.processingPhase;
const hadActiveStream = Boolean(this.currentStreamPromise);
const hadActiveStream = this.hasActiveAgentRun(this.voiceModeAgentId);
this.speechInProgress = true;
this.sessionLogger.debug("Voice speech detected – aborting playback and LLM");
this.ttsManager.cancelPendingPlaybacks("voice speech detected");
this.sessionLogger.debug("Voice speech detected – aborting playback and active agent run");
if (this.pendingAudioSegments.length > 0) {
this.sessionLogger.debug(
packages/server/src/server/speech/providers/local/config.ts
+6 -2
View File
@@ -144,6 +144,10 @@ export function resolveLocalSpeechConfig(params: {
params.persisted.features?.voiceMode?.tts?.speed,
});
const resolvedVoiceTtsSpeakerId =
parsed.voiceLocalTtsSpeakerId ??
(parsed.voiceLocalTtsModel === "kokoro-en-v0_19" ? 0 : undefined);
return {
local:
parsed.includeProviderConfig
@@ -154,8 +158,8 @@ export function resolveLocalSpeechConfig(params: {
dictationStt: parsed.dictationLocalSttModel,
voiceStt: parsed.voiceLocalSttModel,
voiceTts: parsed.voiceLocalTtsModel,
...(parsed.voiceLocalTtsSpeakerId !== undefined
? { voiceTtsSpeakerId: parsed.voiceLocalTtsSpeakerId }
...(resolvedVoiceTtsSpeakerId !== undefined
? { voiceTtsSpeakerId: resolvedVoiceTtsSpeakerId }
: {}),
...(parsed.voiceLocalTtsSpeed !== undefined
? { voiceTtsSpeed: parsed.voiceLocalTtsSpeed }
packages/server/src/server/speech/providers/local/sherpa/model-catalog.ts
+1 -1
View File
@@ -74,6 +74,7 @@ export const SHERPA_ONNX_MODEL_CATALOG = {
requiredFiles: ["model.onnx", "voices.bin", "tokens.txt", "espeak-ng-data"],
description: "Kokoro TTS (higher quality; larger).",
aliases: ["kokoro"],
defaultFor: "tts",
},
"pocket-tts-onnx-int8": {
kind: "tts",
@@ -119,7 +120,6 @@ export const SHERPA_ONNX_MODEL_CATALOG = {
],
description: "Pocket TTS ONNX (INT8) with streaming decode support (via onnxruntime).",
aliases: ["pocket", "pocket-tts"],
defaultFor: "tts",
},
} as const satisfies Record<string, SherpaOnnxCatalogEntry>;
packages/server/src/server/speech/speech-config-resolver.test.ts
+6 -3
View File
@@ -36,12 +36,14 @@ describe("resolveSpeechConfig", () => {
models: {
dictationStt: "parakeet-tdt-0.6b-v2-int8",
voiceStt: "parakeet-tdt-0.6b-v2-int8",
voiceTts: "pocket-tts-onnx-int8",
voiceTts: "kokoro-en-v0_19",
voiceTtsSpeakerId: 0,
},
});
expect(result.speech.local?.models.dictationStt).toBe("parakeet-tdt-0.6b-v2-int8");
expect(result.speech.local?.models.voiceStt).toBe("parakeet-tdt-0.6b-v2-int8");
expect(result.speech.local?.models.voiceTts).toBe("pocket-tts-onnx-int8");
expect(result.speech.local?.models.voiceTts).toBe("kokoro-en-v0_19");
expect(result.speech.local?.models.voiceTtsSpeakerId).toBe(0);
});
test("resolves feature-scoped local model env vars", () => {
@@ -122,6 +124,7 @@ describe("resolveSpeechConfig", () => {
expect(result.speech.local?.models.dictationStt).toBe("parakeet-tdt-0.6b-v2-int8");
expect(result.speech.local?.models.voiceStt).toBe("parakeet-tdt-0.6b-v2-int8");
expect(result.speech.local?.models.voiceTts).toBe("pocket-tts-onnx-int8");
expect(result.speech.local?.models.voiceTts).toBe("kokoro-en-v0_19");
expect(result.speech.local?.models.voiceTtsSpeakerId).toBe(0);
});
});