jb/paseo
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'main' into dev

Browse Source 
Surgical merge of 74 commits from main into dev. Key features ported:

- Provider visibility gating (appVersion filtering for Pi/Copilot)
- Pi agent provider + Copilot re-enabled
- Provider-declared features system (Codex fast mode)
- Workspace dedup by worktree root (adapted to integer IDs)
- Bulk close archiving fix for stored agents
- Agent creation timeout increase to 60s
- Audio/voice crash fixes (external buffer copies)
- Multi-host setup fix
- Reload agent tab action

Dev architecture preserved: SQLite storage, service proxy,
workspace setup dialog, hover cards, removed launcher tabs.
This commit is contained in:
Mohamed Boudra
2026-04-05 12:59:07 +07:00
parent 5e1b24b217 a264058a30
commit fcfb9c99da
173 changed files with 8873 additions and 2316 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/app/src/voice/voice-runtime.ts
+1 -164
View File
@@ -3,7 +3,6 @@ import type { AgentStreamEventPayload, SessionOutboundMessage } from "@server/sh
import { resolveVoiceUnavailableMessage } from "@/utils/server-info-capabilities";
import type { DaemonServerInfo } from "@/stores/session-store";
import type { AudioEngine } from "@/voice/audio-engine-types";
import { REALTIME_VOICE_VAD_CONFIG } from "@/voice/realtime-voice-config";
import {
THINKING_TONE_NATIVE_PCM_BASE64,
THINKING_TONE_NATIVE_PCM_DURATION_MS,
@@ -82,8 +81,6 @@ interface RuntimeState {
segmentDurationTimer: ReturnType<typeof setInterval> | null;
lastDisplayVolumePublishMs: number;
serverSpeechStartedAt: number | null;
lastNoServerSpeechLogMs: number;
localAboveThresholdActive: boolean;
}
type AudioOutputPayload = Extract<SessionOutboundMessage, { type: "audio_output" }>["payload"];
@@ -120,19 +117,6 @@ interface CueState {
playing: boolean;
}
interface RealtimeBridgeStats {
windowStartedAtMs: number;
captureEvents: number;
captureBytes: number;
uplinkEvents: number;
uplinkRawBytes: number;
uplinkBase64Chars: number;
outputEvents: number;
outputBytes: number;
outputGroups: number;
jsLagMaxMs: number;
}
const INITIAL_SNAPSHOT: VoiceRuntimeSnapshot = {
phase: "disabled",
isVoiceMode: false,
@@ -210,8 +194,6 @@ export function createVoiceRuntime(deps: VoiceRuntimeDeps): VoiceRuntime {
segmentDurationTimer: null,
lastDisplayVolumePublishMs: 0,
serverSpeechStartedAt: null,
lastNoServerSpeechLogMs: 0,
localAboveThresholdActive: false,
};
const playback: RuntimePlaybackState = {
groups: new Map(),
@@ -220,18 +202,6 @@ export function createVoiceRuntime(deps: VoiceRuntimeDeps): VoiceRuntime {
processing: false,
generation: 0,
};
const bridgeStats: RealtimeBridgeStats = {
windowStartedAtMs: Date.now(),
captureEvents: 0,
captureBytes: 0,
uplinkEvents: 0,
uplinkRawBytes: 0,
uplinkBase64Chars: 0,
outputEvents: 0,
outputBytes: 0,
outputGroups: 0,
jsLagMaxMs: 0,
};
const cue: CueState = {
active: false,
token: 0,
@@ -246,41 +216,6 @@ export function createVoiceRuntime(deps: VoiceRuntimeDeps): VoiceRuntime {
return cuePcm16.buffer.slice(cuePcm16.byteOffset, cuePcm16.byteOffset + cuePcm16.byteLength);
},
};
let lagProbeLastMs = Date.now();
const lagProbe = setInterval(() => {
const now = Date.now();
const lagMs = Math.max(0, now - lagProbeLastMs - 100);
lagProbeLastMs = now;
if (lagMs > bridgeStats.jsLagMaxMs) {
bridgeStats.jsLagMaxMs = lagMs;
}
}, 100);
function flushBridgeStats(reason: string): void {
const now = Date.now();
const elapsedMs = now - bridgeStats.windowStartedAtMs;
if (elapsedMs < 1000) {
return;
}
console.log(
`[VoiceRuntime#${instanceId}][bridge] ${reason} ` +
`capture=${bridgeStats.captureEvents}ev/${bridgeStats.captureBytes}B ` +
`uplink=${bridgeStats.uplinkEvents}ev/${bridgeStats.uplinkRawBytes}B/${bridgeStats.uplinkBase64Chars}c ` +
`output=${bridgeStats.outputEvents}ev/${bridgeStats.outputBytes}B groups=${bridgeStats.outputGroups} ` +
`jsLagMaxMs=${bridgeStats.jsLagMaxMs} windowMs=${elapsedMs}`,
);
bridgeStats.windowStartedAtMs = now;
bridgeStats.captureEvents = 0;
bridgeStats.captureBytes = 0;
bridgeStats.uplinkEvents = 0;
bridgeStats.uplinkRawBytes = 0;
bridgeStats.uplinkBase64Chars = 0;
bridgeStats.outputEvents = 0;
bridgeStats.outputBytes = 0;
bridgeStats.outputGroups = 0;
bridgeStats.jsLagMaxMs = 0;
}
function emit(): void {
for (const listener of listeners) {
listener();
@@ -354,17 +289,11 @@ export function createVoiceRuntime(deps: VoiceRuntimeDeps): VoiceRuntime {
}
function resetPlaybackState(): void {
const hadGroups = playback.groups.size;
playback.generation += 1;
playback.groups.clear();
playback.orderedGroupIds = [];
playback.activeGroupId = null;
playback.processing = false;
if (hadGroups > 0) {
console.log(
`[VoiceRuntime] resetPlaybackState: cleared ${hadGroups} groups, new gen=${playback.generation}`,
);
}
}
function activateNextPlaybackGroup(): void {
@@ -394,15 +323,9 @@ export function createVoiceRuntime(deps: VoiceRuntimeDeps): VoiceRuntime {
playback.processing = true;
const generation = playback.generation;
console.log(
`[VoiceRuntime] processPlaybackQueue start gen=${generation} activeGroup=${playback.activeGroupId}`,
);
try {
while (playback.activeGroupId) {
if (generation !== playback.generation) {
console.log(
`[VoiceRuntime] processPlaybackQueue abort: generation changed ${generation} -> ${playback.generation}`,
);
return;
}
@@ -415,9 +338,6 @@ export function createVoiceRuntime(deps: VoiceRuntimeDeps): VoiceRuntime {
const nextChunk = group.chunks.get(group.nextChunkToPlay);
if (!nextChunk) {
if (group.finalChunkIndex !== null && group.nextChunkToPlay > group.finalChunkIndex) {
console.log(
`[VoiceRuntime] group=${group.groupId} complete, played=${group.started} chunks=${group.nextChunkToPlay}`,
);
playback.groups.delete(group.groupId);
if (playback.orderedGroupIds[0] === group.groupId) {
playback.orderedGroupIds.shift();
@@ -432,9 +352,6 @@ export function createVoiceRuntime(deps: VoiceRuntimeDeps): VoiceRuntime {
activateNextPlaybackGroup();
continue;
}
console.log(
`[VoiceRuntime] group=${group.groupId} waiting for chunk=${group.nextChunkToPlay} (finalChunkIndex=${group.finalChunkIndex})`,
);
return;
}
@@ -442,39 +359,21 @@ export function createVoiceRuntime(deps: VoiceRuntimeDeps): VoiceRuntime {
if (group.shouldPlay && !group.started && group.isVoiceMode) {
group.started = true;
console.log(
`[VoiceRuntime] group=${group.groupId} first play starting at chunk=${group.nextChunkToPlay}`,
);
api.onAssistantAudioStarted(serverId);
}
const playStart = Date.now();
try {
if (group.shouldPlay) {
await deps.engine.play(nextChunk.source);
console.log(
`[VoiceRuntime] played chunk=${group.nextChunkToPlay} id=${nextChunk.id} took=${Date.now() - playStart}ms`,
);
} else {
console.log(
`[VoiceRuntime] SKIPPED chunk=${group.nextChunkToPlay} id=${nextChunk.id} shouldPlay=false`,
);
}
} catch (error) {
if (generation !== playback.generation) {
console.log(`[VoiceRuntime] play error + generation changed, aborting`);
return;
}
console.error(
`[VoiceRuntime] play error chunk=${group.nextChunkToPlay} took=${Date.now() - playStart}ms:`,
error,
);
console.error(`[VoiceRuntime] play error chunk=${group.nextChunkToPlay}:`, error);
}
if (generation !== playback.generation) {
console.log(
`[VoiceRuntime] post-play generation changed ${generation} -> ${playback.generation}`,
);
return;
}
@@ -491,9 +390,6 @@ export function createVoiceRuntime(deps: VoiceRuntimeDeps): VoiceRuntime {
if (generation === playback.generation) {
playback.processing = false;
}
console.log(
`[VoiceRuntime] processPlaybackQueue exit gen=${generation} currentGen=${playback.generation}`,
);
}
}
@@ -607,12 +503,6 @@ export function createVoiceRuntime(deps: VoiceRuntimeDeps): VoiceRuntime {
}
const base64 = Buffer.from(chunk).toString("base64");
bridgeStats.captureEvents += 1;
bridgeStats.captureBytes += chunk.byteLength;
bridgeStats.uplinkEvents += 1;
bridgeStats.uplinkRawBytes += chunk.byteLength;
bridgeStats.uplinkBase64Chars += base64.length;
flushBridgeStats("uplink");
void activeSession.adapter.sendVoiceAudioChunk(base64, PCM_MIME_TYPE).catch((error) => {
console.error(`[VoiceRuntime#${instanceId}] Failed to send audio chunk:`, error);
@@ -625,8 +515,6 @@ export function createVoiceRuntime(deps: VoiceRuntimeDeps): VoiceRuntime {
state.turnInProgress = false;
state.serverSpeechDetected = false;
state.lastDisplayVolumePublishMs = 0;
state.lastNoServerSpeechLogMs = 0;
state.localAboveThresholdActive = false;
uploader.reset();
resetCaptureTelemetry();
patchSnapshot({ ...INITIAL_SNAPSHOT });
@@ -747,11 +635,6 @@ export function createVoiceRuntime(deps: VoiceRuntimeDeps): VoiceRuntime {
if (!state.snapshot.isVoiceMode || state.snapshot.isMuted) {
return;
}
if (bridgeStats.captureEvents === 0) {
console.log(
`[VoiceRuntime#${instanceId}] firstCapturePcm bytes=${chunk.byteLength} phase=${state.snapshot.phase} transportReady=${state.transportReady}`,
);
}
uploader.pushPcmChunk(chunk);
},
@@ -768,30 +651,6 @@ export function createVoiceRuntime(deps: VoiceRuntimeDeps): VoiceRuntime {
return;
}
const isActive = level > REALTIME_VOICE_VAD_CONFIG.volumeThreshold;
if (isActive && !state.localAboveThresholdActive) {
state.localAboveThresholdActive = true;
console.log(
`[VoiceRuntime#${instanceId}] localSpeechActive level=${level.toFixed(3)} threshold=${REALTIME_VOICE_VAD_CONFIG.volumeThreshold.toFixed(3)} phase=${state.snapshot.phase} transportReady=${state.transportReady}`,
);
}
if (!isActive && state.localAboveThresholdActive) {
state.localAboveThresholdActive = false;
console.log(
`[VoiceRuntime#${instanceId}] localSpeechInactive phase=${state.snapshot.phase} serverSpeaking=${state.serverSpeechDetected}`,
);
}
if (
isActive &&
!state.serverSpeechDetected &&
nowMs - state.lastNoServerSpeechLogMs >= 1500
) {
state.lastNoServerSpeechLogMs = nowMs;
console.log(
`[VoiceRuntime#${instanceId}] localSpeechWithoutServerSpeech level=${level.toFixed(3)} phase=${state.snapshot.phase} turnInProgress=${state.turnInProgress} transportReady=${state.transportReady}`,
);
}
patchTelemetry((prev) => ({
...prev,
isSpeaking: state.serverSpeechDetected,
@@ -806,33 +665,15 @@ export function createVoiceRuntime(deps: VoiceRuntimeDeps): VoiceRuntime {
!state.snapshot.isVoiceMode ||
!payload.isVoiceMode
) {
console.log(
`[VoiceRuntime#${instanceId}] audio_output DROPPED: activeServer=${state.snapshot.activeServerId} serverId=${serverId} isVoiceMode=${state.snapshot.isVoiceMode} payloadVoice=${payload.isVoiceMode}`,
);
return;
}
const groupId = payload.groupId ?? payload.id;
const chunkIndex = payload.chunkIndex ?? 0;
const decoded = decodeAudioChunk(payload.audio);
bridgeStats.outputEvents += 1;
bridgeStats.outputBytes += decoded.byteLength;
bridgeStats.outputGroups += playback.groups.has(groupId) ? 0 : 1;
console.log(
`[VoiceRuntime#${instanceId}] audio_output groupId=${groupId} chunk=${chunkIndex} isLast=${payload.isLastChunk} ` +
`base64Chars=${payload.audio.length} decodedBytes=${decoded.byteLength} format=${payload.format} ` +
`head=${Array.from(decoded.slice(0, 12))
.map((value) => value.toString(16).padStart(2, "0"))
.join(" ")}`,
);
flushBridgeStats("audio_output");
let group = playback.groups.get(groupId);
if (!group) {
const shouldPlay = api.shouldPlayVoiceAudio(serverId);
console.log(
`[VoiceRuntime] new group=${groupId} shouldPlay=${shouldPlay} phase=${state.snapshot.phase} isSpeaking=${state.telemetry.isSpeaking}`,
);
group = {
groupId,
isVoiceMode: payload.isVoiceMode,
@@ -966,7 +807,6 @@ export function createVoiceRuntime(deps: VoiceRuntimeDeps): VoiceRuntime {
async destroy() {
await this.stopVoice().catch(() => undefined);
clearInterval(lagProbe);
await deps.engine.destroy();
listeners.clear();
telemetryListeners.clear();
@@ -1057,9 +897,6 @@ export function createVoiceRuntime(deps: VoiceRuntimeDeps): VoiceRuntime {
return;
}
console.log(
`[VoiceRuntime#${instanceId}] onServerSpeechStateChanged isSpeaking=${isSpeaking} phase=${state.snapshot.phase} volume=${state.telemetry.volume}`,
);
state.serverSpeechDetected = isSpeaking;
state.serverSpeechStartedAt = isSpeaking ? (state.serverSpeechStartedAt ?? Date.now()) : null;
if (isSpeaking) {