jb/paseo
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'main' into dev

Browse Source 
Surgical merge of 74 commits from main into dev. Key features ported:

- Provider visibility gating (appVersion filtering for Pi/Copilot)
- Pi agent provider + Copilot re-enabled
- Provider-declared features system (Codex fast mode)
- Workspace dedup by worktree root (adapted to integer IDs)
- Bulk close archiving fix for stored agents
- Agent creation timeout increase to 60s
- Audio/voice crash fixes (external buffer copies)
- Multi-host setup fix
- Reload agent tab action

Dev architecture preserved: SQLite storage, service proxy,
workspace setup dialog, hover cards, removed launcher tabs.
This commit is contained in:
Mohamed Boudra
2026-04-05 12:59:07 +07:00
parent 5e1b24b217 a264058a30
commit fcfb9c99da
173 changed files with 8873 additions and 2316 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/app/src/voice/audio-engine.native.ts
+2 -116
View File
@@ -14,20 +14,6 @@ interface AudioEngineTraceOptions {
traceLabel?: string;
}
let nextAudioEngineInstanceId = 1;
interface BridgeStats {
windowStartedAtMs: number;
captureEvents: number;
captureBytes: number;
volumeEvents: number;
volumeMax: number;
playbackEvents: number;
playbackInputBytes: number;
playbackResampledBytes: number;
playbackDurationMs: number;
}
function parsePcmSampleRate(mimeType: string): number | null {
const match = /rate=(\d+)/i.exec(mimeType);
if (!match) {
@@ -85,48 +71,6 @@ export function createAudioEngine(
_options?: AudioEngineTraceOptions,
): AudioEngine {
const native = require("@getpaseo/expo-two-way-audio");
const instanceId = nextAudioEngineInstanceId++;
const bridgeStats: BridgeStats = {
windowStartedAtMs: Date.now(),
captureEvents: 0,
captureBytes: 0,
volumeEvents: 0,
volumeMax: 0,
playbackEvents: 0,
playbackInputBytes: 0,
playbackResampledBytes: 0,
playbackDurationMs: 0,
};
const toHexPreview = (bytes: Uint8Array, count = 12): string =>
Array.from(bytes.slice(0, count))
.map((value) => value.toString(16).padStart(2, "0"))
.join(" ");
const maybeFlushBridgeStats = (reason: string): void => {
const now = Date.now();
const elapsedMs = now - bridgeStats.windowStartedAtMs;
if (elapsedMs < 1000) {
return;
}
console.log(
`[AudioEngine.native#${instanceId}][bridge] ${reason} ` +
`capture=${bridgeStats.captureEvents}ev/${bridgeStats.captureBytes}B ` +
`volume=${bridgeStats.volumeEvents}ev max=${bridgeStats.volumeMax.toFixed(3)} ` +
`play=${bridgeStats.playbackEvents}ev/${bridgeStats.playbackInputBytes}B->${bridgeStats.playbackResampledBytes}B ` +
`playMs=${bridgeStats.playbackDurationMs.toFixed(1)} ` +
`windowMs=${elapsedMs}`,
);
bridgeStats.windowStartedAtMs = now;
bridgeStats.captureEvents = 0;
bridgeStats.captureBytes = 0;
bridgeStats.volumeEvents = 0;
bridgeStats.volumeMax = 0;
bridgeStats.playbackEvents = 0;
bridgeStats.playbackInputBytes = 0;
bridgeStats.playbackResampledBytes = 0;
bridgeStats.playbackDurationMs = 0;
};
const refs: {
initialized: boolean;
@@ -140,8 +84,6 @@ export function createAudioEngine(
reject: (error: Error) => void;
settled: boolean;
} | null;
sawFirstMicChunk: boolean;
sawFirstVolumeEvent: boolean;
destroyed: boolean;
} = {
initialized: false,
@@ -151,8 +93,6 @@ export function createAudioEngine(
processingQueue: false,
playbackTimeout: null,
activePlayback: null,
sawFirstMicChunk: false,
sawFirstVolumeEvent: false,
destroyed: false,
};
@@ -163,15 +103,6 @@ export function createAudioEngine(
return;
}
const pcm = event.data as Uint8Array;
if (!refs.sawFirstMicChunk) {
refs.sawFirstMicChunk = true;
console.log(
`[AudioEngine.native#${instanceId}] firstMicChunk bytes=${pcm.byteLength} head=${toHexPreview(pcm)}`,
);
}
bridgeStats.captureEvents += 1;
bridgeStats.captureBytes += pcm.byteLength;
maybeFlushBridgeStats("capture");
callbacks.onCaptureData(pcm);
},
);
@@ -182,26 +113,10 @@ export function createAudioEngine(
return;
}
const level = refs.muted ? 0 : event.data;
bridgeStats.volumeEvents += 1;
bridgeStats.volumeMax = Math.max(bridgeStats.volumeMax, level);
if (!refs.sawFirstVolumeEvent) {
refs.sawFirstVolumeEvent = true;
console.log(
`[AudioEngine.native#${instanceId}] firstInputVolume level=${level.toFixed(3)} muted=${refs.muted}`,
);
}
maybeFlushBridgeStats("volume");
callbacks.onVolumeLevel(level);
},
);
const outputVolumeSubscription = native.addExpoTwoWayAudioEventListener(
"onOutputVolumeLevelData",
(event: any) => {
console.log(`[AudioEngine.native#${instanceId}] outputVolume=${event.data}`);
},
);
async function ensureInitialized(): Promise<void> {
if (refs.initialized) {
return;
@@ -210,20 +125,13 @@ export function createAudioEngine(
if (!success) {
throw new Error("expo-two-way-audio: native initialize() returned false");
}
console.log(`[AudioEngine.native#${instanceId}] initialized successfully`);
refs.initialized = true;
}
async function ensureMicrophonePermission(): Promise<void> {
let permission = await native.getMicrophonePermissionsAsync().catch(() => null);
console.log(
`[AudioEngine.native#${instanceId}] microphonePermission initial=${permission?.status ?? "unknown"} granted=${String(permission?.granted ?? false)}`,
);
if (!permission?.granted) {
permission = await native.requestMicrophonePermissionsAsync().catch(() => null);
console.log(
`[AudioEngine.native#${instanceId}] microphonePermission requested=${permission?.status ?? "unknown"} granted=${String(permission?.granted ?? false)}`,
);
}
if (!permission?.granted) {
throw new Error(
@@ -253,17 +161,6 @@ export function createAudioEngine(
// Native AudioEngine expects 16kHz PCM16
const pcm16k = resamplePcm16(pcm, inputRate, 16000);
const durationSec = pcm16k.length / 2 / 16000;
bridgeStats.playbackEvents += 1;
bridgeStats.playbackInputBytes += pcm.length;
bridgeStats.playbackResampledBytes += pcm16k.length;
bridgeStats.playbackDurationMs += durationSec * 1000;
console.log(
`[AudioEngine.native#${instanceId}] playPCMData: inputRate=${inputRate} inputBytes=${pcm.length} ` +
`resampled=${pcm16k.length} durationSec=${durationSec.toFixed(3)} ` +
`pcmHead=${toHexPreview(pcm)} resampledHead=${toHexPreview(pcm16k)}`,
);
maybeFlushBridgeStats("play");
native.resumePlayback();
native.playPCMData(pcm16k);
@@ -334,26 +231,18 @@ export function createAudioEngine(
}
microphoneSubscription.remove();
volumeSubscription.remove();
outputVolumeSubscription.remove();
},
async startCapture() {
if (refs.captureActive) {
console.log(`[AudioEngine.native#${instanceId}] startCapture skipped: already active`);
return;
}
try {
console.log(`[AudioEngine.native#${instanceId}] startCapture begin`);
await ensureMicrophonePermission();
await ensureInitialized();
refs.sawFirstMicChunk = false;
refs.sawFirstVolumeEvent = false;
const isRecording = native.toggleRecording(true);
native.toggleRecording(true);
refs.captureActive = true;
console.log(
`[AudioEngine.native#${instanceId}] startCapture toggleRecording(true) => ${String(isRecording)}`,
);
} catch (error) {
const wrapped = error instanceof Error ? error : new Error(String(error));
callbacks.onError?.(wrapped);
@@ -363,10 +252,7 @@ export function createAudioEngine(
async stopCapture() {
if (refs.captureActive) {
const isRecording = native.toggleRecording(false);
console.log(
`[AudioEngine.native#${instanceId}] stopCapture toggleRecording(false) => ${String(isRecording)}`,
);
native.toggleRecording(false);
}
refs.captureActive = false;
refs.muted = false;
packages/app/src/voice/voice-runtime.ts
+1 -164
View File
@@ -3,7 +3,6 @@ import type { AgentStreamEventPayload, SessionOutboundMessage } from "@server/sh
import { resolveVoiceUnavailableMessage } from "@/utils/server-info-capabilities";
import type { DaemonServerInfo } from "@/stores/session-store";
import type { AudioEngine } from "@/voice/audio-engine-types";
import { REALTIME_VOICE_VAD_CONFIG } from "@/voice/realtime-voice-config";
import {
THINKING_TONE_NATIVE_PCM_BASE64,
THINKING_TONE_NATIVE_PCM_DURATION_MS,
@@ -82,8 +81,6 @@ interface RuntimeState {
segmentDurationTimer: ReturnType<typeof setInterval> | null;
lastDisplayVolumePublishMs: number;
serverSpeechStartedAt: number | null;
lastNoServerSpeechLogMs: number;
localAboveThresholdActive: boolean;
}
type AudioOutputPayload = Extract<SessionOutboundMessage, { type: "audio_output" }>["payload"];
@@ -120,19 +117,6 @@ interface CueState {
playing: boolean;
}
interface RealtimeBridgeStats {
windowStartedAtMs: number;
captureEvents: number;
captureBytes: number;
uplinkEvents: number;
uplinkRawBytes: number;
uplinkBase64Chars: number;
outputEvents: number;
outputBytes: number;
outputGroups: number;
jsLagMaxMs: number;
}
const INITIAL_SNAPSHOT: VoiceRuntimeSnapshot = {
phase: "disabled",
isVoiceMode: false,
@@ -210,8 +194,6 @@ export function createVoiceRuntime(deps: VoiceRuntimeDeps): VoiceRuntime {
segmentDurationTimer: null,
lastDisplayVolumePublishMs: 0,
serverSpeechStartedAt: null,
lastNoServerSpeechLogMs: 0,
localAboveThresholdActive: false,
};
const playback: RuntimePlaybackState = {
groups: new Map(),
@@ -220,18 +202,6 @@ export function createVoiceRuntime(deps: VoiceRuntimeDeps): VoiceRuntime {
processing: false,
generation: 0,
};
const bridgeStats: RealtimeBridgeStats = {
windowStartedAtMs: Date.now(),
captureEvents: 0,
captureBytes: 0,
uplinkEvents: 0,
uplinkRawBytes: 0,
uplinkBase64Chars: 0,
outputEvents: 0,
outputBytes: 0,
outputGroups: 0,
jsLagMaxMs: 0,
};
const cue: CueState = {
active: false,
token: 0,
@@ -246,41 +216,6 @@ export function createVoiceRuntime(deps: VoiceRuntimeDeps): VoiceRuntime {
return cuePcm16.buffer.slice(cuePcm16.byteOffset, cuePcm16.byteOffset + cuePcm16.byteLength);
},
};
let lagProbeLastMs = Date.now();
const lagProbe = setInterval(() => {
const now = Date.now();
const lagMs = Math.max(0, now - lagProbeLastMs - 100);
lagProbeLastMs = now;
if (lagMs > bridgeStats.jsLagMaxMs) {
bridgeStats.jsLagMaxMs = lagMs;
}
}, 100);
function flushBridgeStats(reason: string): void {
const now = Date.now();
const elapsedMs = now - bridgeStats.windowStartedAtMs;
if (elapsedMs < 1000) {
return;
}
console.log(
`[VoiceRuntime#${instanceId}][bridge] ${reason} ` +
`capture=${bridgeStats.captureEvents}ev/${bridgeStats.captureBytes}B ` +
`uplink=${bridgeStats.uplinkEvents}ev/${bridgeStats.uplinkRawBytes}B/${bridgeStats.uplinkBase64Chars}c ` +
`output=${bridgeStats.outputEvents}ev/${bridgeStats.outputBytes}B groups=${bridgeStats.outputGroups} ` +
`jsLagMaxMs=${bridgeStats.jsLagMaxMs} windowMs=${elapsedMs}`,
);
bridgeStats.windowStartedAtMs = now;
bridgeStats.captureEvents = 0;
bridgeStats.captureBytes = 0;
bridgeStats.uplinkEvents = 0;
bridgeStats.uplinkRawBytes = 0;
bridgeStats.uplinkBase64Chars = 0;
bridgeStats.outputEvents = 0;
bridgeStats.outputBytes = 0;
bridgeStats.outputGroups = 0;
bridgeStats.jsLagMaxMs = 0;
}
function emit(): void {
for (const listener of listeners) {
listener();
@@ -354,17 +289,11 @@ export function createVoiceRuntime(deps: VoiceRuntimeDeps): VoiceRuntime {
}
function resetPlaybackState(): void {
const hadGroups = playback.groups.size;
playback.generation += 1;
playback.groups.clear();
playback.orderedGroupIds = [];
playback.activeGroupId = null;
playback.processing = false;
if (hadGroups > 0) {
console.log(
`[VoiceRuntime] resetPlaybackState: cleared ${hadGroups} groups, new gen=${playback.generation}`,
);
}
}
function activateNextPlaybackGroup(): void {
@@ -394,15 +323,9 @@ export function createVoiceRuntime(deps: VoiceRuntimeDeps): VoiceRuntime {
playback.processing = true;
const generation = playback.generation;
console.log(
`[VoiceRuntime] processPlaybackQueue start gen=${generation} activeGroup=${playback.activeGroupId}`,
);
try {
while (playback.activeGroupId) {
if (generation !== playback.generation) {
console.log(
`[VoiceRuntime] processPlaybackQueue abort: generation changed ${generation} -> ${playback.generation}`,
);
return;
}
@@ -415,9 +338,6 @@ export function createVoiceRuntime(deps: VoiceRuntimeDeps): VoiceRuntime {
const nextChunk = group.chunks.get(group.nextChunkToPlay);
if (!nextChunk) {
if (group.finalChunkIndex !== null && group.nextChunkToPlay > group.finalChunkIndex) {
console.log(
`[VoiceRuntime] group=${group.groupId} complete, played=${group.started} chunks=${group.nextChunkToPlay}`,
);
playback.groups.delete(group.groupId);
if (playback.orderedGroupIds[0] === group.groupId) {
playback.orderedGroupIds.shift();
@@ -432,9 +352,6 @@ export function createVoiceRuntime(deps: VoiceRuntimeDeps): VoiceRuntime {
activateNextPlaybackGroup();
continue;
}
console.log(
`[VoiceRuntime] group=${group.groupId} waiting for chunk=${group.nextChunkToPlay} (finalChunkIndex=${group.finalChunkIndex})`,
);
return;
}
@@ -442,39 +359,21 @@ export function createVoiceRuntime(deps: VoiceRuntimeDeps): VoiceRuntime {
if (group.shouldPlay && !group.started && group.isVoiceMode) {
group.started = true;
console.log(
`[VoiceRuntime] group=${group.groupId} first play starting at chunk=${group.nextChunkToPlay}`,
);
api.onAssistantAudioStarted(serverId);
}
const playStart = Date.now();
try {
if (group.shouldPlay) {
await deps.engine.play(nextChunk.source);
console.log(
`[VoiceRuntime] played chunk=${group.nextChunkToPlay} id=${nextChunk.id} took=${Date.now() - playStart}ms`,
);
} else {
console.log(
`[VoiceRuntime] SKIPPED chunk=${group.nextChunkToPlay} id=${nextChunk.id} shouldPlay=false`,
);
}
} catch (error) {
if (generation !== playback.generation) {
console.log(`[VoiceRuntime] play error + generation changed, aborting`);
return;
}
console.error(
`[VoiceRuntime] play error chunk=${group.nextChunkToPlay} took=${Date.now() - playStart}ms:`,
error,
);
console.error(`[VoiceRuntime] play error chunk=${group.nextChunkToPlay}:`, error);
}
if (generation !== playback.generation) {
console.log(
`[VoiceRuntime] post-play generation changed ${generation} -> ${playback.generation}`,
);
return;
}
@@ -491,9 +390,6 @@ export function createVoiceRuntime(deps: VoiceRuntimeDeps): VoiceRuntime {
if (generation === playback.generation) {
playback.processing = false;
}
console.log(
`[VoiceRuntime] processPlaybackQueue exit gen=${generation} currentGen=${playback.generation}`,
);
}
}
@@ -607,12 +503,6 @@ export function createVoiceRuntime(deps: VoiceRuntimeDeps): VoiceRuntime {
}
const base64 = Buffer.from(chunk).toString("base64");
bridgeStats.captureEvents += 1;
bridgeStats.captureBytes += chunk.byteLength;
bridgeStats.uplinkEvents += 1;
bridgeStats.uplinkRawBytes += chunk.byteLength;
bridgeStats.uplinkBase64Chars += base64.length;
flushBridgeStats("uplink");
void activeSession.adapter.sendVoiceAudioChunk(base64, PCM_MIME_TYPE).catch((error) => {
console.error(`[VoiceRuntime#${instanceId}] Failed to send audio chunk:`, error);
@@ -625,8 +515,6 @@ export function createVoiceRuntime(deps: VoiceRuntimeDeps): VoiceRuntime {
state.turnInProgress = false;
state.serverSpeechDetected = false;
state.lastDisplayVolumePublishMs = 0;
state.lastNoServerSpeechLogMs = 0;
state.localAboveThresholdActive = false;
uploader.reset();
resetCaptureTelemetry();
patchSnapshot({ ...INITIAL_SNAPSHOT });
@@ -747,11 +635,6 @@ export function createVoiceRuntime(deps: VoiceRuntimeDeps): VoiceRuntime {
if (!state.snapshot.isVoiceMode || state.snapshot.isMuted) {
return;
}
if (bridgeStats.captureEvents === 0) {
console.log(
`[VoiceRuntime#${instanceId}] firstCapturePcm bytes=${chunk.byteLength} phase=${state.snapshot.phase} transportReady=${state.transportReady}`,
);
}
uploader.pushPcmChunk(chunk);
},
@@ -768,30 +651,6 @@ export function createVoiceRuntime(deps: VoiceRuntimeDeps): VoiceRuntime {
return;
}
const isActive = level > REALTIME_VOICE_VAD_CONFIG.volumeThreshold;
if (isActive && !state.localAboveThresholdActive) {
state.localAboveThresholdActive = true;
console.log(
`[VoiceRuntime#${instanceId}] localSpeechActive level=${level.toFixed(3)} threshold=${REALTIME_VOICE_VAD_CONFIG.volumeThreshold.toFixed(3)} phase=${state.snapshot.phase} transportReady=${state.transportReady}`,
);
}
if (!isActive && state.localAboveThresholdActive) {
state.localAboveThresholdActive = false;
console.log(
`[VoiceRuntime#${instanceId}] localSpeechInactive phase=${state.snapshot.phase} serverSpeaking=${state.serverSpeechDetected}`,
);
}
if (
isActive &&
!state.serverSpeechDetected &&
nowMs - state.lastNoServerSpeechLogMs >= 1500
) {
state.lastNoServerSpeechLogMs = nowMs;
console.log(
`[VoiceRuntime#${instanceId}] localSpeechWithoutServerSpeech level=${level.toFixed(3)} phase=${state.snapshot.phase} turnInProgress=${state.turnInProgress} transportReady=${state.transportReady}`,
);
}
patchTelemetry((prev) => ({
...prev,
isSpeaking: state.serverSpeechDetected,
@@ -806,33 +665,15 @@ export function createVoiceRuntime(deps: VoiceRuntimeDeps): VoiceRuntime {
!state.snapshot.isVoiceMode ||
!payload.isVoiceMode
) {
console.log(
`[VoiceRuntime#${instanceId}] audio_output DROPPED: activeServer=${state.snapshot.activeServerId} serverId=${serverId} isVoiceMode=${state.snapshot.isVoiceMode} payloadVoice=${payload.isVoiceMode}`,
);
return;
}
const groupId = payload.groupId ?? payload.id;
const chunkIndex = payload.chunkIndex ?? 0;
const decoded = decodeAudioChunk(payload.audio);
bridgeStats.outputEvents += 1;
bridgeStats.outputBytes += decoded.byteLength;
bridgeStats.outputGroups += playback.groups.has(groupId) ? 0 : 1;
console.log(
`[VoiceRuntime#${instanceId}] audio_output groupId=${groupId} chunk=${chunkIndex} isLast=${payload.isLastChunk} ` +
`base64Chars=${payload.audio.length} decodedBytes=${decoded.byteLength} format=${payload.format} ` +
`head=${Array.from(decoded.slice(0, 12))
.map((value) => value.toString(16).padStart(2, "0"))
.join(" ")}`,
);
flushBridgeStats("audio_output");
let group = playback.groups.get(groupId);
if (!group) {
const shouldPlay = api.shouldPlayVoiceAudio(serverId);
console.log(
`[VoiceRuntime] new group=${groupId} shouldPlay=${shouldPlay} phase=${state.snapshot.phase} isSpeaking=${state.telemetry.isSpeaking}`,
);
group = {
groupId,
isVoiceMode: payload.isVoiceMode,
@@ -966,7 +807,6 @@ export function createVoiceRuntime(deps: VoiceRuntimeDeps): VoiceRuntime {
async destroy() {
await this.stopVoice().catch(() => undefined);
clearInterval(lagProbe);
await deps.engine.destroy();
listeners.clear();
telemetryListeners.clear();
@@ -1057,9 +897,6 @@ export function createVoiceRuntime(deps: VoiceRuntimeDeps): VoiceRuntime {
return;
}
console.log(
`[VoiceRuntime#${instanceId}] onServerSpeechStateChanged isSpeaking=${isSpeaking} phase=${state.snapshot.phase} volume=${state.telemetry.volume}`,
);
state.serverSpeechDetected = isSpeaking;
state.serverSpeechStartedAt = isSpeaking ? (state.serverSpeechStartedAt ?? Date.now()) : null;
if (isSpeaking) {