jb/paseo
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add expo-two-way-audio module to workspace

Browse Source
This commit is contained in:
Mohamed Boudra
2026-03-14 16:46:35 +07:00
parent fc7369384b
commit cad4110e24
74 changed files with 48803 additions and 354 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/app/src/screens/settings-screen.tsx
+72 -7
View File
@@ -6,11 +6,13 @@ import {
ScrollView,
Alert,
Platform,
Pressable,
} from "react-native";
import { router, useLocalSearchParams } from "expo-router";
import { useFocusEffect } from "@react-navigation/native";
import { useSafeAreaInsets } from "react-native-safe-area-context";
import { StyleSheet, UnistylesRuntime, useUnistyles } from "react-native-unistyles";
import { Buffer } from "buffer";
import { Sun, Moon, Monitor, Globe, Settings, RotateCw, Trash2 } from "lucide-react-native";
import { useAppSettings, type AppSettings } from "@/hooks/use-settings";
import type { HostProfile, HostConnection } from "@/types/host-connection";
@@ -43,6 +45,8 @@ import { useDesktopAppUpdater } from "@/desktop/updates/use-desktop-app-updater"
import { formatVersionWithPrefix } from "@/desktop/updates/desktop-updates";
import { resolveAppVersion } from "@/utils/app-version";
import { settingsStyles } from "@/styles/settings";
import { THINKING_TONE_NATIVE_PCM_BASE64 } from "@/utils/thinking-tone.native-pcm";
import { useVoiceAudioEngineOptional } from "@/contexts/voice-context";
const delay = (ms: number) =>
new Promise<void>((resolve) => {
@@ -484,6 +488,7 @@ function DesktopAppUpdateRow() {
export default function SettingsScreen() {
const { theme } = useUnistyles();
const insets = useSafeAreaInsets();
const voiceAudioEngine = useVoiceAudioEngineOptional();
const params = useLocalSearchParams<{ editHost?: string; serverId?: string }>();
const routeServerId = typeof params.serverId === "string" ? params.serverId.trim() : "";
const { settings, isLoading: settingsLoading, updateSettings } = useAppSettings();
@@ -500,6 +505,8 @@ export default function SettingsScreen() {
const [isRemovingHost, setIsRemovingHost] = useState(false);
const [editingDaemon, setEditingDaemon] = useState<HostProfile | null>(null);
const [isSavingEdit, setIsSavingEdit] = useState(false);
const [isPlaybackTestRunning, setIsPlaybackTestRunning] = useState(false);
const [playbackTestResult, setPlaybackTestResult] = useState<string | null>(null);
const isLoading = settingsLoading;
const isMountedRef = useRef(true);
const lastHandledEditHostRef = useRef<string | null>(null);
@@ -659,6 +666,35 @@ export default function SettingsScreen() {
const restartConfirmationMessage =
"This will restart the daemon. The app will reconnect automatically.";
const handlePlaybackTest = useCallback(async () => {
if (!voiceAudioEngine || isPlaybackTestRunning) {
return;
}
setIsPlaybackTestRunning(true);
setPlaybackTestResult(null);
try {
const bytes = Buffer.from(THINKING_TONE_NATIVE_PCM_BASE64, "base64");
await voiceAudioEngine.initialize();
voiceAudioEngine.stop();
await voiceAudioEngine.play({
type: "audio/pcm;rate=16000;bits=16",
size: bytes.byteLength,
async arrayBuffer() {
return Uint8Array.from(bytes).buffer;
},
});
setPlaybackTestResult("Playback command completed.");
} catch (error) {
const message = error instanceof Error ? error.message : String(error);
console.error("[Settings] Playback test failed", error);
setPlaybackTestResult(`Playback failed: ${message}`);
} finally {
setIsPlaybackTestRunning(false);
}
}, [isPlaybackTestRunning, voiceAudioEngine]);
if (isLoading) {
return (
<View style={styles.loadingContainer}>
@@ -869,6 +905,31 @@ export default function SettingsScreen() {
{isDesktop ? <DesktopPermissionsSection /> : null}
{isDesktop ? <LocalDaemonSection appVersion={appVersion} /> : null}
<View style={settingsStyles.section}>
<Text style={settingsStyles.sectionTitle}>Diagnostics</Text>
<View style={[settingsStyles.card, styles.audioCard]}>
<View style={styles.audioRow}>
<View style={styles.audioRowContent}>
<Text style={styles.audioRowTitle}>Test audio playback</Text>
<Text style={styles.aboutHintText}>
Plays a built-in PCM sample through the current app audio engine.
</Text>
{playbackTestResult ? (
<Text style={styles.aboutHintText}>{playbackTestResult}</Text>
) : null}
</View>
<Button
variant="secondary"
size="sm"
onPress={() => void handlePlaybackTest()}
disabled={!voiceAudioEngine || isPlaybackTestRunning}
>
{isPlaybackTestRunning ? "Playing..." : "Play test"}
</Button>
</View>
</View>
</View>
{/* About */}
<View style={settingsStyles.section}>
<Text style={settingsStyles.sectionTitle}>About</Text>
@@ -1393,18 +1454,22 @@ function DaemonCard({
</View>
) : null}
<Button
variant="ghost"
size="sm"
<Pressable
style={styles.hostSettingsButton}
textStyle={{ fontSize: 0, lineHeight: 0 }}
leftIcon={<Settings size={theme.iconSize.md} color={theme.colors.foregroundMuted} />}
onPress={() => onOpenSettings(daemon)}
testID={`daemon-card-settings-${daemon.serverId}`}
accessibilityLabel={`Open settings for ${daemon.label}`}
accessibilityRole="button"
>
{" "}
</Button>
{({ hovered = false, pressed = false }) => (
<Settings
size={theme.iconSize.md}
color={
hovered || pressed ? theme.colors.foreground : theme.colors.foregroundMuted
}
/>
)}
</Pressable>
</View>
</View>
{connectionError ? <Text style={styles.hostError}>{connectionError}</Text> : null}
packages/app/src/voice/audio-engine-types.ts
-3
View File
@@ -23,7 +23,4 @@ export interface AudioEngine {
stop(): void;
clearQueue(): void;
isPlaying(): boolean;
playLooping(pcm: Uint8Array, gapMs: number): void;
stopLooping(): void;
}
packages/app/src/voice/audio-engine.native.ts
+149 -176
View File
@@ -1,35 +1,31 @@
import { Audio } from "expo-av";
import * as FileSystem from "expo-file-system";
import {
THINKING_TONE_REPEAT_GAP_MS,
} from "@/utils/thinking-tone";
import {
THINKING_TONE_NATIVE_PCM_DURATION_MS,
} from "@/utils/thinking-tone.native-pcm";
import type {
AudioEngine,
AudioEngineCallbacks,
AudioPlaybackSource,
} from "@/voice/audio-engine-types";
// Use expo-av for playback instead of expo-two-way-audio (temporary test)
const USE_EXPO_AV_PLAYBACK = true;
interface QueuedAudio {
audio: AudioPlaybackSource;
resolve: (duration: number) => void;
reject: (error: Error) => void;
}
interface CuePcm {
pcm16k: Uint8Array;
durationMs: number;
}
interface AudioEngineTraceOptions {
traceLabel?: string;
}
let nextAudioEngineInstanceId = 1;
interface BridgeStats {
windowStartedAtMs: number;
captureEvents: number;
captureBytes: number;
playbackEvents: number;
playbackInputBytes: number;
playbackResampledBytes: number;
playbackDurationMs: number;
}
function parsePcmSampleRate(mimeType: string): number | null {
const match = /rate=(\d+)/i.exec(mimeType);
if (!match) {
@@ -39,10 +35,92 @@ function parsePcmSampleRate(mimeType: string): number | null {
return Number.isFinite(rate) && rate > 0 ? rate : null;
}
function resamplePcm16(pcm: Uint8Array, fromRate: number, toRate: number): Uint8Array {
if (fromRate === toRate) {
return pcm;
}
const inputSamples = Math.floor(pcm.length / 2);
const outputSamples = Math.floor((inputSamples * toRate) / fromRate);
const out = new Uint8Array(outputSamples * 2);
const ratio = fromRate / toRate;
const readInt16 = (sampleIndex: number): number => {
const i = sampleIndex * 2;
if (i + 1 >= pcm.length) {
return 0;
}
const lo = pcm[i]!;
const hi = pcm[i + 1]!;
let value = (hi << 8) | lo;
if (value & 0x8000) {
value = value - 0x10000;
}
return value;
};
const writeInt16 = (sampleIndex: number, value: number): void => {
const clamped = Math.max(-32768, Math.min(32767, Math.round(value)));
const i = sampleIndex * 2;
out[i] = clamped & 0xff;
out[i + 1] = (clamped >> 8) & 0xff;
};
for (let i = 0; i < outputSamples; i++) {
const srcPos = i * ratio;
const i0 = Math.floor(srcPos);
const frac = srcPos - i0;
const s0 = readInt16(i0);
const s1 = readInt16(Math.min(inputSamples - 1, i0 + 1));
writeInt16(i, s0 + (s1 - s0) * frac);
}
return out;
}
export function createAudioEngine(
callbacks: AudioEngineCallbacks,
_options?: AudioEngineTraceOptions
): AudioEngine {
const native = require("@getpaseo/expo-two-way-audio");
const instanceId = nextAudioEngineInstanceId++;
const bridgeStats: BridgeStats = {
windowStartedAtMs: Date.now(),
captureEvents: 0,
captureBytes: 0,
playbackEvents: 0,
playbackInputBytes: 0,
playbackResampledBytes: 0,
playbackDurationMs: 0,
};
const toHexPreview = (bytes: Uint8Array, count = 12): string =>
Array.from(bytes.slice(0, count))
.map((value) => value.toString(16).padStart(2, "0"))
.join(" ");
const maybeFlushBridgeStats = (reason: string): void => {
const now = Date.now();
const elapsedMs = now - bridgeStats.windowStartedAtMs;
if (elapsedMs < 1000) {
return;
}
console.log(
`[AudioEngine.native#${instanceId}][bridge] ${reason} ` +
`capture=${bridgeStats.captureEvents}ev/${bridgeStats.captureBytes}B ` +
`play=${bridgeStats.playbackEvents}ev/${bridgeStats.playbackInputBytes}B->${bridgeStats.playbackResampledBytes}B ` +
`playMs=${bridgeStats.playbackDurationMs.toFixed(1)} ` +
`windowMs=${elapsedMs}`
);
bridgeStats.windowStartedAtMs = now;
bridgeStats.captureEvents = 0;
bridgeStats.captureBytes = 0;
bridgeStats.playbackEvents = 0;
bridgeStats.playbackInputBytes = 0;
bridgeStats.playbackResampledBytes = 0;
bridgeStats.playbackDurationMs = 0;
};
const refs: {
initialized: boolean;
captureActive: boolean;
@@ -55,14 +133,7 @@ export function createAudioEngine(
reject: (error: Error) => void;
settled: boolean;
} | null;
looping: {
active: boolean;
token: number;
timeout: ReturnType<typeof setTimeout> | null;
};
thinkingTone: CuePcm | null;
destroyed: boolean;
mockVolumeInterval: ReturnType<typeof setInterval> | null;
} = {
initialized: false,
captureActive: false,
@@ -71,62 +142,53 @@ export function createAudioEngine(
processingQueue: false,
playbackTimeout: null,
activePlayback: null,
looping: {
active: false,
token: 0,
timeout: null,
},
thinkingTone: null,
destroyed: false,
mockVolumeInterval: null,
};
// Only subscribe to native events if not mocking
let microphoneSubscription: { remove(): void };
let volumeSubscription: { remove(): void };
const microphoneSubscription = native.addExpoTwoWayAudioEventListener(
"onMicrophoneData",
(event: any) => {
if (!refs.captureActive || refs.muted) {
return;
}
const pcm = event.data as Uint8Array;
bridgeStats.captureEvents += 1;
bridgeStats.captureBytes += pcm.byteLength;
maybeFlushBridgeStats("capture");
callbacks.onCaptureData(pcm);
}
);
const volumeSubscription = native.addExpoTwoWayAudioEventListener(
"onInputVolumeLevelData",
(event: any) => {
if (!refs.captureActive) {
return;
}
const level = refs.muted ? 0 : event.data;
callbacks.onVolumeLevel(level);
}
);
if (MOCK_MODE) {
microphoneSubscription = { remove() {} };
volumeSubscription = { remove() {} };
} else {
const native = require("@boudra/expo-two-way-audio");
microphoneSubscription = native.addExpoTwoWayAudioEventListener(
"onMicrophoneData",
(event: any) => {
if (!refs.captureActive || refs.muted) {
return;
}
callbacks.onCaptureData(event.data);
}
);
volumeSubscription = native.addExpoTwoWayAudioEventListener(
"onInputVolumeLevelData",
(event: any) => {
if (!refs.captureActive) {
return;
}
const level = refs.muted ? 0 : event.data;
callbacks.onVolumeLevel(level);
}
);
}
const outputVolumeSubscription = native.addExpoTwoWayAudioEventListener(
"onOutputVolumeLevelData",
(event: any) => {
console.log(`[AudioEngine.native#${instanceId}] outputVolume=${event.data}`);
}
);
async function ensureInitialized(): Promise<void> {
if (refs.initialized) {
return;
}
if (!MOCK_MODE) {
const native = require("@boudra/expo-two-way-audio");
await native.initialize();
const success = await native.initialize();
if (!success) {
throw new Error("expo-two-way-audio: native initialize() returned false");
}
console.log(`[AudioEngine.native#${instanceId}] initialized successfully`);
refs.initialized = true;
}
async function ensureMicrophonePermission(): Promise<void> {
if (MOCK_MODE) {
return;
}
const native = require("@boudra/expo-two-way-audio");
let permission = await native.getMicrophonePermissionsAsync().catch(() => null);
if (!permission?.granted) {
permission = await native.requestMicrophonePermissionsAsync().catch(() => null);
@@ -138,15 +200,6 @@ export function createAudioEngine(
}
}
async function ensureThinkingTone(): Promise<CuePcm> {
if (refs.thinkingTone) {
return refs.thinkingTone;
}
const durationMs = THINKING_TONE_NATIVE_PCM_DURATION_MS;
refs.thinkingTone = { pcm16k: new Uint8Array(0), durationMs };
return refs.thinkingTone;
}
function clearPlaybackTimeout(): void {
if (refs.playbackTimeout) {
clearTimeout(refs.playbackTimeout);
@@ -164,15 +217,24 @@ export function createAudioEngine(
const arrayBuffer = await audio.arrayBuffer();
const pcm = new Uint8Array(arrayBuffer);
const inputRate = parsePcmSampleRate(audio.type || "") ?? 24000;
const durationSec = pcm.length / 2 / inputRate;
console.log(`[AudioEngine.native] playPCMData: MOCK_MODE=${MOCK_MODE} inputRate=${inputRate} inputBytes=${pcm.length} durationSec=${durationSec.toFixed(3)}`);
// Native AudioEngine expects 16kHz PCM16
const pcm16k = resamplePcm16(pcm, inputRate, 16000);
const durationSec = pcm16k.length / 2 / 16000;
bridgeStats.playbackEvents += 1;
bridgeStats.playbackInputBytes += pcm.length;
bridgeStats.playbackResampledBytes += pcm16k.length;
bridgeStats.playbackDurationMs += durationSec * 1000;
if (!MOCK_MODE) {
const native = require("@boudra/expo-two-way-audio");
native.resumePlayback();
native.playPCMData(pcm);
}
console.log(
`[AudioEngine.native#${instanceId}] playPCMData: inputRate=${inputRate} inputBytes=${pcm.length} ` +
`resampled=${pcm16k.length} durationSec=${durationSec.toFixed(3)} ` +
`pcmHead=${toHexPreview(pcm)} resampledHead=${toHexPreview(pcm16k)}`
);
maybeFlushBridgeStats("play");
native.resumePlayback();
native.playPCMData(pcm16k);
clearPlaybackTimeout();
refs.playbackTimeout = setTimeout(() => {
@@ -215,27 +277,9 @@ export function createAudioEngine(
refs.processingQueue = false;
}
function nativeStopPlayback(): void {
if (!MOCK_MODE) {
const native = require("@boudra/expo-two-way-audio");
native.stopPlayback();
}
}
function stopLooping(): void {
refs.looping.active = false;
refs.looping.token += 1;
if (refs.looping.timeout) {
clearTimeout(refs.looping.timeout);
refs.looping.timeout = null;
}
nativeStopPlayback();
}
return {
async initialize() {
await ensureInitialized();
await ensureThinkingTone();
},
async destroy() {
@@ -243,30 +287,22 @@ export function createAudioEngine(
return;
}
refs.destroyed = true;
stopLooping();
this.stop();
this.clearQueue();
if (refs.captureActive) {
if (!MOCK_MODE) {
const native = require("@boudra/expo-two-way-audio");
native.toggleRecording(false);
}
if (refs.mockVolumeInterval) {
clearInterval(refs.mockVolumeInterval);
refs.mockVolumeInterval = null;
}
native.toggleRecording(false);
refs.captureActive = false;
}
clearPlaybackTimeout();
refs.muted = false;
callbacks.onVolumeLevel(0);
if (refs.initialized && !MOCK_MODE) {
const native = require("@boudra/expo-two-way-audio");
if (refs.initialized) {
native.tearDown();
refs.initialized = false;
}
microphoneSubscription.remove();
volumeSubscription.remove();
outputVolumeSubscription.remove();
},
async startCapture() {
@@ -277,22 +313,7 @@ export function createAudioEngine(
try {
await ensureMicrophonePermission();
await ensureInitialized();
if (MOCK_MODE) {
// Emit fake volume levels and empty PCM data periodically
refs.mockVolumeInterval = setInterval(() => {
if (!refs.captureActive) {
return;
}
const fakeVolume = refs.muted ? 0 : 0.15 + Math.random() * 0.5;
callbacks.onVolumeLevel(fakeVolume);
// Send empty PCM chunk to keep the pipeline alive
callbacks.onCaptureData(new Uint8Array(3200));
}, 100);
} else {
const native = require("@boudra/expo-two-way-audio");
native.toggleRecording(true);
}
native.toggleRecording(true);
refs.captureActive = true;
} catch (error) {
const wrapped = error instanceof Error ? error : new Error(String(error));
@@ -303,15 +324,7 @@ export function createAudioEngine(
async stopCapture() {
if (refs.captureActive) {
if (MOCK_MODE) {
if (refs.mockVolumeInterval) {
clearInterval(refs.mockVolumeInterval);
refs.mockVolumeInterval = null;
}
} else {
const native = require("@boudra/expo-two-way-audio");
native.toggleRecording(false);
}
native.toggleRecording(false);
}
refs.captureActive = false;
refs.muted = false;
@@ -340,7 +353,7 @@ export function createAudioEngine(
},
stop() {
nativeStopPlayback();
native.stopPlayback();
clearPlaybackTimeout();
const active = refs.activePlayback;
refs.activePlayback = null;
@@ -360,45 +373,5 @@ export function createAudioEngine(
isPlaying() {
return refs.activePlayback !== null;
},
playLooping(audio, gapMs) {
if (refs.looping.active) {
return;
}
refs.looping.active = true;
const token = refs.looping.token + 1;
refs.looping.token = token;
void (async () => {
try {
await ensureInitialized();
const cue = await ensureThinkingTone();
const loop = () => {
if (!refs.looping.active || refs.looping.token !== token) {
return;
}
if (!MOCK_MODE) {
const native = require("@boudra/expo-two-way-audio");
native.resumePlayback();
native.playPCMData(cue.pcm16k);
}
refs.looping.timeout = setTimeout(
loop,
cue.durationMs + (gapMs || THINKING_TONE_REPEAT_GAP_MS)
);
};
loop();
} catch (error) {
callbacks.onError?.(
error instanceof Error ? error : new Error(String(error))
);
}
})();
},
stopLooping,
};
}
packages/app/src/voice/audio-engine.web.ts
-127
View File
@@ -1,8 +1,4 @@
import { getTauri } from "@/utils/tauri";
import {
loadThinkingToneArrayBuffer,
THINKING_TONE_REPEAT_GAP_MS,
} from "@/utils/thinking-tone";
import type {
AudioEngine,
AudioEngineCallbacks,
@@ -117,16 +113,6 @@ export function createAudioEngine(
reject: (error: Error) => void;
settled: boolean;
} | null;
looping: {
token: number;
source: AudioBufferSourceNode | null;
gain: GainNode | null;
timeout: ReturnType<typeof setTimeout> | null;
};
thinkingTone: {
bytes: Uint8Array | null;
buffer: AudioBuffer | null;
};
} = {
playbackContext: null,
captureContext: null,
@@ -139,16 +125,6 @@ export function createAudioEngine(
queue: [],
processingQueue: false,
activePlayback: null,
looping: {
token: 0,
source: null,
gain: null,
timeout: null,
},
thinkingTone: {
bytes: null,
buffer: null,
},
};
async function ensurePlaybackContext(): Promise<AudioContext> {
@@ -193,48 +169,6 @@ export function createAudioEngine(
return context;
}
async function ensureThinkingToneBytes(): Promise<Uint8Array> {
if (refs.thinkingTone.bytes) {
return refs.thinkingTone.bytes;
}
const wav = await loadThinkingToneArrayBuffer();
refs.thinkingTone.bytes = new Uint8Array(wav);
return refs.thinkingTone.bytes;
}
async function ensureThinkingToneBuffer(): Promise<AudioBuffer> {
if (refs.thinkingTone.buffer) {
return refs.thinkingTone.buffer;
}
const context = await ensurePlaybackContext();
const wav = await loadThinkingToneArrayBuffer();
const audioBuffer = await decodeAudioData(context, wav);
refs.thinkingTone.buffer = audioBuffer;
return audioBuffer;
}
function stopLooping(): void {
refs.looping.token += 1;
if (refs.looping.timeout) {
clearTimeout(refs.looping.timeout);
refs.looping.timeout = null;
}
if (refs.looping.source) {
try {
refs.looping.source.onended = null;
refs.looping.source.stop();
} catch {
// Ignore best-effort stop errors.
}
refs.looping.source.disconnect();
refs.looping.source = null;
}
if (refs.looping.gain) {
refs.looping.gain.disconnect();
refs.looping.gain = null;
}
}
async function playAudio(audio: AudioPlaybackSource): Promise<number> {
const context = await ensurePlaybackContext();
const arrayBuffer = await audio.arrayBuffer();
@@ -335,18 +269,15 @@ export function createAudioEngine(
return {
async initialize() {
await ensurePlaybackContext();
await Promise.all([ensureThinkingToneBytes(), ensureThinkingToneBuffer()]);
},
async destroy() {
stopLooping();
this.stop();
this.clearQueue();
await stopCapture();
const playbackContext = refs.playbackContext;
refs.playbackContext = null;
refs.thinkingTone.buffer = null;
if (playbackContext && playbackContext.state !== "closed") {
await playbackContext.close().catch(() => undefined);
}
@@ -486,63 +417,5 @@ export function createAudioEngine(
isPlaying() {
return refs.activePlayback !== null;
},
playLooping(audio, gapMs) {
if (refs.looping.source || refs.looping.timeout) {
return;
}
const token = refs.looping.token + 1;
refs.looping.token = token;
void (async () => {
try {
const context = await ensurePlaybackContext();
const audioBuffer =
audio.byteLength > 0
? pcm16LeToAudioBuffer(context, audio, 16000)
: await ensureThinkingToneBuffer();
if (refs.looping.token !== token) {
return;
}
const gain = context.createGain();
gain.gain.value = 1;
gain.connect(context.destination);
refs.looping.gain = gain;
const playOnce = () => {
if (refs.looping.token !== token) {
return;
}
const source = context.createBufferSource();
source.buffer = audioBuffer;
source.connect(gain);
refs.looping.source = source;
source.onended = () => {
if (refs.looping.source === source) {
refs.looping.source = null;
}
if (refs.looping.token !== token) {
return;
}
refs.looping.timeout = setTimeout(
playOnce,
gapMs || THINKING_TONE_REPEAT_GAP_MS
);
};
source.start();
};
playOnce();
} catch (error) {
callbacks.onError?.(
error instanceof Error ? error : new Error(String(error))
);
}
})();
},
stopLooping,
};
}
packages/app/src/voice/voice-runtime.test.ts
+3 -7
View File
@@ -20,8 +20,6 @@ function createAudioEngineMock(): AudioEngine {
stop: vi.fn(),
clearQueue: vi.fn(),
isPlaying: vi.fn().mockReturnValue(false),
playLooping: vi.fn(),
stopLooping: vi.fn(),
};
}
@@ -137,7 +135,6 @@ describe("voice runtime", () => {
runtime.onAssistantAudioStarted("server-1");
expect(runtime.getSnapshot().phase).toBe("playing");
expect(engine.stopLooping).toHaveBeenCalled();
expect(adapter.setAssistantAudioPlaying).toHaveBeenCalledWith(true);
});
@@ -256,7 +253,7 @@ describe("voice runtime", () => {
runtime.onAssistantAudioFinished("server-1");
expect(runtime.getSnapshot().phase).toBe("waiting");
expect(engine.playLooping).toHaveBeenCalled();
expect(engine.play).toHaveBeenCalled();
});
it("starts the thinking tone when an agent turn begins before playback", async () => {
@@ -268,7 +265,7 @@ describe("voice runtime", () => {
runtime.onTurnEvent("server-1", "agent-1", "turn_started");
expect(runtime.getSnapshot().phase).toBe("waiting");
expect(engine.playLooping).toHaveBeenCalled();
expect(engine.play).toHaveBeenCalled();
});
it("returns to listening after assistant playback once the turn is complete", async () => {
@@ -283,8 +280,7 @@ describe("voice runtime", () => {
runtime.onAssistantAudioFinished("server-1");
expect(runtime.getSnapshot().phase).toBe("listening");
expect(engine.playLooping).toHaveBeenCalledTimes(1);
expect(engine.stopLooping).toHaveBeenCalled();
expect(engine.play).toHaveBeenCalled();
});
it("keeps local volume alone non-authoritative for playback interruption", async () => {
packages/app/src/voice/voice-runtime.ts
+154 -7
View File
@@ -7,6 +7,10 @@ import { resolveVoiceUnavailableMessage } from "@/utils/server-info-capabilities
import type { DaemonServerInfo } from "@/stores/session-store";
import type { AudioEngine } from "@/voice/audio-engine-types";
import { REALTIME_VOICE_VAD_CONFIG } from "@/voice/realtime-voice-config";
import {
THINKING_TONE_NATIVE_PCM_BASE64,
THINKING_TONE_NATIVE_PCM_DURATION_MS,
} from "@/utils/thinking-tone.native-pcm";
const PCM_MIME_TYPE = "audio/pcm;rate=16000;bits=16";
const KEEP_AWAKE_TAG = "paseo:voice";
@@ -114,6 +118,26 @@ interface RuntimePlaybackState {
generation: number;
}
interface CueState {
active: boolean;
token: number;
timeout: ReturnType<typeof setTimeout> | null;
playing: boolean;
}
interface RealtimeBridgeStats {
windowStartedAtMs: number;
captureEvents: number;
captureBytes: number;
uplinkEvents: number;
uplinkRawBytes: number;
uplinkBase64Chars: number;
outputEvents: number;
outputBytes: number;
outputGroups: number;
jsLagMaxMs: number;
}
const INITIAL_SNAPSHOT: VoiceRuntimeSnapshot = {
phase: "disabled",
isVoiceMode: false,
@@ -130,6 +154,8 @@ const INITIAL_TELEMETRY: VoiceRuntimeTelemetrySnapshot = {
segmentDuration: 0,
};
let nextVoiceRuntimeInstanceId = 1;
function snapshotsEqual(
left: VoiceRuntimeSnapshot,
right: VoiceRuntimeSnapshot
@@ -180,6 +206,7 @@ export interface VoiceRuntime {
}
export function createVoiceRuntime(deps: VoiceRuntimeDeps): VoiceRuntime {
const instanceId = nextVoiceRuntimeInstanceId++;
const listeners = new Set<() => void>();
const telemetryListeners = new Set<() => void>();
const sessions = new Map<string, RuntimeSessionState>();
@@ -201,6 +228,69 @@ export function createVoiceRuntime(deps: VoiceRuntimeDeps): VoiceRuntime {
processing: false,
generation: 0,
};
const bridgeStats: RealtimeBridgeStats = {
windowStartedAtMs: Date.now(),
captureEvents: 0,
captureBytes: 0,
uplinkEvents: 0,
uplinkRawBytes: 0,
uplinkBase64Chars: 0,
outputEvents: 0,
outputBytes: 0,
outputGroups: 0,
jsLagMaxMs: 0,
};
const cue: CueState = {
active: false,
token: 0,
timeout: null,
playing: false,
};
const cuePcm16 = Uint8Array.from(Buffer.from(THINKING_TONE_NATIVE_PCM_BASE64, "base64"));
const cueSource = {
size: cuePcm16.byteLength,
type: "audio/pcm;rate=16000;bits=16",
async arrayBuffer() {
return cuePcm16.buffer.slice(
cuePcm16.byteOffset,
cuePcm16.byteOffset + cuePcm16.byteLength
);
},
};
let lagProbeLastMs = Date.now();
const lagProbe = setInterval(() => {
const now = Date.now();
const lagMs = Math.max(0, now - lagProbeLastMs - 100);
lagProbeLastMs = now;
if (lagMs > bridgeStats.jsLagMaxMs) {
bridgeStats.jsLagMaxMs = lagMs;
}
}, 100);
function flushBridgeStats(reason: string): void {
const now = Date.now();
const elapsedMs = now - bridgeStats.windowStartedAtMs;
if (elapsedMs < 1000) {
return;
}
console.log(
`[VoiceRuntime#${instanceId}][bridge] ${reason} ` +
`capture=${bridgeStats.captureEvents}ev/${bridgeStats.captureBytes}B ` +
`uplink=${bridgeStats.uplinkEvents}ev/${bridgeStats.uplinkRawBytes}B/${bridgeStats.uplinkBase64Chars}c ` +
`output=${bridgeStats.outputEvents}ev/${bridgeStats.outputBytes}B groups=${bridgeStats.outputGroups} ` +
`jsLagMaxMs=${bridgeStats.jsLagMaxMs} windowMs=${elapsedMs}`
);
bridgeStats.windowStartedAtMs = now;
bridgeStats.captureEvents = 0;
bridgeStats.captureBytes = 0;
bridgeStats.uplinkEvents = 0;
bridgeStats.uplinkRawBytes = 0;
bridgeStats.uplinkBase64Chars = 0;
bridgeStats.outputEvents = 0;
bridgeStats.outputBytes = 0;
bridgeStats.outputGroups = 0;
bridgeStats.jsLagMaxMs = 0;
}
function emit(): void {
for (const listener of listeners) {
@@ -441,7 +531,17 @@ export function createVoiceRuntime(deps: VoiceRuntimeDeps): VoiceRuntime {
}
function stopCue(): void {
deps.engine.stopLooping();
const hadActive = cue.active || cue.timeout !== null || cue.playing;
cue.active = false;
cue.token += 1;
if (cue.timeout) {
clearTimeout(cue.timeout);
cue.timeout = null;
}
if (hadActive) {
deps.engine.stop();
deps.engine.clearQueue();
}
}
function resetCaptureTelemetry(): void {
@@ -455,7 +555,36 @@ export function createVoiceRuntime(deps: VoiceRuntimeDeps): VoiceRuntime {
stopCue();
return;
}
deps.engine.playLooping(new Uint8Array(0), THINKING_TONE_REPEAT_GAP_MS);
if (cue.active) {
return;
}
cue.active = true;
cue.token += 1;
const token = cue.token;
const playNext = () => {
if (!cue.active || cue.token !== token) {
return;
}
cue.playing = true;
void deps.engine
.play(cueSource)
.catch((error) => {
if (cue.token !== token) {
return;
}
console.warn(`[VoiceRuntime#${instanceId}] Cue playback failed:`, error);
})
.finally(() => {
cue.playing = false;
if (!cue.active || cue.token !== token) {
return;
}
cue.timeout = setTimeout(playNext, THINKING_TONE_NATIVE_PCM_DURATION_MS + THINKING_TONE_REPEAT_GAP_MS);
});
};
playNext();
}
const uploader: ContinuousVoiceUploader = {
@@ -471,10 +600,18 @@ export function createVoiceRuntime(deps: VoiceRuntimeDeps): VoiceRuntime {
return;
}
const base64 = Buffer.from(chunk).toString("base64");
bridgeStats.captureEvents += 1;
bridgeStats.captureBytes += chunk.byteLength;
bridgeStats.uplinkEvents += 1;
bridgeStats.uplinkRawBytes += chunk.byteLength;
bridgeStats.uplinkBase64Chars += base64.length;
flushBridgeStats("uplink");
void activeSession.adapter
.sendVoiceAudioChunk(Buffer.from(chunk).toString("base64"), PCM_MIME_TYPE)
.sendVoiceAudioChunk(base64, PCM_MIME_TYPE)
.catch((error) => {
console.error("[VoiceRuntime] Failed to send audio chunk:", error);
console.error(`[VoiceRuntime#${instanceId}] Failed to send audio chunk:`, error);
});
},
};
@@ -651,13 +788,22 @@ export function createVoiceRuntime(deps: VoiceRuntimeDeps): VoiceRuntime {
!state.snapshot.isVoiceMode ||
!payload.isVoiceMode
) {
console.log(`[VoiceRuntime] audio_output DROPPED: activeServer=${state.snapshot.activeServerId} serverId=${serverId} isVoiceMode=${state.snapshot.isVoiceMode} payloadVoice=${payload.isVoiceMode}`);
console.log(`[VoiceRuntime#${instanceId}] audio_output DROPPED: activeServer=${state.snapshot.activeServerId} serverId=${serverId} isVoiceMode=${state.snapshot.isVoiceMode} payloadVoice=${payload.isVoiceMode}`);
return;
}
const groupId = payload.groupId ?? payload.id;
const chunkIndex = payload.chunkIndex ?? 0;
console.log(`[VoiceRuntime] audio_output groupId=${groupId} chunk=${chunkIndex} isLast=${payload.isLastChunk} size=${payload.audio.length} format=${payload.format}`);
const decoded = decodeAudioChunk(payload.audio);
bridgeStats.outputEvents += 1;
bridgeStats.outputBytes += decoded.byteLength;
bridgeStats.outputGroups += playback.groups.has(groupId) ? 0 : 1;
console.log(
`[VoiceRuntime#${instanceId}] audio_output groupId=${groupId} chunk=${chunkIndex} isLast=${payload.isLastChunk} ` +
`base64Chars=${payload.audio.length} decodedBytes=${decoded.byteLength} format=${payload.format} ` +
`head=${Array.from(decoded.slice(0, 12)).map((value) => value.toString(16).padStart(2, "0")).join(" ")}`
);
flushBridgeStats("audio_output");
let group = playback.groups.get(groupId);
if (!group) {
@@ -683,7 +829,7 @@ export function createVoiceRuntime(deps: VoiceRuntimeDeps): VoiceRuntime {
group.chunks.set(chunkIndex, {
id: payload.id,
chunkIndex,
source: toPlaybackSource(decodeAudioChunk(payload.audio), payload.format),
source: toPlaybackSource(decoded, payload.format),
});
if (payload.isLastChunk) {
group.finalChunkIndex = chunkIndex;
@@ -796,6 +942,7 @@ export function createVoiceRuntime(deps: VoiceRuntimeDeps): VoiceRuntime {
async destroy() {
await this.stopVoice().catch(() => undefined);
clearInterval(lagProbe);
await deps.engine.destroy();
listeners.clear();
telemetryListeners.clear();