feat: add expo-two-way-audio module to workspace

2026-03-14 16:46:35 +07:00
parent fc7369384b
commit cad4110e24
74 changed files with 48803 additions and 354 deletions
@@ -0,0 +1,35 @@
+# Learn more https://docs.github.com/en/get-started/getting-started-with-git/ignoring-files
+
+# dependencies
+node_modules/
+
+# Expo
+.expo/
+dist/
+web-build/
+
+# Native
+*.orig.*
+*.jks
+*.p8
+*.p12
+*.key
+*.mobileprovision
+
+# Metro
+.metro-health-check*
+
+# debug
+npm-debug.*
+yarn-debug.*
+yarn-error.*
+
+# macOS
+.DS_Store
+*.pem
+
+# local env files
+.env*.local
+
+# typescript
+*.tsbuildinfo
@@ -0,0 +1,270 @@
+// EventTarget polyfill is required for the Flow SDK to work in React Native
+import "event-target-polyfill";
+import { Buffer } from "buffer";
+import { Button, StyleSheet, Text, View } from "react-native";
+import { Platform } from "react-native";
+
+import {
+  type MicrophoneDataCallback,
+  type VolumeLevelCallback,
+  getMicrophoneModeIOS,
+  initialize,
+  playPCMData,
+  toggleRecording,
+  useExpoTwoWayAudioEventListener,
+  useIsRecording,
+  useMicrophonePermissions,
+} from "@speechmatics/expo-two-way-audio";
+
+import {
+  FlowProvider,
+  type RecognitionAlternative,
+  useFlow,
+  useFlowEventListener,
+} from "@speechmatics/flow-client-react";
+import { useCallback, useEffect, useState, useSyncExternalStore } from "react";
+import { useSharedValue } from "react-native-reanimated";
+import { getFlowAPIJwt } from "./auth";
+import VolumeDisplay from "./volume-display";
+
+export default function App() {
+  const [micPermission, requestMicPermission] = useMicrophonePermissions();
+  console.log(micPermission);
+
+  if (!micPermission?.granted) {
+    return (
+      <View style={styles.container}>
+        <Text>Mic permission: {micPermission?.status}</Text>
+        <Button
+          title={micPermission?.canAskAgain ? "Request permission" : "Cannot request permissions"}
+          disabled={!micPermission?.canAskAgain}
+          onPress={requestMicPermission}
+        />
+      </View>
+    );
+  }
+
+  return (
+    <FlowProvider appId="flow-api-example" websocketBinaryType="arraybuffer">
+      <FlowTest />
+    </FlowProvider>
+  );
+}
+
+export function FlowTest() {
+  const [isConnectedToFlow, setIsConnectedToFlow] = useState(false);
+  const [isConnectingToFlow, setIsConnectingToFlow] = useState(false);
+  const [audioInitialized, setAudioInitialized] = useState(false);
+  const inputVolumeLevel = useSharedValue(0.0);
+  const outputVolumeLevel = useSharedValue(0.0);
+  const isRecording = useIsRecording();
+  const { startConversation, endConversation, sendAudio } = useFlow();
+
+  // Initialize Expo Two Way Audio
+  useEffect(() => {
+    const initializeAudio = async () => {
+      await initialize();
+      setAudioInitialized(true);
+    };
+
+    initializeAudio();
+  }, []);
+
+  // Setup a handler for the "message" event from Flow API
+  useFlowEventListener("message", ({ data }) => {
+    // For the purpose of this example, just print the content of the transcript
+    if (data.message === "AddTranscript") {
+      for (const result of data.results as {
+        alternatives: Array<RecognitionAlternative>;
+      }[]) {
+        for (const alternative of result.alternatives as Array<RecognitionAlternative>) {
+          console.log(alternative.content);
+        }
+      }
+    }
+  });
+
+  // Setup a handler for the "agentAudio" event from Flow API
+  useFlowEventListener("agentAudio", (audio) => {
+    // Even though Int16Array is a more natural representation for PCM16_sle,
+    // Expo Modules API uses a convertible type for arrays of bytes and it needs Uint8Array in the JS side.
+    // This is converted to a `Data` type in Swift and to a `kotlin.ByteArray` in Kotlin.
+    // More info here: https://docs.expo.dev/modules/module-api/#convertibles
+    // For this reason, the Expo Two Way Audio library requires a Uint8Array argument for the `playPCMData` function.
+    const byteArray = new Uint8Array(audio.data.buffer);
+    playPCMData(byteArray);
+  });
+
+  // Setup a handler for the "onMicrophoneData" event from Expo Two Way Audio module
+  useExpoTwoWayAudioEventListener(
+    "onMicrophoneData",
+    useCallback<MicrophoneDataCallback>(
+      (event) => {
+        // We send the audio bytes to the Flow API
+        sendAudio(event.data.buffer);
+      },
+      [sendAudio],
+    ),
+  );
+
+  // Setup a handler for the "onInputVolumeLevelData" event from Expo Two Way Audio module.
+  // This event is triggered when the input volume level changes.
+  useExpoTwoWayAudioEventListener(
+    "onInputVolumeLevelData",
+    useCallback<VolumeLevelCallback>(
+      (event) => {
+        // We update the react-native-reanimated shared value for the input volume (microphone).
+        // This allow us to create animations for the microphone volume based on this shared value.
+        inputVolumeLevel.value = event.data;
+      },
+      [inputVolumeLevel],
+    ),
+  );
+
+  // Setup a handler for the "onOutputVolumeLevelData" event from Expo Two Way Audio module.
+  // This event is triggered when the output volume level changes.
+  useExpoTwoWayAudioEventListener(
+    "onOutputVolumeLevelData",
+    useCallback<VolumeLevelCallback>(
+      (event) => {
+        // We update the react-native-reanimated shared value for the output volume (speaker).
+        // This allow us to create animations for the speaker volume based on this shared value.
+        outputVolumeLevel.value = event.data;
+      },
+      [outputVolumeLevel],
+    ),
+  );
+
+  // Handle clicks to the 'Connect/Disconnect' button
+  const handleToggleConnect = useCallback(async () => {
+    setIsConnectingToFlow(true);
+    if (isConnectedToFlow) {
+      endConversation();
+      setIsConnectedToFlow(false);
+    } else {
+      let jwt;
+      try {
+        // WARNING: This function is used for the purpose of this example only.
+        // In a real-world scenario, you should not expose your API key in your client-side code.
+        // Instead, you should create a server-side endpoint that generates the JWT for you.
+        jwt = await getFlowAPIJwt(
+          // EXPO_PUBLIC_SPEECHMATICS_API_KEY could be set in your .env file
+          process.env.EXPO_PUBLIC_SPEECHMATICS_API_KEY,
+        );
+      } catch (error) {
+        console.error("Failed to get JWT:", error);
+        setIsConnectingToFlow(false);
+        return;
+      }
+
+      try {
+        await startConversation(jwt, {
+          config: {
+            template_id: "flow-service-assistant-humphrey",
+            template_variables: {
+              timezone: "Europe/London",
+            },
+          },
+          // `audioFormat` is optional. The value below is the default:
+          audioFormat: {
+            type: "raw",
+            encoding: "pcm_s16le", // this can also be set to 'pcm_f32le' for 32-bit Float
+            sample_rate: 16000,
+          },
+        });
+        setIsConnectedToFlow(true);
+      } catch (error) {
+        console.error("Failed to start conversation:", error);
+      }
+    }
+    setIsConnectingToFlow(false);
+  }, [isConnectedToFlow, startConversation, endConversation]);
+
+  // Handle clicks to the 'Mute/Unmute' button
+  const handleToggleMute = useCallback(() => {
+    toggleRecording(!isRecording);
+  }, [isRecording]);
+
+  return (
+    <View style={styles.container}>
+      <View style={styles.VolumeDisplayContainer}>
+        <View style={styles.volumeDisplay}>
+          <VolumeDisplay
+            volumeLevel={outputVolumeLevel}
+            color="#7ECFBD"
+            minSize={130}
+            maxSize={180}
+          />
+        </View>
+        <View style={styles.volumeDisplay}>
+          <VolumeDisplay
+            volumeLevel={inputVolumeLevel}
+            color="#4CBBA5"
+            minSize={70}
+            maxSize={120}
+          />
+        </View>
+      </View>
+      <View>
+        <Text>
+          {isConnectedToFlow
+            ? isRecording
+              ? "I'm ready to listen. Try saying something!"
+              : "Muted. Unmute to start a conversation"
+            : isConnectingToFlow
+              ? "Connecting..."
+              : "Disconnected"}
+        </Text>
+      </View>
+      <View style={styles.bottomBar}>
+        <View style={styles.buttonContainer}>
+          <Button
+            title={isConnectedToFlow ? "Disconnect" : "Connect"}
+            disabled={isConnectingToFlow || !audioInitialized}
+            onPress={handleToggleConnect}
+          />
+
+          <Button
+            title={isRecording ? "Mute" : "Unmute"}
+            disabled={!isConnectedToFlow || !audioInitialized}
+            onPress={handleToggleMute}
+          />
+        </View>
+      </View>
+    </View>
+  );
+}
+
+const styles = StyleSheet.create({
+  container: {
+    flex: 1,
+    backgroundColor: "#fff",
+    alignItems: "center",
+    justifyContent: "space-evenly",
+    padding: 50,
+  },
+  buttonContainer: {
+    flexDirection: "row",
+    justifyContent: "space-around",
+    width: "100%",
+    marginBottom: 20,
+  },
+  bottomBar: {
+    position: "absolute",
+    bottom: 0,
+    width: "100%",
+    padding: 20,
+    borderTopWidth: 1,
+    borderTopColor: "lightgray",
+  },
+  VolumeDisplayContainer: {
+    position: "relative",
+    width: 150,
+    height: 150,
+    alignItems: "center",
+    justifyContent: "center",
+  },
+  volumeDisplay: {
+    position: "absolute",
+  },
+});
@@ -0,0 +1,59 @@
+# Expo Two-Way-Audio Using Flow API
+
+This example shows how expo-two-way-audio can be used to communicate with [Speechmatics Flow API](https://www.speechmatics.com/flow)
+
+Flow is a conversational AI API and therefore, we need to send audio data in real time and play back the audio responses that we'll get from the API.
+
+## Requirements
+
+- Node.js
+- Expo CLI
+- [iOS](https://docs.expo.dev/workflow/ios-simulator/) and/or [Android](https://docs.expo.dev/workflow/android-studio-emulator/) dev environment setup.
+- [Speechmatics Flow Client (React)](https://github.com/speechmatics/speechmatics-js-sdk/tree/main/packages/flow-client-react)
+- [`event-target-polyfill`](https://www.npmjs.com/package/event-target-polyfill) package, or any other polyfill for the [`EventTarget` class](https://developer.mozilla.org/en-US/docs/Web/API/EventTarget)
+
+A Speechmatics API key is also needed in order to run the example. The key is injected into the app via environment variable. It can be passed by creating a `.env` file in the same directory as the example with the following content:
+
+```
+EXPO_PUBLIC_SPEECHMATICS_API_KEY='YOUR-API-KEY-HERE'
+```
+
+An API key can be obtained from the [Speechmatics Portal](https://portal.speechmatics.com)
+
+> [!WARNING]
+>
+> The example uses the API key to request a temporary token (needed for communicating with flow).
+> In a real-world scenario, you should not expose your API key in your client-side code.
+> Instead, you should create a server-side endpoint that generates the temporary token (JWT) for you.
+
+## Run the app
+
+In the `examples/flow-api` directory:
+
+1. Install dependencies:
+
+   ```sh
+   npm install
+   ```
+
+2. Run the app:
+
+   ```sh
+   # Run iOS
+   npm run ios
+
+   # Run Android
+   npm run android
+   ```
+
+## Notes
+
+Some audio features of expo-two-way-audio like Acoustic Echo Cancelling, noise reduction or microphone modes (iOS) don't work on simulator. Run the example on a real device to test these features.
+
+```bash
+# iOS
+npx expo run:ios --device --configuration Release
+
+# Android
+npx expo run:android --device --variant release
+```
@@ -0,0 +1,47 @@
+{
+  "expo": {
+    "name": "Flow Example",
+    "slug": "expo-two-way-audio-flow-example",
+    "version": "1.0.0",
+    "orientation": "portrait",
+    "icon": "./assets/icon.png",
+    "userInterfaceStyle": "light",
+    "splash": {
+      "image": "./assets/splash.png",
+      "resizeMode": "contain",
+      "backgroundColor": "#ffffff"
+    },
+    "ios": {
+      "supportsTablet": false,
+      "bundleIdentifier": "expo.modules.twowayaudio.flow.example",
+      "infoPlist": {
+        "UIBackgroundModes": ["audio"],
+        "NSMicrophoneUsageDescription": "Allow Speechmatics to access your microphone"
+      }
+    },
+    "android": {
+      "adaptiveIcon": {
+        "foregroundImage": "./assets/adaptive-icon.png",
+        "backgroundColor": "#ffffff"
+      },
+      "permissions": ["RECORD_AUDIO", "MODIFY_AUDIO_SETTINGS"],
+      "package": "expo.modules.twowayaudio.flow.example"
+    },
+    "web": {
+      "favicon": "./assets/favicon.png"
+    },
+    "plugins": [
+      [
+        "expo-build-properties",
+        {
+          "android": {
+            "compileSdkVersion": 34,
+            "targetSdkVersion": 31,
+            "minSdkVersion": 26,
+            "buildToolsVersion": "34.0.0"
+          }
+        }
+      ]
+    ]
+  }
+}
@@ -0,0 +1,15 @@
+import { createSpeechmaticsJWT } from "@speechmatics/auth";
+
+// WARNING: This function is used for the purpose of this example only.
+// In a real-world scenario, you should not expose your API key in your client-side code.
+// Instead, you should create a server-side endpoint that generates the JWT for you.
+export async function getFlowAPIJwt(apiKey: string) {
+  if (!apiKey) {
+    throw new Error("API key is required.");
+  }
+  return await createSpeechmaticsJWT({
+    type: "flow",
+    apiKey,
+    ttl: 60, // 1 minute
+  });
+}
@@ -0,0 +1,6 @@
+module.exports = function (api) {
+  api.cache(true);
+  return {
+    presets: ["babel-preset-expo"],
+  };
+};
@@ -0,0 +1,37 @@
+const { wrapWithReanimatedMetroConfig } = require("react-native-reanimated/metro-config");
+
+// Learn more https://docs.expo.io/guides/customizing-metro
+const { getDefaultConfig } = require("expo/metro-config");
+const path = require("path");
+
+const config = getDefaultConfig(__dirname);
+
+// npm v7+ will install ../node_modules/react and ../node_modules/react-native because of peerDependencies.
+// To prevent the incompatible react-native between ./node_modules/react-native and ../node_modules/react-native,
+// excludes the one from the parent folder when bundling.
+config.resolver.blockList = [
+  ...Array.from(config.resolver.blockList ?? []),
+  new RegExp(path.resolve("../..", "node_modules", "react")),
+  new RegExp(path.resolve("../..", "node_modules", "react-native")),
+];
+
+config.resolver.nodeModulesPaths = [
+  path.resolve(__dirname, "./node_modules"),
+  path.resolve(__dirname, "../../node_modules"),
+];
+
+config.resolver.extraNodeModules = {
+  "expo-two-way-audio": "../..",
+};
+
+config.watchFolders = [path.resolve(__dirname, "../../")];
+
+config.transformer.getTransformOptions = async () => ({
+  transform: {
+    experimentalImportSupport: false,
+    inlineRequires: true,
+  },
+});
+
+module.exports = wrapWithReanimatedMetroConfig(config);
+config;
@@ -0,0 +1,34 @@
+{
+  "name": "expo-two-way-audio-flow-example",
+  "version": "1.0.0",
+  "main": "expo/AppEntry.js",
+  "scripts": {
+    "start": "expo start",
+    "android": "expo run:android",
+    "ios": "expo run:ios",
+    "web": "expo start --web"
+  },
+  "dependencies": {
+    "@speechmatics/auth": "^0.0.1",
+    "@speechmatics/flow-client-react": "^0.2.0",
+    "buffer": "^6.0.3",
+    "event-target-polyfill": "^0.0.4",
+    "expo": "^52.0.25",
+    "expo-build-properties": "~0.13.2",
+    "expo-dev-client": "~5.0.9",
+    "react": "18.3.1",
+    "react-native": "0.76.6",
+    "react-native-reanimated": "~3.16.1"
+  },
+  "devDependencies": {
+    "@babel/core": "^7.20.0",
+    "@types/react": "~18.3.12",
+    "typescript": "^5.1.3"
+  },
+  "private": true,
+  "expo": {
+    "autolinking": {
+      "nativeModulesDir": "../.."
+    }
+  }
+}
@@ -0,0 +1,11 @@
+{
+  "extends": "expo/tsconfig.base",
+  "compilerOptions": {
+    "strict": true,
+    "module": "ESNext",
+    "paths": {
+      "@speechmatics/expo-two-way-audio": ["../../src/index"],
+      "@speechmatics/expo-two-way-audio/*": ["../../src/*"]
+    }
+  }
+}
@@ -0,0 +1,52 @@
+import { StyleSheet, View } from "react-native";
+import Animated, {
+  Easing,
+  interpolate,
+  useAnimatedStyle,
+  withTiming,
+  type SharedValue,
+} from "react-native-reanimated";
+
+export default function VolumeDisplay({
+  volumeLevel,
+  color,
+  minSize,
+  maxSize,
+}: {
+  volumeLevel: SharedValue<number>;
+  color: string;
+  minSize: number;
+  maxSize: number;
+}) {
+  const animatedStyle = useAnimatedStyle(() => {
+    const size = interpolate(volumeLevel.value, [0, 1], [minSize, maxSize]);
+
+    return {
+      width: withTiming(size, {
+        duration: 100,
+        easing: Easing.linear,
+      }),
+      height: withTiming(size, {
+        duration: 100,
+        easing: Easing.linear,
+      }),
+      borderRadius: withTiming(size / 2, {
+        duration: 100,
+        easing: Easing.linear,
+      }),
+    };
+  });
+
+  return (
+    <View style={styles.container}>
+      <Animated.View style={[{ backgroundColor: color }, animatedStyle]} />
+    </View>
+  );
+}
+
+const styles = StyleSheet.create({
+  container: {
+    justifyContent: "center",
+    alignItems: "center",
+  },
+});
@@ -0,0 +1,20 @@
+const createConfigAsync = require("@expo/webpack-config");
+const path = require("path");
+
+module.exports = async (env, argv) => {
+  const config = await createConfigAsync(
+    {
+      ...env,
+      babel: {
+        dangerouslyAddModulePathsToTranspile: ["expo-two-way-audio"],
+      },
+    },
+    argv,
+  );
+  config.resolve.modules = [
+    path.resolve(__dirname, "./node_modules"),
+    path.resolve(__dirname, "../../node_modules"),
+  ];
+
+  return config;
+};