From e2760811dd71c8fc1b1d56bc4348aed71b63a8e7 Mon Sep 17 00:00:00 2001 From: twitchard Date: Fri, 25 Oct 2024 09:10:34 -0700 Subject: [PATCH] wip --- evi-react-native-example/EVIExample/App.tsx | 47 ++--- evi-react-native-example/EVIExample/app.json | 5 +- .../ios/EVIExample.xcodeproj/project.pbxproj | 68 +++---- .../EVIExample/ios/EVIExample/Info.plist | 2 + .../EVIExample/modules/audio/index.ts | 23 ++- .../modules/audio/ios/AudioModule.swift | 191 +++++++++++------- .../modules/audio/src/Audio.types.ts | 4 +- 7 files changed, 190 insertions(+), 150 deletions(-) diff --git a/evi-react-native-example/EVIExample/App.tsx b/evi-react-native-example/EVIExample/App.tsx index 82f1056..5b7aad1 100644 --- a/evi-react-native-example/EVIExample/App.tsx +++ b/evi-react-native-example/EVIExample/App.tsx @@ -9,16 +9,7 @@ import { } from 'react-native'; import { HumeClient, type Hume } from 'hume' -import { hello } from './modules/audio'; - -async function greetUser() { - - const message = hello(); - console.log(message); -} - -greetUser(); - +import * as NativeAudio from './modules/audio'; interface ChatEntry { role: 'user' | 'assistant'; @@ -42,8 +33,12 @@ const App = () => { const chatSocketRef = useRef(null); useEffect(() => { - greetUser(); if (isConnected) { + NativeAudio.getPermissions().then(() => { + NativeAudio.startRecording(); + }).catch((error) => { + console.error('Failed to get permissions:', error); + }) const chatSocket = hume.empathicVoice.chat.connect({ configId: process.env.EXPO_PUBLIC_HUME_CONFIG_ID, }) @@ -60,9 +55,15 @@ const App = () => { chatSocketRef.current = chatSocket; - return () => { - chatSocket.close(); - }; + NativeAudio.onAudioInput(({base64EncodedAudio}: NativeAudio.AudioEventPayload) => { + chatSocket.sendAudioInput({data: base64EncodedAudio}); + }) + } else { + NativeAudio.stopRecording(); + } + return () => { + NativeAudio.stopRecording(); + chatSocketRef.current?.close(); } }, [isConnected]); @@ -88,33 +89,25 @@ const App = () => { const disconnectFromWebSocket = () => { if (chatSocketRef.current) { chatSocketRef.current.close(); - setIsConnected(false); } - }; - - const startRecording = async () => { - console.log('TODO: record things') - }; - - const stopRecording = async () => { - console.log('TODO: stop recording things') + setIsConnected(false); }; const muteInput = () => { setIsMuted(true); - stopRecording(); + NativeAudio.stopRecording(); }; const unmuteInput = () => { setIsMuted(false); - startRecording(); + NativeAudio.startRecording(); }; - const playAudio = (audioData: any) => { + const playAudio = (audioData: string) => { if (playbackQueue.length > 0) { setPlaybackQueue((prev) => [...prev, audioData]); } else { - console.log('TODO: enqueue audio for playback'); + NativeAudio.playAudio(audioData); } }; diff --git a/evi-react-native-example/EVIExample/app.json b/evi-react-native-example/EVIExample/app.json index 9dd71a8..fb7ebbf 100644 --- a/evi-react-native-example/EVIExample/app.json +++ b/evi-react-native-example/EVIExample/app.json @@ -13,7 +13,10 @@ }, "ios": { "supportsTablet": true, - "bundleIdentifier": "com.example.EVIExample" + "bundleIdentifier": "com.example.EVIExample", + "infoPlist": { + "NSMicrophoneUsageDescription": "This app uses the microphone to allow the user to talk to the EVI conversational AI interface" + } }, "android": { "adaptiveIcon": { diff --git a/evi-react-native-example/EVIExample/ios/EVIExample.xcodeproj/project.pbxproj b/evi-react-native-example/EVIExample/ios/EVIExample.xcodeproj/project.pbxproj index 4ee3b2e..03cf406 100644 --- a/evi-react-native-example/EVIExample/ios/EVIExample.xcodeproj/project.pbxproj +++ b/evi-react-native-example/EVIExample/ios/EVIExample.xcodeproj/project.pbxproj @@ -10,29 +10,29 @@ 13B07FBC1A68108700A75B9A /* AppDelegate.mm in Sources */ = {isa = PBXBuildFile; fileRef = 13B07FB01A68108700A75B9A /* AppDelegate.mm */; }; 13B07FBF1A68108700A75B9A /* Images.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = 13B07FB51A68108700A75B9A /* Images.xcassets */; }; 13B07FC11A68108700A75B9A /* main.m in Sources */ = {isa = PBXBuildFile; fileRef = 13B07FB71A68108700A75B9A /* main.m */; }; + 2D7A14CEDB964A6BB821853B /* noop-file.swift in Sources */ = {isa = PBXBuildFile; fileRef = 02F03A40837F4F48A83A9FA5 /* noop-file.swift */; }; 3E461D99554A48A4959DE609 /* SplashScreen.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = AA286B85B6C04FC6940260E9 /* SplashScreen.storyboard */; }; - 416628AE244C47183BE4080B /* PrivacyInfo.xcprivacy in Resources */ = {isa = PBXBuildFile; fileRef = D670C7844B2C922241619D64 /* PrivacyInfo.xcprivacy */; }; - 54987443346E4D38A033F481 /* noop-file.swift in Sources */ = {isa = PBXBuildFile; fileRef = 86634FD337404C0E88181964 /* noop-file.swift */; }; 96905EF65AED1B983A6B3ABC /* libPods-EVIExample.a in Frameworks */ = {isa = PBXBuildFile; fileRef = 58EEBF8E8E6FB1BC6CAF49B5 /* libPods-EVIExample.a */; }; B18059E884C0ABDD17F3DC3D /* ExpoModulesProvider.swift in Sources */ = {isa = PBXBuildFile; fileRef = FAC715A2D49A985799AEE119 /* ExpoModulesProvider.swift */; }; + BA094F021E8F5F331736B1B9 /* PrivacyInfo.xcprivacy in Resources */ = {isa = PBXBuildFile; fileRef = 641C73F996BC9042D9CC297D /* PrivacyInfo.xcprivacy */; }; BB2F792D24A3F905000567C9 /* Expo.plist in Resources */ = {isa = PBXBuildFile; fileRef = BB2F792C24A3F905000567C9 /* Expo.plist */; }; /* End PBXBuildFile section */ /* Begin PBXFileReference section */ + 02F03A40837F4F48A83A9FA5 /* noop-file.swift */ = {isa = PBXFileReference; explicitFileType = undefined; fileEncoding = 4; includeInIndex = 0; lastKnownFileType = sourcecode.swift; name = "noop-file.swift"; path = "EVIExample/noop-file.swift"; sourceTree = ""; }; 13B07F961A680F5B00A75B9A /* EVIExample.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = EVIExample.app; sourceTree = BUILT_PRODUCTS_DIR; }; 13B07FAF1A68108700A75B9A /* AppDelegate.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = AppDelegate.h; path = EVIExample/AppDelegate.h; sourceTree = ""; }; 13B07FB01A68108700A75B9A /* AppDelegate.mm */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.objcpp; name = AppDelegate.mm; path = EVIExample/AppDelegate.mm; sourceTree = ""; }; 13B07FB51A68108700A75B9A /* Images.xcassets */ = {isa = PBXFileReference; lastKnownFileType = folder.assetcatalog; name = Images.xcassets; path = EVIExample/Images.xcassets; sourceTree = ""; }; 13B07FB61A68108700A75B9A /* Info.plist */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.plist.xml; name = Info.plist; path = EVIExample/Info.plist; sourceTree = ""; }; 13B07FB71A68108700A75B9A /* main.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; name = main.m; path = EVIExample/main.m; sourceTree = ""; }; - 4B9E4F5EB5344F42B3209D48 /* EVIExample-Bridging-Header.h */ = {isa = PBXFileReference; explicitFileType = undefined; fileEncoding = 4; includeInIndex = 0; lastKnownFileType = sourcecode.c.h; name = "EVIExample-Bridging-Header.h"; path = "EVIExample/EVIExample-Bridging-Header.h"; sourceTree = ""; }; 58EEBF8E8E6FB1BC6CAF49B5 /* libPods-EVIExample.a */ = {isa = PBXFileReference; explicitFileType = archive.ar; includeInIndex = 0; path = "libPods-EVIExample.a"; sourceTree = BUILT_PRODUCTS_DIR; }; + 641C73F996BC9042D9CC297D /* PrivacyInfo.xcprivacy */ = {isa = PBXFileReference; includeInIndex = 1; name = PrivacyInfo.xcprivacy; path = EVIExample/PrivacyInfo.xcprivacy; sourceTree = ""; }; 6C2E3173556A471DD304B334 /* Pods-EVIExample.debug.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-EVIExample.debug.xcconfig"; path = "Target Support Files/Pods-EVIExample/Pods-EVIExample.debug.xcconfig"; sourceTree = ""; }; 7A4D352CD337FB3A3BF06240 /* Pods-EVIExample.release.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-EVIExample.release.xcconfig"; path = "Target Support Files/Pods-EVIExample/Pods-EVIExample.release.xcconfig"; sourceTree = ""; }; - 86634FD337404C0E88181964 /* noop-file.swift */ = {isa = PBXFileReference; explicitFileType = undefined; fileEncoding = 4; includeInIndex = 0; lastKnownFileType = sourcecode.swift; name = "noop-file.swift"; path = "EVIExample/noop-file.swift"; sourceTree = ""; }; + A84258F5A02B4837BDA5156D /* EVIExample-Bridging-Header.h */ = {isa = PBXFileReference; explicitFileType = undefined; fileEncoding = 4; includeInIndex = 0; lastKnownFileType = sourcecode.c.h; name = "EVIExample-Bridging-Header.h"; path = "EVIExample/EVIExample-Bridging-Header.h"; sourceTree = ""; }; AA286B85B6C04FC6940260E9 /* SplashScreen.storyboard */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = file.storyboard; name = SplashScreen.storyboard; path = EVIExample/SplashScreen.storyboard; sourceTree = ""; }; BB2F792C24A3F905000567C9 /* Expo.plist */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.plist.xml; path = Expo.plist; sourceTree = ""; }; - D670C7844B2C922241619D64 /* PrivacyInfo.xcprivacy */ = {isa = PBXFileReference; includeInIndex = 1; name = PrivacyInfo.xcprivacy; path = EVIExample/PrivacyInfo.xcprivacy; sourceTree = ""; }; ED297162215061F000B7C4FE /* JavaScriptCore.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = JavaScriptCore.framework; path = System/Library/Frameworks/JavaScriptCore.framework; sourceTree = SDKROOT; }; FAC715A2D49A985799AEE119 /* ExpoModulesProvider.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; name = ExpoModulesProvider.swift; path = "Pods/Target Support Files/Pods-EVIExample/ExpoModulesProvider.swift"; sourceTree = ""; }; /* End PBXFileReference section */ @@ -59,9 +59,9 @@ 13B07FB61A68108700A75B9A /* Info.plist */, 13B07FB71A68108700A75B9A /* main.m */, AA286B85B6C04FC6940260E9 /* SplashScreen.storyboard */, - 86634FD337404C0E88181964 /* noop-file.swift */, - 4B9E4F5EB5344F42B3209D48 /* EVIExample-Bridging-Header.h */, - D670C7844B2C922241619D64 /* PrivacyInfo.xcprivacy */, + 02F03A40837F4F48A83A9FA5 /* noop-file.swift */, + A84258F5A02B4837BDA5156D /* EVIExample-Bridging-Header.h */, + 641C73F996BC9042D9CC297D /* PrivacyInfo.xcprivacy */, ); name = EVIExample; sourceTree = ""; @@ -147,13 +147,13 @@ buildConfigurationList = 13B07F931A680F5B00A75B9A /* Build configuration list for PBXNativeTarget "EVIExample" */; buildPhases = ( 08A4A3CD28434E44B6B9DE2E /* [CP] Check Pods Manifest.lock */, - 2B73FDCF9D1B06C1C82B650B /* [Expo] Configure project */, + B686FFB6D6778EC6C0E932AB /* [Expo] Configure project */, 13B07F871A680F5B00A75B9A /* Sources */, 13B07F8C1A680F5B00A75B9A /* Frameworks */, 13B07F8E1A680F5B00A75B9A /* Resources */, 00DD1BFF1BD5951E006B06BC /* Bundle React Native code and images */, 800E24972A6A228C8D4807E9 /* [CP] Copy Pods Resources */, - C8513F70D957F286002F02EC /* [CP] Embed Pods Frameworks */, + 8024DE7DD9C34304BE2FBE90 /* [CP] Embed Pods Frameworks */, ); buildRules = ( ); @@ -203,7 +203,7 @@ BB2F792D24A3F905000567C9 /* Expo.plist in Resources */, 13B07FBF1A68108700A75B9A /* Images.xcassets in Resources */, 3E461D99554A48A4959DE609 /* SplashScreen.storyboard in Resources */, - 416628AE244C47183BE4080B /* PrivacyInfo.xcprivacy in Resources */, + BA094F021E8F5F331736B1B9 /* PrivacyInfo.xcprivacy in Resources */, ); runOnlyForDeploymentPostprocessing = 0; }; @@ -247,25 +247,6 @@ shellScript = "diff \"${PODS_PODFILE_DIR_PATH}/Podfile.lock\" \"${PODS_ROOT}/Manifest.lock\" > /dev/null\nif [ $? != 0 ] ; then\n # print error to STDERR\n echo \"error: The sandbox is not in sync with the Podfile.lock. Run 'pod install' or update your CocoaPods installation.\" >&2\n exit 1\nfi\n# This output is used by Xcode 'outputs' to avoid re-running this script phase.\necho \"SUCCESS\" > \"${SCRIPT_OUTPUT_FILE_0}\"\n"; showEnvVarsInLog = 0; }; - 2B73FDCF9D1B06C1C82B650B /* [Expo] Configure project */ = { - isa = PBXShellScriptBuildPhase; - alwaysOutOfDate = 1; - buildActionMask = 2147483647; - files = ( - ); - inputFileListPaths = ( - ); - inputPaths = ( - ); - name = "[Expo] Configure project"; - outputFileListPaths = ( - ); - outputPaths = ( - ); - runOnlyForDeploymentPostprocessing = 0; - shellPath = /bin/sh; - shellScript = "# This script configures Expo modules and generates the modules provider file.\nbash -l -c \"./Pods/Target\\ Support\\ Files/Pods-EVIExample/expo-configure-project.sh\"\n"; - }; 800E24972A6A228C8D4807E9 /* [CP] Copy Pods Resources */ = { isa = PBXShellScriptBuildPhase; buildActionMask = 2147483647; @@ -290,7 +271,7 @@ shellScript = "\"${PODS_ROOT}/Target Support Files/Pods-EVIExample/Pods-EVIExample-resources.sh\"\n"; showEnvVarsInLog = 0; }; - C8513F70D957F286002F02EC /* [CP] Embed Pods Frameworks */ = { + 8024DE7DD9C34304BE2FBE90 /* [CP] Embed Pods Frameworks */ = { isa = PBXShellScriptBuildPhase; buildActionMask = 2147483647; files = ( @@ -308,6 +289,25 @@ shellScript = "\"${PODS_ROOT}/Target Support Files/Pods-EVIExample/Pods-EVIExample-frameworks.sh\"\n"; showEnvVarsInLog = 0; }; + B686FFB6D6778EC6C0E932AB /* [Expo] Configure project */ = { + isa = PBXShellScriptBuildPhase; + alwaysOutOfDate = 1; + buildActionMask = 2147483647; + files = ( + ); + inputFileListPaths = ( + ); + inputPaths = ( + ); + name = "[Expo] Configure project"; + outputFileListPaths = ( + ); + outputPaths = ( + ); + runOnlyForDeploymentPostprocessing = 0; + shellPath = /bin/sh; + shellScript = "# This script configures Expo modules and generates the modules provider file.\nbash -l -c \"./Pods/Target\\ Support\\ Files/Pods-EVIExample/expo-configure-project.sh\"\n"; + }; /* End PBXShellScriptBuildPhase section */ /* Begin PBXSourcesBuildPhase section */ @@ -318,7 +318,7 @@ 13B07FBC1A68108700A75B9A /* AppDelegate.mm in Sources */, 13B07FC11A68108700A75B9A /* main.m in Sources */, B18059E884C0ABDD17F3DC3D /* ExpoModulesProvider.swift in Sources */, - 54987443346E4D38A033F481 /* noop-file.swift in Sources */, + 2D7A14CEDB964A6BB821853B /* noop-file.swift in Sources */, ); runOnlyForDeploymentPostprocessing = 0; }; @@ -349,7 +349,7 @@ ); OTHER_SWIFT_FLAGS = "$(inherited) -D EXPO_CONFIGURATION_DEBUG"; PRODUCT_BUNDLE_IDENTIFIER = com.example.EVIExample; - PRODUCT_NAME = EVIExample; + PRODUCT_NAME = "EVIExample"; SWIFT_OBJC_BRIDGING_HEADER = "EVIExample/EVIExample-Bridging-Header.h"; SWIFT_OPTIMIZATION_LEVEL = "-Onone"; SWIFT_VERSION = 5.0; @@ -377,7 +377,7 @@ ); OTHER_SWIFT_FLAGS = "$(inherited) -D EXPO_CONFIGURATION_RELEASE"; PRODUCT_BUNDLE_IDENTIFIER = com.example.EVIExample; - PRODUCT_NAME = EVIExample; + PRODUCT_NAME = "EVIExample"; SWIFT_OBJC_BRIDGING_HEADER = "EVIExample/EVIExample-Bridging-Header.h"; SWIFT_VERSION = 5.0; TARGETED_DEVICE_FAMILY = "1,2"; diff --git a/evi-react-native-example/EVIExample/ios/EVIExample/Info.plist b/evi-react-native-example/EVIExample/ios/EVIExample/Info.plist index 4ad39d4..69a5676 100644 --- a/evi-react-native-example/EVIExample/ios/EVIExample/Info.plist +++ b/evi-react-native-example/EVIExample/ios/EVIExample/Info.plist @@ -42,6 +42,8 @@ NSAllowsLocalNetworking + NSMicrophoneUsageDescription + This app uses the microphone to allow the user to talk to the EVI conversational AI interface UILaunchStoryboardName SplashScreen UIRequiredDeviceCapabilities diff --git a/evi-react-native-example/EVIExample/modules/audio/index.ts b/evi-react-native-example/EVIExample/modules/audio/index.ts index f99a606..0365ed5 100644 --- a/evi-react-native-example/EVIExample/modules/audio/index.ts +++ b/evi-react-native-example/EVIExample/modules/audio/index.ts @@ -3,25 +3,28 @@ import { NativeModulesProxy, EventEmitter, Subscription } from 'expo-modules-cor // Import the native module. On web, it will be resolved to Audio.web.ts // and on native platforms to Audio.ts import AudioModule from './src/AudioModule'; -import { BytesEventPayload } from './src/Audio.types'; +import { AudioEventPayload } from './src/Audio.types'; -export async function getPermissions(value: string) { - return await AudioModule.setValueAsync(value); +export async function getPermissions(): Promise { + return await AudioModule.getPermissions(); } export async function startRecording(): Promise { - return await AudioModule.startRecording(); + await AudioModule.startRecording(); } -export async function stopRecording(): Promise { - return await AudioModule.stopRecording(); +export async function playAudio(base64EncodedAudio: string): Promise { + await AudioModule.playAudio(base64EncodedAudio); } - const emitter = new EventEmitter(AudioModule ?? NativeModulesProxy.Audio); +export async function stopRecording(): Promise { + emitter.removeAllListeners('onAudioInput'); + return await AudioModule.stopRecording(); +} -export function onBytes(listener: (event: BytesEventPayload) => void): Subscription { - return emitter.addListener('onBytes', listener); +export function onAudioInput(listener: (event: AudioEventPayload) => void): Subscription { + return emitter.addListener('onAudioInput', listener); } -export { BytesEventPayload }; +export { AudioEventPayload }; diff --git a/evi-react-native-example/EVIExample/modules/audio/ios/AudioModule.swift b/evi-react-native-example/EVIExample/modules/audio/ios/AudioModule.swift index 74f6e85..d017b1d 100644 --- a/evi-react-native-example/EVIExample/modules/audio/ios/AudioModule.swift +++ b/evi-react-native-example/EVIExample/modules/audio/ios/AudioModule.swift @@ -1,92 +1,131 @@ import ExpoModulesCore +import AVFoundation public class AudioModule: Module { - public func definition() -> ModuleDefinition { - Name("Audio") - - // Sets constant properties on the module. Can take a dictionary or a closure that returns a dictionary. - Constants([]) - - // Defines event names that the module can send to JavaScript. - Events("onBytes") - - AsyncFunction("getPermissions") { (resolve: @escaping (Bool) -> Void, reject: @escaping (String) -> Void) in - self.getPermissions(resolve: resolve, reject: reject) - } - - AsyncFunction("startRecording") { - self.startRecording() - } - - AsyncFunction("stopRecording") { - self.stopRecording() - } - - // Defines a JavaScript function that always returns a Promise and whose native code - // is by default dispatched on the different thread than the JavaScript runtime runs on. - AsyncFunction("setValueAsync") { (value: String) in - // Send an event to JavaScript. - self.sendEvent("onChange", [ - "value": value - ]) + private var audioEngine: AVAudioEngine? + private var inputNode: AVAudioInputNode? + private var audioPlayerNode: AVAudioPlayerNode? + + private static let sampleRate: Double = 44100 + private static let channels: AVAudioChannelCount = 2 + private static let playbackAudioFormat = AVAudioFormat( + standardFormatWithSampleRate: sampleRate, + channels: channels + ) + private static let bytesPerFrame = playbackAudioFormat?.streamDescription.pointee.mBytesPerFrame ?? 1 + + public func definition() -> ModuleDefinition { + Name("Audio") + + Constants([:]) + + Events("onAudioInput") + + AsyncFunction("getPermissions") { + return try await self.getPermissions(); + } + + AsyncFunction("startRecording") { + self.startRecording() + } + + AsyncFunction("stopRecording") { + self.stopRecording() + } + + AsyncFunction("playAudio") { (base64EncodedAudio: String) in + return try await self.playBase64Audio(base64EncodedAudio) + } } - } - - private func getPermissions(resolve: @escaping (Bool) -> Void, reject: @escaping (String) -> Void) { - let audioSession = AVAudioSession.sharedInstance() - switch audioSession.recordPermission { + + private func getPermissions() async throws -> Bool { + print("Attempting to get audio permissions") + let audioSession = AVAudioSession.sharedInstance() + switch audioSession.recordPermission { case .granted: - resolve(true) + return true case .denied: - resolve(false) + return false case .undetermined: - audioSession.requestRecordPermission { granted in - resolve(granted) - } + return await withCheckedContinuation { continuation in + audioSession.requestRecordPermission { granted in + continuation.resume(returning: granted) + } + } @unknown default: - reject("Unknown permission state") - } + throw NSError(domain: "AudioModule", code: 1, userInfo: [NSLocalizedDescriptionKey: "Unknown permission state"]) + } } - + private func startRecording() { - audioEngine = AVAudioEngine() - guard let audioEngine = audioEngine else { return } - - inputNode = audioEngine.inputNode - guard let inputNode = inputNode else { return } - - recordingFormat = inputNode.inputFormat(forBus: 0) - guard let recordingFormat = recordingFormat else { return } - - inputNode.installTap(onBus: 0, bufferSize: 1024, format: recordingFormat) { [weak self] (buffer, time) in - self?.processAudioBuffer(buffer) - } - - do { - try audioEngine.start() - } catch { - // Handle audio engine start error - } + do { + let audioSession = AVAudioSession.sharedInstance() + try audioSession.setCategory(.playAndRecord, mode: .default, options: []) + try audioSession.setActive(true) + } catch { + print("Failed to activate audio session: \(error.localizedDescription)") + return + } + + let audioEngine = AVAudioEngine() + self.audioEngine = audioEngine + let inputNode = audioEngine.inputNode + self.inputNode = inputNode + let format = inputNode.inputFormat(forBus: 0) + let bufferSize = AVAudioFrameCount(format.sampleRate * 0.1) // 100ms of audio + + inputNode.installTap(onBus: 0, bufferSize: bufferSize, format: format) { [weak self] (buffer, time) in + let byteLength = Int(buffer.frameLength) * Int(buffer.format.streamDescription.pointee.mBytesPerFrame) + let audioData = Data(bytes: buffer.audioBufferList.pointee.mBuffers.mData!, count: byteLength) + let base64String = audioData.base64EncodedString() + self?.sendEvent("onAudioInput", ["base64EncodedAudio": base64String]) + } + + do { + try audioEngine.start() + print("Audio engine started successfully") + } catch { + print("Failed to start audio engine: \(error.localizedDescription)") + } } - + private func stopRecording() { - audioEngine?.stop() - inputNode?.removeTap(onBus: 0) + audioEngine?.stop() + self.inputNode?.removeTap(onBus: 0) } - - private func processAudioBuffer(_ buffer: AVAudioPCMBuffer) { - let byteLength = buffer.frameLength * buffer.format.streamDescription.pointee.mBytesPerFrame - var byteArray = [UInt8](repeating: 0, count: Int(byteLength)) - - // Copy audio data from buffer to byteArray - if let channelData = buffer.floatChannelData { - let audioBuffer = UnsafeBufferPointer(start: channelData[0], count: Int(buffer.frameLength)) - for (i, sample) in audioBuffer.enumerated() { - byteArray[i] = UInt8(sample * 255.0) // Convert float to byte (you may want a better scaling method) + + private func initAudioPlayer() { + guard let audioEngine = self.audioEngine else { return } + let audioPlayerNode = AVAudioPlayerNode() + self.audioPlayerNode = audioPlayerNode + audioEngine.connect(audioPlayerNode, to: audioEngine.mainMixerNode, format: audioEngine.mainMixerNode.outputFormat(forBus: 0)) + } + + private func playBase64Audio(_ base64String: String) async throws -> Bool { + if self.audioPlayerNode == nil { + initAudioPlayer() + } + guard let audioPlayerNode else { return false } + + guard let audioData = Data(base64Encoded: base64String) else { + throw NSError(domain: "AudioModule", code: 2, userInfo: [NSLocalizedDescriptionKey: "Invalid Base64 string"]) + } + + let frameLength = UInt32(audioData.count) / AudioModule.bytesPerFrame + guard let buffer = AVAudioPCMBuffer(pcmFormat: AudioModule.playbackAudioFormat!, frameCapacity: frameLength) else { + throw NSError(domain: "AudioPlaybackEngine", code: 2, userInfo: [NSLocalizedDescriptionKey: "Unable to create buffer"]) + } + buffer.frameLength = frameLength + + audioData.withUnsafeBytes { bufferPtr in + guard let audioPtr = bufferPtr.bindMemory(to: Float.self).baseAddress else { return } + buffer.floatChannelData?.pointee.assign(from: audioPtr, count: Int(frameLength)) + } + + return try await withCheckedThrowingContinuation { continuation in + audioPlayerNode.scheduleBuffer(buffer, at: nil, options: .interrupts) { + continuation.resume(returning: true) + } } - } - - // Send bytes to JavaScript - self.sendEvent("onBytes", ["bytes": byteArray]) } } diff --git a/evi-react-native-example/EVIExample/modules/audio/src/Audio.types.ts b/evi-react-native-example/EVIExample/modules/audio/src/Audio.types.ts index 72a04d5..f88af6f 100644 --- a/evi-react-native-example/EVIExample/modules/audio/src/Audio.types.ts +++ b/evi-react-native-example/EVIExample/modules/audio/src/Audio.types.ts @@ -1,3 +1,3 @@ -export type BytesEventPayload = { - value: string; +export type AudioEventPayload = { + base64EncodedAudio: string; };