From cb8d37a4784745df94052e5a764cec3b0b9f424e Mon Sep 17 00:00:00 2001 From: twitchard Date: Wed, 23 Oct 2024 18:11:35 -0700 Subject: [PATCH 01/27] wip --- .../EVIExample/.gitignore | 39 + evi-react-native-example/EVIExample/.npmrc | 1 + evi-react-native-example/EVIExample/App.tsx | 207 + evi-react-native-example/EVIExample/app.json | 32 + .../EVIExample/assets/adaptive-icon.png | Bin 0 -> 17547 bytes .../EVIExample/assets/favicon.png | Bin 0 -> 1466 bytes .../EVIExample/assets/icon.png | Bin 0 -> 22380 bytes .../EVIExample/assets/splash.png | Bin 0 -> 47346 bytes .../EVIExample/babel.config.js | 6 + .../modules/audio/android/build.gradle | 43 + .../android/src/main/AndroidManifest.xml | 2 + .../java/expo/modules/audio/AudioModule.kt | 47 + .../modules/audio/expo-module.config.json | 9 + .../EVIExample/modules/audio/index.ts | 30 + .../modules/audio/ios/Audio.podspec | 21 + .../modules/audio/ios/AudioModule.swift | 131 + .../modules/audio/src/Audio.types.ts | 3 + .../modules/audio/src/AudioModule.ts | 5 + .../modules/audio/src/AudioModule.web.ts | 13 + .../EVIExample/package.json | 28 + .../EVIExample/pnpm-lock.yaml | 8606 +++++++++++++++++ .../EVIExample/tsconfig.json | 6 + 22 files changed, 9229 insertions(+) create mode 100644 evi-react-native-example/EVIExample/.gitignore create mode 100644 evi-react-native-example/EVIExample/.npmrc create mode 100644 evi-react-native-example/EVIExample/App.tsx create mode 100644 evi-react-native-example/EVIExample/app.json create mode 100644 evi-react-native-example/EVIExample/assets/adaptive-icon.png create mode 100644 evi-react-native-example/EVIExample/assets/favicon.png create mode 100644 evi-react-native-example/EVIExample/assets/icon.png create mode 100644 evi-react-native-example/EVIExample/assets/splash.png create mode 100644 evi-react-native-example/EVIExample/babel.config.js create mode 100644 evi-react-native-example/EVIExample/modules/audio/android/build.gradle create mode 100644 evi-react-native-example/EVIExample/modules/audio/android/src/main/AndroidManifest.xml create mode 100644 evi-react-native-example/EVIExample/modules/audio/android/src/main/java/expo/modules/audio/AudioModule.kt create mode 100644 evi-react-native-example/EVIExample/modules/audio/expo-module.config.json create mode 100644 evi-react-native-example/EVIExample/modules/audio/index.ts create mode 100644 evi-react-native-example/EVIExample/modules/audio/ios/Audio.podspec create mode 100644 evi-react-native-example/EVIExample/modules/audio/ios/AudioModule.swift create mode 100644 evi-react-native-example/EVIExample/modules/audio/src/Audio.types.ts create mode 100644 evi-react-native-example/EVIExample/modules/audio/src/AudioModule.ts create mode 100644 evi-react-native-example/EVIExample/modules/audio/src/AudioModule.web.ts create mode 100644 evi-react-native-example/EVIExample/package.json create mode 100644 evi-react-native-example/EVIExample/pnpm-lock.yaml create mode 100644 evi-react-native-example/EVIExample/tsconfig.json diff --git a/evi-react-native-example/EVIExample/.gitignore b/evi-react-native-example/EVIExample/.gitignore new file mode 100644 index 0000000..9344619 --- /dev/null +++ b/evi-react-native-example/EVIExample/.gitignore @@ -0,0 +1,39 @@ +# Learn more https://docs.github.com/en/get-started/getting-started-with-git/ignoring-files + +# dependencies +node_modules/ + +# Expo +.expo/ +dist/ +web-build/ + +# Native +*.orig.* +*.jks +*.p8 +*.p12 +*.key +*.mobileprovision +.env + +# Metro +.metro-health-check* + +# debug +npm-debug.* +yarn-debug.* +yarn-error.* + +# macOS +.DS_Store +*.pem + +# local env files +.env*.local + +# typescript +*.tsbuildinfo + +ios +android diff --git a/evi-react-native-example/EVIExample/.npmrc b/evi-react-native-example/EVIExample/.npmrc new file mode 100644 index 0000000..d67f374 --- /dev/null +++ b/evi-react-native-example/EVIExample/.npmrc @@ -0,0 +1 @@ +node-linker=hoisted diff --git a/evi-react-native-example/EVIExample/App.tsx b/evi-react-native-example/EVIExample/App.tsx new file mode 100644 index 0000000..5b7aad1 --- /dev/null +++ b/evi-react-native-example/EVIExample/App.tsx @@ -0,0 +1,207 @@ +import React, { useEffect, useState, useRef } from 'react'; +import { + View, + Text, + Button, + StyleSheet, + ScrollView, + SafeAreaView, +} from 'react-native'; +import { HumeClient, type Hume } from 'hume' + +import * as NativeAudio from './modules/audio'; + +interface ChatEntry { + role: 'user' | 'assistant'; + timestamp: string; + content: string; +} + +const hume = new HumeClient({ + apiKey: process.env.EXPO_PUBLIC_HUME_API_KEY || '' +}) +const App = () => { + const [isConnected, setIsConnected] = useState(false); + const [isMuted, setIsMuted] = useState(false); + const [chatEntries, setChatEntries] = useState([ + { role: 'assistant', timestamp: new Date().toString(), content: 'Hello! How can I help you today?' }, + { role: 'user', timestamp: new Date().toString(), content: 'I am beyond help' }, + + ]); + const [playbackQueue, setPlaybackQueue] = useState([]); + + const chatSocketRef = useRef(null); + + useEffect(() => { + if (isConnected) { + NativeAudio.getPermissions().then(() => { + NativeAudio.startRecording(); + }).catch((error) => { + console.error('Failed to get permissions:', error); + }) + const chatSocket = hume.empathicVoice.chat.connect({ + configId: process.env.EXPO_PUBLIC_HUME_CONFIG_ID, + }) + chatSocket.on('message', handleIncomingMessage); + + chatSocket.on('error', (error) => { + console.error("WebSocket Error:", error); + }); + + chatSocket.on('close', () => { + console.log("WebSocket Connection Closed"); + setIsConnected(false); + }); + + chatSocketRef.current = chatSocket; + + NativeAudio.onAudioInput(({base64EncodedAudio}: NativeAudio.AudioEventPayload) => { + chatSocket.sendAudioInput({data: base64EncodedAudio}); + }) + } else { + NativeAudio.stopRecording(); + } + return () => { + NativeAudio.stopRecording(); + chatSocketRef.current?.close(); + } + }, [isConnected]); + + const handleIncomingMessage = (message: any) => { + if (message.type === 'audio_output') { + const audioData = message.data; + const decodedAudio = atob(audioData); + playAudio(decodedAudio); + } else if (message.type === 'chat_message') { + const chatEntry: ChatEntry = { + role: message.role === 'assistant' ? 'assistant' : 'user', + timestamp: new Date().toString(), + content: message.content, + }; + setChatEntries((prev) => [...prev, chatEntry]); + } + }; + + const connectToWebSocket = () => { + setIsConnected(true); + }; + + const disconnectFromWebSocket = () => { + if (chatSocketRef.current) { + chatSocketRef.current.close(); + } + setIsConnected(false); + }; + + const muteInput = () => { + setIsMuted(true); + NativeAudio.stopRecording(); + }; + + const unmuteInput = () => { + setIsMuted(false); + NativeAudio.startRecording(); + }; + + const playAudio = (audioData: string) => { + if (playbackQueue.length > 0) { + setPlaybackQueue((prev) => [...prev, audioData]); + } else { + NativeAudio.playAudio(audioData); + } + }; + + return ( + + + + You are {isConnected ? 'connected' : 'disconnected'} + + + {chatEntries.map((entry, index) => ( + + {entry.content} + + ))} + + +