Skip to content

Commit

Permalink
wip
Browse files Browse the repository at this point in the history
  • Loading branch information
@twitchard
twitchard committed Oct 25, 2024
1 parent 3c785f0 commit f944e44
Show file tree
Hide file tree
Showing 7 changed files with 190 additions and 150 deletions.
47 changes: 20 additions & 27 deletions evi-react-native-example/EVIExample/App.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,16 +9,7 @@ import {
} from 'react-native';
import { HumeClient, type Hume } from 'hume'

import { hello } from './modules/audio';

async function greetUser() {

const message = hello();
console.log(message);
}

greetUser();

import * as NativeAudio from './modules/audio';

interface ChatEntry {
role: 'user' | 'assistant';
Expand All @@ -42,8 +33,12 @@ const App = () => {
const chatSocketRef = useRef<Hume.empathicVoice.chat.ChatSocket | null>(null);

useEffect(() => {
greetUser();
if (isConnected) {
NativeAudio.getPermissions().then(() => {
NativeAudio.startRecording();
}).catch((error) => {
console.error('Failed to get permissions:', error);
})
const chatSocket = hume.empathicVoice.chat.connect({
configId: process.env.EXPO_PUBLIC_HUME_CONFIG_ID,
})
Expand All @@ -60,9 +55,15 @@ const App = () => {

chatSocketRef.current = chatSocket;

return () => {
chatSocket.close();
};
NativeAudio.onAudioInput(({base64EncodedAudio}: NativeAudio.AudioEventPayload) => {
chatSocket.sendAudioInput({data: base64EncodedAudio});
})
} else {
NativeAudio.stopRecording();
}
return () => {
NativeAudio.stopRecording();
chatSocketRef.current?.close();
}
}, [isConnected]);

Expand All @@ -88,33 +89,25 @@ const App = () => {
const disconnectFromWebSocket = () => {
if (chatSocketRef.current) {
chatSocketRef.current.close();
setIsConnected(false);
}
};

const startRecording = async () => {
console.log('TODO: record things')
};

const stopRecording = async () => {
console.log('TODO: stop recording things')
setIsConnected(false);
};

const muteInput = () => {
setIsMuted(true);
stopRecording();
NativeAudio.stopRecording();
};

const unmuteInput = () => {
setIsMuted(false);
startRecording();
NativeAudio.startRecording();
};

const playAudio = (audioData: any) => {
const playAudio = (audioData: string) => {
if (playbackQueue.length > 0) {
setPlaybackQueue((prev) => [...prev, audioData]);
} else {
console.log('TODO: enqueue audio for playback');
NativeAudio.playAudio(audioData);
}
};

Expand Down
5 changes: 4 additions & 1 deletion evi-react-native-example/EVIExample/app.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,10 @@
},
"ios": {
"supportsTablet": true,
"bundleIdentifier": "com.example.EVIExample"
"bundleIdentifier": "com.example.EVIExample",
"infoPlist": {
"NSMicrophoneUsageDescription": "This app uses the microphone to allow the user to talk to the EVI conversational AI interface"
}
},
"android": {
"adaptiveIcon": {
Expand Down
68 changes: 34 additions & 34 deletions evi-react-native-example/EVIExample/ios/EVIExample.xcodeproj/project.pbxproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,29 +10,29 @@
13B07FBC1A68108700A75B9A /* AppDelegate.mm in Sources */ = {isa = PBXBuildFile; fileRef = 13B07FB01A68108700A75B9A /* AppDelegate.mm */; };
13B07FBF1A68108700A75B9A /* Images.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = 13B07FB51A68108700A75B9A /* Images.xcassets */; };
13B07FC11A68108700A75B9A /* main.m in Sources */ = {isa = PBXBuildFile; fileRef = 13B07FB71A68108700A75B9A /* main.m */; };
2D7A14CEDB964A6BB821853B /* noop-file.swift in Sources */ = {isa = PBXBuildFile; fileRef = 02F03A40837F4F48A83A9FA5 /* noop-file.swift */; };
3E461D99554A48A4959DE609 /* SplashScreen.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = AA286B85B6C04FC6940260E9 /* SplashScreen.storyboard */; };
416628AE244C47183BE4080B /* PrivacyInfo.xcprivacy in Resources */ = {isa = PBXBuildFile; fileRef = D670C7844B2C922241619D64 /* PrivacyInfo.xcprivacy */; };
54987443346E4D38A033F481 /* noop-file.swift in Sources */ = {isa = PBXBuildFile; fileRef = 86634FD337404C0E88181964 /* noop-file.swift */; };
96905EF65AED1B983A6B3ABC /* libPods-EVIExample.a in Frameworks */ = {isa = PBXBuildFile; fileRef = 58EEBF8E8E6FB1BC6CAF49B5 /* libPods-EVIExample.a */; };
B18059E884C0ABDD17F3DC3D /* ExpoModulesProvider.swift in Sources */ = {isa = PBXBuildFile; fileRef = FAC715A2D49A985799AEE119 /* ExpoModulesProvider.swift */; };
BA094F021E8F5F331736B1B9 /* PrivacyInfo.xcprivacy in Resources */ = {isa = PBXBuildFile; fileRef = 641C73F996BC9042D9CC297D /* PrivacyInfo.xcprivacy */; };
BB2F792D24A3F905000567C9 /* Expo.plist in Resources */ = {isa = PBXBuildFile; fileRef = BB2F792C24A3F905000567C9 /* Expo.plist */; };
/* End PBXBuildFile section */

/* Begin PBXFileReference section */
02F03A40837F4F48A83A9FA5 /* noop-file.swift */ = {isa = PBXFileReference; explicitFileType = undefined; fileEncoding = 4; includeInIndex = 0; lastKnownFileType = sourcecode.swift; name = "noop-file.swift"; path = "EVIExample/noop-file.swift"; sourceTree = "<group>"; };
13B07F961A680F5B00A75B9A /* EVIExample.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = EVIExample.app; sourceTree = BUILT_PRODUCTS_DIR; };
13B07FAF1A68108700A75B9A /* AppDelegate.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = AppDelegate.h; path = EVIExample/AppDelegate.h; sourceTree = "<group>"; };
13B07FB01A68108700A75B9A /* AppDelegate.mm */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.objcpp; name = AppDelegate.mm; path = EVIExample/AppDelegate.mm; sourceTree = "<group>"; };
13B07FB51A68108700A75B9A /* Images.xcassets */ = {isa = PBXFileReference; lastKnownFileType = folder.assetcatalog; name = Images.xcassets; path = EVIExample/Images.xcassets; sourceTree = "<group>"; };
13B07FB61A68108700A75B9A /* Info.plist */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.plist.xml; name = Info.plist; path = EVIExample/Info.plist; sourceTree = "<group>"; };
13B07FB71A68108700A75B9A /* main.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; name = main.m; path = EVIExample/main.m; sourceTree = "<group>"; };
4B9E4F5EB5344F42B3209D48 /* EVIExample-Bridging-Header.h */ = {isa = PBXFileReference; explicitFileType = undefined; fileEncoding = 4; includeInIndex = 0; lastKnownFileType = sourcecode.c.h; name = "EVIExample-Bridging-Header.h"; path = "EVIExample/EVIExample-Bridging-Header.h"; sourceTree = "<group>"; };
58EEBF8E8E6FB1BC6CAF49B5 /* libPods-EVIExample.a */ = {isa = PBXFileReference; explicitFileType = archive.ar; includeInIndex = 0; path = "libPods-EVIExample.a"; sourceTree = BUILT_PRODUCTS_DIR; };
641C73F996BC9042D9CC297D /* PrivacyInfo.xcprivacy */ = {isa = PBXFileReference; includeInIndex = 1; name = PrivacyInfo.xcprivacy; path = EVIExample/PrivacyInfo.xcprivacy; sourceTree = "<group>"; };
6C2E3173556A471DD304B334 /* Pods-EVIExample.debug.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-EVIExample.debug.xcconfig"; path = "Target Support Files/Pods-EVIExample/Pods-EVIExample.debug.xcconfig"; sourceTree = "<group>"; };
7A4D352CD337FB3A3BF06240 /* Pods-EVIExample.release.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-EVIExample.release.xcconfig"; path = "Target Support Files/Pods-EVIExample/Pods-EVIExample.release.xcconfig"; sourceTree = "<group>"; };
86634FD337404C0E88181964 /* noop-file.swift */ = {isa = PBXFileReference; explicitFileType = undefined; fileEncoding = 4; includeInIndex = 0; lastKnownFileType = sourcecode.swift; name = "noop-file.swift"; path = "EVIExample/noop-file.swift"; sourceTree = "<group>"; };
A84258F5A02B4837BDA5156D /* EVIExample-Bridging-Header.h */ = {isa = PBXFileReference; explicitFileType = undefined; fileEncoding = 4; includeInIndex = 0; lastKnownFileType = sourcecode.c.h; name = "EVIExample-Bridging-Header.h"; path = "EVIExample/EVIExample-Bridging-Header.h"; sourceTree = "<group>"; };
AA286B85B6C04FC6940260E9 /* SplashScreen.storyboard */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = file.storyboard; name = SplashScreen.storyboard; path = EVIExample/SplashScreen.storyboard; sourceTree = "<group>"; };
BB2F792C24A3F905000567C9 /* Expo.plist */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.plist.xml; path = Expo.plist; sourceTree = "<group>"; };
D670C7844B2C922241619D64 /* PrivacyInfo.xcprivacy */ = {isa = PBXFileReference; includeInIndex = 1; name = PrivacyInfo.xcprivacy; path = EVIExample/PrivacyInfo.xcprivacy; sourceTree = "<group>"; };
ED297162215061F000B7C4FE /* JavaScriptCore.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = JavaScriptCore.framework; path = System/Library/Frameworks/JavaScriptCore.framework; sourceTree = SDKROOT; };
FAC715A2D49A985799AEE119 /* ExpoModulesProvider.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; name = ExpoModulesProvider.swift; path = "Pods/Target Support Files/Pods-EVIExample/ExpoModulesProvider.swift"; sourceTree = "<group>"; };
/* End PBXFileReference section */
Expand All @@ -59,9 +59,9 @@
13B07FB61A68108700A75B9A /* Info.plist */,
13B07FB71A68108700A75B9A /* main.m */,
AA286B85B6C04FC6940260E9 /* SplashScreen.storyboard */,
86634FD337404C0E88181964 /* noop-file.swift */,
4B9E4F5EB5344F42B3209D48 /* EVIExample-Bridging-Header.h */,
D670C7844B2C922241619D64 /* PrivacyInfo.xcprivacy */,
02F03A40837F4F48A83A9FA5 /* noop-file.swift */,
A84258F5A02B4837BDA5156D /* EVIExample-Bridging-Header.h */,
641C73F996BC9042D9CC297D /* PrivacyInfo.xcprivacy */,
);
name = EVIExample;
sourceTree = "<group>";
Expand Down Expand Up @@ -147,13 +147,13 @@
buildConfigurationList = 13B07F931A680F5B00A75B9A /* Build configuration list for PBXNativeTarget "EVIExample" */;
buildPhases = (
08A4A3CD28434E44B6B9DE2E /* [CP] Check Pods Manifest.lock */,
2B73FDCF9D1B06C1C82B650B /* [Expo] Configure project */,
B686FFB6D6778EC6C0E932AB /* [Expo] Configure project */,
13B07F871A680F5B00A75B9A /* Sources */,
13B07F8C1A680F5B00A75B9A /* Frameworks */,
13B07F8E1A680F5B00A75B9A /* Resources */,
00DD1BFF1BD5951E006B06BC /* Bundle React Native code and images */,
800E24972A6A228C8D4807E9 /* [CP] Copy Pods Resources */,
C8513F70D957F286002F02EC /* [CP] Embed Pods Frameworks */,
8024DE7DD9C34304BE2FBE90 /* [CP] Embed Pods Frameworks */,
);
buildRules = (
);
Expand Down Expand Up @@ -203,7 +203,7 @@
BB2F792D24A3F905000567C9 /* Expo.plist in Resources */,
13B07FBF1A68108700A75B9A /* Images.xcassets in Resources */,
3E461D99554A48A4959DE609 /* SplashScreen.storyboard in Resources */,
416628AE244C47183BE4080B /* PrivacyInfo.xcprivacy in Resources */,
BA094F021E8F5F331736B1B9 /* PrivacyInfo.xcprivacy in Resources */,
);
runOnlyForDeploymentPostprocessing = 0;
};
Expand Down Expand Up @@ -247,25 +247,6 @@
shellScript = "diff \"${PODS_PODFILE_DIR_PATH}/Podfile.lock\" \"${PODS_ROOT}/Manifest.lock\" > /dev/null\nif [ $? != 0 ] ; then\n # print error to STDERR\n echo \"error: The sandbox is not in sync with the Podfile.lock. Run 'pod install' or update your CocoaPods installation.\" >&2\n exit 1\nfi\n# This output is used by Xcode 'outputs' to avoid re-running this script phase.\necho \"SUCCESS\" > \"${SCRIPT_OUTPUT_FILE_0}\"\n";
showEnvVarsInLog = 0;
};
2B73FDCF9D1B06C1C82B650B /* [Expo] Configure project */ = {
isa = PBXShellScriptBuildPhase;
alwaysOutOfDate = 1;
buildActionMask = 2147483647;
files = (
);
inputFileListPaths = (
);
inputPaths = (
);
name = "[Expo] Configure project";
outputFileListPaths = (
);
outputPaths = (
);
runOnlyForDeploymentPostprocessing = 0;
shellPath = /bin/sh;
shellScript = "# This script configures Expo modules and generates the modules provider file.\nbash -l -c \"./Pods/Target\\ Support\\ Files/Pods-EVIExample/expo-configure-project.sh\"\n";
};
800E24972A6A228C8D4807E9 /* [CP] Copy Pods Resources */ = {
isa = PBXShellScriptBuildPhase;
buildActionMask = 2147483647;
Expand All @@ -290,7 +271,7 @@
shellScript = "\"${PODS_ROOT}/Target Support Files/Pods-EVIExample/Pods-EVIExample-resources.sh\"\n";
showEnvVarsInLog = 0;
};
C8513F70D957F286002F02EC /* [CP] Embed Pods Frameworks */ = {
8024DE7DD9C34304BE2FBE90 /* [CP] Embed Pods Frameworks */ = {
isa = PBXShellScriptBuildPhase;
buildActionMask = 2147483647;
files = (
Expand All @@ -308,6 +289,25 @@
shellScript = "\"${PODS_ROOT}/Target Support Files/Pods-EVIExample/Pods-EVIExample-frameworks.sh\"\n";
showEnvVarsInLog = 0;
};
B686FFB6D6778EC6C0E932AB /* [Expo] Configure project */ = {
isa = PBXShellScriptBuildPhase;
alwaysOutOfDate = 1;
buildActionMask = 2147483647;
files = (
);
inputFileListPaths = (
);
inputPaths = (
);
name = "[Expo] Configure project";
outputFileListPaths = (
);
outputPaths = (
);
runOnlyForDeploymentPostprocessing = 0;
shellPath = /bin/sh;
shellScript = "# This script configures Expo modules and generates the modules provider file.\nbash -l -c \"./Pods/Target\\ Support\\ Files/Pods-EVIExample/expo-configure-project.sh\"\n";
};
/* End PBXShellScriptBuildPhase section */

/* Begin PBXSourcesBuildPhase section */
Expand All @@ -318,7 +318,7 @@
13B07FBC1A68108700A75B9A /* AppDelegate.mm in Sources */,
13B07FC11A68108700A75B9A /* main.m in Sources */,
B18059E884C0ABDD17F3DC3D /* ExpoModulesProvider.swift in Sources */,
54987443346E4D38A033F481 /* noop-file.swift in Sources */,
2D7A14CEDB964A6BB821853B /* noop-file.swift in Sources */,
);
runOnlyForDeploymentPostprocessing = 0;
};
Expand Down Expand Up @@ -349,7 +349,7 @@
);
OTHER_SWIFT_FLAGS = "$(inherited) -D EXPO_CONFIGURATION_DEBUG";
PRODUCT_BUNDLE_IDENTIFIER = com.example.EVIExample;
PRODUCT_NAME = EVIExample;
PRODUCT_NAME = "EVIExample";
SWIFT_OBJC_BRIDGING_HEADER = "EVIExample/EVIExample-Bridging-Header.h";
SWIFT_OPTIMIZATION_LEVEL = "-Onone";
SWIFT_VERSION = 5.0;
Expand Down Expand Up @@ -377,7 +377,7 @@
);
OTHER_SWIFT_FLAGS = "$(inherited) -D EXPO_CONFIGURATION_RELEASE";
PRODUCT_BUNDLE_IDENTIFIER = com.example.EVIExample;
PRODUCT_NAME = EVIExample;
PRODUCT_NAME = "EVIExample";
SWIFT_OBJC_BRIDGING_HEADER = "EVIExample/EVIExample-Bridging-Header.h";
SWIFT_VERSION = 5.0;
TARGETED_DEVICE_FAMILY = "1,2";
Expand Down
2 changes: 2 additions & 0 deletions evi-react-native-example/EVIExample/ios/EVIExample/Info.plist
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,8 @@
<key>NSAllowsLocalNetworking</key>
<true/>
</dict>
<key>NSMicrophoneUsageDescription</key>
<string>This app uses the microphone to allow the user to talk to the EVI conversational AI interface</string>
<key>UILaunchStoryboardName</key>
<string>SplashScreen</string>
<key>UIRequiredDeviceCapabilities</key>
Expand Down
23 changes: 13 additions & 10 deletions evi-react-native-example/EVIExample/modules/audio/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,25 +3,28 @@ import { NativeModulesProxy, EventEmitter, Subscription } from 'expo-modules-cor
// Import the native module. On web, it will be resolved to Audio.web.ts
// and on native platforms to Audio.ts
import AudioModule from './src/AudioModule';
import { BytesEventPayload } from './src/Audio.types';
import { AudioEventPayload } from './src/Audio.types';

export async function getPermissions(value: string) {
return await AudioModule.setValueAsync(value);
export async function getPermissions(): Promise<void> {
return await AudioModule.getPermissions();
}

export async function startRecording(): Promise<void> {
return await AudioModule.startRecording();
await AudioModule.startRecording();
}

export async function stopRecording(): Promise<void> {
return await AudioModule.stopRecording();
export async function playAudio(base64EncodedAudio: string): Promise<void> {
await AudioModule.playAudio(base64EncodedAudio);
}


const emitter = new EventEmitter(AudioModule ?? NativeModulesProxy.Audio);
export async function stopRecording(): Promise<void> {
emitter.removeAllListeners('onAudioInput');
return await AudioModule.stopRecording();
}

export function onBytes(listener: (event: BytesEventPayload) => void): Subscription {
return emitter.addListener<BytesEventPayload>('onBytes', listener);
export function onAudioInput(listener: (event: AudioEventPayload) => void): Subscription {
return emitter.addListener<AudioEventPayload>('onAudioInput', listener);
}

export { BytesEventPayload };
export { AudioEventPayload };
Loading

0 comments on commit f944e44

Please sign in to comment.