diff --git a/evi-react-native-example/EVIExample/App.tsx b/evi-react-native-example/EVIExample/App.tsx index d9978ef..c8ecb23 100644 --- a/evi-react-native-example/EVIExample/App.tsx +++ b/evi-react-native-example/EVIExample/App.tsx @@ -110,13 +110,15 @@ const App = () => { // system audio to linear 16 PCM, a standard format recognized by EVI. For linear16 PCM // you must send a `session_settings` message to EVI to inform EVI of the // correct sampling rate. - chatSocket.sendSessionSettings({ - audio: { - encoding: "linear16", - channels: 1, - sampleRate: NativeAudio.sampleRate, - }, - }); + if (NativeAudio.isLinear16PCM) { + chatSocket.sendSessionSettings({ + audio: { + encoding: "linear16", + channels: 1, + sampleRate: NativeAudio.sampleRate, + }, + }); + } }); chatSocket.on("message", handleIncomingMessage); @@ -135,6 +137,7 @@ const App = () => { if (chatSocket.readyState !== WebSocket.OPEN) { return; } + console.log(typeof base64EncodedAudio) chatSocket.sendAudioInput({ data: base64EncodedAudio }); } ); diff --git a/evi-react-native-example/EVIExample/modules/audio/index.ts b/evi-react-native-example/EVIExample/modules/audio/index.ts index 8bf8dfe..5e5410c 100644 --- a/evi-react-native-example/EVIExample/modules/audio/index.ts +++ b/evi-react-native-example/EVIExample/modules/audio/index.ts @@ -30,6 +30,7 @@ export async function unmute(): Promise { } export const sampleRate = AudioModule.sampleRate +export const isLinear16PCM = AudioModule.isLinear16PCM const emitter = new EventEmitter(AudioModule ?? NativeModulesProxy.Audio); export async function stopRecording(): Promise { diff --git a/evi-react-native-example/EVIExample/modules/audio/ios/AudioModule.swift b/evi-react-native-example/EVIExample/modules/audio/ios/AudioModule.swift index 12c546f..42df0c6 100644 --- a/evi-react-native-example/EVIExample/modules/audio/ios/AudioModule.swift +++ b/evi-react-native-example/EVIExample/modules/audio/ios/AudioModule.swift @@ -20,7 +20,7 @@ public class AudioModule: Module { public func definition() -> ModuleDefinition { Name("Audio") - Constants(["sampleRate": Microphone.sampleRate]) + Constants(["sampleRate": Microphone.sampleRate, "isLinear16PCM": Microphone.isLinear16PCM]) Events("onAudioInput", "onError") diff --git a/evi-react-native-example/EVIExample/modules/audio/src/AudioModule.web.ts b/evi-react-native-example/EVIExample/modules/audio/src/AudioModule.web.ts index 24d3de5..1d10054 100644 --- a/evi-react-native-example/EVIExample/modules/audio/src/AudioModule.web.ts +++ b/evi-react-native-example/EVIExample/modules/audio/src/AudioModule.web.ts @@ -1,25 +1,77 @@ import { EventEmitter } from 'expo-modules-core'; +import { convertBlobToBase64, convertBase64ToBlob, getAudioStream, ensureSingleValidAudioTrack, getBrowserSupportedMimeType, MimeType } from 'hume'; const emitter = new EventEmitter({} as any); +let recorder: MediaRecorder | null = null; +let audioStream: MediaStream | null = null; +let currentAudio: HTMLAudioElement | null = null; +let isMuted = false; + +const mimeType: MimeType = (() => { + const result = getBrowserSupportedMimeType(); + return result.success ? result.mimeType : MimeType.WEBM; +})(); export default { async getPermissions() { - console.log('Pretending to get permissions...') + console.log('Requesting microphone permissions...'); + await navigator.mediaDevices.getUserMedia({ audio: true }); + console.log('Microphone permissions granted.'); }, + async startRecording(): Promise { - console.log('Pretending to start recording...') + console.log('Starting audio recording...'); + + audioStream = await getAudioStream(); + ensureSingleValidAudioTrack(audioStream); + + recorder = new MediaRecorder(audioStream, { mimeType }); + console.log(recorder) + + recorder.ondataavailable = async ({ data }) => { + if (isMuted) return; + if (data.size < 1) return; + + const base64EncodedAudio = await convertBlobToBase64(data); + emitter.emit('onAudioInput', {base64EncodedAudio}); + }; + + recorder.start(100); // Record audio in 100ms slices + console.log('Audio recording started.'); }, + async stopRecording(): Promise { - console.log('Pretending to stop recording...') - //emitter.removeAllListeners('onAudioInput'); + console.log('Stopping audio recording...'); + recorder?.stop(); + recorder = null; + audioStream?.getTracks().forEach(track => track.stop()); + audioStream = null; + console.log('Audio recording stopped.'); }, + async playAudio(base64EncodedAudio: string): Promise { - console.log('Pretending to play audio...') + console.log('Playing audio...'); + return new Promise((resolve) => { + const audioBlob = convertBase64ToBlob(base64EncodedAudio, mimeType!); + const audioUrl = URL.createObjectURL(audioBlob); + currentAudio = new Audio(audioUrl); + currentAudio.onended = () => resolve() + currentAudio.play(); + }) }, + async mute(): Promise { - console.log('Pretending to mute...') + isMuted = true; }, + async unmute(): Promise { - console.log('Pretending to unmute...') - } + isMuted = false; + }, + + async stopPlayback(): Promise { + currentAudio?.pause(); + currentAudio = null; + }, + + isLinear16PCM: false, };