From 4cbe21f1802c3d158d08ee7bf325cd0f1c0c05e1 Mon Sep 17 00:00:00 2001 From: Khushal Agarwal Date: Tue, 28 Nov 2023 10:43:47 +0530 Subject: [PATCH 1/4] fix(react-native): receive audioLevels when joined the call with mic muted --- packages/client/src/Call.ts | 11 +++++++++-- packages/client/src/devices/MicrophoneManager.ts | 10 ++++++++++ packages/client/src/devices/MicrophoneManagerState.ts | 4 ---- 3 files changed, 19 insertions(+), 6 deletions(-) diff --git a/packages/client/src/Call.ts b/packages/client/src/Call.ts index 234f90189a..476d1bd523 100644 --- a/packages/client/src/Call.ts +++ b/packages/client/src/Call.ts @@ -125,6 +125,7 @@ import { ScreenShareManager, SpeakerManager, } from './devices'; +import { isReactNative } from './helpers/platforms'; /** * An object representation of a `Call`. @@ -1869,11 +1870,17 @@ export class Call { if (options.setStatus) { // Publish media stream that was set before we joined if ( - this.microphone.state.status === 'enabled' && this.microphone.state.mediaStream && !this.publisher?.isPublishing(TrackType.AUDIO) ) { - await this.publishAudioStream(this.microphone.state.mediaStream); + if (!isReactNative()) { + if (this.microphone.state.status === 'enabled') { + await this.publishAudioStream(this.microphone.state.mediaStream); + } + } else { + // In case of React Native we need to publish the stream everytime to get the audioLevels + await this.publishAudioStream(this.microphone.state.mediaStream); + } } // Start mic if backend config specifies, and there is no local setting diff --git a/packages/client/src/devices/MicrophoneManager.ts b/packages/client/src/devices/MicrophoneManager.ts index c9d640e25b..e92f058498 100644 --- a/packages/client/src/devices/MicrophoneManager.ts +++ b/packages/client/src/devices/MicrophoneManager.ts @@ -61,6 +61,16 @@ export class MicrophoneManager extends InputMediaDeviceManager { diff --git a/packages/client/src/devices/MicrophoneManagerState.ts b/packages/client/src/devices/MicrophoneManagerState.ts index c7a48bb0da..3a1e388cae 100644 --- a/packages/client/src/devices/MicrophoneManagerState.ts +++ b/packages/client/src/devices/MicrophoneManagerState.ts @@ -6,8 +6,6 @@ export class MicrophoneManagerState extends InputMediaDeviceManagerState { /** * An Observable that emits `true` if the user's microphone is muted but they'are speaking. - * - * This feature is not available in the React Native SDK. */ speakingWhileMuted$: Observable; @@ -26,8 +24,6 @@ export class MicrophoneManagerState extends InputMediaDeviceManagerState { /** * `true` if the user's microphone is muted but they'are speaking. - * - * This feature is not available in the React Native SDK. */ get speakingWhileMuted() { return this.getCurrentValue(this.speakingWhileMuted$); From 14ae3c06302f742ae666dbd93e5a3abaf495256e Mon Sep 17 00:00:00 2001 From: Khushal Agarwal Date: Tue, 28 Nov 2023 10:44:05 +0530 Subject: [PATCH 2/4] fix(react-native): styling fix --- .../dogfood/src/components/CallControlsComponent.tsx | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/sample-apps/react-native/dogfood/src/components/CallControlsComponent.tsx b/sample-apps/react-native/dogfood/src/components/CallControlsComponent.tsx index e51a60e12e..75f9997fd5 100644 --- a/sample-apps/react-native/dogfood/src/components/CallControlsComponent.tsx +++ b/sample-apps/react-native/dogfood/src/components/CallControlsComponent.tsx @@ -42,7 +42,7 @@ export const CallControlsComponent = ({ }; return ( - + <> {isSpeakingWhileMuted && ( You are muted. Unmute to speak. @@ -60,7 +60,7 @@ export const CallControlsComponent = ({ - + ); }; @@ -68,7 +68,6 @@ const styles = StyleSheet.create({ speakingLabelContainer: { backgroundColor: appTheme.colors.static_overlay, paddingVertical: 10, - width: '100%', }, label: { textAlign: 'center', From a11dbc98f365c329cba108fa9ea961210c31511c Mon Sep 17 00:00:00 2001 From: Khushal Agarwal Date: Tue, 28 Nov 2023 17:42:07 +0530 Subject: [PATCH 3/4] refactor: muteStream and unmuteStream to muteOrDestrouStream and unmuteOrCreateStream --- packages/client/src/Call.ts | 12 ++- .../src/devices/InputMediaDeviceManager.ts | 94 ++++++++++++------- .../client/src/devices/MicrophoneManager.ts | 22 ++--- packages/client/src/rtc/Publisher.ts | 6 +- packages/client/src/types.ts | 4 + .../react-native/dogfood/ios/Podfile.lock | 4 +- 6 files changed, 90 insertions(+), 52 deletions(-) diff --git a/packages/client/src/Call.ts b/packages/client/src/Call.ts index 476d1bd523..0c3f1d77a0 100644 --- a/packages/client/src/Call.ts +++ b/packages/client/src/Call.ts @@ -1110,8 +1110,12 @@ export class Call { * The previous audio stream will be stopped. * * @param audioStream the audio stream to publish. + * @param disableTrackWhilePublish if the track should be disabled while published. This is mainly useful for React Native SDK. */ - publishAudioStream = async (audioStream: MediaStream) => { + publishAudioStream = async ( + audioStream: MediaStream, + disableTrackWhilePublish?: boolean, + ) => { // we should wait until we get a JoinResponse from the SFU, // otherwise we risk breaking the ICETrickle flow. await this.assertCallJoined(); @@ -1130,6 +1134,7 @@ export class Call { audioStream, audioTrack, TrackType.AUDIO, + { disableTrackWhilePublish }, ); }; @@ -1879,7 +1884,10 @@ export class Call { } } else { // In case of React Native we need to publish the stream everytime to get the audioLevels - await this.publishAudioStream(this.microphone.state.mediaStream); + await this.publishAudioStream( + this.microphone.state.mediaStream, + true, + ); } } diff --git a/packages/client/src/devices/InputMediaDeviceManager.ts b/packages/client/src/devices/InputMediaDeviceManager.ts index eb345ca370..0a353d034c 100644 --- a/packages/client/src/devices/InputMediaDeviceManager.ts +++ b/packages/client/src/devices/InputMediaDeviceManager.ts @@ -8,6 +8,13 @@ import { getLogger } from '../logger'; import { TrackType } from '../gen/video/sfu/models/models'; import { deviceIds$ } from './devices'; +export type UnmuteOrCreateStreamSettings = { + /** + * Create a stream that is muted. Useful for React Native SDK audioLevel Stats. + */ + createMutedStream?: boolean; +}; + export abstract class InputMediaDeviceManager< T extends InputMediaDeviceManagerState, C = MediaTrackConstraints, @@ -55,7 +62,7 @@ export abstract class InputMediaDeviceManager< */ async enable() { if (this.state.status === 'enabled') return; - this.enablePromise = this.unmuteStream(); + this.enablePromise = this.unmuteOrCreateStream(); try { await this.enablePromise; this.state.setStatus('enabled'); @@ -73,7 +80,7 @@ export abstract class InputMediaDeviceManager< this.state.prevStatus = this.state.status; if (this.state.status === 'disabled') return; const stopTracks = this.state.disableMode === 'stop-tracks'; - this.disablePromise = this.muteStream(stopTracks); + this.disablePromise = this.muteOrDestroyStream(stopTracks); try { await this.disablePromise; this.state.setStatus('disabled'); @@ -140,8 +147,8 @@ export abstract class InputMediaDeviceManager< protected async applySettingsToStream() { if (this.state.status === 'enabled') { - await this.muteStream(); - await this.unmuteStream(); + await this.muteOrDestroyStream(); + await this.unmuteOrCreateStream(); } } @@ -149,7 +156,10 @@ export abstract class InputMediaDeviceManager< protected abstract getStream(constraints: C): Promise; - protected abstract publishStream(stream: MediaStream): Promise; + protected abstract publishStream( + stream: MediaStream, + disableTrackWhilePublish?: boolean, + ): Promise; protected abstract stopPublishStream(stopTracks: boolean): Promise; @@ -157,27 +167,6 @@ export abstract class InputMediaDeviceManager< return this.state.mediaStream?.getTracks() ?? []; } - protected async muteStream(stopTracks: boolean = true) { - if (!this.state.mediaStream) return; - this.logger('debug', `${stopTracks ? 'Stopping' : 'Disabling'} stream`); - if (this.call.state.callingState === CallingState.JOINED) { - await this.stopPublishStream(stopTracks); - } - this.muteLocalStream(stopTracks); - const allEnded = this.getTracks().every((t) => t.readyState === 'ended'); - if (allEnded) { - if ( - this.state.mediaStream && - // @ts-expect-error release() is present in react-native-webrtc - typeof this.state.mediaStream.release === 'function' - ) { - // @ts-expect-error called to dispose the stream in RN - this.state.mediaStream.release(); - } - this.state.setMediaStream(undefined); - } - } - private muteTracks() { this.getTracks().forEach((track) => { if (track.enabled) track.enabled = false; @@ -207,25 +196,60 @@ export abstract class InputMediaDeviceManager< } } - protected async unmuteStream() { + protected async muteOrDestroyStream(stopTracks: boolean = true) { + if (!this.state.mediaStream) return; + this.logger('debug', `${stopTracks ? 'Stopping' : 'Disabling'} stream`); + if (this.call.state.callingState === CallingState.JOINED) { + await this.stopPublishStream(stopTracks); + } + this.muteLocalStream(stopTracks); + const allEnded = this.getTracks().every((t) => t.readyState === 'ended'); + if (allEnded) { + if ( + this.state.mediaStream && + // @ts-expect-error release() is present in react-native-webrtc + typeof this.state.mediaStream.release === 'function' + ) { + // @ts-expect-error called to dispose the stream in RN + this.state.mediaStream.release(); + } + this.state.setMediaStream(undefined); + } + } + + protected async unmuteOrCreateStream( + settings?: UnmuteOrCreateStreamSettings, + ) { this.logger('debug', 'Starting stream'); let stream: MediaStream; - if ( - this.state.mediaStream && - this.getTracks().every((t) => t.readyState === 'live') - ) { - stream = this.state.mediaStream; - this.unmuteTracks(); - } else { + if (!this.state.mediaStream && settings && settings.createMutedStream) { const defaultConstraints = this.state.defaultConstraints; const constraints: MediaTrackConstraints = { ...defaultConstraints, deviceId: this.state.selectedDevice, }; stream = await this.getStream(constraints as C); + this.state.setMediaStream(stream); + await this.muteOrDestroyStream(this.state.disableMode === 'stop-tracks'); + } else { + if ( + this.state.mediaStream && + this.getTracks().every((t) => t.readyState === 'live') + ) { + stream = this.state.mediaStream; + this.unmuteTracks(); + } else { + const defaultConstraints = this.state.defaultConstraints; + const constraints: MediaTrackConstraints = { + ...defaultConstraints, + deviceId: this.state.selectedDevice, + }; + stream = await this.getStream(constraints as C); + } } + if (this.call.state.callingState === CallingState.JOINED) { - await this.publishStream(stream); + await this.publishStream(stream, settings?.createMutedStream); } if (this.state.mediaStream !== stream) { this.state.setMediaStream(stream); diff --git a/packages/client/src/devices/MicrophoneManager.ts b/packages/client/src/devices/MicrophoneManager.ts index e92f058498..81c035b782 100644 --- a/packages/client/src/devices/MicrophoneManager.ts +++ b/packages/client/src/devices/MicrophoneManager.ts @@ -50,8 +50,11 @@ export class MicrophoneManager extends InputMediaDeviceManager { - return this.call.publishAudioStream(stream); + protected publishStream( + stream: MediaStream, + disableTrackWhilePublish?: boolean, + ): Promise { + return this.call.publishAudioStream(stream, disableTrackWhilePublish); } protected stopPublishStream(stopTracks: boolean): Promise { @@ -61,15 +64,12 @@ export class MicrophoneManager extends InputMediaDeviceManager 118.0.0) - React-Core - - stream-video-react-native (0.2.11): + - stream-video-react-native (0.2.14): - React-Core - stream-react-native-webrtc - TOCropViewController (2.6.1) @@ -835,7 +835,7 @@ SPEC CHECKSUMS: RNVoipPushNotification: 543e18f83089134a35e7f1d2eba4c8b1f7776b08 SocketRocket: fccef3f9c5cedea1353a9ef6ada904fde10d6608 stream-react-native-webrtc: 31fe9ee69d5b4fc191380a78efa292377494b7ac - stream-video-react-native: 87db9cd0f19ea8052db8ec74602f63847fe1816f + stream-video-react-native: 77590f3dfcdc79352de2b60996caad9b6f564829 TOCropViewController: edfd4f25713d56905ad1e0b9f5be3fbe0f59c863 Yoga: f7decafdc5e8c125e6fa0da38a687e35238420fa YogaKit: f782866e155069a2cca2517aafea43200b01fd5a From b5b1ecdb20961cf8337a5e6dad9f78e1150d190b Mon Sep 17 00:00:00 2001 From: Khushal Agarwal Date: Tue, 28 Nov 2023 17:52:12 +0530 Subject: [PATCH 4/4] fix: tests for microphone manager and inputdevicemanager --- .../client/src/devices/__tests__/InputMediaDeviceManager.test.ts | 1 + packages/client/src/devices/__tests__/MicrophoneManager.test.ts | 1 + 2 files changed, 2 insertions(+) diff --git a/packages/client/src/devices/__tests__/InputMediaDeviceManager.test.ts b/packages/client/src/devices/__tests__/InputMediaDeviceManager.test.ts index 9cab524b7d..479f3a07ad 100644 --- a/packages/client/src/devices/__tests__/InputMediaDeviceManager.test.ts +++ b/packages/client/src/devices/__tests__/InputMediaDeviceManager.test.ts @@ -96,6 +96,7 @@ describe('InputMediaDeviceManager.test', () => { expect(manager.publishStream).toHaveBeenCalledWith( manager.state.mediaStream, + undefined, ); }); diff --git a/packages/client/src/devices/__tests__/MicrophoneManager.test.ts b/packages/client/src/devices/__tests__/MicrophoneManager.test.ts index 3637dd7982..e8fe7e5ada 100644 --- a/packages/client/src/devices/__tests__/MicrophoneManager.test.ts +++ b/packages/client/src/devices/__tests__/MicrophoneManager.test.ts @@ -88,6 +88,7 @@ describe('MicrophoneManager', () => { expect(manager['call'].publishAudioStream).toHaveBeenCalledWith( manager.state.mediaStream, + undefined, ); });