feat(react-sdk)!: Universal Device Management API (#1127)

Removes the React-specific device management API and replaces it with the universal API that Plain-JS and React Native uses. ### New features - Introduced/improved the following `useCallStateHooks()`: ```ts const { useMicrophoneState, useCameraState, useSpeakerState, useScreenShareState } = useCallStateHooks(); const { microphone, // a shortcut to `call.microphone` -> `call.microphone.toggle()` vs. `microphone.toggle()` isMute, // mic mute state devices, // a list of available microphones to the system selectedDevice, // current mic in use isSpeakingWhileMuted, // true when current user is speaking, while being muted for the others mediaStream, // the current `MediaStream` instance } = useMicrophoneState(); // the other hooks follow a similar pattern ``` - Added `usePersistedDevicePreferences()`: Allows integrators to opt-in to our SDK-default device preferences management - Exposes `useRequestPermission()`: A hook that makes it easier to work with permission requests ### Breaking Changes #### Core SDK - `StreamVideoLocalParticipant` type is removed, as now `audioDeviceId`, `videoDeviceId`, and `audioOutputDeviceId` are managed by the universal device management API - `call.state.localParticipant` now has `StreamVideoParticipant` type - `call.setAudioOutputDevice`, `call.setAudioDevice` and `call.setVideoDevice` are replaced with `call.speakers.select`, `call.microphone.select` and `call.camera.select` APIs #### React SDK - `useAudioPublisher` hook is removed, use `call.microphone.*` APIs. - `useVideoPublisher` hook is removed, use `call.camera.*` APIs - `useToggleAudioMuteState` hook is now replaced with `call.microphone.toggle()` - `useToggleVideoMuteState` hook is now replaced with `call.camera.toggle()` - `useToggleScreenShare` hook is now replaced with `call.screenShare.toggle()` - `<MediaDevicesProvider />` and `useMediaDevices()` hook are removed from the SDK. Respectively, `<StreamCall />` doesn't accept a `mediaDevicesProviderProps` prop anymore. Device initialization can now be done through the exposed `call` instance APIs: ```ts const call = client.call('my-call-type', 'my-call-id'); call.microphone.select(initialAudioInputDeviceId); // replacement for `initialAudioInputDeviceId` call.microphone.enable(); // replacement for `initialAudioEnabled = true` call.camera.select(initialVideoInputDeviceId); // replacement for `initialVideoInputDeviceId` call.camera.enable(); // replacement for `initialVideoEnabled = true` ``` - `useDevices()`, `useVideoDevices()`, `useAudioInputDevices()`, and `useAudioOutputDevices()` are removed. Replacement APIs: ```ts const { useMicrophoneState, useCameraState, useSpeakerState } = useCallStateHooks(); const { devices: mics } = useMicrophoneState(); const { devices: cameras } = useCameraState(); const { devices: speakers } = useSpeakerState(); ```
GetStream · Oct 27, 2023 · aeb3561 · aeb3561
1 parent c537812
commit aeb3561
Show file tree

Hide file tree

Showing 102 changed files with 961 additions and 3,106 deletions.
diff --git a/packages/client/src/Call.ts b/packages/client/src/Call.ts
@@ -9,11 +9,7 @@ import {
   Subscriber,
 } from './rtc';
 import { muteTypeToTrackType } from './rtc/helpers/tracks';
-import {
-  GoAwayReason,
-  SdkType,
-  TrackType,
-} from './gen/video/sfu/models/models';
+import { GoAwayReason, TrackType } from './gen/video/sfu/models/models';
 import {
   registerEventHandlers,
   registerRingingCallEventHandlers,
@@ -120,7 +116,7 @@ import {
   Logger,
   StreamCallEvent,
 } from './coordinator/connection/types';
-import { getClientDetails, getSdkInfo } from './client-details';
+import { getClientDetails } from './client-details';
 import { getLogger } from './logger';
 import {
   CameraDirection,
@@ -1004,14 +1000,11 @@ export class Call {
       this.reconnectAttempts = 0; // reset the reconnect attempts counter
       this.state.setCallingState(CallingState.JOINED);
 
-      // React uses a different device management for now
-      if (getSdkInfo()?.type !== SdkType.REACT) {
-        try {
-          await this.initCamera();
-          await this.initMic();
-        } catch (error) {
-          this.logger('warn', 'Camera and/or mic init failed during join call');
-        }
+      try {
+        await this.initCamera();
+        await this.initMic();
+      } catch (error) {
+        this.logger('warn', 'Camera and/or mic init failed during join call');
       }
 
       // 3. once we have the "joinResponse", and possibly reconciled the local state
@@ -1322,56 +1315,6 @@ export class Call {
     return this.statsReporter?.stopReportingStatsFor(sessionId);
   };
 
-  /**
-   * Sets the used audio output device (`audioOutputDeviceId` of the [`localParticipant$`](./StreamVideoClient.md/#readonlystatestore).
-   *
-   * This method only stores the selection, if you're using custom UI components, you'll have to implement the audio switching, for more information see: https://developer.mozilla.org/en-US/docs/Web/API/HTMLMediaElement/sinkId.
-   *
-   *
-   * @param deviceId the selected device, `undefined` means the user wants to use the system's default audio output
-   *
-   * @deprecated use `call.speaker` instead
-   */
-  setAudioOutputDevice = (deviceId?: string) => {
-    if (!this.sfuClient) return;
-    this.state.updateParticipant(this.sfuClient.sessionId, {
-      audioOutputDeviceId: deviceId,
-    });
-  };
-
-  /**
-   * Sets the `audioDeviceId` property of the [`localParticipant$`](./StreamVideoClient.md/#readonlystatestore)).
-   *
-   * This method only stores the selection, if you want to start publishing a media stream call the [`publishAudioStream` method](#publishaudiostream) that will set `audioDeviceId` as well.
-   *
-   *
-   * @param deviceId the selected device, pass `undefined` to clear the device selection
-   *
-   * @deprecated use call.microphone.select
-   */
-  setAudioDevice = (deviceId?: string) => {
-    if (!this.sfuClient) return;
-    this.state.updateParticipant(this.sfuClient.sessionId, {
-      audioDeviceId: deviceId,
-    });
-  };
-
-  /**
-   * Sets the `videoDeviceId` property of the [`localParticipant$`](./StreamVideoClient.md/#readonlystatestore).
-   *
-   * This method only stores the selection, if you want to start publishing a media stream call the [`publishVideoStream` method](#publishvideostream) that will set `videoDeviceId` as well.
-   *
-   * @param deviceId the selected device, pass `undefined` to clear the device selection
-   *
-   * @deprecated use call.camera.select
-   */
-  setVideoDevice = (deviceId?: string) => {
-    if (!this.sfuClient) return;
-    this.state.updateParticipant(this.sfuClient.sessionId, {
-      videoDeviceId: deviceId,
-    });
-  };
-
   /**
    * Resets the last sent reaction for the user holding the given `sessionId`. This is a local action, it won't reset the reaction on the backend.
    *

diff --git a/packages/client/src/devices/InputMediaDeviceManager.ts b/packages/client/src/devices/InputMediaDeviceManager.ts
@@ -136,7 +136,7 @@ export abstract class InputMediaDeviceManager<
     }
   }
 
-  protected abstract getDevices(): Observable<MediaDeviceInfo[]>;
+  protected abstract getDevices(): Observable<MediaDeviceInfo[] | undefined>;
 
   protected abstract getStream(constraints: C): Promise<MediaStream>;
 

diff --git a/packages/client/src/helpers/DynascaleManager.ts b/packages/client/src/helpers/DynascaleManager.ts
@@ -2,18 +2,12 @@ import { Call } from '../Call';
 import {
   AudioTrackType,
   DebounceType,
-  StreamVideoLocalParticipant,
   StreamVideoParticipant,
   VideoTrackType,
   VisibilityState,
 } from '../types';
+import { TrackType, VideoDimension } from '../gen/video/sfu/models/models';
 import {
-  SdkType,
-  TrackType,
-  VideoDimension,
-} from '../gen/video/sfu/models/models';
-import {
-  combineLatest,
   distinctUntilChanged,
   distinctUntilKeyChanged,
   map,
@@ -22,7 +16,6 @@ import {
 } from 'rxjs';
 import { ViewportTracker } from './ViewportTracker';
 import { getLogger } from '../logger';
-import { getSdkInfo } from '../client-details';
 import { isFirefox, isSafari } from './browsers';
 
 const DEFAULT_VIEWPORT_VISIBILITY_STATE: Record<
@@ -174,7 +167,7 @@ export class DynascaleManager {
         (participants) =>
           participants.find(
             (participant) => participant.sessionId === sessionId,
-          ) as StreamVideoLocalParticipant | StreamVideoParticipant,
+          ) as StreamVideoParticipant,
       ),
       takeWhile((participant) => !!participant),
       distinctUntilChanged(),
@@ -339,9 +332,9 @@ export class DynascaleManager {
     const participant$ = this.call.state.participants$.pipe(
       map(
         (participants) =>
-          participants.find((p) => p.sessionId === sessionId) as
-            | StreamVideoLocalParticipant
-            | StreamVideoParticipant,
+          participants.find(
+            (p) => p.sessionId === sessionId,
+          ) as StreamVideoParticipant,
       ),
       takeWhile((p) => !!p),
       distinctUntilChanged(),
@@ -373,20 +366,14 @@ export class DynascaleManager {
         });
       });
 
-    const sinkIdSubscription = combineLatest([
-      this.call.state.localParticipant$,
-      this.call.speaker.state.selectedDevice$,
-    ]).subscribe(([p, selectedDevice]) => {
-      const deviceId =
-        getSdkInfo()?.type === SdkType.REACT
-          ? p?.audioOutputDeviceId
-          : selectedDevice;
-
-      if ('setSinkId' in audioElement && typeof deviceId === 'string') {
-        // @ts-expect-error setSinkId is not yet in the lib
-        audioElement.setSinkId(deviceId);
-      }
-    });
+    const sinkIdSubscription = !('setSinkId' in audioElement)
+      ? null
+      : this.call.speaker.state.selectedDevice$.subscribe((deviceId) => {
+          if (deviceId) {
+            // @ts-expect-error setSinkId is not yet in the lib
+            audioElement.setSinkId(deviceId);
+          }
+        });
 
     const volumeSubscription = this.call.speaker.state.volume$.subscribe(
       (volume) => {
@@ -397,7 +384,7 @@ export class DynascaleManager {
     audioElement.autoplay = true;
 
     return () => {
-      sinkIdSubscription.unsubscribe();
+      sinkIdSubscription?.unsubscribe();
       volumeSubscription.unsubscribe();
       updateMediaStreamSubscription.unsubscribe();
     };

diff --git a/packages/client/src/helpers/__tests__/DynascaleManager.test.ts b/packages/client/src/helpers/__tests__/DynascaleManager.test.ts
@@ -4,21 +4,14 @@
 
 import '../../rtc/__tests__/mocks/webrtc.mocks';
 
-import { afterEach, beforeEach, describe, expect, it, Mock, vi } from 'vitest';
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
 import { DynascaleManager } from '../DynascaleManager';
 import { Call } from '../../Call';
 import { StreamClient } from '../../coordinator/connection/client';
 import { StreamVideoWriteableStateStore } from '../../store';
 import { DebounceType, VisibilityState } from '../../types';
 import { noopComparator } from '../../sorting';
-import { SdkType, TrackType } from '../../gen/video/sfu/models/models';
-import { getSdkInfo } from '../../client-details';
-
-vi.mock('../../client-details.ts', () => {
-  return {
-    getSdkInfo: vi.fn(),
-  };
-});
+import { TrackType } from '../../gen/video/sfu/models/models';
 
 describe('DynascaleManager', () => {
   let dynascaleManager: DynascaleManager;
@@ -145,7 +138,7 @@ describe('DynascaleManager', () => {
       expect(audioElement.volume).toBe(1);
 
       // @ts-expect-error setSinkId is not defined in types
-      expect(audioElement.setSinkId).toHaveBeenCalledWith('');
+      expect(audioElement.setSinkId).not.toHaveBeenCalled();
 
       call.speaker.select('different-device-id');
 
@@ -154,18 +147,6 @@ describe('DynascaleManager', () => {
         'different-device-id',
       );
 
-      const mock = getSdkInfo as Mock;
-      mock.mockImplementation(() => ({
-        type: SdkType.REACT,
-      }));
-
-      call.state.updateParticipant('session-id-local', {
-        audioOutputDeviceId: 'new-device-id',
-      });
-
-      // @ts-expect-error setSinkId is not defined in types
-      expect(audioElement.setSinkId).toHaveBeenCalledWith('new-device-id');
-
       call.speaker.setVolume(0.5);
 
       expect(audioElement.volume).toBe(0.5);

diff --git a/packages/client/src/permissions/PermissionsContext.ts b/packages/client/src/permissions/PermissionsContext.ts
@@ -44,11 +44,15 @@ export class PermissionsContext {
    * within the call.
    *
    * @param permission the permission to check for.
+   * @param settings the call settings to check against (optional).
    */
-  canRequest = (permission: OwnCapability) => {
-    if (!this.settings) return false;
+  canRequest = (
+    permission: OwnCapability,
+    settings: CallSettingsResponse | undefined = this.settings,
+  ) => {
+    if (!settings) return false;
 
-    const { audio, video, screensharing } = this.settings;
+    const { audio, video, screensharing } = settings;
     switch (permission) {
       case OwnCapability.SEND_AUDIO:
         return audio.access_request_enabled;

diff --git a/packages/client/src/rtc/Publisher.ts b/packages/client/src/rtc/Publisher.ts
@@ -14,10 +14,7 @@ import {
   OptimalVideoLayer,
 } from './videoLayers';
 import { getPreferredCodecs } from './codecs';
-import {
-  trackTypeToDeviceIdKey,
-  trackTypeToParticipantStreamKey,
-} from './helpers/tracks';
+import { trackTypeToParticipantStreamKey } from './helpers/tracks';
 import { CallState } from '../store';
 import { PublishOptions } from '../types';
 import { isReactNative } from '../helpers/platforms';
@@ -388,14 +385,11 @@ export class Publisher {
         [audioOrVideoOrScreenShareStream]: undefined,
       }));
     } else {
-      const deviceId = track.getSettings().deviceId;
-      const audioOrVideoDeviceKey = trackTypeToDeviceIdKey(trackType);
       this.state.updateParticipant(this.sfuClient.sessionId, (p) => {
         return {
           publishedTracks: p.publishedTracks.includes(trackType)
             ? p.publishedTracks
             : [...p.publishedTracks, trackType],
-          ...(audioOrVideoDeviceKey && { [audioOrVideoDeviceKey]: deviceId }),
           [audioOrVideoOrScreenShareStream]: mediaStream,
         };
       });

diff --git a/packages/client/src/rtc/__tests__/Publisher.test.ts b/packages/client/src/rtc/__tests__/Publisher.test.ts
@@ -99,7 +99,6 @@ describe('Publisher', () => {
     // initial publish
     await publisher.publishStream(mediaStream, track, TrackType.VIDEO);
 
-    expect(state.localParticipant?.videoDeviceId).toEqual('test-device-id');
     expect(state.localParticipant?.publishedTracks).toContain(TrackType.VIDEO);
     expect(state.localParticipant?.videoStream).toEqual(mediaStream);
     expect(transceiver.setCodecPreferences).toHaveBeenCalled();
@@ -136,15 +135,13 @@ describe('Publisher', () => {
       expect.any(Function),
     );
     expect(transceiver.sender.replaceTrack).toHaveBeenCalledWith(newTrack);
-    expect(state.localParticipant?.videoDeviceId).toEqual('test-device-id-2');
 
     // stop publishing
     await publisher.unpublishStream(TrackType.VIDEO, true);
     expect(newTrack.stop).toHaveBeenCalled();
     expect(state.localParticipant?.publishedTracks).not.toContain(
       TrackType.VIDEO,
     );
-    expect(state.localParticipant?.videoDeviceId).toEqual('test-device-id-2');
   });
 
   it('can publish and un-pubish with just enabling and disabling tracks', async () => {
@@ -178,7 +175,6 @@ describe('Publisher', () => {
     // initial publish
     await publisher.publishStream(mediaStream, track, TrackType.VIDEO);
 
-    expect(state.localParticipant?.videoDeviceId).toEqual('test-device-id');
     expect(state.localParticipant?.publishedTracks).toContain(TrackType.VIDEO);
     expect(track.enabled).toBe(true);
     expect(state.localParticipant?.videoStream).toEqual(mediaStream);

diff --git a/packages/client/src/rtc/flows/join.ts b/packages/client/src/rtc/flows/join.ts
@@ -3,12 +3,7 @@ import {
   JoinCallRequest,
   JoinCallResponse,
 } from '../../gen/coordinator';
-import {
-  isStreamVideoLocalParticipant,
-  JoinCallData,
-  StreamVideoLocalParticipant,
-  StreamVideoParticipant,
-} from '../../types';
+import { JoinCallData, StreamVideoParticipant } from '../../types';
 import { StreamClient } from '../../coordinator/connection/client';
 
 /**
@@ -107,21 +102,11 @@ const getCascadingModeParams = () => {
  * @param source the participant to reconcile from.
  */
 export const reconcileParticipantLocalState = (
-  target: StreamVideoParticipant | StreamVideoLocalParticipant,
-  source?: StreamVideoParticipant | StreamVideoLocalParticipant,
+  target: StreamVideoParticipant,
+  source?: StreamVideoParticipant,
 ) => {
   if (!source) return target;
 
   // copy everything from source to target
-  Object.assign(target, source);
-
-  if (
-    isStreamVideoLocalParticipant(source) &&
-    isStreamVideoLocalParticipant(target)
-  ) {
-    target.audioDeviceId = source.audioDeviceId;
-    target.videoDeviceId = source.videoDeviceId;
-    target.audioOutputDeviceId = source.audioOutputDeviceId;
-  }
-  return target;
+  return Object.assign(target, source);
 };
diff --git a/packages/client/src/rtc/helpers/tracks.ts b/packages/client/src/rtc/helpers/tracks.ts
@@ -1,8 +1,5 @@
 import { TrackType } from '../../gen/video/sfu/models/models';
-import type {
-  StreamVideoLocalParticipant,
-  StreamVideoParticipant,
-} from '../../types';
+import type { StreamVideoParticipant } from '../../types';
 import { TrackMuteType } from '../../types';
 
 export const trackTypeToParticipantStreamKey = (
@@ -25,24 +22,6 @@ export const trackTypeToParticipantStreamKey = (
   }
 };
 
-export const trackTypeToDeviceIdKey = (
-  trackType: TrackType,
-): keyof StreamVideoLocalParticipant | undefined => {
-  switch (trackType) {
-    case TrackType.AUDIO:
-      return 'audioDeviceId';
-    case TrackType.VIDEO:
-      return 'videoDeviceId';
-    case TrackType.SCREEN_SHARE:
-    case TrackType.SCREEN_SHARE_AUDIO:
-    case TrackType.UNSPECIFIED:
-      return undefined;
-    default:
-      const exhaustiveTrackTypeCheck: never = trackType;
-      throw new Error(`Unknown track type: ${exhaustiveTrackTypeCheck}`);
-  }
-};
-
 export const muteTypeToTrackType = (muteType: TrackMuteType): TrackType => {
   switch (muteType) {
     case 'audio':
-Original file line number
+Diff line change
@@ Expand Up / @@ -136,7 +136,7 @@ export abstract class InputMediaDeviceManager< @@
         }
       }
-      protected abstract getDevices(): Observable<MediaDeviceInfo[]>;
+      protected abstract getDevices(): Observable<MediaDeviceInfo[] | undefined>;
       protected abstract getStream(constraints: C): Promise<MediaStream>;
@@ Expand Down @@