diff --git a/packages/client/docusaurus/docs/javascript/02-guides/03-call-and-participant-state.mdx b/packages/client/docusaurus/docs/javascript/02-guides/03-call-and-participant-state.mdx index 694b1b5674..c2df0738bc 100644 --- a/packages/client/docusaurus/docs/javascript/02-guides/03-call-and-participant-state.mdx +++ b/packages/client/docusaurus/docs/javascript/02-guides/03-call-and-participant-state.mdx @@ -95,6 +95,7 @@ The `StreamVideoParticipant` object contains the following information: | `audioStream` | The published audio `MediaStream`. | | `videoStream` | The published video `MediaStream`. | | `screenShareStream` | The published screen share `MediaStream`. | +| `screenShareAudioStream` | The published screen share audio `MediaStream`. | | `isLocalParticipant` | It's `true` if the participant is the local participant. | | `pin` | Holds pinning information. | | `reaction` | The last reaction this user has sent to this call. | diff --git a/packages/client/docusaurus/docs/javascript/02-guides/04-camera-and-microphone.mdx b/packages/client/docusaurus/docs/javascript/02-guides/04-camera-and-microphone.mdx index aab4ddc4eb..4b01b0f9e5 100644 --- a/packages/client/docusaurus/docs/javascript/02-guides/04-camera-and-microphone.mdx +++ b/packages/client/docusaurus/docs/javascript/02-guides/04-camera-and-microphone.mdx @@ -11,13 +11,11 @@ If you want to see the device management API in action, you can check out [the s ### Start-stop camera ```typescript -const toggleCamera = () => { - call.camera.toggle(); +call.camera.toggle(); - // or - call.camera.enable(); - call.camera.disable(); -}; +// or +call.camera.enable(); +call.camera.disable(); ``` Here is how you can access the status: diff --git a/packages/client/docusaurus/docs/javascript/02-guides/04-screensharing.mdx b/packages/client/docusaurus/docs/javascript/02-guides/04-screensharing.mdx new file mode 100644 index 0000000000..ae293f20ff --- /dev/null +++ b/packages/client/docusaurus/docs/javascript/02-guides/04-screensharing.mdx @@ -0,0 +1,77 @@ +--- +id: screensharing +title: Screen Sharing +description: Managing Screen Sharing +--- + +If you want to see the device management API in action, you can check out [the sample app](https://github.com/GetStream/stream-video-js/tree/main/sample-apps/client/ts-quickstart). + +## Screen Sharing + +### Start/Stop Screen Sharing + +```typescript +call.screenShare.toggle(); + +// or +call.screenShare.enable(); +call.screenShare.disable(); +``` + +### Screen Sharing Status + +Here is how you can access the status of screen sharing: + +```typescript +call.screenShare.state.status; // enabled, disabled or undefined + +// or, if you want to subscribe to changes +call.screenShare.state.status$.subscribe((status) => { + // enabled, disabled or undefined +}); +``` + +### Screen Sharing Settings + +The behavior of the screen share video track can be customized, and a few parameters can be set: + +```typescript +call.screenShare.setSettings({ + maxFramerate: 15, // will be clamped between 1 and 15 fps + maxBitrate: 1500000, // will use at most 1.5Mbps +}); + +call.screenShare.enable(); +``` + +### Render Screen Share + +Please follow our [Playing Video and Audio guide](../../guides/playing-video-and-audio/). + +## Screen Share Audio + +### Start/Stop Screen Share Audio + +```typescript +// enable it +call.screenShare.enableScreenShareAudio(); + +// publish video and audio (if available, and supported by the browser) +call.screenShare.enable(); + +// disable it +call.screenShare.disableScreenShareAudio(); +``` + +### Play Screen Share Audio + +Please follow our [Playing Video and Audio guide](../../guides/playing-video-and-audio/). + +### Caveats + +Screen Share Audio has limited support across browsers and platforms. +For most up-to-date information, please take a look at [Browser Compatibility](https://developer.mozilla.org/en-US/docs/Web/API/Screen_Capture_API/Using_Screen_Capture#browser_compatibility). + +In addition to that, there are a [few caveats](https://caniuse.com/?search=getDisplayMedia) that you should be aware of: + +- On Windows, the entire system audio can be captured, but on MacOS and Lunux, only the audio of a tab can be captured. diff --git a/packages/client/docusaurus/docs/javascript/02-guides/11-playing-video-and-audio.mdx b/packages/client/docusaurus/docs/javascript/02-guides/11-playing-video-and-audio.mdx index af6a6843f7..ecb91c116a 100644 --- a/packages/client/docusaurus/docs/javascript/02-guides/11-playing-video-and-audio.mdx +++ b/packages/client/docusaurus/docs/javascript/02-guides/11-playing-video-and-audio.mdx @@ -61,6 +61,7 @@ This method can be found in `call.bindAudioElement`. It takes two arguments: - the audio element to bind to - the participant's `sessionId` +- the kind of track to bind to (either `audioTrack` or `screenShareAudioTrack` for screen sharing) This method needs to be called only once, usually after the element is mounted in the DOM. @@ -73,6 +74,10 @@ if (!audioElement) { // bind the audio element to the participant's audio track // use the returned `unbind()` function to unbind the audio element - const unbind = call.bindAudioElement(audioElement, participant.sessionId); + const unbind = call.bindAudioElement( + audioElement, + participant.sessionId, + 'audioTrack', + ); } ``` diff --git a/packages/client/src/Call.ts b/packages/client/src/Call.ts index 43024deaae..6943837ffd 100644 --- a/packages/client/src/Call.ts +++ b/packages/client/src/Call.ts @@ -68,6 +68,7 @@ import { } from './gen/coordinator'; import { join, reconcileParticipantLocalState } from './rtc/flows/join'; import { + AudioTrackType, CallConstructor, CallLeaveOptions, DebounceType, @@ -76,6 +77,7 @@ import { StreamVideoParticipant, StreamVideoParticipantPatches, SubscriptionChanges, + TrackMuteType, VideoTrackType, VisibilityState, } from './types'; @@ -120,6 +122,7 @@ import { CameraDirection, CameraManager, MicrophoneManager, + ScreenShareManager, SpeakerManager, } from './devices'; @@ -168,6 +171,11 @@ export class Call { */ readonly speaker: SpeakerManager; + /** + * Device manager for the screen. + */ + readonly screenShare: ScreenShareManager; + /** * The DynascaleManager instance. */ @@ -281,6 +289,7 @@ export class Call { this.camera = new CameraManager(this); this.microphone = new MicrophoneManager(this); this.speaker = new SpeakerManager(); + this.screenShare = new ScreenShareManager(this); } private registerEffects() { @@ -768,9 +777,21 @@ export class Call { const { audioStream, videoStream, - screenShareStream: screenShare, + screenShareStream, + screenShareAudioStream, } = localParticipant; + let screenShare: MediaStream | undefined; + if (screenShareStream || screenShareAudioStream) { + screenShare = new MediaStream(); + screenShareStream?.getVideoTracks().forEach((track) => { + screenShare?.addTrack(track); + }); + screenShareAudioStream?.getAudioTracks().forEach((track) => { + screenShare?.addTrack(track); + }); + } + // restore previous publishing state if (audioStream) await this.publishAudioStream(audioStream); if (videoStream) await this.publishVideoStream(videoStream); @@ -1081,7 +1102,6 @@ export class Call { * Consecutive calls to this method will replace the audio stream that is currently being published. * The previous audio stream will be stopped. * - * * @param audioStream the audio stream to publish. */ publishAudioStream = async (audioStream: MediaStream) => { @@ -1112,10 +1132,13 @@ export class Call { * Consecutive calls to this method will replace the previous screen-share stream. * The previous screen-share stream will be stopped. * - * * @param screenShareStream the screen-share stream to publish. + * @param opts the options to use when publishing the stream. */ - publishScreenShareStream = async (screenShareStream: MediaStream) => { + publishScreenShareStream = async ( + screenShareStream: MediaStream, + opts: PublishOptions = {}, + ) => { // we should wait until we get a JoinResponse from the SFU, // otherwise we risk breaking the ICETrickle flow. await this.assertCallJoined(); @@ -1140,7 +1163,18 @@ export class Call { screenShareStream, screenShareTrack, TrackType.SCREEN_SHARE, + opts, ); + + const [screenShareAudioTrack] = screenShareStream.getAudioTracks(); + if (screenShareAudioTrack) { + await this.publisher.publishStream( + screenShareStream, + screenShareAudioTrack, + TrackType.SCREEN_SHARE_AUDIO, + opts, + ); + } }; /** @@ -1252,6 +1286,13 @@ export class Call { dimension: p.screenShareDimension, }); } + if (p.publishedTracks.includes(TrackType.SCREEN_SHARE_AUDIO)) { + subscriptions.push({ + userId: p.userId, + sessionId: p.sessionId, + trackType: TrackType.SCREEN_SHARE_AUDIO, + }); + } } // schedule update this.trackSubscriptionsSubject.next({ type, data: subscriptions }); @@ -1414,7 +1455,7 @@ export class Call { * * @param type the type of the mute operation. */ - muteSelf = (type: 'audio' | 'video' | 'screenshare') => { + muteSelf = (type: TrackMuteType) => { const myUserId = this.currentUserId; if (myUserId) { return this.muteUser(myUserId, type); @@ -1426,7 +1467,7 @@ export class Call { * * @param type the type of the mute operation. */ - muteOthers = (type: 'audio' | 'video' | 'screenshare') => { + muteOthers = (type: TrackMuteType) => { const trackType = muteTypeToTrackType(type); if (!trackType) return; const userIdsToMute: string[] = []; @@ -1445,10 +1486,7 @@ export class Call { * @param userId the id of the user to mute. * @param type the type of the mute operation. */ - muteUser = ( - userId: string | string[], - type: 'audio' | 'video' | 'screenshare', - ) => { + muteUser = (userId: string | string[], type: TrackMuteType) => { return this.streamClient.post( `${this.streamClientBasePath}/mute_users`, { @@ -1463,7 +1501,7 @@ export class Call { * * @param type the type of the mute operation. */ - muteAllUsers = (type: 'audio' | 'video' | 'screenshare') => { + muteAllUsers = (type: TrackMuteType) => { return this.streamClient.post( `${this.streamClientBasePath}/mute_users`, { @@ -1952,11 +1990,17 @@ export class Call { * * @param audioElement the audio element to bind to. * @param sessionId the session id. + * @param trackType the kind of audio. */ - bindAudioElement = (audioElement: HTMLAudioElement, sessionId: string) => { + bindAudioElement = ( + audioElement: HTMLAudioElement, + sessionId: string, + trackType: AudioTrackType = 'audioTrack', + ) => { const unbind = this.dynascaleManager.bindAudioElement( audioElement, sessionId, + trackType, ); if (!unbind) return; diff --git a/packages/client/src/devices/CameraManager.ts b/packages/client/src/devices/CameraManager.ts index 7cf4492c04..01d416c5ed 100644 --- a/packages/client/src/devices/CameraManager.ts +++ b/packages/client/src/devices/CameraManager.ts @@ -69,6 +69,7 @@ export class CameraManager extends InputMediaDeviceManager { protected getDevices(): Observable { return getVideoDevices(); } + protected getStream( constraints: MediaTrackConstraints, ): Promise { @@ -82,14 +83,12 @@ export class CameraManager extends InputMediaDeviceManager { } return getVideoStream(constraints); } + protected publishStream(stream: MediaStream): Promise { return this.call.publishVideoStream(stream); } + protected stopPublishStream(stopTracks: boolean): Promise { return this.call.stopPublish(TrackType.VIDEO, stopTracks); } - - protected getTrack() { - return this.state.mediaStream?.getVideoTracks()[0]; - } } diff --git a/packages/client/src/devices/InputMediaDeviceManager.ts b/packages/client/src/devices/InputMediaDeviceManager.ts index 1bd58f8609..90c156e9fb 100644 --- a/packages/client/src/devices/InputMediaDeviceManager.ts +++ b/packages/client/src/devices/InputMediaDeviceManager.ts @@ -8,7 +8,8 @@ import { getLogger } from '../logger'; import { TrackType } from '../gen/video/sfu/models/models'; export abstract class InputMediaDeviceManager< - T extends InputMediaDeviceManagerState, + T extends InputMediaDeviceManagerState, + C = MediaTrackConstraints, > { /** * @internal @@ -20,7 +21,7 @@ export abstract class InputMediaDeviceManager< disablePromise?: Promise; logger: Logger; - constructor( + protected constructor( protected readonly call: Call, public readonly state: T, protected readonly trackType: TrackType, @@ -40,7 +41,7 @@ export abstract class InputMediaDeviceManager< } /** - * Starts camera/microphone + * Starts stream. */ async enable() { if (this.state.status === 'enabled') { @@ -57,9 +58,7 @@ export abstract class InputMediaDeviceManager< } /** - * Stops camera/microphone - * - * @returns + * Stops the stream. */ async disable() { this.state.prevStatus = this.state.status; @@ -80,7 +79,7 @@ export abstract class InputMediaDeviceManager< } /** - * If status was previously enabled, it will reenable the device. + * If status was previously enabled, it will re-enable the device. */ async resume() { if ( @@ -92,9 +91,8 @@ export abstract class InputMediaDeviceManager< } /** - * If current device statis is disabled, it will enable the device, else it will disable it. - * - * @returns + * If the current device status is disabled, it will enable the device, + * else it will disable it. */ async toggle() { if (this.state.status === 'enabled') { @@ -104,6 +102,15 @@ export abstract class InputMediaDeviceManager< } } + /** + * Will set the default constraints for the device. + * + * @param constraints the constraints to set. + */ + async setDefaultConstraints(constraints: C) { + this.state.setDefaultConstraints(constraints); + } + /** * Select device * @@ -131,15 +138,15 @@ export abstract class InputMediaDeviceManager< protected abstract getDevices(): Observable; - protected abstract getStream( - constraints: MediaTrackConstraints, - ): Promise; + protected abstract getStream(constraints: C): Promise; protected abstract publishStream(stream: MediaStream): Promise; protected abstract stopPublishStream(stopTracks: boolean): Promise; - protected abstract getTrack(): undefined | MediaStreamTrack; + protected getTracks(): MediaStreamTrack[] { + return this.state.mediaStream?.getTracks() ?? []; + } protected async muteStream(stopTracks: boolean = true) { if (!this.state.mediaStream) { @@ -150,59 +157,67 @@ export abstract class InputMediaDeviceManager< await this.stopPublishStream(stopTracks); } this.muteLocalStream(stopTracks); - if (this.getTrack()?.readyState === 'ended') { - // @ts-expect-error release() is present in react-native-webrtc and must be called to dispose the stream - if (typeof this.state.mediaStream.release === 'function') { - // @ts-expect-error - this.state.mediaStream.release(); + this.getTracks().forEach((track) => { + if (track.readyState === 'ended') { + // @ts-expect-error release() is present in react-native-webrtc + // and must be called to dispose the stream + if (typeof this.state.mediaStream.release === 'function') { + // @ts-expect-error + this.state.mediaStream.release(); + } + this.state.setMediaStream(undefined); } - this.state.setMediaStream(undefined); - } + }); } - private muteTrack() { - const track = this.getTrack(); - if (!track || !track.enabled) { - return; - } - track.enabled = false; + private muteTracks() { + this.getTracks().forEach((track) => { + if (track.enabled) track.enabled = false; + }); } - private unmuteTrack() { - const track = this.getTrack(); - if (!track || track.enabled) { - return; - } - track.enabled = true; + private unmuteTracks() { + this.getTracks().forEach((track) => { + if (!track.enabled) track.enabled = true; + }); } - private stopTrack() { - const track = this.getTrack(); - if (!track || track.readyState === 'ended') { - return; - } - track.stop(); + private stopTracks() { + this.getTracks().forEach((track) => { + if (track.readyState === 'live') track.stop(); + }); } private muteLocalStream(stopTracks: boolean) { if (!this.state.mediaStream) { return; } - stopTracks ? this.stopTrack() : this.muteTrack(); + if (stopTracks) { + this.stopTracks(); + } else { + this.muteTracks(); + } } protected async unmuteStream() { this.logger('debug', 'Starting stream'); let stream: MediaStream; - if (this.state.mediaStream && this.getTrack()?.readyState === 'live') { + if ( + this.state.mediaStream && + this.getTracks().every((t) => t.readyState === 'live') + ) { stream = this.state.mediaStream; - this.unmuteTrack(); + this.unmuteTracks(); } else { if (this.state.mediaStream) { - this.stopTrack(); + this.stopTracks(); } - const constraints = { deviceId: this.state.selectedDevice }; - stream = await this.getStream(constraints); + const defaultConstraints = this.state.defaultConstraints; + const constraints: MediaTrackConstraints = { + ...defaultConstraints, + deviceId: this.state.selectedDevice, + }; + stream = await this.getStream(constraints as C); } if (this.call.state.callingState === CallingState.JOINED) { await this.publishStream(stream); diff --git a/packages/client/src/devices/InputMediaDeviceManagerState.ts b/packages/client/src/devices/InputMediaDeviceManagerState.ts index c688414d9c..9c9b1b3184 100644 --- a/packages/client/src/devices/InputMediaDeviceManagerState.ts +++ b/packages/client/src/devices/InputMediaDeviceManagerState.ts @@ -1,9 +1,9 @@ -import { BehaviorSubject, distinctUntilChanged, Observable } from 'rxjs'; +import { BehaviorSubject, distinctUntilChanged } from 'rxjs'; import { RxUtils } from '../store'; export type InputDeviceStatus = 'enabled' | 'disabled' | undefined; -export abstract class InputMediaDeviceManagerState { +export abstract class InputMediaDeviceManagerState { protected statusSubject = new BehaviorSubject(undefined); protected mediaStreamSubject = new BehaviorSubject( undefined, @@ -11,6 +11,10 @@ export abstract class InputMediaDeviceManagerState { protected selectedDeviceSubject = new BehaviorSubject( undefined, ); + protected defaultConstraintsSubject = new BehaviorSubject( + undefined, + ); + /** * @internal */ @@ -20,31 +24,30 @@ export abstract class InputMediaDeviceManagerState { * An Observable that emits the current media stream, or `undefined` if the device is currently disabled. * */ - mediaStream$: Observable; + mediaStream$ = this.mediaStreamSubject.asObservable(); /** * An Observable that emits the currently selected device */ - selectedDevice$: Observable; + selectedDevice$ = this.selectedDeviceSubject + .asObservable() + .pipe(distinctUntilChanged()); /** * An Observable that emits the device status */ - status$: Observable; + status$ = this.statusSubject.asObservable().pipe(distinctUntilChanged()); + + /** + * The default constraints for the device. + */ + defaultConstraints$ = this.defaultConstraintsSubject.asObservable(); constructor( public readonly disableMode: | 'stop-tracks' | 'disable-tracks' = 'stop-tracks', - ) { - this.mediaStream$ = this.mediaStreamSubject.asObservable(); - this.selectedDevice$ = this.selectedDeviceSubject - .asObservable() - .pipe(distinctUntilChanged()); - this.status$ = this.statusSubject - .asObservable() - .pipe(distinctUntilChanged()); - } + ) {} /** * The device status @@ -102,6 +105,23 @@ export abstract class InputMediaDeviceManagerState { this.setCurrentValue(this.selectedDeviceSubject, deviceId); } + /** + * Gets the default constraints for the device. + */ + get defaultConstraints() { + return this.getCurrentValue(this.defaultConstraints$); + } + + /** + * Sets the default constraints for the device. + * + * @internal + * @param constraints the constraints to set. + */ + setDefaultConstraints(constraints: C | undefined) { + this.setCurrentValue(this.defaultConstraintsSubject, constraints); + } + /** * Updates the value of the provided Subject. * An `update` can either be a new value or a function which takes diff --git a/packages/client/src/devices/MicrophoneManager.ts b/packages/client/src/devices/MicrophoneManager.ts index a7a6b853b3..2f2caf8554 100644 --- a/packages/client/src/devices/MicrophoneManager.ts +++ b/packages/client/src/devices/MicrophoneManager.ts @@ -42,22 +42,21 @@ export class MicrophoneManager extends InputMediaDeviceManager { return getAudioDevices(); } + protected getStream( constraints: MediaTrackConstraints, ): Promise { return getAudioStream(constraints); } + protected publishStream(stream: MediaStream): Promise { return this.call.publishAudioStream(stream); } + protected stopPublishStream(stopTracks: boolean): Promise { return this.call.stopPublish(TrackType.AUDIO, stopTracks); } - protected getTrack() { - return this.state.mediaStream?.getAudioTracks()[0]; - } - private async startSpeakingWhileMutedDetection(deviceId?: string) { if (isReactNative()) { return; diff --git a/packages/client/src/devices/ScreenShareManager.ts b/packages/client/src/devices/ScreenShareManager.ts new file mode 100644 index 0000000000..0498651b71 --- /dev/null +++ b/packages/client/src/devices/ScreenShareManager.ts @@ -0,0 +1,85 @@ +import { Observable, of } from 'rxjs'; +import { InputMediaDeviceManager } from './InputMediaDeviceManager'; +import { ScreenShareState } from './ScreenShareState'; +import { Call } from '../Call'; +import { TrackType } from '../gen/video/sfu/models/models'; +import { getScreenShareStream } from './devices'; +import { ScreenShareSettings } from '../types'; + +export class ScreenShareManager extends InputMediaDeviceManager< + ScreenShareState, + DisplayMediaStreamOptions +> { + constructor(call: Call) { + super(call, new ScreenShareState(), TrackType.SCREEN_SHARE); + } + + /** + * Will enable screen share audio options on supported platforms. + * + * Note: for ongoing screen share, audio won't be enabled until you + * re-publish the screen share stream. + */ + enableScreenShareAudio(): void { + this.state.setAudioEnabled(true); + } + + /** + * Will disable screen share audio options on supported platforms. + */ + async disableScreenShareAudio(): Promise { + this.state.setAudioEnabled(false); + if (this.call.publisher?.isPublishing(TrackType.SCREEN_SHARE_AUDIO)) { + await this.call.stopPublish(TrackType.SCREEN_SHARE_AUDIO, true); + } + } + + /** + * Returns the current screen share settings. + */ + getSettings(): ScreenShareSettings | undefined { + return this.state.settings; + } + + /** + * Sets the current screen share settings. + * + * @param settings the settings to set. + */ + setSettings(settings: ScreenShareSettings | undefined): void { + this.state.setSettings(settings); + } + + protected getDevices(): Observable { + return of([]); // there are no devices to be listed for Screen Share + } + + protected getStream( + constraints: DisplayMediaStreamOptions, + ): Promise { + if (!this.state.audioEnabled) { + constraints.audio = false; + } + return getScreenShareStream(constraints); + } + + protected publishStream(stream: MediaStream): Promise { + return this.call.publishScreenShareStream(stream, { + screenShareSettings: this.state.settings, + }); + } + + protected async stopPublishStream(stopTracks: boolean): Promise { + await this.call.stopPublish(TrackType.SCREEN_SHARE, stopTracks); + await this.call.stopPublish(TrackType.SCREEN_SHARE_AUDIO, stopTracks); + } + + /** + * Overrides the default `select` method to throw an error. + * + * @param deviceId ignored. + */ + async select(deviceId: string | undefined): Promise { + throw new Error('This method is not supported in for Screen Share'); + } +} diff --git a/packages/client/src/devices/ScreenShareState.ts b/packages/client/src/devices/ScreenShareState.ts new file mode 100644 index 0000000000..5efeb9b622 --- /dev/null +++ b/packages/client/src/devices/ScreenShareState.ts @@ -0,0 +1,63 @@ +import { BehaviorSubject } from 'rxjs'; +import { InputMediaDeviceManagerState } from './InputMediaDeviceManagerState'; +import { distinctUntilChanged } from 'rxjs/operators'; +import { ScreenShareSettings } from '../types'; + +export class ScreenShareState extends InputMediaDeviceManagerState { + private audioEnabledSubject = new BehaviorSubject(true); + private settingsSubject = new BehaviorSubject< + ScreenShareSettings | undefined + >(undefined); + + /** + * An Observable that emits the current screen share audio status. + */ + audioEnabled$ = this.audioEnabledSubject + .asObservable() + .pipe(distinctUntilChanged()); + + /** + * An Observable that emits the current screen share settings. + */ + settings$ = this.settingsSubject.asObservable(); + + /** + * @internal + */ + protected getDeviceIdFromStream = ( + stream: MediaStream, + ): string | undefined => { + const [track] = stream.getTracks(); + return track?.getSettings().deviceId; + }; + + /** + * The current screen share audio status. + */ + get audioEnabled() { + return this.getCurrentValue(this.audioEnabled$); + } + + /** + * Set the current screen share audio status. + */ + setAudioEnabled(isEnabled: boolean) { + this.setCurrentValue(this.audioEnabledSubject, isEnabled); + } + + /** + * The current screen share settings. + */ + get settings() { + return this.getCurrentValue(this.settings$); + } + + /** + * Set the current screen share settings. + * + * @param settings the screen share settings to set. + */ + setSettings(settings: ScreenShareSettings | undefined) { + this.setCurrentValue(this.settingsSubject, settings); + } +} diff --git a/packages/client/src/devices/__tests__/InputMediaDeviceManager.test.ts b/packages/client/src/devices/__tests__/InputMediaDeviceManager.test.ts index bcc2e0f530..550ae51b5a 100644 --- a/packages/client/src/devices/__tests__/InputMediaDeviceManager.test.ts +++ b/packages/client/src/devices/__tests__/InputMediaDeviceManager.test.ts @@ -25,7 +25,7 @@ class TestInputMediaDeviceManager extends InputMediaDeviceManager Promise.resolve(mockVideoStream())); public publishStream = vi.fn(); public stopPublishStream = vi.fn(); - public getTrack = () => this.state.mediaStream!.getVideoTracks()[0]; + public getTracks = () => this.state.mediaStream?.getTracks() ?? []; constructor(call: Call) { super(call, new TestInputMediaDeviceManagerState(), TrackType.VIDEO); @@ -202,6 +202,21 @@ describe('InputMediaDeviceManager.test', () => { expect(manager.enable).toHaveBeenCalledOnce(); }); + it('should provide default constraints to `getStream` method', () => { + manager.setDefaultConstraints({ + echoCancellation: true, + autoGainControl: false, + }); + + manager.enable(); + + expect(manager.getStream).toHaveBeenCalledWith({ + deviceId: undefined, + echoCancellation: true, + autoGainControl: false, + }); + }); + afterEach(() => { vi.clearAllMocks(); vi.resetModules(); diff --git a/packages/client/src/devices/__tests__/ScreenShareManager.test.ts b/packages/client/src/devices/__tests__/ScreenShareManager.test.ts new file mode 100644 index 0000000000..fa771f1159 --- /dev/null +++ b/packages/client/src/devices/__tests__/ScreenShareManager.test.ts @@ -0,0 +1,119 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { ScreenShareManager } from '../ScreenShareManager'; +import { Call } from '../../Call'; +import { StreamClient } from '../../coordinator/connection/client'; +import { CallingState, StreamVideoWriteableStateStore } from '../../store'; +import * as RxUtils from '../../store/rxUtils'; +import { mockCall, mockScreenShareStream } from './mocks'; +import { getScreenShareStream } from '../devices'; +import { TrackType } from '../../gen/video/sfu/models/models'; + +vi.mock('../devices.ts', () => { + console.log('MOCKING devices API'); + return { + disposeOfMediaStream: vi.fn(), + getScreenShareStream: vi.fn(() => Promise.resolve(mockScreenShareStream())), + checkIfAudioOutputChangeSupported: vi.fn(() => Promise.resolve(true)), + }; +}); + +vi.mock('../../Call.ts', () => { + console.log('MOCKING Call'); + return { + Call: vi.fn(() => mockCall()), + }; +}); + +describe('ScreenShareManager', () => { + let manager: ScreenShareManager; + + beforeEach(() => { + manager = new ScreenShareManager( + new Call({ + id: '', + type: '', + streamClient: new StreamClient('abc123'), + clientStore: new StreamVideoWriteableStateStore(), + }), + ); + }); + + afterEach(() => { + vi.clearAllMocks(); + vi.resetModules(); + }); + + it('list devices', () => { + const devices = manager.listDevices(); + expect(RxUtils.getCurrentValue(devices)).toEqual([]); + }); + + it('select device', () => { + expect(manager.select('any-device-id')).rejects.toThrowError(); + }); + + it('get stream', async () => { + manager.enableScreenShareAudio(); + await manager.enable(); + expect(manager.state.status).toEqual('enabled'); + + expect(getScreenShareStream).toHaveBeenCalledWith({ + deviceId: undefined, + }); + }); + + it('get stream with no audio', async () => { + await manager.disableScreenShareAudio(); + await manager.enable(); + expect(manager.state.status).toEqual('enabled'); + + expect(getScreenShareStream).toHaveBeenCalledWith({ + deviceId: undefined, + audio: false, + }); + }); + + it('should get device id from stream', async () => { + expect(manager.state.selectedDevice).toBeUndefined(); + await manager.enable(); + expect(manager.state.selectedDevice).toBeDefined(); + expect(manager.state.selectedDevice).toEqual('screen'); + }); + + it('publishes screen share stream', async () => { + const call = manager['call']; + call.state.setCallingState(CallingState.JOINED); + await manager.enable(); + expect(call.publishScreenShareStream).toHaveBeenCalledWith( + manager.state.mediaStream, + { screenShareSettings: undefined }, + ); + }); + + it('publishes screen share stream with settings', async () => { + const call = manager['call']; + call.state.setCallingState(CallingState.JOINED); + + manager.setSettings({ maxFramerate: 15, maxBitrate: 1000 }); + + await manager.enable(); + expect(call.publishScreenShareStream).toHaveBeenCalledWith( + manager.state.mediaStream, + { screenShareSettings: { maxFramerate: 15, maxBitrate: 1000 } }, + ); + }); + + it('stop publish stream', async () => { + const call = manager['call']; + call.state.setCallingState(CallingState.JOINED); + await manager.enable(); + + await manager.disable(); + expect(manager.state.status).toEqual('disabled'); + expect(call.stopPublish).toHaveBeenCalledWith(TrackType.SCREEN_SHARE, true); + expect(call.stopPublish).toHaveBeenCalledWith( + TrackType.SCREEN_SHARE_AUDIO, + true, + ); + }); +}); diff --git a/packages/client/src/devices/__tests__/mocks.ts b/packages/client/src/devices/__tests__/mocks.ts index 7039eb221b..1fbe1dc364 100644 --- a/packages/client/src/devices/__tests__/mocks.ts +++ b/packages/client/src/devices/__tests__/mocks.ts @@ -1,6 +1,7 @@ import { vi } from 'vitest'; import { CallingState, CallState } from '../../store'; import { OwnCapability } from '../../gen/coordinator'; +import { Call } from '../../Call'; export const mockVideoDevices = [ { @@ -63,7 +64,7 @@ export const mockAudioDevices = [ }, ] as MediaDeviceInfo[]; -export const mockCall = () => { +export const mockCall = (): Partial => { const callState = new CallState(); callState.setCallingState(CallingState.JOINED); callState.setOwnCapabilities([ @@ -74,6 +75,7 @@ export const mockCall = () => { state: callState, publishVideoStream: vi.fn(), publishAudioStream: vi.fn(), + publishScreenShareStream: vi.fn(), stopPublish: vi.fn(), }; }; @@ -90,6 +92,7 @@ export const mockAudioStream = () => { }, }; return { + getTracks: () => [track], getAudioTracks: () => [track], } as MediaStream; }; @@ -108,6 +111,40 @@ export const mockVideoStream = () => { }, }; return { + getTracks: () => [track], getVideoTracks: () => [track], } as MediaStream; }; + +export const mockScreenShareStream = (includeAudio: boolean = true) => { + const track = { + getSettings: () => ({ + deviceId: 'screen', + }), + enabled: true, + readyState: 'live', + stop: () => { + track.readyState = 'ended'; + }, + }; + + const tracks = [track]; + if (includeAudio) { + tracks.push({ + getSettings: () => ({ + deviceId: 'screen-audio', + }), + enabled: true, + readyState: 'live', + stop: () => { + track.readyState = 'ended'; + }, + }); + } + + return { + getTracks: () => tracks, + getVideoTracks: () => tracks, + getAudioTracks: () => tracks, + } as MediaStream; +}; diff --git a/packages/client/src/devices/devices.ts b/packages/client/src/devices/devices.ts index 73bc5b101f..6e0c3bdab5 100644 --- a/packages/client/src/devices/devices.ts +++ b/packages/client/src/devices/devices.ts @@ -199,7 +199,16 @@ export const getScreenShareStream = async ( try { return await navigator.mediaDevices.getDisplayMedia({ video: true, - audio: false, + audio: { + channelCount: { + ideal: 2, + }, + echoCancellation: false, + autoGainControl: false, + noiseSuppression: false, + }, + // @ts-expect-error - not present in types yet + systemAudio: 'include', ...options, }); } catch (e) { diff --git a/packages/client/src/devices/index.ts b/packages/client/src/devices/index.ts index a75dd4c1f4..3e1705367e 100644 --- a/packages/client/src/devices/index.ts +++ b/packages/client/src/devices/index.ts @@ -5,5 +5,7 @@ export * from './CameraManager'; export * from './CameraManagerState'; export * from './MicrophoneManager'; export * from './MicrophoneManagerState'; +export * from './ScreenShareManager'; +export * from './ScreenShareState'; export * from './SpeakerManager'; export * from './SpeakerState'; diff --git a/packages/client/src/helpers/DynascaleManager.ts b/packages/client/src/helpers/DynascaleManager.ts index e8f57593e7..981df39fc0 100644 --- a/packages/client/src/helpers/DynascaleManager.ts +++ b/packages/client/src/helpers/DynascaleManager.ts @@ -1,5 +1,6 @@ import { Call } from '../Call'; import { + AudioTrackType, DebounceType, StreamVideoLocalParticipant, StreamVideoParticipant, @@ -324,9 +325,14 @@ export class DynascaleManager { * * @param audioElement the audio element to bind to. * @param sessionId the session id. + * @param trackType the kind of audio. * @returns a cleanup function that will unbind the audio element. */ - bindAudioElement = (audioElement: HTMLAudioElement, sessionId: string) => { + bindAudioElement = ( + audioElement: HTMLAudioElement, + sessionId: string, + trackType: AudioTrackType, + ) => { const participant = this.call.state.findParticipantBySessionId(sessionId); if (!participant || participant.isLocalParticipant) return; @@ -343,9 +349,18 @@ export class DynascaleManager { ); const updateMediaStreamSubscription = participant$ - .pipe(distinctUntilKeyChanged('audioStream')) + .pipe( + distinctUntilKeyChanged( + trackType === 'screenShareAudioTrack' + ? 'screenShareAudioStream' + : 'audioStream', + ), + ) .subscribe((p) => { - const source = p.audioStream; + const source = + trackType === 'screenShareAudioTrack' + ? p.screenShareAudioStream + : p.audioStream; if (audioElement.srcObject === source) return; setTimeout(() => { diff --git a/packages/client/src/helpers/__tests__/DynascaleManager.test.ts b/packages/client/src/helpers/__tests__/DynascaleManager.test.ts index 3b8753afc1..e81d042fd0 100644 --- a/packages/client/src/helpers/__tests__/DynascaleManager.test.ts +++ b/packages/client/src/helpers/__tests__/DynascaleManager.test.ts @@ -103,7 +103,7 @@ describe('DynascaleManager', () => { videoElement.clientHeight = 100; }); - it('should bind audio element', () => { + it('audio: should bind audio element', () => { vi.useFakeTimers(); const audioElement = document.createElement('audio'); const play = vi.spyOn(audioElement, 'play').mockResolvedValue(); @@ -128,6 +128,7 @@ describe('DynascaleManager', () => { const cleanup = dynascaleManager.bindAudioElement( audioElement, 'session-id', + 'audioTrack', ); expect(audioElement.autoplay).toBe(true); @@ -172,6 +173,40 @@ describe('DynascaleManager', () => { cleanup?.(); }); + it('audio: should bind screenShare audio element', () => { + vi.useFakeTimers(); + const audioElement = document.createElement('audio'); + const play = vi.spyOn(audioElement, 'play').mockResolvedValue(); + + // @ts-ignore + call.state.updateOrAddParticipant('session-id', { + userId: 'user-id', + sessionId: 'session-id', + publishedTracks: [TrackType.SCREEN_SHARE_AUDIO], + }); + + const cleanup = dynascaleManager.bindAudioElement( + audioElement, + 'session-id', + 'screenShareAudioTrack', + ); + expect(audioElement.autoplay).toBe(true); + + const audioMediaStream = new MediaStream(); + const screenShareAudioMediaStream = new MediaStream(); + call.state.updateParticipant('session-id', { + audioStream: audioMediaStream, + screenShareAudioStream: screenShareAudioMediaStream, + }); + + vi.runAllTimers(); + + expect(play).toHaveBeenCalled(); + expect(audioElement.srcObject).toBe(screenShareAudioMediaStream); + + cleanup?.(); + }); + it('video: should update subscription when track becomes available', () => { const updateSubscription = vi.spyOn(call, 'updateSubscriptionsPartial'); diff --git a/packages/client/src/helpers/__tests__/hq-audio-sdp.ts b/packages/client/src/helpers/__tests__/hq-audio-sdp.ts new file mode 100644 index 0000000000..c4bad821df --- /dev/null +++ b/packages/client/src/helpers/__tests__/hq-audio-sdp.ts @@ -0,0 +1,332 @@ +export const initialSdp = ` +v=0 +o=- 898697271686242868 5 IN IP4 127.0.0.1 +s=- +t=0 0 +a=group:BUNDLE 0 1 2 3 +a=extmap-allow-mixed +a=msid-semantic: WMS e893e3ad-d9e8-4b56-998f-0d89213dd857 +m=video 60017 UDP/TLS/RTP/SAVPF 96 97 102 103 104 105 106 107 108 109 127 125 39 40 45 46 98 99 100 101 112 113 116 117 118 +c=IN IP4 79.125.240.146 +a=rtcp:9 IN IP4 0.0.0.0 +a=candidate:2824354870 1 udp 2122260223 192.168.1.102 60017 typ host generation 0 network-id 2 +a=candidate:427386432 1 udp 2122194687 192.168.1.244 63221 typ host generation 0 network-id 1 network-cost 10 +a=candidate:2841136656 1 udp 1686052607 79.125.240.146 60017 typ srflx raddr 192.168.1.102 rport 60017 generation 0 network-id 2 +a=candidate:410588262 1 udp 1685987071 79.125.240.146 63221 typ srflx raddr 192.168.1.244 rport 63221 generation 0 network-id 1 network-cost 10 +a=candidate:3600277166 1 tcp 1518280447 192.168.1.102 9 typ host tcptype active generation 0 network-id 2 +a=candidate:1740014808 1 tcp 1518214911 192.168.1.244 9 typ host tcptype active generation 0 network-id 1 network-cost 10 +a=ice-ufrag:GM64 +a=ice-pwd:ANZFilRRlZJ3bg9AD40eRu7n +a=ice-options:trickle +a=fingerprint:sha-256 38:1F:02:E5:2A:49:9A:2A:D9:8E:B9:9B:4C:40:21:B7:F1:C4:27:8E:B5:68:D6:E0:91:08:D9:CB:2B:AC:B3:87 +a=setup:actpass +a=mid:0 +a=extmap:1 urn:ietf:params:rtp-hdrext:toffset +a=extmap:2 http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time +a=extmap:3 urn:3gpp:video-orientation +a=extmap:4 http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01 +a=extmap:5 http://www.webrtc.org/experiments/rtp-hdrext/playout-delay +a=extmap:6 http://www.webrtc.org/experiments/rtp-hdrext/video-content-type +a=extmap:7 http://www.webrtc.org/experiments/rtp-hdrext/video-timing +a=extmap:8 http://www.webrtc.org/experiments/rtp-hdrext/color-space +a=extmap:9 urn:ietf:params:rtp-hdrext:sdes:mid +a=extmap:10 urn:ietf:params:rtp-hdrext:sdes:rtp-stream-id +a=extmap:11 urn:ietf:params:rtp-hdrext:sdes:repaired-rtp-stream-id +a=sendonly +a=msid:e893e3ad-d9e8-4b56-998f-0d89213dd857 44e5d53f-ce6a-4bf4-824a-335113d86e6e +a=rtcp-mux +a=rtcp-rsize +a=rtpmap:96 VP8/90000 +a=rtcp-fb:96 goog-remb +a=rtcp-fb:96 transport-cc +a=rtcp-fb:96 ccm fir +a=rtcp-fb:96 nack +a=rtcp-fb:96 nack pli +a=rtpmap:97 rtx/90000 +a=fmtp:97 apt=96 +a=rtpmap:102 H264/90000 +a=rtcp-fb:102 goog-remb +a=rtcp-fb:102 transport-cc +a=rtcp-fb:102 ccm fir +a=rtcp-fb:102 nack +a=rtcp-fb:102 nack pli +a=fmtp:102 level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42001f +a=rtpmap:103 rtx/90000 +a=fmtp:103 apt=102 +a=rtpmap:104 H264/90000 +a=rtcp-fb:104 goog-remb +a=rtcp-fb:104 transport-cc +a=rtcp-fb:104 ccm fir +a=rtcp-fb:104 nack +a=rtcp-fb:104 nack pli +a=fmtp:104 level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=42001f +a=rtpmap:105 rtx/90000 +a=fmtp:105 apt=104 +a=rtpmap:106 H264/90000 +a=rtcp-fb:106 goog-remb +a=rtcp-fb:106 transport-cc +a=rtcp-fb:106 ccm fir +a=rtcp-fb:106 nack +a=rtcp-fb:106 nack pli +a=fmtp:106 level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f +a=rtpmap:107 rtx/90000 +a=fmtp:107 apt=106 +a=rtpmap:108 H264/90000 +a=rtcp-fb:108 goog-remb +a=rtcp-fb:108 transport-cc +a=rtcp-fb:108 ccm fir +a=rtcp-fb:108 nack +a=rtcp-fb:108 nack pli +a=fmtp:108 level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=42e01f +a=rtpmap:109 rtx/90000 +a=fmtp:109 apt=108 +a=rtpmap:127 H264/90000 +a=rtcp-fb:127 goog-remb +a=rtcp-fb:127 transport-cc +a=rtcp-fb:127 ccm fir +a=rtcp-fb:127 nack +a=rtcp-fb:127 nack pli +a=fmtp:127 level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=4d001f +a=rtpmap:125 rtx/90000 +a=fmtp:125 apt=127 +a=rtpmap:39 H264/90000 +a=rtcp-fb:39 goog-remb +a=rtcp-fb:39 transport-cc +a=rtcp-fb:39 ccm fir +a=rtcp-fb:39 nack +a=rtcp-fb:39 nack pli +a=fmtp:39 level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=4d001f +a=rtpmap:40 rtx/90000 +a=fmtp:40 apt=39 +a=rtpmap:45 AV1/90000 +a=rtcp-fb:45 goog-remb +a=rtcp-fb:45 transport-cc +a=rtcp-fb:45 ccm fir +a=rtcp-fb:45 nack +a=rtcp-fb:45 nack pli +a=rtpmap:46 rtx/90000 +a=fmtp:46 apt=45 +a=rtpmap:98 VP9/90000 +a=rtcp-fb:98 goog-remb +a=rtcp-fb:98 transport-cc +a=rtcp-fb:98 ccm fir +a=rtcp-fb:98 nack +a=rtcp-fb:98 nack pli +a=fmtp:98 profile-id=0 +a=rtpmap:99 rtx/90000 +a=fmtp:99 apt=98 +a=rtpmap:100 VP9/90000 +a=rtcp-fb:100 goog-remb +a=rtcp-fb:100 transport-cc +a=rtcp-fb:100 ccm fir +a=rtcp-fb:100 nack +a=rtcp-fb:100 nack pli +a=fmtp:100 profile-id=2 +a=rtpmap:101 rtx/90000 +a=fmtp:101 apt=100 +a=rtpmap:112 H264/90000 +a=rtcp-fb:112 goog-remb +a=rtcp-fb:112 transport-cc +a=rtcp-fb:112 ccm fir +a=rtcp-fb:112 nack +a=rtcp-fb:112 nack pli +a=fmtp:112 level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=64001f +a=rtpmap:113 rtx/90000 +a=fmtp:113 apt=112 +a=rtpmap:116 red/90000 +a=rtpmap:117 rtx/90000 +a=fmtp:117 apt=116 +a=rtpmap:118 ulpfec/90000 +a=rid:q send +a=rid:h send +a=rid:f send +a=simulcast:send q;h;f +m=audio 9 UDP/TLS/RTP/SAVPF 63 111 9 0 8 13 110 126 +c=IN IP4 0.0.0.0 +a=rtcp:9 IN IP4 0.0.0.0 +a=ice-ufrag:GM64 +a=ice-pwd:ANZFilRRlZJ3bg9AD40eRu7n +a=ice-options:trickle +a=fingerprint:sha-256 38:1F:02:E5:2A:49:9A:2A:D9:8E:B9:9B:4C:40:21:B7:F1:C4:27:8E:B5:68:D6:E0:91:08:D9:CB:2B:AC:B3:87 +a=setup:actpass +a=mid:1 +a=extmap:14 urn:ietf:params:rtp-hdrext:ssrc-audio-level +a=extmap:2 http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time +a=extmap:4 http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01 +a=extmap:9 urn:ietf:params:rtp-hdrext:sdes:mid +a=sendonly +a=msid:- cb02416c-8035-4e80-8b7e-becd6b7f600f +a=rtcp-mux +a=rtpmap:63 red/48000/2 +a=fmtp:63 111/111 +a=rtpmap:111 opus/48000/2 +a=rtcp-fb:111 transport-cc +a=fmtp:111 minptime=10;usedtx=1;useinbandfec=1 +a=rtpmap:9 G722/8000 +a=rtpmap:0 PCMU/8000 +a=rtpmap:8 PCMA/8000 +a=rtpmap:13 CN/8000 +a=rtpmap:110 telephone-event/48000 +a=rtpmap:126 telephone-event/8000 +a=ssrc:743121750 cname:mbGW+aeWMLFhPbBC +a=ssrc:743121750 msid:- cb02416c-8035-4e80-8b7e-becd6b7f600f +m=video 9 UDP/TLS/RTP/SAVPF 96 97 102 103 104 105 106 107 108 109 127 125 39 40 45 46 98 99 100 101 112 113 116 117 118 +c=IN IP4 0.0.0.0 +a=rtcp:9 IN IP4 0.0.0.0 +a=ice-ufrag:GM64 +a=ice-pwd:ANZFilRRlZJ3bg9AD40eRu7n +a=ice-options:trickle +a=fingerprint:sha-256 38:1F:02:E5:2A:49:9A:2A:D9:8E:B9:9B:4C:40:21:B7:F1:C4:27:8E:B5:68:D6:E0:91:08:D9:CB:2B:AC:B3:87 +a=setup:actpass +a=mid:2 +a=extmap:1 urn:ietf:params:rtp-hdrext:toffset +a=extmap:2 http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time +a=extmap:3 urn:3gpp:video-orientation +a=extmap:4 http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01 +a=extmap:5 http://www.webrtc.org/experiments/rtp-hdrext/playout-delay +a=extmap:6 http://www.webrtc.org/experiments/rtp-hdrext/video-content-type +a=extmap:7 http://www.webrtc.org/experiments/rtp-hdrext/video-timing +a=extmap:8 http://www.webrtc.org/experiments/rtp-hdrext/color-space +a=extmap:9 urn:ietf:params:rtp-hdrext:sdes:mid +a=extmap:10 urn:ietf:params:rtp-hdrext:sdes:rtp-stream-id +a=extmap:11 urn:ietf:params:rtp-hdrext:sdes:repaired-rtp-stream-id +a=sendonly +a=msid:04f96a2a-d107-4a12-b545-cfe1dc2c4d37 4a1ae8a7-6c89-44d3-a9a1-02abce0012c7 +a=rtcp-mux +a=rtcp-rsize +a=rtpmap:96 VP8/90000 +a=rtcp-fb:96 goog-remb +a=rtcp-fb:96 transport-cc +a=rtcp-fb:96 ccm fir +a=rtcp-fb:96 nack +a=rtcp-fb:96 nack pli +a=rtpmap:97 rtx/90000 +a=fmtp:97 apt=96 +a=rtpmap:102 H264/90000 +a=rtcp-fb:102 goog-remb +a=rtcp-fb:102 transport-cc +a=rtcp-fb:102 ccm fir +a=rtcp-fb:102 nack +a=rtcp-fb:102 nack pli +a=fmtp:102 level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42001f +a=rtpmap:103 rtx/90000 +a=fmtp:103 apt=102 +a=rtpmap:104 H264/90000 +a=rtcp-fb:104 goog-remb +a=rtcp-fb:104 transport-cc +a=rtcp-fb:104 ccm fir +a=rtcp-fb:104 nack +a=rtcp-fb:104 nack pli +a=fmtp:104 level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=42001f +a=rtpmap:105 rtx/90000 +a=fmtp:105 apt=104 +a=rtpmap:106 H264/90000 +a=rtcp-fb:106 goog-remb +a=rtcp-fb:106 transport-cc +a=rtcp-fb:106 ccm fir +a=rtcp-fb:106 nack +a=rtcp-fb:106 nack pli +a=fmtp:106 level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f +a=rtpmap:107 rtx/90000 +a=fmtp:107 apt=106 +a=rtpmap:108 H264/90000 +a=rtcp-fb:108 goog-remb +a=rtcp-fb:108 transport-cc +a=rtcp-fb:108 ccm fir +a=rtcp-fb:108 nack +a=rtcp-fb:108 nack pli +a=fmtp:108 level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=42e01f +a=rtpmap:109 rtx/90000 +a=fmtp:109 apt=108 +a=rtpmap:127 H264/90000 +a=rtcp-fb:127 goog-remb +a=rtcp-fb:127 transport-cc +a=rtcp-fb:127 ccm fir +a=rtcp-fb:127 nack +a=rtcp-fb:127 nack pli +a=fmtp:127 level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=4d001f +a=rtpmap:125 rtx/90000 +a=fmtp:125 apt=127 +a=rtpmap:39 H264/90000 +a=rtcp-fb:39 goog-remb +a=rtcp-fb:39 transport-cc +a=rtcp-fb:39 ccm fir +a=rtcp-fb:39 nack +a=rtcp-fb:39 nack pli +a=fmtp:39 level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=4d001f +a=rtpmap:40 rtx/90000 +a=fmtp:40 apt=39 +a=rtpmap:45 AV1/90000 +a=rtcp-fb:45 goog-remb +a=rtcp-fb:45 transport-cc +a=rtcp-fb:45 ccm fir +a=rtcp-fb:45 nack +a=rtcp-fb:45 nack pli +a=rtpmap:46 rtx/90000 +a=fmtp:46 apt=45 +a=rtpmap:98 VP9/90000 +a=rtcp-fb:98 goog-remb +a=rtcp-fb:98 transport-cc +a=rtcp-fb:98 ccm fir +a=rtcp-fb:98 nack +a=rtcp-fb:98 nack pli +a=fmtp:98 profile-id=0 +a=rtpmap:99 rtx/90000 +a=fmtp:99 apt=98 +a=rtpmap:100 VP9/90000 +a=rtcp-fb:100 goog-remb +a=rtcp-fb:100 transport-cc +a=rtcp-fb:100 ccm fir +a=rtcp-fb:100 nack +a=rtcp-fb:100 nack pli +a=fmtp:100 profile-id=2 +a=rtpmap:101 rtx/90000 +a=fmtp:101 apt=100 +a=rtpmap:112 H264/90000 +a=rtcp-fb:112 goog-remb +a=rtcp-fb:112 transport-cc +a=rtcp-fb:112 ccm fir +a=rtcp-fb:112 nack +a=rtcp-fb:112 nack pli +a=fmtp:112 level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=64001f +a=rtpmap:113 rtx/90000 +a=fmtp:113 apt=112 +a=rtpmap:116 red/90000 +a=rtpmap:117 rtx/90000 +a=fmtp:117 apt=116 +a=rtpmap:118 ulpfec/90000 +a=ssrc-group:FID 4072017687 3466187747 +a=ssrc:4072017687 cname:mbGW+aeWMLFhPbBC +a=ssrc:4072017687 msid:04f96a2a-d107-4a12-b545-cfe1dc2c4d37 4a1ae8a7-6c89-44d3-a9a1-02abce0012c7 +a=ssrc:3466187747 cname:mbGW+aeWMLFhPbBC +a=ssrc:3466187747 msid:04f96a2a-d107-4a12-b545-cfe1dc2c4d37 4a1ae8a7-6c89-44d3-a9a1-02abce0012c7 +m=audio 9 UDP/TLS/RTP/SAVPF 111 63 9 0 8 13 110 126 +c=IN IP4 0.0.0.0 +a=rtcp:9 IN IP4 0.0.0.0 +a=ice-ufrag:GM64 +a=ice-pwd:ANZFilRRlZJ3bg9AD40eRu7n +a=ice-options:trickle +a=fingerprint:sha-256 38:1F:02:E5:2A:49:9A:2A:D9:8E:B9:9B:4C:40:21:B7:F1:C4:27:8E:B5:68:D6:E0:91:08:D9:CB:2B:AC:B3:87 +a=setup:actpass +a=mid:3 +a=extmap:14 urn:ietf:params:rtp-hdrext:ssrc-audio-level +a=extmap:2 http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time +a=extmap:4 http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01 +a=extmap:9 urn:ietf:params:rtp-hdrext:sdes:mid +a=sendonly +a=msid:- 75b8c290-1f46-464c-8dc5-a4a3bced313a +a=rtcp-mux +a=rtpmap:111 opus/48000/2 +a=rtcp-fb:111 transport-cc +a=fmtp:111 minptime=10;usedtx=1;useinbandfec=1 +a=rtpmap:63 red/48000/2 +a=fmtp:63 111/111 +a=rtpmap:9 G722/8000 +a=rtpmap:0 PCMU/8000 +a=rtpmap:8 PCMA/8000 +a=rtpmap:13 CN/8000 +a=rtpmap:110 telephone-event/48000 +a=rtpmap:126 telephone-event/8000 +a=ssrc:1281279951 cname:mbGW+aeWMLFhPbBC +a=ssrc:1281279951 msid:- 75b8c290-1f46-464c-8dc5-a4a3bced313a +`.trim(); diff --git a/packages/client/src/helpers/__tests__/sdp-munging.test.ts b/packages/client/src/helpers/__tests__/sdp-munging.test.ts index 22e6d58954..462ccd2292 100644 --- a/packages/client/src/helpers/__tests__/sdp-munging.test.ts +++ b/packages/client/src/helpers/__tests__/sdp-munging.test.ts @@ -1,5 +1,11 @@ import { describe, expect, it } from 'vitest'; -import { setPreferredCodec, removeCodec, toggleDtx } from '../sdp-munging'; +import { + enableHighQualityAudio, + removeCodec, + setPreferredCodec, + toggleDtx, +} from '../sdp-munging'; +import { initialSdp as HQAudioSDP } from './hq-audio-sdp'; const sdpWithRed = `v=0 o=- 3265541491372987511 2 IN IP4 127.0.0.1 @@ -275,4 +281,10 @@ a=maxptime:40`; const dtxDisabledSdp = toggleDtx(dtxEnabledSdp, false); expect(dtxDisabledSdp.search('usedtx=0') !== -1).toBeTruthy(); }); + + it('enables HighQuality audio for Opus', () => { + const sdpWithHighQualityAudio = enableHighQualityAudio(HQAudioSDP, '3'); + expect(sdpWithHighQualityAudio).toContain('maxaveragebitrate=510000'); + expect(sdpWithHighQualityAudio).toContain('stereo=1'); + }); }); diff --git a/packages/client/src/helpers/sdp-munging.ts b/packages/client/src/helpers/sdp-munging.ts index abee224fd9..6af1af7fae 100644 --- a/packages/client/src/helpers/sdp-munging.ts +++ b/packages/client/src/helpers/sdp-munging.ts @@ -1,3 +1,5 @@ +import * as SDP from 'sdp-transform'; + type Media = { original: string; mediaWithPorts: string; @@ -227,3 +229,50 @@ export const toggleDtx = (sdp: string, enable: boolean): string => { } return sdp; }; + +/** + * Enables high-quality audio through SDP munging for the given trackMid. + * + * @param sdp the SDP to munge. + * @param trackMid the trackMid. + * @param maxBitrate the max bitrate to set. + */ +export const enableHighQualityAudio = ( + sdp: string, + trackMid: string, + maxBitrate: number = 510000, +): string => { + maxBitrate = Math.max(Math.min(maxBitrate, 510000), 96000); + + const parsedSdp = SDP.parse(sdp); + const audioMedia = parsedSdp.media.find( + (m) => m.type === 'audio' && String(m.mid) === trackMid, + ); + + if (!audioMedia) return sdp; + + const opusRtp = audioMedia.rtp.find((r) => r.codec === 'opus'); + if (!opusRtp) return sdp; + + const opusFmtp = audioMedia.fmtp.find((f) => f.payload === opusRtp.payload); + if (!opusFmtp) return sdp; + + // enable stereo, if not already enabled + if (opusFmtp.config.match(/stereo=(\d)/)) { + opusFmtp.config = opusFmtp.config.replace(/stereo=(\d)/, 'stereo=1'); + } else { + opusFmtp.config = `${opusFmtp.config};stereo=1`; + } + + // set maxaveragebitrate, to the given value + if (opusFmtp.config.match(/maxaveragebitrate=(\d*)/)) { + opusFmtp.config = opusFmtp.config.replace( + /maxaveragebitrate=(\d*)/, + `maxaveragebitrate=${maxBitrate}`, + ); + } else { + opusFmtp.config = `${opusFmtp.config};maxaveragebitrate=${maxBitrate}`; + } + + return SDP.write(parsedSdp); +}; diff --git a/packages/client/src/rtc/Publisher.ts b/packages/client/src/rtc/Publisher.ts index 4d8b370b38..70db0dd9f4 100644 --- a/packages/client/src/rtc/Publisher.ts +++ b/packages/client/src/rtc/Publisher.ts @@ -21,7 +21,7 @@ import { import { CallState } from '../store'; import { PublishOptions } from '../types'; import { isReactNative } from '../helpers/platforms'; -import { toggleDtx } from '../helpers/sdp-munging'; +import { enableHighQualityAudio, toggleDtx } from '../helpers/sdp-munging'; import { Logger } from '../coordinator/connection/types'; import { getLogger } from '../logger'; import { Dispatcher } from './Dispatcher'; @@ -56,6 +56,12 @@ export class Publisher { [TrackType.SCREEN_SHARE_AUDIO]: undefined, [TrackType.UNSPECIFIED]: undefined, }; + + private readonly publishOptionsPerTrackType = new Map< + TrackType, + PublishOptions + >(); + /** * An array maintaining the order how transceivers were added to the peer connection. * This is needed because some browsers (Firefox) don't reliably report @@ -71,7 +77,7 @@ export class Publisher { [TrackType.AUDIO]: 'audio', [TrackType.VIDEO]: 'video', [TrackType.SCREEN_SHARE]: 'video', - [TrackType.SCREEN_SHARE_AUDIO]: undefined, + [TrackType.SCREEN_SHARE_AUDIO]: 'audio', [TrackType.UNSPECIFIED]: undefined, }; @@ -180,10 +186,11 @@ export class Publisher { * * Consecutive calls to this method will replace the stream. * The previous stream will be stopped. - * @param mediaStream - * @param track - * @param trackType - * @param opts + * + * @param mediaStream the media stream to publish. + * @param track the track to publish. + * @param trackType the track type to publish. + * @param opts the optional publish options to use. */ publishStream = async ( mediaStream: MediaStream, @@ -229,6 +236,8 @@ export class Publisher { const videoEncodings = trackType === TrackType.VIDEO ? findOptimalVideoLayers(track, targetResolution) + : trackType === TrackType.SCREEN_SHARE + ? findOptimalScreenSharingLayers(track, opts.screenShareSettings) : undefined; let preferredCodec = opts.preferredCodec; @@ -264,6 +273,7 @@ export class Publisher { logger('debug', `Added ${TrackType[trackType]} transceiver`); this.transceiverInitOrder.push(trackType); this.transceiverRegistry[trackType] = transceiver; + this.publishOptionsPerTrackType.set(trackType, opts); if ('setCodecPreferences' in transceiver && codecPreferences) { logger( @@ -535,7 +545,24 @@ export class Publisher { this.isIceRestarting = options?.iceRestart ?? false; const offer = await this.pc.createOffer(options); - offer.sdp = this.mungeCodecs(offer.sdp); + let sdp = this.mungeCodecs(offer.sdp); + if (sdp && this.isPublishing(TrackType.SCREEN_SHARE_AUDIO)) { + const transceiver = + this.transceiverRegistry[TrackType.SCREEN_SHARE_AUDIO]; + if (transceiver && transceiver.sender.track) { + const mid = + transceiver.mid ?? + this.extractMid( + sdp, + transceiver.sender.track, + TrackType.SCREEN_SHARE_AUDIO, + ); + sdp = enableHighQualityAudio(sdp, mid); + } + } + + // set the munged SDP back to the offer + offer.sdp = sdp; const trackInfos = this.getCurrentTrackInfos(offer.sdp); if (trackInfos.length === 0) { @@ -584,48 +611,48 @@ export class Publisher { return sdp; }; - getCurrentTrackInfos = (sdp?: string) => { - sdp = sdp || this.pc.localDescription?.sdp; - const extractMid = ( - defaultMid: string | null, - track: MediaStreamTrack, - trackType: TrackType, - ): string => { - if (defaultMid) return defaultMid; - if (!sdp) { - logger('warn', 'No SDP found. Returning empty mid'); - return ''; - } + private extractMid = ( + sdp: string | undefined, + track: MediaStreamTrack, + trackType: TrackType, + ): string => { + if (!sdp) { + logger('warn', 'No SDP found. Returning empty mid'); + return ''; + } + + logger( + 'debug', + `No 'mid' found for track. Trying to find it from the Offer SDP`, + ); + const parsedSdp = SDP.parse(sdp); + const media = parsedSdp.media.find((m) => { + return ( + m.type === track.kind && + // if `msid` is not present, we assume that the track is the first one + (m.msid?.includes(track.id) ?? true) + ); + }); + if (typeof media?.mid === 'undefined') { logger( 'debug', - `No 'mid' found for track. Trying to find it from the Offer SDP`, + `No mid found in SDP for track type ${track.kind} and id ${track.id}. Attempting to find a heuristic mid`, ); - const parsedSdp = SDP.parse(sdp); - const media = parsedSdp.media.find((m) => { - return ( - m.type === track.kind && - // if `msid` is not present, we assume that the track is the first one - (m.msid?.includes(track.id) ?? true) - ); - }); - if (typeof media?.mid === 'undefined') { - logger( - 'debug', - `No mid found in SDP for track type ${track.kind} and id ${track.id}. Attempting to find a heuristic mid`, - ); + const heuristicMid = this.transceiverInitOrder.indexOf(trackType); + if (heuristicMid !== -1) { + return String(heuristicMid); + } - const heuristicMid = this.transceiverInitOrder.indexOf(trackType); - if (heuristicMid !== -1) { - return String(heuristicMid); - } + logger('debug', 'No heuristic mid found. Returning empty mid'); + return ''; + } + return String(media.mid); + }; - logger('debug', 'No heuristic mid found. Returning empty mid'); - return ''; - } - return String(media.mid); - }; + getCurrentTrackInfos = (sdp?: string) => { + sdp = sdp || this.pc.localDescription?.sdp; const { settings } = this.state; const targetResolution = settings?.video.target_resolution; @@ -642,11 +669,15 @@ export class Publisher { const track = transceiver.sender.track!; let optimalLayers: OptimalVideoLayer[]; if (track.readyState === 'live') { + const publishOpts = this.publishOptionsPerTrackType.get(trackType); optimalLayers = trackType === TrackType.VIDEO ? findOptimalVideoLayers(track, targetResolution) : trackType === TrackType.SCREEN_SHARE - ? findOptimalScreenSharingLayers(track) + ? findOptimalScreenSharingLayers( + track, + publishOpts?.screenShareSettings, + ) : []; this.trackLayersCache[trackType] = optimalLayers; } else { @@ -670,16 +701,24 @@ export class Publisher { }, })); + const isAudioTrack = [ + TrackType.AUDIO, + TrackType.SCREEN_SHARE_AUDIO, + ].includes(trackType); + + const trackSettings = track.getSettings(); + // @ts-expect-error - `channelCount` is not defined on `MediaTrackSettings` + const isStereo = isAudioTrack && trackSettings.channelCount === 2; + return { trackId: track.id, layers: layers, trackType, - mid: extractMid(transceiver.mid, track, trackType), + mid: transceiver.mid ?? this.extractMid(sdp, track, trackType), - // FIXME OL: adjust these values - stereo: false, - dtx: TrackType.AUDIO === trackType && this.isDtxEnabled, - red: TrackType.AUDIO === trackType && this.isRedEnabled, + stereo: isStereo, + dtx: isAudioTrack && this.isDtxEnabled, + red: isAudioTrack && this.isRedEnabled, }; }); }; diff --git a/packages/client/src/rtc/Subscriber.ts b/packages/client/src/rtc/Subscriber.ts index 3c16412a84..f50e6ccf66 100644 --- a/packages/client/src/rtc/Subscriber.ts +++ b/packages/client/src/rtc/Subscriber.ts @@ -264,6 +264,7 @@ export class Subscriber { TRACK_TYPE_AUDIO: 'audioStream', TRACK_TYPE_VIDEO: 'videoStream', TRACK_TYPE_SCREEN_SHARE: 'screenShareStream', + TRACK_TYPE_SCREEN_SHARE_AUDIO: 'screenShareAudioStream', } as const )[trackType]; @@ -374,6 +375,8 @@ export class Subscriber { const errorMessage = e instanceof RTCPeerConnectionIceErrorEvent && `${e.errorCode}: ${e.errorText}`; - logger('error', `ICE Candidate error`, errorMessage); + const logLevel = + this.pc.iceConnectionState === 'connected' ? 'debug' : 'error'; + logger(logLevel, `ICE Candidate error`, errorMessage); }; } diff --git a/packages/client/src/rtc/helpers/tracks.ts b/packages/client/src/rtc/helpers/tracks.ts index ae7d4ef1ab..4791711ce5 100644 --- a/packages/client/src/rtc/helpers/tracks.ts +++ b/packages/client/src/rtc/helpers/tracks.ts @@ -3,6 +3,7 @@ import type { StreamVideoLocalParticipant, StreamVideoParticipant, } from '../../types'; +import { TrackMuteType } from '../../types'; export const trackTypeToParticipantStreamKey = ( trackType: TrackType, @@ -10,12 +11,17 @@ export const trackTypeToParticipantStreamKey = ( switch (trackType) { case TrackType.SCREEN_SHARE: return 'screenShareStream'; + case TrackType.SCREEN_SHARE_AUDIO: + return 'screenShareAudioStream'; case TrackType.VIDEO: return 'videoStream'; case TrackType.AUDIO: return 'audioStream'; + case TrackType.UNSPECIFIED: + throw new Error('Track type is unspecified'); default: - throw new Error(`Unknown track type: ${trackType}`); + const exhaustiveTrackTypeCheck: never = trackType; + throw new Error(`Unknown track type: ${exhaustiveTrackTypeCheck}`); } }; @@ -28,15 +34,16 @@ export const trackTypeToDeviceIdKey = ( case TrackType.VIDEO: return 'videoDeviceId'; case TrackType.SCREEN_SHARE: + case TrackType.SCREEN_SHARE_AUDIO: + case TrackType.UNSPECIFIED: return undefined; default: - throw new Error(`Unknown track type: ${trackType}`); + const exhaustiveTrackTypeCheck: never = trackType; + throw new Error(`Unknown track type: ${exhaustiveTrackTypeCheck}`); } }; -export const muteTypeToTrackType = ( - muteType: 'audio' | 'video' | 'screenshare', -): TrackType => { +export const muteTypeToTrackType = (muteType: TrackMuteType): TrackType => { switch (muteType) { case 'audio': return TrackType.AUDIO; @@ -44,7 +51,10 @@ export const muteTypeToTrackType = ( return TrackType.VIDEO; case 'screenshare': return TrackType.SCREEN_SHARE; + case 'screenshare_audio': + return TrackType.SCREEN_SHARE_AUDIO; default: - throw new Error(`Unknown mute type: ${muteType}`); + const exhaustiveMuteTypeCheck: never = muteType; + throw new Error(`Unknown mute type: ${exhaustiveMuteTypeCheck}`); } }; diff --git a/packages/client/src/rtc/videoLayers.ts b/packages/client/src/rtc/videoLayers.ts index 74a9d4757d..6a992f57cb 100644 --- a/packages/client/src/rtc/videoLayers.ts +++ b/packages/client/src/rtc/videoLayers.ts @@ -1,4 +1,5 @@ import { getOSInfo } from '../client-details'; +import { ScreenShareSettings } from '../types'; import { TargetResolution } from '../gen/coordinator'; import { isReactNative } from '../helpers/platforms'; @@ -122,6 +123,7 @@ const withSimulcastConstraints = ( export const findOptimalScreenSharingLayers = ( videoTrack: MediaStreamTrack, + preferences?: ScreenShareSettings, ): OptimalVideoLayer[] => { const settings = videoTrack.getSettings(); return [ @@ -130,9 +132,9 @@ export const findOptimalScreenSharingLayers = ( rid: 'q', // single track, start from 'q' width: settings.width || 0, height: settings.height || 0, - maxBitrate: 3000000, scaleResolutionDownBy: 1, - maxFramerate: 30, + maxBitrate: preferences?.maxBitrate ?? 3000000, + maxFramerate: preferences?.maxFramerate ?? 30, }, ]; }; diff --git a/packages/client/src/types.ts b/packages/client/src/types.ts index 2c487042cf..aceb3fa8d4 100644 --- a/packages/client/src/types.ts +++ b/packages/client/src/types.ts @@ -50,6 +50,12 @@ export interface StreamVideoParticipant extends Participant { */ screenShareStream?: MediaStream; + /** + * The participant's screen audio stream, if they are sharing their audio, + * and we are subscribed to it. + */ + screenShareAudioStream?: MediaStream; + /** * The preferred video dimensions for this participant. * Set it to `undefined` to unsubscribe from this participant's video. @@ -110,6 +116,12 @@ export interface StreamVideoLocalParticipant extends StreamVideoParticipant { } export type VideoTrackType = 'videoTrack' | 'screenShareTrack'; +export type AudioTrackType = 'audioTrack' | 'screenShareAudioTrack'; +export type TrackMuteType = + | 'audio' + | 'video' + | 'screenshare' + | 'screenshare_audio'; /** * Represents a participant's pin state. @@ -161,6 +173,21 @@ export type SubscriptionChanges = { export type PublishOptions = { preferredCodec?: string | null; + screenShareSettings?: ScreenShareSettings; +}; + +export type ScreenShareSettings = { + /** + * Limits the maximum framerate (in frames per second) of the screen share. + * Defaults to 30. + */ + maxFramerate?: number; + + /** + * Limits the maximum bitrate (in bits per second) of the screen share. + * Defaults to 3000000 (3Mbps). + */ + maxBitrate?: number; }; export type CallLeaveOptions = { diff --git a/packages/react-bindings/src/hooks/callStateHooks.ts b/packages/react-bindings/src/hooks/callStateHooks.ts index 761ccee384..a70751ae01 100644 --- a/packages/react-bindings/src/hooks/callStateHooks.ts +++ b/packages/react-bindings/src/hooks/callStateHooks.ts @@ -6,11 +6,9 @@ import { CallSettingsResponse, CallState, CallStatsReport, - CameraManagerState, Comparator, EgressResponse, MemberResponse, - MicrophoneManagerState, StreamVideoParticipant, UserResponse, } from '@stream-io/video-client'; @@ -330,12 +328,7 @@ export const useCallThumbnail = () => { */ export const useCameraState = () => { const call = useCall(); - - const { - camera = { - state: new CameraManagerState(), - }, - } = call as Call; + const { camera } = call as Call; const status = useObservableValue(camera.state.status$); const direction = useObservableValue(camera.state.direction$); @@ -353,12 +346,7 @@ export const useCameraState = () => { */ export const useMicrophoneState = () => { const call = useCall(); - - const { - microphone = { - state: new MicrophoneManagerState(), - }, - } = call as Call; + const { microphone } = call as Call; const status = useObservableValue(microphone.state.status$); const selectedDevice = useObservableValue(microphone.state.selectedDevice$); @@ -368,3 +356,14 @@ export const useMicrophoneState = () => { selectedDevice, }; }; + +export const useScreenShareState = () => { + const call = useCall(); + const { screenShare } = call as Call; + + const status = useObservableValue(screenShare.state.status$); + + return { + status, + }; +}; diff --git a/packages/react-native-sdk/docusaurus/docs/reactnative/03-core/03-call-and-participant-state.mdx b/packages/react-native-sdk/docusaurus/docs/reactnative/03-core/03-call-and-participant-state.mdx index 70294de429..c4ef461237 100644 --- a/packages/react-native-sdk/docusaurus/docs/reactnative/03-core/03-call-and-participant-state.mdx +++ b/packages/react-native-sdk/docusaurus/docs/reactnative/03-core/03-call-and-participant-state.mdx @@ -150,6 +150,7 @@ The `StreamVideoParticipant` object contains the following information: | `audioStream` | The published audio `MediaStream`. | | `videoStream` | The published video `MediaStream`. | | `screenShareStream` | The published screen share `MediaStream`. | +| `screenShareAudioStream` | The published screen share audio `MediaStream`. | | `isLocalParticipant` | It's `true` if the participant is the local participant. | | `pin` | Holds pinning information. | | `reaction` | The last reaction this user has sent to this call. | diff --git a/packages/react-sdk/README.md b/packages/react-sdk/README.md index 87de5ced13..8454085740 100644 --- a/packages/react-sdk/README.md +++ b/packages/react-sdk/README.md @@ -58,8 +58,8 @@ Here are some of the features we support: - [x] Enhanced device management API - [x] Composite layout for streaming and recording - [x] Livestream Player -- [ ] Screenshare Audio -- [ ] Screen-sharing resolution and FPS control +- [x] Screenshare Audio +- [x] Screen-sharing bitrate and FPS control - [ ] Fast-reconnects - [ ] New Device Management API - [x] SFU retries diff --git a/packages/react-sdk/docusaurus/docs/React/02-guides/03-call-and-participant-state.mdx b/packages/react-sdk/docusaurus/docs/React/02-guides/03-call-and-participant-state.mdx index be90e52854..7b0b29520c 100644 --- a/packages/react-sdk/docusaurus/docs/React/02-guides/03-call-and-participant-state.mdx +++ b/packages/react-sdk/docusaurus/docs/React/02-guides/03-call-and-participant-state.mdx @@ -143,6 +143,7 @@ The `StreamVideoParticipant` object contains the following information: | `audioStream` | The published audio `MediaStream`. | | `videoStream` | The published video `MediaStream`. | | `screenShareStream` | The published screen share `MediaStream`. | +| `screenShareAudioStream` | The published screen share audio `MediaStream`. | | `isLocalParticipant` | It's `true` if the participant is the local participant. | | `pin` | Holds pinning information. | | `reaction` | The last reaction this user has sent to this call. | diff --git a/packages/react-sdk/docusaurus/docs/React/02-guides/04-screensharing.mdx b/packages/react-sdk/docusaurus/docs/React/02-guides/04-screensharing.mdx new file mode 100644 index 0000000000..457e09ecdf --- /dev/null +++ b/packages/react-sdk/docusaurus/docs/React/02-guides/04-screensharing.mdx @@ -0,0 +1,76 @@ +--- +id: screensharing +title: Screen Sharing +description: Managing Screen Sharing +--- + +## Screen Sharing + +### Start/Stop Screen Sharing + +```typescript +call.screenShare.toggle(); + +// or +call.screenShare.enable(); +call.screenShare.disable(); +``` + +### Screen Sharing Status + +Here is how you can access the status of screen sharing: + +```typescript +import { useCallStateHooks } from '@stream-io/video-react-sdk'; + +call.screenShare.state.status; // enabled, disabled or undefined + +// or, if you want to subscribe to changes +const { useScreenShareState } = useCallStateHooks(); +const { status } = useScreenShareState(); +``` + +### Screen Sharing Settings + +The behavior of the screen share video track can be customized, and a few parameters can be set: + +```typescript +call.screenShare.setSettings({ + maxFramerate: 15, // will be clamped between 1 and 15 fps + maxBitrate: 1500000, // will use at most 1.5Mbps +}); + +call.screenShare.enable(); +``` + +### Render Screen Share + +Our SDK provided [`ParticipantView`](../../ui-components/core/participant-view/) component can automatically render the screen share video track. + +## Screen Share Audio + +### Start/Stop Screen Share Audio + +```typescript +// enable it +call.screenShare.enableScreenShareAudio(); + +// publish video and audio (if available, and supported by the browser) +call.screenShare.enable(); + +// disable it +call.screenShare.disableScreenShareAudio(); +``` + +### Play Screen Share Audio + +Our SDK provided [`ParticipantView`](../../ui-components/core/participant-view/) component can automatically play the screen share audio track. + +### Caveats + +Screen Share Audio has limited support across browsers and platforms. +For most up-to-date information, please take a look at [Browser Compatibility](https://developer.mozilla.org/en-US/docs/Web/API/Screen_Capture_API/Using_Screen_Capture#browser_compatibility). + +In addition to that, there are a [few caveats](https://caniuse.com/?search=getDisplayMedia) that you should be aware of: + +- On Windows, the entire system audio can be captured, but on MacOS and Lunux, only the audio of a tab can be captured. diff --git a/packages/react-sdk/src/components/CallParticipantsList/CallParticipantListingItem.tsx b/packages/react-sdk/src/components/CallParticipantsList/CallParticipantListingItem.tsx index 4071c7b20c..b6c6badbba 100644 --- a/packages/react-sdk/src/components/CallParticipantsList/CallParticipantListingItem.tsx +++ b/packages/react-sdk/src/components/CallParticipantsList/CallParticipantListingItem.tsx @@ -142,7 +142,7 @@ export const ParticipantActionsContextMenu = ({ const [pictureInPictureElement, setPictureInPictureElement] = useState( document.pictureInPictureElement, ); - const activeCall = useCall(); + const call = useCall(); const { t } = useI18n(); const { pin, publishedTracks, sessionId, userId } = participant; @@ -152,29 +152,26 @@ export const ParticipantActionsContextMenu = ({ const hasScreenShare = publishedTracks.includes( SfuModels.TrackType.SCREEN_SHARE, ); + const hasScreenShareAudio = publishedTracks.includes( + SfuModels.TrackType.SCREEN_SHARE_AUDIO, + ); - const blockUser = () => { - activeCall?.blockUser(userId); - }; - const muteAudio = () => { - activeCall?.muteUser(userId, 'audio'); - }; - const muteVideo = () => { - activeCall?.muteUser(userId, 'video'); - }; - const muteScreenShare = () => { - activeCall?.muteUser(userId, 'screenshare'); - }; + const blockUser = () => call?.blockUser(userId); + const muteAudio = () => call?.muteUser(userId, 'audio'); + const muteVideo = () => call?.muteUser(userId, 'video'); + const muteScreenShare = () => call?.muteUser(userId, 'screenshare'); + const muteScreenShareAudio = () => + call?.muteUser(userId, 'screenshare_audio'); const grantPermission = (permission: string) => () => { - activeCall?.updateUserPermissions({ + call?.updateUserPermissions({ user_id: userId, grant_permissions: [permission], }); }; const revokePermission = (permission: string) => () => { - activeCall?.updateUserPermissions({ + call?.updateUserPermissions({ user_id: userId, revoke_permissions: [permission], }); @@ -182,14 +179,14 @@ export const ParticipantActionsContextMenu = ({ const toggleParticipantPinnedAt = () => { if (pin) { - activeCall?.unpin(sessionId); + call?.unpin(sessionId); } else { - activeCall?.pin(sessionId); + call?.pin(sessionId); } }; const pinForEveryone = () => { - activeCall + call ?.pinForEveryone({ user_id: userId, session_id: sessionId, @@ -200,7 +197,7 @@ export const ParticipantActionsContextMenu = ({ }; const unpinForEveryone = () => { - activeCall + call ?.unpinForEveryone({ user_id: userId, session_id: sessionId, @@ -253,10 +250,11 @@ export const ParticipantActionsContextMenu = ({ }, [videoElement]); const togglePictureInPicture = () => { - if (videoElement && pictureInPictureElement !== videoElement) + if (videoElement && pictureInPictureElement !== videoElement) { return videoElement .requestPictureInPicture() .catch(console.error) as Promise; + } document.exitPictureInPicture().catch(console.error); }; @@ -308,6 +306,13 @@ export const ParticipantActionsContextMenu = ({ {t('Mute audio')} + + + {t('Mute screen share audio')} + {participantViewElement && ( diff --git a/packages/react-sdk/src/core/components/Audio/Audio.tsx b/packages/react-sdk/src/core/components/Audio/Audio.tsx index da16da6cc9..ae795edde4 100644 --- a/packages/react-sdk/src/core/components/Audio/Audio.tsx +++ b/packages/react-sdk/src/core/components/Audio/Audio.tsx @@ -1,12 +1,28 @@ import { ComponentPropsWithoutRef, useEffect, useState } from 'react'; -import { StreamVideoParticipant } from '@stream-io/video-client'; +import { + AudioTrackType, + StreamVideoParticipant, +} from '@stream-io/video-client'; import { useCall } from '@stream-io/video-react-bindings'; export type AudioProps = ComponentPropsWithoutRef<'audio'> & { + /** + * The participant whose audio stream we want to play. + */ participant: StreamVideoParticipant; + + /** + * The type of audio stream to play for the given participant. + * The default value is `audioTrack`. + */ + trackType?: AudioTrackType; }; -export const Audio = ({ participant, ...rest }: AudioProps) => { +export const Audio = ({ + participant, + trackType = 'audioTrack', + ...rest +}: AudioProps) => { const call = useCall(); const [audioElement, setAudioElement] = useState( null, @@ -15,13 +31,11 @@ export const Audio = ({ participant, ...rest }: AudioProps) => { useEffect(() => { if (!call || !audioElement) return; - - const cleanup = call.bindAudioElement(audioElement, sessionId); - + const cleanup = call.bindAudioElement(audioElement, sessionId, trackType); return () => { cleanup?.(); }; - }, [call, sessionId, audioElement]); + }, [call, sessionId, audioElement, trackType]); return (