From 89b1ea02617ab8b405a825ddb48fe0134feb16ca Mon Sep 17 00:00:00 2001 From: Oliver Lazoroski Date: Tue, 3 Oct 2023 14:53:36 +0200 Subject: [PATCH] feat: ScreenShare for Plain-JS --- packages/client/src/Call.ts | 17 ++- packages/client/src/devices/CameraManager.ts | 7 +- .../src/devices/InputMediaDeviceManager.ts | 96 +++++++------- .../devices/InputMediaDeviceManagerState.ts | 48 ++++--- .../client/src/devices/MicrophoneManager.ts | 7 +- .../client/src/devices/ScreenShareManager.ts | 85 +++++++++++++ .../client/src/devices/ScreenShareState.ts | 63 ++++++++++ .../__tests__/InputMediaDeviceManager.test.ts | 2 +- .../__tests__/ScreenShareManager.test.ts | 119 ++++++++++++++++++ .../client/src/devices/__tests__/mocks.ts | 39 +++++- packages/client/src/devices/index.ts | 2 + packages/client/src/rtc/Publisher.ts | 24 +++- packages/client/src/rtc/videoLayers.ts | 6 +- packages/client/src/types.ts | 15 +++ .../client/ts-quickstart/src/controls.ts | 16 +++ sample-apps/client/ts-quickstart/src/main.ts | 8 ++ 16 files changed, 475 insertions(+), 79 deletions(-) create mode 100644 packages/client/src/devices/ScreenShareManager.ts create mode 100644 packages/client/src/devices/ScreenShareState.ts create mode 100644 packages/client/src/devices/__tests__/ScreenShareManager.test.ts diff --git a/packages/client/src/Call.ts b/packages/client/src/Call.ts index 4bf57aace0..6943837ffd 100644 --- a/packages/client/src/Call.ts +++ b/packages/client/src/Call.ts @@ -122,6 +122,7 @@ import { CameraDirection, CameraManager, MicrophoneManager, + ScreenShareManager, SpeakerManager, } from './devices'; @@ -170,6 +171,11 @@ export class Call { */ readonly speaker: SpeakerManager; + /** + * Device manager for the screen. + */ + readonly screenShare: ScreenShareManager; + /** * The DynascaleManager instance. */ @@ -283,6 +289,7 @@ export class Call { this.camera = new CameraManager(this); this.microphone = new MicrophoneManager(this); this.speaker = new SpeakerManager(); + this.screenShare = new ScreenShareManager(this); } private registerEffects() { @@ -1095,7 +1102,6 @@ export class Call { * Consecutive calls to this method will replace the audio stream that is currently being published. * The previous audio stream will be stopped. * - * * @param audioStream the audio stream to publish. */ publishAudioStream = async (audioStream: MediaStream) => { @@ -1126,10 +1132,13 @@ export class Call { * Consecutive calls to this method will replace the previous screen-share stream. * The previous screen-share stream will be stopped. * - * * @param screenShareStream the screen-share stream to publish. + * @param opts the options to use when publishing the stream. */ - publishScreenShareStream = async (screenShareStream: MediaStream) => { + publishScreenShareStream = async ( + screenShareStream: MediaStream, + opts: PublishOptions = {}, + ) => { // we should wait until we get a JoinResponse from the SFU, // otherwise we risk breaking the ICETrickle flow. await this.assertCallJoined(); @@ -1154,6 +1163,7 @@ export class Call { screenShareStream, screenShareTrack, TrackType.SCREEN_SHARE, + opts, ); const [screenShareAudioTrack] = screenShareStream.getAudioTracks(); @@ -1162,6 +1172,7 @@ export class Call { screenShareStream, screenShareAudioTrack, TrackType.SCREEN_SHARE_AUDIO, + opts, ); } }; diff --git a/packages/client/src/devices/CameraManager.ts b/packages/client/src/devices/CameraManager.ts index 7cf4492c04..01d416c5ed 100644 --- a/packages/client/src/devices/CameraManager.ts +++ b/packages/client/src/devices/CameraManager.ts @@ -69,6 +69,7 @@ export class CameraManager extends InputMediaDeviceManager { protected getDevices(): Observable { return getVideoDevices(); } + protected getStream( constraints: MediaTrackConstraints, ): Promise { @@ -82,14 +83,12 @@ export class CameraManager extends InputMediaDeviceManager { } return getVideoStream(constraints); } + protected publishStream(stream: MediaStream): Promise { return this.call.publishVideoStream(stream); } + protected stopPublishStream(stopTracks: boolean): Promise { return this.call.stopPublish(TrackType.VIDEO, stopTracks); } - - protected getTrack() { - return this.state.mediaStream?.getVideoTracks()[0]; - } } diff --git a/packages/client/src/devices/InputMediaDeviceManager.ts b/packages/client/src/devices/InputMediaDeviceManager.ts index 1bd58f8609..5154542e29 100644 --- a/packages/client/src/devices/InputMediaDeviceManager.ts +++ b/packages/client/src/devices/InputMediaDeviceManager.ts @@ -8,7 +8,8 @@ import { getLogger } from '../logger'; import { TrackType } from '../gen/video/sfu/models/models'; export abstract class InputMediaDeviceManager< - T extends InputMediaDeviceManagerState, + T extends InputMediaDeviceManagerState, + C = MediaTrackConstraints, > { /** * @internal @@ -20,7 +21,7 @@ export abstract class InputMediaDeviceManager< disablePromise?: Promise; logger: Logger; - constructor( + protected constructor( protected readonly call: Call, public readonly state: T, protected readonly trackType: TrackType, @@ -40,7 +41,7 @@ export abstract class InputMediaDeviceManager< } /** - * Starts camera/microphone + * Starts stream. */ async enable() { if (this.state.status === 'enabled') { @@ -57,9 +58,7 @@ export abstract class InputMediaDeviceManager< } /** - * Stops camera/microphone - * - * @returns + * Stops the stream. */ async disable() { this.state.prevStatus = this.state.status; @@ -80,7 +79,7 @@ export abstract class InputMediaDeviceManager< } /** - * If status was previously enabled, it will reenable the device. + * If status was previously enabled, it will re-enable the device. */ async resume() { if ( @@ -92,9 +91,8 @@ export abstract class InputMediaDeviceManager< } /** - * If current device statis is disabled, it will enable the device, else it will disable it. - * - * @returns + * If the current device status is disabled, it will enable the device, + * else it will disable it. */ async toggle() { if (this.state.status === 'enabled') { @@ -131,15 +129,15 @@ export abstract class InputMediaDeviceManager< protected abstract getDevices(): Observable; - protected abstract getStream( - constraints: MediaTrackConstraints, - ): Promise; + protected abstract getStream(constraints: C): Promise; protected abstract publishStream(stream: MediaStream): Promise; protected abstract stopPublishStream(stopTracks: boolean): Promise; - protected abstract getTrack(): undefined | MediaStreamTrack; + protected getTracks(): MediaStreamTrack[] { + return this.state.mediaStream?.getTracks() ?? []; + } protected async muteStream(stopTracks: boolean = true) { if (!this.state.mediaStream) { @@ -150,59 +148,67 @@ export abstract class InputMediaDeviceManager< await this.stopPublishStream(stopTracks); } this.muteLocalStream(stopTracks); - if (this.getTrack()?.readyState === 'ended') { - // @ts-expect-error release() is present in react-native-webrtc and must be called to dispose the stream - if (typeof this.state.mediaStream.release === 'function') { - // @ts-expect-error - this.state.mediaStream.release(); + this.getTracks().forEach((track) => { + if (track.readyState === 'ended') { + // @ts-expect-error release() is present in react-native-webrtc + // and must be called to dispose the stream + if (typeof this.state.mediaStream.release === 'function') { + // @ts-expect-error + this.state.mediaStream.release(); + } + this.state.setMediaStream(undefined); } - this.state.setMediaStream(undefined); - } + }); } - private muteTrack() { - const track = this.getTrack(); - if (!track || !track.enabled) { - return; - } - track.enabled = false; + private muteTracks() { + this.getTracks().forEach((track) => { + if (track.enabled) track.enabled = false; + }); } - private unmuteTrack() { - const track = this.getTrack(); - if (!track || track.enabled) { - return; - } - track.enabled = true; + private unmuteTracks() { + this.getTracks().forEach((track) => { + if (!track.enabled) track.enabled = true; + }); } - private stopTrack() { - const track = this.getTrack(); - if (!track || track.readyState === 'ended') { - return; - } - track.stop(); + private stopTracks() { + this.getTracks().forEach((track) => { + if (track.readyState === 'live') track.stop(); + }); } private muteLocalStream(stopTracks: boolean) { if (!this.state.mediaStream) { return; } - stopTracks ? this.stopTrack() : this.muteTrack(); + if (stopTracks) { + this.stopTracks(); + } else { + this.muteTracks(); + } } protected async unmuteStream() { this.logger('debug', 'Starting stream'); let stream: MediaStream; - if (this.state.mediaStream && this.getTrack()?.readyState === 'live') { + if ( + this.state.mediaStream && + this.getTracks().every((t) => t.readyState === 'live') + ) { stream = this.state.mediaStream; - this.unmuteTrack(); + this.unmuteTracks(); } else { if (this.state.mediaStream) { - this.stopTrack(); + this.stopTracks(); } - const constraints = { deviceId: this.state.selectedDevice }; - stream = await this.getStream(constraints); + const defaultConstraints = this.state.defaultConstraints; + const constraints: MediaTrackConstraints = { + ...defaultConstraints, + deviceId: this.state.selectedDevice, + }; + stream = await this.getStream(constraints as C); } if (this.call.state.callingState === CallingState.JOINED) { await this.publishStream(stream); diff --git a/packages/client/src/devices/InputMediaDeviceManagerState.ts b/packages/client/src/devices/InputMediaDeviceManagerState.ts index c688414d9c..9c9b1b3184 100644 --- a/packages/client/src/devices/InputMediaDeviceManagerState.ts +++ b/packages/client/src/devices/InputMediaDeviceManagerState.ts @@ -1,9 +1,9 @@ -import { BehaviorSubject, distinctUntilChanged, Observable } from 'rxjs'; +import { BehaviorSubject, distinctUntilChanged } from 'rxjs'; import { RxUtils } from '../store'; export type InputDeviceStatus = 'enabled' | 'disabled' | undefined; -export abstract class InputMediaDeviceManagerState { +export abstract class InputMediaDeviceManagerState { protected statusSubject = new BehaviorSubject(undefined); protected mediaStreamSubject = new BehaviorSubject( undefined, @@ -11,6 +11,10 @@ export abstract class InputMediaDeviceManagerState { protected selectedDeviceSubject = new BehaviorSubject( undefined, ); + protected defaultConstraintsSubject = new BehaviorSubject( + undefined, + ); + /** * @internal */ @@ -20,31 +24,30 @@ export abstract class InputMediaDeviceManagerState { * An Observable that emits the current media stream, or `undefined` if the device is currently disabled. * */ - mediaStream$: Observable; + mediaStream$ = this.mediaStreamSubject.asObservable(); /** * An Observable that emits the currently selected device */ - selectedDevice$: Observable; + selectedDevice$ = this.selectedDeviceSubject + .asObservable() + .pipe(distinctUntilChanged()); /** * An Observable that emits the device status */ - status$: Observable; + status$ = this.statusSubject.asObservable().pipe(distinctUntilChanged()); + + /** + * The default constraints for the device. + */ + defaultConstraints$ = this.defaultConstraintsSubject.asObservable(); constructor( public readonly disableMode: | 'stop-tracks' | 'disable-tracks' = 'stop-tracks', - ) { - this.mediaStream$ = this.mediaStreamSubject.asObservable(); - this.selectedDevice$ = this.selectedDeviceSubject - .asObservable() - .pipe(distinctUntilChanged()); - this.status$ = this.statusSubject - .asObservable() - .pipe(distinctUntilChanged()); - } + ) {} /** * The device status @@ -102,6 +105,23 @@ export abstract class InputMediaDeviceManagerState { this.setCurrentValue(this.selectedDeviceSubject, deviceId); } + /** + * Gets the default constraints for the device. + */ + get defaultConstraints() { + return this.getCurrentValue(this.defaultConstraints$); + } + + /** + * Sets the default constraints for the device. + * + * @internal + * @param constraints the constraints to set. + */ + setDefaultConstraints(constraints: C | undefined) { + this.setCurrentValue(this.defaultConstraintsSubject, constraints); + } + /** * Updates the value of the provided Subject. * An `update` can either be a new value or a function which takes diff --git a/packages/client/src/devices/MicrophoneManager.ts b/packages/client/src/devices/MicrophoneManager.ts index a7a6b853b3..2f2caf8554 100644 --- a/packages/client/src/devices/MicrophoneManager.ts +++ b/packages/client/src/devices/MicrophoneManager.ts @@ -42,22 +42,21 @@ export class MicrophoneManager extends InputMediaDeviceManager { return getAudioDevices(); } + protected getStream( constraints: MediaTrackConstraints, ): Promise { return getAudioStream(constraints); } + protected publishStream(stream: MediaStream): Promise { return this.call.publishAudioStream(stream); } + protected stopPublishStream(stopTracks: boolean): Promise { return this.call.stopPublish(TrackType.AUDIO, stopTracks); } - protected getTrack() { - return this.state.mediaStream?.getAudioTracks()[0]; - } - private async startSpeakingWhileMutedDetection(deviceId?: string) { if (isReactNative()) { return; diff --git a/packages/client/src/devices/ScreenShareManager.ts b/packages/client/src/devices/ScreenShareManager.ts new file mode 100644 index 0000000000..0498651b71 --- /dev/null +++ b/packages/client/src/devices/ScreenShareManager.ts @@ -0,0 +1,85 @@ +import { Observable, of } from 'rxjs'; +import { InputMediaDeviceManager } from './InputMediaDeviceManager'; +import { ScreenShareState } from './ScreenShareState'; +import { Call } from '../Call'; +import { TrackType } from '../gen/video/sfu/models/models'; +import { getScreenShareStream } from './devices'; +import { ScreenShareSettings } from '../types'; + +export class ScreenShareManager extends InputMediaDeviceManager< + ScreenShareState, + DisplayMediaStreamOptions +> { + constructor(call: Call) { + super(call, new ScreenShareState(), TrackType.SCREEN_SHARE); + } + + /** + * Will enable screen share audio options on supported platforms. + * + * Note: for ongoing screen share, audio won't be enabled until you + * re-publish the screen share stream. + */ + enableScreenShareAudio(): void { + this.state.setAudioEnabled(true); + } + + /** + * Will disable screen share audio options on supported platforms. + */ + async disableScreenShareAudio(): Promise { + this.state.setAudioEnabled(false); + if (this.call.publisher?.isPublishing(TrackType.SCREEN_SHARE_AUDIO)) { + await this.call.stopPublish(TrackType.SCREEN_SHARE_AUDIO, true); + } + } + + /** + * Returns the current screen share settings. + */ + getSettings(): ScreenShareSettings | undefined { + return this.state.settings; + } + + /** + * Sets the current screen share settings. + * + * @param settings the settings to set. + */ + setSettings(settings: ScreenShareSettings | undefined): void { + this.state.setSettings(settings); + } + + protected getDevices(): Observable { + return of([]); // there are no devices to be listed for Screen Share + } + + protected getStream( + constraints: DisplayMediaStreamOptions, + ): Promise { + if (!this.state.audioEnabled) { + constraints.audio = false; + } + return getScreenShareStream(constraints); + } + + protected publishStream(stream: MediaStream): Promise { + return this.call.publishScreenShareStream(stream, { + screenShareSettings: this.state.settings, + }); + } + + protected async stopPublishStream(stopTracks: boolean): Promise { + await this.call.stopPublish(TrackType.SCREEN_SHARE, stopTracks); + await this.call.stopPublish(TrackType.SCREEN_SHARE_AUDIO, stopTracks); + } + + /** + * Overrides the default `select` method to throw an error. + * + * @param deviceId ignored. + */ + async select(deviceId: string | undefined): Promise { + throw new Error('This method is not supported in for Screen Share'); + } +} diff --git a/packages/client/src/devices/ScreenShareState.ts b/packages/client/src/devices/ScreenShareState.ts new file mode 100644 index 0000000000..5efeb9b622 --- /dev/null +++ b/packages/client/src/devices/ScreenShareState.ts @@ -0,0 +1,63 @@ +import { BehaviorSubject } from 'rxjs'; +import { InputMediaDeviceManagerState } from './InputMediaDeviceManagerState'; +import { distinctUntilChanged } from 'rxjs/operators'; +import { ScreenShareSettings } from '../types'; + +export class ScreenShareState extends InputMediaDeviceManagerState { + private audioEnabledSubject = new BehaviorSubject(true); + private settingsSubject = new BehaviorSubject< + ScreenShareSettings | undefined + >(undefined); + + /** + * An Observable that emits the current screen share audio status. + */ + audioEnabled$ = this.audioEnabledSubject + .asObservable() + .pipe(distinctUntilChanged()); + + /** + * An Observable that emits the current screen share settings. + */ + settings$ = this.settingsSubject.asObservable(); + + /** + * @internal + */ + protected getDeviceIdFromStream = ( + stream: MediaStream, + ): string | undefined => { + const [track] = stream.getTracks(); + return track?.getSettings().deviceId; + }; + + /** + * The current screen share audio status. + */ + get audioEnabled() { + return this.getCurrentValue(this.audioEnabled$); + } + + /** + * Set the current screen share audio status. + */ + setAudioEnabled(isEnabled: boolean) { + this.setCurrentValue(this.audioEnabledSubject, isEnabled); + } + + /** + * The current screen share settings. + */ + get settings() { + return this.getCurrentValue(this.settings$); + } + + /** + * Set the current screen share settings. + * + * @param settings the screen share settings to set. + */ + setSettings(settings: ScreenShareSettings | undefined) { + this.setCurrentValue(this.settingsSubject, settings); + } +} diff --git a/packages/client/src/devices/__tests__/InputMediaDeviceManager.test.ts b/packages/client/src/devices/__tests__/InputMediaDeviceManager.test.ts index bcc2e0f530..922a5d8bce 100644 --- a/packages/client/src/devices/__tests__/InputMediaDeviceManager.test.ts +++ b/packages/client/src/devices/__tests__/InputMediaDeviceManager.test.ts @@ -25,7 +25,7 @@ class TestInputMediaDeviceManager extends InputMediaDeviceManager Promise.resolve(mockVideoStream())); public publishStream = vi.fn(); public stopPublishStream = vi.fn(); - public getTrack = () => this.state.mediaStream!.getVideoTracks()[0]; + public getTracks = () => this.state.mediaStream?.getTracks() ?? []; constructor(call: Call) { super(call, new TestInputMediaDeviceManagerState(), TrackType.VIDEO); diff --git a/packages/client/src/devices/__tests__/ScreenShareManager.test.ts b/packages/client/src/devices/__tests__/ScreenShareManager.test.ts new file mode 100644 index 0000000000..fa771f1159 --- /dev/null +++ b/packages/client/src/devices/__tests__/ScreenShareManager.test.ts @@ -0,0 +1,119 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { ScreenShareManager } from '../ScreenShareManager'; +import { Call } from '../../Call'; +import { StreamClient } from '../../coordinator/connection/client'; +import { CallingState, StreamVideoWriteableStateStore } from '../../store'; +import * as RxUtils from '../../store/rxUtils'; +import { mockCall, mockScreenShareStream } from './mocks'; +import { getScreenShareStream } from '../devices'; +import { TrackType } from '../../gen/video/sfu/models/models'; + +vi.mock('../devices.ts', () => { + console.log('MOCKING devices API'); + return { + disposeOfMediaStream: vi.fn(), + getScreenShareStream: vi.fn(() => Promise.resolve(mockScreenShareStream())), + checkIfAudioOutputChangeSupported: vi.fn(() => Promise.resolve(true)), + }; +}); + +vi.mock('../../Call.ts', () => { + console.log('MOCKING Call'); + return { + Call: vi.fn(() => mockCall()), + }; +}); + +describe('ScreenShareManager', () => { + let manager: ScreenShareManager; + + beforeEach(() => { + manager = new ScreenShareManager( + new Call({ + id: '', + type: '', + streamClient: new StreamClient('abc123'), + clientStore: new StreamVideoWriteableStateStore(), + }), + ); + }); + + afterEach(() => { + vi.clearAllMocks(); + vi.resetModules(); + }); + + it('list devices', () => { + const devices = manager.listDevices(); + expect(RxUtils.getCurrentValue(devices)).toEqual([]); + }); + + it('select device', () => { + expect(manager.select('any-device-id')).rejects.toThrowError(); + }); + + it('get stream', async () => { + manager.enableScreenShareAudio(); + await manager.enable(); + expect(manager.state.status).toEqual('enabled'); + + expect(getScreenShareStream).toHaveBeenCalledWith({ + deviceId: undefined, + }); + }); + + it('get stream with no audio', async () => { + await manager.disableScreenShareAudio(); + await manager.enable(); + expect(manager.state.status).toEqual('enabled'); + + expect(getScreenShareStream).toHaveBeenCalledWith({ + deviceId: undefined, + audio: false, + }); + }); + + it('should get device id from stream', async () => { + expect(manager.state.selectedDevice).toBeUndefined(); + await manager.enable(); + expect(manager.state.selectedDevice).toBeDefined(); + expect(manager.state.selectedDevice).toEqual('screen'); + }); + + it('publishes screen share stream', async () => { + const call = manager['call']; + call.state.setCallingState(CallingState.JOINED); + await manager.enable(); + expect(call.publishScreenShareStream).toHaveBeenCalledWith( + manager.state.mediaStream, + { screenShareSettings: undefined }, + ); + }); + + it('publishes screen share stream with settings', async () => { + const call = manager['call']; + call.state.setCallingState(CallingState.JOINED); + + manager.setSettings({ maxFramerate: 15, maxBitrate: 1000 }); + + await manager.enable(); + expect(call.publishScreenShareStream).toHaveBeenCalledWith( + manager.state.mediaStream, + { screenShareSettings: { maxFramerate: 15, maxBitrate: 1000 } }, + ); + }); + + it('stop publish stream', async () => { + const call = manager['call']; + call.state.setCallingState(CallingState.JOINED); + await manager.enable(); + + await manager.disable(); + expect(manager.state.status).toEqual('disabled'); + expect(call.stopPublish).toHaveBeenCalledWith(TrackType.SCREEN_SHARE, true); + expect(call.stopPublish).toHaveBeenCalledWith( + TrackType.SCREEN_SHARE_AUDIO, + true, + ); + }); +}); diff --git a/packages/client/src/devices/__tests__/mocks.ts b/packages/client/src/devices/__tests__/mocks.ts index 7039eb221b..1fbe1dc364 100644 --- a/packages/client/src/devices/__tests__/mocks.ts +++ b/packages/client/src/devices/__tests__/mocks.ts @@ -1,6 +1,7 @@ import { vi } from 'vitest'; import { CallingState, CallState } from '../../store'; import { OwnCapability } from '../../gen/coordinator'; +import { Call } from '../../Call'; export const mockVideoDevices = [ { @@ -63,7 +64,7 @@ export const mockAudioDevices = [ }, ] as MediaDeviceInfo[]; -export const mockCall = () => { +export const mockCall = (): Partial => { const callState = new CallState(); callState.setCallingState(CallingState.JOINED); callState.setOwnCapabilities([ @@ -74,6 +75,7 @@ export const mockCall = () => { state: callState, publishVideoStream: vi.fn(), publishAudioStream: vi.fn(), + publishScreenShareStream: vi.fn(), stopPublish: vi.fn(), }; }; @@ -90,6 +92,7 @@ export const mockAudioStream = () => { }, }; return { + getTracks: () => [track], getAudioTracks: () => [track], } as MediaStream; }; @@ -108,6 +111,40 @@ export const mockVideoStream = () => { }, }; return { + getTracks: () => [track], getVideoTracks: () => [track], } as MediaStream; }; + +export const mockScreenShareStream = (includeAudio: boolean = true) => { + const track = { + getSettings: () => ({ + deviceId: 'screen', + }), + enabled: true, + readyState: 'live', + stop: () => { + track.readyState = 'ended'; + }, + }; + + const tracks = [track]; + if (includeAudio) { + tracks.push({ + getSettings: () => ({ + deviceId: 'screen-audio', + }), + enabled: true, + readyState: 'live', + stop: () => { + track.readyState = 'ended'; + }, + }); + } + + return { + getTracks: () => tracks, + getVideoTracks: () => tracks, + getAudioTracks: () => tracks, + } as MediaStream; +}; diff --git a/packages/client/src/devices/index.ts b/packages/client/src/devices/index.ts index a75dd4c1f4..3e1705367e 100644 --- a/packages/client/src/devices/index.ts +++ b/packages/client/src/devices/index.ts @@ -5,5 +5,7 @@ export * from './CameraManager'; export * from './CameraManagerState'; export * from './MicrophoneManager'; export * from './MicrophoneManagerState'; +export * from './ScreenShareManager'; +export * from './ScreenShareState'; export * from './SpeakerManager'; export * from './SpeakerState'; diff --git a/packages/client/src/rtc/Publisher.ts b/packages/client/src/rtc/Publisher.ts index 0518ad2cd9..70db0dd9f4 100644 --- a/packages/client/src/rtc/Publisher.ts +++ b/packages/client/src/rtc/Publisher.ts @@ -56,6 +56,12 @@ export class Publisher { [TrackType.SCREEN_SHARE_AUDIO]: undefined, [TrackType.UNSPECIFIED]: undefined, }; + + private readonly publishOptionsPerTrackType = new Map< + TrackType, + PublishOptions + >(); + /** * An array maintaining the order how transceivers were added to the peer connection. * This is needed because some browsers (Firefox) don't reliably report @@ -180,10 +186,11 @@ export class Publisher { * * Consecutive calls to this method will replace the stream. * The previous stream will be stopped. - * @param mediaStream - * @param track - * @param trackType - * @param opts + * + * @param mediaStream the media stream to publish. + * @param track the track to publish. + * @param trackType the track type to publish. + * @param opts the optional publish options to use. */ publishStream = async ( mediaStream: MediaStream, @@ -229,6 +236,8 @@ export class Publisher { const videoEncodings = trackType === TrackType.VIDEO ? findOptimalVideoLayers(track, targetResolution) + : trackType === TrackType.SCREEN_SHARE + ? findOptimalScreenSharingLayers(track, opts.screenShareSettings) : undefined; let preferredCodec = opts.preferredCodec; @@ -264,6 +273,7 @@ export class Publisher { logger('debug', `Added ${TrackType[trackType]} transceiver`); this.transceiverInitOrder.push(trackType); this.transceiverRegistry[trackType] = transceiver; + this.publishOptionsPerTrackType.set(trackType, opts); if ('setCodecPreferences' in transceiver && codecPreferences) { logger( @@ -659,11 +669,15 @@ export class Publisher { const track = transceiver.sender.track!; let optimalLayers: OptimalVideoLayer[]; if (track.readyState === 'live') { + const publishOpts = this.publishOptionsPerTrackType.get(trackType); optimalLayers = trackType === TrackType.VIDEO ? findOptimalVideoLayers(track, targetResolution) : trackType === TrackType.SCREEN_SHARE - ? findOptimalScreenSharingLayers(track) + ? findOptimalScreenSharingLayers( + track, + publishOpts?.screenShareSettings, + ) : []; this.trackLayersCache[trackType] = optimalLayers; } else { diff --git a/packages/client/src/rtc/videoLayers.ts b/packages/client/src/rtc/videoLayers.ts index 74a9d4757d..6a992f57cb 100644 --- a/packages/client/src/rtc/videoLayers.ts +++ b/packages/client/src/rtc/videoLayers.ts @@ -1,4 +1,5 @@ import { getOSInfo } from '../client-details'; +import { ScreenShareSettings } from '../types'; import { TargetResolution } from '../gen/coordinator'; import { isReactNative } from '../helpers/platforms'; @@ -122,6 +123,7 @@ const withSimulcastConstraints = ( export const findOptimalScreenSharingLayers = ( videoTrack: MediaStreamTrack, + preferences?: ScreenShareSettings, ): OptimalVideoLayer[] => { const settings = videoTrack.getSettings(); return [ @@ -130,9 +132,9 @@ export const findOptimalScreenSharingLayers = ( rid: 'q', // single track, start from 'q' width: settings.width || 0, height: settings.height || 0, - maxBitrate: 3000000, scaleResolutionDownBy: 1, - maxFramerate: 30, + maxBitrate: preferences?.maxBitrate ?? 3000000, + maxFramerate: preferences?.maxFramerate ?? 30, }, ]; }; diff --git a/packages/client/src/types.ts b/packages/client/src/types.ts index 9b974b1e20..aceb3fa8d4 100644 --- a/packages/client/src/types.ts +++ b/packages/client/src/types.ts @@ -173,6 +173,21 @@ export type SubscriptionChanges = { export type PublishOptions = { preferredCodec?: string | null; + screenShareSettings?: ScreenShareSettings; +}; + +export type ScreenShareSettings = { + /** + * Limits the maximum framerate (in frames per second) of the screen share. + * Defaults to 30. + */ + maxFramerate?: number; + + /** + * Limits the maximum bitrate (in bits per second) of the screen share. + * Defaults to 3000000 (3Mbps). + */ + maxBitrate?: number; }; export type CallLeaveOptions = { diff --git a/sample-apps/client/ts-quickstart/src/controls.ts b/sample-apps/client/ts-quickstart/src/controls.ts index c6ef0cfefd..afb132a049 100644 --- a/sample-apps/client/ts-quickstart/src/controls.ts +++ b/sample-apps/client/ts-quickstart/src/controls.ts @@ -30,6 +30,21 @@ const renderVideoButton = (call: Call) => { return videoButton; }; +const renderScreenShareButton = (call: Call) => { + const screenShareButton = document.createElement('button'); + + screenShareButton.addEventListener('click', async () => { + await call.screenShare.toggle(); + }); + + call.screenShare.state.status$.subscribe((status) => { + screenShareButton.innerText = + status === 'enabled' ? 'Turn off screen share' : 'Turn on screen share'; + }); + + return screenShareButton; +}; + const renderFlipButton = (call: Call) => { const flipButton = document.createElement('button'); @@ -49,6 +64,7 @@ export const renderControls = (call: Call) => { return { audioButton: renderAudioButton(call), videoButton: renderVideoButton(call), + screenShareButton: renderScreenShareButton(call), flipButton: renderFlipButton(call), }; }; diff --git a/sample-apps/client/ts-quickstart/src/main.ts b/sample-apps/client/ts-quickstart/src/main.ts index 4b9ad65eb4..81f5a969ad 100644 --- a/sample-apps/client/ts-quickstart/src/main.ts +++ b/sample-apps/client/ts-quickstart/src/main.ts @@ -38,12 +38,20 @@ const client = new StreamVideoClient({ options: { logLevel: import.meta.env.VITE_STREAM_LOG_LEVEL }, }); const call = client.call('default', callId); + +call.screenShare.enableScreenShareAudio(); +call.screenShare.setSettings({ + maxFramerate: 10, + maxBitrate: 1500000, +}); + call.join({ create: true }).then(async () => { // render mic and camera controls const controls = renderControls(call); const container = document.getElementById('call-controls')!; container.appendChild(controls.audioButton); container.appendChild(controls.videoButton); + container.appendChild(controls.screenShareButton); container.appendChild(renderAudioDeviceSelector(call));