From cf4ed2fd0e18b3c90547ad49a518b90aa0bd96b9 Mon Sep 17 00:00:00 2001 From: Oliver Lazoroski Date: Tue, 3 Dec 2024 12:35:34 +0100 Subject: [PATCH] feat: update to the newest protocol, support publishing multiple tracks of the same kind --- .../client/src/gen/video/sfu/event/events.ts | 76 +++- .../client/src/gen/video/sfu/models/models.ts | 18 + packages/client/src/rtc/Dispatcher.ts | 1 - packages/client/src/rtc/Publisher.ts | 270 +++++++------- packages/client/src/rtc/TransceiverCache.ts | 120 +++++++ .../src/rtc/__tests__/Publisher.test.ts | 178 +++++----- .../src/rtc/__tests__/mocks/webrtc.mocks.ts | 1 + .../src/rtc/__tests__/videoLayers.test.ts | 33 ++ .../src/rtc/helpers/__tests__/hq-audio-sdp.ts | 332 ------------------ .../src/rtc/helpers/__tests__/sdp.test.ts | 9 +- packages/client/src/rtc/helpers/sdp.ts | 47 --- packages/client/src/rtc/videoLayers.ts | 16 + 12 files changed, 467 insertions(+), 634 deletions(-) create mode 100644 packages/client/src/rtc/TransceiverCache.ts delete mode 100644 packages/client/src/rtc/helpers/__tests__/hq-audio-sdp.ts diff --git a/packages/client/src/gen/video/sfu/event/events.ts b/packages/client/src/gen/video/sfu/event/events.ts index d6f7baa7c4..353c82a62d 100644 --- a/packages/client/src/gen/video/sfu/event/events.ts +++ b/packages/client/src/gen/video/sfu/event/events.ts @@ -243,15 +243,6 @@ export interface SfuEvent { */ participantMigrationComplete: ParticipantMigrationComplete; } - | { - oneofKind: 'changePublishOptionsComplete'; - /** - * ChangePublishOptionsComplete is sent to signal the completion of a ChangePublishOptions request. - * - * @generated from protobuf field: stream.video.sfu.event.ChangePublishOptionsComplete change_publish_options_complete = 26; - */ - changePublishOptionsComplete: ChangePublishOptionsComplete; - } | { oneofKind: 'changePublishOptions'; /** @@ -270,9 +261,13 @@ export interface SfuEvent { */ export interface ChangePublishOptions { /** - * @generated from protobuf field: stream.video.sfu.models.PublishOption publish_option = 1; + * @generated from protobuf field: repeated stream.video.sfu.models.PublishOption publish_options = 1; + */ + publishOptions: PublishOption[]; + /** + * @generated from protobuf field: string reason = 2; */ - publishOption?: PublishOption; + reason: string; } /** * @generated from protobuf message stream.video.sfu.event.ChangePublishOptionsComplete @@ -731,6 +726,14 @@ export interface AudioSender { * @generated from protobuf field: stream.video.sfu.models.Codec codec = 2; */ codec?: Codec; + /** + * @generated from protobuf field: stream.video.sfu.models.TrackType track_type = 3; + */ + trackType: TrackType; + /** + * @generated from protobuf field: int32 publish_option_id = 4; + */ + publishOptionId: number; } /** * VideoLayerSetting is used to specify various parameters of a particular encoding in simulcast. @@ -781,6 +784,14 @@ export interface VideoSender { * @generated from protobuf field: repeated stream.video.sfu.event.VideoLayerSetting layers = 3; */ layers: VideoLayerSetting[]; + /** + * @generated from protobuf field: stream.video.sfu.models.TrackType track_type = 4; + */ + trackType: TrackType; + /** + * @generated from protobuf field: int32 publish_option_id = 5; + */ + publishOptionId: number; } /** * sent to users when they need to change the quality of their video @@ -1002,13 +1013,6 @@ class SfuEvent$Type extends MessageType { oneof: 'eventPayload', T: () => ParticipantMigrationComplete, }, - { - no: 26, - name: 'change_publish_options_complete', - kind: 'message', - oneof: 'eventPayload', - T: () => ChangePublishOptionsComplete, - }, { no: 27, name: 'change_publish_options', @@ -1029,10 +1033,12 @@ class ChangePublishOptions$Type extends MessageType { super('stream.video.sfu.event.ChangePublishOptions', [ { no: 1, - name: 'publish_option', + name: 'publish_options', kind: 'message', + repeat: 1 /*RepeatType.PACKED*/, T: () => PublishOption, }, + { no: 2, name: 'reason', kind: 'scalar', T: 9 /*ScalarType.STRING*/ }, ]); } } @@ -1589,6 +1595,22 @@ class AudioSender$Type extends MessageType { constructor() { super('stream.video.sfu.event.AudioSender', [ { no: 2, name: 'codec', kind: 'message', T: () => Codec }, + { + no: 3, + name: 'track_type', + kind: 'enum', + T: () => [ + 'stream.video.sfu.models.TrackType', + TrackType, + 'TRACK_TYPE_', + ], + }, + { + no: 4, + name: 'publish_option_id', + kind: 'scalar', + T: 5 /*ScalarType.INT32*/, + }, ]); } } @@ -1641,6 +1663,22 @@ class VideoSender$Type extends MessageType { repeat: 1 /*RepeatType.PACKED*/, T: () => VideoLayerSetting, }, + { + no: 4, + name: 'track_type', + kind: 'enum', + T: () => [ + 'stream.video.sfu.models.TrackType', + TrackType, + 'TRACK_TYPE_', + ], + }, + { + no: 5, + name: 'publish_option_id', + kind: 'scalar', + T: 5 /*ScalarType.INT32*/, + }, ]); } } diff --git a/packages/client/src/gen/video/sfu/models/models.ts b/packages/client/src/gen/video/sfu/models/models.ts index a17c5b9224..8ddeb5dfdd 100644 --- a/packages/client/src/gen/video/sfu/models/models.ts +++ b/packages/client/src/gen/video/sfu/models/models.ts @@ -253,6 +253,23 @@ export interface PublishOption { * @generated from protobuf field: stream.video.sfu.models.VideoDimension video_dimension = 7; */ videoDimension?: VideoDimension; + /** + * The unique identifier for the publish request. + * - This `id` is assigned exclusively by the SFU. Any `id` set by the client + * in the `PublishOption` will be ignored and overwritten by the SFU. + * - The primary purpose of this `id` is to uniquely identify each publish + * request, even in scenarios where multiple publish requests for the same + * `track_type` and `codec` are active simultaneously. + * For example: + * - A user may publish two tracks of the same type (e.g., video) and codec + * (e.g., VP9) concurrently. + * - This uniqueness ensures that individual requests can be managed + * independently. For instance, an `id` is critical when stopping a specific + * publish request without affecting others. + * + * @generated from protobuf field: int32 id = 8; + */ + id: number; } /** * @generated from protobuf message stream.video.sfu.models.Codec @@ -1159,6 +1176,7 @@ class PublishOption$Type extends MessageType { kind: 'message', T: () => VideoDimension, }, + { no: 8, name: 'id', kind: 'scalar', T: 5 /*ScalarType.INT32*/ }, ]); } } diff --git a/packages/client/src/rtc/Dispatcher.ts b/packages/client/src/rtc/Dispatcher.ts index 8a5265b34e..232fc961d8 100644 --- a/packages/client/src/rtc/Dispatcher.ts +++ b/packages/client/src/rtc/Dispatcher.ts @@ -42,7 +42,6 @@ const sfuEventKinds: { [key in SfuEventKinds]: undefined } = { callEnded: undefined, participantUpdated: undefined, participantMigrationComplete: undefined, - changePublishOptionsComplete: undefined, changePublishOptions: undefined, }; diff --git a/packages/client/src/rtc/Publisher.ts b/packages/client/src/rtc/Publisher.ts index d525dd4d9b..9b9ba46bfc 100644 --- a/packages/client/src/rtc/Publisher.ts +++ b/packages/client/src/rtc/Publisher.ts @@ -2,26 +2,26 @@ import { BasePeerConnection, BasePeerConnectionOpts, } from './BasePeerConnection'; +import { TransceiverCache } from './TransceiverCache'; import { PeerType, PublishOption, TrackInfo, TrackType, - VideoLayer, } from '../gen/video/sfu/models/models'; import { findOptimalVideoLayers, OptimalVideoLayer, - ridToVideoQuality, toSvcEncodings, + toVideoLayers, } from './videoLayers'; import { isSvcCodec } from './codecs'; import { isAudioTrackType, trackTypeToParticipantStreamKey, } from './helpers/tracks'; -import { enableHighQualityAudio, extractMid } from './helpers/sdp'; -import { VideoLayerSetting } from '../gen/video/sfu/event/events'; +import { extractMid } from './helpers/sdp'; +import { VideoSender } from '../gen/video/sfu/event/events'; import { withoutConcurrency } from '../helpers/concurrency'; export type PublisherConstructorOpts = BasePeerConnectionOpts & { @@ -34,20 +34,12 @@ export type PublisherConstructorOpts = BasePeerConnectionOpts & { * @internal */ export class Publisher extends BasePeerConnection { - private readonly transceiverCache = new Map(); - private readonly trackLayersCache = new Map(); - - /** - * An array maintaining the order how transceivers were added to the peer connection. - * This is needed because some browsers (Firefox) don't reliably report - * trackId and `mid` parameters. - */ - private readonly transceiverOrder: RTCRtpTransceiver[] = []; + private readonly transceiverCache = new TransceiverCache(); + private readonly knownTrackIds = new Set(); private readonly unsubscribeOnIceRestart: () => void; private readonly unsubscribeChangePublishQuality: () => void; private readonly unsubscribeChangePublishOptions: () => void; - private unsubscribeCodecNegotiationComplete?: () => void; private publishOptions: PublishOption[]; @@ -75,9 +67,7 @@ export class Publisher extends BasePeerConnection { ({ videoSenders }) => { withoutConcurrency('publisher.changePublishQuality', async () => { for (const videoSender of videoSenders) { - const { layers } = videoSender; - const enabledLayers = layers.filter((l) => l.active); - await this.changePublishQuality(enabledLayers); + await this.changePublishQuality(videoSender); } }).catch((err) => { this.logger('warn', 'Failed to change publish quality', err); @@ -87,15 +77,10 @@ export class Publisher extends BasePeerConnection { this.unsubscribeChangePublishOptions = this.dispatcher.on( 'changePublishOptions', - ({ publishOption }) => { + (event) => { withoutConcurrency('publisher.changePublishOptions', async () => { - if (!publishOption) return; - this.publishOptions = this.publishOptions.map((o) => - o.trackType === publishOption.trackType ? publishOption : o, - ); - if (this.isPublishing(publishOption.trackType)) { - this.switchCodec(publishOption); - } + this.publishOptions = event.publishOptions; + return this.syncPublishOptions(); }).catch((err) => { this.logger('warn', 'Failed to change publish options', err); }); @@ -109,8 +94,6 @@ export class Publisher extends BasePeerConnection { close = ({ stopTracks }: { stopTracks: boolean }) => { if (stopTracks) { this.stopPublishing(); - this.transceiverCache.clear(); - this.trackLayersCache.clear(); } this.dispose(); @@ -125,7 +108,6 @@ export class Publisher extends BasePeerConnection { this.unsubscribeOnIceRestart(); this.unsubscribeChangePublishQuality(); this.unsubscribeChangePublishOptions(); - this.unsubscribeCodecNegotiationComplete?.(); super.detachEventHandlers(); this.pc.removeEventListener('negotiationneeded', this.onNegotiationNeeded); @@ -153,8 +135,7 @@ export class Publisher extends BasePeerConnection { // enable the track if it is disabled if (!track.enabled) track.enabled = true; - const transceiver = this.transceiverCache.get(trackType); - if (!transceiver || !transceiver.sender.track) { + if (!this.knownTrackIds.has(track.id)) { // listen for 'ended' event on the track as it might be ended abruptly // by an external factors such as permission revokes, a disconnected device, etc. // keep in mind that `track.stop()` doesn't trigger this event. @@ -166,10 +147,25 @@ export class Publisher extends BasePeerConnection { ); }; track.addEventListener('ended', handleTrackEnded); - const publishOption = this.getPublishOptionFor(trackType); - this.addTransceiver(trackType, track, publishOption); - } else { - await this.updateTransceiver(transceiver, track); + + // we now publish clones, hence we need to keep track of the original track ids + // to avoid assigning the same event listener multiple times + this.knownTrackIds.add(track.id); + } + + for (const publishOption of this.publishOptions) { + if (publishOption.trackType !== trackType) continue; + + // create a clone of the track as otherwise the same trackId will + // appear in the SDP in multiple transceivers + const trackToPublish = track.clone(); + + const transceiver = this.transceiverCache.get(publishOption); + if (!transceiver || !transceiver.sender.track) { + this.addTransceiver(trackToPublish, publishOption); + } else { + await this.updateTransceiver(transceiver, trackToPublish); + } } await this.notifyTrackMuteStateChanged(mediaStream, trackType, false); @@ -181,7 +177,6 @@ export class Publisher extends BasePeerConnection { * In other cases, use `updateTransceiver` method. */ private addTransceiver = ( - trackType: TrackType, track: MediaStreamTrack, publishOption: PublishOption, ) => { @@ -194,9 +189,9 @@ export class Publisher extends BasePeerConnection { sendEncodings, }); + const trackType = publishOption.trackType; this.logger('debug', `Added ${TrackType[trackType]} transceiver`); - this.transceiverOrder.push(transceiver); - this.transceiverCache.set(trackType, transceiver); + this.transceiverCache.add(publishOption, transceiver); }; /** @@ -218,28 +213,39 @@ export class Publisher extends BasePeerConnection { /** * Switches the codec of the given track type. */ - private switchCodec = (publishOption: PublishOption) => { - const trackType = publishOption.trackType; - const transceiver = this.transceiverCache.get(trackType); - if (!transceiver || !transceiver.sender.track) return; - - const onChangePublishOptionsComplete = async () => { - this.logger('info', 'Codec negotiation complete'); - this.dispatcher.off( - 'changePublishOptionsComplete', - onChangePublishOptionsComplete, + private syncPublishOptions = async () => { + // enable publishing with new options -> [av1, vp9] + for (const publishOption of this.publishOptions) { + const { trackType } = publishOption; + if (!this.isPublishing(trackType)) continue; + if (this.transceiverCache.has(publishOption)) continue; + + const item = this.transceiverCache.find( + (i) => + !!i.transceiver.sender.track && + i.publishOption.trackType === trackType, ); + if (!item || !item.transceiver) continue; - await transceiver.sender.replaceTrack(null); - }; - this.unsubscribeCodecNegotiationComplete?.(); - this.unsubscribeCodecNegotiationComplete = this.dispatcher.on( - 'changePublishOptionsComplete', - onChangePublishOptionsComplete, - ); + // take the track from the existing transceiver for the same track type, + // clone it and publish it with the new publish options + const track = item.transceiver.sender.track!.clone(); + this.addTransceiver(track, publishOption); + } - const track = transceiver.sender.track.clone(); - this.addTransceiver(trackType, track, publishOption); + // stop publishing with options not required anymore -> [vp9] + for (const item of this.transceiverCache.items()) { + const { publishOption, transceiver } = item; + const hasPublishOption = this.publishOptions.some( + (option) => + option.id === publishOption.id && + option.trackType === publishOption.trackType, + ); + if (hasPublishOption) continue; + // it is safe to stop the track here, it is a clone + transceiver.sender.track?.stop(); + await transceiver.sender.replaceTrack(null); + } }; /** @@ -249,13 +255,18 @@ export class Publisher extends BasePeerConnection { * @param stopTrack specifies whether track should be stopped or just disabled */ unpublishStream = async (trackType: TrackType, stopTrack: boolean) => { - const transceiver = this.transceiverCache.get(trackType); - if (!transceiver || !transceiver.sender.track) return; + for (const option of this.publishOptions) { + if (option.trackType !== trackType) continue; - if (stopTrack && transceiver.sender.track.readyState === 'live') { - transceiver.sender.track.stop(); - } else if (transceiver.sender.track.enabled) { - transceiver.sender.track.enabled = false; + const transceiver = this.transceiverCache.get(option); + const track = transceiver?.sender.track; + if (!track) continue; + + if (stopTrack && track.readyState === 'live') { + track.stop(); + } else if (track.enabled) { + track.enabled = false; + } } if (this.state.localParticipant?.publishedTracks.includes(trackType)) { @@ -269,19 +280,25 @@ export class Publisher extends BasePeerConnection { * @param trackType the track type to check. */ isPublishing = (trackType: TrackType): boolean => { - const transceiver = this.transceiverCache.get(trackType); - if (!transceiver || !transceiver.sender.track) return false; - const track = transceiver.sender.track; - return track.readyState === 'live' && track.enabled; + for (const item of this.transceiverCache.items()) { + if (item.publishOption.trackType !== trackType) continue; + + const track = item.transceiver?.sender.track; + if (!track) continue; + + if (track.readyState === 'live' && track.enabled) return true; + } + return false; }; /** * Maps the given track ID to the corresponding track type. */ getTrackType = (trackId: string): TrackType | undefined => { - for (const [trackType, transceiver] of this.transceiverCache) { + for (const transceiverId of this.transceiverCache.items()) { + const { publishOption, transceiver } = transceiverId; if (transceiver.sender.track?.id === trackId) { - return trackType; + return publishOption.trackType; } } return undefined; @@ -328,21 +345,25 @@ export class Publisher extends BasePeerConnection { }); }; - private changePublishQuality = async (enabledLayers: VideoLayerSetting[]) => { + private changePublishQuality = async (videoSender: VideoSender) => { + const { trackType, layers, publishOptionId } = videoSender; + const enabledLayers = layers.filter((l) => l.active); this.logger( 'info', 'Update publish quality, requested layers by SFU:', enabledLayers, ); - const trackType = TrackType.VIDEO; - const videoSender = this.transceiverCache.get(trackType)?.sender; - if (!videoSender) { + const sender = this.transceiverCache.getWith( + trackType, + publishOptionId, + )?.sender; + if (!sender) { this.logger('warn', 'Update publish quality, no video sender found.'); return; } - const params = videoSender.getParameters(); + const params = sender.getParameters(); if (params.encodings.length === 0) { this.logger( 'warn', @@ -408,7 +429,7 @@ export class Publisher extends BasePeerConnection { return; } - await videoSender.setParameters(params); + await sender.setParameters(params); this.logger('info', `Update publish quality, enabled rids:`, activeLayers); }; @@ -441,10 +462,6 @@ export class Publisher extends BasePeerConnection { */ private negotiate = async (options?: RTCOfferOptions) => { const offer = await this.pc.createOffer(options); - if (offer.sdp && this.isPublishing(TrackType.SCREEN_SHARE_AUDIO)) { - offer.sdp = this.enableHighQualityAudio(offer.sdp); - } - const trackInfos = this.getAnnouncedTracks(offer.sdp); if (trackInfos.length === 0) { throw new Error(`Can't negotiate without announcing any tracks`); @@ -477,15 +494,6 @@ export class Publisher extends BasePeerConnection { ); }; - private enableHighQualityAudio = (sdp: string) => { - const transceiver = this.transceiverCache.get(TrackType.SCREEN_SHARE_AUDIO); - if (!transceiver) return sdp; - - const transceiverInitIndex = this.transceiverOrder.indexOf(transceiver); - const mid = extractMid(transceiver, transceiverInitIndex, sdp); - return enableHighQualityAudio(sdp, mid); - }; - /** * Returns a list of tracks that are currently being published. * @@ -494,66 +502,36 @@ export class Publisher extends BasePeerConnection { */ getAnnouncedTracks = (sdp?: string): TrackInfo[] => { sdp = sdp || this.pc.localDescription?.sdp; - return this.pc - .getTransceivers() - .filter((t) => t.direction === 'sendonly' && t.sender.track) - .map((transceiver) => { - let trackType!: TrackType; - this.transceiverCache.forEach((value, key) => { - if (value === transceiver) trackType = key; - }); - - if (!trackType) return undefined; - const track = transceiver.sender.track!; - - const publishOption = this.getPublishOptionFor(trackType); - const isTrackLive = track.readyState === 'live'; - const optimalLayers = isTrackLive - ? this.computeLayers(track, publishOption) || [] - : this.trackLayersCache.get(trackType) || []; - this.trackLayersCache.set(trackType, optimalLayers); - - const layers = optimalLayers.map((optimalLayer) => ({ - rid: optimalLayer.rid || '', - bitrate: optimalLayer.maxBitrate || 0, - fps: optimalLayer.maxFramerate || 0, - quality: ridToVideoQuality(optimalLayer.rid || ''), - videoDimension: { - width: optimalLayer.width, - height: optimalLayer.height, - }, - })); - - const isAudioTrack = isAudioTrackType(trackType); - const trackSettings = track.getSettings(); - const isStereo = isAudioTrack && trackSettings.channelCount === 2; - const transceiverIndex = this.transceiverOrder.indexOf(transceiver); - const mid = extractMid(transceiver, transceiverIndex, sdp); - - const audioSettings = this.state.settings?.audio; - return { - trackId: track.id, - layers, - trackType, - mid, - stereo: isStereo, - dtx: isAudioTrack && !!audioSettings?.opus_dtx_enabled, - red: isAudioTrack && !!audioSettings?.redundant_coding_enabled, - muted: !isTrackLive, - }; - }) - .filter(Boolean) as TrackInfo[]; - }; - - /** - * Returns the publish options for the given track type. - */ - private getPublishOptionFor = (trackType: TrackType): PublishOption => { - const option = this.publishOptions.find((o) => o.trackType === trackType); - if (!option) { - throw new Error(`No publish options found for ${TrackType[trackType]}`); + const trackInfos: TrackInfo[] = []; + for (const transceiverId of this.transceiverCache.items()) { + const { publishOption, transceiver } = transceiverId; + const track = transceiver.sender.track; + if (!track) continue; + + const isTrackLive = track.readyState === 'live'; + const layers = isTrackLive + ? this.computeLayers(track, publishOption) + : this.transceiverCache.getLayers(publishOption); + this.transceiverCache.setLayers(publishOption, layers); + + const isAudioTrack = isAudioTrackType(publishOption.trackType); + const isStereo = isAudioTrack && track.getSettings().channelCount === 2; + const transceiverIndex = this.transceiverCache.indexOf(transceiver); + const mid = extractMid(transceiver, transceiverIndex, sdp); + + const audioSettings = this.state.settings?.audio; + trackInfos.push({ + trackId: track.id, + layers: toVideoLayers(layers), + trackType: publishOption.trackType, + mid, + stereo: isStereo, + dtx: isAudioTrack && !!audioSettings?.opus_dtx_enabled, + red: isAudioTrack && !!audioSettings?.redundant_coding_enabled, + muted: !isTrackLive, + }); } - return option; + return trackInfos; }; private computeLayers = ( diff --git a/packages/client/src/rtc/TransceiverCache.ts b/packages/client/src/rtc/TransceiverCache.ts new file mode 100644 index 0000000000..72e500af74 --- /dev/null +++ b/packages/client/src/rtc/TransceiverCache.ts @@ -0,0 +1,120 @@ +import { PublishOption, TrackType } from '../gen/video/sfu/models/models'; +import { OptimalVideoLayer } from './videoLayers'; + +type TransceiverId = { + publishOption: PublishOption; + transceiver: RTCRtpTransceiver; +}; +type TrackLayersCache = { + publishOption: PublishOption; + layers: OptimalVideoLayer[]; +}; + +export class TransceiverCache { + private readonly cache: TransceiverId[] = []; + private readonly layers: TrackLayersCache[] = []; + + /** + * An array maintaining the order how transceivers were added to the peer connection. + * This is needed because some browsers (Firefox) don't reliably report + * trackId and `mid` parameters. + */ + private readonly transceiverOrder: RTCRtpTransceiver[] = []; + + /** + * Adds a transceiver to the cache. + */ + add = (publishOption: PublishOption, transceiver: RTCRtpTransceiver) => { + this.cache.push({ publishOption, transceiver }); + this.transceiverOrder.push(transceiver); + }; + + /** + * Gets the transceiver for the given publish option. + */ + get = (publishOption: PublishOption): RTCRtpTransceiver | undefined => { + return this.findTransceiver(publishOption)?.transceiver; + }; + + /** + * Gets the last transceiver for the given track type and publish option id. + */ + getWith = (trackType: TrackType, id: number) => { + return this.findTransceiver({ trackType, id })?.transceiver; + }; + + /** + * Checks if the cache has the given publish option. + */ + has = (publishOption: PublishOption): boolean => { + return !!this.get(publishOption); + }; + + /** + * Finds the first transceiver that satisfies the given predicate. + */ + find = ( + predicate: (item: TransceiverId) => boolean, + ): TransceiverId | undefined => { + return this.cache.find(predicate); + }; + + /** + * Provides all the items in the cache. + */ + items = (): TransceiverId[] => { + return this.cache; + }; + + /** + * Init index of the transceiver in the cache. + */ + indexOf = (transceiver: RTCRtpTransceiver): number => { + return this.transceiverOrder.indexOf(transceiver); + }; + + /** + * Gets cached video layers for the given track. + */ + getLayers = ( + publishOption: PublishOption, + ): OptimalVideoLayer[] | undefined => { + const entry = this.layers.find( + (item) => + item.publishOption.id === publishOption.id && + item.publishOption.trackType === publishOption.trackType, + ); + return entry?.layers; + }; + + /** + * Sets the video layers for the given track. + */ + setLayers = ( + publishOption: PublishOption, + layers: OptimalVideoLayer[] = [], + ) => { + const entry = this.findLayer(publishOption); + if (entry) { + entry.layers = layers; + } else { + this.layers.push({ publishOption, layers }); + } + }; + + private findTransceiver = (publishOption: Partial) => { + return this.cache.find( + (item) => + item.publishOption.id === publishOption.id && + item.publishOption.trackType === publishOption.trackType, + ); + }; + + private findLayer = (publishOption: PublishOption) => { + return this.layers.find( + (item) => + item.publishOption.id === publishOption.id && + item.publishOption.trackType === publishOption.trackType, + ); + }; +} diff --git a/packages/client/src/rtc/__tests__/Publisher.test.ts b/packages/client/src/rtc/__tests__/Publisher.test.ts index 6b7bc7ec18..93389062ee 100644 --- a/packages/client/src/rtc/__tests__/Publisher.test.ts +++ b/packages/client/src/rtc/__tests__/Publisher.test.ts @@ -17,22 +17,6 @@ vi.mock('../../StreamSfuClient', () => { }; }); -vi.mock('../codecs', async () => { - const codecs = await vi.importActual('../codecs'); - return { - getPreferredCodecs: vi.fn((): RTCRtpCodecCapability[] => [ - { - channels: 1, - clockRate: 48000, - mimeType: 'video/h264', - sdpFmtpLine: 'profile-level-id=42e01f', - }, - ]), - getOptimalVideoCodec: codecs.getOptimalVideoCodec, - isSvcCodec: codecs.isSvcCodec, - }; -}); - describe('Publisher', () => { const sessionId = 'session-id-test'; let publisher: Publisher; @@ -72,13 +56,14 @@ describe('Publisher', () => { logTag: 'test', publishOptions: [ { + id: 1, trackType: TrackType.VIDEO, bitrate: 1000, // @ts-expect-error - incomplete data codec: { name: 'vp9' }, fps: 30, maxTemporalLayers: 3, - maxSpatialLayers: 0, + maxSpatialLayers: 3, }, ], }); @@ -110,6 +95,7 @@ describe('Publisher', () => { height: 480, deviceId: 'test-device-id', }); + vi.spyOn(track, 'clone').mockReturnValue(track); const transceiver = new RTCRtpTransceiver(); vi.spyOn(transceiver.sender, 'track', 'get').mockReturnValue(track); @@ -142,6 +128,7 @@ describe('Publisher', () => { height: 720, deviceId: 'test-device-id-2', }); + vi.spyOn(newTrack, 'clone').mockReturnValue(newTrack); await publisher.publishStream(newMediaStream, newTrack, TrackType.VIDEO); vi.spyOn(transceiver.sender, 'track', 'get').mockReturnValue(newTrack); @@ -181,6 +168,7 @@ describe('Publisher', () => { height: 480, deviceId: 'test-device-id', }); + vi.spyOn(track, 'clone').mockReturnValue(track); const transceiver = new RTCRtpTransceiver(); vi.spyOn(transceiver.sender, 'track', 'get').mockReturnValue(track); @@ -311,34 +299,42 @@ describe('Publisher', () => { }); // inject the transceiver - publisher['transceiverCache'].set(TrackType.VIDEO, transceiver); + publisher['transceiverCache'].add( + // @ts-expect-error incomplete data + { trackType: TrackType.VIDEO, id: 1 }, + transceiver, + ); - await publisher['changePublishQuality']([ - { - name: 'q', - active: true, - maxBitrate: 100, - scaleResolutionDownBy: 4, - maxFramerate: 30, - scalabilityMode: '', - }, - { - name: 'h', - active: false, - maxBitrate: 150, - scaleResolutionDownBy: 2, - maxFramerate: 30, - scalabilityMode: '', - }, - { - name: 'f', - active: true, - maxBitrate: 200, - scaleResolutionDownBy: 1, - maxFramerate: 30, - scalabilityMode: '', - }, - ]); + await publisher['changePublishQuality']({ + publishOptionId: 1, + trackType: TrackType.VIDEO, + layers: [ + { + name: 'q', + active: true, + maxBitrate: 100, + scaleResolutionDownBy: 4, + maxFramerate: 30, + scalabilityMode: '', + }, + { + name: 'h', + active: false, + maxBitrate: 150, + scaleResolutionDownBy: 2, + maxFramerate: 30, + scalabilityMode: '', + }, + { + name: 'f', + active: true, + maxBitrate: 200, + scaleResolutionDownBy: 1, + maxFramerate: 30, + scalabilityMode: '', + }, + ], + }); expect(getParametersSpy).toHaveBeenCalled(); expect(setParametersSpy).toHaveBeenCalled(); @@ -353,9 +349,6 @@ describe('Publisher', () => { { rid: 'h', active: false, - maxBitrate: 150, - scaleResolutionDownBy: 2, - maxFramerate: 30, }, { rid: 'f', @@ -381,18 +374,26 @@ describe('Publisher', () => { }); // inject the transceiver - publisher['transceiverCache'].set(TrackType.VIDEO, transceiver); + publisher['transceiverCache'].add( + // @ts-expect-error incomplete data + { trackType: TrackType.VIDEO, id: 1 }, + transceiver, + ); - await publisher['changePublishQuality']([ - { - name: 'q', - active: true, - maxBitrate: 100, - scaleResolutionDownBy: 4, - maxFramerate: 30, - scalabilityMode: '', - }, - ]); + await publisher['changePublishQuality']({ + publishOptionId: 1, + trackType: TrackType.VIDEO, + layers: [ + { + name: 'q', + active: true, + maxBitrate: 100, + scaleResolutionDownBy: 4, + maxFramerate: 30, + scalabilityMode: '', + }, + ], + }); expect(getParametersSpy).toHaveBeenCalled(); expect(setParametersSpy).toHaveBeenCalled(); @@ -436,18 +437,25 @@ describe('Publisher', () => { }); // inject the transceiver - publisher['transceiverCache'].set(TrackType.VIDEO, transceiver); - - await publisher['changePublishQuality']([ - { - name: 'q', - active: true, - maxBitrate: 50, - scaleResolutionDownBy: 1, - maxFramerate: 30, - scalabilityMode: 'L1T3', - }, - ]); + publisher['transceiverCache'].add( + // @ts-expect-error incomplete data + { trackType: TrackType.VIDEO, id: 1 }, + transceiver, + ); + await publisher['changePublishQuality']({ + publishOptionId: 1, + trackType: TrackType.VIDEO, + layers: [ + { + name: 'q', + active: true, + maxBitrate: 50, + scaleResolutionDownBy: 1, + maxFramerate: 30, + scalabilityMode: 'L1T3', + }, + ], + }); expect(getParametersSpy).toHaveBeenCalled(); expect(setParametersSpy).toHaveBeenCalled(); @@ -486,18 +494,26 @@ describe('Publisher', () => { }); // inject the transceiver - publisher['transceiverCache'].set(TrackType.VIDEO, transceiver); + publisher['transceiverCache'].add( + // @ts-expect-error incomplete data + { trackType: TrackType.VIDEO, id: 1 }, + transceiver, + ); - await publisher['changePublishQuality']([ - { - name: 'q', - active: true, - maxBitrate: 50, - scaleResolutionDownBy: 1, - maxFramerate: 30, - scalabilityMode: 'L1T3', - }, - ]); + await publisher['changePublishQuality']({ + publishOptionId: 1, + trackType: TrackType.VIDEO, + layers: [ + { + name: 'q', + active: true, + maxBitrate: 50, + scaleResolutionDownBy: 1, + maxFramerate: 30, + scalabilityMode: 'L1T3', + }, + ], + }); expect(getParametersSpy).toHaveBeenCalled(); expect(setParametersSpy).toHaveBeenCalled(); diff --git a/packages/client/src/rtc/__tests__/mocks/webrtc.mocks.ts b/packages/client/src/rtc/__tests__/mocks/webrtc.mocks.ts index 9af17ff0da..1d9442b540 100644 --- a/packages/client/src/rtc/__tests__/mocks/webrtc.mocks.ts +++ b/packages/client/src/rtc/__tests__/mocks/webrtc.mocks.ts @@ -33,6 +33,7 @@ const MediaStreamTrackMock = vi.fn((): Partial => { removeEventListener: vi.fn(), getSettings: vi.fn(), stop: vi.fn(), + clone: vi.fn(), readyState: 'live', kind: 'video', }; diff --git a/packages/client/src/rtc/__tests__/videoLayers.test.ts b/packages/client/src/rtc/__tests__/videoLayers.test.ts index ac962e4dbc..740989eba6 100644 --- a/packages/client/src/rtc/__tests__/videoLayers.test.ts +++ b/packages/client/src/rtc/__tests__/videoLayers.test.ts @@ -7,6 +7,7 @@ import { OptimalVideoLayer, ridToVideoQuality, toSvcEncodings, + toVideoLayers, } from '../videoLayers'; describe('videoLayers', () => { @@ -160,6 +161,38 @@ describe('videoLayers', () => { expect(ridToVideoQuality('')).toBe(VideoQuality.HIGH); }); + it('should map optimal video layers to SFU VideoLayers', () => { + const layers: Array> = [ + { rid: 'f', width: 1920, height: 1080, maxBitrate: 3000000 }, + { rid: 'h', width: 960, height: 540, maxBitrate: 750000 }, + { rid: 'q', width: 480, height: 270, maxBitrate: 187500 }, + ]; + + const videoLayers = toVideoLayers(layers as OptimalVideoLayer[]); + expect(videoLayers.length).toBe(3); + expect(videoLayers[0]).toEqual({ + rid: 'f', + bitrate: 3000000, + fps: 0, + quality: VideoQuality.HIGH, + videoDimension: { width: 1920, height: 1080 }, + }); + expect(videoLayers[1]).toEqual({ + rid: 'h', + bitrate: 750000, + fps: 0, + quality: VideoQuality.MID, + videoDimension: { width: 960, height: 540 }, + }); + expect(videoLayers[2]).toEqual({ + rid: 'q', + bitrate: 187500, + fps: 0, + quality: VideoQuality.LOW_UNSPECIFIED, + videoDimension: { width: 480, height: 270 }, + }); + }); + it('should map OptimalVideoLayer to SVC encodings', () => { const layers: Array> = [ { rid: 'f', width: 1920, height: 1080, maxBitrate: 3000000 }, diff --git a/packages/client/src/rtc/helpers/__tests__/hq-audio-sdp.ts b/packages/client/src/rtc/helpers/__tests__/hq-audio-sdp.ts deleted file mode 100644 index c4bad821df..0000000000 --- a/packages/client/src/rtc/helpers/__tests__/hq-audio-sdp.ts +++ /dev/null @@ -1,332 +0,0 @@ -export const initialSdp = ` -v=0 -o=- 898697271686242868 5 IN IP4 127.0.0.1 -s=- -t=0 0 -a=group:BUNDLE 0 1 2 3 -a=extmap-allow-mixed -a=msid-semantic: WMS e893e3ad-d9e8-4b56-998f-0d89213dd857 -m=video 60017 UDP/TLS/RTP/SAVPF 96 97 102 103 104 105 106 107 108 109 127 125 39 40 45 46 98 99 100 101 112 113 116 117 118 -c=IN IP4 79.125.240.146 -a=rtcp:9 IN IP4 0.0.0.0 -a=candidate:2824354870 1 udp 2122260223 192.168.1.102 60017 typ host generation 0 network-id 2 -a=candidate:427386432 1 udp 2122194687 192.168.1.244 63221 typ host generation 0 network-id 1 network-cost 10 -a=candidate:2841136656 1 udp 1686052607 79.125.240.146 60017 typ srflx raddr 192.168.1.102 rport 60017 generation 0 network-id 2 -a=candidate:410588262 1 udp 1685987071 79.125.240.146 63221 typ srflx raddr 192.168.1.244 rport 63221 generation 0 network-id 1 network-cost 10 -a=candidate:3600277166 1 tcp 1518280447 192.168.1.102 9 typ host tcptype active generation 0 network-id 2 -a=candidate:1740014808 1 tcp 1518214911 192.168.1.244 9 typ host tcptype active generation 0 network-id 1 network-cost 10 -a=ice-ufrag:GM64 -a=ice-pwd:ANZFilRRlZJ3bg9AD40eRu7n -a=ice-options:trickle -a=fingerprint:sha-256 38:1F:02:E5:2A:49:9A:2A:D9:8E:B9:9B:4C:40:21:B7:F1:C4:27:8E:B5:68:D6:E0:91:08:D9:CB:2B:AC:B3:87 -a=setup:actpass -a=mid:0 -a=extmap:1 urn:ietf:params:rtp-hdrext:toffset -a=extmap:2 http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time -a=extmap:3 urn:3gpp:video-orientation -a=extmap:4 http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01 -a=extmap:5 http://www.webrtc.org/experiments/rtp-hdrext/playout-delay -a=extmap:6 http://www.webrtc.org/experiments/rtp-hdrext/video-content-type -a=extmap:7 http://www.webrtc.org/experiments/rtp-hdrext/video-timing -a=extmap:8 http://www.webrtc.org/experiments/rtp-hdrext/color-space -a=extmap:9 urn:ietf:params:rtp-hdrext:sdes:mid -a=extmap:10 urn:ietf:params:rtp-hdrext:sdes:rtp-stream-id -a=extmap:11 urn:ietf:params:rtp-hdrext:sdes:repaired-rtp-stream-id -a=sendonly -a=msid:e893e3ad-d9e8-4b56-998f-0d89213dd857 44e5d53f-ce6a-4bf4-824a-335113d86e6e -a=rtcp-mux -a=rtcp-rsize -a=rtpmap:96 VP8/90000 -a=rtcp-fb:96 goog-remb -a=rtcp-fb:96 transport-cc -a=rtcp-fb:96 ccm fir -a=rtcp-fb:96 nack -a=rtcp-fb:96 nack pli -a=rtpmap:97 rtx/90000 -a=fmtp:97 apt=96 -a=rtpmap:102 H264/90000 -a=rtcp-fb:102 goog-remb -a=rtcp-fb:102 transport-cc -a=rtcp-fb:102 ccm fir -a=rtcp-fb:102 nack -a=rtcp-fb:102 nack pli -a=fmtp:102 level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42001f -a=rtpmap:103 rtx/90000 -a=fmtp:103 apt=102 -a=rtpmap:104 H264/90000 -a=rtcp-fb:104 goog-remb -a=rtcp-fb:104 transport-cc -a=rtcp-fb:104 ccm fir -a=rtcp-fb:104 nack -a=rtcp-fb:104 nack pli -a=fmtp:104 level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=42001f -a=rtpmap:105 rtx/90000 -a=fmtp:105 apt=104 -a=rtpmap:106 H264/90000 -a=rtcp-fb:106 goog-remb -a=rtcp-fb:106 transport-cc -a=rtcp-fb:106 ccm fir -a=rtcp-fb:106 nack -a=rtcp-fb:106 nack pli -a=fmtp:106 level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f -a=rtpmap:107 rtx/90000 -a=fmtp:107 apt=106 -a=rtpmap:108 H264/90000 -a=rtcp-fb:108 goog-remb -a=rtcp-fb:108 transport-cc -a=rtcp-fb:108 ccm fir -a=rtcp-fb:108 nack -a=rtcp-fb:108 nack pli -a=fmtp:108 level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=42e01f -a=rtpmap:109 rtx/90000 -a=fmtp:109 apt=108 -a=rtpmap:127 H264/90000 -a=rtcp-fb:127 goog-remb -a=rtcp-fb:127 transport-cc -a=rtcp-fb:127 ccm fir -a=rtcp-fb:127 nack -a=rtcp-fb:127 nack pli -a=fmtp:127 level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=4d001f -a=rtpmap:125 rtx/90000 -a=fmtp:125 apt=127 -a=rtpmap:39 H264/90000 -a=rtcp-fb:39 goog-remb -a=rtcp-fb:39 transport-cc -a=rtcp-fb:39 ccm fir -a=rtcp-fb:39 nack -a=rtcp-fb:39 nack pli -a=fmtp:39 level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=4d001f -a=rtpmap:40 rtx/90000 -a=fmtp:40 apt=39 -a=rtpmap:45 AV1/90000 -a=rtcp-fb:45 goog-remb -a=rtcp-fb:45 transport-cc -a=rtcp-fb:45 ccm fir -a=rtcp-fb:45 nack -a=rtcp-fb:45 nack pli -a=rtpmap:46 rtx/90000 -a=fmtp:46 apt=45 -a=rtpmap:98 VP9/90000 -a=rtcp-fb:98 goog-remb -a=rtcp-fb:98 transport-cc -a=rtcp-fb:98 ccm fir -a=rtcp-fb:98 nack -a=rtcp-fb:98 nack pli -a=fmtp:98 profile-id=0 -a=rtpmap:99 rtx/90000 -a=fmtp:99 apt=98 -a=rtpmap:100 VP9/90000 -a=rtcp-fb:100 goog-remb -a=rtcp-fb:100 transport-cc -a=rtcp-fb:100 ccm fir -a=rtcp-fb:100 nack -a=rtcp-fb:100 nack pli -a=fmtp:100 profile-id=2 -a=rtpmap:101 rtx/90000 -a=fmtp:101 apt=100 -a=rtpmap:112 H264/90000 -a=rtcp-fb:112 goog-remb -a=rtcp-fb:112 transport-cc -a=rtcp-fb:112 ccm fir -a=rtcp-fb:112 nack -a=rtcp-fb:112 nack pli -a=fmtp:112 level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=64001f -a=rtpmap:113 rtx/90000 -a=fmtp:113 apt=112 -a=rtpmap:116 red/90000 -a=rtpmap:117 rtx/90000 -a=fmtp:117 apt=116 -a=rtpmap:118 ulpfec/90000 -a=rid:q send -a=rid:h send -a=rid:f send -a=simulcast:send q;h;f -m=audio 9 UDP/TLS/RTP/SAVPF 63 111 9 0 8 13 110 126 -c=IN IP4 0.0.0.0 -a=rtcp:9 IN IP4 0.0.0.0 -a=ice-ufrag:GM64 -a=ice-pwd:ANZFilRRlZJ3bg9AD40eRu7n -a=ice-options:trickle -a=fingerprint:sha-256 38:1F:02:E5:2A:49:9A:2A:D9:8E:B9:9B:4C:40:21:B7:F1:C4:27:8E:B5:68:D6:E0:91:08:D9:CB:2B:AC:B3:87 -a=setup:actpass -a=mid:1 -a=extmap:14 urn:ietf:params:rtp-hdrext:ssrc-audio-level -a=extmap:2 http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time -a=extmap:4 http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01 -a=extmap:9 urn:ietf:params:rtp-hdrext:sdes:mid -a=sendonly -a=msid:- cb02416c-8035-4e80-8b7e-becd6b7f600f -a=rtcp-mux -a=rtpmap:63 red/48000/2 -a=fmtp:63 111/111 -a=rtpmap:111 opus/48000/2 -a=rtcp-fb:111 transport-cc -a=fmtp:111 minptime=10;usedtx=1;useinbandfec=1 -a=rtpmap:9 G722/8000 -a=rtpmap:0 PCMU/8000 -a=rtpmap:8 PCMA/8000 -a=rtpmap:13 CN/8000 -a=rtpmap:110 telephone-event/48000 -a=rtpmap:126 telephone-event/8000 -a=ssrc:743121750 cname:mbGW+aeWMLFhPbBC -a=ssrc:743121750 msid:- cb02416c-8035-4e80-8b7e-becd6b7f600f -m=video 9 UDP/TLS/RTP/SAVPF 96 97 102 103 104 105 106 107 108 109 127 125 39 40 45 46 98 99 100 101 112 113 116 117 118 -c=IN IP4 0.0.0.0 -a=rtcp:9 IN IP4 0.0.0.0 -a=ice-ufrag:GM64 -a=ice-pwd:ANZFilRRlZJ3bg9AD40eRu7n -a=ice-options:trickle -a=fingerprint:sha-256 38:1F:02:E5:2A:49:9A:2A:D9:8E:B9:9B:4C:40:21:B7:F1:C4:27:8E:B5:68:D6:E0:91:08:D9:CB:2B:AC:B3:87 -a=setup:actpass -a=mid:2 -a=extmap:1 urn:ietf:params:rtp-hdrext:toffset -a=extmap:2 http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time -a=extmap:3 urn:3gpp:video-orientation -a=extmap:4 http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01 -a=extmap:5 http://www.webrtc.org/experiments/rtp-hdrext/playout-delay -a=extmap:6 http://www.webrtc.org/experiments/rtp-hdrext/video-content-type -a=extmap:7 http://www.webrtc.org/experiments/rtp-hdrext/video-timing -a=extmap:8 http://www.webrtc.org/experiments/rtp-hdrext/color-space -a=extmap:9 urn:ietf:params:rtp-hdrext:sdes:mid -a=extmap:10 urn:ietf:params:rtp-hdrext:sdes:rtp-stream-id -a=extmap:11 urn:ietf:params:rtp-hdrext:sdes:repaired-rtp-stream-id -a=sendonly -a=msid:04f96a2a-d107-4a12-b545-cfe1dc2c4d37 4a1ae8a7-6c89-44d3-a9a1-02abce0012c7 -a=rtcp-mux -a=rtcp-rsize -a=rtpmap:96 VP8/90000 -a=rtcp-fb:96 goog-remb -a=rtcp-fb:96 transport-cc -a=rtcp-fb:96 ccm fir -a=rtcp-fb:96 nack -a=rtcp-fb:96 nack pli -a=rtpmap:97 rtx/90000 -a=fmtp:97 apt=96 -a=rtpmap:102 H264/90000 -a=rtcp-fb:102 goog-remb -a=rtcp-fb:102 transport-cc -a=rtcp-fb:102 ccm fir -a=rtcp-fb:102 nack -a=rtcp-fb:102 nack pli -a=fmtp:102 level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42001f -a=rtpmap:103 rtx/90000 -a=fmtp:103 apt=102 -a=rtpmap:104 H264/90000 -a=rtcp-fb:104 goog-remb -a=rtcp-fb:104 transport-cc -a=rtcp-fb:104 ccm fir -a=rtcp-fb:104 nack -a=rtcp-fb:104 nack pli -a=fmtp:104 level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=42001f -a=rtpmap:105 rtx/90000 -a=fmtp:105 apt=104 -a=rtpmap:106 H264/90000 -a=rtcp-fb:106 goog-remb -a=rtcp-fb:106 transport-cc -a=rtcp-fb:106 ccm fir -a=rtcp-fb:106 nack -a=rtcp-fb:106 nack pli -a=fmtp:106 level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f -a=rtpmap:107 rtx/90000 -a=fmtp:107 apt=106 -a=rtpmap:108 H264/90000 -a=rtcp-fb:108 goog-remb -a=rtcp-fb:108 transport-cc -a=rtcp-fb:108 ccm fir -a=rtcp-fb:108 nack -a=rtcp-fb:108 nack pli -a=fmtp:108 level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=42e01f -a=rtpmap:109 rtx/90000 -a=fmtp:109 apt=108 -a=rtpmap:127 H264/90000 -a=rtcp-fb:127 goog-remb -a=rtcp-fb:127 transport-cc -a=rtcp-fb:127 ccm fir -a=rtcp-fb:127 nack -a=rtcp-fb:127 nack pli -a=fmtp:127 level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=4d001f -a=rtpmap:125 rtx/90000 -a=fmtp:125 apt=127 -a=rtpmap:39 H264/90000 -a=rtcp-fb:39 goog-remb -a=rtcp-fb:39 transport-cc -a=rtcp-fb:39 ccm fir -a=rtcp-fb:39 nack -a=rtcp-fb:39 nack pli -a=fmtp:39 level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=4d001f -a=rtpmap:40 rtx/90000 -a=fmtp:40 apt=39 -a=rtpmap:45 AV1/90000 -a=rtcp-fb:45 goog-remb -a=rtcp-fb:45 transport-cc -a=rtcp-fb:45 ccm fir -a=rtcp-fb:45 nack -a=rtcp-fb:45 nack pli -a=rtpmap:46 rtx/90000 -a=fmtp:46 apt=45 -a=rtpmap:98 VP9/90000 -a=rtcp-fb:98 goog-remb -a=rtcp-fb:98 transport-cc -a=rtcp-fb:98 ccm fir -a=rtcp-fb:98 nack -a=rtcp-fb:98 nack pli -a=fmtp:98 profile-id=0 -a=rtpmap:99 rtx/90000 -a=fmtp:99 apt=98 -a=rtpmap:100 VP9/90000 -a=rtcp-fb:100 goog-remb -a=rtcp-fb:100 transport-cc -a=rtcp-fb:100 ccm fir -a=rtcp-fb:100 nack -a=rtcp-fb:100 nack pli -a=fmtp:100 profile-id=2 -a=rtpmap:101 rtx/90000 -a=fmtp:101 apt=100 -a=rtpmap:112 H264/90000 -a=rtcp-fb:112 goog-remb -a=rtcp-fb:112 transport-cc -a=rtcp-fb:112 ccm fir -a=rtcp-fb:112 nack -a=rtcp-fb:112 nack pli -a=fmtp:112 level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=64001f -a=rtpmap:113 rtx/90000 -a=fmtp:113 apt=112 -a=rtpmap:116 red/90000 -a=rtpmap:117 rtx/90000 -a=fmtp:117 apt=116 -a=rtpmap:118 ulpfec/90000 -a=ssrc-group:FID 4072017687 3466187747 -a=ssrc:4072017687 cname:mbGW+aeWMLFhPbBC -a=ssrc:4072017687 msid:04f96a2a-d107-4a12-b545-cfe1dc2c4d37 4a1ae8a7-6c89-44d3-a9a1-02abce0012c7 -a=ssrc:3466187747 cname:mbGW+aeWMLFhPbBC -a=ssrc:3466187747 msid:04f96a2a-d107-4a12-b545-cfe1dc2c4d37 4a1ae8a7-6c89-44d3-a9a1-02abce0012c7 -m=audio 9 UDP/TLS/RTP/SAVPF 111 63 9 0 8 13 110 126 -c=IN IP4 0.0.0.0 -a=rtcp:9 IN IP4 0.0.0.0 -a=ice-ufrag:GM64 -a=ice-pwd:ANZFilRRlZJ3bg9AD40eRu7n -a=ice-options:trickle -a=fingerprint:sha-256 38:1F:02:E5:2A:49:9A:2A:D9:8E:B9:9B:4C:40:21:B7:F1:C4:27:8E:B5:68:D6:E0:91:08:D9:CB:2B:AC:B3:87 -a=setup:actpass -a=mid:3 -a=extmap:14 urn:ietf:params:rtp-hdrext:ssrc-audio-level -a=extmap:2 http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time -a=extmap:4 http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01 -a=extmap:9 urn:ietf:params:rtp-hdrext:sdes:mid -a=sendonly -a=msid:- 75b8c290-1f46-464c-8dc5-a4a3bced313a -a=rtcp-mux -a=rtpmap:111 opus/48000/2 -a=rtcp-fb:111 transport-cc -a=fmtp:111 minptime=10;usedtx=1;useinbandfec=1 -a=rtpmap:63 red/48000/2 -a=fmtp:63 111/111 -a=rtpmap:9 G722/8000 -a=rtpmap:0 PCMU/8000 -a=rtpmap:8 PCMA/8000 -a=rtpmap:13 CN/8000 -a=rtpmap:110 telephone-event/48000 -a=rtpmap:126 telephone-event/8000 -a=ssrc:1281279951 cname:mbGW+aeWMLFhPbBC -a=ssrc:1281279951 msid:- 75b8c290-1f46-464c-8dc5-a4a3bced313a -`.trim(); diff --git a/packages/client/src/rtc/helpers/__tests__/sdp.test.ts b/packages/client/src/rtc/helpers/__tests__/sdp.test.ts index 4b3e8269ae..96893c06a4 100644 --- a/packages/client/src/rtc/helpers/__tests__/sdp.test.ts +++ b/packages/client/src/rtc/helpers/__tests__/sdp.test.ts @@ -1,15 +1,8 @@ import { describe, expect, it } from 'vitest'; -import { enableHighQualityAudio, getPayloadTypeForCodec } from '../sdp'; -import { initialSdp as HQAudioSDP } from './hq-audio-sdp'; +import { getPayloadTypeForCodec } from '../sdp'; import { publisherSDP } from './publisher-sdp.mock'; describe('sdp-munging', () => { - it('enables HighQuality audio for Opus', () => { - const sdpWithHighQualityAudio = enableHighQualityAudio(HQAudioSDP, '3'); - expect(sdpWithHighQualityAudio).toContain('maxaveragebitrate=510000'); - expect(sdpWithHighQualityAudio).toContain('stereo=1'); - }); - it('extracts payload type for codec', () => { const payload = getPayloadTypeForCodec( publisherSDP, diff --git a/packages/client/src/rtc/helpers/sdp.ts b/packages/client/src/rtc/helpers/sdp.ts index d0e799cb5d..c0687c5794 100644 --- a/packages/client/src/rtc/helpers/sdp.ts +++ b/packages/client/src/rtc/helpers/sdp.ts @@ -1,52 +1,5 @@ import * as SDP from 'sdp-transform'; -/** - * Enables high-quality audio through SDP munging for the given trackMid. - * - * @param sdp the SDP to munge. - * @param trackMid the trackMid. - * @param maxBitrate the max bitrate to set. - */ -export const enableHighQualityAudio = ( - sdp: string, - trackMid: string, - maxBitrate: number = 510000, -): string => { - maxBitrate = Math.max(Math.min(maxBitrate, 510000), 96000); - - const parsedSdp = SDP.parse(sdp); - const audioMedia = parsedSdp.media.find( - (m) => m.type === 'audio' && String(m.mid) === trackMid, - ); - - if (!audioMedia) return sdp; - - const opusRtp = audioMedia.rtp.find((r) => r.codec === 'opus'); - if (!opusRtp) return sdp; - - const opusFmtp = audioMedia.fmtp.find((f) => f.payload === opusRtp.payload); - if (!opusFmtp) return sdp; - - // enable stereo, if not already enabled - if (opusFmtp.config.match(/stereo=(\d)/)) { - opusFmtp.config = opusFmtp.config.replace(/stereo=(\d)/, 'stereo=1'); - } else { - opusFmtp.config = `${opusFmtp.config};stereo=1`; - } - - // set maxaveragebitrate, to the given value - if (opusFmtp.config.match(/maxaveragebitrate=(\d*)/)) { - opusFmtp.config = opusFmtp.config.replace( - /maxaveragebitrate=(\d*)/, - `maxaveragebitrate=${maxBitrate}`, - ); - } else { - opusFmtp.config = `${opusFmtp.config};maxaveragebitrate=${maxBitrate}`; - } - - return SDP.write(parsedSdp); -}; - /** * Gets the payload type for the given codec. */ diff --git a/packages/client/src/rtc/videoLayers.ts b/packages/client/src/rtc/videoLayers.ts index 425ae8d7dc..0bd5e60fde 100644 --- a/packages/client/src/rtc/videoLayers.ts +++ b/packages/client/src/rtc/videoLayers.ts @@ -2,6 +2,7 @@ import { isSvcCodec } from './codecs'; import { PublishOption, VideoDimension, + VideoLayer, VideoQuality, } from '../gen/video/sfu/models/models'; @@ -43,6 +44,21 @@ export const ridToVideoQuality = (rid: string): VideoQuality => { : VideoQuality.HIGH; // default to HIGH }; +/** + * Converts the given video layers to SFU video layers. + */ +export const toVideoLayers = ( + layers: OptimalVideoLayer[] | undefined = [], +): VideoLayer[] => { + return layers.map((layer) => ({ + rid: layer.rid || '', + bitrate: layer.maxBitrate || 0, + fps: layer.maxFramerate || 0, + quality: ridToVideoQuality(layer.rid || ''), + videoDimension: { width: layer.width, height: layer.height }, + })); +}; + /** * Converts the spatial and temporal layers to a scalability mode. */