From f6b63362d30b0eb76d2810ec152d34f05b8d3995 Mon Sep 17 00:00:00 2001 From: Oliver Lazoroski Date: Fri, 18 Oct 2024 17:51:13 +0200 Subject: [PATCH 01/52] feat: move preferred codec negotiation to the SFU --- .../client/src/gen/video/sfu/models/models.ts | 62 ++++++------------ packages/client/src/rtc/Publisher.ts | 64 +++++++++---------- .../src/rtc/__tests__/Publisher.test.ts | 2 - packages/client/src/rtc/codecs.ts | 4 +- 4 files changed, 54 insertions(+), 78 deletions(-) diff --git a/packages/client/src/gen/video/sfu/models/models.ts b/packages/client/src/gen/video/sfu/models/models.ts index b555626a74..7d33a838b2 100644 --- a/packages/client/src/gen/video/sfu/models/models.ts +++ b/packages/client/src/gen/video/sfu/models/models.ts @@ -199,29 +199,17 @@ export interface VideoLayer { */ export interface Codec { /** - * @generated from protobuf field: uint32 payload_type = 1; + * @generated from protobuf field: string mime_type = 1; */ - payloadType: number; + mimeType: string; /** - * @generated from protobuf field: string name = 2; + * @generated from protobuf field: string scalability_mode = 2; */ - name: string; - /** - * @generated from protobuf field: string fmtp_line = 3; - */ - fmtpLine: string; - /** - * @generated from protobuf field: uint32 clock_rate = 4; - */ - clockRate: number; - /** - * @generated from protobuf field: string encoding_parameters = 5; - */ - encodingParameters: string; + scalabilityMode: string; /** - * @generated from protobuf field: repeated string feedbacks = 6; + * @generated from protobuf field: string fmtp = 3; */ - feedbacks: string[]; + fmtp: string; } /** * @generated from protobuf message stream.video.sfu.models.ICETrickle @@ -278,6 +266,10 @@ export interface TrackInfo { * @generated from protobuf field: bool muted = 10; */ muted: boolean; + /** + * @generated from protobuf field: repeated stream.video.sfu.models.Codec preferred_codecs = 11; + */ + preferredCodecs: Codec[]; } /** * @generated from protobuf message stream.video.sfu.models.Error @@ -959,33 +951,14 @@ export const VideoLayer = new VideoLayer$Type(); class Codec$Type extends MessageType { constructor() { super('stream.video.sfu.models.Codec', [ + { no: 1, name: 'mime_type', kind: 'scalar', T: 9 /*ScalarType.STRING*/ }, { - no: 1, - name: 'payload_type', - kind: 'scalar', - T: 13 /*ScalarType.UINT32*/, - }, - { no: 2, name: 'name', kind: 'scalar', T: 9 /*ScalarType.STRING*/ }, - { no: 3, name: 'fmtp_line', kind: 'scalar', T: 9 /*ScalarType.STRING*/ }, - { - no: 4, - name: 'clock_rate', - kind: 'scalar', - T: 13 /*ScalarType.UINT32*/, - }, - { - no: 5, - name: 'encoding_parameters', - kind: 'scalar', - T: 9 /*ScalarType.STRING*/, - }, - { - no: 6, - name: 'feedbacks', + no: 2, + name: 'scalability_mode', kind: 'scalar', - repeat: 2 /*RepeatType.UNPACKED*/, T: 9 /*ScalarType.STRING*/, }, + { no: 3, name: 'fmtp', kind: 'scalar', T: 9 /*ScalarType.STRING*/ }, ]); } } @@ -1044,6 +1017,13 @@ class TrackInfo$Type extends MessageType { { no: 8, name: 'stereo', kind: 'scalar', T: 8 /*ScalarType.BOOL*/ }, { no: 9, name: 'red', kind: 'scalar', T: 8 /*ScalarType.BOOL*/ }, { no: 10, name: 'muted', kind: 'scalar', T: 8 /*ScalarType.BOOL*/ }, + { + no: 11, + name: 'preferred_codecs', + kind: 'message', + repeat: 1 /*RepeatType.PACKED*/, + T: () => Codec, + }, ]); } } diff --git a/packages/client/src/rtc/Publisher.ts b/packages/client/src/rtc/Publisher.ts index a55de6411e..1f72d159f1 100644 --- a/packages/client/src/rtc/Publisher.ts +++ b/packages/client/src/rtc/Publisher.ts @@ -1,5 +1,6 @@ import { StreamSfuClient } from '../StreamSfuClient'; import { + Codec, PeerType, TrackInfo, TrackType, @@ -52,6 +53,8 @@ export class Publisher { private readonly transceiverCache = new Map(); private readonly trackLayersCache = new Map(); private readonly publishOptsForTrack = new Map(); + private readonly codecsForTrack = new Map(); + private readonly scalabilityModeForTrack = new Map(); /** * An array maintaining the order how transceivers were added to the peer connection. @@ -211,7 +214,7 @@ export class Publisher { ); }; track.addEventListener('ended', handleTrackEnded); - this.addTransceiver(trackType, track, opts, mediaStream); + this.addTransceiver(trackType, track, opts); } else { await this.updateTransceiver(transceiver, track); } @@ -228,46 +231,27 @@ export class Publisher { trackType: TrackType, track: MediaStreamTrack, opts: PublishOptions, - mediaStream: MediaStream, ) => { const { forceCodec, preferredCodec } = opts; const codecInUse = forceCodec || getOptimalVideoCodec(preferredCodec); const videoEncodings = this.computeLayers(trackType, track, opts); + const sendEncodings = isSvcCodec(codecInUse) + ? toSvcEncodings(videoEncodings) + : videoEncodings; const transceiver = this.pc.addTransceiver(track, { direction: 'sendonly', - streams: - trackType === TrackType.VIDEO || trackType === TrackType.SCREEN_SHARE - ? [mediaStream] - : undefined, - sendEncodings: isSvcCodec(codecInUse) - ? toSvcEncodings(videoEncodings) - : videoEncodings, + sendEncodings, }); this.logger('debug', `Added ${TrackType[trackType]} transceiver`); this.transceiverInitOrder.push(trackType); this.transceiverCache.set(trackType, transceiver); this.publishOptsForTrack.set(trackType, opts); - - // handle codec preferences - if (!('setCodecPreferences' in transceiver)) return; - - const codecPreferences = this.getCodecPreferences( + this.codecsForTrack.set(trackType, codecInUse); + this.scalabilityModeForTrack.set( trackType, - trackType === TrackType.VIDEO ? codecInUse : undefined, + sendEncodings?.[0]?.scalabilityMode || '', ); - if (!codecPreferences) return; - - try { - this.logger( - 'info', - `Setting ${TrackType[trackType]} codec preferences`, - codecPreferences, - ); - transceiver.setCodecPreferences(codecPreferences); - } catch (err) { - this.logger('warn', `Couldn't set codec preferences`, err); - } }; /** @@ -370,7 +354,8 @@ export class Publisher { enabledLayers, ); - const videoSender = this.transceiverCache.get(TrackType.VIDEO)?.sender; + const trackType = TrackType.VIDEO; + const videoSender = this.transceiverCache.get(trackType)?.sender; if (!videoSender) { this.logger('warn', 'Update publish quality, no video sender found.'); return; @@ -431,6 +416,7 @@ export class Publisher { if (scalabilityMode && scalabilityMode !== encoder.scalabilityMode) { // @ts-expect-error scalabilityMode is not in the typedefs yet encoder.scalabilityMode = scalabilityMode; + this.scalabilityModeForTrack.set(trackType, scalabilityMode); changed = true; } } @@ -617,24 +603,36 @@ export class Publisher { }, })); - const isAudioTrack = [ - TrackType.AUDIO, - TrackType.SCREEN_SHARE_AUDIO, - ].includes(trackType); + const isAudioTrack = + trackType === TrackType.AUDIO || + trackType === TrackType.SCREEN_SHARE_AUDIO; const trackSettings = track.getSettings(); const isStereo = isAudioTrack && trackSettings.channelCount === 2; const transceiverInitIndex = this.transceiverInitOrder.indexOf(trackType); + + const codecInUse = + trackType === TrackType.VIDEO + ? this.codecsForTrack.get(trackType) + : undefined; + const preferredCodecs = this.getCodecPreferences(trackType, codecInUse); return { trackId: track.id, - layers: layers, + layers, trackType, mid: extractMid(transceiver, transceiverInitIndex, sdp), stereo: isStereo, dtx: isAudioTrack && this.isDtxEnabled, red: isAudioTrack && this.isRedEnabled, muted: !isTrackLive, + preferredCodecs: (preferredCodecs || []).map((codec) => ({ + mimeType: codec.mimeType, + fmtp: codec.sdpFmtpLine || '', + scalabilityMode: isSvcCodec(codec.mimeType) + ? this.scalabilityModeForTrack.get(trackType) || '' + : '', + })), }; }); }; diff --git a/packages/client/src/rtc/__tests__/Publisher.test.ts b/packages/client/src/rtc/__tests__/Publisher.test.ts index 9b73af5b95..d558a6c4c5 100644 --- a/packages/client/src/rtc/__tests__/Publisher.test.ts +++ b/packages/client/src/rtc/__tests__/Publisher.test.ts @@ -114,7 +114,6 @@ describe('Publisher', () => { expect(state.localParticipant?.publishedTracks).toContain(TrackType.VIDEO); expect(state.localParticipant?.videoStream).toEqual(mediaStream); - expect(transceiver.setCodecPreferences).toHaveBeenCalled(); expect(sfuClient.updateMuteState).toHaveBeenCalledWith( TrackType.VIDEO, false, @@ -187,7 +186,6 @@ describe('Publisher', () => { expect(state.localParticipant?.publishedTracks).toContain(TrackType.VIDEO); expect(track.enabled).toBe(true); expect(state.localParticipant?.videoStream).toEqual(mediaStream); - expect(transceiver.setCodecPreferences).toHaveBeenCalled(); expect(sfuClient.updateMuteState).toHaveBeenCalledWith( TrackType.VIDEO, false, diff --git a/packages/client/src/rtc/codecs.ts b/packages/client/src/rtc/codecs.ts index 8c498c029c..2b86b00dd3 100644 --- a/packages/client/src/rtc/codecs.ts +++ b/packages/client/src/rtc/codecs.ts @@ -15,9 +15,9 @@ export const getPreferredCodecs = ( preferredCodec: string, codecToRemove?: string, ): RTCRtpCodecCapability[] | undefined => { - if (!('getCapabilities' in RTCRtpReceiver)) return; + if (!('getCapabilities' in RTCRtpSender)) return; - const capabilities = RTCRtpReceiver.getCapabilities(kind); + const capabilities = RTCRtpSender.getCapabilities(kind); if (!capabilities) return; const preferred: RTCRtpCodecCapability[] = []; From 5105235878f10a07527c08b9927556b4061f1f7e Mon Sep 17 00:00:00 2001 From: Oliver Lazoroski Date: Fri, 18 Oct 2024 18:12:41 +0200 Subject: [PATCH 02/52] feat: move opus, dtx and red negotiation to the SFU too --- packages/client/src/Call.ts | 5 - .../src/helpers/__tests__/sdp-munging.test.ts | 15 +- packages/client/src/helpers/sdp-munging.ts | 129 ------------------ packages/client/src/rtc/Publisher.ts | 37 ++--- .../client/src/rtc/__tests__/codecs.test.ts | 10 +- .../src/rtc/__tests__/mocks/webrtc.mocks.ts | 7 + packages/client/src/rtc/codecs.ts | 8 -- 7 files changed, 22 insertions(+), 189 deletions(-) diff --git a/packages/client/src/Call.ts b/packages/client/src/Call.ts index cbf052dd10..31419af4ba 100644 --- a/packages/client/src/Call.ts +++ b/packages/client/src/Call.ts @@ -955,16 +955,11 @@ export class Call { if (closePreviousInstances && this.publisher) { this.publisher.close({ stopTracks: false }); } - const audioSettings = this.state.settings?.audio; - const isDtxEnabled = !!audioSettings?.opus_dtx_enabled; - const isRedEnabled = !!audioSettings?.redundant_coding_enabled; this.publisher = new Publisher({ sfuClient, dispatcher: this.dispatcher, state: this.state, connectionConfig, - isDtxEnabled, - isRedEnabled, logTag: String(this.sfuClientTag), onUnrecoverableError: () => { this.reconnect(WebsocketReconnectStrategy.REJOIN).catch((err) => { diff --git a/packages/client/src/helpers/__tests__/sdp-munging.test.ts b/packages/client/src/helpers/__tests__/sdp-munging.test.ts index 37dae3d1fb..8798fbcf51 100644 --- a/packages/client/src/helpers/__tests__/sdp-munging.test.ts +++ b/packages/client/src/helpers/__tests__/sdp-munging.test.ts @@ -1,21 +1,8 @@ import { describe, expect, it } from 'vitest'; -import { enableHighQualityAudio, toggleDtx } from '../sdp-munging'; +import { enableHighQualityAudio } from '../sdp-munging'; import { initialSdp as HQAudioSDP } from './hq-audio-sdp'; describe('sdp-munging', () => { - it('Supporting the enabling and disabling of DTX audio codec', () => { - const sdp = `m=audio 54312 RTP/AVP 101 -a=rtpmap:101 opus/48000/2 -a=fmtp:101 maxplaybackrate=16000; sprop-maxcapturerate=16000; -maxaveragebitrate=20000; stereo=1; useinbandfec=1; usedtx=0 -a=ptime:40 -a=maxptime:40`; - const dtxEnabledSdp = toggleDtx(sdp, true); - expect(dtxEnabledSdp.search('usedtx=1') !== -1).toBeTruthy(); - const dtxDisabledSdp = toggleDtx(dtxEnabledSdp, false); - expect(dtxDisabledSdp.search('usedtx=0') !== -1).toBeTruthy(); - }); - it('enables HighQuality audio for Opus', () => { const sdpWithHighQualityAudio = enableHighQualityAudio(HQAudioSDP, '3'); expect(sdpWithHighQualityAudio).toContain('maxaveragebitrate=510000'); diff --git a/packages/client/src/helpers/sdp-munging.ts b/packages/client/src/helpers/sdp-munging.ts index b8bdfd66af..bca3fd10df 100644 --- a/packages/client/src/helpers/sdp-munging.ts +++ b/packages/client/src/helpers/sdp-munging.ts @@ -1,134 +1,5 @@ import * as SDP from 'sdp-transform'; -type Media = { - original: string; - mediaWithPorts: string; - codecOrder: string; -}; - -type RtpMap = { - original: string; - payload: string; - codec: string; -}; - -type Fmtp = { - original: string; - payload: string; - config: string; -}; - -const getRtpMap = (line: string): RtpMap | undefined => { - // Example: a=rtpmap:110 opus/48000/2 - const rtpRegex = /^a=rtpmap:(\d*) ([\w\-.]*)(?:\s*\/(\d*)(?:\s*\/(\S*))?)?/; - // The first captured group is the payload type number, the second captured group is the encoding name, the third captured group is the clock rate, and the fourth captured group is any additional parameters. - const rtpMatch = rtpRegex.exec(line); - if (rtpMatch) { - return { - original: rtpMatch[0], - payload: rtpMatch[1], - codec: rtpMatch[2], - }; - } -}; - -const getFmtp = (line: string): Fmtp | undefined => { - // Example: a=fmtp:111 minptime=10; useinbandfec=1 - const fmtpRegex = /^a=fmtp:(\d*) (.*)/; - const fmtpMatch = fmtpRegex.exec(line); - // The first captured group is the payload type number, the second captured group is any additional parameters. - if (fmtpMatch) { - return { - original: fmtpMatch[0], - payload: fmtpMatch[1], - config: fmtpMatch[2], - }; - } -}; - -/** - * gets the media section for the specified media type. - * The media section contains the media type, port, codec, and payload type. - * Example: m=video 9 UDP/TLS/RTP/SAVPF 100 101 96 97 35 36 102 125 127 - */ -const getMedia = (line: string, mediaType: string): Media | undefined => { - const regex = new RegExp(`(m=${mediaType} \\d+ [\\w/]+) ([\\d\\s]+)`); - const match = regex.exec(line); - if (match) { - return { - original: match[0], - mediaWithPorts: match[1], - codecOrder: match[2], - }; - } -}; - -const getMediaSection = (sdp: string, mediaType: 'video' | 'audio') => { - let media: Media | undefined; - const rtpMap: RtpMap[] = []; - const fmtp: Fmtp[] = []; - let isTheRequiredMediaSection = false; - sdp.split(/(\r\n|\r|\n)/).forEach((line) => { - const isValidLine = /^([a-z])=(.*)/.test(line); - if (!isValidLine) return; - /* - NOTE: according to https://www.rfc-editor.org/rfc/rfc8866.pdf - Each media description starts with an "m=" line and continues to the next media description or the end of the whole session description, whichever comes first - */ - const type = line[0]; - if (type === 'm') { - const _media = getMedia(line, mediaType); - isTheRequiredMediaSection = !!_media; - if (_media) { - media = _media; - } - } else if (isTheRequiredMediaSection && type === 'a') { - const rtpMapLine = getRtpMap(line); - const fmtpLine = getFmtp(line); - if (rtpMapLine) { - rtpMap.push(rtpMapLine); - } else if (fmtpLine) { - fmtp.push(fmtpLine); - } - } - }); - if (media) { - return { - media, - rtpMap, - fmtp, - }; - } -}; - -/** - * Gets the fmtp line corresponding to opus - */ -const getOpusFmtp = (sdp: string): Fmtp | undefined => { - const section = getMediaSection(sdp, 'audio'); - const rtpMap = section?.rtpMap.find((r) => r.codec.toLowerCase() === 'opus'); - const codecId = rtpMap?.payload; - if (codecId) { - return section?.fmtp.find((f) => f.payload === codecId); - } -}; - -/** - * Returns an SDP with DTX enabled or disabled. - */ -export const toggleDtx = (sdp: string, enable: boolean): string => { - const opusFmtp = getOpusFmtp(sdp); - if (!opusFmtp) return sdp; - - const matchDtx = /usedtx=(\d)/.exec(opusFmtp.config); - const requiredDtxConfig = `usedtx=${enable ? '1' : '0'}`; - const newFmtp = matchDtx - ? opusFmtp.original.replace(/usedtx=(\d)/, requiredDtxConfig) - : `${opusFmtp.original};${requiredDtxConfig}`; - - return sdp.replace(opusFmtp.original, newFmtp); -}; - /** * Enables high-quality audio through SDP munging for the given trackMid. * diff --git a/packages/client/src/rtc/Publisher.ts b/packages/client/src/rtc/Publisher.ts index 1f72d159f1..94a2977f2f 100644 --- a/packages/client/src/rtc/Publisher.ts +++ b/packages/client/src/rtc/Publisher.ts @@ -18,11 +18,7 @@ import { getOptimalVideoCodec, getPreferredCodecs, isSvcCodec } from './codecs'; import { trackTypeToParticipantStreamKey } from './helpers/tracks'; import { CallingState, CallState } from '../store'; import { PublishOptions } from '../types'; -import { - enableHighQualityAudio, - extractMid, - toggleDtx, -} from '../helpers/sdp-munging'; +import { enableHighQualityAudio, extractMid } from '../helpers/sdp-munging'; import { Logger } from '../coordinator/connection/types'; import { getLogger } from '../logger'; import { Dispatcher } from './Dispatcher'; @@ -35,8 +31,6 @@ export type PublisherConstructorOpts = { state: CallState; dispatcher: Dispatcher; connectionConfig?: RTCConfiguration; - isDtxEnabled: boolean; - isRedEnabled: boolean; onUnrecoverableError?: () => void; logTag: string; }; @@ -64,8 +58,6 @@ export class Publisher { * @internal */ private readonly transceiverInitOrder: TrackType[] = []; - private readonly isDtxEnabled: boolean; - private readonly isRedEnabled: boolean; private readonly unsubscribeOnIceRestart: () => void; private readonly unsubscribeChangePublishQuality: () => void; @@ -82,8 +74,6 @@ export class Publisher { sfuClient, dispatcher, state, - isDtxEnabled, - isRedEnabled, onUnrecoverableError, logTag, }: PublisherConstructorOpts) { @@ -91,8 +81,6 @@ export class Publisher { this.pc = this.createPeerConnection(connectionConfig); this.sfuClient = sfuClient; this.state = state; - this.isDtxEnabled = isDtxEnabled; - this.isRedEnabled = isRedEnabled; this.onUnrecoverableError = onUnrecoverableError; this.unsubscribeOnIceRestart = dispatcher.on('iceRestart', (iceRestart) => { @@ -448,13 +436,7 @@ export class Publisher { return getPreferredCodecs('video', preferredCodec || 'vp8'); } if (trackType === TrackType.AUDIO) { - const defaultAudioCodec = this.isRedEnabled ? 'red' : 'opus'; - const codecToRemove = !this.isRedEnabled ? 'red' : undefined; - return getPreferredCodecs( - 'audio', - preferredCodec ?? defaultAudioCodec, - codecToRemove, - ); + return getPreferredCodecs('audio', 'opus'); } }; @@ -510,11 +492,8 @@ export class Publisher { */ private negotiate = async (options?: RTCOfferOptions) => { const offer = await this.pc.createOffer(options); - if (offer.sdp) { - offer.sdp = toggleDtx(offer.sdp, this.isDtxEnabled); - if (this.isPublishing(TrackType.SCREEN_SHARE_AUDIO)) { - offer.sdp = this.enableHighQualityAudio(offer.sdp); - } + if (offer.sdp && this.isPublishing(TrackType.SCREEN_SHARE_AUDIO)) { + offer.sdp = this.enableHighQualityAudio(offer.sdp); } const trackInfos = this.getAnnouncedTracks(offer.sdp); @@ -607,6 +586,10 @@ export class Publisher { trackType === TrackType.AUDIO || trackType === TrackType.SCREEN_SHARE_AUDIO; + const audioSettings = this.state.settings?.audio; + const isDtxEnabled = !!audioSettings?.opus_dtx_enabled; + const isRedEnabled = !!audioSettings?.redundant_coding_enabled; + const trackSettings = track.getSettings(); const isStereo = isAudioTrack && trackSettings.channelCount === 2; const transceiverInitIndex = @@ -623,8 +606,8 @@ export class Publisher { trackType, mid: extractMid(transceiver, transceiverInitIndex, sdp), stereo: isStereo, - dtx: isAudioTrack && this.isDtxEnabled, - red: isAudioTrack && this.isRedEnabled, + dtx: isAudioTrack && isDtxEnabled, + red: isAudioTrack && isRedEnabled, muted: !isTrackLive, preferredCodecs: (preferredCodecs || []).map((codec) => ({ mimeType: codec.mimeType, diff --git a/packages/client/src/rtc/__tests__/codecs.test.ts b/packages/client/src/rtc/__tests__/codecs.test.ts index afa8a1be94..61e1c58233 100644 --- a/packages/client/src/rtc/__tests__/codecs.test.ts +++ b/packages/client/src/rtc/__tests__/codecs.test.ts @@ -4,7 +4,7 @@ import './mocks/webrtc.mocks'; describe('codecs', () => { it('should return preferred audio codec', () => { - RTCRtpReceiver.getCapabilities = vi.fn().mockReturnValue(audioCodecs); + RTCRtpSender.getCapabilities = vi.fn().mockReturnValue(audioCodecs); const codecs = getPreferredCodecs('audio', 'red'); expect(codecs).toBeDefined(); expect(codecs?.map((c) => c.mimeType)).toEqual([ @@ -19,7 +19,7 @@ describe('codecs', () => { }); it('should return preferred video codec', () => { - RTCRtpReceiver.getCapabilities = vi.fn().mockReturnValue(videoCodecs); + RTCRtpSender.getCapabilities = vi.fn().mockReturnValue(videoCodecs); const codecs = getPreferredCodecs('video', 'vp8'); expect(codecs).toBeDefined(); // prettier-ignore @@ -39,7 +39,7 @@ describe('codecs', () => { }); it('should pick the baseline H264 codec', () => { - RTCRtpReceiver.getCapabilities = vi.fn().mockReturnValue(videoCodecs); + RTCRtpSender.getCapabilities = vi.fn().mockReturnValue(videoCodecs); const codecs = getPreferredCodecs('video', 'h264'); expect(codecs).toBeDefined(); // prettier-ignore @@ -59,9 +59,7 @@ describe('codecs', () => { }); it('should pick the baseline H264 codec with optional packetization-mode', () => { - RTCRtpReceiver.getCapabilities = vi - .fn() - .mockReturnValue(videoCodecsFirefox); + RTCRtpSender.getCapabilities = vi.fn().mockReturnValue(videoCodecsFirefox); const codecs = getPreferredCodecs('video', 'h264'); expect(codecs).toBeDefined(); // prettier-ignore diff --git a/packages/client/src/rtc/__tests__/mocks/webrtc.mocks.ts b/packages/client/src/rtc/__tests__/mocks/webrtc.mocks.ts index 5a4424fbba..9af17ff0da 100644 --- a/packages/client/src/rtc/__tests__/mocks/webrtc.mocks.ts +++ b/packages/client/src/rtc/__tests__/mocks/webrtc.mocks.ts @@ -69,3 +69,10 @@ const RTCRtpReceiverMock = vi.fn((): Partial => { }; }); vi.stubGlobal('RTCRtpReceiver', RTCRtpReceiverMock); + +const RTCRtpSenderMock = vi.fn((): Partial => { + return { + getCapabilities: vi.fn(), + }; +}); +vi.stubGlobal('RTCRtpSender', RTCRtpSenderMock); diff --git a/packages/client/src/rtc/codecs.ts b/packages/client/src/rtc/codecs.ts index 2b86b00dd3..0954bc52be 100644 --- a/packages/client/src/rtc/codecs.ts +++ b/packages/client/src/rtc/codecs.ts @@ -8,12 +8,10 @@ import type { PreferredCodec } from '../types'; * * @param kind the kind of codec to get. * @param preferredCodec the codec to prioritize (vp8, h264, vp9, av1...). - * @param codecToRemove the codec to exclude from the list. */ export const getPreferredCodecs = ( kind: 'audio' | 'video', preferredCodec: string, - codecToRemove?: string, ): RTCRtpCodecCapability[] | undefined => { if (!('getCapabilities' in RTCRtpSender)) return; @@ -25,15 +23,9 @@ export const getPreferredCodecs = ( const unpreferred: RTCRtpCodecCapability[] = []; const preferredCodecMimeType = `${kind}/${preferredCodec.toLowerCase()}`; - const codecToRemoveMimeType = - codecToRemove && `${kind}/${codecToRemove.toLowerCase()}`; - for (const codec of capabilities.codecs) { const codecMimeType = codec.mimeType.toLowerCase(); - const shouldRemoveCodec = codecMimeType === codecToRemoveMimeType; - if (shouldRemoveCodec) continue; // skip this codec - const isPreferredCodec = codecMimeType === preferredCodecMimeType; if (!isPreferredCodec) { unpreferred.push(codec); From 70791ae894d3a9640c6131d8d76be093f32e108e Mon Sep 17 00:00:00 2001 From: Oliver Lazoroski Date: Mon, 21 Oct 2024 12:24:25 +0200 Subject: [PATCH 03/52] feat: correctly sort audio codec preferences --- packages/client/src/rtc/Publisher.ts | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/packages/client/src/rtc/Publisher.ts b/packages/client/src/rtc/Publisher.ts index 94a2977f2f..6daf0817bf 100644 --- a/packages/client/src/rtc/Publisher.ts +++ b/packages/client/src/rtc/Publisher.ts @@ -436,7 +436,7 @@ export class Publisher { return getPreferredCodecs('video', preferredCodec || 'vp8'); } if (trackType === TrackType.AUDIO) { - return getPreferredCodecs('audio', 'opus'); + return getPreferredCodecs('audio', preferredCodec || 'opus'); } }; @@ -595,11 +595,13 @@ export class Publisher { const transceiverInitIndex = this.transceiverInitOrder.indexOf(trackType); - const codecInUse = - trackType === TrackType.VIDEO - ? this.codecsForTrack.get(trackType) - : undefined; - const preferredCodecs = this.getCodecPreferences(trackType, codecInUse); + let codecToUse: string | undefined = undefined; + if (trackType === TrackType.VIDEO) { + codecToUse = this.codecsForTrack.get(trackType); + } else if (trackType === TrackType.AUDIO) { + codecToUse = isRedEnabled ? 'red' : 'opus'; + } + const preferredCodecs = this.getCodecPreferences(trackType, codecToUse); return { trackId: track.id, layers, From 6a05ca4d40f93a0f2bd35524e0c597c07d345150 Mon Sep 17 00:00:00 2001 From: Oliver Lazoroski Date: Tue, 22 Oct 2024 23:15:18 +0200 Subject: [PATCH 04/52] fix: don't announce RED if unsupported by the platform --- packages/client/src/rtc/Publisher.ts | 25 ++++---- .../client/src/rtc/__tests__/codecs.test.ts | 8 +-- packages/client/src/rtc/codecs.ts | 57 +++++++++++++------ 3 files changed, 54 insertions(+), 36 deletions(-) diff --git a/packages/client/src/rtc/Publisher.ts b/packages/client/src/rtc/Publisher.ts index 6daf0817bf..a02024de6d 100644 --- a/packages/client/src/rtc/Publisher.ts +++ b/packages/client/src/rtc/Publisher.ts @@ -14,7 +14,12 @@ import { ridToVideoQuality, toSvcEncodings, } from './videoLayers'; -import { getOptimalVideoCodec, getPreferredCodecs, isSvcCodec } from './codecs'; +import { + getCodecPreferences, + getOptimalVideoCodec, + isCodecSupported, + isSvcCodec, +} from './codecs'; import { trackTypeToParticipantStreamKey } from './helpers/tracks'; import { CallingState, CallState } from '../store'; import { PublishOptions } from '../types'; @@ -428,18 +433,6 @@ export class Publisher { return this.pc.getStats(selector); }; - private getCodecPreferences = ( - trackType: TrackType, - preferredCodec?: string, - ) => { - if (trackType === TrackType.VIDEO) { - return getPreferredCodecs('video', preferredCodec || 'vp8'); - } - if (trackType === TrackType.AUDIO) { - return getPreferredCodecs('audio', preferredCodec || 'opus'); - } - }; - private onIceCandidate = (e: RTCPeerConnectionIceEvent) => { const { candidate } = e; if (!candidate) { @@ -588,7 +581,9 @@ export class Publisher { const audioSettings = this.state.settings?.audio; const isDtxEnabled = !!audioSettings?.opus_dtx_enabled; - const isRedEnabled = !!audioSettings?.redundant_coding_enabled; + const isRedEnabled = + !!audioSettings?.redundant_coding_enabled && + isCodecSupported('audio/red'); const trackSettings = track.getSettings(); const isStereo = isAudioTrack && trackSettings.channelCount === 2; @@ -601,7 +596,7 @@ export class Publisher { } else if (trackType === TrackType.AUDIO) { codecToUse = isRedEnabled ? 'red' : 'opus'; } - const preferredCodecs = this.getCodecPreferences(trackType, codecToUse); + const preferredCodecs = getCodecPreferences(trackType, codecToUse); return { trackId: track.id, layers, diff --git a/packages/client/src/rtc/__tests__/codecs.test.ts b/packages/client/src/rtc/__tests__/codecs.test.ts index 61e1c58233..62a4892c48 100644 --- a/packages/client/src/rtc/__tests__/codecs.test.ts +++ b/packages/client/src/rtc/__tests__/codecs.test.ts @@ -5,7 +5,7 @@ import './mocks/webrtc.mocks'; describe('codecs', () => { it('should return preferred audio codec', () => { RTCRtpSender.getCapabilities = vi.fn().mockReturnValue(audioCodecs); - const codecs = getPreferredCodecs('audio', 'red'); + const codecs = getPreferredCodecs('audio/red'); expect(codecs).toBeDefined(); expect(codecs?.map((c) => c.mimeType)).toEqual([ 'audio/red', @@ -20,7 +20,7 @@ describe('codecs', () => { it('should return preferred video codec', () => { RTCRtpSender.getCapabilities = vi.fn().mockReturnValue(videoCodecs); - const codecs = getPreferredCodecs('video', 'vp8'); + const codecs = getPreferredCodecs('video/vp8'); expect(codecs).toBeDefined(); // prettier-ignore expect(codecs?.map((c) => [c.mimeType, c.sdpFmtpLine])).toEqual([ @@ -40,7 +40,7 @@ describe('codecs', () => { it('should pick the baseline H264 codec', () => { RTCRtpSender.getCapabilities = vi.fn().mockReturnValue(videoCodecs); - const codecs = getPreferredCodecs('video', 'h264'); + const codecs = getPreferredCodecs('video/h264'); expect(codecs).toBeDefined(); // prettier-ignore expect(codecs?.map((c) => [c.mimeType, c.sdpFmtpLine])).toEqual([ @@ -60,7 +60,7 @@ describe('codecs', () => { it('should pick the baseline H264 codec with optional packetization-mode', () => { RTCRtpSender.getCapabilities = vi.fn().mockReturnValue(videoCodecsFirefox); - const codecs = getPreferredCodecs('video', 'h264'); + const codecs = getPreferredCodecs('video/h264'); expect(codecs).toBeDefined(); // prettier-ignore expect(codecs?.map((c) => [c.mimeType, c.sdpFmtpLine])).toEqual([ diff --git a/packages/client/src/rtc/codecs.ts b/packages/client/src/rtc/codecs.ts index 0954bc52be..12f7410a50 100644 --- a/packages/client/src/rtc/codecs.ts +++ b/packages/client/src/rtc/codecs.ts @@ -1,20 +1,20 @@ import { getOSInfo } from '../client-details'; import { isReactNative } from '../helpers/platforms'; import { isFirefox, isSafari } from '../helpers/browsers'; +import { TrackType } from '../gen/video/sfu/models/models'; import type { PreferredCodec } from '../types'; /** * Returns back a list of sorted codecs, with the preferred codec first. * - * @param kind the kind of codec to get. - * @param preferredCodec the codec to prioritize (vp8, h264, vp9, av1...). + * @param preferredCodecMimeType the codec to prioritize (video/vp8, video/h264, video/vp9, video/av1...). */ export const getPreferredCodecs = ( - kind: 'audio' | 'video', - preferredCodec: string, + preferredCodecMimeType: string, ): RTCRtpCodecCapability[] | undefined => { if (!('getCapabilities' in RTCRtpSender)) return; + const [kind] = preferredCodecMimeType.split('/'); const capabilities = RTCRtpSender.getCapabilities(kind); if (!capabilities) return; @@ -22,7 +22,7 @@ export const getPreferredCodecs = ( const partiallyPreferred: RTCRtpCodecCapability[] = []; const unpreferred: RTCRtpCodecCapability[] = []; - const preferredCodecMimeType = `${kind}/${preferredCodec.toLowerCase()}`; + preferredCodecMimeType = preferredCodecMimeType.toLowerCase(); for (const codec of capabilities.codecs) { const codecMimeType = codec.mimeType.toLowerCase(); @@ -64,10 +64,27 @@ export const getPreferredCodecs = ( return [...preferred, ...partiallyPreferred, ...unpreferred]; }; +/** + * Returns an ordered list of preferred codecs for the given track type. + * + * @param trackType the type of track. + * @param preferredCodec the preferred codec to prioritize. + */ +export const getCodecPreferences = ( + trackType: TrackType, + preferredCodec?: string, +): RTCRtpCodecCapability[] | undefined => { + return trackType === TrackType.VIDEO + ? getPreferredCodecs(`video/${preferredCodec || 'vp8'}`) + : trackType === TrackType.AUDIO + ? getPreferredCodecs(`audio/${preferredCodec || 'opus'}`) + : undefined; +}; + /** * Returns a generic SDP for the given direction. * We use this SDP to send it as part of our JoinRequest so that the SFU - * can use it to determine client's codec capabilities. + * can use it to determine the client's codec capabilities. * * @param direction the direction of the transceiver. */ @@ -98,6 +115,8 @@ export const getOptimalVideoCodec = ( if (os === 'ios' || os === 'ipados') return 'h264'; return preferredOr(preferredCodec, 'h264'); } + // Safari and Firefox do not have a good support encoding to SVC codecs, + // so we disable it for them. if (isSafari()) return 'h264'; if (isFirefox()) return 'vp8'; return preferredOr(preferredCodec, 'vp8'); @@ -111,20 +130,24 @@ const preferredOr = ( codec: PreferredCodec | undefined, fallback: PreferredCodec, ): PreferredCodec => { - if (!codec) return fallback; - if (!('getCapabilities' in RTCRtpSender)) return fallback; - const capabilities = RTCRtpSender.getCapabilities('video'); - if (!capabilities) return fallback; + return codec && isCodecSupported(`video/${codec}`) ? codec : fallback; +}; - // Safari and Firefox do not have a good support encoding to SVC codecs, - // so we disable it for them. - if (isSvcCodec(codec) && (isSafari() || isFirefox())) return fallback; +/** + * Returns whether the codec is supported by the platform. + * + * @param codecMimeType the codec to check. + */ +export const isCodecSupported = (codecMimeType: string): boolean => { + if (!('getCapabilities' in RTCRtpSender)) return false; + + codecMimeType = codecMimeType.toLowerCase(); + const [kind] = codecMimeType.split('/'); + const capabilities = RTCRtpSender.getCapabilities(kind); + if (!capabilities) return false; const { codecs } = capabilities; - const codecMimeType = `video/${codec}`.toLowerCase(); - return codecs.some((c) => c.mimeType.toLowerCase() === codecMimeType) - ? codec - : fallback; + return codecs.some((c) => c.mimeType.toLowerCase() === codecMimeType); }; /** From b4cf770c7357abac83e662f4db68d374f350d5b6 Mon Sep 17 00:00:00 2001 From: Oliver Lazoroski Date: Tue, 22 Oct 2024 23:21:58 +0200 Subject: [PATCH 05/52] fix: remove unnecessary iceRestart event handler on the subscriber --- packages/client/src/rtc/Subscriber.ts | 21 ---------- .../src/rtc/__tests__/Subscriber.test.ts | 39 +------------------ 2 files changed, 2 insertions(+), 58 deletions(-) diff --git a/packages/client/src/rtc/Subscriber.ts b/packages/client/src/rtc/Subscriber.ts index 229fcb3cc8..ea68a0ea8d 100644 --- a/packages/client/src/rtc/Subscriber.ts +++ b/packages/client/src/rtc/Subscriber.ts @@ -31,21 +31,12 @@ export class Subscriber { private state: CallState; private readonly unregisterOnSubscriberOffer: () => void; - private readonly unregisterOnIceRestart: () => void; private readonly onUnrecoverableError?: () => void; private isIceRestarting = false; /** * Constructs a new `Subscriber` instance. - * - * @param sfuClient the SFU client to use. - * @param dispatcher the dispatcher to use. - * @param state the state of the call. - * @param connectionConfig the connection configuration to use. - * @param iceRestartDelay the delay in milliseconds to wait before restarting ICE when connection goes to `disconnected` state. - * @param onUnrecoverableError a callback to call when an unrecoverable error occurs. - * @param logTag a tag to use for logging. */ constructor({ sfuClient, @@ -73,17 +64,6 @@ export class Subscriber { }); }, ); - - const iceRestartConcurrencyTag = Symbol('iceRestart'); - this.unregisterOnIceRestart = dispatcher.on('iceRestart', (iceRestart) => { - withoutConcurrency(iceRestartConcurrencyTag, async () => { - if (iceRestart.peerType !== PeerType.SUBSCRIBER) return; - await this.restartIce(); - }).catch((err) => { - this.logger('error', `ICERestart failed`, err); - this.onUnrecoverableError?.(); - }); - }); } /** @@ -124,7 +104,6 @@ export class Subscriber { */ detachEventHandlers = () => { this.unregisterOnSubscriberOffer(); - this.unregisterOnIceRestart(); this.pc.removeEventListener('icecandidate', this.onIceCandidate); this.pc.removeEventListener('track', this.handleOnTrack); diff --git a/packages/client/src/rtc/__tests__/Subscriber.test.ts b/packages/client/src/rtc/__tests__/Subscriber.test.ts index d8b16ca886..ce949358aa 100644 --- a/packages/client/src/rtc/__tests__/Subscriber.test.ts +++ b/packages/client/src/rtc/__tests__/Subscriber.test.ts @@ -1,12 +1,11 @@ import './mocks/webrtc.mocks'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { DispatchableMessage, Dispatcher } from '../Dispatcher'; +import { Dispatcher } from '../Dispatcher'; import { StreamSfuClient } from '../../StreamSfuClient'; import { Subscriber } from '../Subscriber'; import { CallState } from '../../store'; -import { SfuEvent } from '../../gen/video/sfu/event/events'; -import { PeerType, TrackType } from '../../gen/video/sfu/models/models'; +import { TrackType } from '../../gen/video/sfu/models/models'; import { IceTrickleBuffer } from '../IceTrickleBuffer'; import { StreamClient } from '../../coordinator/connection/client'; @@ -59,40 +58,6 @@ describe('Subscriber', () => { }); describe('Subscriber ICE restart', () => { - it('should perform ICE restart when iceRestart event is received', () => { - sfuClient.iceRestart = vi.fn(); - dispatcher.dispatch( - SfuEvent.create({ - eventPayload: { - oneofKind: 'iceRestart', - iceRestart: { - peerType: PeerType.SUBSCRIBER, - }, - }, - }) as DispatchableMessage<'iceRestart'>, - ); - - expect(sfuClient.iceRestart).toHaveBeenCalledWith({ - peerType: PeerType.SUBSCRIBER, - }); - }); - - it('should not perform ICE restart when iceRestart event is received for a different peer type', () => { - sfuClient.iceRestart = vi.fn(); - dispatcher.dispatch( - SfuEvent.create({ - eventPayload: { - oneofKind: 'iceRestart', - iceRestart: { - peerType: PeerType.PUBLISHER_UNSPECIFIED, - }, - }, - }) as DispatchableMessage<'iceRestart'>, - ); - - expect(sfuClient.iceRestart).not.toHaveBeenCalled(); - }); - it(`should drop consequent ICE restart requests`, async () => { sfuClient.iceRestart = vi.fn(); // @ts-ignore From 3de122bcd0ca189cabca3d7dc54424c591c9083a Mon Sep 17 00:00:00 2001 From: Oliver Lazoroski Date: Thu, 24 Oct 2024 14:11:47 +0200 Subject: [PATCH 06/52] fix: regenerate models --- packages/client/src/gen/video/sfu/models/models.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/client/src/gen/video/sfu/models/models.ts b/packages/client/src/gen/video/sfu/models/models.ts index 7d33a838b2..50c3877fd1 100644 --- a/packages/client/src/gen/video/sfu/models/models.ts +++ b/packages/client/src/gen/video/sfu/models/models.ts @@ -199,15 +199,15 @@ export interface VideoLayer { */ export interface Codec { /** - * @generated from protobuf field: string mime_type = 1; + * @generated from protobuf field: string mime_type = 10; */ mimeType: string; /** - * @generated from protobuf field: string scalability_mode = 2; + * @generated from protobuf field: string scalability_mode = 11; */ scalabilityMode: string; /** - * @generated from protobuf field: string fmtp = 3; + * @generated from protobuf field: string fmtp = 12; */ fmtp: string; } @@ -951,14 +951,14 @@ export const VideoLayer = new VideoLayer$Type(); class Codec$Type extends MessageType { constructor() { super('stream.video.sfu.models.Codec', [ - { no: 1, name: 'mime_type', kind: 'scalar', T: 9 /*ScalarType.STRING*/ }, + { no: 10, name: 'mime_type', kind: 'scalar', T: 9 /*ScalarType.STRING*/ }, { - no: 2, + no: 11, name: 'scalability_mode', kind: 'scalar', T: 9 /*ScalarType.STRING*/, }, - { no: 3, name: 'fmtp', kind: 'scalar', T: 9 /*ScalarType.STRING*/ }, + { no: 12, name: 'fmtp', kind: 'scalar', T: 9 /*ScalarType.STRING*/ }, ]); } } From 6c797754cdd84274b7c06a45c4f137044f597e2c Mon Sep 17 00:00:00 2001 From: Oliver Lazoroski Date: Thu, 31 Oct 2024 16:09:17 +0100 Subject: [PATCH 07/52] feat: prevent flickering when a new stream arrives for an existing participant --- packages/client/src/rtc/Subscriber.ts | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/packages/client/src/rtc/Subscriber.ts b/packages/client/src/rtc/Subscriber.ts index ea68a0ea8d..0f43e40e34 100644 --- a/packages/client/src/rtc/Subscriber.ts +++ b/packages/client/src/rtc/Subscriber.ts @@ -218,7 +218,15 @@ export class Subscriber { this.logger('error', `Unknown track type: ${rawTrackType}`); return; } + const previousStream = participantToUpdate[streamKindProp]; + + // replace the previous stream with the new one, prevents flickering + this.state.updateParticipant(participantToUpdate.sessionId, { + [streamKindProp]: primaryStream, + }); + + // now, dispose the previous stream if (previousStream) { this.logger( 'info', @@ -229,9 +237,6 @@ export class Subscriber { previousStream.removeTrack(t); }); } - this.state.updateParticipant(participantToUpdate.sessionId, { - [streamKindProp]: primaryStream, - }); }; private onIceCandidate = (e: RTCPeerConnectionIceEvent) => { From 4025e8e723ff07c861d3207e71339069dedeb9f5 Mon Sep 17 00:00:00 2001 From: Oliver Lazoroski Date: Thu, 31 Oct 2024 23:21:59 +0100 Subject: [PATCH 08/52] fix: prevent flickering of participant's video when migrating --- packages/client/src/rtc/Subscriber.ts | 35 +++++++++------------------ 1 file changed, 11 insertions(+), 24 deletions(-) diff --git a/packages/client/src/rtc/Subscriber.ts b/packages/client/src/rtc/Subscriber.ts index 229fcb3cc8..5d2f69d336 100644 --- a/packages/client/src/rtc/Subscriber.ts +++ b/packages/client/src/rtc/Subscriber.ts @@ -31,21 +31,12 @@ export class Subscriber { private state: CallState; private readonly unregisterOnSubscriberOffer: () => void; - private readonly unregisterOnIceRestart: () => void; private readonly onUnrecoverableError?: () => void; private isIceRestarting = false; /** * Constructs a new `Subscriber` instance. - * - * @param sfuClient the SFU client to use. - * @param dispatcher the dispatcher to use. - * @param state the state of the call. - * @param connectionConfig the connection configuration to use. - * @param iceRestartDelay the delay in milliseconds to wait before restarting ICE when connection goes to `disconnected` state. - * @param onUnrecoverableError a callback to call when an unrecoverable error occurs. - * @param logTag a tag to use for logging. */ constructor({ sfuClient, @@ -73,17 +64,6 @@ export class Subscriber { }); }, ); - - const iceRestartConcurrencyTag = Symbol('iceRestart'); - this.unregisterOnIceRestart = dispatcher.on('iceRestart', (iceRestart) => { - withoutConcurrency(iceRestartConcurrencyTag, async () => { - if (iceRestart.peerType !== PeerType.SUBSCRIBER) return; - await this.restartIce(); - }).catch((err) => { - this.logger('error', `ICERestart failed`, err); - this.onUnrecoverableError?.(); - }); - }); } /** @@ -124,7 +104,6 @@ export class Subscriber { */ detachEventHandlers = () => { this.unregisterOnSubscriberOffer(); - this.unregisterOnIceRestart(); this.pc.removeEventListener('icecandidate', this.onIceCandidate); this.pc.removeEventListener('track', this.handleOnTrack); @@ -239,7 +218,18 @@ export class Subscriber { this.logger('error', `Unknown track type: ${rawTrackType}`); return; } + + // get the previous stream to dispose it later + // usually this happens during migration, when the stream is replaced + // with a new one but the old one is still in the state const previousStream = participantToUpdate[streamKindProp]; + + // replace the previous stream with the new one, prevents flickering + this.state.updateParticipant(participantToUpdate.sessionId, { + [streamKindProp]: primaryStream, + }); + + // now, dispose the previous stream if it exists if (previousStream) { this.logger( 'info', @@ -250,9 +240,6 @@ export class Subscriber { previousStream.removeTrack(t); }); } - this.state.updateParticipant(participantToUpdate.sessionId, { - [streamKindProp]: primaryStream, - }); }; private onIceCandidate = (e: RTCPeerConnectionIceEvent) => { From aae6cf8aecbc742fd7e3fbfa2279441596b3b1ad Mon Sep 17 00:00:00 2001 From: Oliver Lazoroski Date: Thu, 31 Oct 2024 23:29:39 +0100 Subject: [PATCH 09/52] chore: adjust tests --- .../src/rtc/__tests__/Subscriber.test.ts | 39 +------------------ 1 file changed, 2 insertions(+), 37 deletions(-) diff --git a/packages/client/src/rtc/__tests__/Subscriber.test.ts b/packages/client/src/rtc/__tests__/Subscriber.test.ts index d8b16ca886..ce949358aa 100644 --- a/packages/client/src/rtc/__tests__/Subscriber.test.ts +++ b/packages/client/src/rtc/__tests__/Subscriber.test.ts @@ -1,12 +1,11 @@ import './mocks/webrtc.mocks'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { DispatchableMessage, Dispatcher } from '../Dispatcher'; +import { Dispatcher } from '../Dispatcher'; import { StreamSfuClient } from '../../StreamSfuClient'; import { Subscriber } from '../Subscriber'; import { CallState } from '../../store'; -import { SfuEvent } from '../../gen/video/sfu/event/events'; -import { PeerType, TrackType } from '../../gen/video/sfu/models/models'; +import { TrackType } from '../../gen/video/sfu/models/models'; import { IceTrickleBuffer } from '../IceTrickleBuffer'; import { StreamClient } from '../../coordinator/connection/client'; @@ -59,40 +58,6 @@ describe('Subscriber', () => { }); describe('Subscriber ICE restart', () => { - it('should perform ICE restart when iceRestart event is received', () => { - sfuClient.iceRestart = vi.fn(); - dispatcher.dispatch( - SfuEvent.create({ - eventPayload: { - oneofKind: 'iceRestart', - iceRestart: { - peerType: PeerType.SUBSCRIBER, - }, - }, - }) as DispatchableMessage<'iceRestart'>, - ); - - expect(sfuClient.iceRestart).toHaveBeenCalledWith({ - peerType: PeerType.SUBSCRIBER, - }); - }); - - it('should not perform ICE restart when iceRestart event is received for a different peer type', () => { - sfuClient.iceRestart = vi.fn(); - dispatcher.dispatch( - SfuEvent.create({ - eventPayload: { - oneofKind: 'iceRestart', - iceRestart: { - peerType: PeerType.PUBLISHER_UNSPECIFIED, - }, - }, - }) as DispatchableMessage<'iceRestart'>, - ); - - expect(sfuClient.iceRestart).not.toHaveBeenCalled(); - }); - it(`should drop consequent ICE restart requests`, async () => { sfuClient.iceRestart = vi.fn(); // @ts-ignore From f31351504f80f282b00dd41412f5a4ac0f291e15 Mon Sep 17 00:00:00 2001 From: Oliver Lazoroski Date: Fri, 1 Nov 2024 13:37:30 +0100 Subject: [PATCH 10/52] chore: reduce code duplication --- packages/client/src/rtc/BasePeerConnection.ts | 187 ++++++++++++++++++ packages/client/src/rtc/Publisher.ts | 148 ++------------ packages/client/src/rtc/Subscriber.ts | 151 ++------------ .../client/src/rtc/helpers/iceCandidate.ts | 16 -- 4 files changed, 212 insertions(+), 290 deletions(-) create mode 100644 packages/client/src/rtc/BasePeerConnection.ts delete mode 100644 packages/client/src/rtc/helpers/iceCandidate.ts diff --git a/packages/client/src/rtc/BasePeerConnection.ts b/packages/client/src/rtc/BasePeerConnection.ts new file mode 100644 index 0000000000..d34054fd65 --- /dev/null +++ b/packages/client/src/rtc/BasePeerConnection.ts @@ -0,0 +1,187 @@ +import { getLogger } from '../logger'; +import type { Logger } from '../coordinator/connection/types'; +import { CallingState, CallState } from '../store'; +import { PeerType } from '../gen/video/sfu/models/models'; +import { StreamSfuClient } from '../StreamSfuClient'; + +export type BasePeerConnectionOpts = { + sfuClient: StreamSfuClient; + state: CallState; + connectionConfig?: RTCConfiguration; + onUnrecoverableError?: () => void; + logTag: string; +}; + +/** + * A base class for the `Publisher` and `Subscriber` classes. + * @internal + */ +export abstract class BasePeerConnection { + protected readonly logger: Logger; + protected readonly state: CallState; + protected readonly pc: RTCPeerConnection; + protected readonly peerType: PeerType; + protected sfuClient: StreamSfuClient; + + protected readonly onUnrecoverableError?: () => void; + protected isIceRestarting = false; + + /** + * Constructs a new `BasePeerConnection` instance. + */ + protected constructor( + peerType: PeerType, + { + sfuClient, + connectionConfig, + state, + onUnrecoverableError, + logTag, + }: BasePeerConnectionOpts, + ) { + this.peerType = peerType; + this.sfuClient = sfuClient; + this.state = state; + this.onUnrecoverableError = onUnrecoverableError; + this.logger = getLogger([ + peerType === PeerType.SUBSCRIBER ? 'Subscriber' : 'Publisher', + logTag, + ]); + + this.pc = new RTCPeerConnection(connectionConfig); + this.pc.addEventListener('icecandidate', this.onIceCandidate); + this.pc.addEventListener('icecandidateerror', this.onIceCandidateError); + this.pc.addEventListener( + 'iceconnectionstatechange', + this.onIceConnectionStateChange, + ); + this.pc.addEventListener('icegatheringstatechange', this.onIceGatherChange); + this.pc.addEventListener('signalingstatechange', this.onSignalingChange); + } + + /** + * Disposes the `RTCPeerConnection` instance. + */ + protected dispose = () => { + this.detachEventHandlers(); + this.pc.close(); + }; + + /** + * Detaches the event handlers from the `RTCPeerConnection`. + */ + protected detachEventHandlers() { + this.pc.removeEventListener('icecandidate', this.onIceCandidate); + this.pc.removeEventListener('icecandidateerror', this.onIceCandidateError); + this.pc.removeEventListener('signalingstatechange', this.onSignalingChange); + this.pc.removeEventListener( + 'iceconnectionstatechange', + this.onIceConnectionStateChange, + ); + this.pc.removeEventListener( + 'icegatheringstatechange', + this.onIceGatherChange, + ); + } + + /** + * Performs an ICE restart on the `RTCPeerConnection`. + */ + protected abstract restartIce(): Promise; + + /** + * Sets the SFU client to use. + * + * @param sfuClient the SFU client to use. + */ + setSfuClient = (sfuClient: StreamSfuClient) => { + this.sfuClient = sfuClient; + }; + + /** + * Returns the result of the `RTCPeerConnection.getStats()` method + * @param selector an optional `MediaStreamTrack` to get the stats for. + */ + getStats = (selector?: MediaStreamTrack | null) => { + return this.pc.getStats(selector); + }; + + /** + * Handles the ICECandidate event and + * Initiates an ICE Trickle process with the SFU. + */ + private onIceCandidate = (e: RTCPeerConnectionIceEvent) => { + const { candidate } = e; + if (!candidate) { + this.logger('debug', 'null ice candidate'); + return; + } + + const iceCandidate = this.toJSON(candidate); + this.sfuClient + .iceTrickle({ peerType: this.peerType, iceCandidate }) + .catch((err) => this.logger('warn', `ICETrickle failed`, err)); + }; + + /** + * Converts the ICE candidate to a JSON string. + */ + private toJSON = (candidate: RTCIceCandidate): string => { + if (!candidate.usernameFragment) { + // react-native-webrtc doesn't include usernameFragment in the candidate + const segments = candidate.candidate.split(' '); + const ufragIndex = segments.findIndex((s) => s === 'ufrag') + 1; + const usernameFragment = segments[ufragIndex]; + return JSON.stringify({ ...candidate, usernameFragment }); + } + return JSON.stringify(candidate.toJSON()); + }; + + /** + * Handles the ICE connection state change event. + */ + private onIceConnectionStateChange = () => { + const state = this.pc.iceConnectionState; + this.logger('debug', `ICE connection state changed`, state); + + if (this.state.callingState === CallingState.RECONNECTING) return; + + // do nothing when ICE is restarting + if (this.isIceRestarting) return; + + if (state === 'failed' || state === 'disconnected') { + this.logger('debug', `Attempting to restart ICE`); + this.restartIce().catch((e) => { + this.logger('error', `ICE restart failed`, e); + this.onUnrecoverableError?.(); + }); + } + }; + + /** + * Handles the ICE candidate error event. + */ + private onIceCandidateError = (e: Event) => { + const errorMessage = + e instanceof RTCPeerConnectionIceErrorEvent && + `${e.errorCode}: ${e.errorText}`; + const iceState = this.pc.iceConnectionState; + const logLevel = + iceState === 'connected' || iceState === 'checking' ? 'debug' : 'warn'; + this.logger(logLevel, `ICE Candidate error`, errorMessage); + }; + + /** + * Handles the ICE gathering state change event. + */ + private onIceGatherChange = () => { + this.logger('debug', `ICE Gathering State`, this.pc.iceGatheringState); + }; + + /** + * Handles the signaling state change event. + */ + private onSignalingChange = () => { + this.logger('debug', `Signaling state changed`, this.pc.signalingState); + }; +} diff --git a/packages/client/src/rtc/Publisher.ts b/packages/client/src/rtc/Publisher.ts index a02024de6d..d6e06736f4 100644 --- a/packages/client/src/rtc/Publisher.ts +++ b/packages/client/src/rtc/Publisher.ts @@ -1,4 +1,7 @@ -import { StreamSfuClient } from '../StreamSfuClient'; +import { + BasePeerConnection, + BasePeerConnectionOpts, +} from './BasePeerConnection'; import { Codec, PeerType, @@ -6,7 +9,6 @@ import { TrackType, VideoLayer, } from '../gen/video/sfu/models/models'; -import { getIceCandidate } from './helpers/iceCandidate'; import { findOptimalScreenSharingLayers, findOptimalVideoLayers, @@ -21,23 +23,15 @@ import { isSvcCodec, } from './codecs'; import { trackTypeToParticipantStreamKey } from './helpers/tracks'; -import { CallingState, CallState } from '../store'; import { PublishOptions } from '../types'; import { enableHighQualityAudio, extractMid } from '../helpers/sdp-munging'; -import { Logger } from '../coordinator/connection/types'; -import { getLogger } from '../logger'; import { Dispatcher } from './Dispatcher'; import { VideoLayerSetting } from '../gen/video/sfu/event/events'; import { TargetResolutionResponse } from '../gen/shims'; import { withoutConcurrency } from '../helpers/concurrency'; -export type PublisherConstructorOpts = { - sfuClient: StreamSfuClient; - state: CallState; +export type PublisherConstructorOpts = BasePeerConnectionOpts & { dispatcher: Dispatcher; - connectionConfig?: RTCConfiguration; - onUnrecoverableError?: () => void; - logTag: string; }; /** @@ -45,10 +39,7 @@ export type PublisherConstructorOpts = { * * @internal */ -export class Publisher { - private readonly logger: Logger; - private pc: RTCPeerConnection; - private readonly state: CallState; +export class Publisher extends BasePeerConnection { private readonly transceiverCache = new Map(); private readonly trackLayersCache = new Map(); private readonly publishOptsForTrack = new Map(); @@ -66,27 +57,13 @@ export class Publisher { private readonly unsubscribeOnIceRestart: () => void; private readonly unsubscribeChangePublishQuality: () => void; - private readonly onUnrecoverableError?: () => void; - - private isIceRestarting = false; - private sfuClient: StreamSfuClient; /** * Constructs a new `Publisher` instance. */ - constructor({ - connectionConfig, - sfuClient, - dispatcher, - state, - onUnrecoverableError, - logTag, - }: PublisherConstructorOpts) { - this.logger = getLogger(['Publisher', logTag]); - this.pc = this.createPeerConnection(connectionConfig); - this.sfuClient = sfuClient; - this.state = state; - this.onUnrecoverableError = onUnrecoverableError; + constructor({ dispatcher, ...baseOptions }: PublisherConstructorOpts) { + super(PeerType.PUBLISHER_UNSPECIFIED, baseOptions); + this.pc.addEventListener('negotiationneeded', this.onNegotiationNeeded); this.unsubscribeOnIceRestart = dispatcher.on('iceRestart', (iceRestart) => { if (iceRestart.peerType !== PeerType.PUBLISHER_UNSPECIFIED) return; @@ -112,24 +89,6 @@ export class Publisher { ); } - private createPeerConnection = (connectionConfig?: RTCConfiguration) => { - const pc = new RTCPeerConnection(connectionConfig); - pc.addEventListener('icecandidate', this.onIceCandidate); - pc.addEventListener('negotiationneeded', this.onNegotiationNeeded); - - pc.addEventListener('icecandidateerror', this.onIceCandidateError); - pc.addEventListener( - 'iceconnectionstatechange', - this.onIceConnectionStateChange, - ); - pc.addEventListener( - 'icegatheringstatechange', - this.onIceGatheringStateChange, - ); - pc.addEventListener('signalingstatechange', this.onSignalingStateChange); - return pc; - }; - /** * Closes the publisher PeerConnection and cleans up the resources. */ @@ -140,8 +99,7 @@ export class Publisher { this.trackLayersCache.clear(); } - this.detachEventHandlers(); - this.pc.close(); + this.dispose(); }; /** @@ -149,26 +107,13 @@ export class Publisher { * This is useful when we want to replace the `RTCPeerConnection` * instance with a new one (in case of migration). */ - detachEventHandlers = () => { + detachEventHandlers() { this.unsubscribeOnIceRestart(); this.unsubscribeChangePublishQuality(); - this.pc.removeEventListener('icecandidate', this.onIceCandidate); + super.detachEventHandlers(); this.pc.removeEventListener('negotiationneeded', this.onNegotiationNeeded); - this.pc.removeEventListener('icecandidateerror', this.onIceCandidateError); - this.pc.removeEventListener( - 'iceconnectionstatechange', - this.onIceConnectionStateChange, - ); - this.pc.removeEventListener( - 'icegatheringstatechange', - this.onIceGatheringStateChange, - ); - this.pc.removeEventListener( - 'signalingstatechange', - this.onSignalingStateChange, - ); - }; + } /** * Starts publishing the given track of the given media stream. @@ -424,40 +369,6 @@ export class Publisher { this.logger('info', `Update publish quality, enabled rids:`, activeLayers); }; - /** - * Returns the result of the `RTCPeerConnection.getStats()` method - * @param selector - * @returns - */ - getStats = (selector?: MediaStreamTrack | null | undefined) => { - return this.pc.getStats(selector); - }; - - private onIceCandidate = (e: RTCPeerConnectionIceEvent) => { - const { candidate } = e; - if (!candidate) { - this.logger('debug', 'null ice candidate'); - return; - } - this.sfuClient - .iceTrickle({ - iceCandidate: getIceCandidate(candidate), - peerType: PeerType.PUBLISHER_UNSPECIFIED, - }) - .catch((err) => { - this.logger('warn', `ICETrickle failed`, err); - }); - }; - - /** - * Sets the SFU client to use. - * - * @param sfuClient the SFU client to use. - */ - setSfuClient = (sfuClient: StreamSfuClient) => { - this.sfuClient = sfuClient; - }; - /** * Restarts the ICE connection and renegotiates with the SFU. */ @@ -636,37 +547,4 @@ export class Publisher { ? findOptimalScreenSharingLayers(track, publishOpts, screenShareBitrate) : undefined; }; - - private onIceCandidateError = (e: Event) => { - const errorMessage = - e instanceof RTCPeerConnectionIceErrorEvent && - `${e.errorCode}: ${e.errorText}`; - const iceState = this.pc.iceConnectionState; - const logLevel = - iceState === 'connected' || iceState === 'checking' ? 'debug' : 'warn'; - this.logger(logLevel, `ICE Candidate error`, errorMessage); - }; - - private onIceConnectionStateChange = () => { - const state = this.pc.iceConnectionState; - this.logger('debug', `ICE Connection state changed to`, state); - - if (this.state.callingState === CallingState.RECONNECTING) return; - - if (state === 'failed' || state === 'disconnected') { - this.logger('debug', `Attempting to restart ICE`); - this.restartIce().catch((e) => { - this.logger('error', `ICE restart error`, e); - this.onUnrecoverableError?.(); - }); - } - }; - - private onIceGatheringStateChange = () => { - this.logger('debug', `ICE Gathering State`, this.pc.iceGatheringState); - }; - - private onSignalingStateChange = () => { - this.logger('debug', `Signaling state changed`, this.pc.signalingState); - }; } diff --git a/packages/client/src/rtc/Subscriber.ts b/packages/client/src/rtc/Subscriber.ts index 0f43e40e34..6008e9268e 100644 --- a/packages/client/src/rtc/Subscriber.ts +++ b/packages/client/src/rtc/Subscriber.ts @@ -1,20 +1,15 @@ -import { StreamSfuClient } from '../StreamSfuClient'; -import { getIceCandidate } from './helpers/iceCandidate'; +import { + BasePeerConnection, + BasePeerConnectionOpts, +} from './BasePeerConnection'; import { PeerType } from '../gen/video/sfu/models/models'; import { SubscriberOffer } from '../gen/video/sfu/event/events'; import { Dispatcher } from './Dispatcher'; -import { getLogger } from '../logger'; -import { CallingState, CallState } from '../store'; import { withoutConcurrency } from '../helpers/concurrency'; import { toTrackType, trackTypeToParticipantStreamKey } from './helpers/tracks'; -import { Logger } from '../coordinator/connection/types'; -export type SubscriberOpts = { - sfuClient: StreamSfuClient; +export type SubscriberOpts = BasePeerConnectionOpts & { dispatcher: Dispatcher; - state: CallState; - connectionConfig?: RTCConfiguration; - onUnrecoverableError?: () => void; logTag: string; }; @@ -24,34 +19,15 @@ export type SubscriberOpts = { * * @internal */ -export class Subscriber { - private readonly logger: Logger; - private pc: RTCPeerConnection; - private sfuClient: StreamSfuClient; - private state: CallState; - +export class Subscriber extends BasePeerConnection { private readonly unregisterOnSubscriberOffer: () => void; - private readonly onUnrecoverableError?: () => void; - - private isIceRestarting = false; /** * Constructs a new `Subscriber` instance. */ - constructor({ - sfuClient, - dispatcher, - state, - connectionConfig, - onUnrecoverableError, - logTag, - }: SubscriberOpts) { - this.logger = getLogger(['Subscriber', logTag]); - this.sfuClient = sfuClient; - this.state = state; - this.onUnrecoverableError = onUnrecoverableError; - - this.pc = this.createPeerConnection(connectionConfig); + constructor({ dispatcher, ...baseOptions }: SubscriberOpts) { + super(PeerType.SUBSCRIBER, baseOptions); + this.pc.addEventListener('track', this.handleOnTrack); const subscriberOfferConcurrencyTag = Symbol('subscriberOffer'); this.unregisterOnSubscriberOffer = dispatcher.on( @@ -66,29 +42,6 @@ export class Subscriber { ); } - /** - * Creates a new `RTCPeerConnection` instance with the given configuration. - * - * @param connectionConfig the connection configuration to use. - */ - private createPeerConnection = (connectionConfig?: RTCConfiguration) => { - const pc = new RTCPeerConnection(connectionConfig); - pc.addEventListener('icecandidate', this.onIceCandidate); - pc.addEventListener('track', this.handleOnTrack); - - pc.addEventListener('icecandidateerror', this.onIceCandidateError); - pc.addEventListener( - 'iceconnectionstatechange', - this.onIceConnectionStateChange, - ); - pc.addEventListener( - 'icegatheringstatechange', - this.onIceGatheringStateChange, - ); - - return pc; - }; - /** * Closes the `RTCPeerConnection` and unsubscribes from the dispatcher. */ @@ -102,39 +55,12 @@ export class Subscriber { * This is useful when we want to replace the `RTCPeerConnection` * instance with a new one (in case of migration). */ - detachEventHandlers = () => { + detachEventHandlers() { this.unregisterOnSubscriberOffer(); - this.pc.removeEventListener('icecandidate', this.onIceCandidate); + super.detachEventHandlers(); this.pc.removeEventListener('track', this.handleOnTrack); - this.pc.removeEventListener('icecandidateerror', this.onIceCandidateError); - this.pc.removeEventListener( - 'iceconnectionstatechange', - this.onIceConnectionStateChange, - ); - this.pc.removeEventListener( - 'icegatheringstatechange', - this.onIceGatheringStateChange, - ); - }; - - /** - * Returns the result of the `RTCPeerConnection.getStats()` method - * @param selector - * @returns - */ - getStats = (selector?: MediaStreamTrack | null | undefined) => { - return this.pc.getStats(selector); - }; - - /** - * Sets the SFU client to use. - * - * @param sfuClient the SFU client to use. - */ - setSfuClient = (sfuClient: StreamSfuClient) => { - this.sfuClient = sfuClient; - }; + } /** * Restarts the ICE connection and renegotiates with the SFU. @@ -239,23 +165,6 @@ export class Subscriber { } }; - private onIceCandidate = (e: RTCPeerConnectionIceEvent) => { - const { candidate } = e; - if (!candidate) { - this.logger('debug', 'null ice candidate'); - return; - } - - this.sfuClient - .iceTrickle({ - iceCandidate: getIceCandidate(candidate), - peerType: PeerType.SUBSCRIBER, - }) - .catch((err) => { - this.logger('warn', `ICETrickle failed`, err); - }); - }; - private negotiate = async (subscriberOffer: SubscriberOffer) => { this.logger('info', `Received subscriberOffer`, subscriberOffer); @@ -285,40 +194,4 @@ export class Subscriber { this.isIceRestarting = false; }; - - private onIceConnectionStateChange = () => { - const state = this.pc.iceConnectionState; - this.logger('debug', `ICE connection state changed`, state); - - if (this.state.callingState === CallingState.RECONNECTING) return; - - // do nothing when ICE is restarting - if (this.isIceRestarting) return; - - if (state === 'failed' || state === 'disconnected') { - this.logger('debug', `Attempting to restart ICE`); - this.restartIce().catch((e) => { - this.logger('error', `ICE restart failed`, e); - this.onUnrecoverableError?.(); - }); - } - }; - - private onIceGatheringStateChange = () => { - this.logger( - 'debug', - `ICE gathering state changed`, - this.pc.iceGatheringState, - ); - }; - - private onIceCandidateError = (e: Event) => { - const errorMessage = - e instanceof RTCPeerConnectionIceErrorEvent && - `${e.errorCode}: ${e.errorText}`; - const iceState = this.pc.iceConnectionState; - const logLevel = - iceState === 'connected' || iceState === 'checking' ? 'debug' : 'warn'; - this.logger(logLevel, `ICE Candidate error`, errorMessage); - }; } diff --git a/packages/client/src/rtc/helpers/iceCandidate.ts b/packages/client/src/rtc/helpers/iceCandidate.ts deleted file mode 100644 index 5b78cbe19f..0000000000 --- a/packages/client/src/rtc/helpers/iceCandidate.ts +++ /dev/null @@ -1,16 +0,0 @@ -import { ICETrickle } from '../../gen/video/sfu/models/models'; - -export function getIceCandidate( - candidate: RTCIceCandidate, -): ICETrickle['iceCandidate'] { - if (!candidate.usernameFragment) { - // react-native-webrtc doesn't include usernameFragment in the candidate - const splittedCandidate = candidate.candidate.split(' '); - const ufragIndex = - splittedCandidate.findIndex((s: string) => s === 'ufrag') + 1; - const usernameFragment = splittedCandidate[ufragIndex]; - return JSON.stringify({ ...candidate, usernameFragment }); - } else { - return JSON.stringify(candidate.toJSON()); - } -} From 08459e3469dd7ce060bf706587e007465c2593b3 Mon Sep 17 00:00:00 2001 From: Oliver Lazoroski Date: Fri, 1 Nov 2024 14:19:25 +0100 Subject: [PATCH 11/52] wip: server-side codec prioritization --- packages/client/src/Call.ts | 6 ++ packages/client/src/rtc/Publisher.ts | 20 +++--- packages/client/src/rtc/codecs.ts | 88 +++-------------------- packages/client/src/rtc/helpers/tracks.ts | 17 +++++ 4 files changed, 40 insertions(+), 91 deletions(-) diff --git a/packages/client/src/Call.ts b/packages/client/src/Call.ts index d4f3072a05..3f4aefb128 100644 --- a/packages/client/src/Call.ts +++ b/packages/client/src/Call.ts @@ -1458,6 +1458,12 @@ export class Call { * @param options the options to use. */ updatePublishOptions(options: PublishOptions) { + if (this.state.callingState === CallingState.JOINED) { + this.logger( + 'warn', + 'Cannot update publish options after joining the call', + ); + } this.publishOptions = { ...this.publishOptions, ...options }; } diff --git a/packages/client/src/rtc/Publisher.ts b/packages/client/src/rtc/Publisher.ts index d6e06736f4..4afac01778 100644 --- a/packages/client/src/rtc/Publisher.ts +++ b/packages/client/src/rtc/Publisher.ts @@ -17,12 +17,12 @@ import { toSvcEncodings, } from './videoLayers'; import { - getCodecPreferences, getOptimalVideoCodec, + getSupportedCodecs, isCodecSupported, isSvcCodec, } from './codecs'; -import { trackTypeToParticipantStreamKey } from './helpers/tracks'; +import { toTrackKind, trackTypeToParticipantStreamKey } from './helpers/tracks'; import { PublishOptions } from '../types'; import { enableHighQualityAudio, extractMid } from '../helpers/sdp-munging'; import { Dispatcher } from './Dispatcher'; @@ -43,7 +43,6 @@ export class Publisher extends BasePeerConnection { private readonly transceiverCache = new Map(); private readonly trackLayersCache = new Map(); private readonly publishOptsForTrack = new Map(); - private readonly codecsForTrack = new Map(); private readonly scalabilityModeForTrack = new Map(); /** @@ -185,7 +184,6 @@ export class Publisher extends BasePeerConnection { this.transceiverInitOrder.push(trackType); this.transceiverCache.set(trackType, transceiver); this.publishOptsForTrack.set(trackType, opts); - this.codecsForTrack.set(trackType, codecInUse); this.scalabilityModeForTrack.set( trackType, sendEncodings?.[0]?.scalabilityMode || '', @@ -501,13 +499,11 @@ export class Publisher extends BasePeerConnection { const transceiverInitIndex = this.transceiverInitOrder.indexOf(trackType); - let codecToUse: string | undefined = undefined; - if (trackType === TrackType.VIDEO) { - codecToUse = this.codecsForTrack.get(trackType); - } else if (trackType === TrackType.AUDIO) { - codecToUse = isRedEnabled ? 'red' : 'opus'; - } - const preferredCodecs = getCodecPreferences(trackType, codecToUse); + // FIXME OL: + // instead of sending all supported codecs, we should send a prioritized list + // [video/vp9, video/h264, video/vp8] or [audio/opus, audio/red] + const trackKind = toTrackKind(trackType); + const preferredCodecs = trackKind ? getSupportedCodecs(trackKind) : []; return { trackId: track.id, layers, @@ -517,7 +513,7 @@ export class Publisher extends BasePeerConnection { dtx: isAudioTrack && isDtxEnabled, red: isAudioTrack && isRedEnabled, muted: !isTrackLive, - preferredCodecs: (preferredCodecs || []).map((codec) => ({ + preferredCodecs: preferredCodecs.map((codec) => ({ mimeType: codec.mimeType, fmtp: codec.sdpFmtpLine || '', scalabilityMode: isSvcCodec(codec.mimeType) diff --git a/packages/client/src/rtc/codecs.ts b/packages/client/src/rtc/codecs.ts index 12f7410a50..b70929aa59 100644 --- a/packages/client/src/rtc/codecs.ts +++ b/packages/client/src/rtc/codecs.ts @@ -1,84 +1,19 @@ import { getOSInfo } from '../client-details'; import { isReactNative } from '../helpers/platforms'; import { isFirefox, isSafari } from '../helpers/browsers'; -import { TrackType } from '../gen/video/sfu/models/models'; import type { PreferredCodec } from '../types'; /** - * Returns back a list of sorted codecs, with the preferred codec first. - * - * @param preferredCodecMimeType the codec to prioritize (video/vp8, video/h264, video/vp9, video/av1...). + * Returns back a list of supported publish codecs for the given kind. */ -export const getPreferredCodecs = ( - preferredCodecMimeType: string, -): RTCRtpCodecCapability[] | undefined => { - if (!('getCapabilities' in RTCRtpSender)) return; - - const [kind] = preferredCodecMimeType.split('/'); +export const getSupportedCodecs = ( + kind: 'audio' | 'video', +): RTCRtpCodecCapability[] => { + if (!('getCapabilities' in RTCRtpSender)) return []; const capabilities = RTCRtpSender.getCapabilities(kind); - if (!capabilities) return; - - const preferred: RTCRtpCodecCapability[] = []; - const partiallyPreferred: RTCRtpCodecCapability[] = []; - const unpreferred: RTCRtpCodecCapability[] = []; - - preferredCodecMimeType = preferredCodecMimeType.toLowerCase(); - for (const codec of capabilities.codecs) { - const codecMimeType = codec.mimeType.toLowerCase(); - - const isPreferredCodec = codecMimeType === preferredCodecMimeType; - if (!isPreferredCodec) { - unpreferred.push(codec); - continue; - } - - // h264 is a special case, we want to prioritize the baseline codec with - // profile-level-id is 42e01f and packetization-mode=0 for maximum - // cross-browser compatibility. - // this branch covers the other cases, such as vp8. - if (codecMimeType !== 'video/h264') { - preferred.push(codec); - continue; - } - - const sdpFmtpLine = codec.sdpFmtpLine; - if (!sdpFmtpLine || !sdpFmtpLine.includes('profile-level-id=42e01f')) { - // this is not the baseline h264 codec, prioritize it lower - partiallyPreferred.push(codec); - continue; - } - - // packetization-mode mode is optional; when not present it defaults to 0: - // https://datatracker.ietf.org/doc/html/rfc6184#section-6.2 - if ( - sdpFmtpLine.includes('packetization-mode=0') || - !sdpFmtpLine.includes('packetization-mode') - ) { - preferred.unshift(codec); - } else { - preferred.push(codec); - } - } + if (!capabilities) return []; - // return a sorted list of codecs, with the preferred codecs first - return [...preferred, ...partiallyPreferred, ...unpreferred]; -}; - -/** - * Returns an ordered list of preferred codecs for the given track type. - * - * @param trackType the type of track. - * @param preferredCodec the preferred codec to prioritize. - */ -export const getCodecPreferences = ( - trackType: TrackType, - preferredCodec?: string, -): RTCRtpCodecCapability[] | undefined => { - return trackType === TrackType.VIDEO - ? getPreferredCodecs(`video/${preferredCodec || 'vp8'}`) - : trackType === TrackType.AUDIO - ? getPreferredCodecs(`audio/${preferredCodec || 'opus'}`) - : undefined; + return capabilities.codecs; }; /** @@ -139,14 +74,9 @@ const preferredOr = ( * @param codecMimeType the codec to check. */ export const isCodecSupported = (codecMimeType: string): boolean => { - if (!('getCapabilities' in RTCRtpSender)) return false; - codecMimeType = codecMimeType.toLowerCase(); - const [kind] = codecMimeType.split('/'); - const capabilities = RTCRtpSender.getCapabilities(kind); - if (!capabilities) return false; - - const { codecs } = capabilities; + const [kind] = codecMimeType.split('/') as ('audio' | 'video')[]; + const codecs = getSupportedCodecs(kind); return codecs.some((c) => c.mimeType.toLowerCase() === codecMimeType); }; diff --git a/packages/client/src/rtc/helpers/tracks.ts b/packages/client/src/rtc/helpers/tracks.ts index 8de63a345d..58e2f64162 100644 --- a/packages/client/src/rtc/helpers/tracks.ts +++ b/packages/client/src/rtc/helpers/tracks.ts @@ -57,3 +57,20 @@ export const toTrackType = (trackType: string): TrackType | undefined => { return undefined; } }; + +export const toTrackKind = ( + trackType: TrackType, +): 'audio' | 'video' | undefined => { + switch (trackType) { + case TrackType.AUDIO: + return 'audio'; + case TrackType.VIDEO: + return 'video'; + case TrackType.UNSPECIFIED: + case TrackType.SCREEN_SHARE: + case TrackType.SCREEN_SHARE_AUDIO: + return undefined; + default: + ensureExhausted(trackType, 'Unknown track type'); + } +}; From c00193ec2db9dbda15a82e63e9860fce99ee408e Mon Sep 17 00:00:00 2001 From: Oliver Lazoroski Date: Fri, 1 Nov 2024 14:21:31 +0100 Subject: [PATCH 12/52] fix: close -> dispose --- packages/client/src/rtc/Subscriber.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/client/src/rtc/Subscriber.ts b/packages/client/src/rtc/Subscriber.ts index 6008e9268e..f332eb354e 100644 --- a/packages/client/src/rtc/Subscriber.ts +++ b/packages/client/src/rtc/Subscriber.ts @@ -46,8 +46,7 @@ export class Subscriber extends BasePeerConnection { * Closes the `RTCPeerConnection` and unsubscribes from the dispatcher. */ close = () => { - this.detachEventHandlers(); - this.pc.close(); + this.dispose(); }; /** From ddf5f43e292bcbbcf2487589898f95a886e417c6 Mon Sep 17 00:00:00 2001 From: Oliver Lazoroski Date: Fri, 8 Nov 2024 20:11:24 +0100 Subject: [PATCH 13/52] feat: Codec Switching --- .../client/src/gen/video/sfu/event/events.ts | 31 ++++++++++++++++ packages/client/src/rtc/Dispatcher.ts | 1 + packages/client/src/rtc/Publisher.ts | 36 ++++++++++++++++--- 3 files changed, 64 insertions(+), 4 deletions(-) diff --git a/packages/client/src/gen/video/sfu/event/events.ts b/packages/client/src/gen/video/sfu/event/events.ts index 4519a1e0b0..6b216e4c4b 100644 --- a/packages/client/src/gen/video/sfu/event/events.ts +++ b/packages/client/src/gen/video/sfu/event/events.ts @@ -245,10 +245,24 @@ export interface SfuEvent { */ participantMigrationComplete: ParticipantMigrationComplete; } + | { + oneofKind: 'codecNegotiationComplete'; + /** + * CodecNegotiationComplete is sent to signal the completion of a codec negotiation. + * SDKs can safely stop previous transceivers + * + * @generated from protobuf field: stream.video.sfu.event.CodecNegotiationComplete codec_negotiation_complete = 26; + */ + codecNegotiationComplete: CodecNegotiationComplete; + } | { oneofKind: undefined; }; } +/** + * @generated from protobuf message stream.video.sfu.event.CodecNegotiationComplete + */ +export interface CodecNegotiationComplete {} /** * @generated from protobuf message stream.video.sfu.event.ParticipantMigrationComplete */ @@ -961,6 +975,13 @@ class SfuEvent$Type extends MessageType { oneof: 'eventPayload', T: () => ParticipantMigrationComplete, }, + { + no: 26, + name: 'codec_negotiation_complete', + kind: 'message', + oneof: 'eventPayload', + T: () => CodecNegotiationComplete, + }, ]); } } @@ -969,6 +990,16 @@ class SfuEvent$Type extends MessageType { */ export const SfuEvent = new SfuEvent$Type(); // @generated message type with reflection information, may provide speed optimized methods +class CodecNegotiationComplete$Type extends MessageType { + constructor() { + super('stream.video.sfu.event.CodecNegotiationComplete', []); + } +} +/** + * @generated MessageType for protobuf message stream.video.sfu.event.CodecNegotiationComplete + */ +export const CodecNegotiationComplete = new CodecNegotiationComplete$Type(); +// @generated message type with reflection information, may provide speed optimized methods class ParticipantMigrationComplete$Type extends MessageType { constructor() { super('stream.video.sfu.event.ParticipantMigrationComplete', []); diff --git a/packages/client/src/rtc/Dispatcher.ts b/packages/client/src/rtc/Dispatcher.ts index 31ee45c920..ccc48a2274 100644 --- a/packages/client/src/rtc/Dispatcher.ts +++ b/packages/client/src/rtc/Dispatcher.ts @@ -42,6 +42,7 @@ const sfuEventKinds: { [key in SfuEventKinds]: undefined } = { callEnded: undefined, participantUpdated: undefined, participantMigrationComplete: undefined, + codecNegotiationComplete: undefined, }; export const isSfuEvent = ( diff --git a/packages/client/src/rtc/Publisher.ts b/packages/client/src/rtc/Publisher.ts index aaf48c037a..708d483cb4 100644 --- a/packages/client/src/rtc/Publisher.ts +++ b/packages/client/src/rtc/Publisher.ts @@ -16,7 +16,7 @@ import { import { getOptimalVideoCodec, getPreferredCodecs, isSvcCodec } from './codecs'; import { trackTypeToParticipantStreamKey } from './helpers/tracks'; import { CallingState, CallState } from '../store'; -import { PublishOptions } from '../types'; +import { PreferredCodec, PublishOptions } from '../types'; import { enableHighQualityAudio, extractMid, @@ -61,6 +61,7 @@ export class Publisher { * @internal */ private readonly transceiverInitOrder: TrackType[] = []; + private readonly transceiverOrder: RTCRtpTransceiver[] = []; private readonly isDtxEnabled: boolean; private readonly isRedEnabled: boolean; @@ -71,6 +72,8 @@ export class Publisher { private isIceRestarting = false; private sfuClient: StreamSfuClient; + private readonly dispatcher: Dispatcher; + /** * Constructs a new `Publisher` instance. */ @@ -90,6 +93,7 @@ export class Publisher { this.state = state; this.isDtxEnabled = isDtxEnabled; this.isRedEnabled = isRedEnabled; + this.dispatcher = dispatcher; this.onUnrecoverableError = onUnrecoverableError; this.unsubscribeOnIceRestart = dispatcher.on('iceRestart', (iceRestart) => { @@ -246,6 +250,7 @@ export class Publisher { this.logger('debug', `Added ${TrackType[trackType]} transceiver`); this.transceiverInitOrder.push(trackType); + this.transceiverOrder.push(transceiver); this.transceiverCache.set(trackType, transceiver); this.publishOptsForTrack.set(trackType, opts); @@ -286,6 +291,24 @@ export class Publisher { await transceiver.sender.replaceTrack(track); }; + publishTrack = (codec: PreferredCodec) => { + const currentTransceiver = this.transceiverCache.get(TrackType.VIDEO); + if (!currentTransceiver || !currentTransceiver.sender.track) return; + const track = currentTransceiver.sender.track; + + const negotiationComplete = async () => { + this.dispatcher.off('codecNegotiationComplete', negotiationComplete); + this.logger('info', 'Codec negotiation complete'); + + await currentTransceiver.sender.replaceTrack(null); + currentTransceiver.stop(); + }; + this.dispatcher.on('codecNegotiationComplete', negotiationComplete); + + const ms = new MediaStream([track]); + this.addTransceiver(TrackType.VIDEO, track, { preferredCodec: codec }, ms); + }; + /** * Stops publishing the given track type to the SFU, if it is currently being published. * Underlying track will be stopped and removed from the publisher. @@ -586,11 +609,12 @@ export class Publisher { return this.pc .getTransceivers() .filter((t) => t.direction === 'sendonly' && t.sender.track) - .map((transceiver) => { + .map((transceiver) => { let trackType!: TrackType; this.transceiverCache.forEach((value, key) => { if (value === transceiver) trackType = key; }); + if (!trackType) return; const track = transceiver.sender.track!; let optimalLayers: OptimalVideoLayer[]; const isTrackLive = track.readyState === 'live'; @@ -627,17 +651,21 @@ export class Publisher { const isStereo = isAudioTrack && trackSettings.channelCount === 2; const transceiverInitIndex = this.transceiverInitOrder.indexOf(trackType); + const mid = this.transceiverOrder.indexOf(transceiver); + return { trackId: track.id, layers: layers, trackType, - mid: extractMid(transceiver, transceiverInitIndex, sdp), + mid: + String(mid) ?? extractMid(transceiver, transceiverInitIndex, sdp), stereo: isStereo, dtx: isAudioTrack && this.isDtxEnabled, red: isAudioTrack && this.isRedEnabled, muted: !isTrackLive, }; - }); + }) + .filter(Boolean) as TrackInfo[]; }; private computeLayers = ( From 8e3f91f4cc62fbbd424a39c734efd4a745156ed3 Mon Sep 17 00:00:00 2001 From: Oliver Lazoroski Date: Fri, 8 Nov 2024 20:14:45 +0100 Subject: [PATCH 14/52] fix: lint --- packages/client/src/rtc/Publisher.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/client/src/rtc/Publisher.ts b/packages/client/src/rtc/Publisher.ts index 708d483cb4..9b6c6fb904 100644 --- a/packages/client/src/rtc/Publisher.ts +++ b/packages/client/src/rtc/Publisher.ts @@ -614,7 +614,7 @@ export class Publisher { this.transceiverCache.forEach((value, key) => { if (value === transceiver) trackType = key; }); - if (!trackType) return; + if (!trackType) return undefined; const track = transceiver.sender.track!; let optimalLayers: OptimalVideoLayer[]; const isTrackLive = track.readyState === 'live'; From e38560d5b4cf43bd03b04132729baf2160d9690b Mon Sep 17 00:00:00 2001 From: thesyncim Date: Sun, 10 Nov 2024 14:03:48 +0000 Subject: [PATCH 15/52] do not stop the previous transceiver --- packages/client/src/rtc/Publisher.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/client/src/rtc/Publisher.ts b/packages/client/src/rtc/Publisher.ts index 9b6c6fb904..946f6f5f53 100644 --- a/packages/client/src/rtc/Publisher.ts +++ b/packages/client/src/rtc/Publisher.ts @@ -301,7 +301,6 @@ export class Publisher { this.logger('info', 'Codec negotiation complete'); await currentTransceiver.sender.replaceTrack(null); - currentTransceiver.stop(); }; this.dispatcher.on('codecNegotiationComplete', negotiationComplete); From 4f9c7de0e09233748e04ce508bbfdc5529346b48 Mon Sep 17 00:00:00 2001 From: Oliver Lazoroski Date: Mon, 11 Nov 2024 23:03:50 +0100 Subject: [PATCH 16/52] fix: update protocol --- packages/client/src/Call.ts | 21 +++++++++++++++++-- .../client/src/gen/video/sfu/event/events.ts | 20 ++++++++++++++++++ .../client/src/gen/video/sfu/models/models.ts | 11 ---------- packages/client/src/rtc/Subscriber.ts | 1 - packages/client/src/types.ts | 18 ++++++++++++++-- 5 files changed, 55 insertions(+), 16 deletions(-) diff --git a/packages/client/src/Call.ts b/packages/client/src/Call.ts index 3f4aefb128..3ee06ab140 100644 --- a/packages/client/src/Call.ts +++ b/packages/client/src/Call.ts @@ -82,6 +82,7 @@ import { CallConstructor, CallLeaveOptions, JoinCallData, + PreferredCodec, PublishOptions, TrackMuteType, VideoTrackType, @@ -786,18 +787,34 @@ export class Call { // prepare a generic SDP and send it to the SFU. // this is a throw-away SDP that the SFU will use to determine // the capabilities of the client (codec support, etc.) - const receivingCapabilitiesSdp = await getGenericSdp('recvonly'); + const [receivingCapabilitiesSdp, publishingCapabilitiesSdp] = + await Promise.all([ + getGenericSdp('recvonly'), + getGenericSdp('sendonly'), + ]); const reconnectDetails = this.reconnectStrategy !== WebsocketReconnectStrategy.UNSPECIFIED ? this.getReconnectDetails(data?.migrating_from, previousSessionId) : undefined; - const { callState, fastReconnectDeadlineSeconds } = await sfuClient.join({ + const { + callState, + fastReconnectDeadlineSeconds, + publishAudioCodec, + publishVideoCodec, + } = await sfuClient.join({ subscriberSdp: receivingCapabilitiesSdp, + publisherSdp: publishingCapabilitiesSdp, clientDetails, fastReconnect: performingFastReconnect, reconnectDetails, }); + this.updatePublishOptions({ + preferredAudioCodec: + publishAudioCodec?.mimeType.toLowerCase() as PreferredCodec, + preferredCodec: + publishVideoCodec?.mimeType.toLowerCase() as PreferredCodec, + }); this.fastReconnectDeadlineSeconds = fastReconnectDeadlineSeconds; if (callState) { this.state.updateFromSfuCallState( diff --git a/packages/client/src/gen/video/sfu/event/events.ts b/packages/client/src/gen/video/sfu/event/events.ts index 6b216e4c4b..12749d1c8c 100644 --- a/packages/client/src/gen/video/sfu/event/events.ts +++ b/packages/client/src/gen/video/sfu/event/events.ts @@ -453,6 +453,10 @@ export interface JoinRequest { * @generated from protobuf field: string subscriber_sdp = 3; */ subscriberSdp: string; + /** + * @generated from protobuf field: string publisher_sdp = 8; + */ + publisherSdp: string; /** * @generated from protobuf field: stream.video.sfu.models.ClientDetails client_details = 4; */ @@ -548,6 +552,14 @@ export interface JoinResponse { * @generated from protobuf field: int32 fast_reconnect_deadline_seconds = 3; */ fastReconnectDeadlineSeconds: number; + /** + * @generated from protobuf field: stream.video.sfu.models.Codec publish_audio_codec = 4; + */ + publishAudioCodec?: Codec; + /** + * @generated from protobuf field: stream.video.sfu.models.Codec publish_video_codec = 5; + */ + publishVideoCodec?: Codec; } /** * ParticipantJoined is fired when a user joins a call @@ -1232,6 +1244,12 @@ class JoinRequest$Type extends MessageType { kind: 'scalar', T: 9 /*ScalarType.STRING*/, }, + { + no: 8, + name: 'publisher_sdp', + kind: 'scalar', + T: 9 /*ScalarType.STRING*/, + }, { no: 4, name: 'client_details', @@ -1354,6 +1372,8 @@ class JoinResponse$Type extends MessageType { kind: 'scalar', T: 5 /*ScalarType.INT32*/, }, + { no: 4, name: 'publish_audio_codec', kind: 'message', T: () => Codec }, + { no: 5, name: 'publish_video_codec', kind: 'message', T: () => Codec }, ]); } } diff --git a/packages/client/src/gen/video/sfu/models/models.ts b/packages/client/src/gen/video/sfu/models/models.ts index fb58b7f145..91861f5c27 100644 --- a/packages/client/src/gen/video/sfu/models/models.ts +++ b/packages/client/src/gen/video/sfu/models/models.ts @@ -266,10 +266,6 @@ export interface TrackInfo { * @generated from protobuf field: bool muted = 10; */ muted: boolean; - /** - * @generated from protobuf field: repeated stream.video.sfu.models.Codec preferred_codecs = 11; - */ - preferredCodecs: Codec[]; } /** * @generated from protobuf message stream.video.sfu.models.Error @@ -1130,13 +1126,6 @@ class TrackInfo$Type extends MessageType { { no: 8, name: 'stereo', kind: 'scalar', T: 8 /*ScalarType.BOOL*/ }, { no: 9, name: 'red', kind: 'scalar', T: 8 /*ScalarType.BOOL*/ }, { no: 10, name: 'muted', kind: 'scalar', T: 8 /*ScalarType.BOOL*/ }, - { - no: 11, - name: 'preferred_codecs', - kind: 'message', - repeat: 1 /*RepeatType.PACKED*/, - T: () => Codec, - }, ]); } } diff --git a/packages/client/src/rtc/Subscriber.ts b/packages/client/src/rtc/Subscriber.ts index ef20967581..d3ba5fd1f3 100644 --- a/packages/client/src/rtc/Subscriber.ts +++ b/packages/client/src/rtc/Subscriber.ts @@ -10,7 +10,6 @@ import { toTrackType, trackTypeToParticipantStreamKey } from './helpers/tracks'; export type SubscriberOpts = BasePeerConnectionOpts & { dispatcher: Dispatcher; - logTag: string; }; /** diff --git a/packages/client/src/types.ts b/packages/client/src/types.ts index 3cb8a59858..da459b9b07 100644 --- a/packages/client/src/types.ts +++ b/packages/client/src/types.ts @@ -147,10 +147,20 @@ export type SubscriptionChanges = { }; /** - * A preferred codec to use when publishing a video track. + * A preferred codec to use when publishing a video or audio track. * @internal */ -export type PreferredCodec = 'vp8' | 'h264' | 'vp9' | 'av1'; +export type PreferredCodec = + | 'vp8' + | 'h264' + | 'vp9' + | 'av1' + | 'video/vp8' + | 'video/h264' + | 'video/vp9' + | 'video/av1' + | 'audio/opus' + | 'audio/red'; /** * A collection of track publication options. @@ -167,6 +177,10 @@ export type PublishOptions = { * Use with caution. */ forceCodec?: PreferredCodec; + /** + * The preferred audio codec to use when publishing the audio stream. + */ + preferredAudioCodec?: PreferredCodec; /** * The preferred scalability to use when publishing the video stream. * Applicable only for SVC codecs. From 3c34f1a5d7a2df3bc2104b457888bc955b9a7d3b Mon Sep 17 00:00:00 2001 From: Oliver Lazoroski Date: Tue, 12 Nov 2024 16:51:54 +0100 Subject: [PATCH 17/52] feat: handle ChangePublishOptions --- packages/client/src/Call.ts | 51 +++--- packages/client/src/devices/CameraManager.ts | 12 -- .../client/src/gen/video/sfu/event/events.ts | 60 ++++++- .../client/src/gen/video/sfu/models/models.ts | 124 ++++++++++++- packages/client/src/rtc/BasePeerConnection.ts | 9 +- packages/client/src/rtc/Dispatcher.ts | 1 + packages/client/src/rtc/Publisher.ts | 163 ++++++++---------- packages/client/src/rtc/Subscriber.ts | 11 +- .../src/rtc/__tests__/Publisher.test.ts | 15 +- .../src/rtc/__tests__/bitrateLookup.test.ts | 12 -- .../client/src/rtc/__tests__/codecs.test.ts | 143 --------------- .../src/rtc/__tests__/videoLayers.test.ts | 96 +++++++++-- packages/client/src/rtc/bitrateLookup.ts | 61 ------- packages/client/src/rtc/codecs.ts | 60 ------- packages/client/src/rtc/videoLayers.ts | 70 ++++---- packages/client/src/types.ts | 38 +--- 16 files changed, 403 insertions(+), 523 deletions(-) delete mode 100644 packages/client/src/rtc/__tests__/bitrateLookup.test.ts delete mode 100644 packages/client/src/rtc/__tests__/codecs.test.ts delete mode 100644 packages/client/src/rtc/bitrateLookup.ts diff --git a/packages/client/src/Call.ts b/packages/client/src/Call.ts index 3ee06ab140..076cd49b06 100644 --- a/packages/client/src/Call.ts +++ b/packages/client/src/Call.ts @@ -81,9 +81,8 @@ import { AudioTrackType, CallConstructor, CallLeaveOptions, + ClientPublishOptions, JoinCallData, - PreferredCodec, - PublishOptions, TrackMuteType, VideoTrackType, } from './types'; @@ -91,6 +90,7 @@ import { BehaviorSubject, Subject, takeWhile } from 'rxjs'; import { ReconnectDetails } from './gen/video/sfu/event/events'; import { ClientDetails, + PublishOptions, TrackType, WebsocketReconnectStrategy, } from './gen/video/sfu/models/models'; @@ -200,6 +200,7 @@ export class Call { */ private readonly dispatcher = new Dispatcher(); + private clientPublishOptions?: ClientPublishOptions; private publishOptions?: PublishOptions; private statsReporter?: StatsReporter; private sfuStatsReporter?: SfuStatsReporter; @@ -796,25 +797,16 @@ export class Call { this.reconnectStrategy !== WebsocketReconnectStrategy.UNSPECIFIED ? this.getReconnectDetails(data?.migrating_from, previousSessionId) : undefined; - const { - callState, - fastReconnectDeadlineSeconds, - publishAudioCodec, - publishVideoCodec, - } = await sfuClient.join({ - subscriberSdp: receivingCapabilitiesSdp, - publisherSdp: publishingCapabilitiesSdp, - clientDetails, - fastReconnect: performingFastReconnect, - reconnectDetails, - }); + const { callState, fastReconnectDeadlineSeconds, publishOptions } = + await sfuClient.join({ + subscriberSdp: receivingCapabilitiesSdp, + publisherSdp: publishingCapabilitiesSdp, + clientDetails, + fastReconnect: performingFastReconnect, + reconnectDetails, + }); - this.updatePublishOptions({ - preferredAudioCodec: - publishAudioCodec?.mimeType.toLowerCase() as PreferredCodec, - preferredCodec: - publishVideoCodec?.mimeType.toLowerCase() as PreferredCodec, - }); + this.publishOptions = publishOptions; this.fastReconnectDeadlineSeconds = fastReconnectDeadlineSeconds; if (callState) { this.state.updateFromSfuCallState( @@ -844,6 +836,7 @@ export class Call { connectionConfig, clientDetails, statsOptions, + publishOptions: this.publishOptions!, closePreviousInstances: !performingMigration, }); } @@ -936,6 +929,7 @@ export class Call { connectionConfig: RTCConfiguration; statsOptions: StatsOptions; clientDetails: ClientDetails; + publishOptions: PublishOptions; closePreviousInstances: boolean; }) => { const { @@ -943,6 +937,7 @@ export class Call { connectionConfig, clientDetails, statsOptions, + publishOptions, closePreviousInstances, } = opts; if (closePreviousInstances && this.subscriber) { @@ -977,6 +972,7 @@ export class Call { dispatcher: this.dispatcher, state: this.state, connectionConfig, + publishOptions, logTag: String(this.sfuClientTag), onUnrecoverableError: () => { this.reconnect(WebsocketReconnectStrategy.REJOIN).catch((err) => { @@ -1365,7 +1361,6 @@ export class Call { videoStream, videoTrack, TrackType.VIDEO, - this.publishOptions, ); }; @@ -1429,14 +1424,14 @@ export class Call { if (!this.trackPublishOrder.includes(TrackType.SCREEN_SHARE)) { this.trackPublishOrder.push(TrackType.SCREEN_SHARE); } - const opts: PublishOptions = { - screenShareSettings: this.screenShare.getSettings(), - }; + // const opts: ClientPublishOptions = { + // screenShareSettings: this.screenShare.getSettings(), + // }; await this.publisher.publishStream( screenShareStream, screenShareTrack, TrackType.SCREEN_SHARE, - opts, + // opts, ); const [screenShareAudioTrack] = screenShareStream.getAudioTracks(); @@ -1448,7 +1443,7 @@ export class Call { screenShareStream, screenShareAudioTrack, TrackType.SCREEN_SHARE_AUDIO, - opts, + // opts, ); } }; @@ -1474,14 +1469,14 @@ export class Call { * @internal * @param options the options to use. */ - updatePublishOptions(options: PublishOptions) { + updatePublishOptions(options: ClientPublishOptions) { if (this.state.callingState === CallingState.JOINED) { this.logger( 'warn', 'Cannot update publish options after joining the call', ); } - this.publishOptions = { ...this.publishOptions, ...options }; + this.clientPublishOptions = { ...this.clientPublishOptions, ...options }; } /** diff --git a/packages/client/src/devices/CameraManager.ts b/packages/client/src/devices/CameraManager.ts index fc63d9974b..0a5b749eeb 100644 --- a/packages/client/src/devices/CameraManager.ts +++ b/packages/client/src/devices/CameraManager.ts @@ -4,7 +4,6 @@ import { CameraDirection, CameraManagerState } from './CameraManagerState'; import { InputMediaDeviceManager } from './InputMediaDeviceManager'; import { getVideoDevices, getVideoStream } from './devices'; import { TrackType } from '../gen/video/sfu/models/models'; -import { PreferredCodec } from '../types'; import { isMobile } from '../compatibility'; import { isReactNative } from '../helpers/platforms'; @@ -85,17 +84,6 @@ export class CameraManager extends InputMediaDeviceManager { } } - /** - * Sets the preferred codec for encoding the video. - * - * @internal internal use only, not part of the public API. - * @deprecated use {@link call.updatePublishOptions} instead. - * @param codec the codec to use for encoding the video. - */ - setPreferredCodec(codec: PreferredCodec | undefined) { - this.call.updatePublishOptions({ preferredCodec: codec }); - } - protected getDevices(): Observable { return getVideoDevices(); } diff --git a/packages/client/src/gen/video/sfu/event/events.ts b/packages/client/src/gen/video/sfu/event/events.ts index 12749d1c8c..ffbc4aec01 100644 --- a/packages/client/src/gen/video/sfu/event/events.ts +++ b/packages/client/src/gen/video/sfu/event/events.ts @@ -17,6 +17,8 @@ import { ParticipantCount, PeerType, Pin, + PublishOption, + PublishOptions, TrackInfo, TrackType, TrackUnpublishReason, @@ -255,10 +257,28 @@ export interface SfuEvent { */ codecNegotiationComplete: CodecNegotiationComplete; } + | { + oneofKind: 'changePublishOptions'; + /** + * ChangePublishOptions is sent to signal the change in publish options such as a new codec or simulcast layers + * + * @generated from protobuf field: stream.video.sfu.event.ChangePublishOptions change_publish_options = 27; + */ + changePublishOptions: ChangePublishOptions; + } | { oneofKind: undefined; }; } +/** + * @generated from protobuf message stream.video.sfu.event.ChangePublishOptions + */ +export interface ChangePublishOptions { + /** + * @generated from protobuf field: stream.video.sfu.models.PublishOption publish_option = 1; + */ + publishOption?: PublishOption; +} /** * @generated from protobuf message stream.video.sfu.event.CodecNegotiationComplete */ @@ -553,13 +573,9 @@ export interface JoinResponse { */ fastReconnectDeadlineSeconds: number; /** - * @generated from protobuf field: stream.video.sfu.models.Codec publish_audio_codec = 4; - */ - publishAudioCodec?: Codec; - /** - * @generated from protobuf field: stream.video.sfu.models.Codec publish_video_codec = 5; + * @generated from protobuf field: stream.video.sfu.models.PublishOptions publish_options = 4; */ - publishVideoCodec?: Codec; + publishOptions?: PublishOptions; } /** * ParticipantJoined is fired when a user joins a call @@ -994,6 +1010,13 @@ class SfuEvent$Type extends MessageType { oneof: 'eventPayload', T: () => CodecNegotiationComplete, }, + { + no: 27, + name: 'change_publish_options', + kind: 'message', + oneof: 'eventPayload', + T: () => ChangePublishOptions, + }, ]); } } @@ -1002,6 +1025,23 @@ class SfuEvent$Type extends MessageType { */ export const SfuEvent = new SfuEvent$Type(); // @generated message type with reflection information, may provide speed optimized methods +class ChangePublishOptions$Type extends MessageType { + constructor() { + super('stream.video.sfu.event.ChangePublishOptions', [ + { + no: 1, + name: 'publish_option', + kind: 'message', + T: () => PublishOption, + }, + ]); + } +} +/** + * @generated MessageType for protobuf message stream.video.sfu.event.ChangePublishOptions + */ +export const ChangePublishOptions = new ChangePublishOptions$Type(); +// @generated message type with reflection information, may provide speed optimized methods class CodecNegotiationComplete$Type extends MessageType { constructor() { super('stream.video.sfu.event.CodecNegotiationComplete', []); @@ -1372,8 +1412,12 @@ class JoinResponse$Type extends MessageType { kind: 'scalar', T: 5 /*ScalarType.INT32*/, }, - { no: 4, name: 'publish_audio_codec', kind: 'message', T: () => Codec }, - { no: 5, name: 'publish_video_codec', kind: 'message', T: () => Codec }, + { + no: 4, + name: 'publish_options', + kind: 'message', + T: () => PublishOptions, + }, ]); } } diff --git a/packages/client/src/gen/video/sfu/models/models.ts b/packages/client/src/gen/video/sfu/models/models.ts index 91861f5c27..061d159bf1 100644 --- a/packages/client/src/gen/video/sfu/models/models.ts +++ b/packages/client/src/gen/video/sfu/models/models.ts @@ -194,18 +194,64 @@ export interface VideoLayer { */ quality: VideoQuality; } +/** + * @generated from protobuf message stream.video.sfu.models.PublishOptions + */ +export interface PublishOptions { + /** + * @generated from protobuf field: repeated stream.video.sfu.models.PublishOption codecs = 1; + */ + codecs: PublishOption[]; +} +/** + * @generated from protobuf message stream.video.sfu.models.PublishOption + */ +export interface PublishOption { + /** + * @generated from protobuf field: stream.video.sfu.models.TrackType track_type = 1; + */ + trackType: TrackType; + /** + * @generated from protobuf field: stream.video.sfu.models.Codec codec = 2; + */ + codec?: Codec; + /** + * @generated from protobuf field: int32 bitrate = 3; + */ + bitrate: number; + /** + * @generated from protobuf field: int32 fps = 4; + */ + fps: number; + /** + * @generated from protobuf field: int32 max_spatial_layers = 5; + */ + maxSpatialLayers: number; + /** + * @generated from protobuf field: int32 max_temporal_layers = 6; + */ + maxTemporalLayers: number; +} /** * @generated from protobuf message stream.video.sfu.models.Codec */ export interface Codec { /** - * @generated from protobuf field: string mime_type = 10; + * @generated from protobuf field: uint32 payload_type = 11; + */ + payloadType: number; + /** + * @generated from protobuf field: string name = 10; + */ + name: string; + /** + * @generated from protobuf field: uint32 clock_rate = 14; */ - mimeType: string; + clockRate: number; /** - * @generated from protobuf field: string scalability_mode = 11; + * @generated from protobuf field: string encoding_parameters = 13; */ - scalabilityMode: string; + encodingParameters: string; /** * @generated from protobuf field: string fmtp = 12; */ @@ -1057,13 +1103,79 @@ class VideoLayer$Type extends MessageType { */ export const VideoLayer = new VideoLayer$Type(); // @generated message type with reflection information, may provide speed optimized methods +class PublishOptions$Type extends MessageType { + constructor() { + super('stream.video.sfu.models.PublishOptions', [ + { + no: 1, + name: 'codecs', + kind: 'message', + repeat: 1 /*RepeatType.PACKED*/, + T: () => PublishOption, + }, + ]); + } +} +/** + * @generated MessageType for protobuf message stream.video.sfu.models.PublishOptions + */ +export const PublishOptions = new PublishOptions$Type(); +// @generated message type with reflection information, may provide speed optimized methods +class PublishOption$Type extends MessageType { + constructor() { + super('stream.video.sfu.models.PublishOption', [ + { + no: 1, + name: 'track_type', + kind: 'enum', + T: () => [ + 'stream.video.sfu.models.TrackType', + TrackType, + 'TRACK_TYPE_', + ], + }, + { no: 2, name: 'codec', kind: 'message', T: () => Codec }, + { no: 3, name: 'bitrate', kind: 'scalar', T: 5 /*ScalarType.INT32*/ }, + { no: 4, name: 'fps', kind: 'scalar', T: 5 /*ScalarType.INT32*/ }, + { + no: 5, + name: 'max_spatial_layers', + kind: 'scalar', + T: 5 /*ScalarType.INT32*/, + }, + { + no: 6, + name: 'max_temporal_layers', + kind: 'scalar', + T: 5 /*ScalarType.INT32*/, + }, + ]); + } +} +/** + * @generated MessageType for protobuf message stream.video.sfu.models.PublishOption + */ +export const PublishOption = new PublishOption$Type(); +// @generated message type with reflection information, may provide speed optimized methods class Codec$Type extends MessageType { constructor() { super('stream.video.sfu.models.Codec', [ - { no: 10, name: 'mime_type', kind: 'scalar', T: 9 /*ScalarType.STRING*/ }, { no: 11, - name: 'scalability_mode', + name: 'payload_type', + kind: 'scalar', + T: 13 /*ScalarType.UINT32*/, + }, + { no: 10, name: 'name', kind: 'scalar', T: 9 /*ScalarType.STRING*/ }, + { + no: 14, + name: 'clock_rate', + kind: 'scalar', + T: 13 /*ScalarType.UINT32*/, + }, + { + no: 13, + name: 'encoding_parameters', kind: 'scalar', T: 9 /*ScalarType.STRING*/, }, diff --git a/packages/client/src/rtc/BasePeerConnection.ts b/packages/client/src/rtc/BasePeerConnection.ts index d34054fd65..e38b228afe 100644 --- a/packages/client/src/rtc/BasePeerConnection.ts +++ b/packages/client/src/rtc/BasePeerConnection.ts @@ -3,11 +3,13 @@ import type { Logger } from '../coordinator/connection/types'; import { CallingState, CallState } from '../store'; import { PeerType } from '../gen/video/sfu/models/models'; import { StreamSfuClient } from '../StreamSfuClient'; +import { Dispatcher } from './Dispatcher'; export type BasePeerConnectionOpts = { sfuClient: StreamSfuClient; state: CallState; connectionConfig?: RTCConfiguration; + dispatcher: Dispatcher; onUnrecoverableError?: () => void; logTag: string; }; @@ -18,9 +20,10 @@ export type BasePeerConnectionOpts = { */ export abstract class BasePeerConnection { protected readonly logger: Logger; - protected readonly state: CallState; - protected readonly pc: RTCPeerConnection; protected readonly peerType: PeerType; + protected readonly pc: RTCPeerConnection; + protected readonly state: CallState; + protected readonly dispatcher: Dispatcher; protected sfuClient: StreamSfuClient; protected readonly onUnrecoverableError?: () => void; @@ -35,6 +38,7 @@ export abstract class BasePeerConnection { sfuClient, connectionConfig, state, + dispatcher, onUnrecoverableError, logTag, }: BasePeerConnectionOpts, @@ -42,6 +46,7 @@ export abstract class BasePeerConnection { this.peerType = peerType; this.sfuClient = sfuClient; this.state = state; + this.dispatcher = dispatcher; this.onUnrecoverableError = onUnrecoverableError; this.logger = getLogger([ peerType === PeerType.SUBSCRIBER ? 'Subscriber' : 'Publisher', diff --git a/packages/client/src/rtc/Dispatcher.ts b/packages/client/src/rtc/Dispatcher.ts index ccc48a2274..5af5ce2ab1 100644 --- a/packages/client/src/rtc/Dispatcher.ts +++ b/packages/client/src/rtc/Dispatcher.ts @@ -43,6 +43,7 @@ const sfuEventKinds: { [key in SfuEventKinds]: undefined } = { participantUpdated: undefined, participantMigrationComplete: undefined, codecNegotiationComplete: undefined, + changePublishOptions: undefined, }; export const isSfuEvent = ( diff --git a/packages/client/src/rtc/Publisher.ts b/packages/client/src/rtc/Publisher.ts index cf33026766..0134cb8afd 100644 --- a/packages/client/src/rtc/Publisher.ts +++ b/packages/client/src/rtc/Publisher.ts @@ -4,6 +4,8 @@ import { } from './BasePeerConnection'; import { PeerType, + PublishOption, + PublishOptions, TrackInfo, TrackType, VideoLayer, @@ -15,17 +17,15 @@ import { ridToVideoQuality, toSvcEncodings, } from './videoLayers'; -import { getOptimalVideoCodec, isCodecSupported, isSvcCodec } from './codecs'; +import { isSvcCodec } from './codecs'; import { trackTypeToParticipantStreamKey } from './helpers/tracks'; -import { PreferredCodec, PublishOptions } from '../types'; import { enableHighQualityAudio, extractMid } from '../helpers/sdp-munging'; -import { Dispatcher } from './Dispatcher'; import { VideoLayerSetting } from '../gen/video/sfu/event/events'; import { TargetResolutionResponse } from '../gen/shims'; import { withoutConcurrency } from '../helpers/concurrency'; export type PublisherConstructorOpts = BasePeerConnectionOpts & { - dispatcher: Dispatcher; + publishOptions: PublishOptions; }; /** @@ -36,41 +36,40 @@ export type PublisherConstructorOpts = BasePeerConnectionOpts & { export class Publisher extends BasePeerConnection { private readonly transceiverCache = new Map(); private readonly trackLayersCache = new Map(); - private readonly publishOptsForTrack = new Map(); - private readonly scalabilityModeForTrack = new Map(); /** * An array maintaining the order how transceivers were added to the peer connection. * This is needed because some browsers (Firefox) don't reliably report * trackId and `mid` parameters. - * - * @internal */ - private readonly transceiverInitOrder: TrackType[] = []; private readonly transceiverOrder: RTCRtpTransceiver[] = []; private readonly unsubscribeOnIceRestart: () => void; private readonly unsubscribeChangePublishQuality: () => void; + private readonly unsubscribeChangePublishOptions: () => void; - private readonly dispatcher: Dispatcher; + private publishOptions: PublishOptions; /** * Constructs a new `Publisher` instance. */ - constructor({ dispatcher, ...baseOptions }: PublisherConstructorOpts) { + constructor({ publishOptions, ...baseOptions }: PublisherConstructorOpts) { super(PeerType.PUBLISHER_UNSPECIFIED, baseOptions); - this.dispatcher = dispatcher; + this.publishOptions = publishOptions; this.pc.addEventListener('negotiationneeded', this.onNegotiationNeeded); - this.unsubscribeOnIceRestart = dispatcher.on('iceRestart', (iceRestart) => { - if (iceRestart.peerType !== PeerType.PUBLISHER_UNSPECIFIED) return; - this.restartIce().catch((err) => { - this.logger('warn', `ICERestart failed`, err); - this.onUnrecoverableError?.(); - }); - }); + this.unsubscribeOnIceRestart = this.dispatcher.on( + 'iceRestart', + (iceRestart) => { + if (iceRestart.peerType !== PeerType.PUBLISHER_UNSPECIFIED) return; + this.restartIce().catch((err) => { + this.logger('warn', `ICERestart failed`, err); + this.onUnrecoverableError?.(); + }); + }, + ); - this.unsubscribeChangePublishQuality = dispatcher.on( + this.unsubscribeChangePublishQuality = this.dispatcher.on( 'changePublishQuality', ({ videoSenders }) => { withoutConcurrency('publisher.changePublishQuality', async () => { @@ -84,6 +83,17 @@ export class Publisher extends BasePeerConnection { }); }, ); + + this.unsubscribeChangePublishOptions = this.dispatcher.on( + 'changePublishOptions', + (event) => { + const { publishOption } = event; + if (!publishOption) return; + this.publishOptions.codecs = this.publishOptions.codecs.map((option) => + option.trackType === publishOption.trackType ? publishOption : option, + ); + }, + ); } /** @@ -107,6 +117,7 @@ export class Publisher extends BasePeerConnection { detachEventHandlers() { this.unsubscribeOnIceRestart(); this.unsubscribeChangePublishQuality(); + this.unsubscribeChangePublishOptions(); super.detachEventHandlers(); this.pc.removeEventListener('negotiationneeded', this.onNegotiationNeeded); @@ -121,13 +132,11 @@ export class Publisher extends BasePeerConnection { * @param mediaStream the media stream to publish. * @param track the track to publish. * @param trackType the track type to publish. - * @param opts the optional publish options to use. */ publishStream = async ( mediaStream: MediaStream, track: MediaStreamTrack, trackType: TrackType, - opts: PublishOptions = {}, ) => { if (track.readyState === 'ended') { throw new Error(`Can't publish a track that has ended already.`); @@ -149,7 +158,7 @@ export class Publisher extends BasePeerConnection { ); }; track.addEventListener('ended', handleTrackEnded); - this.addTransceiver(trackType, track, opts); + this.addTransceiver(trackType, track); } else { await this.updateTransceiver(transceiver, track); } @@ -162,15 +171,11 @@ export class Publisher extends BasePeerConnection { * This needs to be called when a new track kind is added to the peer connection. * In other cases, use `updateTransceiver` method. */ - private addTransceiver = ( - trackType: TrackType, - track: MediaStreamTrack, - opts: PublishOptions, - ) => { - const { forceCodec, preferredCodec } = opts; - const codecInUse = forceCodec || getOptimalVideoCodec(preferredCodec); - const videoEncodings = this.computeLayers(trackType, track, opts); - const sendEncodings = isSvcCodec(codecInUse) + private addTransceiver = (trackType: TrackType, track: MediaStreamTrack) => { + const publishOption = this.getPublishOptionFor(trackType); + + const videoEncodings = this.computeLayers(trackType, track, publishOption); + const sendEncodings = isSvcCodec(publishOption.codec?.name) ? toSvcEncodings(videoEncodings) : videoEncodings; const transceiver = this.pc.addTransceiver(track, { @@ -179,14 +184,8 @@ export class Publisher extends BasePeerConnection { }); this.logger('debug', `Added ${TrackType[trackType]} transceiver`); - this.transceiverInitOrder.push(trackType); this.transceiverOrder.push(transceiver); this.transceiverCache.set(trackType, transceiver); - this.publishOptsForTrack.set(trackType, opts); - this.scalabilityModeForTrack.set( - trackType, - sendEncodings?.[0]?.scalabilityMode || '', - ); }; /** @@ -205,7 +204,7 @@ export class Publisher extends BasePeerConnection { await transceiver.sender.replaceTrack(track); }; - publishTrack = (codec: PreferredCodec) => { + publishTrack = () => { const currentTransceiver = this.transceiverCache.get(TrackType.VIDEO); if (!currentTransceiver || !currentTransceiver.sender.track) return; const track = currentTransceiver.sender.track; @@ -218,7 +217,7 @@ export class Publisher extends BasePeerConnection { }; this.dispatcher.on('codecNegotiationComplete', negotiationComplete); - this.addTransceiver(TrackType.VIDEO, track, { preferredCodec: codec }); + this.addTransceiver(TrackType.VIDEO, track); }; /** @@ -229,20 +228,16 @@ export class Publisher extends BasePeerConnection { */ unpublishStream = async (trackType: TrackType, stopTrack: boolean) => { const transceiver = this.transceiverCache.get(trackType); - if ( - transceiver && - transceiver.sender.track && - (stopTrack - ? transceiver.sender.track.readyState === 'live' - : transceiver.sender.track.enabled) - ) { - stopTrack - ? transceiver.sender.track.stop() - : (transceiver.sender.track.enabled = false); - // We don't need to notify SFU if unpublishing in response to remote soft mute - if (this.state.localParticipant?.publishedTracks.includes(trackType)) { - await this.notifyTrackMuteStateChanged(undefined, trackType, true); - } + if (!transceiver || !transceiver.sender.track) return; + + if (stopTrack && transceiver.sender.track.readyState === 'live') { + transceiver.sender.track.stop(); + } else if (transceiver.sender.track.enabled) { + transceiver.sender.track.enabled = false; + } + + if (this.state.localParticipant?.publishedTracks.includes(trackType)) { + await this.notifyTrackMuteStateChanged(undefined, trackType, true); } }; @@ -253,9 +248,9 @@ export class Publisher extends BasePeerConnection { */ isPublishing = (trackType: TrackType): boolean => { const transceiver = this.transceiverCache.get(trackType); - if (!transceiver || !transceiver.sender) return false; + if (!transceiver || !transceiver.sender.track) return false; const track = transceiver.sender.track; - return !!track && track.readyState === 'live' && track.enabled; + return track.readyState === 'live' && track.enabled; }; private notifyTrackMuteStateChanged = async ( @@ -368,7 +363,6 @@ export class Publisher extends BasePeerConnection { if (scalabilityMode && scalabilityMode !== encoder.scalabilityMode) { // @ts-expect-error scalabilityMode is not in the typedefs yet encoder.scalabilityMode = scalabilityMode; - this.scalabilityModeForTrack.set(trackType, scalabilityMode); changed = true; } } @@ -450,9 +444,7 @@ export class Publisher extends BasePeerConnection { const transceiver = this.transceiverCache.get(TrackType.SCREEN_SHARE_AUDIO); if (!transceiver) return sdp; - const transceiverInitIndex = this.transceiverInitOrder.indexOf( - TrackType.SCREEN_SHARE_AUDIO, - ); + const transceiverInitIndex = this.transceiverOrder.indexOf(transceiver); const mid = extractMid(transceiver, transceiverInitIndex, sdp); return enableHighQualityAudio(sdp, mid); }; @@ -473,22 +465,16 @@ export class Publisher extends BasePeerConnection { this.transceiverCache.forEach((value, key) => { if (value === transceiver) trackType = key; }); + if (!trackType) return undefined; const track = transceiver.sender.track!; - let optimalLayers: OptimalVideoLayer[]; + + const publishOption = this.getPublishOptionFor(trackType); const isTrackLive = track.readyState === 'live'; - if (isTrackLive) { - optimalLayers = this.computeLayers(trackType, track) || []; - this.trackLayersCache.set(trackType, optimalLayers); - } else { - // we report the last known optimal layers for ended tracks - optimalLayers = this.trackLayersCache.get(trackType) || []; - this.logger( - 'debug', - `Track ${TrackType[trackType]} is ended. Announcing last known optimal layers`, - optimalLayers, - ); - } + const optimalLayers = isTrackLive + ? this.computeLayers(trackType, track, publishOption) || [] + : this.trackLayersCache.get(trackType) || []; + this.trackLayersCache.set(trackType, optimalLayers); const layers = optimalLayers.map((optimalLayer) => ({ rid: optimalLayer.rid || '', @@ -507,22 +493,18 @@ export class Publisher extends BasePeerConnection { const audioSettings = this.state.settings?.audio; const isDtxEnabled = !!audioSettings?.opus_dtx_enabled; - const isRedEnabled = - !!audioSettings?.redundant_coding_enabled && - isCodecSupported('audio/red'); + const isRedEnabled = !!audioSettings?.redundant_coding_enabled; const trackSettings = track.getSettings(); const isStereo = isAudioTrack && trackSettings.channelCount === 2; - const transceiverInitIndex = - this.transceiverInitOrder.indexOf(trackType); - const mid = this.transceiverOrder.indexOf(transceiver); + const transceiverIndex = this.transceiverOrder.indexOf(transceiver); + const mid = extractMid(transceiver, transceiverIndex, sdp); return { trackId: track.id, layers, trackType, - mid: - String(mid) ?? extractMid(transceiver, transceiverInitIndex, sdp), + mid, stereo: isStereo, dtx: isAudioTrack && isDtxEnabled, red: isAudioTrack && isRedEnabled, @@ -532,23 +514,28 @@ export class Publisher extends BasePeerConnection { .filter(Boolean) as TrackInfo[]; }; + private getPublishOptionFor = (trackType: TrackType) => { + const options = this.publishOptions.codecs; + const publishOption = options.find((o) => o.trackType === trackType); + if (!publishOption) { + throw new Error(`No publish options found for ${TrackType[trackType]}`); + } + return publishOption; + }; + private computeLayers = ( trackType: TrackType, track: MediaStreamTrack, - opts?: PublishOptions, + opts: PublishOption, ): OptimalVideoLayer[] | undefined => { const { settings } = this.state; const targetResolution = settings?.video .target_resolution as TargetResolutionResponse; - const screenShareBitrate = - settings?.screensharing.target_resolution?.bitrate; - const publishOpts = opts || this.publishOptsForTrack.get(trackType); - const codecInUse = getOptimalVideoCodec(publishOpts?.preferredCodec); return trackType === TrackType.VIDEO - ? findOptimalVideoLayers(track, targetResolution, codecInUse, publishOpts) + ? findOptimalVideoLayers(track, targetResolution, opts) : trackType === TrackType.SCREEN_SHARE - ? findOptimalScreenSharingLayers(track, publishOpts, screenShareBitrate) + ? findOptimalScreenSharingLayers(track, undefined, opts.bitrate) : undefined; }; } diff --git a/packages/client/src/rtc/Subscriber.ts b/packages/client/src/rtc/Subscriber.ts index d3ba5fd1f3..58426cb442 100644 --- a/packages/client/src/rtc/Subscriber.ts +++ b/packages/client/src/rtc/Subscriber.ts @@ -4,14 +4,9 @@ import { } from './BasePeerConnection'; import { PeerType } from '../gen/video/sfu/models/models'; import { SubscriberOffer } from '../gen/video/sfu/event/events'; -import { Dispatcher } from './Dispatcher'; import { withoutConcurrency } from '../helpers/concurrency'; import { toTrackType, trackTypeToParticipantStreamKey } from './helpers/tracks'; -export type SubscriberOpts = BasePeerConnectionOpts & { - dispatcher: Dispatcher; -}; - /** * A wrapper around the `RTCPeerConnection` that handles the incoming * media streams from the SFU. @@ -24,12 +19,12 @@ export class Subscriber extends BasePeerConnection { /** * Constructs a new `Subscriber` instance. */ - constructor({ dispatcher, ...baseOptions }: SubscriberOpts) { - super(PeerType.SUBSCRIBER, baseOptions); + constructor(opts: BasePeerConnectionOpts) { + super(PeerType.SUBSCRIBER, opts); this.pc.addEventListener('track', this.handleOnTrack); const subscriberOfferConcurrencyTag = Symbol('subscriberOffer'); - this.unregisterOnSubscriberOffer = dispatcher.on( + this.unregisterOnSubscriberOffer = this.dispatcher.on( 'subscriberOffer', (subscriberOffer) => { withoutConcurrency(subscriberOfferConcurrencyTag, () => { diff --git a/packages/client/src/rtc/__tests__/Publisher.test.ts b/packages/client/src/rtc/__tests__/Publisher.test.ts index 39493abd45..9022b67b75 100644 --- a/packages/client/src/rtc/__tests__/Publisher.test.ts +++ b/packages/client/src/rtc/__tests__/Publisher.test.ts @@ -69,9 +69,20 @@ describe('Publisher', () => { sfuClient, dispatcher, state, - isDtxEnabled: true, - isRedEnabled: true, logTag: 'test', + publishOptions: { + codecs: [ + { + trackType: TrackType.VIDEO, + bitrate: 1000, + // @ts-expect-error - incomplete data + codec: { name: 'vp9' }, + fps: 30, + maxTemporalLayers: 3, + maxSpatialLayers: 0, + }, + ], + }, }); }); diff --git a/packages/client/src/rtc/__tests__/bitrateLookup.test.ts b/packages/client/src/rtc/__tests__/bitrateLookup.test.ts deleted file mode 100644 index 6ec4c67426..0000000000 --- a/packages/client/src/rtc/__tests__/bitrateLookup.test.ts +++ /dev/null @@ -1,12 +0,0 @@ -import { describe, expect, it } from 'vitest'; -import { getOptimalBitrate } from '../bitrateLookup'; - -describe('bitrateLookup', () => { - it('should return optimal bitrate', () => { - expect(getOptimalBitrate('vp9', 720)).toBe(1_250_000); - }); - - it('should return nearest bitrate for exotic dimensions', () => { - expect(getOptimalBitrate('vp9', 1000)).toBe(1_500_000); - }); -}); diff --git a/packages/client/src/rtc/__tests__/codecs.test.ts b/packages/client/src/rtc/__tests__/codecs.test.ts deleted file mode 100644 index 62a4892c48..0000000000 --- a/packages/client/src/rtc/__tests__/codecs.test.ts +++ /dev/null @@ -1,143 +0,0 @@ -import { describe, expect, it, vi } from 'vitest'; -import { getPreferredCodecs } from '../codecs'; -import './mocks/webrtc.mocks'; - -describe('codecs', () => { - it('should return preferred audio codec', () => { - RTCRtpSender.getCapabilities = vi.fn().mockReturnValue(audioCodecs); - const codecs = getPreferredCodecs('audio/red'); - expect(codecs).toBeDefined(); - expect(codecs?.map((c) => c.mimeType)).toEqual([ - 'audio/red', - 'audio/opus', - 'audio/G722', - 'audio/PCMU', - 'audio/PCMA', - 'audio/CN', - 'audio/telephone-event', - ]); - }); - - it('should return preferred video codec', () => { - RTCRtpSender.getCapabilities = vi.fn().mockReturnValue(videoCodecs); - const codecs = getPreferredCodecs('video/vp8'); - expect(codecs).toBeDefined(); - // prettier-ignore - expect(codecs?.map((c) => [c.mimeType, c.sdpFmtpLine])).toEqual([ - ['video/VP8', undefined], - ['video/H264', 'level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=640c1f'], - ['video/rtx', undefined], - ['video/H264', 'level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f'], - ['video/H264', 'level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=640c1f'], - ['video/H264', 'level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=42e01f'], - ['video/VP9', 'profile-id=0'], - ['video/VP9', 'profile-id=2'], - ['video/red', undefined], - ['video/ulpfec', undefined], - ['video/flexfec-03', 'repair-window=10000000'], - ]); - }); - - it('should pick the baseline H264 codec', () => { - RTCRtpSender.getCapabilities = vi.fn().mockReturnValue(videoCodecs); - const codecs = getPreferredCodecs('video/h264'); - expect(codecs).toBeDefined(); - // prettier-ignore - expect(codecs?.map((c) => [c.mimeType, c.sdpFmtpLine])).toEqual([ - ['video/H264', 'level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=42e01f'], - ['video/H264', 'level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f'], - ['video/H264', 'level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=640c1f'], - ['video/H264', 'level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=640c1f'], - ['video/rtx', undefined], - ['video/VP8', undefined], - ['video/VP9', 'profile-id=0'], - ['video/VP9', 'profile-id=2'], - ['video/red', undefined], - ['video/ulpfec', undefined], - ['video/flexfec-03', 'repair-window=10000000'], - ]); - }); - - it('should pick the baseline H264 codec with optional packetization-mode', () => { - RTCRtpSender.getCapabilities = vi.fn().mockReturnValue(videoCodecsFirefox); - const codecs = getPreferredCodecs('video/h264'); - expect(codecs).toBeDefined(); - // prettier-ignore - expect(codecs?.map((c) => [c.mimeType, c.sdpFmtpLine])).toEqual([ - ['video/H264', 'profile-level-id=42e01f;level-asymmetry-allowed=1'], - ['video/H264', 'profile-level-id=42e01f;level-asymmetry-allowed=1;packetization-mode=1'], - ['video/VP8', 'max-fs=12288;max-fr=60'], - ['video/rtx', undefined], - ['video/VP9', 'max-fs=12288;max-fr=60'], - ['video/ulpfec', undefined], - ['video/red', undefined], - ]); - }); -}); - -// prettier-ignore -const videoCodecsFirefox: RTCRtpCapabilities = { - codecs: [ - { mimeType: 'video/VP8', sdpFmtpLine: 'max-fs=12288;max-fr=60', clockRate: 90000 }, - { mimeType: 'video/rtx', clockRate: 90000 }, - { mimeType: 'video/VP9', sdpFmtpLine: 'max-fs=12288;max-fr=60', clockRate: 90000 }, - { mimeType: 'video/H264', sdpFmtpLine: 'profile-level-id=42e01f;level-asymmetry-allowed=1;packetization-mode=1', clockRate: 90000 }, - { mimeType: 'video/H264', sdpFmtpLine: 'profile-level-id=42e01f;level-asymmetry-allowed=1', clockRate: 90000 }, - { mimeType: 'video/ulpfec', clockRate: 90000 }, - { mimeType: 'video/red', clockRate: 90000 }, - ], - headerExtensions: [ - { uri: 'urn:ietf:params:rtp-hdrext:sdes:mid' }, - { uri: 'http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time' }, - { uri: 'urn:ietf:params:rtp-hdrext:toffset' }, - { uri: 'http://www.webrtc.org/experiments/rtp-hdrext/playout-delay' }, - ], -}; - -// prettier-ignore -const videoCodecs: RTCRtpCapabilities = { - codecs: [ - { mimeType: 'video/H264', sdpFmtpLine: 'level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=640c1f', clockRate: 90000 }, - { mimeType: 'video/rtx', clockRate: 90000 }, - { mimeType: 'video/H264', sdpFmtpLine: 'level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f', clockRate: 90000 }, - { mimeType: 'video/H264', sdpFmtpLine: 'level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=640c1f', clockRate: 90000 }, - { mimeType: 'video/H264', sdpFmtpLine: 'level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=42e01f', clockRate: 90000 }, - { mimeType: 'video/VP8', clockRate: 90000 }, - { mimeType: 'video/VP9', sdpFmtpLine: 'profile-id=0', clockRate: 90000 }, - { mimeType: 'video/VP9', sdpFmtpLine: 'profile-id=2', clockRate: 90000 }, - { mimeType: 'video/red', clockRate: 90000 }, - { mimeType: 'video/ulpfec', clockRate: 90000 }, - { mimeType: 'video/flexfec-03', sdpFmtpLine: 'repair-window=10000000', clockRate: 90000 }, - ], - headerExtensions: [ - { uri: 'urn:ietf:params:rtp-hdrext:toffset' }, - { uri: 'http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time' }, - { uri: 'urn:3gpp:video-orientation' }, - { uri: 'http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01' }, - { uri: 'http://www.webrtc.org/experiments/rtp-hdrext/playout-delay' }, - { uri: 'http://www.webrtc.org/experiments/rtp-hdrext/video-content-type' }, - { uri: 'http://www.webrtc.org/experiments/rtp-hdrext/video-timing' }, - { uri: 'http://www.webrtc.org/experiments/rtp-hdrext/color-space' }, - { uri: 'urn:ietf:params:rtp-hdrext:sdes:mid' }, - { uri: 'urn:ietf:params:rtp-hdrext:sdes:rtp-stream-id' }, - { uri: 'urn:ietf:params:rtp-hdrext:sdes:repaired-rtp-stream-id' }, - ], -}; - -// prettier-ignore -const audioCodecs: RTCRtpCapabilities = { - codecs: [ - { mimeType: 'audio/opus', sdpFmtpLine: 'minptime=10;useinbandfec=1', clockRate: 48000 }, - { mimeType: 'audio/red', sdpFmtpLine: '=111/111', clockRate: 48000 }, - { mimeType: 'audio/G722', clockRate: 8000, channels: 1 }, - { mimeType: 'audio/PCMU', clockRate: 8000, channels: 1 }, - { mimeType: 'audio/PCMA', clockRate: 8000, channels: 1 }, - { mimeType: 'audio/CN', clockRate: 8000, channels: 1 }, - { mimeType: 'audio/telephone-event', clockRate: 8000, channels: 1 }, - ], - headerExtensions: [ - { uri: 'urn:ietf:params:rtp-hdrext:ssrc-audio-level' }, - { uri: 'http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time' }, - { uri: 'urn:ietf:params:rtp-hdrext:sdes:mid' }, - ], -}; diff --git a/packages/client/src/rtc/__tests__/videoLayers.test.ts b/packages/client/src/rtc/__tests__/videoLayers.test.ts index 49a0f80339..c0c8e695e2 100644 --- a/packages/client/src/rtc/__tests__/videoLayers.test.ts +++ b/packages/client/src/rtc/__tests__/videoLayers.test.ts @@ -1,5 +1,6 @@ import './mocks/webrtc.mocks'; import { describe, expect, it, vi } from 'vitest'; +import { PublishOption, VideoQuality } from '../../gen/video/sfu/models/models'; import { findOptimalScreenSharingLayers, findOptimalVideoLayers, @@ -8,7 +9,6 @@ import { ridToVideoQuality, toSvcEncodings, } from '../videoLayers'; -import { VideoQuality } from '../../gen/video/sfu/models/models'; describe('videoLayers', () => { it('should find optimal screen sharing layers', () => { @@ -50,11 +50,17 @@ describe('videoLayers', () => { const targetBitrate = 3000000; vi.spyOn(track, 'getSettings').mockReturnValue({ width, height }); - const layers = findOptimalVideoLayers(track, { - width, - height, + const publishOption: PublishOption = { bitrate: targetBitrate, - }); + // @ts-expect-error - incomplete data + codec: { name: 'vp8' }, + fps: 30, + }; + const layers = findOptimalVideoLayers( + track, + { width, height, bitrate: targetBitrate }, + publishOption, + ); expect(layers).toEqual([ { active: true, @@ -92,7 +98,12 @@ describe('videoLayers', () => { const bitrate = 3000000; const track = new MediaStreamTrack(); vi.spyOn(track, 'getSettings').mockReturnValue({ width, height }); - const layers = findOptimalVideoLayers(track, { width, height, bitrate }); + const layers = findOptimalVideoLayers( + track, + { width, height, bitrate }, + // @ts-expect-error - incomplete data + { bitrate, codec: { name: 'vp8' }, fps: 30 }, + ); expect(layers).toEqual([ { active: true, @@ -111,7 +122,12 @@ describe('videoLayers', () => { const width = 320; const height = 240; vi.spyOn(track, 'getSettings').mockReturnValue({ width, height }); - const layers = findOptimalVideoLayers(track); + const layers = findOptimalVideoLayers( + track, + undefined, + // @ts-expect-error - incomplete data + { bitrate: 0, codec: { name: 'vp8' }, fps: 30 }, + ); expect(layers.length).toBe(1); expect(layers[0].rid).toBe('q'); }); @@ -121,7 +137,12 @@ describe('videoLayers', () => { const width = 640; const height = 480; vi.spyOn(track, 'getSettings').mockReturnValue({ width, height }); - const layers = findOptimalVideoLayers(track); + const layers = findOptimalVideoLayers( + track, + undefined, + // @ts-expect-error - incomplete data + { bitrate: 0, codec: { name: 'vp8' }, fps: 30 }, + ); expect(layers.length).toBe(2); expect(layers[0].rid).toBe('q'); expect(layers[1].rid).toBe('h'); @@ -132,7 +153,12 @@ describe('videoLayers', () => { const width = 1280; const height = 720; vi.spyOn(track, 'getSettings').mockReturnValue({ width, height }); - const layers = findOptimalVideoLayers(track); + const layers = findOptimalVideoLayers( + track, + undefined, + // @ts-expect-error - incomplete data + { bitrate: 0, codec: { name: 'vp8' }, fps: 30 }, + ); expect(layers.length).toBe(3); expect(layers[0].rid).toBe('q'); expect(layers[1].rid).toBe('h'); @@ -145,12 +171,14 @@ describe('videoLayers', () => { width: 1280, height: 720, }); - const layers = findOptimalVideoLayers(track, undefined, 'vp9', { - preferredCodec: 'vp9', - scalabilityMode: 'L3T3', + const layers = findOptimalVideoLayers(track, undefined, { + maxTemporalLayers: 3, + maxSpatialLayers: 3, + // @ts-expect-error - incomplete data + codec: { name: 'vp9' }, }); expect(layers.length).toBe(3); - expect(layers[0].scalabilityMode).toBe('L3T3'); + expect(layers[0].scalabilityMode).toBe('L3T3_KEY'); expect(layers[0].rid).toBe('q'); expect(layers[1].rid).toBe('h'); expect(layers[2].rid).toBe('f'); @@ -183,7 +211,12 @@ describe('videoLayers', () => { describe('getComputedMaxBitrate', () => { it('should scale target bitrate down if resolution is smaller than target resolution', () => { const targetResolution = { width: 1920, height: 1080, bitrate: 3000000 }; - const scaledBitrate = getComputedMaxBitrate(targetResolution, 1280, 720); + const scaledBitrate = getComputedMaxBitrate( + targetResolution, + 1280, + 720, + 3000000, + ); expect(scaledBitrate).toBe(1333333); }); @@ -193,7 +226,12 @@ describe('videoLayers', () => { const targetBitrates = ['f', 'h', 'q'].map((rid) => { const width = targetResolution.width / downscaleFactor; const height = targetResolution.height / downscaleFactor; - const bitrate = getComputedMaxBitrate(targetResolution, width, height); + const bitrate = getComputedMaxBitrate( + targetResolution, + width, + height, + 3000000, + ); downscaleFactor *= 2; return { rid, @@ -211,25 +249,45 @@ describe('videoLayers', () => { it('should not scale target bitrate if resolution is larger than target resolution', () => { const targetResolution = { width: 1280, height: 720, bitrate: 1000000 }; - const scaledBitrate = getComputedMaxBitrate(targetResolution, 2560, 1440); + const scaledBitrate = getComputedMaxBitrate( + targetResolution, + 2560, + 1440, + 1000000, + ); expect(scaledBitrate).toBe(1000000); }); it('should not scale target bitrate if resolution is equal to target resolution', () => { const targetResolution = { width: 1280, height: 720, bitrate: 1000000 }; - const scaledBitrate = getComputedMaxBitrate(targetResolution, 1280, 720); + const scaledBitrate = getComputedMaxBitrate( + targetResolution, + 1280, + 720, + 1000000, + ); expect(scaledBitrate).toBe(1000000); }); it('should handle 0 width and height', () => { const targetResolution = { width: 1280, height: 720, bitrate: 1000000 }; - const scaledBitrate = getComputedMaxBitrate(targetResolution, 0, 0); + const scaledBitrate = getComputedMaxBitrate( + targetResolution, + 0, + 0, + 1000000, + ); expect(scaledBitrate).toBe(0); }); it('should handle 4k target resolution', () => { const targetResolution = { width: 3840, height: 2160, bitrate: 15000000 }; - const scaledBitrate = getComputedMaxBitrate(targetResolution, 1280, 720); + const scaledBitrate = getComputedMaxBitrate( + targetResolution, + 1280, + 720, + 15000000, + ); expect(scaledBitrate).toBe(1666667); }); }); diff --git a/packages/client/src/rtc/bitrateLookup.ts b/packages/client/src/rtc/bitrateLookup.ts deleted file mode 100644 index 448b1df922..0000000000 --- a/packages/client/src/rtc/bitrateLookup.ts +++ /dev/null @@ -1,61 +0,0 @@ -import { PreferredCodec } from '../types'; - -const bitrateLookupTable: Record< - PreferredCodec, - Record | undefined -> = { - h264: { - 2160: 5_000_000, - 1440: 3_000_000, - 1080: 2_000_000, - 720: 1_250_000, - 540: 750_000, - 360: 400_000, - default: 1_250_000, - }, - vp8: { - 2160: 5_000_000, - 1440: 2_750_000, - 1080: 2_000_000, - 720: 1_250_000, - 540: 600_000, - 360: 350_000, - default: 1_250_000, - }, - vp9: { - 2160: 3_000_000, - 1440: 2_000_000, - 1080: 1_500_000, - 720: 1_250_000, - 540: 500_000, - 360: 275_000, - default: 1_250_000, - }, - av1: { - 2160: 2_000_000, - 1440: 1_550_000, - 1080: 1_000_000, - 720: 600_000, - 540: 350_000, - 360: 200_000, - default: 600_000, - }, -}; - -export const getOptimalBitrate = ( - codec: PreferredCodec, - frameHeight: number, -): number => { - const codecLookup = bitrateLookupTable[codec]; - if (!codecLookup) throw new Error(`Unknown codec: ${codec}`); - - let bitrate = codecLookup[frameHeight]; - if (!bitrate) { - const keys = Object.keys(codecLookup).map(Number); - const nearest = keys.reduce((a, b) => - Math.abs(b - frameHeight) < Math.abs(a - frameHeight) ? b : a, - ); - bitrate = codecLookup[nearest]; - } - return bitrate ?? codecLookup.default!; -}; diff --git a/packages/client/src/rtc/codecs.ts b/packages/client/src/rtc/codecs.ts index b70929aa59..6de9fe50dd 100644 --- a/packages/client/src/rtc/codecs.ts +++ b/packages/client/src/rtc/codecs.ts @@ -1,21 +1,3 @@ -import { getOSInfo } from '../client-details'; -import { isReactNative } from '../helpers/platforms'; -import { isFirefox, isSafari } from '../helpers/browsers'; -import type { PreferredCodec } from '../types'; - -/** - * Returns back a list of supported publish codecs for the given kind. - */ -export const getSupportedCodecs = ( - kind: 'audio' | 'video', -): RTCRtpCodecCapability[] => { - if (!('getCapabilities' in RTCRtpSender)) return []; - const capabilities = RTCRtpSender.getCapabilities(kind); - if (!capabilities) return []; - - return capabilities.codecs; -}; - /** * Returns a generic SDP for the given direction. * We use this SDP to send it as part of our JoinRequest so that the SFU @@ -38,48 +20,6 @@ export const getGenericSdp = async (direction: RTCRtpTransceiverDirection) => { return sdp; }; -/** - * Returns the optimal video codec for the device. - */ -export const getOptimalVideoCodec = ( - preferredCodec: PreferredCodec | undefined, -): PreferredCodec => { - if (isReactNative()) { - const os = getOSInfo()?.name.toLowerCase(); - if (os === 'android') return preferredOr(preferredCodec, 'vp8'); - if (os === 'ios' || os === 'ipados') return 'h264'; - return preferredOr(preferredCodec, 'h264'); - } - // Safari and Firefox do not have a good support encoding to SVC codecs, - // so we disable it for them. - if (isSafari()) return 'h264'; - if (isFirefox()) return 'vp8'; - return preferredOr(preferredCodec, 'vp8'); -}; - -/** - * Determines if the platform supports the preferred codec. - * If not, it returns the fallback codec. - */ -const preferredOr = ( - codec: PreferredCodec | undefined, - fallback: PreferredCodec, -): PreferredCodec => { - return codec && isCodecSupported(`video/${codec}`) ? codec : fallback; -}; - -/** - * Returns whether the codec is supported by the platform. - * - * @param codecMimeType the codec to check. - */ -export const isCodecSupported = (codecMimeType: string): boolean => { - codecMimeType = codecMimeType.toLowerCase(); - const [kind] = codecMimeType.split('/') as ('audio' | 'video')[]; - const codecs = getSupportedCodecs(kind); - return codecs.some((c) => c.mimeType.toLowerCase() === codecMimeType); -}; - /** * Returns whether the codec is an SVC codec. * diff --git a/packages/client/src/rtc/videoLayers.ts b/packages/client/src/rtc/videoLayers.ts index bbc2a359f1..60efb976c3 100644 --- a/packages/client/src/rtc/videoLayers.ts +++ b/packages/client/src/rtc/videoLayers.ts @@ -1,8 +1,7 @@ -import { PreferredCodec, PublishOptions } from '../types'; +import { ScreenShareSettings } from '../types'; import { TargetResolutionResponse } from '../gen/shims'; import { isSvcCodec } from './codecs'; -import { getOptimalBitrate } from './bitrateLookup'; -import { VideoQuality } from '../gen/video/sfu/models/models'; +import { PublishOption, VideoQuality } from '../gen/video/sfu/models/models'; export type OptimalVideoLayer = RTCRtpEncodingParameters & { width: number; @@ -47,61 +46,68 @@ export const ridToVideoQuality = (rid: string): VideoQuality => { : VideoQuality.HIGH; // default to HIGH }; +/** + * Converts the spatial and temporal layers to a scalability mode. + */ +const toScalabilityMode = (spatialLayers: number, temporalLayers: number) => + `L${spatialLayers}T${temporalLayers}${spatialLayers > 1 ? '_KEY' : ''}`; + /** * Determines the most optimal video layers for simulcasting * for the given track. * * @param videoTrack the video track to find optimal layers for. * @param targetResolution the expected target resolution. - * @param codecInUse the codec in use. - * @param publishOptions the publish options for the track. + * @param publishOption the publish options for the track. */ export const findOptimalVideoLayers = ( videoTrack: MediaStreamTrack, targetResolution: TargetResolutionResponse = defaultTargetResolution, - codecInUse?: PreferredCodec, - publishOptions?: PublishOptions, + publishOption: PublishOption, ) => { const optimalVideoLayers: OptimalVideoLayer[] = []; const settings = videoTrack.getSettings(); const { width = 0, height = 0 } = settings; const { - scalabilityMode, - bitrateDownscaleFactor = 2, - maxSimulcastLayers = 3, - } = publishOptions || {}; + bitrate, + codec, + fps, + maxSpatialLayers = 3, + maxTemporalLayers = 3, + } = publishOption; const maxBitrate = getComputedMaxBitrate( targetResolution, width, height, - codecInUse, - publishOptions, + bitrate, ); let downscaleFactor = 1; let bitrateFactor = 1; - const svcCodec = isSvcCodec(codecInUse); - const totalLayers = svcCodec ? 3 : Math.min(3, maxSimulcastLayers); + const svcCodec = isSvcCodec(codec?.name); + const totalLayers = svcCodec ? 3 : Math.min(3, maxSpatialLayers); for (const rid of ['f', 'h', 'q'].slice(0, totalLayers)) { const layer: OptimalVideoLayer = { active: true, rid, width: Math.round(width / downscaleFactor), height: Math.round(height / downscaleFactor), - maxBitrate: - Math.round(maxBitrate / bitrateFactor) || defaultBitratePerRid[rid], - maxFramerate: 30, + maxBitrate: maxBitrate / bitrateFactor || defaultBitratePerRid[rid], + maxFramerate: fps, }; if (svcCodec) { // for SVC codecs, we need to set the scalability mode, and the // codec will handle the rest (layers, temporal layers, etc.) - layer.scalabilityMode = scalabilityMode || 'L3T2_KEY'; + layer.scalabilityMode = toScalabilityMode( + maxSpatialLayers, + maxTemporalLayers, + ); } else { // for non-SVC codecs, we need to downscale proportionally (simulcast) layer.scaleResolutionDownBy = downscaleFactor; } downscaleFactor *= 2; - bitrateFactor *= bitrateDownscaleFactor; + bitrateFactor *= 2; // Reversing the order [f, h, q] to [q, h, f] as Chrome uses encoding index // when deciding which layer to disable when CPU or bandwidth is constrained. @@ -124,29 +130,17 @@ export const findOptimalVideoLayers = ( * @param targetResolution the target resolution. * @param currentWidth the current width of the track. * @param currentHeight the current height of the track. - * @param codecInUse the codec in use. - * @param publishOptions the publish options. + * @param bitrate the target bitrate. */ export const getComputedMaxBitrate = ( targetResolution: TargetResolutionResponse, currentWidth: number, currentHeight: number, - codecInUse?: PreferredCodec, - publishOptions?: PublishOptions, + bitrate: number, ): number => { // if the current resolution is lower than the target resolution, // we want to proportionally reduce the target bitrate - const { - width: targetWidth, - height: targetHeight, - bitrate: targetBitrate, - } = targetResolution; - const { preferredBitrate } = publishOptions || {}; - const frameHeight = - currentWidth > currentHeight ? currentHeight : currentWidth; - const bitrate = - preferredBitrate || - (codecInUse ? getOptimalBitrate(codecInUse, frameHeight) : targetBitrate); + const { width: targetWidth, height: targetHeight } = targetResolution; if (currentWidth < targetWidth || currentHeight < targetHeight) { const currentPixels = currentWidth * currentHeight; const targetPixels = targetWidth * targetHeight; @@ -189,12 +183,14 @@ const withSimulcastConstraints = ( })); }; +/** + * Determines the most optimal video layers for screen sharing. + */ export const findOptimalScreenSharingLayers = ( videoTrack: MediaStreamTrack, - publishOptions?: PublishOptions, + preferences?: ScreenShareSettings, defaultMaxBitrate = 3000000, ): OptimalVideoLayer[] => { - const { screenShareSettings: preferences } = publishOptions || {}; const settings = videoTrack.getSettings(); return [ { diff --git a/packages/client/src/types.ts b/packages/client/src/types.ts index da459b9b07..2f04ceb1dd 100644 --- a/packages/client/src/types.ts +++ b/packages/client/src/types.ts @@ -151,10 +151,6 @@ export type SubscriptionChanges = { * @internal */ export type PreferredCodec = - | 'vp8' - | 'h264' - | 'vp9' - | 'av1' | 'video/vp8' | 'video/h264' | 'video/vp9' @@ -166,39 +162,7 @@ export type PreferredCodec = * A collection of track publication options. * @internal */ -export type PublishOptions = { - /** - * The preferred codec to use when publishing the video stream. - */ - preferredCodec?: PreferredCodec; - /** - * Force the codec to use when publishing the video stream. - * This will override the preferred codec and the internal codec selection logic. - * Use with caution. - */ - forceCodec?: PreferredCodec; - /** - * The preferred audio codec to use when publishing the audio stream. - */ - preferredAudioCodec?: PreferredCodec; - /** - * The preferred scalability to use when publishing the video stream. - * Applicable only for SVC codecs. - */ - scalabilityMode?: string; - /** - * The preferred bitrate to use when publishing the video stream. - */ - preferredBitrate?: number; - /** - * The preferred downscale factor to use when publishing the video stream - * in simulcast mode (non-SVC). - */ - bitrateDownscaleFactor?: number; - /** - * The maximum number of simulcast layers to use when publishing the video stream. - */ - maxSimulcastLayers?: number; +export type ClientPublishOptions = { /** * Screen share settings. */ From 01495ce39ec2bfc563462b4e3e2636c051eb61b9 Mon Sep 17 00:00:00 2001 From: Oliver Lazoroski Date: Tue, 12 Nov 2024 16:52:03 +0100 Subject: [PATCH 18/52] feat: handle ChangePublishOptions --- packages/react-native-sdk/src/utils/StreamVideoRN/types.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/packages/react-native-sdk/src/utils/StreamVideoRN/types.ts b/packages/react-native-sdk/src/utils/StreamVideoRN/types.ts index 6a45788974..3ee1387f2e 100644 --- a/packages/react-native-sdk/src/utils/StreamVideoRN/types.ts +++ b/packages/react-native-sdk/src/utils/StreamVideoRN/types.ts @@ -1,4 +1,7 @@ -import { PublishOptions, StreamVideoClient } from '@stream-io/video-client'; +import { + ClientPublishOptions, + StreamVideoClient, +} from '@stream-io/video-client'; import type { AndroidChannel } from '@notifee/react-native'; export type NonRingingPushEvent = 'call.live_started' | 'call.notification'; @@ -16,7 +19,7 @@ export type StreamVideoConfig = { * * @internal */ - publishOptions?: PublishOptions; + publishOptions?: ClientPublishOptions; ios: { /** * The name for the alias of push provider used for iOS From a95c9eda859cd6bfeb65d679bda4729165ee940f Mon Sep 17 00:00:00 2001 From: Oliver Lazoroski Date: Tue, 12 Nov 2024 16:53:51 +0100 Subject: [PATCH 19/52] fix: remove outdated code --- .../dogfood/src/components/MeetingUI.tsx | 3 -- .../react-dogfood/components/MeetingUI.tsx | 40 +------------------ 2 files changed, 1 insertion(+), 42 deletions(-) diff --git a/sample-apps/react-native/dogfood/src/components/MeetingUI.tsx b/sample-apps/react-native/dogfood/src/components/MeetingUI.tsx index 378270ab03..dcae405b3e 100644 --- a/sample-apps/react-native/dogfood/src/components/MeetingUI.tsx +++ b/sample-apps/react-native/dogfood/src/components/MeetingUI.tsx @@ -59,9 +59,6 @@ export const MeetingUI = ({ callId, navigation, route }: Props) => { const onJoinCallHandler = useCallback(async () => { try { setShow('loading'); - call?.updatePublishOptions({ - preferredCodec: 'vp9', - }); await call?.join({ create: true }); appStoreSetState({ chatLabelNoted: false }); setShow('active-call'); diff --git a/sample-apps/react/react-dogfood/components/MeetingUI.tsx b/sample-apps/react/react-dogfood/components/MeetingUI.tsx index 7b74d416d6..afc60c8563 100644 --- a/sample-apps/react/react-dogfood/components/MeetingUI.tsx +++ b/sample-apps/react/react-dogfood/components/MeetingUI.tsx @@ -6,7 +6,6 @@ import { defaultSortPreset, LoadingIndicator, noopComparator, - PreferredCodec, useCall, useCallStateHooks, usePersistedDevicePreferences, @@ -46,42 +45,11 @@ export const MeetingUI = ({ chatClient, mode }: MeetingUIProps) => { const { useCallCallingState } = useCallStateHooks(); const callState = useCallCallingState(); - const videoCodecOverride = router.query['video_codec'] as - | PreferredCodec - | undefined; - const bitrateOverride = router.query['bitrate'] as string | undefined; - const bitrateFactorOverride = router.query['bitrate_factor'] as - | string - | undefined; - const scalabilityMode = router.query['scalability_mode'] as - | string - | undefined; - const maxSimulcastLayers = router.query['max_simulcast_layers'] as - | string - | undefined; - const onJoin = useCallback( async ({ fastJoin = false } = {}) => { if (!fastJoin) setShow('loading'); if (!call) throw new Error('No active call found'); try { - const preferredBitrate = bitrateOverride - ? parseInt(bitrateOverride, 10) - : undefined; - - call.updatePublishOptions({ - preferredCodec: 'vp9', - forceCodec: videoCodecOverride, - scalabilityMode, - preferredBitrate, - bitrateDownscaleFactor: bitrateFactorOverride - ? parseInt(bitrateFactorOverride, 10) - : 2, // default to 2 - maxSimulcastLayers: maxSimulcastLayers - ? parseInt(maxSimulcastLayers, 10) - : 3, // default to 3 - }); - await call.join({ create: true }); setShow('active-call'); } catch (e) { @@ -90,13 +58,7 @@ export const MeetingUI = ({ chatClient, mode }: MeetingUIProps) => { setShow('error-join'); } }, - [ - bitrateFactorOverride, - bitrateOverride, - call, - scalabilityMode, - videoCodecOverride, - ], + [call], ); const onLeave = useCallback( From 7ff22217a51a2a386dee0045de453c79e3d1c72f Mon Sep 17 00:00:00 2001 From: Oliver Lazoroski Date: Wed, 13 Nov 2024 17:09:55 +0100 Subject: [PATCH 20/52] feat: handle changePublishOptions --- packages/client/src/Call.ts | 6 +-- packages/client/src/rtc/Publisher.ts | 60 +++++++++++++++++++--------- 2 files changed, 44 insertions(+), 22 deletions(-) diff --git a/packages/client/src/Call.ts b/packages/client/src/Call.ts index b8dc57dd30..3b516e1c1b 100644 --- a/packages/client/src/Call.ts +++ b/packages/client/src/Call.ts @@ -201,7 +201,7 @@ export class Call { private readonly dispatcher = new Dispatcher(); private clientPublishOptions?: ClientPublishOptions; - private publishOptions?: PublishOptions; + private initialPublishOptions?: PublishOptions; private statsReporter?: StatsReporter; private sfuStatsReporter?: SfuStatsReporter; private dropTimeout: ReturnType | undefined; @@ -807,7 +807,7 @@ export class Call { reconnectDetails, }); - this.publishOptions = publishOptions; + this.initialPublishOptions = publishOptions; this.fastReconnectDeadlineSeconds = fastReconnectDeadlineSeconds; if (callState) { this.state.updateFromSfuCallState( @@ -837,7 +837,7 @@ export class Call { connectionConfig, clientDetails, statsOptions, - publishOptions: this.publishOptions!, + publishOptions: this.initialPublishOptions!, closePreviousInstances: !performingMigration, }); } diff --git a/packages/client/src/rtc/Publisher.ts b/packages/client/src/rtc/Publisher.ts index 0134cb8afd..715251ed39 100644 --- a/packages/client/src/rtc/Publisher.ts +++ b/packages/client/src/rtc/Publisher.ts @@ -47,6 +47,7 @@ export class Publisher extends BasePeerConnection { private readonly unsubscribeOnIceRestart: () => void; private readonly unsubscribeChangePublishQuality: () => void; private readonly unsubscribeChangePublishOptions: () => void; + private unsubscribeCodecNegotiationComplete?: () => void; private publishOptions: PublishOptions; @@ -86,12 +87,21 @@ export class Publisher extends BasePeerConnection { this.unsubscribeChangePublishOptions = this.dispatcher.on( 'changePublishOptions', - (event) => { - const { publishOption } = event; - if (!publishOption) return; - this.publishOptions.codecs = this.publishOptions.codecs.map((option) => - option.trackType === publishOption.trackType ? publishOption : option, - ); + ({ publishOption }) => { + withoutConcurrency('publisher.changePublishOptions', async () => { + if (!publishOption) return; + this.publishOptions.codecs = this.publishOptions.codecs.map( + (option) => + option.trackType === publishOption.trackType + ? publishOption + : option, + ); + if (this.isPublishing(publishOption.trackType)) { + this.switchCodec(publishOption); + } + }).catch((err) => { + this.logger('warn', 'Failed to change publish options', err); + }); }, ); } @@ -118,6 +128,7 @@ export class Publisher extends BasePeerConnection { this.unsubscribeOnIceRestart(); this.unsubscribeChangePublishQuality(); this.unsubscribeChangePublishOptions(); + this.unsubscribeCodecNegotiationComplete?.(); super.detachEventHandlers(); this.pc.removeEventListener('negotiationneeded', this.onNegotiationNeeded); @@ -158,7 +169,8 @@ export class Publisher extends BasePeerConnection { ); }; track.addEventListener('ended', handleTrackEnded); - this.addTransceiver(trackType, track); + const publishOption = this.getPublishOptionFor(trackType); + this.addTransceiver(trackType, track, publishOption); } else { await this.updateTransceiver(transceiver, track); } @@ -171,9 +183,11 @@ export class Publisher extends BasePeerConnection { * This needs to be called when a new track kind is added to the peer connection. * In other cases, use `updateTransceiver` method. */ - private addTransceiver = (trackType: TrackType, track: MediaStreamTrack) => { - const publishOption = this.getPublishOptionFor(trackType); - + private addTransceiver = ( + trackType: TrackType, + track: MediaStreamTrack, + publishOption: PublishOption, + ) => { const videoEncodings = this.computeLayers(trackType, track, publishOption); const sendEncodings = isSvcCodec(publishOption.codec?.name) ? toSvcEncodings(videoEncodings) @@ -204,20 +218,28 @@ export class Publisher extends BasePeerConnection { await transceiver.sender.replaceTrack(track); }; - publishTrack = () => { - const currentTransceiver = this.transceiverCache.get(TrackType.VIDEO); - if (!currentTransceiver || !currentTransceiver.sender.track) return; - const track = currentTransceiver.sender.track; + /** + * Switches the codec of the given track type. + */ + private switchCodec = (publishOption: PublishOption) => { + const trackType = publishOption.trackType; + const transceiver = this.transceiverCache.get(trackType); + if (!transceiver || !transceiver.sender.track) return; - const negotiationComplete = async () => { - this.dispatcher.off('codecNegotiationComplete', negotiationComplete); + const onNegotiationComplete = async () => { this.logger('info', 'Codec negotiation complete'); + this.dispatcher.off('codecNegotiationComplete', onNegotiationComplete); - await currentTransceiver.sender.replaceTrack(null); + await transceiver.sender.replaceTrack(null); }; - this.dispatcher.on('codecNegotiationComplete', negotiationComplete); + this.unsubscribeCodecNegotiationComplete?.(); + this.unsubscribeCodecNegotiationComplete = this.dispatcher.on( + 'codecNegotiationComplete', + onNegotiationComplete, + ); - this.addTransceiver(TrackType.VIDEO, track); + const track = transceiver.sender.track; + this.addTransceiver(trackType, track, publishOption); }; /** From cda1dcd93890d87fae025c553e46937461feabdb Mon Sep 17 00:00:00 2001 From: Oliver Lazoroski Date: Thu, 14 Nov 2024 14:45:50 +0100 Subject: [PATCH 21/52] feat: preferredCodec hint --- packages/client/src/Call.ts | 29 ++- .../client/src/gen/video/sfu/event/events.ts | 46 +++-- .../client/src/gen/video/sfu/models/models.ts | 9 +- .../helpers/__tests__/publisher-sdp.mock.ts | 170 ++++++++++++++++++ .../src/helpers/__tests__/sdp-munging.test.ts | 12 +- packages/client/src/helpers/sdp-munging.ts | 21 +++ packages/client/src/rtc/codecs.ts | 15 ++ packages/client/src/types.ts | 12 +- .../react-dogfood/components/MeetingUI.tsx | 10 +- 9 files changed, 290 insertions(+), 34 deletions(-) create mode 100644 packages/client/src/helpers/__tests__/publisher-sdp.mock.ts diff --git a/packages/client/src/Call.ts b/packages/client/src/Call.ts index 3b516e1c1b..c52e661b93 100644 --- a/packages/client/src/Call.ts +++ b/packages/client/src/Call.ts @@ -1,6 +1,7 @@ import { StreamSfuClient } from './StreamSfuClient'; import { Dispatcher, + findCodec, getGenericSdp, isSfuEvent, Publisher, @@ -90,6 +91,7 @@ import { BehaviorSubject, Subject, takeWhile } from 'rxjs'; import { ReconnectDetails } from './gen/video/sfu/event/events'; import { ClientDetails, + Codec, PublishOptions, TrackType, WebsocketReconnectStrategy, @@ -120,6 +122,7 @@ import { import { getSdkSignature } from './stats/utils'; import { withoutConcurrency } from './helpers/concurrency'; import { ensureExhausted } from './helpers/ensureExhausted'; +import { getPayloadTypeForCodec } from './helpers/sdp-munging'; import { PromiseWithResolvers, promiseWithResolvers, @@ -805,6 +808,7 @@ export class Call { clientDetails, fastReconnect: performingFastReconnect, reconnectDetails, + preferredCodec: this.getPreferredCodec(publishingCapabilitiesSdp), }); this.initialPublishOptions = publishOptions; @@ -902,6 +906,27 @@ export class Call { }; }; + /** + * Prepares the preferred codec for the call. + * This is an experimental client feature and subject to change. + * @internal + */ + private getPreferredCodec = (sdp: string): Codec | undefined => { + const { preferredCodec } = this.clientPublishOptions || {}; + if (!preferredCodec) return; + + const codec = findCodec(`video/${preferredCodec}`); + if (!codec) return; + + const { clockRate, mimeType, sdpFmtpLine } = codec; + return Codec.create({ + name: preferredCodec, // e.g. 'vp9' + fmtp: sdpFmtpLine || '', + clockRate: clockRate, + payloadType: getPayloadTypeForCodec(sdp, mimeType, sdpFmtpLine), + }); + }; + /** * Performs an ICE restart on both the Publisher and Subscriber Peer Connections. * Uses the provided SFU client to restore the ICE connection. @@ -1510,7 +1535,7 @@ export class Call { * @internal * @param options the options to use. */ - updatePublishOptions(options: ClientPublishOptions) { + updatePublishOptions = (options: ClientPublishOptions) => { if (this.state.callingState === CallingState.JOINED) { this.logger( 'warn', @@ -1518,7 +1543,7 @@ export class Call { ); } this.clientPublishOptions = { ...this.clientPublishOptions, ...options }; - } + }; /** * Notifies the SFU that a noise cancellation process has started. diff --git a/packages/client/src/gen/video/sfu/event/events.ts b/packages/client/src/gen/video/sfu/event/events.ts index 22f487b350..a51e701c15 100644 --- a/packages/client/src/gen/video/sfu/event/events.ts +++ b/packages/client/src/gen/video/sfu/event/events.ts @@ -3,26 +3,29 @@ // @generated from protobuf file "video/sfu/event/events.proto" (package "stream.video.sfu.event", syntax proto3) // tslint:disable import { MessageType } from '@protobuf-ts/runtime'; -import { CallEndedReason } from '../models/models'; -import { GoAwayReason } from '../models/models'; -import { CallGrants } from '../models/models'; -import { Codec } from '../models/models'; -import { ConnectionQuality } from '../models/models'; -import { PublishOptions } from '../models/models'; -import { CallState } from '../models/models'; +import { + CallEndedReason, + CallGrants, + CallState, + ClientDetails, + Codec, + ConnectionQuality, + Error as Error$, + GoAwayReason, + ICETrickle as ICETrickle$, + Participant, + ParticipantCount, + PeerType, + Pin, + PublishOption, + PublishOptions, + TrackInfo, + TrackType, + TrackUnpublishReason, + WebsocketReconnectStrategy, +} from '../models/models'; import { TrackSubscriptionDetails } from '../signal_rpc/signal'; -import { TrackInfo } from '../models/models'; -import { ClientDetails } from '../models/models'; -import { TrackUnpublishReason } from '../models/models'; -import { Participant } from '../models/models'; -import { TrackType } from '../models/models'; -import { ParticipantCount } from '../models/models'; -import { PeerType } from '../models/models'; -import { WebsocketReconnectStrategy } from '../models/models'; -import { Error as Error$ } from '../models/models'; -import { Pin } from '../models/models'; -import { PublishOption } from '../models/models'; -import { ICETrickle as ICETrickle$ } from '../models/models'; + /** * SFUEvent is a message that is sent from the SFU to the client. * @@ -504,6 +507,10 @@ export interface JoinRequest { * @generated from protobuf field: stream.video.sfu.event.ReconnectDetails reconnect_details = 7; */ reconnectDetails?: ReconnectDetails; + /** + * @generated from protobuf field: stream.video.sfu.models.Codec preferred_codec = 9; + */ + preferredCodec?: Codec; } /** * @generated from protobuf message stream.video.sfu.event.ReconnectDetails @@ -1306,6 +1313,7 @@ class JoinRequest$Type extends MessageType { kind: 'message', T: () => ReconnectDetails, }, + { no: 9, name: 'preferred_codec', kind: 'message', T: () => Codec }, ]); } } diff --git a/packages/client/src/gen/video/sfu/models/models.ts b/packages/client/src/gen/video/sfu/models/models.ts index d7a4f2ba11..f8849e39b2 100644 --- a/packages/client/src/gen/video/sfu/models/models.ts +++ b/packages/client/src/gen/video/sfu/models/models.ts @@ -5,6 +5,7 @@ import { MessageType } from '@protobuf-ts/runtime'; import { Struct } from '../../../google/protobuf/struct'; import { Timestamp } from '../../../google/protobuf/timestamp'; + /** * CallState is the current state of the call * as seen by an SFU. @@ -236,7 +237,7 @@ export interface PublishOption { */ export interface Codec { /** - * @generated from protobuf field: uint32 payload_type = 11; + * @generated from protobuf field: uint32 payload_type = 16; */ payloadType: number; /** @@ -248,7 +249,7 @@ export interface Codec { */ clockRate: number; /** - * @generated from protobuf field: string encoding_parameters = 13; + * @generated from protobuf field: string encoding_parameters = 15; */ encodingParameters: string; /** @@ -1160,7 +1161,7 @@ class Codec$Type extends MessageType { constructor() { super('stream.video.sfu.models.Codec', [ { - no: 11, + no: 16, name: 'payload_type', kind: 'scalar', T: 13 /*ScalarType.UINT32*/, @@ -1173,7 +1174,7 @@ class Codec$Type extends MessageType { T: 13 /*ScalarType.UINT32*/, }, { - no: 13, + no: 15, name: 'encoding_parameters', kind: 'scalar', T: 9 /*ScalarType.STRING*/, diff --git a/packages/client/src/helpers/__tests__/publisher-sdp.mock.ts b/packages/client/src/helpers/__tests__/publisher-sdp.mock.ts new file mode 100644 index 0000000000..4d4f19d294 --- /dev/null +++ b/packages/client/src/helpers/__tests__/publisher-sdp.mock.ts @@ -0,0 +1,170 @@ +export const publisherSDP = ` +v=0 +o=- 6786973741786618802 2 IN IP4 127.0.0.1 +s=- +t=0 0 +a=group:BUNDLE 0 1 +a=extmap-allow-mixed +a=msid-semantic: WMS +m=video 9 UDP/TLS/RTP/SAVPF 96 97 102 103 104 105 106 107 108 109 127 125 39 40 45 46 98 99 100 101 112 113 116 117 118 +c=IN IP4 0.0.0.0 +a=rtcp:9 IN IP4 0.0.0.0 +a=ice-ufrag:PeYM +a=ice-pwd:GDZQ30hwMK+5m/uVP3C0EQvf +a=ice-options:trickle +a=fingerprint:sha-256 E4:D3:D1:D8:70:08:6C:44:51:C5:0C:7E:AB:01:06:FF:10:A7:39:38:5B:43:C4:BA:FA:68:DA:AA:43:95:2C:32 +a=setup:actpass +a=mid:0 +a=extmap:1 urn:ietf:params:rtp-hdrext:toffset +a=extmap:2 http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time +a=extmap:3 urn:3gpp:video-orientation +a=extmap:4 http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01 +a=extmap:5 http://www.webrtc.org/experiments/rtp-hdrext/playout-delay +a=extmap:6 http://www.webrtc.org/experiments/rtp-hdrext/video-content-type +a=extmap:7 http://www.webrtc.org/experiments/rtp-hdrext/video-timing +a=extmap:8 http://www.webrtc.org/experiments/rtp-hdrext/color-space +a=extmap:9 urn:ietf:params:rtp-hdrext:sdes:mid +a=extmap:10 urn:ietf:params:rtp-hdrext:sdes:rtp-stream-id +a=extmap:11 urn:ietf:params:rtp-hdrext:sdes:repaired-rtp-stream-id +a=sendonly +a=msid:- 22bcd911-0a07-4b69-9557-8f4746411089 +a=rtcp-mux +a=rtcp-rsize +a=rtpmap:96 VP8/90000 +a=rtcp-fb:96 goog-remb +a=rtcp-fb:96 transport-cc +a=rtcp-fb:96 ccm fir +a=rtcp-fb:96 nack +a=rtcp-fb:96 nack pli +a=rtpmap:97 rtx/90000 +a=fmtp:97 apt=96 +a=rtpmap:102 H264/90000 +a=rtcp-fb:102 goog-remb +a=rtcp-fb:102 transport-cc +a=rtcp-fb:102 ccm fir +a=rtcp-fb:102 nack +a=rtcp-fb:102 nack pli +a=fmtp:102 level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42001f +a=rtpmap:103 rtx/90000 +a=fmtp:103 apt=102 +a=rtpmap:104 H264/90000 +a=rtcp-fb:104 goog-remb +a=rtcp-fb:104 transport-cc +a=rtcp-fb:104 ccm fir +a=rtcp-fb:104 nack +a=rtcp-fb:104 nack pli +a=fmtp:104 level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=42001f +a=rtpmap:105 rtx/90000 +a=fmtp:105 apt=104 +a=rtpmap:106 H264/90000 +a=rtcp-fb:106 goog-remb +a=rtcp-fb:106 transport-cc +a=rtcp-fb:106 ccm fir +a=rtcp-fb:106 nack +a=rtcp-fb:106 nack pli +a=fmtp:106 level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f +a=rtpmap:107 rtx/90000 +a=fmtp:107 apt=106 +a=rtpmap:108 H264/90000 +a=rtcp-fb:108 goog-remb +a=rtcp-fb:108 transport-cc +a=rtcp-fb:108 ccm fir +a=rtcp-fb:108 nack +a=rtcp-fb:108 nack pli +a=fmtp:108 level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=42e01f +a=rtpmap:109 rtx/90000 +a=fmtp:109 apt=108 +a=rtpmap:127 H264/90000 +a=rtcp-fb:127 goog-remb +a=rtcp-fb:127 transport-cc +a=rtcp-fb:127 ccm fir +a=rtcp-fb:127 nack +a=rtcp-fb:127 nack pli +a=fmtp:127 level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=4d001f +a=rtpmap:125 rtx/90000 +a=fmtp:125 apt=127 +a=rtpmap:39 H264/90000 +a=rtcp-fb:39 goog-remb +a=rtcp-fb:39 transport-cc +a=rtcp-fb:39 ccm fir +a=rtcp-fb:39 nack +a=rtcp-fb:39 nack pli +a=fmtp:39 level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=4d001f +a=rtpmap:40 rtx/90000 +a=fmtp:40 apt=39 +a=rtpmap:45 AV1/90000 +a=rtcp-fb:45 goog-remb +a=rtcp-fb:45 transport-cc +a=rtcp-fb:45 ccm fir +a=rtcp-fb:45 nack +a=rtcp-fb:45 nack pli +a=fmtp:45 level-idx=5;profile=0;tier=0 +a=rtpmap:46 rtx/90000 +a=fmtp:46 apt=45 +a=rtpmap:98 VP9/90000 +a=rtcp-fb:98 goog-remb +a=rtcp-fb:98 transport-cc +a=rtcp-fb:98 ccm fir +a=rtcp-fb:98 nack +a=rtcp-fb:98 nack pli +a=fmtp:98 profile-id=0 +a=rtpmap:99 rtx/90000 +a=fmtp:99 apt=98 +a=rtpmap:100 VP9/90000 +a=rtcp-fb:100 goog-remb +a=rtcp-fb:100 transport-cc +a=rtcp-fb:100 ccm fir +a=rtcp-fb:100 nack +a=rtcp-fb:100 nack pli +a=fmtp:100 profile-id=2 +a=rtpmap:101 rtx/90000 +a=fmtp:101 apt=100 +a=rtpmap:112 H264/90000 +a=rtcp-fb:112 goog-remb +a=rtcp-fb:112 transport-cc +a=rtcp-fb:112 ccm fir +a=rtcp-fb:112 nack +a=rtcp-fb:112 nack pli +a=fmtp:112 level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=64001f +a=rtpmap:113 rtx/90000 +a=fmtp:113 apt=112 +a=rtpmap:116 red/90000 +a=rtpmap:117 rtx/90000 +a=fmtp:117 apt=116 +a=rtpmap:118 ulpfec/90000 +a=ssrc-group:FID 3856340930 368490694 +a=ssrc:3856340930 cname:BnMV9wuzGoHGeFUb +a=ssrc:3856340930 msid:- 22bcd911-0a07-4b69-9557-8f4746411089 +a=ssrc:368490694 cname:BnMV9wuzGoHGeFUb +a=ssrc:368490694 msid:- 22bcd911-0a07-4b69-9557-8f4746411089 +m=audio 9 UDP/TLS/RTP/SAVPF 111 63 9 0 8 13 110 126 +c=IN IP4 0.0.0.0 +a=rtcp:9 IN IP4 0.0.0.0 +a=ice-ufrag:PeYM +a=ice-pwd:GDZQ30hwMK+5m/uVP3C0EQvf +a=ice-options:trickle +a=fingerprint:sha-256 E4:D3:D1:D8:70:08:6C:44:51:C5:0C:7E:AB:01:06:FF:10:A7:39:38:5B:43:C4:BA:FA:68:DA:AA:43:95:2C:32 +a=setup:actpass +a=mid:1 +a=extmap:14 urn:ietf:params:rtp-hdrext:ssrc-audio-level +a=extmap:2 http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time +a=extmap:4 http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01 +a=extmap:9 urn:ietf:params:rtp-hdrext:sdes:mid +a=sendonly +a=msid:- 440e2b2c-de05-4a72-9c30-317a0d18d32c +a=rtcp-mux +a=rtcp-rsize +a=rtpmap:111 opus/48000/2 +a=rtcp-fb:111 transport-cc +a=fmtp:111 minptime=10;useinbandfec=1 +a=rtpmap:63 red/48000/2 +a=fmtp:63 111/111 +a=rtpmap:9 G722/8000 +a=rtpmap:0 PCMU/8000 +a=rtpmap:8 PCMA/8000 +a=rtpmap:13 CN/8000 +a=rtpmap:110 telephone-event/48000 +a=rtpmap:126 telephone-event/8000 +a=ssrc:2019539886 cname:BnMV9wuzGoHGeFUb +a=ssrc:2019539886 msid:- 440e2b2c-de05-4a72-9c30-317a0d18d32c +`; diff --git a/packages/client/src/helpers/__tests__/sdp-munging.test.ts b/packages/client/src/helpers/__tests__/sdp-munging.test.ts index 8798fbcf51..1abfa86316 100644 --- a/packages/client/src/helpers/__tests__/sdp-munging.test.ts +++ b/packages/client/src/helpers/__tests__/sdp-munging.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from 'vitest'; -import { enableHighQualityAudio } from '../sdp-munging'; +import { enableHighQualityAudio, getPayloadTypeForCodec } from '../sdp-munging'; import { initialSdp as HQAudioSDP } from './hq-audio-sdp'; +import { publisherSDP } from './publisher-sdp.mock'; describe('sdp-munging', () => { it('enables HighQuality audio for Opus', () => { @@ -8,4 +9,13 @@ describe('sdp-munging', () => { expect(sdpWithHighQualityAudio).toContain('maxaveragebitrate=510000'); expect(sdpWithHighQualityAudio).toContain('stereo=1'); }); + + it('extracts payload type for codec', () => { + const payload = getPayloadTypeForCodec( + publisherSDP, + 'video/vp9', + 'profile-id=2', + ); + expect(payload).toBe(100); + }); }); diff --git a/packages/client/src/helpers/sdp-munging.ts b/packages/client/src/helpers/sdp-munging.ts index bca3fd10df..d0e799cb5d 100644 --- a/packages/client/src/helpers/sdp-munging.ts +++ b/packages/client/src/helpers/sdp-munging.ts @@ -47,6 +47,27 @@ export const enableHighQualityAudio = ( return SDP.write(parsedSdp); }; +/** + * Gets the payload type for the given codec. + */ +export const getPayloadTypeForCodec = ( + sdp: string, + mimeType: string, + fmtpLine: string | undefined, +): number => { + mimeType = mimeType.toLowerCase(); + const parsedSdp = SDP.parse(sdp); + const [kind, codec] = mimeType.split('/'); + const media = parsedSdp.media.find((m) => m.type === kind); + if (!media) return 0; + + const fmtp = media.fmtp.find((f) => f.config === fmtpLine); + const rtp = media.rtp.find( + (r) => r.codec.toLowerCase() === codec && r.payload === fmtp?.payload, + ); + return rtp?.payload ?? 0; +}; + /** * Extracts the mid from the transceiver or the SDP. * diff --git a/packages/client/src/rtc/codecs.ts b/packages/client/src/rtc/codecs.ts index 6de9fe50dd..b794b3d877 100644 --- a/packages/client/src/rtc/codecs.ts +++ b/packages/client/src/rtc/codecs.ts @@ -35,3 +35,18 @@ export const isSvcCodec = (codecOrMimeType: string | undefined) => { codecOrMimeType === 'video/av1' ); }; + +/** + * Returns whether the codec is supported + */ +export const findCodec = (mimeType: string): RTCRtpCodec | undefined => { + if (!('getCapabilities' in RTCRtpSender)) return; + + mimeType = mimeType.toLowerCase(); + const [kind] = mimeType.split('/'); + const capabilities = RTCRtpSender.getCapabilities(kind); + if (!capabilities) return; + + const { codecs } = capabilities; + return codecs.find((c) => c.mimeType.toLowerCase() === mimeType); +}; diff --git a/packages/client/src/types.ts b/packages/client/src/types.ts index 2f04ceb1dd..1a53fedc49 100644 --- a/packages/client/src/types.ts +++ b/packages/client/src/types.ts @@ -150,19 +150,17 @@ export type SubscriptionChanges = { * A preferred codec to use when publishing a video or audio track. * @internal */ -export type PreferredCodec = - | 'video/vp8' - | 'video/h264' - | 'video/vp9' - | 'video/av1' - | 'audio/opus' - | 'audio/red'; +export type PreferredCodec = 'vp8' | 'h264' | 'vp9' | 'av1'; /** * A collection of track publication options. * @internal */ export type ClientPublishOptions = { + /** + * The preferred codec to use when publishing the video stream. + */ + preferredCodec?: PreferredCodec; /** * Screen share settings. */ diff --git a/sample-apps/react/react-dogfood/components/MeetingUI.tsx b/sample-apps/react/react-dogfood/components/MeetingUI.tsx index afc60c8563..fd32bcb620 100644 --- a/sample-apps/react/react-dogfood/components/MeetingUI.tsx +++ b/sample-apps/react/react-dogfood/components/MeetingUI.tsx @@ -6,6 +6,7 @@ import { defaultSortPreset, LoadingIndicator, noopComparator, + PreferredCodec, useCall, useCallStateHooks, usePersistedDevicePreferences, @@ -45,11 +46,18 @@ export const MeetingUI = ({ chatClient, mode }: MeetingUIProps) => { const { useCallCallingState } = useCallStateHooks(); const callState = useCallCallingState(); + const videoCodecOverride = router.query['video_codec'] as + | PreferredCodec + | undefined; + const onJoin = useCallback( async ({ fastJoin = false } = {}) => { if (!fastJoin) setShow('loading'); if (!call) throw new Error('No active call found'); try { + call.updatePublishOptions({ + preferredCodec: videoCodecOverride || 'vp9', + }); await call.join({ create: true }); setShow('active-call'); } catch (e) { @@ -58,7 +66,7 @@ export const MeetingUI = ({ chatClient, mode }: MeetingUIProps) => { setShow('error-join'); } }, - [call], + [call, videoCodecOverride], ); const onLeave = useCallback( From b463392d5cb4fad6328749b9c2d4b6169b0d0d8e Mon Sep 17 00:00:00 2001 From: Oliver Lazoroski Date: Thu, 14 Nov 2024 15:17:16 +0100 Subject: [PATCH 22/52] fix: correct mid --- packages/client/src/rtc/Publisher.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/client/src/rtc/Publisher.ts b/packages/client/src/rtc/Publisher.ts index 715251ed39..7765512cdd 100644 --- a/packages/client/src/rtc/Publisher.ts +++ b/packages/client/src/rtc/Publisher.ts @@ -520,7 +520,9 @@ export class Publisher extends BasePeerConnection { const trackSettings = track.getSettings(); const isStereo = isAudioTrack && trackSettings.channelCount === 2; const transceiverIndex = this.transceiverOrder.indexOf(transceiver); - const mid = extractMid(transceiver, transceiverIndex, sdp); + const mid = + String(transceiverIndex) || + extractMid(transceiver, transceiverIndex, sdp); return { trackId: track.id, From fd81f09d336ca49de863342693b19f4452566fee Mon Sep 17 00:00:00 2001 From: Oliver Lazoroski Date: Thu, 14 Nov 2024 15:40:11 +0100 Subject: [PATCH 23/52] feat: use the latest protocol (preferred_publish_options) --- packages/client/src/Call.ts | 35 ++++++----- .../client/src/gen/video/sfu/event/events.ts | 59 ++++++++++--------- .../client/src/gen/video/sfu/models/models.ts | 28 --------- packages/client/src/rtc/Publisher.ts | 19 +++--- 4 files changed, 61 insertions(+), 80 deletions(-) diff --git a/packages/client/src/Call.ts b/packages/client/src/Call.ts index aa16820c15..a2007f7588 100644 --- a/packages/client/src/Call.ts +++ b/packages/client/src/Call.ts @@ -92,7 +92,7 @@ import { ReconnectDetails } from './gen/video/sfu/event/events'; import { ClientDetails, Codec, - PublishOptions, + PublishOption, TrackType, WebsocketReconnectStrategy, } from './gen/video/sfu/models/models'; @@ -204,7 +204,7 @@ export class Call { private readonly dispatcher = new Dispatcher(); private clientPublishOptions?: ClientPublishOptions; - private initialPublishOptions?: PublishOptions; + private initialPublishOptions?: PublishOption[]; private statsReporter?: StatsReporter; private sfuStatsReporter?: SfuStatsReporter; private dropTimeout: ReturnType | undefined; @@ -803,7 +803,9 @@ export class Call { clientDetails, fastReconnect: performingFastReconnect, reconnectDetails, - preferredCodec: this.getPreferredCodec(publishingCapabilitiesSdp), + preferredPublishOptions: this.getPreferredCodec( + publishingCapabilitiesSdp, + ), }); this.initialPublishOptions = publishOptions; @@ -836,7 +838,7 @@ export class Call { connectionConfig, clientDetails, statsOptions, - publishOptions: this.initialPublishOptions!, + publishOptions: this.initialPublishOptions || [], closePreviousInstances: !performingMigration, }); } @@ -906,20 +908,25 @@ export class Call { * This is an experimental client feature and subject to change. * @internal */ - private getPreferredCodec = (sdp: string): Codec | undefined => { + private getPreferredCodec = (sdp: string): PublishOption[] => { const { preferredCodec } = this.clientPublishOptions || {}; - if (!preferredCodec) return; + if (!preferredCodec) return []; const codec = findCodec(`video/${preferredCodec}`); - if (!codec) return; + if (!codec) return []; const { clockRate, mimeType, sdpFmtpLine } = codec; - return Codec.create({ - name: preferredCodec, // e.g. 'vp9' - fmtp: sdpFmtpLine || '', - clockRate: clockRate, - payloadType: getPayloadTypeForCodec(sdp, mimeType, sdpFmtpLine), - }); + return [ + PublishOption.create({ + trackType: TrackType.VIDEO, + codec: Codec.create({ + name: preferredCodec, // e.g. 'vp9' + fmtp: sdpFmtpLine || '', + clockRate: clockRate, + payloadType: getPayloadTypeForCodec(sdp, mimeType, sdpFmtpLine), + }), + }), + ]; }; /** @@ -960,7 +967,7 @@ export class Call { connectionConfig: RTCConfiguration; statsOptions: StatsOptions; clientDetails: ClientDetails; - publishOptions: PublishOptions; + publishOptions: PublishOption[]; closePreviousInstances: boolean; }) => { const { diff --git a/packages/client/src/gen/video/sfu/event/events.ts b/packages/client/src/gen/video/sfu/event/events.ts index a51e701c15..4ac6478bff 100644 --- a/packages/client/src/gen/video/sfu/event/events.ts +++ b/packages/client/src/gen/video/sfu/event/events.ts @@ -3,29 +3,25 @@ // @generated from protobuf file "video/sfu/event/events.proto" (package "stream.video.sfu.event", syntax proto3) // tslint:disable import { MessageType } from '@protobuf-ts/runtime'; -import { - CallEndedReason, - CallGrants, - CallState, - ClientDetails, - Codec, - ConnectionQuality, - Error as Error$, - GoAwayReason, - ICETrickle as ICETrickle$, - Participant, - ParticipantCount, - PeerType, - Pin, - PublishOption, - PublishOptions, - TrackInfo, - TrackType, - TrackUnpublishReason, - WebsocketReconnectStrategy, -} from '../models/models'; +import { CallEndedReason } from '../models/models'; +import { GoAwayReason } from '../models/models'; +import { CallGrants } from '../models/models'; +import { Codec } from '../models/models'; +import { ConnectionQuality } from '../models/models'; +import { CallState } from '../models/models'; import { TrackSubscriptionDetails } from '../signal_rpc/signal'; - +import { TrackInfo } from '../models/models'; +import { ClientDetails } from '../models/models'; +import { TrackUnpublishReason } from '../models/models'; +import { Participant } from '../models/models'; +import { TrackType } from '../models/models'; +import { ParticipantCount } from '../models/models'; +import { PeerType } from '../models/models'; +import { WebsocketReconnectStrategy } from '../models/models'; +import { Error as Error$ } from '../models/models'; +import { Pin } from '../models/models'; +import { PublishOption } from '../models/models'; +import { ICETrickle as ICETrickle$ } from '../models/models'; /** * SFUEvent is a message that is sent from the SFU to the client. * @@ -508,9 +504,9 @@ export interface JoinRequest { */ reconnectDetails?: ReconnectDetails; /** - * @generated from protobuf field: stream.video.sfu.models.Codec preferred_codec = 9; + * @generated from protobuf field: repeated stream.video.sfu.models.PublishOption preferred_publish_options = 9; */ - preferredCodec?: Codec; + preferredPublishOptions: PublishOption[]; } /** * @generated from protobuf message stream.video.sfu.event.ReconnectDetails @@ -577,9 +573,9 @@ export interface JoinResponse { */ fastReconnectDeadlineSeconds: number; /** - * @generated from protobuf field: stream.video.sfu.models.PublishOptions publish_options = 4; + * @generated from protobuf field: repeated stream.video.sfu.models.PublishOption publish_options = 4; */ - publishOptions?: PublishOptions; + publishOptions: PublishOption[]; } /** * ParticipantJoined is fired when a user joins a call @@ -1313,7 +1309,13 @@ class JoinRequest$Type extends MessageType { kind: 'message', T: () => ReconnectDetails, }, - { no: 9, name: 'preferred_codec', kind: 'message', T: () => Codec }, + { + no: 9, + name: 'preferred_publish_options', + kind: 'message', + repeat: 1 /*RepeatType.PACKED*/, + T: () => PublishOption, + }, ]); } } @@ -1421,7 +1423,8 @@ class JoinResponse$Type extends MessageType { no: 4, name: 'publish_options', kind: 'message', - T: () => PublishOptions, + repeat: 1 /*RepeatType.PACKED*/, + T: () => PublishOption, }, ]); } diff --git a/packages/client/src/gen/video/sfu/models/models.ts b/packages/client/src/gen/video/sfu/models/models.ts index f8849e39b2..a2edbbadcd 100644 --- a/packages/client/src/gen/video/sfu/models/models.ts +++ b/packages/client/src/gen/video/sfu/models/models.ts @@ -5,7 +5,6 @@ import { MessageType } from '@protobuf-ts/runtime'; import { Struct } from '../../../google/protobuf/struct'; import { Timestamp } from '../../../google/protobuf/timestamp'; - /** * CallState is the current state of the call * as seen by an SFU. @@ -194,15 +193,6 @@ export interface VideoLayer { */ quality: VideoQuality; } -/** - * @generated from protobuf message stream.video.sfu.models.PublishOptions - */ -export interface PublishOptions { - /** - * @generated from protobuf field: repeated stream.video.sfu.models.PublishOption codecs = 1; - */ - codecs: PublishOption[]; -} /** * @generated from protobuf message stream.video.sfu.models.PublishOption */ @@ -1103,24 +1093,6 @@ class VideoLayer$Type extends MessageType { */ export const VideoLayer = new VideoLayer$Type(); // @generated message type with reflection information, may provide speed optimized methods -class PublishOptions$Type extends MessageType { - constructor() { - super('stream.video.sfu.models.PublishOptions', [ - { - no: 1, - name: 'codecs', - kind: 'message', - repeat: 1 /*RepeatType.PACKED*/, - T: () => PublishOption, - }, - ]); - } -} -/** - * @generated MessageType for protobuf message stream.video.sfu.models.PublishOptions - */ -export const PublishOptions = new PublishOptions$Type(); -// @generated message type with reflection information, may provide speed optimized methods class PublishOption$Type extends MessageType { constructor() { super('stream.video.sfu.models.PublishOption', [ diff --git a/packages/client/src/rtc/Publisher.ts b/packages/client/src/rtc/Publisher.ts index 7765512cdd..bfe2f10594 100644 --- a/packages/client/src/rtc/Publisher.ts +++ b/packages/client/src/rtc/Publisher.ts @@ -5,7 +5,6 @@ import { import { PeerType, PublishOption, - PublishOptions, TrackInfo, TrackType, VideoLayer, @@ -25,7 +24,7 @@ import { TargetResolutionResponse } from '../gen/shims'; import { withoutConcurrency } from '../helpers/concurrency'; export type PublisherConstructorOpts = BasePeerConnectionOpts & { - publishOptions: PublishOptions; + publishOptions: PublishOption[]; }; /** @@ -49,7 +48,7 @@ export class Publisher extends BasePeerConnection { private readonly unsubscribeChangePublishOptions: () => void; private unsubscribeCodecNegotiationComplete?: () => void; - private publishOptions: PublishOptions; + private publishOptions: PublishOption[]; /** * Constructs a new `Publisher` instance. @@ -90,11 +89,10 @@ export class Publisher extends BasePeerConnection { ({ publishOption }) => { withoutConcurrency('publisher.changePublishOptions', async () => { if (!publishOption) return; - this.publishOptions.codecs = this.publishOptions.codecs.map( - (option) => - option.trackType === publishOption.trackType - ? publishOption - : option, + this.publishOptions = this.publishOptions.map((option) => + option.trackType === publishOption.trackType + ? publishOption + : option, ); if (this.isPublishing(publishOption.trackType)) { this.switchCodec(publishOption); @@ -539,8 +537,9 @@ export class Publisher extends BasePeerConnection { }; private getPublishOptionFor = (trackType: TrackType) => { - const options = this.publishOptions.codecs; - const publishOption = options.find((o) => o.trackType === trackType); + const publishOption = this.publishOptions.find( + (option) => option.trackType === trackType, + ); if (!publishOption) { throw new Error(`No publish options found for ${TrackType[trackType]}`); } From 4f8121e9010faef42a9c83e291c00c43ce3a37d9 Mon Sep 17 00:00:00 2001 From: Oliver Lazoroski Date: Thu, 14 Nov 2024 17:30:01 +0100 Subject: [PATCH 24/52] chore: restore overrides --- packages/client/src/Call.ts | 30 +++++++++++-------- packages/client/src/types.ts | 8 +++++ .../dogfood/src/components/MeetingUI.tsx | 3 ++ .../react-dogfood/components/MeetingUI.tsx | 13 +++++++- 4 files changed, 41 insertions(+), 13 deletions(-) diff --git a/packages/client/src/Call.ts b/packages/client/src/Call.ts index a2007f7588..5f6b84f7bf 100644 --- a/packages/client/src/Call.ts +++ b/packages/client/src/Call.ts @@ -803,7 +803,7 @@ export class Call { clientDetails, fastReconnect: performingFastReconnect, reconnectDetails, - preferredPublishOptions: this.getPreferredCodec( + preferredPublishOptions: this.getPreferredCodecs( publishingCapabilitiesSdp, ), }); @@ -908,23 +908,29 @@ export class Call { * This is an experimental client feature and subject to change. * @internal */ - private getPreferredCodec = (sdp: string): PublishOption[] => { - const { preferredCodec } = this.clientPublishOptions || {}; - if (!preferredCodec) return []; + private getPreferredCodecs = (sdp: string): PublishOption[] => { + const { preferredCodec, preferredBitrate, maxSimulcastLayers } = + this.clientPublishOptions || {}; + if (!preferredCodec || !preferredBitrate || !maxSimulcastLayers) return []; + let sfuCodec: Codec | undefined; const codec = findCodec(`video/${preferredCodec}`); - if (!codec) return []; + if (codec) { + const { clockRate, mimeType, sdpFmtpLine } = codec; + sfuCodec = Codec.create({ + name: preferredCodec, // e.g. 'vp9' + fmtp: sdpFmtpLine || '', + clockRate: clockRate, + payloadType: getPayloadTypeForCodec(sdp, mimeType, sdpFmtpLine), + }); + } - const { clockRate, mimeType, sdpFmtpLine } = codec; return [ PublishOption.create({ trackType: TrackType.VIDEO, - codec: Codec.create({ - name: preferredCodec, // e.g. 'vp9' - fmtp: sdpFmtpLine || '', - clockRate: clockRate, - payloadType: getPayloadTypeForCodec(sdp, mimeType, sdpFmtpLine), - }), + codec: sfuCodec, + bitrate: preferredBitrate, + maxSpatialLayers: maxSimulcastLayers, }), ]; }; diff --git a/packages/client/src/types.ts b/packages/client/src/types.ts index 1a53fedc49..a410501dab 100644 --- a/packages/client/src/types.ts +++ b/packages/client/src/types.ts @@ -161,6 +161,14 @@ export type ClientPublishOptions = { * The preferred codec to use when publishing the video stream. */ preferredCodec?: PreferredCodec; + /** + * The preferred bitrate to use when publishing the video stream. + */ + preferredBitrate?: number; + /** + * The maximum number of simulcast layers to use when publishing the video stream. + */ + maxSimulcastLayers?: number; /** * Screen share settings. */ diff --git a/sample-apps/react-native/dogfood/src/components/MeetingUI.tsx b/sample-apps/react-native/dogfood/src/components/MeetingUI.tsx index dcae405b3e..378270ab03 100644 --- a/sample-apps/react-native/dogfood/src/components/MeetingUI.tsx +++ b/sample-apps/react-native/dogfood/src/components/MeetingUI.tsx @@ -59,6 +59,9 @@ export const MeetingUI = ({ callId, navigation, route }: Props) => { const onJoinCallHandler = useCallback(async () => { try { setShow('loading'); + call?.updatePublishOptions({ + preferredCodec: 'vp9', + }); await call?.join({ create: true }); appStoreSetState({ chatLabelNoted: false }); setShow('active-call'); diff --git a/sample-apps/react/react-dogfood/components/MeetingUI.tsx b/sample-apps/react/react-dogfood/components/MeetingUI.tsx index fd32bcb620..8cf694fda5 100644 --- a/sample-apps/react/react-dogfood/components/MeetingUI.tsx +++ b/sample-apps/react/react-dogfood/components/MeetingUI.tsx @@ -49,14 +49,25 @@ export const MeetingUI = ({ chatClient, mode }: MeetingUIProps) => { const videoCodecOverride = router.query['video_codec'] as | PreferredCodec | undefined; + const bitrateOverride = router.query['bitrate'] as string | undefined; + const maxSimulcastLayers = router.query['max_simulcast_layers'] as + | string + | undefined; const onJoin = useCallback( async ({ fastJoin = false } = {}) => { if (!fastJoin) setShow('loading'); if (!call) throw new Error('No active call found'); try { + const preferredBitrate = bitrateOverride + ? parseInt(bitrateOverride, 10) + : undefined; call.updatePublishOptions({ preferredCodec: videoCodecOverride || 'vp9', + preferredBitrate, + maxSimulcastLayers: maxSimulcastLayers + ? parseInt(maxSimulcastLayers, 10) + : 3, // default to 3 }); await call.join({ create: true }); setShow('active-call'); @@ -66,7 +77,7 @@ export const MeetingUI = ({ chatClient, mode }: MeetingUIProps) => { setShow('error-join'); } }, - [call, videoCodecOverride], + [bitrateOverride, call, maxSimulcastLayers, videoCodecOverride], ); const onLeave = useCallback( From 8422c04c5a91a851da382a1f9c3df226a6f69293 Mon Sep 17 00:00:00 2001 From: Oliver Lazoroski Date: Thu, 14 Nov 2024 17:34:47 +0100 Subject: [PATCH 25/52] chore: fix condition --- packages/client/src/Call.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/client/src/Call.ts b/packages/client/src/Call.ts index 5f6b84f7bf..0e6ce7028b 100644 --- a/packages/client/src/Call.ts +++ b/packages/client/src/Call.ts @@ -911,7 +911,7 @@ export class Call { private getPreferredCodecs = (sdp: string): PublishOption[] => { const { preferredCodec, preferredBitrate, maxSimulcastLayers } = this.clientPublishOptions || {}; - if (!preferredCodec || !preferredBitrate || !maxSimulcastLayers) return []; + if (!preferredCodec && !preferredBitrate && !maxSimulcastLayers) return []; let sfuCodec: Codec | undefined; const codec = findCodec(`video/${preferredCodec}`); From 9885b11522258e7af85c90383362b9dfdc6ac72d Mon Sep 17 00:00:00 2001 From: Oliver Lazoroski Date: Thu, 14 Nov 2024 19:07:28 +0100 Subject: [PATCH 26/52] chore: clone track before republishing, prevent concurrent negotiations --- packages/client/src/rtc/Publisher.ts | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/packages/client/src/rtc/Publisher.ts b/packages/client/src/rtc/Publisher.ts index bfe2f10594..dd5df63d2a 100644 --- a/packages/client/src/rtc/Publisher.ts +++ b/packages/client/src/rtc/Publisher.ts @@ -236,7 +236,7 @@ export class Publisher extends BasePeerConnection { onNegotiationComplete, ); - const track = transceiver.sender.track; + const track = transceiver.sender.track.clone(); this.addTransceiver(trackType, track, publishOption); }; @@ -411,10 +411,12 @@ export class Publisher extends BasePeerConnection { }; private onNegotiationNeeded = () => { - this.negotiate().catch((err) => { - this.logger('error', `Negotiation failed.`, err); - this.onUnrecoverableError?.(); - }); + withoutConcurrency('publisher.negotiate', () => this.negotiate()).catch( + (err) => { + this.logger('error', `Negotiation failed.`, err); + this.onUnrecoverableError?.(); + }, + ); }; /** From 9acd4939448da60a4ab71a39dc2b119295305f23 Mon Sep 17 00:00:00 2001 From: Oliver Lazoroski Date: Mon, 18 Nov 2024 16:35:12 +0100 Subject: [PATCH 27/52] fix tests --- .../src/rtc/__tests__/Publisher.test.ts | 24 +++++++++---------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/packages/client/src/rtc/__tests__/Publisher.test.ts b/packages/client/src/rtc/__tests__/Publisher.test.ts index 9022b67b75..6b7bc7ec18 100644 --- a/packages/client/src/rtc/__tests__/Publisher.test.ts +++ b/packages/client/src/rtc/__tests__/Publisher.test.ts @@ -70,19 +70,17 @@ describe('Publisher', () => { dispatcher, state, logTag: 'test', - publishOptions: { - codecs: [ - { - trackType: TrackType.VIDEO, - bitrate: 1000, - // @ts-expect-error - incomplete data - codec: { name: 'vp9' }, - fps: 30, - maxTemporalLayers: 3, - maxSpatialLayers: 0, - }, - ], - }, + publishOptions: [ + { + trackType: TrackType.VIDEO, + bitrate: 1000, + // @ts-expect-error - incomplete data + codec: { name: 'vp9' }, + fps: 30, + maxTemporalLayers: 3, + maxSpatialLayers: 0, + }, + ], }); }); From 6a32edbca8e03cfb112db49d736add45b0503505 Mon Sep 17 00:00:00 2001 From: Oliver Lazoroski Date: Mon, 18 Nov 2024 16:37:49 +0100 Subject: [PATCH 28/52] feat: per track codec report --- packages/client/src/rtc/Publisher.ts | 16 +++++++-- .../src/stats/stateStoreStatsReporter.ts | 34 +++++++++++++++++-- packages/client/src/stats/types.ts | 12 +++++++ .../src/components/CallStats/CallStats.tsx | 9 +++-- 4 files changed, 63 insertions(+), 8 deletions(-) diff --git a/packages/client/src/rtc/Publisher.ts b/packages/client/src/rtc/Publisher.ts index dd5df63d2a..f7cb654a88 100644 --- a/packages/client/src/rtc/Publisher.ts +++ b/packages/client/src/rtc/Publisher.ts @@ -273,6 +273,18 @@ export class Publisher extends BasePeerConnection { return track.readyState === 'live' && track.enabled; }; + /** + * Maps the given track ID to the corresponding track type. + */ + getTrackType = (trackId: string): TrackType | undefined => { + for (const [trackType, transceiver] of this.transceiverCache) { + if (transceiver.sender.track?.id === trackId) { + return trackType; + } + } + return undefined; + }; + private notifyTrackMuteStateChanged = async ( mediaStream: MediaStream | undefined, trackType: TrackType, @@ -520,9 +532,7 @@ export class Publisher extends BasePeerConnection { const trackSettings = track.getSettings(); const isStereo = isAudioTrack && trackSettings.channelCount === 2; const transceiverIndex = this.transceiverOrder.indexOf(transceiver); - const mid = - String(transceiverIndex) || - extractMid(transceiver, transceiverIndex, sdp); + const mid = extractMid(transceiver, transceiverIndex, sdp); return { trackId: track.id, diff --git a/packages/client/src/stats/stateStoreStatsReporter.ts b/packages/client/src/stats/stateStoreStatsReporter.ts index b3bfa4a1ec..4284a6d5b6 100644 --- a/packages/client/src/stats/stateStoreStatsReporter.ts +++ b/packages/client/src/stats/stateStoreStatsReporter.ts @@ -2,12 +2,14 @@ import type { AggregatedStatsReport, BaseStats, ParticipantsStatsReport, + RTCMediaSourceStats, StatsReport, } from './types'; import { CallState } from '../store'; import { Publisher, Subscriber } from '../rtc'; import { getLogger } from '../logger'; import { flatten } from './utils'; +import { TrackType } from '../gen/video/sfu/models/models'; export type StatsReporterOpts = { subscriber: Subscriber; @@ -157,6 +159,7 @@ export const createStatsReporter = ({ transform(report, { kind: 'subscriber', trackKind: 'video', + publisher, }), ) .then(aggregate), @@ -167,6 +170,7 @@ export const createStatsReporter = ({ transform(report, { kind: 'publisher', trackKind: 'video', + publisher, }), ) .then(aggregate) @@ -220,11 +224,14 @@ export type StatsTransformOpts = { * The kind of track we are transforming stats for. */ trackKind: 'audio' | 'video'; - /** * The kind of peer connection we are transforming stats for. */ kind: 'subscriber' | 'publisher'; + /** + * The publisher instance. + */ + publisher: Publisher | undefined; }; /** @@ -237,7 +244,7 @@ const transform = ( report: RTCStatsReport, opts: StatsTransformOpts, ): StatsReport => { - const { trackKind, kind } = opts; + const { trackKind, kind, publisher } = opts; const direction = kind === 'subscriber' ? 'inbound-rtp' : 'outbound-rtp'; const stats = flatten(report); const streams = stats @@ -268,6 +275,17 @@ const transform = ( roundTripTime = candidatePair?.currentRoundTripTime; } + let trackType: TrackType | undefined; + if (kind === 'publisher' && publisher && rtcStreamStats.mediaSourceId) { + const mediaSource = stats.find( + (s) => + s.type === 'media-source' && s.id === rtcStreamStats.mediaSourceId, + ) as RTCMediaSourceStats | undefined; + if (mediaSource) { + trackType = publisher.getTrackType(mediaSource.trackIdentifier); + } + } + return { bytesSent: rtcStreamStats.bytesSent, bytesReceived: rtcStreamStats.bytesReceived, @@ -278,10 +296,12 @@ const transform = ( framesPerSecond: rtcStreamStats.framesPerSecond, jitter: rtcStreamStats.jitter, kind: rtcStreamStats.kind, + mediaSourceId: rtcStreamStats.mediaSourceId, // @ts-ignore: available in Chrome only, TS doesn't recognize this qualityLimitationReason: rtcStreamStats.qualityLimitationReason, rid: rtcStreamStats.rid, ssrc: rtcStreamStats.ssrc, + trackType, }; }); @@ -304,6 +324,7 @@ const getEmptyStats = (stats?: StatsReport): AggregatedStatsReport => { highestFrameHeight: 0, highestFramesPerSecond: 0, codec: '', + codecPerTrackType: {}, timestamp: Date.now(), }; }; @@ -349,6 +370,15 @@ const aggregate = (stats: StatsReport): AggregatedStatsReport => { ); // we take the first codec we find, as it should be the same for all streams report.codec = streams[0].codec || ''; + report.codecPerTrackType = streams.reduce( + (acc, stream) => { + if (stream.trackType) { + acc[stream.trackType] = stream.codec || ''; + } + return acc; + }, + {} as Record, + ); } const qualityLimitationReason = [ diff --git a/packages/client/src/stats/types.ts b/packages/client/src/stats/types.ts index 6a1914e6fa..33731b6154 100644 --- a/packages/client/src/stats/types.ts +++ b/packages/client/src/stats/types.ts @@ -1,3 +1,5 @@ +import { TrackType } from '../gen/video/sfu/models/models'; + export type BaseStats = { audioLevel?: number; bytesSent?: number; @@ -9,9 +11,11 @@ export type BaseStats = { framesPerSecond?: number; jitter?: number; kind?: string; + mediaSourceId?: string; qualityLimitationReason?: string; rid?: string; ssrc?: number; + trackType?: TrackType; }; export type StatsReport = { @@ -30,6 +34,7 @@ export type AggregatedStatsReport = { highestFrameHeight: number; highestFramesPerSecond: number; codec: string; + codecPerTrackType: Partial>; timestamp: number; rawReport: StatsReport; }; @@ -48,3 +53,10 @@ export type CallStatsReport = { participants: ParticipantsStatsReport; timestamp: number; }; + +// shim for RTCMediaSourceStats, not yet available in the standard types +// https://www.w3.org/TR/webrtc-stats/#mediasourcestats-dict* +export interface RTCMediaSourceStats { + kind: string; + trackIdentifier: string; +} diff --git a/packages/react-sdk/src/components/CallStats/CallStats.tsx b/packages/react-sdk/src/components/CallStats/CallStats.tsx index dad0a886cd..c5b8ded760 100644 --- a/packages/react-sdk/src/components/CallStats/CallStats.tsx +++ b/packages/react-sdk/src/components/CallStats/CallStats.tsx @@ -3,6 +3,7 @@ import clsx from 'clsx'; import { AggregatedStatsReport, CallStatsReport, + SfuModels, } from '@stream-io/video-client'; import { useCallStateHooks, useI18n } from '@stream-io/video-react-bindings'; import { useFloating, useHover, useInteractions } from '@floating-ui/react'; @@ -270,9 +271,11 @@ const toFrameSize = (stats: AggregatedStatsReport) => { }; const formatCodec = (callStatsReport: CallStatsReport): string => { - const { codec } = callStatsReport.publisherStats; - if (!codec) return ''; - const [, name] = codec.split('/'); + const { codecPerTrackType } = callStatsReport.publisherStats; + if (!codecPerTrackType || !codecPerTrackType[SfuModels.TrackType.VIDEO]) { + return ''; + } + const [, name] = codecPerTrackType[SfuModels.TrackType.VIDEO].split('/'); return name ? ` (${name})` : ''; }; From fa9d1ffbe0113083312a157a98e75f099044ca31 Mon Sep 17 00:00:00 2001 From: Oliver Lazoroski Date: Mon, 18 Nov 2024 17:24:49 +0100 Subject: [PATCH 29/52] feat: per track codec report (firefox) --- packages/client/src/stats/stateStoreStatsReporter.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/packages/client/src/stats/stateStoreStatsReporter.ts b/packages/client/src/stats/stateStoreStatsReporter.ts index 4284a6d5b6..7b238fafe7 100644 --- a/packages/client/src/stats/stateStoreStatsReporter.ts +++ b/packages/client/src/stats/stateStoreStatsReporter.ts @@ -10,6 +10,7 @@ import { Publisher, Subscriber } from '../rtc'; import { getLogger } from '../logger'; import { flatten } from './utils'; import { TrackType } from '../gen/video/sfu/models/models'; +import { isFirefox } from '../helpers/browsers'; export type StatsReporterOpts = { subscriber: Subscriber; @@ -276,10 +277,13 @@ const transform = ( } let trackType: TrackType | undefined; - if (kind === 'publisher' && publisher && rtcStreamStats.mediaSourceId) { + if (kind === 'publisher' && publisher) { + const firefox = isFirefox(); const mediaSource = stats.find( (s) => - s.type === 'media-source' && s.id === rtcStreamStats.mediaSourceId, + s.type === 'media-source' && + // Firefox doesn't have mediaSourceId, so we need to guess the media source + (firefox ? true : s.id === rtcStreamStats.mediaSourceId), ) as RTCMediaSourceStats | undefined; if (mediaSource) { trackType = publisher.getTrackType(mediaSource.trackIdentifier); From 147356b9088c2ab9f95ca794eafa287c1779c3af Mon Sep 17 00:00:00 2001 From: Oliver Lazoroski Date: Thu, 21 Nov 2024 17:39:38 +0100 Subject: [PATCH 30/52] fix: screen share support --- packages/client/src/Call.ts | 20 +++++-- packages/client/src/rtc/Publisher.ts | 55 ++++++++----------- .../src/rtc/__tests__/videoLayers.test.ts | 33 ----------- packages/client/src/rtc/helpers/tracks.ts | 18 +----- packages/client/src/rtc/videoLayers.ts | 35 +++--------- 5 files changed, 45 insertions(+), 116 deletions(-) diff --git a/packages/client/src/Call.ts b/packages/client/src/Call.ts index 0e6ce7028b..b46c02d5a0 100644 --- a/packages/client/src/Call.ts +++ b/packages/client/src/Call.ts @@ -925,7 +925,7 @@ export class Call { }); } - return [ + const preferredPublishOptions = [ PublishOption.create({ trackType: TrackType.VIDEO, codec: sfuCodec, @@ -933,6 +933,19 @@ export class Call { maxSpatialLayers: maxSimulcastLayers, }), ]; + + const screenShareSettings = this.screenShare.getSettings(); + if (screenShareSettings) { + preferredPublishOptions.push( + PublishOption.create({ + trackType: TrackType.SCREEN_SHARE, + fps: screenShareSettings.maxFramerate, + bitrate: screenShareSettings.maxBitrate, + }), + ); + } + + return preferredPublishOptions; }; /** @@ -1498,14 +1511,10 @@ export class Call { if (!this.trackPublishOrder.includes(TrackType.SCREEN_SHARE)) { this.trackPublishOrder.push(TrackType.SCREEN_SHARE); } - // const opts: ClientPublishOptions = { - // screenShareSettings: this.screenShare.getSettings(), - // }; await this.publisher.publishStream( screenShareStream, screenShareTrack, TrackType.SCREEN_SHARE, - // opts, ); const [screenShareAudioTrack] = screenShareStream.getAudioTracks(); @@ -1517,7 +1526,6 @@ export class Call { screenShareStream, screenShareAudioTrack, TrackType.SCREEN_SHARE_AUDIO, - // opts, ); } }; diff --git a/packages/client/src/rtc/Publisher.ts b/packages/client/src/rtc/Publisher.ts index f7cb654a88..5a012e824c 100644 --- a/packages/client/src/rtc/Publisher.ts +++ b/packages/client/src/rtc/Publisher.ts @@ -10,17 +10,18 @@ import { VideoLayer, } from '../gen/video/sfu/models/models'; import { - findOptimalScreenSharingLayers, findOptimalVideoLayers, OptimalVideoLayer, ridToVideoQuality, toSvcEncodings, } from './videoLayers'; import { isSvcCodec } from './codecs'; -import { trackTypeToParticipantStreamKey } from './helpers/tracks'; +import { + isAudioTrackType, + trackTypeToParticipantStreamKey, +} from './helpers/tracks'; import { enableHighQualityAudio, extractMid } from '../helpers/sdp-munging'; import { VideoLayerSetting } from '../gen/video/sfu/event/events'; -import { TargetResolutionResponse } from '../gen/shims'; import { withoutConcurrency } from '../helpers/concurrency'; export type PublisherConstructorOpts = BasePeerConnectionOpts & { @@ -89,10 +90,8 @@ export class Publisher extends BasePeerConnection { ({ publishOption }) => { withoutConcurrency('publisher.changePublishOptions', async () => { if (!publishOption) return; - this.publishOptions = this.publishOptions.map((option) => - option.trackType === publishOption.trackType - ? publishOption - : option, + this.publishOptions = this.publishOptions.map((o) => + o.trackType === publishOption.trackType ? publishOption : o, ); if (this.isPublishing(publishOption.trackType)) { this.switchCodec(publishOption); @@ -285,6 +284,7 @@ export class Publisher extends BasePeerConnection { return undefined; }; + // FIXME move to InputMediaDeviceManager private notifyTrackMuteStateChanged = async ( mediaStream: MediaStream | undefined, trackType: TrackType, @@ -521,56 +521,45 @@ export class Publisher extends BasePeerConnection { }, })); - const isAudioTrack = - trackType === TrackType.AUDIO || - trackType === TrackType.SCREEN_SHARE_AUDIO; - - const audioSettings = this.state.settings?.audio; - const isDtxEnabled = !!audioSettings?.opus_dtx_enabled; - const isRedEnabled = !!audioSettings?.redundant_coding_enabled; - + const isAudioTrack = isAudioTrackType(trackType); const trackSettings = track.getSettings(); const isStereo = isAudioTrack && trackSettings.channelCount === 2; const transceiverIndex = this.transceiverOrder.indexOf(transceiver); const mid = extractMid(transceiver, transceiverIndex, sdp); + const audioSettings = this.state.settings?.audio; return { trackId: track.id, layers, trackType, mid, stereo: isStereo, - dtx: isAudioTrack && isDtxEnabled, - red: isAudioTrack && isRedEnabled, + dtx: isAudioTrack && !!audioSettings?.opus_dtx_enabled, + red: isAudioTrack && !!audioSettings?.redundant_coding_enabled, muted: !isTrackLive, }; }) .filter(Boolean) as TrackInfo[]; }; - private getPublishOptionFor = (trackType: TrackType) => { - const publishOption = this.publishOptions.find( - (option) => option.trackType === trackType, - ); - if (!publishOption) { + /** + * Returns the publish options for the given track type. + */ + private getPublishOptionFor = (trackType: TrackType): PublishOption => { + const option = this.publishOptions.find((o) => o.trackType === trackType); + if (!option) { throw new Error(`No publish options found for ${TrackType[trackType]}`); } - return publishOption; + return option; }; private computeLayers = ( trackType: TrackType, track: MediaStreamTrack, - opts: PublishOption, + publishOption: PublishOption, ): OptimalVideoLayer[] | undefined => { - const { settings } = this.state; - const targetResolution = settings?.video - .target_resolution as TargetResolutionResponse; - - return trackType === TrackType.VIDEO - ? findOptimalVideoLayers(track, targetResolution, opts) - : trackType === TrackType.SCREEN_SHARE - ? findOptimalScreenSharingLayers(track, undefined, opts.bitrate) - : undefined; + if (isAudioTrackType(trackType)) return; + const targetResolution = this.state.settings?.video.target_resolution; + return findOptimalVideoLayers(track, targetResolution, publishOption); }; } diff --git a/packages/client/src/rtc/__tests__/videoLayers.test.ts b/packages/client/src/rtc/__tests__/videoLayers.test.ts index c0c8e695e2..041d389a61 100644 --- a/packages/client/src/rtc/__tests__/videoLayers.test.ts +++ b/packages/client/src/rtc/__tests__/videoLayers.test.ts @@ -2,7 +2,6 @@ import './mocks/webrtc.mocks'; import { describe, expect, it, vi } from 'vitest'; import { PublishOption, VideoQuality } from '../../gen/video/sfu/models/models'; import { - findOptimalScreenSharingLayers, findOptimalVideoLayers, getComputedMaxBitrate, OptimalVideoLayer, @@ -11,38 +10,6 @@ import { } from '../videoLayers'; describe('videoLayers', () => { - it('should find optimal screen sharing layers', () => { - const track = new MediaStreamTrack(); - vi.spyOn(track, 'getSettings').mockReturnValue({ - width: 1920, - height: 1080, - }); - - const layers = findOptimalScreenSharingLayers(track); - expect(layers).toEqual([ - { - active: true, - rid: 'q', - width: 1920, - height: 1080, - maxBitrate: 3000000, - scaleResolutionDownBy: 1, - maxFramerate: 30, - }, - ]); - }); - - it('should use default max bitrate if none is provided in preferences', () => { - const track = new MediaStreamTrack(); - vi.spyOn(track, 'getSettings').mockReturnValue({ - width: 1920, - height: 1080, - }); - - const layers = findOptimalScreenSharingLayers(track, undefined, 192000); - expect(layers).toMatchObject([{ maxBitrate: 192000 }]); - }); - it('should find optimal video layers', () => { const track = new MediaStreamTrack(); const width = 1920; diff --git a/packages/client/src/rtc/helpers/tracks.ts b/packages/client/src/rtc/helpers/tracks.ts index 58e2f64162..694ed11645 100644 --- a/packages/client/src/rtc/helpers/tracks.ts +++ b/packages/client/src/rtc/helpers/tracks.ts @@ -58,19 +58,5 @@ export const toTrackType = (trackType: string): TrackType | undefined => { } }; -export const toTrackKind = ( - trackType: TrackType, -): 'audio' | 'video' | undefined => { - switch (trackType) { - case TrackType.AUDIO: - return 'audio'; - case TrackType.VIDEO: - return 'video'; - case TrackType.UNSPECIFIED: - case TrackType.SCREEN_SHARE: - case TrackType.SCREEN_SHARE_AUDIO: - return undefined; - default: - ensureExhausted(trackType, 'Unknown track type'); - } -}; +export const isAudioTrackType = (trackType: TrackType): boolean => + trackType === TrackType.AUDIO || trackType === TrackType.SCREEN_SHARE_AUDIO; diff --git a/packages/client/src/rtc/videoLayers.ts b/packages/client/src/rtc/videoLayers.ts index 60efb976c3..95289fad57 100644 --- a/packages/client/src/rtc/videoLayers.ts +++ b/packages/client/src/rtc/videoLayers.ts @@ -1,5 +1,4 @@ -import { ScreenShareSettings } from '../types'; -import { TargetResolutionResponse } from '../gen/shims'; +import type { TargetResolution } from '../gen/coordinator'; import { isSvcCodec } from './codecs'; import { PublishOption, VideoQuality } from '../gen/video/sfu/models/models'; @@ -11,7 +10,7 @@ export type OptimalVideoLayer = RTCRtpEncodingParameters & { }; const DEFAULT_BITRATE = 1250000; -const defaultTargetResolution: TargetResolutionResponse = { +const defaultTargetResolution: TargetResolution = { bitrate: DEFAULT_BITRATE, width: 1280, height: 720, @@ -30,7 +29,9 @@ const defaultBitratePerRid: Record = { * * @param layers the layers to process. */ -export const toSvcEncodings = (layers: OptimalVideoLayer[] | undefined) => { +export const toSvcEncodings = ( + layers: OptimalVideoLayer[] | undefined, +): RTCRtpEncodingParameters[] | undefined => { // we take the `f` layer, and we rename it to `q`. return layers?.filter((l) => l.rid === 'f').map((l) => ({ ...l, rid: 'q' })); }; @@ -62,7 +63,7 @@ const toScalabilityMode = (spatialLayers: number, temporalLayers: number) => */ export const findOptimalVideoLayers = ( videoTrack: MediaStreamTrack, - targetResolution: TargetResolutionResponse = defaultTargetResolution, + targetResolution: TargetResolution = defaultTargetResolution, publishOption: PublishOption, ) => { const optimalVideoLayers: OptimalVideoLayer[] = []; @@ -133,7 +134,7 @@ export const findOptimalVideoLayers = ( * @param bitrate the target bitrate. */ export const getComputedMaxBitrate = ( - targetResolution: TargetResolutionResponse, + targetResolution: TargetResolution, currentWidth: number, currentHeight: number, bitrate: number, @@ -182,25 +183,3 @@ const withSimulcastConstraints = ( rid: ridMapping[index], // reassign rid })); }; - -/** - * Determines the most optimal video layers for screen sharing. - */ -export const findOptimalScreenSharingLayers = ( - videoTrack: MediaStreamTrack, - preferences?: ScreenShareSettings, - defaultMaxBitrate = 3000000, -): OptimalVideoLayer[] => { - const settings = videoTrack.getSettings(); - return [ - { - active: true, - rid: 'q', // single track, start from 'q' - width: settings.width || 0, - height: settings.height || 0, - scaleResolutionDownBy: 1, - maxBitrate: preferences?.maxBitrate ?? defaultMaxBitrate, - maxFramerate: preferences?.maxFramerate ?? 30, - }, - ]; -}; From dde66201ce3aedf004b5f8fc852d2a6bdeb8c481 Mon Sep 17 00:00:00 2001 From: Oliver Lazoroski Date: Wed, 27 Nov 2024 22:42:12 +0100 Subject: [PATCH 31/52] chore: remove unused code --- .../src/helpers/__tests__/sdp-munging.test.ts | 92 ------------------- packages/client/src/helpers/sdp-munging.ts | 59 ------------ 2 files changed, 151 deletions(-) diff --git a/packages/client/src/helpers/__tests__/sdp-munging.test.ts b/packages/client/src/helpers/__tests__/sdp-munging.test.ts index 0a30a7c712..1abfa86316 100644 --- a/packages/client/src/helpers/__tests__/sdp-munging.test.ts +++ b/packages/client/src/helpers/__tests__/sdp-munging.test.ts @@ -18,96 +18,4 @@ describe('sdp-munging', () => { ); expect(payload).toBe(100); }); - - it('works with iOS RN vp8', () => { - const sdp = `v=0 -o=- 2055959380019004946 2 IN IP4 127.0.0.1 -s=- -t=0 0 -a=group:BUNDLE 0 -a=extmap-allow-mixed -a=msid-semantic: WMS FE2B3B06-61D7-4ACC-A4EF-76441C116E47 -m=video 9 UDP/TLS/RTP/SAVPF 96 97 98 99 100 101 127 103 35 36 104 105 106 -c=IN IP4 0.0.0.0 -a=rtcp:9 IN IP4 0.0.0.0 -a=ice-ufrag:gCgh -a=ice-pwd:bz18EOLBL9+kSJfLiVOyU4RP -a=ice-options:trickle renomination -a=fingerprint:sha-256 6B:04:36:6D:E6:92:B5:68:DA:30:CF:53:46:14:49:5B:48:3E:B9:F7:06:B4:E8:85:B1:8C:B3:1C:EB:E8:F8:16 -a=setup:actpass -a=mid:0 -a=extmap:1 urn:ietf:params:rtp-hdrext:toffset -a=extmap:2 http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time -a=extmap:3 urn:3gpp:video-orientation -a=extmap:4 http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01 -a=extmap:5 http://www.webrtc.org/experiments/rtp-hdrext/playout-delay -a=extmap:6 http://www.webrtc.org/experiments/rtp-hdrext/video-content-type -a=extmap:7 http://www.webrtc.org/experiments/rtp-hdrext/video-timing -a=extmap:8 http://www.webrtc.org/experiments/rtp-hdrext/color-space -a=extmap:9 urn:ietf:params:rtp-hdrext:sdes:mid -a=extmap:10 urn:ietf:params:rtp-hdrext:sdes:rtp-stream-id -a=extmap:11 urn:ietf:params:rtp-hdrext:sdes:repaired-rtp-stream-id -a=extmap:12 https://aomediacodec.github.io/av1-rtp-spec/#dependency-descriptor-rtp-header-extension -a=extmap:14 http://www.webrtc.org/experiments/rtp-hdrext/video-layers-allocation00 -a=sendonly -a=msid:FE2B3B06-61D7-4ACC-A4EF-76441C116E47 93FCE555-1DA2-4721-901C-5D263E11DF23 -a=rtcp-mux -a=rtcp-rsize -a=rtpmap:96 H264/90000 -a=rtcp-fb:96 goog-remb -a=rtcp-fb:96 transport-cc -a=rtcp-fb:96 ccm fir -a=rtcp-fb:96 nack -a=rtcp-fb:96 nack pli -a=fmtp:96 level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=640c29 -a=rtpmap:97 rtx/90000 -a=fmtp:97 apt=96 -a=rtpmap:98 H264/90000 -a=rtcp-fb:98 goog-remb -a=rtcp-fb:98 transport-cc -a=rtcp-fb:98 ccm fir -a=rtcp-fb:98 nack -a=rtcp-fb:98 nack pli -a=fmtp:98 level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e029 -a=rtpmap:99 rtx/90000 -a=fmtp:99 apt=98 -a=rtpmap:100 VP8/90000 -a=rtcp-fb:100 goog-remb -a=rtcp-fb:100 transport-cc -a=rtcp-fb:100 ccm fir -a=rtcp-fb:100 nack -a=rtcp-fb:100 nack pli -a=rtpmap:101 rtx/90000 -a=fmtp:101 apt=100 -a=rtpmap:127 VP9/90000 -a=rtcp-fb:127 goog-remb -a=rtcp-fb:127 transport-cc -a=rtcp-fb:127 ccm fir -a=rtcp-fb:127 nack -a=rtcp-fb:127 nack pli -a=rtpmap:103 rtx/90000 -a=fmtp:103 apt=127 -a=rtpmap:35 AV1/90000 -a=rtcp-fb:35 goog-remb -a=rtcp-fb:35 transport-cc -a=rtcp-fb:35 ccm fir -a=rtcp-fb:35 nack -a=rtcp-fb:35 nack pli -a=rtpmap:36 rtx/90000 -a=fmtp:36 apt=35 -a=rtpmap:104 red/90000 -a=rtpmap:105 rtx/90000 -a=fmtp:105 apt=104 -a=rtpmap:106 ulpfec/90000 -a=rid:q send -a=rid:h send -a=rid:f send -a=simulcast:send q;h;f`; - const target = preserveCodec(sdp, '0', { - clockRate: 90000, - mimeType: 'video/VP8', - }); - expect(target).toContain('VP8'); - expect(target).not.toContain('VP9'); - }); }); diff --git a/packages/client/src/helpers/sdp-munging.ts b/packages/client/src/helpers/sdp-munging.ts index d344fff742..d0e799cb5d 100644 --- a/packages/client/src/helpers/sdp-munging.ts +++ b/packages/client/src/helpers/sdp-munging.ts @@ -1,64 +1,5 @@ import * as SDP from 'sdp-transform'; -/** - * Returns and SDP with all the codecs except the given codec removed. - */ -export const preserveCodec = ( - sdp: string, - mid: string, - codec: RTCRtpCodec, -): string => { - const [kind, codecName] = codec.mimeType.toLowerCase().split('/'); - - const toSet = (fmtpLine: string) => - new Set(fmtpLine.split(';').map((f) => f.trim().toLowerCase())); - - const equal = (a: Set, b: Set) => { - if (a.size !== b.size) return false; - for (const item of a) if (!b.has(item)) return false; - return true; - }; - - const codecFmtp = toSet(codec.sdpFmtpLine || ''); - const parsedSdp = SDP.parse(sdp); - for (const media of parsedSdp.media) { - if (media.type !== kind || String(media.mid) !== mid) continue; - - // find the payload id of the desired codec - const payloads = new Set(); - for (const rtp of media.rtp) { - if (rtp.codec.toLowerCase() !== codecName) continue; - const match = - // vp8 doesn't have any fmtp, we preserve it without any additional checks - codecName === 'vp8' - ? true - : media.fmtp.some( - (f) => - f.payload === rtp.payload && equal(toSet(f.config), codecFmtp), - ); - if (match) { - payloads.add(rtp.payload); - } - } - - // find the corresponding rtx codec by matching apt= - for (const fmtp of media.fmtp) { - const match = fmtp.config.match(/(apt)=(\d+)/); - if (!match) continue; - const [, , preservedCodecPayload] = match; - if (payloads.has(Number(preservedCodecPayload))) { - payloads.add(fmtp.payload); - } - } - - media.rtp = media.rtp.filter((r) => payloads.has(r.payload)); - media.fmtp = media.fmtp.filter((f) => payloads.has(f.payload)); - media.rtcpFb = media.rtcpFb?.filter((f) => payloads.has(f.payload)); - media.payloads = Array.from(payloads).join(' '); - } - return SDP.write(parsedSdp); -}; - /** * Enables high-quality audio through SDP munging for the given trackMid. * From 07530ee966a43308c8c3ba2738d6b01fa3028ede Mon Sep 17 00:00:00 2001 From: Oliver Lazoroski Date: Wed, 27 Nov 2024 22:54:56 +0100 Subject: [PATCH 32/52] chore: reorganize file structure --- packages/client/src/Call.ts | 6 +++--- packages/client/src/rtc/Publisher.ts | 2 +- .../client/src/{ => rtc}/helpers/__tests__/hq-audio-sdp.ts | 0 .../src/{ => rtc}/helpers/__tests__/publisher-sdp.mock.ts | 0 .../helpers/__tests__/sdp.test.ts} | 2 +- .../src/{helpers/sdp-munging.ts => rtc/helpers/sdp.ts} | 0 packages/client/src/rtc/index.ts | 4 ++++ 7 files changed, 9 insertions(+), 5 deletions(-) rename packages/client/src/{ => rtc}/helpers/__tests__/hq-audio-sdp.ts (100%) rename packages/client/src/{ => rtc}/helpers/__tests__/publisher-sdp.mock.ts (100%) rename packages/client/src/{helpers/__tests__/sdp-munging.test.ts => rtc/helpers/__tests__/sdp.test.ts} (97%) rename packages/client/src/{helpers/sdp-munging.ts => rtc/helpers/sdp.ts} (100%) diff --git a/packages/client/src/Call.ts b/packages/client/src/Call.ts index b46c02d5a0..460657ddb5 100644 --- a/packages/client/src/Call.ts +++ b/packages/client/src/Call.ts @@ -3,12 +3,13 @@ import { Dispatcher, findCodec, getGenericSdp, + getPayloadTypeForCodec, isSfuEvent, + muteTypeToTrackType, Publisher, Subscriber, + toRtcConfiguration, } from './rtc'; -import { muteTypeToTrackType } from './rtc/helpers/tracks'; -import { toRtcConfiguration } from './rtc/helpers/rtcConfiguration'; import { registerEventHandlers, registerRingingCallEventHandlers, @@ -122,7 +123,6 @@ import { import { getSdkSignature } from './stats/utils'; import { withoutConcurrency } from './helpers/concurrency'; import { ensureExhausted } from './helpers/ensureExhausted'; -import { getPayloadTypeForCodec } from './helpers/sdp-munging'; import { PromiseWithResolvers, promiseWithResolvers, diff --git a/packages/client/src/rtc/Publisher.ts b/packages/client/src/rtc/Publisher.ts index 5a012e824c..3d035b9a88 100644 --- a/packages/client/src/rtc/Publisher.ts +++ b/packages/client/src/rtc/Publisher.ts @@ -20,7 +20,7 @@ import { isAudioTrackType, trackTypeToParticipantStreamKey, } from './helpers/tracks'; -import { enableHighQualityAudio, extractMid } from '../helpers/sdp-munging'; +import { enableHighQualityAudio, extractMid } from './helpers/sdp'; import { VideoLayerSetting } from '../gen/video/sfu/event/events'; import { withoutConcurrency } from '../helpers/concurrency'; diff --git a/packages/client/src/helpers/__tests__/hq-audio-sdp.ts b/packages/client/src/rtc/helpers/__tests__/hq-audio-sdp.ts similarity index 100% rename from packages/client/src/helpers/__tests__/hq-audio-sdp.ts rename to packages/client/src/rtc/helpers/__tests__/hq-audio-sdp.ts diff --git a/packages/client/src/helpers/__tests__/publisher-sdp.mock.ts b/packages/client/src/rtc/helpers/__tests__/publisher-sdp.mock.ts similarity index 100% rename from packages/client/src/helpers/__tests__/publisher-sdp.mock.ts rename to packages/client/src/rtc/helpers/__tests__/publisher-sdp.mock.ts diff --git a/packages/client/src/helpers/__tests__/sdp-munging.test.ts b/packages/client/src/rtc/helpers/__tests__/sdp.test.ts similarity index 97% rename from packages/client/src/helpers/__tests__/sdp-munging.test.ts rename to packages/client/src/rtc/helpers/__tests__/sdp.test.ts index 1abfa86316..4b3e8269ae 100644 --- a/packages/client/src/helpers/__tests__/sdp-munging.test.ts +++ b/packages/client/src/rtc/helpers/__tests__/sdp.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest'; -import { enableHighQualityAudio, getPayloadTypeForCodec } from '../sdp-munging'; +import { enableHighQualityAudio, getPayloadTypeForCodec } from '../sdp'; import { initialSdp as HQAudioSDP } from './hq-audio-sdp'; import { publisherSDP } from './publisher-sdp.mock'; diff --git a/packages/client/src/helpers/sdp-munging.ts b/packages/client/src/rtc/helpers/sdp.ts similarity index 100% rename from packages/client/src/helpers/sdp-munging.ts rename to packages/client/src/rtc/helpers/sdp.ts diff --git a/packages/client/src/rtc/index.ts b/packages/client/src/rtc/index.ts index a604753849..cb6ad6efd5 100644 --- a/packages/client/src/rtc/index.ts +++ b/packages/client/src/rtc/index.ts @@ -5,3 +5,7 @@ export * from './Publisher'; export * from './Subscriber'; export * from './signal'; export * from './videoLayers'; + +export * from './helpers/sdp'; +export * from './helpers/tracks'; +export * from './helpers/rtcConfiguration'; From d9fce8ad8bd46f8ebd6c32300adda1d5d6d712e2 Mon Sep 17 00:00:00 2001 From: Oliver Lazoroski Date: Wed, 27 Nov 2024 22:56:00 +0100 Subject: [PATCH 33/52] chore: remove unused field --- sample-apps/react-native/dogfood/src/components/MeetingUI.tsx | 1 - 1 file changed, 1 deletion(-) diff --git a/sample-apps/react-native/dogfood/src/components/MeetingUI.tsx b/sample-apps/react-native/dogfood/src/components/MeetingUI.tsx index 406067da11..f650841981 100644 --- a/sample-apps/react-native/dogfood/src/components/MeetingUI.tsx +++ b/sample-apps/react-native/dogfood/src/components/MeetingUI.tsx @@ -61,7 +61,6 @@ export const MeetingUI = ({ callId, navigation, route }: Props) => { try { call?.updatePublishOptions({ preferredCodec: 'vp9', - forceSingleCodec: true, }); await call?.join({ create: true }); appStoreSetState({ chatLabelNoted: false }); From 777d1e7a092a96b47cd802567494eb5d343fd229 Mon Sep 17 00:00:00 2001 From: Oliver Lazoroski Date: Wed, 27 Nov 2024 22:56:14 +0100 Subject: [PATCH 34/52] chore: reorganize file structure --- packages/client/src/events/participant.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/client/src/events/participant.ts b/packages/client/src/events/participant.ts index 003983b335..4da4658daf 100644 --- a/packages/client/src/events/participant.ts +++ b/packages/client/src/events/participant.ts @@ -12,7 +12,7 @@ import { VisibilityState, } from '../types'; import { CallState } from '../store'; -import { trackTypeToParticipantStreamKey } from '../rtc/helpers/tracks'; +import { trackTypeToParticipantStreamKey } from '../rtc'; /** * An event responder which handles the `participantJoined` event. From 9bb0030daa9ec837399e46b3d78e3686b57000fe Mon Sep 17 00:00:00 2001 From: Oliver Lazoroski Date: Fri, 29 Nov 2024 11:10:10 +0100 Subject: [PATCH 35/52] fix: update to the newest protocol --- .../client/src/gen/video/sfu/event/events.ts | 26 ++++----- .../client/src/gen/video/sfu/models/models.ts | 38 +++++++++++++ packages/client/src/rtc/Dispatcher.ts | 2 +- packages/client/src/rtc/Publisher.ts | 21 ++++---- .../src/rtc/__tests__/videoLayers.test.ts | 54 ++++++++++--------- packages/client/src/rtc/videoLayers.ts | 26 ++++----- 6 files changed, 101 insertions(+), 66 deletions(-) diff --git a/packages/client/src/gen/video/sfu/event/events.ts b/packages/client/src/gen/video/sfu/event/events.ts index 4ac6478bff..d6f7baa7c4 100644 --- a/packages/client/src/gen/video/sfu/event/events.ts +++ b/packages/client/src/gen/video/sfu/event/events.ts @@ -244,14 +244,13 @@ export interface SfuEvent { participantMigrationComplete: ParticipantMigrationComplete; } | { - oneofKind: 'codecNegotiationComplete'; + oneofKind: 'changePublishOptionsComplete'; /** - * CodecNegotiationComplete is sent to signal the completion of a codec negotiation. - * SDKs can safely stop previous transceivers + * ChangePublishOptionsComplete is sent to signal the completion of a ChangePublishOptions request. * - * @generated from protobuf field: stream.video.sfu.event.CodecNegotiationComplete codec_negotiation_complete = 26; + * @generated from protobuf field: stream.video.sfu.event.ChangePublishOptionsComplete change_publish_options_complete = 26; */ - codecNegotiationComplete: CodecNegotiationComplete; + changePublishOptionsComplete: ChangePublishOptionsComplete; } | { oneofKind: 'changePublishOptions'; @@ -276,9 +275,9 @@ export interface ChangePublishOptions { publishOption?: PublishOption; } /** - * @generated from protobuf message stream.video.sfu.event.CodecNegotiationComplete + * @generated from protobuf message stream.video.sfu.event.ChangePublishOptionsComplete */ -export interface CodecNegotiationComplete {} +export interface ChangePublishOptionsComplete {} /** * @generated from protobuf message stream.video.sfu.event.ParticipantMigrationComplete */ @@ -1005,10 +1004,10 @@ class SfuEvent$Type extends MessageType { }, { no: 26, - name: 'codec_negotiation_complete', + name: 'change_publish_options_complete', kind: 'message', oneof: 'eventPayload', - T: () => CodecNegotiationComplete, + T: () => ChangePublishOptionsComplete, }, { no: 27, @@ -1042,15 +1041,16 @@ class ChangePublishOptions$Type extends MessageType { */ export const ChangePublishOptions = new ChangePublishOptions$Type(); // @generated message type with reflection information, may provide speed optimized methods -class CodecNegotiationComplete$Type extends MessageType { +class ChangePublishOptionsComplete$Type extends MessageType { constructor() { - super('stream.video.sfu.event.CodecNegotiationComplete', []); + super('stream.video.sfu.event.ChangePublishOptionsComplete', []); } } /** - * @generated MessageType for protobuf message stream.video.sfu.event.CodecNegotiationComplete + * @generated MessageType for protobuf message stream.video.sfu.event.ChangePublishOptionsComplete */ -export const CodecNegotiationComplete = new CodecNegotiationComplete$Type(); +export const ChangePublishOptionsComplete = + new ChangePublishOptionsComplete$Type(); // @generated message type with reflection information, may provide speed optimized methods class ParticipantMigrationComplete$Type extends MessageType { constructor() { diff --git a/packages/client/src/gen/video/sfu/models/models.ts b/packages/client/src/gen/video/sfu/models/models.ts index a2edbbadcd..a17c5b9224 100644 --- a/packages/client/src/gen/video/sfu/models/models.ts +++ b/packages/client/src/gen/video/sfu/models/models.ts @@ -194,33 +194,65 @@ export interface VideoLayer { quality: VideoQuality; } /** + * PublishOption represents the configuration options for publishing a track. + * * @generated from protobuf message stream.video.sfu.models.PublishOption */ export interface PublishOption { /** + * The type of the track being published (e.g., video, screenshare). + * * @generated from protobuf field: stream.video.sfu.models.TrackType track_type = 1; */ trackType: TrackType; /** + * The codec to be used for encoding the track (e.g., VP8, VP9, H264). + * * @generated from protobuf field: stream.video.sfu.models.Codec codec = 2; */ codec?: Codec; /** + * The target bitrate for the published track, in bits per second. + * * @generated from protobuf field: int32 bitrate = 3; */ bitrate: number; /** + * The target frames per second (FPS) for video encoding. + * * @generated from protobuf field: int32 fps = 4; */ fps: number; /** + * The maximum number of spatial layers to send. + * - For SVC (e.g., VP9), spatial layers downscale by a factor of 2: + * - 1 layer: full resolution + * - 2 layers: full resolution + half resolution + * - 3 layers: full resolution + half resolution + quarter resolution + * - For non-SVC codecs (e.g., VP8/H264), this determines the number of + * encoded resolutions (e.g., quarter, half, full) sent for simulcast. + * * @generated from protobuf field: int32 max_spatial_layers = 5; */ maxSpatialLayers: number; /** + * The maximum number of temporal layers for scalable video coding (SVC). + * Temporal layers allow varying frame rates for different bandwidths. + * * @generated from protobuf field: int32 max_temporal_layers = 6; */ maxTemporalLayers: number; + /** + * The dimensions of the video (e.g., width and height in pixels). + * Spatial layers are based on this base resolution. For example, if the base + * resolution is 1280x720: + * - Full resolution (1 layer) = 1280x720 + * - Half resolution (2 layers) = 640x360 + * - Quarter resolution (3 layers) = 320x180 + * + * @generated from protobuf field: stream.video.sfu.models.VideoDimension video_dimension = 7; + */ + videoDimension?: VideoDimension; } /** * @generated from protobuf message stream.video.sfu.models.Codec @@ -1121,6 +1153,12 @@ class PublishOption$Type extends MessageType { kind: 'scalar', T: 5 /*ScalarType.INT32*/, }, + { + no: 7, + name: 'video_dimension', + kind: 'message', + T: () => VideoDimension, + }, ]); } } diff --git a/packages/client/src/rtc/Dispatcher.ts b/packages/client/src/rtc/Dispatcher.ts index 5af5ce2ab1..8a5265b34e 100644 --- a/packages/client/src/rtc/Dispatcher.ts +++ b/packages/client/src/rtc/Dispatcher.ts @@ -42,7 +42,7 @@ const sfuEventKinds: { [key in SfuEventKinds]: undefined } = { callEnded: undefined, participantUpdated: undefined, participantMigrationComplete: undefined, - codecNegotiationComplete: undefined, + changePublishOptionsComplete: undefined, changePublishOptions: undefined, }; diff --git a/packages/client/src/rtc/Publisher.ts b/packages/client/src/rtc/Publisher.ts index 3d035b9a88..d525dd4d9b 100644 --- a/packages/client/src/rtc/Publisher.ts +++ b/packages/client/src/rtc/Publisher.ts @@ -185,7 +185,7 @@ export class Publisher extends BasePeerConnection { track: MediaStreamTrack, publishOption: PublishOption, ) => { - const videoEncodings = this.computeLayers(trackType, track, publishOption); + const videoEncodings = this.computeLayers(track, publishOption); const sendEncodings = isSvcCodec(publishOption.codec?.name) ? toSvcEncodings(videoEncodings) : videoEncodings; @@ -223,16 +223,19 @@ export class Publisher extends BasePeerConnection { const transceiver = this.transceiverCache.get(trackType); if (!transceiver || !transceiver.sender.track) return; - const onNegotiationComplete = async () => { + const onChangePublishOptionsComplete = async () => { this.logger('info', 'Codec negotiation complete'); - this.dispatcher.off('codecNegotiationComplete', onNegotiationComplete); + this.dispatcher.off( + 'changePublishOptionsComplete', + onChangePublishOptionsComplete, + ); await transceiver.sender.replaceTrack(null); }; this.unsubscribeCodecNegotiationComplete?.(); this.unsubscribeCodecNegotiationComplete = this.dispatcher.on( - 'codecNegotiationComplete', - onNegotiationComplete, + 'changePublishOptionsComplete', + onChangePublishOptionsComplete, ); const track = transceiver.sender.track.clone(); @@ -506,7 +509,7 @@ export class Publisher extends BasePeerConnection { const publishOption = this.getPublishOptionFor(trackType); const isTrackLive = track.readyState === 'live'; const optimalLayers = isTrackLive - ? this.computeLayers(trackType, track, publishOption) || [] + ? this.computeLayers(track, publishOption) || [] : this.trackLayersCache.get(trackType) || []; this.trackLayersCache.set(trackType, optimalLayers); @@ -554,12 +557,10 @@ export class Publisher extends BasePeerConnection { }; private computeLayers = ( - trackType: TrackType, track: MediaStreamTrack, publishOption: PublishOption, ): OptimalVideoLayer[] | undefined => { - if (isAudioTrackType(trackType)) return; - const targetResolution = this.state.settings?.video.target_resolution; - return findOptimalVideoLayers(track, targetResolution, publishOption); + if (isAudioTrackType(publishOption.trackType)) return; + return findOptimalVideoLayers(track, publishOption); }; } diff --git a/packages/client/src/rtc/__tests__/videoLayers.test.ts b/packages/client/src/rtc/__tests__/videoLayers.test.ts index 041d389a61..ac962e4dbc 100644 --- a/packages/client/src/rtc/__tests__/videoLayers.test.ts +++ b/packages/client/src/rtc/__tests__/videoLayers.test.ts @@ -21,13 +21,10 @@ describe('videoLayers', () => { bitrate: targetBitrate, // @ts-expect-error - incomplete data codec: { name: 'vp8' }, + videoDimension: { width, height }, fps: 30, }; - const layers = findOptimalVideoLayers( - track, - { width, height, bitrate: targetBitrate }, - publishOption, - ); + const layers = findOptimalVideoLayers(track, publishOption); expect(layers).toEqual([ { active: true, @@ -65,12 +62,13 @@ describe('videoLayers', () => { const bitrate = 3000000; const track = new MediaStreamTrack(); vi.spyOn(track, 'getSettings').mockReturnValue({ width, height }); - const layers = findOptimalVideoLayers( - track, - { width, height, bitrate }, + const layers = findOptimalVideoLayers(track, { + bitrate, // @ts-expect-error - incomplete data - { bitrate, codec: { name: 'vp8' }, fps: 30 }, - ); + codec: { name: 'vp8' }, + fps: 30, + videoDimension: { width, height }, + }); expect(layers).toEqual([ { active: true, @@ -89,12 +87,13 @@ describe('videoLayers', () => { const width = 320; const height = 240; vi.spyOn(track, 'getSettings').mockReturnValue({ width, height }); - const layers = findOptimalVideoLayers( - track, - undefined, + const layers = findOptimalVideoLayers(track, { + bitrate: 0, // @ts-expect-error - incomplete data - { bitrate: 0, codec: { name: 'vp8' }, fps: 30 }, - ); + codec: { name: 'vp8' }, + fps: 30, + videoDimension: { width, height }, + }); expect(layers.length).toBe(1); expect(layers[0].rid).toBe('q'); }); @@ -104,12 +103,13 @@ describe('videoLayers', () => { const width = 640; const height = 480; vi.spyOn(track, 'getSettings').mockReturnValue({ width, height }); - const layers = findOptimalVideoLayers( - track, - undefined, + const layers = findOptimalVideoLayers(track, { + bitrate: 0, // @ts-expect-error - incomplete data - { bitrate: 0, codec: { name: 'vp8' }, fps: 30 }, - ); + codec: { name: 'vp8' }, + fps: 30, + videoDimension: { width, height }, + }); expect(layers.length).toBe(2); expect(layers[0].rid).toBe('q'); expect(layers[1].rid).toBe('h'); @@ -120,12 +120,13 @@ describe('videoLayers', () => { const width = 1280; const height = 720; vi.spyOn(track, 'getSettings').mockReturnValue({ width, height }); - const layers = findOptimalVideoLayers( - track, - undefined, + const layers = findOptimalVideoLayers(track, { + bitrate: 0, // @ts-expect-error - incomplete data - { bitrate: 0, codec: { name: 'vp8' }, fps: 30 }, - ); + codec: { name: 'vp8' }, + fps: 30, + videoDimension: { width, height }, + }); expect(layers.length).toBe(3); expect(layers[0].rid).toBe('q'); expect(layers[1].rid).toBe('h'); @@ -138,11 +139,12 @@ describe('videoLayers', () => { width: 1280, height: 720, }); - const layers = findOptimalVideoLayers(track, undefined, { + const layers = findOptimalVideoLayers(track, { maxTemporalLayers: 3, maxSpatialLayers: 3, // @ts-expect-error - incomplete data codec: { name: 'vp9' }, + videoDimension: { width: 1280, height: 720 }, }); expect(layers.length).toBe(3); expect(layers[0].scalabilityMode).toBe('L3T3_KEY'); diff --git a/packages/client/src/rtc/videoLayers.ts b/packages/client/src/rtc/videoLayers.ts index 95289fad57..425ae8d7dc 100644 --- a/packages/client/src/rtc/videoLayers.ts +++ b/packages/client/src/rtc/videoLayers.ts @@ -1,6 +1,9 @@ -import type { TargetResolution } from '../gen/coordinator'; import { isSvcCodec } from './codecs'; -import { PublishOption, VideoQuality } from '../gen/video/sfu/models/models'; +import { + PublishOption, + VideoDimension, + VideoQuality, +} from '../gen/video/sfu/models/models'; export type OptimalVideoLayer = RTCRtpEncodingParameters & { width: number; @@ -9,17 +12,10 @@ export type OptimalVideoLayer = RTCRtpEncodingParameters & { scalabilityMode?: string; }; -const DEFAULT_BITRATE = 1250000; -const defaultTargetResolution: TargetResolution = { - bitrate: DEFAULT_BITRATE, - width: 1280, - height: 720, -}; - const defaultBitratePerRid: Record = { q: 300000, h: 750000, - f: DEFAULT_BITRATE, + f: 1250000, }; /** @@ -58,12 +54,10 @@ const toScalabilityMode = (spatialLayers: number, temporalLayers: number) => * for the given track. * * @param videoTrack the video track to find optimal layers for. - * @param targetResolution the expected target resolution. * @param publishOption the publish options for the track. */ export const findOptimalVideoLayers = ( videoTrack: MediaStreamTrack, - targetResolution: TargetResolution = defaultTargetResolution, publishOption: PublishOption, ) => { const optimalVideoLayers: OptimalVideoLayer[] = []; @@ -75,9 +69,10 @@ export const findOptimalVideoLayers = ( fps, maxSpatialLayers = 3, maxTemporalLayers = 3, + videoDimension = { width: 1280, height: 720 }, } = publishOption; const maxBitrate = getComputedMaxBitrate( - targetResolution, + videoDimension, width, height, bitrate, @@ -85,8 +80,7 @@ export const findOptimalVideoLayers = ( let downscaleFactor = 1; let bitrateFactor = 1; const svcCodec = isSvcCodec(codec?.name); - const totalLayers = svcCodec ? 3 : Math.min(3, maxSpatialLayers); - for (const rid of ['f', 'h', 'q'].slice(0, totalLayers)) { + for (const rid of ['f', 'h', 'q'].slice(0, maxSpatialLayers)) { const layer: OptimalVideoLayer = { active: true, rid, @@ -134,7 +128,7 @@ export const findOptimalVideoLayers = ( * @param bitrate the target bitrate. */ export const getComputedMaxBitrate = ( - targetResolution: TargetResolution, + targetResolution: VideoDimension, currentWidth: number, currentHeight: number, bitrate: number, From cf4ed2fd0e18b3c90547ad49a518b90aa0bd96b9 Mon Sep 17 00:00:00 2001 From: Oliver Lazoroski Date: Tue, 3 Dec 2024 12:35:34 +0100 Subject: [PATCH 36/52] feat: update to the newest protocol, support publishing multiple tracks of the same kind --- .../client/src/gen/video/sfu/event/events.ts | 76 +++- .../client/src/gen/video/sfu/models/models.ts | 18 + packages/client/src/rtc/Dispatcher.ts | 1 - packages/client/src/rtc/Publisher.ts | 270 +++++++------- packages/client/src/rtc/TransceiverCache.ts | 120 +++++++ .../src/rtc/__tests__/Publisher.test.ts | 178 +++++----- .../src/rtc/__tests__/mocks/webrtc.mocks.ts | 1 + .../src/rtc/__tests__/videoLayers.test.ts | 33 ++ .../src/rtc/helpers/__tests__/hq-audio-sdp.ts | 332 ------------------ .../src/rtc/helpers/__tests__/sdp.test.ts | 9 +- packages/client/src/rtc/helpers/sdp.ts | 47 --- packages/client/src/rtc/videoLayers.ts | 16 + 12 files changed, 467 insertions(+), 634 deletions(-) create mode 100644 packages/client/src/rtc/TransceiverCache.ts delete mode 100644 packages/client/src/rtc/helpers/__tests__/hq-audio-sdp.ts diff --git a/packages/client/src/gen/video/sfu/event/events.ts b/packages/client/src/gen/video/sfu/event/events.ts index d6f7baa7c4..353c82a62d 100644 --- a/packages/client/src/gen/video/sfu/event/events.ts +++ b/packages/client/src/gen/video/sfu/event/events.ts @@ -243,15 +243,6 @@ export interface SfuEvent { */ participantMigrationComplete: ParticipantMigrationComplete; } - | { - oneofKind: 'changePublishOptionsComplete'; - /** - * ChangePublishOptionsComplete is sent to signal the completion of a ChangePublishOptions request. - * - * @generated from protobuf field: stream.video.sfu.event.ChangePublishOptionsComplete change_publish_options_complete = 26; - */ - changePublishOptionsComplete: ChangePublishOptionsComplete; - } | { oneofKind: 'changePublishOptions'; /** @@ -270,9 +261,13 @@ export interface SfuEvent { */ export interface ChangePublishOptions { /** - * @generated from protobuf field: stream.video.sfu.models.PublishOption publish_option = 1; + * @generated from protobuf field: repeated stream.video.sfu.models.PublishOption publish_options = 1; + */ + publishOptions: PublishOption[]; + /** + * @generated from protobuf field: string reason = 2; */ - publishOption?: PublishOption; + reason: string; } /** * @generated from protobuf message stream.video.sfu.event.ChangePublishOptionsComplete @@ -731,6 +726,14 @@ export interface AudioSender { * @generated from protobuf field: stream.video.sfu.models.Codec codec = 2; */ codec?: Codec; + /** + * @generated from protobuf field: stream.video.sfu.models.TrackType track_type = 3; + */ + trackType: TrackType; + /** + * @generated from protobuf field: int32 publish_option_id = 4; + */ + publishOptionId: number; } /** * VideoLayerSetting is used to specify various parameters of a particular encoding in simulcast. @@ -781,6 +784,14 @@ export interface VideoSender { * @generated from protobuf field: repeated stream.video.sfu.event.VideoLayerSetting layers = 3; */ layers: VideoLayerSetting[]; + /** + * @generated from protobuf field: stream.video.sfu.models.TrackType track_type = 4; + */ + trackType: TrackType; + /** + * @generated from protobuf field: int32 publish_option_id = 5; + */ + publishOptionId: number; } /** * sent to users when they need to change the quality of their video @@ -1002,13 +1013,6 @@ class SfuEvent$Type extends MessageType { oneof: 'eventPayload', T: () => ParticipantMigrationComplete, }, - { - no: 26, - name: 'change_publish_options_complete', - kind: 'message', - oneof: 'eventPayload', - T: () => ChangePublishOptionsComplete, - }, { no: 27, name: 'change_publish_options', @@ -1029,10 +1033,12 @@ class ChangePublishOptions$Type extends MessageType { super('stream.video.sfu.event.ChangePublishOptions', [ { no: 1, - name: 'publish_option', + name: 'publish_options', kind: 'message', + repeat: 1 /*RepeatType.PACKED*/, T: () => PublishOption, }, + { no: 2, name: 'reason', kind: 'scalar', T: 9 /*ScalarType.STRING*/ }, ]); } } @@ -1589,6 +1595,22 @@ class AudioSender$Type extends MessageType { constructor() { super('stream.video.sfu.event.AudioSender', [ { no: 2, name: 'codec', kind: 'message', T: () => Codec }, + { + no: 3, + name: 'track_type', + kind: 'enum', + T: () => [ + 'stream.video.sfu.models.TrackType', + TrackType, + 'TRACK_TYPE_', + ], + }, + { + no: 4, + name: 'publish_option_id', + kind: 'scalar', + T: 5 /*ScalarType.INT32*/, + }, ]); } } @@ -1641,6 +1663,22 @@ class VideoSender$Type extends MessageType { repeat: 1 /*RepeatType.PACKED*/, T: () => VideoLayerSetting, }, + { + no: 4, + name: 'track_type', + kind: 'enum', + T: () => [ + 'stream.video.sfu.models.TrackType', + TrackType, + 'TRACK_TYPE_', + ], + }, + { + no: 5, + name: 'publish_option_id', + kind: 'scalar', + T: 5 /*ScalarType.INT32*/, + }, ]); } } diff --git a/packages/client/src/gen/video/sfu/models/models.ts b/packages/client/src/gen/video/sfu/models/models.ts index a17c5b9224..8ddeb5dfdd 100644 --- a/packages/client/src/gen/video/sfu/models/models.ts +++ b/packages/client/src/gen/video/sfu/models/models.ts @@ -253,6 +253,23 @@ export interface PublishOption { * @generated from protobuf field: stream.video.sfu.models.VideoDimension video_dimension = 7; */ videoDimension?: VideoDimension; + /** + * The unique identifier for the publish request. + * - This `id` is assigned exclusively by the SFU. Any `id` set by the client + * in the `PublishOption` will be ignored and overwritten by the SFU. + * - The primary purpose of this `id` is to uniquely identify each publish + * request, even in scenarios where multiple publish requests for the same + * `track_type` and `codec` are active simultaneously. + * For example: + * - A user may publish two tracks of the same type (e.g., video) and codec + * (e.g., VP9) concurrently. + * - This uniqueness ensures that individual requests can be managed + * independently. For instance, an `id` is critical when stopping a specific + * publish request without affecting others. + * + * @generated from protobuf field: int32 id = 8; + */ + id: number; } /** * @generated from protobuf message stream.video.sfu.models.Codec @@ -1159,6 +1176,7 @@ class PublishOption$Type extends MessageType { kind: 'message', T: () => VideoDimension, }, + { no: 8, name: 'id', kind: 'scalar', T: 5 /*ScalarType.INT32*/ }, ]); } } diff --git a/packages/client/src/rtc/Dispatcher.ts b/packages/client/src/rtc/Dispatcher.ts index 8a5265b34e..232fc961d8 100644 --- a/packages/client/src/rtc/Dispatcher.ts +++ b/packages/client/src/rtc/Dispatcher.ts @@ -42,7 +42,6 @@ const sfuEventKinds: { [key in SfuEventKinds]: undefined } = { callEnded: undefined, participantUpdated: undefined, participantMigrationComplete: undefined, - changePublishOptionsComplete: undefined, changePublishOptions: undefined, }; diff --git a/packages/client/src/rtc/Publisher.ts b/packages/client/src/rtc/Publisher.ts index d525dd4d9b..9b9ba46bfc 100644 --- a/packages/client/src/rtc/Publisher.ts +++ b/packages/client/src/rtc/Publisher.ts @@ -2,26 +2,26 @@ import { BasePeerConnection, BasePeerConnectionOpts, } from './BasePeerConnection'; +import { TransceiverCache } from './TransceiverCache'; import { PeerType, PublishOption, TrackInfo, TrackType, - VideoLayer, } from '../gen/video/sfu/models/models'; import { findOptimalVideoLayers, OptimalVideoLayer, - ridToVideoQuality, toSvcEncodings, + toVideoLayers, } from './videoLayers'; import { isSvcCodec } from './codecs'; import { isAudioTrackType, trackTypeToParticipantStreamKey, } from './helpers/tracks'; -import { enableHighQualityAudio, extractMid } from './helpers/sdp'; -import { VideoLayerSetting } from '../gen/video/sfu/event/events'; +import { extractMid } from './helpers/sdp'; +import { VideoSender } from '../gen/video/sfu/event/events'; import { withoutConcurrency } from '../helpers/concurrency'; export type PublisherConstructorOpts = BasePeerConnectionOpts & { @@ -34,20 +34,12 @@ export type PublisherConstructorOpts = BasePeerConnectionOpts & { * @internal */ export class Publisher extends BasePeerConnection { - private readonly transceiverCache = new Map(); - private readonly trackLayersCache = new Map(); - - /** - * An array maintaining the order how transceivers were added to the peer connection. - * This is needed because some browsers (Firefox) don't reliably report - * trackId and `mid` parameters. - */ - private readonly transceiverOrder: RTCRtpTransceiver[] = []; + private readonly transceiverCache = new TransceiverCache(); + private readonly knownTrackIds = new Set(); private readonly unsubscribeOnIceRestart: () => void; private readonly unsubscribeChangePublishQuality: () => void; private readonly unsubscribeChangePublishOptions: () => void; - private unsubscribeCodecNegotiationComplete?: () => void; private publishOptions: PublishOption[]; @@ -75,9 +67,7 @@ export class Publisher extends BasePeerConnection { ({ videoSenders }) => { withoutConcurrency('publisher.changePublishQuality', async () => { for (const videoSender of videoSenders) { - const { layers } = videoSender; - const enabledLayers = layers.filter((l) => l.active); - await this.changePublishQuality(enabledLayers); + await this.changePublishQuality(videoSender); } }).catch((err) => { this.logger('warn', 'Failed to change publish quality', err); @@ -87,15 +77,10 @@ export class Publisher extends BasePeerConnection { this.unsubscribeChangePublishOptions = this.dispatcher.on( 'changePublishOptions', - ({ publishOption }) => { + (event) => { withoutConcurrency('publisher.changePublishOptions', async () => { - if (!publishOption) return; - this.publishOptions = this.publishOptions.map((o) => - o.trackType === publishOption.trackType ? publishOption : o, - ); - if (this.isPublishing(publishOption.trackType)) { - this.switchCodec(publishOption); - } + this.publishOptions = event.publishOptions; + return this.syncPublishOptions(); }).catch((err) => { this.logger('warn', 'Failed to change publish options', err); }); @@ -109,8 +94,6 @@ export class Publisher extends BasePeerConnection { close = ({ stopTracks }: { stopTracks: boolean }) => { if (stopTracks) { this.stopPublishing(); - this.transceiverCache.clear(); - this.trackLayersCache.clear(); } this.dispose(); @@ -125,7 +108,6 @@ export class Publisher extends BasePeerConnection { this.unsubscribeOnIceRestart(); this.unsubscribeChangePublishQuality(); this.unsubscribeChangePublishOptions(); - this.unsubscribeCodecNegotiationComplete?.(); super.detachEventHandlers(); this.pc.removeEventListener('negotiationneeded', this.onNegotiationNeeded); @@ -153,8 +135,7 @@ export class Publisher extends BasePeerConnection { // enable the track if it is disabled if (!track.enabled) track.enabled = true; - const transceiver = this.transceiverCache.get(trackType); - if (!transceiver || !transceiver.sender.track) { + if (!this.knownTrackIds.has(track.id)) { // listen for 'ended' event on the track as it might be ended abruptly // by an external factors such as permission revokes, a disconnected device, etc. // keep in mind that `track.stop()` doesn't trigger this event. @@ -166,10 +147,25 @@ export class Publisher extends BasePeerConnection { ); }; track.addEventListener('ended', handleTrackEnded); - const publishOption = this.getPublishOptionFor(trackType); - this.addTransceiver(trackType, track, publishOption); - } else { - await this.updateTransceiver(transceiver, track); + + // we now publish clones, hence we need to keep track of the original track ids + // to avoid assigning the same event listener multiple times + this.knownTrackIds.add(track.id); + } + + for (const publishOption of this.publishOptions) { + if (publishOption.trackType !== trackType) continue; + + // create a clone of the track as otherwise the same trackId will + // appear in the SDP in multiple transceivers + const trackToPublish = track.clone(); + + const transceiver = this.transceiverCache.get(publishOption); + if (!transceiver || !transceiver.sender.track) { + this.addTransceiver(trackToPublish, publishOption); + } else { + await this.updateTransceiver(transceiver, trackToPublish); + } } await this.notifyTrackMuteStateChanged(mediaStream, trackType, false); @@ -181,7 +177,6 @@ export class Publisher extends BasePeerConnection { * In other cases, use `updateTransceiver` method. */ private addTransceiver = ( - trackType: TrackType, track: MediaStreamTrack, publishOption: PublishOption, ) => { @@ -194,9 +189,9 @@ export class Publisher extends BasePeerConnection { sendEncodings, }); + const trackType = publishOption.trackType; this.logger('debug', `Added ${TrackType[trackType]} transceiver`); - this.transceiverOrder.push(transceiver); - this.transceiverCache.set(trackType, transceiver); + this.transceiverCache.add(publishOption, transceiver); }; /** @@ -218,28 +213,39 @@ export class Publisher extends BasePeerConnection { /** * Switches the codec of the given track type. */ - private switchCodec = (publishOption: PublishOption) => { - const trackType = publishOption.trackType; - const transceiver = this.transceiverCache.get(trackType); - if (!transceiver || !transceiver.sender.track) return; - - const onChangePublishOptionsComplete = async () => { - this.logger('info', 'Codec negotiation complete'); - this.dispatcher.off( - 'changePublishOptionsComplete', - onChangePublishOptionsComplete, + private syncPublishOptions = async () => { + // enable publishing with new options -> [av1, vp9] + for (const publishOption of this.publishOptions) { + const { trackType } = publishOption; + if (!this.isPublishing(trackType)) continue; + if (this.transceiverCache.has(publishOption)) continue; + + const item = this.transceiverCache.find( + (i) => + !!i.transceiver.sender.track && + i.publishOption.trackType === trackType, ); + if (!item || !item.transceiver) continue; - await transceiver.sender.replaceTrack(null); - }; - this.unsubscribeCodecNegotiationComplete?.(); - this.unsubscribeCodecNegotiationComplete = this.dispatcher.on( - 'changePublishOptionsComplete', - onChangePublishOptionsComplete, - ); + // take the track from the existing transceiver for the same track type, + // clone it and publish it with the new publish options + const track = item.transceiver.sender.track!.clone(); + this.addTransceiver(track, publishOption); + } - const track = transceiver.sender.track.clone(); - this.addTransceiver(trackType, track, publishOption); + // stop publishing with options not required anymore -> [vp9] + for (const item of this.transceiverCache.items()) { + const { publishOption, transceiver } = item; + const hasPublishOption = this.publishOptions.some( + (option) => + option.id === publishOption.id && + option.trackType === publishOption.trackType, + ); + if (hasPublishOption) continue; + // it is safe to stop the track here, it is a clone + transceiver.sender.track?.stop(); + await transceiver.sender.replaceTrack(null); + } }; /** @@ -249,13 +255,18 @@ export class Publisher extends BasePeerConnection { * @param stopTrack specifies whether track should be stopped or just disabled */ unpublishStream = async (trackType: TrackType, stopTrack: boolean) => { - const transceiver = this.transceiverCache.get(trackType); - if (!transceiver || !transceiver.sender.track) return; + for (const option of this.publishOptions) { + if (option.trackType !== trackType) continue; - if (stopTrack && transceiver.sender.track.readyState === 'live') { - transceiver.sender.track.stop(); - } else if (transceiver.sender.track.enabled) { - transceiver.sender.track.enabled = false; + const transceiver = this.transceiverCache.get(option); + const track = transceiver?.sender.track; + if (!track) continue; + + if (stopTrack && track.readyState === 'live') { + track.stop(); + } else if (track.enabled) { + track.enabled = false; + } } if (this.state.localParticipant?.publishedTracks.includes(trackType)) { @@ -269,19 +280,25 @@ export class Publisher extends BasePeerConnection { * @param trackType the track type to check. */ isPublishing = (trackType: TrackType): boolean => { - const transceiver = this.transceiverCache.get(trackType); - if (!transceiver || !transceiver.sender.track) return false; - const track = transceiver.sender.track; - return track.readyState === 'live' && track.enabled; + for (const item of this.transceiverCache.items()) { + if (item.publishOption.trackType !== trackType) continue; + + const track = item.transceiver?.sender.track; + if (!track) continue; + + if (track.readyState === 'live' && track.enabled) return true; + } + return false; }; /** * Maps the given track ID to the corresponding track type. */ getTrackType = (trackId: string): TrackType | undefined => { - for (const [trackType, transceiver] of this.transceiverCache) { + for (const transceiverId of this.transceiverCache.items()) { + const { publishOption, transceiver } = transceiverId; if (transceiver.sender.track?.id === trackId) { - return trackType; + return publishOption.trackType; } } return undefined; @@ -328,21 +345,25 @@ export class Publisher extends BasePeerConnection { }); }; - private changePublishQuality = async (enabledLayers: VideoLayerSetting[]) => { + private changePublishQuality = async (videoSender: VideoSender) => { + const { trackType, layers, publishOptionId } = videoSender; + const enabledLayers = layers.filter((l) => l.active); this.logger( 'info', 'Update publish quality, requested layers by SFU:', enabledLayers, ); - const trackType = TrackType.VIDEO; - const videoSender = this.transceiverCache.get(trackType)?.sender; - if (!videoSender) { + const sender = this.transceiverCache.getWith( + trackType, + publishOptionId, + )?.sender; + if (!sender) { this.logger('warn', 'Update publish quality, no video sender found.'); return; } - const params = videoSender.getParameters(); + const params = sender.getParameters(); if (params.encodings.length === 0) { this.logger( 'warn', @@ -408,7 +429,7 @@ export class Publisher extends BasePeerConnection { return; } - await videoSender.setParameters(params); + await sender.setParameters(params); this.logger('info', `Update publish quality, enabled rids:`, activeLayers); }; @@ -441,10 +462,6 @@ export class Publisher extends BasePeerConnection { */ private negotiate = async (options?: RTCOfferOptions) => { const offer = await this.pc.createOffer(options); - if (offer.sdp && this.isPublishing(TrackType.SCREEN_SHARE_AUDIO)) { - offer.sdp = this.enableHighQualityAudio(offer.sdp); - } - const trackInfos = this.getAnnouncedTracks(offer.sdp); if (trackInfos.length === 0) { throw new Error(`Can't negotiate without announcing any tracks`); @@ -477,15 +494,6 @@ export class Publisher extends BasePeerConnection { ); }; - private enableHighQualityAudio = (sdp: string) => { - const transceiver = this.transceiverCache.get(TrackType.SCREEN_SHARE_AUDIO); - if (!transceiver) return sdp; - - const transceiverInitIndex = this.transceiverOrder.indexOf(transceiver); - const mid = extractMid(transceiver, transceiverInitIndex, sdp); - return enableHighQualityAudio(sdp, mid); - }; - /** * Returns a list of tracks that are currently being published. * @@ -494,66 +502,36 @@ export class Publisher extends BasePeerConnection { */ getAnnouncedTracks = (sdp?: string): TrackInfo[] => { sdp = sdp || this.pc.localDescription?.sdp; - return this.pc - .getTransceivers() - .filter((t) => t.direction === 'sendonly' && t.sender.track) - .map((transceiver) => { - let trackType!: TrackType; - this.transceiverCache.forEach((value, key) => { - if (value === transceiver) trackType = key; - }); - - if (!trackType) return undefined; - const track = transceiver.sender.track!; - - const publishOption = this.getPublishOptionFor(trackType); - const isTrackLive = track.readyState === 'live'; - const optimalLayers = isTrackLive - ? this.computeLayers(track, publishOption) || [] - : this.trackLayersCache.get(trackType) || []; - this.trackLayersCache.set(trackType, optimalLayers); - - const layers = optimalLayers.map((optimalLayer) => ({ - rid: optimalLayer.rid || '', - bitrate: optimalLayer.maxBitrate || 0, - fps: optimalLayer.maxFramerate || 0, - quality: ridToVideoQuality(optimalLayer.rid || ''), - videoDimension: { - width: optimalLayer.width, - height: optimalLayer.height, - }, - })); - - const isAudioTrack = isAudioTrackType(trackType); - const trackSettings = track.getSettings(); - const isStereo = isAudioTrack && trackSettings.channelCount === 2; - const transceiverIndex = this.transceiverOrder.indexOf(transceiver); - const mid = extractMid(transceiver, transceiverIndex, sdp); - - const audioSettings = this.state.settings?.audio; - return { - trackId: track.id, - layers, - trackType, - mid, - stereo: isStereo, - dtx: isAudioTrack && !!audioSettings?.opus_dtx_enabled, - red: isAudioTrack && !!audioSettings?.redundant_coding_enabled, - muted: !isTrackLive, - }; - }) - .filter(Boolean) as TrackInfo[]; - }; - - /** - * Returns the publish options for the given track type. - */ - private getPublishOptionFor = (trackType: TrackType): PublishOption => { - const option = this.publishOptions.find((o) => o.trackType === trackType); - if (!option) { - throw new Error(`No publish options found for ${TrackType[trackType]}`); + const trackInfos: TrackInfo[] = []; + for (const transceiverId of this.transceiverCache.items()) { + const { publishOption, transceiver } = transceiverId; + const track = transceiver.sender.track; + if (!track) continue; + + const isTrackLive = track.readyState === 'live'; + const layers = isTrackLive + ? this.computeLayers(track, publishOption) + : this.transceiverCache.getLayers(publishOption); + this.transceiverCache.setLayers(publishOption, layers); + + const isAudioTrack = isAudioTrackType(publishOption.trackType); + const isStereo = isAudioTrack && track.getSettings().channelCount === 2; + const transceiverIndex = this.transceiverCache.indexOf(transceiver); + const mid = extractMid(transceiver, transceiverIndex, sdp); + + const audioSettings = this.state.settings?.audio; + trackInfos.push({ + trackId: track.id, + layers: toVideoLayers(layers), + trackType: publishOption.trackType, + mid, + stereo: isStereo, + dtx: isAudioTrack && !!audioSettings?.opus_dtx_enabled, + red: isAudioTrack && !!audioSettings?.redundant_coding_enabled, + muted: !isTrackLive, + }); } - return option; + return trackInfos; }; private computeLayers = ( diff --git a/packages/client/src/rtc/TransceiverCache.ts b/packages/client/src/rtc/TransceiverCache.ts new file mode 100644 index 0000000000..72e500af74 --- /dev/null +++ b/packages/client/src/rtc/TransceiverCache.ts @@ -0,0 +1,120 @@ +import { PublishOption, TrackType } from '../gen/video/sfu/models/models'; +import { OptimalVideoLayer } from './videoLayers'; + +type TransceiverId = { + publishOption: PublishOption; + transceiver: RTCRtpTransceiver; +}; +type TrackLayersCache = { + publishOption: PublishOption; + layers: OptimalVideoLayer[]; +}; + +export class TransceiverCache { + private readonly cache: TransceiverId[] = []; + private readonly layers: TrackLayersCache[] = []; + + /** + * An array maintaining the order how transceivers were added to the peer connection. + * This is needed because some browsers (Firefox) don't reliably report + * trackId and `mid` parameters. + */ + private readonly transceiverOrder: RTCRtpTransceiver[] = []; + + /** + * Adds a transceiver to the cache. + */ + add = (publishOption: PublishOption, transceiver: RTCRtpTransceiver) => { + this.cache.push({ publishOption, transceiver }); + this.transceiverOrder.push(transceiver); + }; + + /** + * Gets the transceiver for the given publish option. + */ + get = (publishOption: PublishOption): RTCRtpTransceiver | undefined => { + return this.findTransceiver(publishOption)?.transceiver; + }; + + /** + * Gets the last transceiver for the given track type and publish option id. + */ + getWith = (trackType: TrackType, id: number) => { + return this.findTransceiver({ trackType, id })?.transceiver; + }; + + /** + * Checks if the cache has the given publish option. + */ + has = (publishOption: PublishOption): boolean => { + return !!this.get(publishOption); + }; + + /** + * Finds the first transceiver that satisfies the given predicate. + */ + find = ( + predicate: (item: TransceiverId) => boolean, + ): TransceiverId | undefined => { + return this.cache.find(predicate); + }; + + /** + * Provides all the items in the cache. + */ + items = (): TransceiverId[] => { + return this.cache; + }; + + /** + * Init index of the transceiver in the cache. + */ + indexOf = (transceiver: RTCRtpTransceiver): number => { + return this.transceiverOrder.indexOf(transceiver); + }; + + /** + * Gets cached video layers for the given track. + */ + getLayers = ( + publishOption: PublishOption, + ): OptimalVideoLayer[] | undefined => { + const entry = this.layers.find( + (item) => + item.publishOption.id === publishOption.id && + item.publishOption.trackType === publishOption.trackType, + ); + return entry?.layers; + }; + + /** + * Sets the video layers for the given track. + */ + setLayers = ( + publishOption: PublishOption, + layers: OptimalVideoLayer[] = [], + ) => { + const entry = this.findLayer(publishOption); + if (entry) { + entry.layers = layers; + } else { + this.layers.push({ publishOption, layers }); + } + }; + + private findTransceiver = (publishOption: Partial) => { + return this.cache.find( + (item) => + item.publishOption.id === publishOption.id && + item.publishOption.trackType === publishOption.trackType, + ); + }; + + private findLayer = (publishOption: PublishOption) => { + return this.layers.find( + (item) => + item.publishOption.id === publishOption.id && + item.publishOption.trackType === publishOption.trackType, + ); + }; +} diff --git a/packages/client/src/rtc/__tests__/Publisher.test.ts b/packages/client/src/rtc/__tests__/Publisher.test.ts index 6b7bc7ec18..93389062ee 100644 --- a/packages/client/src/rtc/__tests__/Publisher.test.ts +++ b/packages/client/src/rtc/__tests__/Publisher.test.ts @@ -17,22 +17,6 @@ vi.mock('../../StreamSfuClient', () => { }; }); -vi.mock('../codecs', async () => { - const codecs = await vi.importActual('../codecs'); - return { - getPreferredCodecs: vi.fn((): RTCRtpCodecCapability[] => [ - { - channels: 1, - clockRate: 48000, - mimeType: 'video/h264', - sdpFmtpLine: 'profile-level-id=42e01f', - }, - ]), - getOptimalVideoCodec: codecs.getOptimalVideoCodec, - isSvcCodec: codecs.isSvcCodec, - }; -}); - describe('Publisher', () => { const sessionId = 'session-id-test'; let publisher: Publisher; @@ -72,13 +56,14 @@ describe('Publisher', () => { logTag: 'test', publishOptions: [ { + id: 1, trackType: TrackType.VIDEO, bitrate: 1000, // @ts-expect-error - incomplete data codec: { name: 'vp9' }, fps: 30, maxTemporalLayers: 3, - maxSpatialLayers: 0, + maxSpatialLayers: 3, }, ], }); @@ -110,6 +95,7 @@ describe('Publisher', () => { height: 480, deviceId: 'test-device-id', }); + vi.spyOn(track, 'clone').mockReturnValue(track); const transceiver = new RTCRtpTransceiver(); vi.spyOn(transceiver.sender, 'track', 'get').mockReturnValue(track); @@ -142,6 +128,7 @@ describe('Publisher', () => { height: 720, deviceId: 'test-device-id-2', }); + vi.spyOn(newTrack, 'clone').mockReturnValue(newTrack); await publisher.publishStream(newMediaStream, newTrack, TrackType.VIDEO); vi.spyOn(transceiver.sender, 'track', 'get').mockReturnValue(newTrack); @@ -181,6 +168,7 @@ describe('Publisher', () => { height: 480, deviceId: 'test-device-id', }); + vi.spyOn(track, 'clone').mockReturnValue(track); const transceiver = new RTCRtpTransceiver(); vi.spyOn(transceiver.sender, 'track', 'get').mockReturnValue(track); @@ -311,34 +299,42 @@ describe('Publisher', () => { }); // inject the transceiver - publisher['transceiverCache'].set(TrackType.VIDEO, transceiver); + publisher['transceiverCache'].add( + // @ts-expect-error incomplete data + { trackType: TrackType.VIDEO, id: 1 }, + transceiver, + ); - await publisher['changePublishQuality']([ - { - name: 'q', - active: true, - maxBitrate: 100, - scaleResolutionDownBy: 4, - maxFramerate: 30, - scalabilityMode: '', - }, - { - name: 'h', - active: false, - maxBitrate: 150, - scaleResolutionDownBy: 2, - maxFramerate: 30, - scalabilityMode: '', - }, - { - name: 'f', - active: true, - maxBitrate: 200, - scaleResolutionDownBy: 1, - maxFramerate: 30, - scalabilityMode: '', - }, - ]); + await publisher['changePublishQuality']({ + publishOptionId: 1, + trackType: TrackType.VIDEO, + layers: [ + { + name: 'q', + active: true, + maxBitrate: 100, + scaleResolutionDownBy: 4, + maxFramerate: 30, + scalabilityMode: '', + }, + { + name: 'h', + active: false, + maxBitrate: 150, + scaleResolutionDownBy: 2, + maxFramerate: 30, + scalabilityMode: '', + }, + { + name: 'f', + active: true, + maxBitrate: 200, + scaleResolutionDownBy: 1, + maxFramerate: 30, + scalabilityMode: '', + }, + ], + }); expect(getParametersSpy).toHaveBeenCalled(); expect(setParametersSpy).toHaveBeenCalled(); @@ -353,9 +349,6 @@ describe('Publisher', () => { { rid: 'h', active: false, - maxBitrate: 150, - scaleResolutionDownBy: 2, - maxFramerate: 30, }, { rid: 'f', @@ -381,18 +374,26 @@ describe('Publisher', () => { }); // inject the transceiver - publisher['transceiverCache'].set(TrackType.VIDEO, transceiver); + publisher['transceiverCache'].add( + // @ts-expect-error incomplete data + { trackType: TrackType.VIDEO, id: 1 }, + transceiver, + ); - await publisher['changePublishQuality']([ - { - name: 'q', - active: true, - maxBitrate: 100, - scaleResolutionDownBy: 4, - maxFramerate: 30, - scalabilityMode: '', - }, - ]); + await publisher['changePublishQuality']({ + publishOptionId: 1, + trackType: TrackType.VIDEO, + layers: [ + { + name: 'q', + active: true, + maxBitrate: 100, + scaleResolutionDownBy: 4, + maxFramerate: 30, + scalabilityMode: '', + }, + ], + }); expect(getParametersSpy).toHaveBeenCalled(); expect(setParametersSpy).toHaveBeenCalled(); @@ -436,18 +437,25 @@ describe('Publisher', () => { }); // inject the transceiver - publisher['transceiverCache'].set(TrackType.VIDEO, transceiver); - - await publisher['changePublishQuality']([ - { - name: 'q', - active: true, - maxBitrate: 50, - scaleResolutionDownBy: 1, - maxFramerate: 30, - scalabilityMode: 'L1T3', - }, - ]); + publisher['transceiverCache'].add( + // @ts-expect-error incomplete data + { trackType: TrackType.VIDEO, id: 1 }, + transceiver, + ); + await publisher['changePublishQuality']({ + publishOptionId: 1, + trackType: TrackType.VIDEO, + layers: [ + { + name: 'q', + active: true, + maxBitrate: 50, + scaleResolutionDownBy: 1, + maxFramerate: 30, + scalabilityMode: 'L1T3', + }, + ], + }); expect(getParametersSpy).toHaveBeenCalled(); expect(setParametersSpy).toHaveBeenCalled(); @@ -486,18 +494,26 @@ describe('Publisher', () => { }); // inject the transceiver - publisher['transceiverCache'].set(TrackType.VIDEO, transceiver); + publisher['transceiverCache'].add( + // @ts-expect-error incomplete data + { trackType: TrackType.VIDEO, id: 1 }, + transceiver, + ); - await publisher['changePublishQuality']([ - { - name: 'q', - active: true, - maxBitrate: 50, - scaleResolutionDownBy: 1, - maxFramerate: 30, - scalabilityMode: 'L1T3', - }, - ]); + await publisher['changePublishQuality']({ + publishOptionId: 1, + trackType: TrackType.VIDEO, + layers: [ + { + name: 'q', + active: true, + maxBitrate: 50, + scaleResolutionDownBy: 1, + maxFramerate: 30, + scalabilityMode: 'L1T3', + }, + ], + }); expect(getParametersSpy).toHaveBeenCalled(); expect(setParametersSpy).toHaveBeenCalled(); diff --git a/packages/client/src/rtc/__tests__/mocks/webrtc.mocks.ts b/packages/client/src/rtc/__tests__/mocks/webrtc.mocks.ts index 9af17ff0da..1d9442b540 100644 --- a/packages/client/src/rtc/__tests__/mocks/webrtc.mocks.ts +++ b/packages/client/src/rtc/__tests__/mocks/webrtc.mocks.ts @@ -33,6 +33,7 @@ const MediaStreamTrackMock = vi.fn((): Partial => { removeEventListener: vi.fn(), getSettings: vi.fn(), stop: vi.fn(), + clone: vi.fn(), readyState: 'live', kind: 'video', }; diff --git a/packages/client/src/rtc/__tests__/videoLayers.test.ts b/packages/client/src/rtc/__tests__/videoLayers.test.ts index ac962e4dbc..740989eba6 100644 --- a/packages/client/src/rtc/__tests__/videoLayers.test.ts +++ b/packages/client/src/rtc/__tests__/videoLayers.test.ts @@ -7,6 +7,7 @@ import { OptimalVideoLayer, ridToVideoQuality, toSvcEncodings, + toVideoLayers, } from '../videoLayers'; describe('videoLayers', () => { @@ -160,6 +161,38 @@ describe('videoLayers', () => { expect(ridToVideoQuality('')).toBe(VideoQuality.HIGH); }); + it('should map optimal video layers to SFU VideoLayers', () => { + const layers: Array> = [ + { rid: 'f', width: 1920, height: 1080, maxBitrate: 3000000 }, + { rid: 'h', width: 960, height: 540, maxBitrate: 750000 }, + { rid: 'q', width: 480, height: 270, maxBitrate: 187500 }, + ]; + + const videoLayers = toVideoLayers(layers as OptimalVideoLayer[]); + expect(videoLayers.length).toBe(3); + expect(videoLayers[0]).toEqual({ + rid: 'f', + bitrate: 3000000, + fps: 0, + quality: VideoQuality.HIGH, + videoDimension: { width: 1920, height: 1080 }, + }); + expect(videoLayers[1]).toEqual({ + rid: 'h', + bitrate: 750000, + fps: 0, + quality: VideoQuality.MID, + videoDimension: { width: 960, height: 540 }, + }); + expect(videoLayers[2]).toEqual({ + rid: 'q', + bitrate: 187500, + fps: 0, + quality: VideoQuality.LOW_UNSPECIFIED, + videoDimension: { width: 480, height: 270 }, + }); + }); + it('should map OptimalVideoLayer to SVC encodings', () => { const layers: Array> = [ { rid: 'f', width: 1920, height: 1080, maxBitrate: 3000000 }, diff --git a/packages/client/src/rtc/helpers/__tests__/hq-audio-sdp.ts b/packages/client/src/rtc/helpers/__tests__/hq-audio-sdp.ts deleted file mode 100644 index c4bad821df..0000000000 --- a/packages/client/src/rtc/helpers/__tests__/hq-audio-sdp.ts +++ /dev/null @@ -1,332 +0,0 @@ -export const initialSdp = ` -v=0 -o=- 898697271686242868 5 IN IP4 127.0.0.1 -s=- -t=0 0 -a=group:BUNDLE 0 1 2 3 -a=extmap-allow-mixed -a=msid-semantic: WMS e893e3ad-d9e8-4b56-998f-0d89213dd857 -m=video 60017 UDP/TLS/RTP/SAVPF 96 97 102 103 104 105 106 107 108 109 127 125 39 40 45 46 98 99 100 101 112 113 116 117 118 -c=IN IP4 79.125.240.146 -a=rtcp:9 IN IP4 0.0.0.0 -a=candidate:2824354870 1 udp 2122260223 192.168.1.102 60017 typ host generation 0 network-id 2 -a=candidate:427386432 1 udp 2122194687 192.168.1.244 63221 typ host generation 0 network-id 1 network-cost 10 -a=candidate:2841136656 1 udp 1686052607 79.125.240.146 60017 typ srflx raddr 192.168.1.102 rport 60017 generation 0 network-id 2 -a=candidate:410588262 1 udp 1685987071 79.125.240.146 63221 typ srflx raddr 192.168.1.244 rport 63221 generation 0 network-id 1 network-cost 10 -a=candidate:3600277166 1 tcp 1518280447 192.168.1.102 9 typ host tcptype active generation 0 network-id 2 -a=candidate:1740014808 1 tcp 1518214911 192.168.1.244 9 typ host tcptype active generation 0 network-id 1 network-cost 10 -a=ice-ufrag:GM64 -a=ice-pwd:ANZFilRRlZJ3bg9AD40eRu7n -a=ice-options:trickle -a=fingerprint:sha-256 38:1F:02:E5:2A:49:9A:2A:D9:8E:B9:9B:4C:40:21:B7:F1:C4:27:8E:B5:68:D6:E0:91:08:D9:CB:2B:AC:B3:87 -a=setup:actpass -a=mid:0 -a=extmap:1 urn:ietf:params:rtp-hdrext:toffset -a=extmap:2 http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time -a=extmap:3 urn:3gpp:video-orientation -a=extmap:4 http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01 -a=extmap:5 http://www.webrtc.org/experiments/rtp-hdrext/playout-delay -a=extmap:6 http://www.webrtc.org/experiments/rtp-hdrext/video-content-type -a=extmap:7 http://www.webrtc.org/experiments/rtp-hdrext/video-timing -a=extmap:8 http://www.webrtc.org/experiments/rtp-hdrext/color-space -a=extmap:9 urn:ietf:params:rtp-hdrext:sdes:mid -a=extmap:10 urn:ietf:params:rtp-hdrext:sdes:rtp-stream-id -a=extmap:11 urn:ietf:params:rtp-hdrext:sdes:repaired-rtp-stream-id -a=sendonly -a=msid:e893e3ad-d9e8-4b56-998f-0d89213dd857 44e5d53f-ce6a-4bf4-824a-335113d86e6e -a=rtcp-mux -a=rtcp-rsize -a=rtpmap:96 VP8/90000 -a=rtcp-fb:96 goog-remb -a=rtcp-fb:96 transport-cc -a=rtcp-fb:96 ccm fir -a=rtcp-fb:96 nack -a=rtcp-fb:96 nack pli -a=rtpmap:97 rtx/90000 -a=fmtp:97 apt=96 -a=rtpmap:102 H264/90000 -a=rtcp-fb:102 goog-remb -a=rtcp-fb:102 transport-cc -a=rtcp-fb:102 ccm fir -a=rtcp-fb:102 nack -a=rtcp-fb:102 nack pli -a=fmtp:102 level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42001f -a=rtpmap:103 rtx/90000 -a=fmtp:103 apt=102 -a=rtpmap:104 H264/90000 -a=rtcp-fb:104 goog-remb -a=rtcp-fb:104 transport-cc -a=rtcp-fb:104 ccm fir -a=rtcp-fb:104 nack -a=rtcp-fb:104 nack pli -a=fmtp:104 level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=42001f -a=rtpmap:105 rtx/90000 -a=fmtp:105 apt=104 -a=rtpmap:106 H264/90000 -a=rtcp-fb:106 goog-remb -a=rtcp-fb:106 transport-cc -a=rtcp-fb:106 ccm fir -a=rtcp-fb:106 nack -a=rtcp-fb:106 nack pli -a=fmtp:106 level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f -a=rtpmap:107 rtx/90000 -a=fmtp:107 apt=106 -a=rtpmap:108 H264/90000 -a=rtcp-fb:108 goog-remb -a=rtcp-fb:108 transport-cc -a=rtcp-fb:108 ccm fir -a=rtcp-fb:108 nack -a=rtcp-fb:108 nack pli -a=fmtp:108 level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=42e01f -a=rtpmap:109 rtx/90000 -a=fmtp:109 apt=108 -a=rtpmap:127 H264/90000 -a=rtcp-fb:127 goog-remb -a=rtcp-fb:127 transport-cc -a=rtcp-fb:127 ccm fir -a=rtcp-fb:127 nack -a=rtcp-fb:127 nack pli -a=fmtp:127 level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=4d001f -a=rtpmap:125 rtx/90000 -a=fmtp:125 apt=127 -a=rtpmap:39 H264/90000 -a=rtcp-fb:39 goog-remb -a=rtcp-fb:39 transport-cc -a=rtcp-fb:39 ccm fir -a=rtcp-fb:39 nack -a=rtcp-fb:39 nack pli -a=fmtp:39 level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=4d001f -a=rtpmap:40 rtx/90000 -a=fmtp:40 apt=39 -a=rtpmap:45 AV1/90000 -a=rtcp-fb:45 goog-remb -a=rtcp-fb:45 transport-cc -a=rtcp-fb:45 ccm fir -a=rtcp-fb:45 nack -a=rtcp-fb:45 nack pli -a=rtpmap:46 rtx/90000 -a=fmtp:46 apt=45 -a=rtpmap:98 VP9/90000 -a=rtcp-fb:98 goog-remb -a=rtcp-fb:98 transport-cc -a=rtcp-fb:98 ccm fir -a=rtcp-fb:98 nack -a=rtcp-fb:98 nack pli -a=fmtp:98 profile-id=0 -a=rtpmap:99 rtx/90000 -a=fmtp:99 apt=98 -a=rtpmap:100 VP9/90000 -a=rtcp-fb:100 goog-remb -a=rtcp-fb:100 transport-cc -a=rtcp-fb:100 ccm fir -a=rtcp-fb:100 nack -a=rtcp-fb:100 nack pli -a=fmtp:100 profile-id=2 -a=rtpmap:101 rtx/90000 -a=fmtp:101 apt=100 -a=rtpmap:112 H264/90000 -a=rtcp-fb:112 goog-remb -a=rtcp-fb:112 transport-cc -a=rtcp-fb:112 ccm fir -a=rtcp-fb:112 nack -a=rtcp-fb:112 nack pli -a=fmtp:112 level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=64001f -a=rtpmap:113 rtx/90000 -a=fmtp:113 apt=112 -a=rtpmap:116 red/90000 -a=rtpmap:117 rtx/90000 -a=fmtp:117 apt=116 -a=rtpmap:118 ulpfec/90000 -a=rid:q send -a=rid:h send -a=rid:f send -a=simulcast:send q;h;f -m=audio 9 UDP/TLS/RTP/SAVPF 63 111 9 0 8 13 110 126 -c=IN IP4 0.0.0.0 -a=rtcp:9 IN IP4 0.0.0.0 -a=ice-ufrag:GM64 -a=ice-pwd:ANZFilRRlZJ3bg9AD40eRu7n -a=ice-options:trickle -a=fingerprint:sha-256 38:1F:02:E5:2A:49:9A:2A:D9:8E:B9:9B:4C:40:21:B7:F1:C4:27:8E:B5:68:D6:E0:91:08:D9:CB:2B:AC:B3:87 -a=setup:actpass -a=mid:1 -a=extmap:14 urn:ietf:params:rtp-hdrext:ssrc-audio-level -a=extmap:2 http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time -a=extmap:4 http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01 -a=extmap:9 urn:ietf:params:rtp-hdrext:sdes:mid -a=sendonly -a=msid:- cb02416c-8035-4e80-8b7e-becd6b7f600f -a=rtcp-mux -a=rtpmap:63 red/48000/2 -a=fmtp:63 111/111 -a=rtpmap:111 opus/48000/2 -a=rtcp-fb:111 transport-cc -a=fmtp:111 minptime=10;usedtx=1;useinbandfec=1 -a=rtpmap:9 G722/8000 -a=rtpmap:0 PCMU/8000 -a=rtpmap:8 PCMA/8000 -a=rtpmap:13 CN/8000 -a=rtpmap:110 telephone-event/48000 -a=rtpmap:126 telephone-event/8000 -a=ssrc:743121750 cname:mbGW+aeWMLFhPbBC -a=ssrc:743121750 msid:- cb02416c-8035-4e80-8b7e-becd6b7f600f -m=video 9 UDP/TLS/RTP/SAVPF 96 97 102 103 104 105 106 107 108 109 127 125 39 40 45 46 98 99 100 101 112 113 116 117 118 -c=IN IP4 0.0.0.0 -a=rtcp:9 IN IP4 0.0.0.0 -a=ice-ufrag:GM64 -a=ice-pwd:ANZFilRRlZJ3bg9AD40eRu7n -a=ice-options:trickle -a=fingerprint:sha-256 38:1F:02:E5:2A:49:9A:2A:D9:8E:B9:9B:4C:40:21:B7:F1:C4:27:8E:B5:68:D6:E0:91:08:D9:CB:2B:AC:B3:87 -a=setup:actpass -a=mid:2 -a=extmap:1 urn:ietf:params:rtp-hdrext:toffset -a=extmap:2 http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time -a=extmap:3 urn:3gpp:video-orientation -a=extmap:4 http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01 -a=extmap:5 http://www.webrtc.org/experiments/rtp-hdrext/playout-delay -a=extmap:6 http://www.webrtc.org/experiments/rtp-hdrext/video-content-type -a=extmap:7 http://www.webrtc.org/experiments/rtp-hdrext/video-timing -a=extmap:8 http://www.webrtc.org/experiments/rtp-hdrext/color-space -a=extmap:9 urn:ietf:params:rtp-hdrext:sdes:mid -a=extmap:10 urn:ietf:params:rtp-hdrext:sdes:rtp-stream-id -a=extmap:11 urn:ietf:params:rtp-hdrext:sdes:repaired-rtp-stream-id -a=sendonly -a=msid:04f96a2a-d107-4a12-b545-cfe1dc2c4d37 4a1ae8a7-6c89-44d3-a9a1-02abce0012c7 -a=rtcp-mux -a=rtcp-rsize -a=rtpmap:96 VP8/90000 -a=rtcp-fb:96 goog-remb -a=rtcp-fb:96 transport-cc -a=rtcp-fb:96 ccm fir -a=rtcp-fb:96 nack -a=rtcp-fb:96 nack pli -a=rtpmap:97 rtx/90000 -a=fmtp:97 apt=96 -a=rtpmap:102 H264/90000 -a=rtcp-fb:102 goog-remb -a=rtcp-fb:102 transport-cc -a=rtcp-fb:102 ccm fir -a=rtcp-fb:102 nack -a=rtcp-fb:102 nack pli -a=fmtp:102 level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42001f -a=rtpmap:103 rtx/90000 -a=fmtp:103 apt=102 -a=rtpmap:104 H264/90000 -a=rtcp-fb:104 goog-remb -a=rtcp-fb:104 transport-cc -a=rtcp-fb:104 ccm fir -a=rtcp-fb:104 nack -a=rtcp-fb:104 nack pli -a=fmtp:104 level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=42001f -a=rtpmap:105 rtx/90000 -a=fmtp:105 apt=104 -a=rtpmap:106 H264/90000 -a=rtcp-fb:106 goog-remb -a=rtcp-fb:106 transport-cc -a=rtcp-fb:106 ccm fir -a=rtcp-fb:106 nack -a=rtcp-fb:106 nack pli -a=fmtp:106 level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f -a=rtpmap:107 rtx/90000 -a=fmtp:107 apt=106 -a=rtpmap:108 H264/90000 -a=rtcp-fb:108 goog-remb -a=rtcp-fb:108 transport-cc -a=rtcp-fb:108 ccm fir -a=rtcp-fb:108 nack -a=rtcp-fb:108 nack pli -a=fmtp:108 level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=42e01f -a=rtpmap:109 rtx/90000 -a=fmtp:109 apt=108 -a=rtpmap:127 H264/90000 -a=rtcp-fb:127 goog-remb -a=rtcp-fb:127 transport-cc -a=rtcp-fb:127 ccm fir -a=rtcp-fb:127 nack -a=rtcp-fb:127 nack pli -a=fmtp:127 level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=4d001f -a=rtpmap:125 rtx/90000 -a=fmtp:125 apt=127 -a=rtpmap:39 H264/90000 -a=rtcp-fb:39 goog-remb -a=rtcp-fb:39 transport-cc -a=rtcp-fb:39 ccm fir -a=rtcp-fb:39 nack -a=rtcp-fb:39 nack pli -a=fmtp:39 level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=4d001f -a=rtpmap:40 rtx/90000 -a=fmtp:40 apt=39 -a=rtpmap:45 AV1/90000 -a=rtcp-fb:45 goog-remb -a=rtcp-fb:45 transport-cc -a=rtcp-fb:45 ccm fir -a=rtcp-fb:45 nack -a=rtcp-fb:45 nack pli -a=rtpmap:46 rtx/90000 -a=fmtp:46 apt=45 -a=rtpmap:98 VP9/90000 -a=rtcp-fb:98 goog-remb -a=rtcp-fb:98 transport-cc -a=rtcp-fb:98 ccm fir -a=rtcp-fb:98 nack -a=rtcp-fb:98 nack pli -a=fmtp:98 profile-id=0 -a=rtpmap:99 rtx/90000 -a=fmtp:99 apt=98 -a=rtpmap:100 VP9/90000 -a=rtcp-fb:100 goog-remb -a=rtcp-fb:100 transport-cc -a=rtcp-fb:100 ccm fir -a=rtcp-fb:100 nack -a=rtcp-fb:100 nack pli -a=fmtp:100 profile-id=2 -a=rtpmap:101 rtx/90000 -a=fmtp:101 apt=100 -a=rtpmap:112 H264/90000 -a=rtcp-fb:112 goog-remb -a=rtcp-fb:112 transport-cc -a=rtcp-fb:112 ccm fir -a=rtcp-fb:112 nack -a=rtcp-fb:112 nack pli -a=fmtp:112 level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=64001f -a=rtpmap:113 rtx/90000 -a=fmtp:113 apt=112 -a=rtpmap:116 red/90000 -a=rtpmap:117 rtx/90000 -a=fmtp:117 apt=116 -a=rtpmap:118 ulpfec/90000 -a=ssrc-group:FID 4072017687 3466187747 -a=ssrc:4072017687 cname:mbGW+aeWMLFhPbBC -a=ssrc:4072017687 msid:04f96a2a-d107-4a12-b545-cfe1dc2c4d37 4a1ae8a7-6c89-44d3-a9a1-02abce0012c7 -a=ssrc:3466187747 cname:mbGW+aeWMLFhPbBC -a=ssrc:3466187747 msid:04f96a2a-d107-4a12-b545-cfe1dc2c4d37 4a1ae8a7-6c89-44d3-a9a1-02abce0012c7 -m=audio 9 UDP/TLS/RTP/SAVPF 111 63 9 0 8 13 110 126 -c=IN IP4 0.0.0.0 -a=rtcp:9 IN IP4 0.0.0.0 -a=ice-ufrag:GM64 -a=ice-pwd:ANZFilRRlZJ3bg9AD40eRu7n -a=ice-options:trickle -a=fingerprint:sha-256 38:1F:02:E5:2A:49:9A:2A:D9:8E:B9:9B:4C:40:21:B7:F1:C4:27:8E:B5:68:D6:E0:91:08:D9:CB:2B:AC:B3:87 -a=setup:actpass -a=mid:3 -a=extmap:14 urn:ietf:params:rtp-hdrext:ssrc-audio-level -a=extmap:2 http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time -a=extmap:4 http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01 -a=extmap:9 urn:ietf:params:rtp-hdrext:sdes:mid -a=sendonly -a=msid:- 75b8c290-1f46-464c-8dc5-a4a3bced313a -a=rtcp-mux -a=rtpmap:111 opus/48000/2 -a=rtcp-fb:111 transport-cc -a=fmtp:111 minptime=10;usedtx=1;useinbandfec=1 -a=rtpmap:63 red/48000/2 -a=fmtp:63 111/111 -a=rtpmap:9 G722/8000 -a=rtpmap:0 PCMU/8000 -a=rtpmap:8 PCMA/8000 -a=rtpmap:13 CN/8000 -a=rtpmap:110 telephone-event/48000 -a=rtpmap:126 telephone-event/8000 -a=ssrc:1281279951 cname:mbGW+aeWMLFhPbBC -a=ssrc:1281279951 msid:- 75b8c290-1f46-464c-8dc5-a4a3bced313a -`.trim(); diff --git a/packages/client/src/rtc/helpers/__tests__/sdp.test.ts b/packages/client/src/rtc/helpers/__tests__/sdp.test.ts index 4b3e8269ae..96893c06a4 100644 --- a/packages/client/src/rtc/helpers/__tests__/sdp.test.ts +++ b/packages/client/src/rtc/helpers/__tests__/sdp.test.ts @@ -1,15 +1,8 @@ import { describe, expect, it } from 'vitest'; -import { enableHighQualityAudio, getPayloadTypeForCodec } from '../sdp'; -import { initialSdp as HQAudioSDP } from './hq-audio-sdp'; +import { getPayloadTypeForCodec } from '../sdp'; import { publisherSDP } from './publisher-sdp.mock'; describe('sdp-munging', () => { - it('enables HighQuality audio for Opus', () => { - const sdpWithHighQualityAudio = enableHighQualityAudio(HQAudioSDP, '3'); - expect(sdpWithHighQualityAudio).toContain('maxaveragebitrate=510000'); - expect(sdpWithHighQualityAudio).toContain('stereo=1'); - }); - it('extracts payload type for codec', () => { const payload = getPayloadTypeForCodec( publisherSDP, diff --git a/packages/client/src/rtc/helpers/sdp.ts b/packages/client/src/rtc/helpers/sdp.ts index d0e799cb5d..c0687c5794 100644 --- a/packages/client/src/rtc/helpers/sdp.ts +++ b/packages/client/src/rtc/helpers/sdp.ts @@ -1,52 +1,5 @@ import * as SDP from 'sdp-transform'; -/** - * Enables high-quality audio through SDP munging for the given trackMid. - * - * @param sdp the SDP to munge. - * @param trackMid the trackMid. - * @param maxBitrate the max bitrate to set. - */ -export const enableHighQualityAudio = ( - sdp: string, - trackMid: string, - maxBitrate: number = 510000, -): string => { - maxBitrate = Math.max(Math.min(maxBitrate, 510000), 96000); - - const parsedSdp = SDP.parse(sdp); - const audioMedia = parsedSdp.media.find( - (m) => m.type === 'audio' && String(m.mid) === trackMid, - ); - - if (!audioMedia) return sdp; - - const opusRtp = audioMedia.rtp.find((r) => r.codec === 'opus'); - if (!opusRtp) return sdp; - - const opusFmtp = audioMedia.fmtp.find((f) => f.payload === opusRtp.payload); - if (!opusFmtp) return sdp; - - // enable stereo, if not already enabled - if (opusFmtp.config.match(/stereo=(\d)/)) { - opusFmtp.config = opusFmtp.config.replace(/stereo=(\d)/, 'stereo=1'); - } else { - opusFmtp.config = `${opusFmtp.config};stereo=1`; - } - - // set maxaveragebitrate, to the given value - if (opusFmtp.config.match(/maxaveragebitrate=(\d*)/)) { - opusFmtp.config = opusFmtp.config.replace( - /maxaveragebitrate=(\d*)/, - `maxaveragebitrate=${maxBitrate}`, - ); - } else { - opusFmtp.config = `${opusFmtp.config};maxaveragebitrate=${maxBitrate}`; - } - - return SDP.write(parsedSdp); -}; - /** * Gets the payload type for the given codec. */ diff --git a/packages/client/src/rtc/videoLayers.ts b/packages/client/src/rtc/videoLayers.ts index 425ae8d7dc..0bd5e60fde 100644 --- a/packages/client/src/rtc/videoLayers.ts +++ b/packages/client/src/rtc/videoLayers.ts @@ -2,6 +2,7 @@ import { isSvcCodec } from './codecs'; import { PublishOption, VideoDimension, + VideoLayer, VideoQuality, } from '../gen/video/sfu/models/models'; @@ -43,6 +44,21 @@ export const ridToVideoQuality = (rid: string): VideoQuality => { : VideoQuality.HIGH; // default to HIGH }; +/** + * Converts the given video layers to SFU video layers. + */ +export const toVideoLayers = ( + layers: OptimalVideoLayer[] | undefined = [], +): VideoLayer[] => { + return layers.map((layer) => ({ + rid: layer.rid || '', + bitrate: layer.maxBitrate || 0, + fps: layer.maxFramerate || 0, + quality: ridToVideoQuality(layer.rid || ''), + videoDimension: { width: layer.width, height: layer.height }, + })); +}; + /** * Converts the spatial and temporal layers to a scalability mode. */ From daf20711da26a0b821dde9c490c9b1cf0e479c97 Mon Sep 17 00:00:00 2001 From: Oliver Lazoroski Date: Tue, 3 Dec 2024 18:21:48 +0100 Subject: [PATCH 37/52] feat: support fmtp line for preferred codecs --- packages/client/src/Call.ts | 4 +- .../client/src/rtc/__tests__/codecs.test.ts | 58 +++++++++++++++++++ packages/client/src/rtc/codecs.ts | 21 ++++++- packages/client/src/types.ts | 4 ++ .../react-dogfood/components/MeetingUI.tsx | 15 +++-- 5 files changed, 94 insertions(+), 8 deletions(-) create mode 100644 packages/client/src/rtc/__tests__/codecs.test.ts diff --git a/packages/client/src/Call.ts b/packages/client/src/Call.ts index 58469624db..560a357061 100644 --- a/packages/client/src/Call.ts +++ b/packages/client/src/Call.ts @@ -912,12 +912,12 @@ export class Call { * @internal */ private getPreferredCodecs = (sdp: string): PublishOption[] => { - const { preferredCodec, preferredBitrate, maxSimulcastLayers } = + const { preferredCodec, fmtpLine, preferredBitrate, maxSimulcastLayers } = this.clientPublishOptions || {}; if (!preferredCodec && !preferredBitrate && !maxSimulcastLayers) return []; let sfuCodec: Codec | undefined; - const codec = findCodec(`video/${preferredCodec}`); + const codec = findCodec(`video/${preferredCodec}`, fmtpLine); if (codec) { const { clockRate, mimeType, sdpFmtpLine } = codec; sfuCodec = Codec.create({ diff --git a/packages/client/src/rtc/__tests__/codecs.test.ts b/packages/client/src/rtc/__tests__/codecs.test.ts new file mode 100644 index 0000000000..7174ca2517 --- /dev/null +++ b/packages/client/src/rtc/__tests__/codecs.test.ts @@ -0,0 +1,58 @@ +import './mocks/webrtc.mocks'; +import { beforeAll, describe, expect, it, vi } from 'vitest'; +import { findCodec } from '../codecs'; + +describe('findCodec', () => { + beforeAll(() => { + RTCRtpSender.getCapabilities = vi.fn().mockReturnValue({ + codecs: codecsMock, + }); + }); + + it('should pick the first matching codec when fmtpLine is not provided', () => { + expect(findCodec('video/H264', undefined)).toEqual({ + mimeType: 'video/H264', + sdpFmtpLine: + 'level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42001f', + }); + + expect(findCodec('video/VP8', undefined)).toEqual({ + mimeType: 'video/VP8', + }); + }); + + it('should match fmtpLine when provided', () => { + const codec = findCodec( + 'video/H264', + 'profile-level-id=640034;level-asymmetry-allowed=1;packetization-mode=1', + ); + expect(codec).toEqual({ + mimeType: 'video/H264', + sdpFmtpLine: + 'level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=640034', + }); + }); + + it('should return undefined when matching codec cannot be found', () => { + expect(findCodec('video/VP9', 'profile-id=1000')).toBeUndefined(); + expect(findCodec('video/VP101', undefined)).toBeUndefined(); + }); +}); + +// prettier-ignore +const codecsMock: Partial[] = [ + { mimeType: 'video/VP8' }, + { mimeType: 'video/rtx' }, + { mimeType: 'video/H264', sdpFmtpLine: 'level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42001f' }, + { mimeType: 'video/H264', sdpFmtpLine: 'level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=42001f' }, + { mimeType: 'video/H264', sdpFmtpLine: 'level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f' }, + { mimeType: 'video/H264', sdpFmtpLine: 'level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=42e01f' }, + { mimeType: 'video/H264', sdpFmtpLine: 'level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=4d001f' }, + { mimeType: 'video/H264', sdpFmtpLine: 'level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=4d001f' }, + { mimeType: 'video/AV1', sdpFmtpLine: 'level-idx=5;profile=0;tier=0' }, + { mimeType: 'video/VP9', sdpFmtpLine: 'profile-id=0' }, + { mimeType: 'video/VP9', sdpFmtpLine: 'profile-id=2' }, + { mimeType: 'video/H264',sdpFmtpLine: 'level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=640034' }, + { mimeType: 'video/red' }, + { mimeType: 'video/ulpfec' }, +]; diff --git a/packages/client/src/rtc/codecs.ts b/packages/client/src/rtc/codecs.ts index b794b3d877..c5396ca203 100644 --- a/packages/client/src/rtc/codecs.ts +++ b/packages/client/src/rtc/codecs.ts @@ -39,14 +39,31 @@ export const isSvcCodec = (codecOrMimeType: string | undefined) => { /** * Returns whether the codec is supported */ -export const findCodec = (mimeType: string): RTCRtpCodec | undefined => { +export const findCodec = ( + mimeType: string, + fmtpLine: string | undefined, +): RTCRtpCodec | undefined => { if (!('getCapabilities' in RTCRtpSender)) return; + const toSet = (fmtp: string = '') => + new Set(fmtp.split(';').map((f) => f.trim().toLowerCase())); + + const equal = (a: Set, b: Set) => { + if (a.size !== b.size) return false; + for (const item of a) if (!b.has(item)) return false; + return true; + }; + + const fmtp = fmtpLine ? toSet(fmtpLine) : new Set(); mimeType = mimeType.toLowerCase(); const [kind] = mimeType.split('/'); const capabilities = RTCRtpSender.getCapabilities(kind); if (!capabilities) return; const { codecs } = capabilities; - return codecs.find((c) => c.mimeType.toLowerCase() === mimeType); + return codecs.find((c) => + c.mimeType.toLowerCase() === mimeType && fmtp.size === 0 + ? true + : equal(fmtp, toSet(c.sdpFmtpLine)), + ); }; diff --git a/packages/client/src/types.ts b/packages/client/src/types.ts index 3978c270fd..7c6685712d 100644 --- a/packages/client/src/types.ts +++ b/packages/client/src/types.ts @@ -161,6 +161,10 @@ export type ClientPublishOptions = { * The preferred codec to use when publishing the video stream. */ preferredCodec?: PreferredCodec; + /** + * The fmtp line for the video codec. + */ + fmtpLine?: string; /** * The preferred bitrate to use when publishing the video stream. */ diff --git a/sample-apps/react/react-dogfood/components/MeetingUI.tsx b/sample-apps/react/react-dogfood/components/MeetingUI.tsx index 8cf694fda5..50d76510c2 100644 --- a/sample-apps/react/react-dogfood/components/MeetingUI.tsx +++ b/sample-apps/react/react-dogfood/components/MeetingUI.tsx @@ -46,9 +46,9 @@ export const MeetingUI = ({ chatClient, mode }: MeetingUIProps) => { const { useCallCallingState } = useCallStateHooks(); const callState = useCallCallingState(); - const videoCodecOverride = router.query['video_codec'] as - | PreferredCodec - | undefined; + const videoCodecOverride = (router.query['video_encoder'] || + router.query['video_codec']) as PreferredCodec | undefined; + const fmtpOverride = router.query['fmtp'] as string | undefined; const bitrateOverride = router.query['bitrate'] as string | undefined; const maxSimulcastLayers = router.query['max_simulcast_layers'] as | string @@ -64,6 +64,7 @@ export const MeetingUI = ({ chatClient, mode }: MeetingUIProps) => { : undefined; call.updatePublishOptions({ preferredCodec: videoCodecOverride || 'vp9', + fmtpLine: fmtpOverride, preferredBitrate, maxSimulcastLayers: maxSimulcastLayers ? parseInt(maxSimulcastLayers, 10) @@ -77,7 +78,13 @@ export const MeetingUI = ({ chatClient, mode }: MeetingUIProps) => { setShow('error-join'); } }, - [bitrateOverride, call, maxSimulcastLayers, videoCodecOverride], + [ + bitrateOverride, + call, + fmtpOverride, + maxSimulcastLayers, + videoCodecOverride, + ], ); const onLeave = useCallback( From 77b2fceff893a0c5966fdba3bc7999a9e3216350 Mon Sep 17 00:00:00 2001 From: Oliver Lazoroski Date: Wed, 4 Dec 2024 10:46:30 +0100 Subject: [PATCH 38/52] feat: support target resolution overrides --- .../helpers/axiosApiTransformers.ts | 33 ++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/sample-apps/react/react-dogfood/helpers/axiosApiTransformers.ts b/sample-apps/react/react-dogfood/helpers/axiosApiTransformers.ts index 070d63afcb..2741fc245d 100644 --- a/sample-apps/react/react-dogfood/helpers/axiosApiTransformers.ts +++ b/sample-apps/react/react-dogfood/helpers/axiosApiTransformers.ts @@ -3,7 +3,11 @@ import { AxiosResponseTransformer, default as axios, } from 'axios'; -import { JoinCallRequest, JoinCallResponse } from '@stream-io/video-react-sdk'; +import { + GetOrCreateCallResponse, + JoinCallRequest, + JoinCallResponse, +} from '@stream-io/video-react-sdk'; const cascadingTransformer: AxiosRequestTransformer = /** @@ -115,6 +119,32 @@ const sfuOverrideTransformer: AxiosResponseTransformer = return data; }; +const targetResolutionOverrideTransformer: AxiosResponseTransformer = + /** + * This transformer is used to override the target resolution returned by the + * backend with the one provided in the URL query params. + * + * Useful for testing with a specific target resolution. + * + * Note: it needs to be declared as a `function` instead of an arrow function + * as it executes in the context of the current axios instance. + */ + function targetResolutionOverrideTransformer(data: GetOrCreateCallResponse) { + if (typeof window !== 'undefined') { + const params = new URLSearchParams(window.location.search); + const width = params.get('width') || params.get('w'); + const height = params.get('height') || params.get('h'); + if (width && height && data?.call?.settings?.video?.target_resolution) { + data.call.settings.video.target_resolution = { + ...data.call.settings.video.target_resolution, + width: parseInt(width, 10), + height: parseInt(height, 10), + }; + } + } + return data; + }; + /** * Default request transformers. */ @@ -130,4 +160,5 @@ export const defaultRequestTransformers: AxiosRequestTransformer[] = [ export const defaultResponseTransformers: AxiosResponseTransformer[] = [ ...(axios.defaults.transformResponse as AxiosResponseTransformer[]), sfuOverrideTransformer, + targetResolutionOverrideTransformer, ]; From f6ce98775e4357834271f1ed297719568b5f4e0f Mon Sep 17 00:00:00 2001 From: Oliver Lazoroski Date: Thu, 5 Dec 2024 14:28:40 +0100 Subject: [PATCH 39/52] feat: pass-through fmtp line --- packages/client/src/Call.ts | 24 +-- .../client/src/rtc/__tests__/codecs.test.ts | 58 ------ packages/client/src/rtc/codecs.ts | 32 ---- .../helpers/__tests__/publisher-sdp.mock.ts | 170 ------------------ .../src/rtc/helpers/__tests__/sdp.test.ts | 14 -- packages/client/src/rtc/helpers/sdp.ts | 21 --- .../react-dogfood/components/MeetingUI.tsx | 4 +- 7 files changed, 8 insertions(+), 315 deletions(-) delete mode 100644 packages/client/src/rtc/__tests__/codecs.test.ts delete mode 100644 packages/client/src/rtc/helpers/__tests__/publisher-sdp.mock.ts delete mode 100644 packages/client/src/rtc/helpers/__tests__/sdp.test.ts diff --git a/packages/client/src/Call.ts b/packages/client/src/Call.ts index 560a357061..8573aebf91 100644 --- a/packages/client/src/Call.ts +++ b/packages/client/src/Call.ts @@ -1,9 +1,7 @@ import { StreamSfuClient } from './StreamSfuClient'; import { Dispatcher, - findCodec, getGenericSdp, - getPayloadTypeForCodec, isSfuEvent, muteTypeToTrackType, Publisher, @@ -806,9 +804,7 @@ export class Call { clientDetails, fastReconnect: performingFastReconnect, reconnectDetails, - preferredPublishOptions: this.getPreferredCodecs( - publishingCapabilitiesSdp, - ), + preferredPublishOptions: this.getPreferredCodecs(), }); this.initialPublishOptions = publishOptions; @@ -911,27 +907,19 @@ export class Call { * This is an experimental client feature and subject to change. * @internal */ - private getPreferredCodecs = (sdp: string): PublishOption[] => { + private getPreferredCodecs = (): PublishOption[] => { const { preferredCodec, fmtpLine, preferredBitrate, maxSimulcastLayers } = this.clientPublishOptions || {}; if (!preferredCodec && !preferredBitrate && !maxSimulcastLayers) return []; - let sfuCodec: Codec | undefined; - const codec = findCodec(`video/${preferredCodec}`, fmtpLine); - if (codec) { - const { clockRate, mimeType, sdpFmtpLine } = codec; - sfuCodec = Codec.create({ - name: preferredCodec, // e.g. 'vp9' - fmtp: sdpFmtpLine || '', - clockRate: clockRate, - payloadType: getPayloadTypeForCodec(sdp, mimeType, sdpFmtpLine), - }); - } + const codec = preferredCodec + ? Codec.create({ name: preferredCodec.split('/').pop(), fmtp: fmtpLine }) + : undefined; const preferredPublishOptions = [ PublishOption.create({ trackType: TrackType.VIDEO, - codec: sfuCodec, + codec, bitrate: preferredBitrate, maxSpatialLayers: maxSimulcastLayers, }), diff --git a/packages/client/src/rtc/__tests__/codecs.test.ts b/packages/client/src/rtc/__tests__/codecs.test.ts deleted file mode 100644 index 7174ca2517..0000000000 --- a/packages/client/src/rtc/__tests__/codecs.test.ts +++ /dev/null @@ -1,58 +0,0 @@ -import './mocks/webrtc.mocks'; -import { beforeAll, describe, expect, it, vi } from 'vitest'; -import { findCodec } from '../codecs'; - -describe('findCodec', () => { - beforeAll(() => { - RTCRtpSender.getCapabilities = vi.fn().mockReturnValue({ - codecs: codecsMock, - }); - }); - - it('should pick the first matching codec when fmtpLine is not provided', () => { - expect(findCodec('video/H264', undefined)).toEqual({ - mimeType: 'video/H264', - sdpFmtpLine: - 'level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42001f', - }); - - expect(findCodec('video/VP8', undefined)).toEqual({ - mimeType: 'video/VP8', - }); - }); - - it('should match fmtpLine when provided', () => { - const codec = findCodec( - 'video/H264', - 'profile-level-id=640034;level-asymmetry-allowed=1;packetization-mode=1', - ); - expect(codec).toEqual({ - mimeType: 'video/H264', - sdpFmtpLine: - 'level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=640034', - }); - }); - - it('should return undefined when matching codec cannot be found', () => { - expect(findCodec('video/VP9', 'profile-id=1000')).toBeUndefined(); - expect(findCodec('video/VP101', undefined)).toBeUndefined(); - }); -}); - -// prettier-ignore -const codecsMock: Partial[] = [ - { mimeType: 'video/VP8' }, - { mimeType: 'video/rtx' }, - { mimeType: 'video/H264', sdpFmtpLine: 'level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42001f' }, - { mimeType: 'video/H264', sdpFmtpLine: 'level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=42001f' }, - { mimeType: 'video/H264', sdpFmtpLine: 'level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f' }, - { mimeType: 'video/H264', sdpFmtpLine: 'level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=42e01f' }, - { mimeType: 'video/H264', sdpFmtpLine: 'level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=4d001f' }, - { mimeType: 'video/H264', sdpFmtpLine: 'level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=4d001f' }, - { mimeType: 'video/AV1', sdpFmtpLine: 'level-idx=5;profile=0;tier=0' }, - { mimeType: 'video/VP9', sdpFmtpLine: 'profile-id=0' }, - { mimeType: 'video/VP9', sdpFmtpLine: 'profile-id=2' }, - { mimeType: 'video/H264',sdpFmtpLine: 'level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=640034' }, - { mimeType: 'video/red' }, - { mimeType: 'video/ulpfec' }, -]; diff --git a/packages/client/src/rtc/codecs.ts b/packages/client/src/rtc/codecs.ts index c5396ca203..6de9fe50dd 100644 --- a/packages/client/src/rtc/codecs.ts +++ b/packages/client/src/rtc/codecs.ts @@ -35,35 +35,3 @@ export const isSvcCodec = (codecOrMimeType: string | undefined) => { codecOrMimeType === 'video/av1' ); }; - -/** - * Returns whether the codec is supported - */ -export const findCodec = ( - mimeType: string, - fmtpLine: string | undefined, -): RTCRtpCodec | undefined => { - if (!('getCapabilities' in RTCRtpSender)) return; - - const toSet = (fmtp: string = '') => - new Set(fmtp.split(';').map((f) => f.trim().toLowerCase())); - - const equal = (a: Set, b: Set) => { - if (a.size !== b.size) return false; - for (const item of a) if (!b.has(item)) return false; - return true; - }; - - const fmtp = fmtpLine ? toSet(fmtpLine) : new Set(); - mimeType = mimeType.toLowerCase(); - const [kind] = mimeType.split('/'); - const capabilities = RTCRtpSender.getCapabilities(kind); - if (!capabilities) return; - - const { codecs } = capabilities; - return codecs.find((c) => - c.mimeType.toLowerCase() === mimeType && fmtp.size === 0 - ? true - : equal(fmtp, toSet(c.sdpFmtpLine)), - ); -}; diff --git a/packages/client/src/rtc/helpers/__tests__/publisher-sdp.mock.ts b/packages/client/src/rtc/helpers/__tests__/publisher-sdp.mock.ts deleted file mode 100644 index 4d4f19d294..0000000000 --- a/packages/client/src/rtc/helpers/__tests__/publisher-sdp.mock.ts +++ /dev/null @@ -1,170 +0,0 @@ -export const publisherSDP = ` -v=0 -o=- 6786973741786618802 2 IN IP4 127.0.0.1 -s=- -t=0 0 -a=group:BUNDLE 0 1 -a=extmap-allow-mixed -a=msid-semantic: WMS -m=video 9 UDP/TLS/RTP/SAVPF 96 97 102 103 104 105 106 107 108 109 127 125 39 40 45 46 98 99 100 101 112 113 116 117 118 -c=IN IP4 0.0.0.0 -a=rtcp:9 IN IP4 0.0.0.0 -a=ice-ufrag:PeYM -a=ice-pwd:GDZQ30hwMK+5m/uVP3C0EQvf -a=ice-options:trickle -a=fingerprint:sha-256 E4:D3:D1:D8:70:08:6C:44:51:C5:0C:7E:AB:01:06:FF:10:A7:39:38:5B:43:C4:BA:FA:68:DA:AA:43:95:2C:32 -a=setup:actpass -a=mid:0 -a=extmap:1 urn:ietf:params:rtp-hdrext:toffset -a=extmap:2 http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time -a=extmap:3 urn:3gpp:video-orientation -a=extmap:4 http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01 -a=extmap:5 http://www.webrtc.org/experiments/rtp-hdrext/playout-delay -a=extmap:6 http://www.webrtc.org/experiments/rtp-hdrext/video-content-type -a=extmap:7 http://www.webrtc.org/experiments/rtp-hdrext/video-timing -a=extmap:8 http://www.webrtc.org/experiments/rtp-hdrext/color-space -a=extmap:9 urn:ietf:params:rtp-hdrext:sdes:mid -a=extmap:10 urn:ietf:params:rtp-hdrext:sdes:rtp-stream-id -a=extmap:11 urn:ietf:params:rtp-hdrext:sdes:repaired-rtp-stream-id -a=sendonly -a=msid:- 22bcd911-0a07-4b69-9557-8f4746411089 -a=rtcp-mux -a=rtcp-rsize -a=rtpmap:96 VP8/90000 -a=rtcp-fb:96 goog-remb -a=rtcp-fb:96 transport-cc -a=rtcp-fb:96 ccm fir -a=rtcp-fb:96 nack -a=rtcp-fb:96 nack pli -a=rtpmap:97 rtx/90000 -a=fmtp:97 apt=96 -a=rtpmap:102 H264/90000 -a=rtcp-fb:102 goog-remb -a=rtcp-fb:102 transport-cc -a=rtcp-fb:102 ccm fir -a=rtcp-fb:102 nack -a=rtcp-fb:102 nack pli -a=fmtp:102 level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42001f -a=rtpmap:103 rtx/90000 -a=fmtp:103 apt=102 -a=rtpmap:104 H264/90000 -a=rtcp-fb:104 goog-remb -a=rtcp-fb:104 transport-cc -a=rtcp-fb:104 ccm fir -a=rtcp-fb:104 nack -a=rtcp-fb:104 nack pli -a=fmtp:104 level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=42001f -a=rtpmap:105 rtx/90000 -a=fmtp:105 apt=104 -a=rtpmap:106 H264/90000 -a=rtcp-fb:106 goog-remb -a=rtcp-fb:106 transport-cc -a=rtcp-fb:106 ccm fir -a=rtcp-fb:106 nack -a=rtcp-fb:106 nack pli -a=fmtp:106 level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f -a=rtpmap:107 rtx/90000 -a=fmtp:107 apt=106 -a=rtpmap:108 H264/90000 -a=rtcp-fb:108 goog-remb -a=rtcp-fb:108 transport-cc -a=rtcp-fb:108 ccm fir -a=rtcp-fb:108 nack -a=rtcp-fb:108 nack pli -a=fmtp:108 level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=42e01f -a=rtpmap:109 rtx/90000 -a=fmtp:109 apt=108 -a=rtpmap:127 H264/90000 -a=rtcp-fb:127 goog-remb -a=rtcp-fb:127 transport-cc -a=rtcp-fb:127 ccm fir -a=rtcp-fb:127 nack -a=rtcp-fb:127 nack pli -a=fmtp:127 level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=4d001f -a=rtpmap:125 rtx/90000 -a=fmtp:125 apt=127 -a=rtpmap:39 H264/90000 -a=rtcp-fb:39 goog-remb -a=rtcp-fb:39 transport-cc -a=rtcp-fb:39 ccm fir -a=rtcp-fb:39 nack -a=rtcp-fb:39 nack pli -a=fmtp:39 level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=4d001f -a=rtpmap:40 rtx/90000 -a=fmtp:40 apt=39 -a=rtpmap:45 AV1/90000 -a=rtcp-fb:45 goog-remb -a=rtcp-fb:45 transport-cc -a=rtcp-fb:45 ccm fir -a=rtcp-fb:45 nack -a=rtcp-fb:45 nack pli -a=fmtp:45 level-idx=5;profile=0;tier=0 -a=rtpmap:46 rtx/90000 -a=fmtp:46 apt=45 -a=rtpmap:98 VP9/90000 -a=rtcp-fb:98 goog-remb -a=rtcp-fb:98 transport-cc -a=rtcp-fb:98 ccm fir -a=rtcp-fb:98 nack -a=rtcp-fb:98 nack pli -a=fmtp:98 profile-id=0 -a=rtpmap:99 rtx/90000 -a=fmtp:99 apt=98 -a=rtpmap:100 VP9/90000 -a=rtcp-fb:100 goog-remb -a=rtcp-fb:100 transport-cc -a=rtcp-fb:100 ccm fir -a=rtcp-fb:100 nack -a=rtcp-fb:100 nack pli -a=fmtp:100 profile-id=2 -a=rtpmap:101 rtx/90000 -a=fmtp:101 apt=100 -a=rtpmap:112 H264/90000 -a=rtcp-fb:112 goog-remb -a=rtcp-fb:112 transport-cc -a=rtcp-fb:112 ccm fir -a=rtcp-fb:112 nack -a=rtcp-fb:112 nack pli -a=fmtp:112 level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=64001f -a=rtpmap:113 rtx/90000 -a=fmtp:113 apt=112 -a=rtpmap:116 red/90000 -a=rtpmap:117 rtx/90000 -a=fmtp:117 apt=116 -a=rtpmap:118 ulpfec/90000 -a=ssrc-group:FID 3856340930 368490694 -a=ssrc:3856340930 cname:BnMV9wuzGoHGeFUb -a=ssrc:3856340930 msid:- 22bcd911-0a07-4b69-9557-8f4746411089 -a=ssrc:368490694 cname:BnMV9wuzGoHGeFUb -a=ssrc:368490694 msid:- 22bcd911-0a07-4b69-9557-8f4746411089 -m=audio 9 UDP/TLS/RTP/SAVPF 111 63 9 0 8 13 110 126 -c=IN IP4 0.0.0.0 -a=rtcp:9 IN IP4 0.0.0.0 -a=ice-ufrag:PeYM -a=ice-pwd:GDZQ30hwMK+5m/uVP3C0EQvf -a=ice-options:trickle -a=fingerprint:sha-256 E4:D3:D1:D8:70:08:6C:44:51:C5:0C:7E:AB:01:06:FF:10:A7:39:38:5B:43:C4:BA:FA:68:DA:AA:43:95:2C:32 -a=setup:actpass -a=mid:1 -a=extmap:14 urn:ietf:params:rtp-hdrext:ssrc-audio-level -a=extmap:2 http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time -a=extmap:4 http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01 -a=extmap:9 urn:ietf:params:rtp-hdrext:sdes:mid -a=sendonly -a=msid:- 440e2b2c-de05-4a72-9c30-317a0d18d32c -a=rtcp-mux -a=rtcp-rsize -a=rtpmap:111 opus/48000/2 -a=rtcp-fb:111 transport-cc -a=fmtp:111 minptime=10;useinbandfec=1 -a=rtpmap:63 red/48000/2 -a=fmtp:63 111/111 -a=rtpmap:9 G722/8000 -a=rtpmap:0 PCMU/8000 -a=rtpmap:8 PCMA/8000 -a=rtpmap:13 CN/8000 -a=rtpmap:110 telephone-event/48000 -a=rtpmap:126 telephone-event/8000 -a=ssrc:2019539886 cname:BnMV9wuzGoHGeFUb -a=ssrc:2019539886 msid:- 440e2b2c-de05-4a72-9c30-317a0d18d32c -`; diff --git a/packages/client/src/rtc/helpers/__tests__/sdp.test.ts b/packages/client/src/rtc/helpers/__tests__/sdp.test.ts deleted file mode 100644 index 96893c06a4..0000000000 --- a/packages/client/src/rtc/helpers/__tests__/sdp.test.ts +++ /dev/null @@ -1,14 +0,0 @@ -import { describe, expect, it } from 'vitest'; -import { getPayloadTypeForCodec } from '../sdp'; -import { publisherSDP } from './publisher-sdp.mock'; - -describe('sdp-munging', () => { - it('extracts payload type for codec', () => { - const payload = getPayloadTypeForCodec( - publisherSDP, - 'video/vp9', - 'profile-id=2', - ); - expect(payload).toBe(100); - }); -}); diff --git a/packages/client/src/rtc/helpers/sdp.ts b/packages/client/src/rtc/helpers/sdp.ts index c0687c5794..527e271445 100644 --- a/packages/client/src/rtc/helpers/sdp.ts +++ b/packages/client/src/rtc/helpers/sdp.ts @@ -1,26 +1,5 @@ import * as SDP from 'sdp-transform'; -/** - * Gets the payload type for the given codec. - */ -export const getPayloadTypeForCodec = ( - sdp: string, - mimeType: string, - fmtpLine: string | undefined, -): number => { - mimeType = mimeType.toLowerCase(); - const parsedSdp = SDP.parse(sdp); - const [kind, codec] = mimeType.split('/'); - const media = parsedSdp.media.find((m) => m.type === kind); - if (!media) return 0; - - const fmtp = media.fmtp.find((f) => f.config === fmtpLine); - const rtp = media.rtp.find( - (r) => r.codec.toLowerCase() === codec && r.payload === fmtp?.payload, - ); - return rtp?.payload ?? 0; -}; - /** * Extracts the mid from the transceiver or the SDP. * diff --git a/sample-apps/react/react-dogfood/components/MeetingUI.tsx b/sample-apps/react/react-dogfood/components/MeetingUI.tsx index 50d76510c2..1b792c07f4 100644 --- a/sample-apps/react/react-dogfood/components/MeetingUI.tsx +++ b/sample-apps/react/react-dogfood/components/MeetingUI.tsx @@ -63,12 +63,12 @@ export const MeetingUI = ({ chatClient, mode }: MeetingUIProps) => { ? parseInt(bitrateOverride, 10) : undefined; call.updatePublishOptions({ - preferredCodec: videoCodecOverride || 'vp9', + preferredCodec: videoCodecOverride, fmtpLine: fmtpOverride, preferredBitrate, maxSimulcastLayers: maxSimulcastLayers ? parseInt(maxSimulcastLayers, 10) - : 3, // default to 3 + : undefined, }); await call.join({ create: true }); setShow('active-call'); From 3ac0abfff4284b6d0c7b0fbaf1101d73c0da8bd8 Mon Sep 17 00:00:00 2001 From: Oliver Lazoroski Date: Fri, 6 Dec 2024 15:19:12 +0100 Subject: [PATCH 40/52] fix: listen for changePublishOptions in Call as well --- packages/client/src/Call.ts | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/packages/client/src/Call.ts b/packages/client/src/Call.ts index 8573aebf91..8f8bb36a30 100644 --- a/packages/client/src/Call.ts +++ b/packages/client/src/Call.ts @@ -289,7 +289,7 @@ export class Call { this.dynascaleManager = new DynascaleManager(this.state, this.speaker); } - private async setup() { + private setup = async () => { await withoutConcurrency(this.joinLeaveConcurrencyTag, async () => { if (this.initialized) return; @@ -300,6 +300,12 @@ export class Call { }), ); + this.leaveCallHooks.add( + this.on('changePublishOptions', (event) => { + this.initialPublishOptions = event.publishOptions; + }), + ); + this.leaveCallHooks.add(registerEventHandlers(this, this.dispatcher)); this.registerEffects(); this.registerReconnectHandlers(); @@ -310,9 +316,9 @@ export class Call { this.initialized = true; }); - } + }; - private registerEffects() { + private registerEffects = () => { this.leaveCallHooks.add( // handles updating the permissions context when the settings change. createSubscription(this.state.settings$, (settings) => { @@ -401,7 +407,7 @@ export class Call { } }), ); - } + }; private handleOwnCapabilitiesUpdated = async ( ownCapabilities: OwnCapability[], From 191bb93d993becf70ce610295a029d9dc36f93e5 Mon Sep 17 00:00:00 2001 From: Oliver Lazoroski Date: Fri, 6 Dec 2024 15:19:57 +0100 Subject: [PATCH 41/52] fix: don't override codecs in RN --- sample-apps/react-native/dogfood/App.tsx | 1 - .../react-native/dogfood/ios/Podfile.lock | 136 +++++++++--------- .../dogfood/src/components/MeetingUI.tsx | 3 - .../dogfood/src/contexts/AppContext.tsx | 3 + .../src/screens/Meeting/JoinMeetingScreen.tsx | 13 +- 5 files changed, 79 insertions(+), 77 deletions(-) diff --git a/sample-apps/react-native/dogfood/App.tsx b/sample-apps/react-native/dogfood/App.tsx index 82c2cbb5e2..31ad0612cb 100755 --- a/sample-apps/react-native/dogfood/App.tsx +++ b/sample-apps/react-native/dogfood/App.tsx @@ -34,7 +34,6 @@ import { NavigationHeader } from './src/components/NavigationHeader'; import { GestureHandlerRootView } from 'react-native-gesture-handler'; import { LogBox } from 'react-native'; import { LiveStream } from './src/navigators/Livestream'; -import { REACT_NATIVE_DOGFOOD_APP_ENVIRONMENT } from '@env'; import PushNotificationIOS from '@react-native-community/push-notification-ios'; import { defaultTheme, diff --git a/sample-apps/react-native/dogfood/ios/Podfile.lock b/sample-apps/react-native/dogfood/ios/Podfile.lock index 331d338aa3..64f54adf5a 100644 --- a/sample-apps/react-native/dogfood/ios/Podfile.lock +++ b/sample-apps/react-native/dogfood/ios/Podfile.lock @@ -2012,7 +2012,7 @@ PODS: - stream-react-native-webrtc (125.0.1-rc.1): - React-Core - StreamWebRTC (~> 125.6422.064) - - stream-video-react-native (1.4.10): + - stream-video-react-native (1.4.11): - DoubleConversion - glog - hermes-engine @@ -2333,84 +2333,84 @@ SPEC CHECKSUMS: React: 10ad41b51f981992714011b6a4e081234c28dc2e React-callinvoker: 58b51494f8b2cca07a27fc6f69273239c30a1e70 React-Codegen: 4b8b4817cea7a54b83851d4c1f91f79aa73de30a - React-Core: 7a5e9897daf0189c0233b25243d6704e5b9025d8 - React-CoreModules: 09d4f4ddd85ce9301c4b06dfe68750a82ee4b4f5 - React-cxxreact: 29bfe097a993c73a314f569998fe863eb6fb8a18 + React-Core: 54860c16fb5873a6f00dd44d8979bbb648b34c7c + React-CoreModules: 443101f113a7b5d51b93e061574dcadf7850f8cc + React-cxxreact: 5407ecb854a755de34c0e6b03965d3a51c28c933 React-debug: a01fc4f077430d1a73f333ee2838f4d766e2d58b - React-defaultsnativemodule: 421fc755e08d5ad7726a252cc38f6925cd2bf919 - React-domnativemodule: 64a00466588556262f7a238f0600d201a5d27b86 - React-Fabric: 22f4287daa4187e2a10f9742dc74f3af9d9b2254 - React-FabricComponents: 9295f2fabf5495c87621cea38cbd4dc445f43650 - React-FabricImage: 5caf84d721e28747c53823825a551528c20ff875 + React-defaultsnativemodule: fad7dd0129357e9012b734d641bdd7c7d8b95c8c + React-domnativemodule: b026c1578ffaada6c62ed8453a44ba263743830c + React-Fabric: 5ffd7ec9e7bf3d9e98358cbfbb1ef1b26954b6f5 + React-FabricComponents: 6306fe9587c4a460017595749f6bfd7979817f43 + React-FabricImage: 189f860814c8d2581ddc784d08addcb2e86ba5be React-featureflags: 0845d47c3314ba87f2f6315bd33d6be8d23d2be8 - React-featureflagsnativemodule: 6220f08c9c51a407375e5025421b06b7aa696ca0 - React-graphics: d9f0317c34579ce4f14d9933b8033fe9ef61c72b - React-hermes: ab8705477c497a5839966bd57471ee3611f864f8 - React-idlecallbacksnativemodule: 28c85b4c689eccc6d3fffe6fb5d010a18f48f629 - React-ImageManager: 9005e783cfae8c521d59c33b8a4df2b78d3a6345 - React-jserrorhandler: a14500014e8cd4d2f960cf16b69d2edbd32547ff - React-jsi: de2c6119671b281671fabf9e96eb11110207fe9d - React-jsiexecutor: 936132921f4d991af7b4faa7424fc54e67791dd0 - React-jsinspector: 12d33a2f643ea2fd08ff4443f3b6c8b9fc5c4976 - React-jsitracing: 1724696aadc78fca5c66ec8d2ce3b8f04d2799bc - React-logger: addd140841248966c2547eb94836399cc1061f4d - React-Mapbuffer: 1bc8e611871f4965dac0bc47a4561421a6e20f69 - React-microtasksnativemodule: cff02bc87f8a1d5b9985c1c92ea8e84e854229d9 - react-native-blob-util: 4f935148b217389fff952096c0f1a6ff67f4bdea - react-native-image-picker: 5c4cfe25630a6ec9105c16693abe8373a6f36d9a - react-native-mmkv: 36d57903d6b78677f6b7ec90c900df6e43d7d3e4 - react-native-netinfo: a65f803f0e7dfa2fd70d093546161357d9326581 - react-native-safe-area-context: ee4151c59dc010b5211fe68eca73b0f98d17b224 - react-native-video: 6d495634696a5580559828544e3454c6ac27fbd7 + React-featureflagsnativemodule: 5dfb68d7678e0fa395deac55f2a1b241a7a9dbd5 + React-graphics: e8288c4a68627349c834eaf2fcc92108f2dbefb6 + React-hermes: 8f31f252aff98a4cb711ccf6644cccfe35d8edd1 + React-idlecallbacksnativemodule: a6780c405376fa6db47619deb44ff3f8dec6c3c9 + React-ImageManager: 0b553a8e762b75d4cf9176474629f2d39cdb2aad + React-jserrorhandler: 2c47610e18594ed6b9c52995308afdbec0f59b71 + React-jsi: b96853ac12c1dab5fe3ea131f959fda0bbaf1151 + React-jsiexecutor: e38748a0e9d899f63dec562f93ac06c7acbc813d + React-jsinspector: b707427ae4772f34dab943a7343feddb155e8add + React-jsitracing: f4028bf2f09cd8707ad8befb35a8b78221d1673d + React-logger: 81d58ca6f1d93fca9a770bda6cc1c4fbfcc99c9c + React-Mapbuffer: b9bfad03a24c3ff440557e9011a6a09864849eae + React-microtasksnativemodule: 853dae5be1372b3ab52b21e29f86f2e1a0c61f37 + react-native-blob-util: 356047c561b3506396852bc0d7988243f74dd77d + react-native-image-picker: df6597d4b1878a443796be11eb2b7286ed10ece6 + react-native-mmkv: fb501d25ce65d16a1fad3296f7fc69150a1f0788 + react-native-netinfo: 299dad906cdbf3b67bcc6f693c807f98bdd127cc + react-native-safe-area-context: f826417dadd1c1042c59355fb050682a9839f990 + react-native-video: e5e752b62458690667276df947aee93b394d3e20 React-nativeconfig: 7f8cd6cae21f8bb18c53b746c495e72795fc5cb0 - React-NativeModulesApple: 3210b7177c11145bb8e0d6f24aae102a221c4ddc - React-perflogger: c8860eaab4fe60d628b27bf0086a372c429fc74f - React-performancetimeline: 6b072ee07b20f71ca7aa443d7c78b9cb2a934ead + React-NativeModulesApple: 4fb24980fec9a94c9e9c1de3cdfd38ff3b87361c + React-perflogger: f2c94413cfad44817c96cab33753831e73f0d0dd + React-performancetimeline: c3ad160557d7406ceb5bb4dbc62834b1e61ee797 React-RCTActionSheet: 2eb26cbf384f3d3b2cb2e23be850a956d83f77ab - React-RCTAnimation: aa0a663829963ca72f4c722e71bd5debbecc1348 - React-RCTAppDelegate: 12688b64e1e28e0eb1c628690678ae5d3ab356b4 - React-RCTBlob: bef788ef3433170f9748d0e00d1afc7be64bc51d - React-RCTFabric: 5f335f0643a84dd888bf7ba70d6d60484c981d87 - React-RCTImage: a9de66d305fa02008759a2aa5a723b68d18907e5 - React-RCTLinking: 15fe8ccad84a4a5274d55b9d43e223896718772d - React-RCTNetwork: 7635ab6b7617648e5b5e35cdb3a4edab6fa309a6 - React-RCTSettings: 18e666705ea62aac59f2a8d50ced87b9b8902c7b - React-RCTText: 5cf76f649b4781362d23f9ee3d52e8d12a74dd18 - React-RCTVibration: bd72dc267866c8cd524c9a61d15060949ff24cf9 + React-RCTAnimation: 59463699a92edc6705ce5306bb789d6a0ca4df0b + React-RCTAppDelegate: 4d9efca7caa477b106e3d55af339d0e071441536 + React-RCTBlob: 0883f5363069ad30f628c970fcb413a619e42804 + React-RCTFabric: 8cd047489627f322e491cf21d91ea242c8068fe3 + React-RCTImage: 78884b7ea6ef4f7bb9655614bf09a40054f282ce + React-RCTLinking: b9beba7465fd9a1ed7a88a4e7fc403d26e17ab95 + React-RCTNetwork: 701d9c050077596b15a11b6b573ed95c309d2315 + React-RCTSettings: e700a82e3e923c10060b8f65297f9d321b93d8eb + React-RCTText: e782ce1c3f9d915daf50d97157f8c226e8f3d206 + React-RCTVibration: 2a19c56be78cb7afce9f4f3471aacfb063f32a00 React-rendererconsistency: bbb7bafd25f3a41f4ea604be846dc2da8180e840 - React-rendererdebug: 7c262ecec4bcddf7c9b8782f836fa68864d3d5f7 + React-rendererdebug: 5cd463cfe5c4c174a8aa6abd67f190ad07a03e24 React-rncore: f2e8940f20f97f900f359861adf3a96f30dc82a3 - React-RuntimeApple: e98509dfdc3c1da7560ac10637e077a05fc438d0 - React-RuntimeCore: 89bd1ffca294f5fb437466e32b394f17bae28b31 + React-RuntimeApple: 4ce7c4cc1ee24608b40a22667250e32e4171eef0 + React-RuntimeCore: c3e89760391557d91b72bba1078d3e2ce26e963d React-runtimeexecutor: 69e27948ee2127400297c7de50b809a7cd127a15 - React-RuntimeHermes: 52f1738a3864f40445b0a5362e232eba29dcbcb1 - React-runtimescheduler: 98d80589939956811f3ff51cb1ab720e6b3b1290 + React-RuntimeHermes: 446adf8034db4b8f9d53b0140fdab832e475e7c9 + React-runtimescheduler: 18e831b141db320f5ee13e0a6ecfd486a0e3de0c React-timing: 5627775f1ccecc1d56bfc1247f942eec82069d1f - React-utils: 7ce63e32e4ca425cc73cfb84e656bfb9e02e58b3 - ReactCodegen: 76542015d808938c67640540879b99413001fe42 - ReactCommon: a1c914f7575011239a63603a95fb341d0331953c - ReactNativeIncallManager: ef7b845e166f04cf8ddf433d8a23227b01cef87a - RNCallKeep: 516281f03461e6be68f21a4634dbeee85d3fb730 - RNCClipboard: efe1b27ad1ea378c60c8c8aabfd130961bbeb474 - RNCPushNotificationIOS: 6c4ca3388c7434e4a662b92e4dfeeee858e6f440 - RNDeviceInfo: afc27b3f24bd0e97181bf3e9f23cfa4c9040dd32 - RNGestureHandler: 932e0f07ccf470940afa9d0a8b6d8221e7e19cff - RNGoogleSignin: 9b083b6a575e7c2401aac339c93f8292d0542d29 - RNNotifee: 52b319634ba89a2eacdfbadc01e059fd18505f04 - RNPermissions: a9ea34e4f88ced1f9664d6589f57b4931b23e3aa - RNReactNativeHapticFeedback: d557285118551f215efe4b8fbbd24d2c4ae6c1b9 - RNReanimated: 9ebacf9fbe1b8aeb2fc19de93d4a6779b89d0b89 - RNScreens: d6b3735a362dab6a8cef14d032bdbdaf6e1b8dfa - RNSVG: b24f7dfe496d5463a79b330713226587e4b13956 - RNVoipPushNotification: e5edde96849c0133ebc7e900dc30c8c221cfb690 + React-utils: 2431220eeebc884010eb8df65335cb16c5849a55 + ReactCodegen: 7ffe695604dd4aa69ac6d1baa4e51d2f1a9c610f + ReactCommon: 555c6f17f322bf4e7b9ce48990b252723170d158 + ReactNativeIncallManager: bfc9c67358cd524882a7c4116dcb311ac2293d4b + RNCallKeep: 7bfa8f502067be6650eeca5ec0ebbf795314c5c3 + RNCClipboard: 3f0451a8100393908bea5c5c5b16f96d45f30bfc + RNCPushNotificationIOS: 64218f3c776c03d7408284a819b2abfda1834bc8 + RNDeviceInfo: 825f0d2f4381327317f12d1522720a8f76e6a19e + RNGestureHandler: 15ee1ab573a954c92641877ca946e2680f2e58da + RNGoogleSignin: fc408799f1990a12497a32f64280c0fe353ffcc1 + RNNotifee: bc20a5e3d581f629db988075944fdd944d363dfe + RNPermissions: 13cf580b8ac0f6e36ff8f61eb3a955dcffdbd9ab + RNReactNativeHapticFeedback: 70dd302f264d06d1a2e0632a717d0b3ed10a0f35 + RNReanimated: 7f11fff1964b5d073961b54167c22ebf3bd5aaff + RNScreens: 61099dac2e3cd356d2f7818ef15e9b6ad2769408 + RNSVG: feeb4eb546e718d6c6fb70d10fd31e0c5c2d0d90 + RNVoipPushNotification: 543e18f83089134a35e7f1d2eba4c8b1f7776b08 SocketRocket: d4aabe649be1e368d1318fdf28a022d714d65748 - stream-chat-react-native: 470c37e2bed295da2ed9148f398694995d98f5af - stream-io-video-filters-react-native: 8e3d06506767c1a58576bb9d0229cbfc51a709d6 - stream-react-native-webrtc: a4826a7f9dbe990dc4a1de9d94c3c8f35d272d61 - stream-video-react-native: 682e4b962cd775bd21bb5cec4fc1c91779d3fc3f + stream-chat-react-native: e515fc7d0951a32627f75ccca0699c139ba34579 + stream-io-video-filters-react-native: 4153f006b31f805f6a4e42556d8e4ae768231900 + stream-react-native-webrtc: 80b7b6b577d77c72bb40dcc642d8160a9d2cd9da + stream-video-react-native: abceecd788492965f3c8eaef10b2d727ce9e72d3 StreamWebRTC: a77d2450a19a9d7d9a28b416cc0cb336d998dc62 Yoga: 513b871d622689bd53b51481bbcfb6b8f1a3de5b PODFILE CHECKSUM: 22e502ced1a8b5a5e637f60837d3de140b3387b8 -COCOAPODS: 1.15.2 +COCOAPODS: 1.16.2 diff --git a/sample-apps/react-native/dogfood/src/components/MeetingUI.tsx b/sample-apps/react-native/dogfood/src/components/MeetingUI.tsx index f650841981..00f1f44781 100644 --- a/sample-apps/react-native/dogfood/src/components/MeetingUI.tsx +++ b/sample-apps/react-native/dogfood/src/components/MeetingUI.tsx @@ -59,9 +59,6 @@ export const MeetingUI = ({ callId, navigation, route }: Props) => { const onJoinCallHandler = useCallback(async () => { try { - call?.updatePublishOptions({ - preferredCodec: 'vp9', - }); await call?.join({ create: true }); appStoreSetState({ chatLabelNoted: false }); setShow('active-call'); diff --git a/sample-apps/react-native/dogfood/src/contexts/AppContext.tsx b/sample-apps/react-native/dogfood/src/contexts/AppContext.tsx index e6d3d60ff9..7d30592066 100644 --- a/sample-apps/react-native/dogfood/src/contexts/AppContext.tsx +++ b/sample-apps/react-native/dogfood/src/contexts/AppContext.tsx @@ -10,6 +10,7 @@ type AppGlobalStore = { userImageUrl?: string; userName: string; appMode: AppMode; + callId?: string; appEnvironment: AppEnvironment; chatLabelNoted?: boolean; themeMode: ThemeMode; @@ -28,6 +29,7 @@ export const { userImageUrl: '', userName: '', appMode: 'None', + callId: Math.random().toString(36).substring(6), appEnvironment: (REACT_NATIVE_DOGFOOD_APP_ENVIRONMENT as AppEnvironment) || 'demo', chatLabelNoted: false, @@ -38,6 +40,7 @@ export const { [ 'apiKey', 'appEnvironment', + 'callId', 'userId', 'userName', 'userImageUrl', diff --git a/sample-apps/react-native/dogfood/src/screens/Meeting/JoinMeetingScreen.tsx b/sample-apps/react-native/dogfood/src/screens/Meeting/JoinMeetingScreen.tsx index 8e74f4d5a7..fe2e0dc8f0 100644 --- a/sample-apps/react-native/dogfood/src/screens/Meeting/JoinMeetingScreen.tsx +++ b/sample-apps/react-native/dogfood/src/screens/Meeting/JoinMeetingScreen.tsx @@ -1,4 +1,4 @@ -import React, { useCallback, useEffect, useState, useMemo } from 'react'; +import React, { useCallback, useEffect, useMemo } from 'react'; import { Image, KeyboardAvoidingView, @@ -8,7 +8,10 @@ import { View, ViewStyle, } from 'react-native'; -import { useAppGlobalStoreValue } from '../../contexts/AppContext'; +import { + useAppGlobalStoreSetState, + useAppGlobalStoreValue, +} from '../../contexts/AppContext'; import { randomId } from '../../modules/helpers/randomId'; import { NativeStackScreenProps } from '@react-navigation/native-stack'; import { MeetingStackParamList } from '../../../types'; @@ -29,7 +32,8 @@ const callIdRegex = /^[A-Za-z0-9_-]*$/g; const isValidCallId = (callId: string) => callId && callId.match(callIdRegex); const JoinMeetingScreen = (props: JoinMeetingScreenProps) => { - const [callId, setCallId] = useState(''); + const setState = useAppGlobalStoreSetState(); + const callId = useAppGlobalStoreValue((store) => store.callId) || ''; const { theme } = useTheme(); const { t } = useI18n(); const orientation = useOrientation(); @@ -71,7 +75,6 @@ const JoinMeetingScreen = (props: JoinMeetingScreenProps) => { }; const isValidCall = isValidCallId(callId); - return ( { autoCapitalize="none" autoCorrect={false} onChangeText={(text) => { - setCallId(text.trim().split(' ').join('-')); + setState({ callId: text.trim().split(' ').join('-') }); }} />