From 35907a198d23e9700681f21efae63d179c4e4fd8 Mon Sep 17 00:00:00 2001 From: Suchith J N Date: Thu, 28 Sep 2023 14:26:39 +0530 Subject: [PATCH 1/6] feat: pick scaleResolutionDownBy parameter from the changePublishQuality message --- packages/client/src/Call.ts | 6 +++--- packages/client/src/events/internal.ts | 4 +--- packages/client/src/rtc/Publisher.ts | 24 ++++++++++++++++-------- 3 files changed, 20 insertions(+), 14 deletions(-) diff --git a/packages/client/src/Call.ts b/packages/client/src/Call.ts index 43024deaae..43260ea8e4 100644 --- a/packages/client/src/Call.ts +++ b/packages/client/src/Call.ts @@ -91,7 +91,7 @@ import { timer, } from 'rxjs'; import { TrackSubscriptionDetails } from './gen/video/sfu/signal_rpc/signal'; -import { JoinResponse, Migration } from './gen/video/sfu/event/events'; +import { JoinResponse, Migration, VideoLayerSetting, VideoSender } from './gen/video/sfu/event/events'; import { Timestamp } from './gen/google/protobuf/timestamp'; import { createStatsReporter, @@ -1352,8 +1352,8 @@ export class Call { * @param enabledRids * @returns */ - updatePublishQuality = async (enabledRids: string[]) => { - return this.publisher?.updateVideoPublishQuality(enabledRids); + updatePublishQuality = async (enabledLayers: VideoLayerSetting[]) => { + return this.publisher?.updateVideoPublishQuality(enabledLayers); }; private assertCallJoined = () => { diff --git a/packages/client/src/events/internal.ts b/packages/client/src/events/internal.ts index 41cd31c3ec..77467d9aef 100644 --- a/packages/client/src/events/internal.ts +++ b/packages/client/src/events/internal.ts @@ -21,9 +21,7 @@ export const watchChangePublishQuality = ( const { videoSenders } = e.eventPayload.changePublishQuality; videoSenders.forEach((videoSender) => { const { layers } = videoSender; - call.updatePublishQuality( - layers.filter((l) => l.active).map((l) => l.name), - ); + call.updatePublishQuality(layers.filter((l) => l.active)); }); }); }; diff --git a/packages/client/src/rtc/Publisher.ts b/packages/client/src/rtc/Publisher.ts index 4d8b370b38..ec4c21f498 100644 --- a/packages/client/src/rtc/Publisher.ts +++ b/packages/client/src/rtc/Publisher.ts @@ -26,6 +26,7 @@ import { Logger } from '../coordinator/connection/types'; import { getLogger } from '../logger'; import { Dispatcher } from './Dispatcher'; import { getOSInfo } from '../client-details'; +import { VideoLayerSetting } from '../gen/video/sfu/event/events'; const logger: Logger = getLogger(['Publisher']); @@ -404,11 +405,11 @@ export class Publisher { }); }; - updateVideoPublishQuality = async (enabledRids: string[]) => { + updateVideoPublishQuality = async (enabledLayers: VideoLayerSetting[]) => { logger( 'info', - 'Update publish quality, requested rids by SFU:', - enabledRids, + 'Update publish quality, requested layers by SFU:', + enabledLayers, ); const videoSender = this.transceiverRegistry[TrackType.VIDEO]?.sender; @@ -427,6 +428,7 @@ export class Publisher { } let changed = false; + let enabledRids = enabledLayers.filter(ly => ly.active).map(ly => ly.name); params.encodings.forEach((enc) => { // flip 'active' flag only when necessary const shouldEnable = enabledRids.includes(enc.rid!); @@ -434,17 +436,23 @@ export class Publisher { enc.active = shouldEnable; changed = true; } + if (shouldEnable) { + let layer = enabledLayers.find(vls => vls.name === enc.rid); + if (layer !== undefined && layer.scaleResolutionDownBy !== enc.scaleResolutionDownBy) { + // TODO: Find a default scale down value if SFU specifies some nonsense or doesn't set it + enc.scaleResolutionDownBy = Math.max(1.0, layer.scaleResolutionDownBy); // min should be 1.0 + changed = true; + } + } }); - const activeRids = params.encodings + const activeLayers = params.encodings .filter((e) => e.active) - .map((e) => e.rid) - .join(', '); if (changed) { await videoSender.setParameters(params); - logger('info', `Update publish quality, enabled rids: ${activeRids}`); + logger('info', `Update publish quality, enabled rids: `, activeLayers); } else { - logger('info', `Update publish quality, no change: ${activeRids}`); + logger('info', `Update publish quality, no change: `, activeLayers); } }; From 5592e0525e22c9549918043ec9ac9873964d426d Mon Sep 17 00:00:00 2001 From: Suchith J N Date: Fri, 29 Sep 2023 14:03:01 +0530 Subject: [PATCH 2/6] pick maxFramerate, maxBitrate and priority parameters from the ChangePublishQuality message --- packages/client/src/rtc/Publisher.ts | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/packages/client/src/rtc/Publisher.ts b/packages/client/src/rtc/Publisher.ts index ec4c21f498..87de83f8f5 100644 --- a/packages/client/src/rtc/Publisher.ts +++ b/packages/client/src/rtc/Publisher.ts @@ -438,6 +438,26 @@ export class Publisher { } if (shouldEnable) { let layer = enabledLayers.find(vls => vls.name === enc.rid); + if (layer !== undefined) { + + // It is possible that the SDK changes are deployed before SFU. Older SFUs + // send 0 and it must not be considered. The minimum valid value is 1.0 + // because resolution scale up is not allowed + if (layer.scaleResolutionDownBy >= 1 && layer.scaleResolutionDownBy !== enc.scaleResolutionDownBy) { + enc.scaleResolutionDownBy = layer.scaleResolutionDownBy; + changed = true; + } + if (layer.maxBitrate > 0 && layer.maxBitrate !== enc.maxBitrate) { + enc.maxBitrate = layer.maxBitrate; + changed = true; + } + // Wait till it is merged in protocol repo (https://github.com/GetStream/protocol/pull/254) + // if (layer.maxFramerate > 0 && layer.maxFramerate !== enc.maxFramerate) { + // enc.maxFramerate = layer.maxFramerate; + // changed = true; + // } + // TODO: Set priority from the ChangePublishQuality message + } if (layer !== undefined && layer.scaleResolutionDownBy !== enc.scaleResolutionDownBy) { // TODO: Find a default scale down value if SFU specifies some nonsense or doesn't set it enc.scaleResolutionDownBy = Math.max(1.0, layer.scaleResolutionDownBy); // min should be 1.0 From 750ce9741d9e02a6e51064b94f04cc3b2d560a16 Mon Sep 17 00:00:00 2001 From: Oliver Lazoroski Date: Wed, 4 Oct 2023 09:57:04 +0200 Subject: [PATCH 3/6] feat: update to the latest protobuf definitions --- .../client/src/gen/video/sfu/event/events.ts | 21 +++++++++++++++++++ .../client/src/gen/video/sfu/models/models.ts | 8 +++++++ 2 files changed, 29 insertions(+) diff --git a/packages/client/src/gen/video/sfu/event/events.ts b/packages/client/src/gen/video/sfu/event/events.ts index 1c279f7bf7..e1e2dac5b3 100644 --- a/packages/client/src/gen/video/sfu/event/events.ts +++ b/packages/client/src/gen/video/sfu/event/events.ts @@ -620,6 +620,10 @@ export interface VideoMediaRequest { idealFrameRate: number; } /** + * VideoLayerSetting is used to specify various parameters of a particular encoding in simulcast. + * The parameters are specified here - https://developer.mozilla.org/en-US/docs/Web/API/RTCRtpEncodingParameters + * SDKs use these parameters sent from the server to dynamically adjust these parameters to save CPU, bandwidth + * * @generated from protobuf message stream.video.sfu.event.VideoLayerSetting */ export interface VideoLayerSetting { @@ -647,6 +651,10 @@ export interface VideoLayerSetting { * @generated from protobuf field: stream.video.sfu.models.Codec codec = 6; */ codec?: Codec; + /** + * @generated from protobuf field: uint32 max_framerate = 7; + */ + maxFramerate: number; } /** * @generated from protobuf enum stream.video.sfu.event.VideoLayerSetting.Priority @@ -3489,6 +3497,12 @@ class VideoLayerSetting$Type extends MessageType { ], }, { no: 6, name: 'codec', kind: 'message', T: () => Codec }, + { + no: 7, + name: 'max_framerate', + kind: 'scalar', + T: 13 /*ScalarType.UINT32*/, + }, ]); } create(value?: PartialMessage): VideoLayerSetting { @@ -3498,6 +3512,7 @@ class VideoLayerSetting$Type extends MessageType { maxBitrate: 0, scaleResolutionDownBy: 0, priority: 0, + maxFramerate: 0, }; globalThis.Object.defineProperty(message, MESSAGE_TYPE, { enumerable: false, @@ -3541,6 +3556,9 @@ class VideoLayerSetting$Type extends MessageType { message.codec, ); break; + case /* uint32 max_framerate */ 7: + message.maxFramerate = reader.uint32(); + break; default: let u = options.readUnknownField; if (u === 'throw') @@ -3587,6 +3605,9 @@ class VideoLayerSetting$Type extends MessageType { writer.tag(6, WireType.LengthDelimited).fork(), options, ).join(); + /* uint32 max_framerate = 7; */ + if (message.maxFramerate !== 0) + writer.tag(7, WireType.Varint).uint32(message.maxFramerate); let u = options.writeUnknownFields; if (u !== false) (u == true ? UnknownFieldHandler.onWrite : u)( diff --git a/packages/client/src/gen/video/sfu/models/models.ts b/packages/client/src/gen/video/sfu/models/models.ts index 2385ff7068..069afafb8e 100644 --- a/packages/client/src/gen/video/sfu/models/models.ts +++ b/packages/client/src/gen/video/sfu/models/models.ts @@ -615,6 +615,10 @@ export enum ErrorCode { * @generated from protobuf enum value: ERROR_CODE_SFU_SHUTTING_DOWN = 600; */ SFU_SHUTTING_DOWN = 600, + /** + * @generated from protobuf enum value: ERROR_CODE_SFU_FULL = 700; + */ + SFU_FULL = 700, } /** * @generated from protobuf enum stream.video.sfu.models.SdkType @@ -648,6 +652,10 @@ export enum SdkType { * @generated from protobuf enum value: SDK_TYPE_REACT_NATIVE = 6; */ REACT_NATIVE = 6, + /** + * @generated from protobuf enum value: SDK_TYPE_UNITY = 7; + */ + UNITY = 7, } /** * @generated from protobuf enum stream.video.sfu.models.TrackUnpublishReason From 97084984a53ef4c0087a7099d05620a406d9ca88 Mon Sep 17 00:00:00 2001 From: Oliver Lazoroski Date: Wed, 4 Oct 2023 10:29:56 +0200 Subject: [PATCH 4/6] fix: linting --- packages/client/src/Call.ts | 6 +++++- packages/client/src/rtc/Publisher.ts | 25 +++++++++++++++++-------- 2 files changed, 22 insertions(+), 9 deletions(-) diff --git a/packages/client/src/Call.ts b/packages/client/src/Call.ts index 43260ea8e4..733d7a4c5c 100644 --- a/packages/client/src/Call.ts +++ b/packages/client/src/Call.ts @@ -91,7 +91,11 @@ import { timer, } from 'rxjs'; import { TrackSubscriptionDetails } from './gen/video/sfu/signal_rpc/signal'; -import { JoinResponse, Migration, VideoLayerSetting, VideoSender } from './gen/video/sfu/event/events'; +import { + JoinResponse, + Migration, + VideoLayerSetting, +} from './gen/video/sfu/event/events'; import { Timestamp } from './gen/google/protobuf/timestamp'; import { createStatsReporter, diff --git a/packages/client/src/rtc/Publisher.ts b/packages/client/src/rtc/Publisher.ts index 87de83f8f5..20645993c6 100644 --- a/packages/client/src/rtc/Publisher.ts +++ b/packages/client/src/rtc/Publisher.ts @@ -428,7 +428,9 @@ export class Publisher { } let changed = false; - let enabledRids = enabledLayers.filter(ly => ly.active).map(ly => ly.name); + let enabledRids = enabledLayers + .filter((ly) => ly.active) + .map((ly) => ly.name); params.encodings.forEach((enc) => { // flip 'active' flag only when necessary const shouldEnable = enabledRids.includes(enc.rid!); @@ -437,13 +439,15 @@ export class Publisher { changed = true; } if (shouldEnable) { - let layer = enabledLayers.find(vls => vls.name === enc.rid); + let layer = enabledLayers.find((vls) => vls.name === enc.rid); if (layer !== undefined) { - // It is possible that the SDK changes are deployed before SFU. Older SFUs // send 0 and it must not be considered. The minimum valid value is 1.0 // because resolution scale up is not allowed - if (layer.scaleResolutionDownBy >= 1 && layer.scaleResolutionDownBy !== enc.scaleResolutionDownBy) { + if ( + layer.scaleResolutionDownBy >= 1 && + layer.scaleResolutionDownBy !== enc.scaleResolutionDownBy + ) { enc.scaleResolutionDownBy = layer.scaleResolutionDownBy; changed = true; } @@ -458,16 +462,21 @@ export class Publisher { // } // TODO: Set priority from the ChangePublishQuality message } - if (layer !== undefined && layer.scaleResolutionDownBy !== enc.scaleResolutionDownBy) { + if ( + layer !== undefined && + layer.scaleResolutionDownBy !== enc.scaleResolutionDownBy + ) { // TODO: Find a default scale down value if SFU specifies some nonsense or doesn't set it - enc.scaleResolutionDownBy = Math.max(1.0, layer.scaleResolutionDownBy); // min should be 1.0 + enc.scaleResolutionDownBy = Math.max( + 1.0, + layer.scaleResolutionDownBy, + ); // min should be 1.0 changed = true; } } }); - const activeLayers = params.encodings - .filter((e) => e.active) + const activeLayers = params.encodings.filter((e) => e.active); if (changed) { await videoSender.setParameters(params); logger('info', `Update publish quality, enabled rids: `, activeLayers); From d57b232fb278b99c1733299ded025032a0e28078 Mon Sep 17 00:00:00 2001 From: Suchith J N Date: Mon, 23 Oct 2023 13:26:31 +0530 Subject: [PATCH 5/6] pick encoding parameters from the server --- packages/client/src/rtc/Publisher.ts | 38 +++++++++++++--------------- 1 file changed, 17 insertions(+), 21 deletions(-) diff --git a/packages/client/src/rtc/Publisher.ts b/packages/client/src/rtc/Publisher.ts index 20645993c6..7827c64a0d 100644 --- a/packages/client/src/rtc/Publisher.ts +++ b/packages/client/src/rtc/Publisher.ts @@ -441,37 +441,33 @@ export class Publisher { if (shouldEnable) { let layer = enabledLayers.find((vls) => vls.name === enc.rid); if (layer !== undefined) { - // It is possible that the SDK changes are deployed before SFU. Older SFUs - // send 0 and it must not be considered. The minimum valid value is 1.0 - // because resolution scale up is not allowed if ( layer.scaleResolutionDownBy >= 1 && layer.scaleResolutionDownBy !== enc.scaleResolutionDownBy ) { + logger('debug', '[dynascale]: setting scaleResolutionDownBy from server', + 'layer', layer.name, 'scale-resolution-down-by', layer.scaleResolutionDownBy) enc.scaleResolutionDownBy = layer.scaleResolutionDownBy; changed = true; } - if (layer.maxBitrate > 0 && layer.maxBitrate !== enc.maxBitrate) { + + if ( + layer.maxBitrate > 0 && + layer.maxBitrate !== enc.maxBitrate + ) { + logger('debug', '[dynascale] setting max-bitrate from the server', + 'layer', layer.name, 'max-bitrate', layer.maxBitrate); enc.maxBitrate = layer.maxBitrate; changed = true; } - // Wait till it is merged in protocol repo (https://github.com/GetStream/protocol/pull/254) - // if (layer.maxFramerate > 0 && layer.maxFramerate !== enc.maxFramerate) { - // enc.maxFramerate = layer.maxFramerate; - // changed = true; - // } - // TODO: Set priority from the ChangePublishQuality message - } - if ( - layer !== undefined && - layer.scaleResolutionDownBy !== enc.scaleResolutionDownBy - ) { - // TODO: Find a default scale down value if SFU specifies some nonsense or doesn't set it - enc.scaleResolutionDownBy = Math.max( - 1.0, - layer.scaleResolutionDownBy, - ); // min should be 1.0 - changed = true; + + if ( + layer.maxFramerate > 0 && + layer.maxFramerate !== enc.maxFramerate + ) { + enc.maxFramerate = layer.maxFramerate; + changed = true; + } } } }); From 9799da66ab3ebd3aa6db083e48f8f34d6dc96ba0 Mon Sep 17 00:00:00 2001 From: Suchith J N Date: Mon, 23 Oct 2023 13:50:03 +0530 Subject: [PATCH 6/6] make lint happy --- packages/client/src/rtc/Publisher.ts | 35 +++++++++++++++++++++------- 1 file changed, 26 insertions(+), 9 deletions(-) diff --git a/packages/client/src/rtc/Publisher.ts b/packages/client/src/rtc/Publisher.ts index 876c765904..231ed434f2 100644 --- a/packages/client/src/rtc/Publisher.ts +++ b/packages/client/src/rtc/Publisher.ts @@ -455,26 +455,43 @@ export class Publisher { layer.scaleResolutionDownBy >= 1 && layer.scaleResolutionDownBy !== enc.scaleResolutionDownBy ) { - logger('debug', '[dynascale]: setting scaleResolutionDownBy from server', - 'layer', layer.name, 'scale-resolution-down-by', layer.scaleResolutionDownBy) + logger( + 'debug', + '[dynascale]: setting scaleResolutionDownBy from server', + 'layer', + layer.name, + 'scale-resolution-down-by', + layer.scaleResolutionDownBy, + ); enc.scaleResolutionDownBy = layer.scaleResolutionDownBy; changed = true; } - if ( - layer.maxBitrate > 0 && - layer.maxBitrate !== enc.maxBitrate - ) { - logger('debug', '[dynascale] setting max-bitrate from the server', - 'layer', layer.name, 'max-bitrate', layer.maxBitrate); + if (layer.maxBitrate > 0 && layer.maxBitrate !== enc.maxBitrate) { + logger( + 'debug', + '[dynascale] setting max-bitrate from the server', + 'layer', + layer.name, + 'max-bitrate', + layer.maxBitrate, + ); enc.maxBitrate = layer.maxBitrate; changed = true; } if ( - layer.maxFramerate > 0 && + layer.maxFramerate > 0 && layer.maxFramerate !== enc.maxFramerate ) { + logger( + 'debug', + '[dynascale]: setting maxFramerate from server', + 'layer', + layer.name, + 'max-framerate', + layer.maxFramerate, + ); enc.maxFramerate = layer.maxFramerate; changed = true; }