feat(dynascale): pick scaleResolutionDownBy parameter from the change…

…PublishQuality message (#1113) With this change, it is also possible to go even below the resolution announced for a layer based on the `scaleResolutionDownBy` parameter sent by the server. This helps us save the bitrate even more. --------- Co-authored-by: Oliver Lazoroski <[email protected]>
GetStream · Oct 25, 2023 · 81b91d4 · 81b91d4
1 parent cbb038c
commit 81b91d4
Show file tree

Hide file tree

Showing 5 changed files with 96 additions and 15 deletions.
diff --git a/packages/client/src/Call.ts b/packages/client/src/Call.ts
@@ -93,7 +93,11 @@ import {
   timer,
 } from 'rxjs';
 import { TrackSubscriptionDetails } from './gen/video/sfu/signal_rpc/signal';
-import { JoinResponse, Migration } from './gen/video/sfu/event/events';
+import {
+  JoinResponse,
+  Migration,
+  VideoLayerSetting,
+} from './gen/video/sfu/event/events';
 import { Timestamp } from './gen/google/protobuf/timestamp';
 import {
   createStatsReporter,
@@ -1393,8 +1397,8 @@ export class Call {
    * @param enabledRids
    * @returns
    */
-  updatePublishQuality = async (enabledRids: string[]) => {
-    return this.publisher?.updateVideoPublishQuality(enabledRids);
+  updatePublishQuality = async (enabledLayers: VideoLayerSetting[]) => {
+    return this.publisher?.updateVideoPublishQuality(enabledLayers);
   };
 
   private assertCallJoined = () => {

diff --git a/packages/client/src/events/internal.ts b/packages/client/src/events/internal.ts
@@ -21,9 +21,7 @@ export const watchChangePublishQuality = (
     const { videoSenders } = e.eventPayload.changePublishQuality;
     videoSenders.forEach((videoSender) => {
       const { layers } = videoSender;
-      call.updatePublishQuality(
-        layers.filter((l) => l.active).map((l) => l.name),
-      );
+      call.updatePublishQuality(layers.filter((l) => l.active));
     });
   });
 };

diff --git a/packages/client/src/gen/video/sfu/event/events.ts b/packages/client/src/gen/video/sfu/event/events.ts
@@ -620,6 +620,10 @@ export interface VideoMediaRequest {
   idealFrameRate: number;
 }
 /**
+ * VideoLayerSetting is used to specify various parameters of a particular encoding in simulcast.
+ * The parameters are specified here - https://developer.mozilla.org/en-US/docs/Web/API/RTCRtpEncodingParameters
+ * SDKs use these parameters sent from the server to dynamically adjust these parameters to save CPU, bandwidth
+ *
  * @generated from protobuf message stream.video.sfu.event.VideoLayerSetting
  */
 export interface VideoLayerSetting {
@@ -647,6 +651,10 @@ export interface VideoLayerSetting {
    * @generated from protobuf field: stream.video.sfu.models.Codec codec = 6;
    */
   codec?: Codec;
+  /**
+   * @generated from protobuf field: uint32 max_framerate = 7;
+   */
+  maxFramerate: number;
 }
 /**
  * @generated from protobuf enum stream.video.sfu.event.VideoLayerSetting.Priority
@@ -3489,6 +3497,12 @@ class VideoLayerSetting$Type extends MessageType<VideoLayerSetting> {
         ],
       },
       { no: 6, name: 'codec', kind: 'message', T: () => Codec },
+      {
+        no: 7,
+        name: 'max_framerate',
+        kind: 'scalar',
+        T: 13 /*ScalarType.UINT32*/,
+      },
     ]);
   }
   create(value?: PartialMessage<VideoLayerSetting>): VideoLayerSetting {
@@ -3498,6 +3512,7 @@ class VideoLayerSetting$Type extends MessageType<VideoLayerSetting> {
       maxBitrate: 0,
       scaleResolutionDownBy: 0,
       priority: 0,
+      maxFramerate: 0,
     };
     globalThis.Object.defineProperty(message, MESSAGE_TYPE, {
       enumerable: false,
@@ -3541,6 +3556,9 @@ class VideoLayerSetting$Type extends MessageType<VideoLayerSetting> {
             message.codec,
           );
           break;
+        case /* uint32 max_framerate */ 7:
+          message.maxFramerate = reader.uint32();
+          break;
         default:
           let u = options.readUnknownField;
           if (u === 'throw')
@@ -3587,6 +3605,9 @@ class VideoLayerSetting$Type extends MessageType<VideoLayerSetting> {
         writer.tag(6, WireType.LengthDelimited).fork(),
         options,
       ).join();
+    /* uint32 max_framerate = 7; */
+    if (message.maxFramerate !== 0)
+      writer.tag(7, WireType.Varint).uint32(message.maxFramerate);
     let u = options.writeUnknownFields;
     if (u !== false)
       (u == true ? UnknownFieldHandler.onWrite : u)(

diff --git a/packages/client/src/gen/video/sfu/models/models.ts b/packages/client/src/gen/video/sfu/models/models.ts
@@ -615,6 +615,10 @@ export enum ErrorCode {
    * @generated from protobuf enum value: ERROR_CODE_SFU_SHUTTING_DOWN = 600;
    */
   SFU_SHUTTING_DOWN = 600,
+  /**
+   * @generated from protobuf enum value: ERROR_CODE_SFU_FULL = 700;
+   */
+  SFU_FULL = 700,
 }
 /**
  * @generated from protobuf enum stream.video.sfu.models.SdkType
@@ -648,6 +652,10 @@ export enum SdkType {
    * @generated from protobuf enum value: SDK_TYPE_REACT_NATIVE = 6;
    */
   REACT_NATIVE = 6,
+  /**
+   * @generated from protobuf enum value: SDK_TYPE_UNITY = 7;
+   */
+  UNITY = 7,
 }
 /**
  * @generated from protobuf enum stream.video.sfu.models.TrackUnpublishReason

diff --git a/packages/client/src/rtc/Publisher.ts b/packages/client/src/rtc/Publisher.ts
@@ -26,6 +26,7 @@ import { Logger } from '../coordinator/connection/types';
 import { getLogger } from '../logger';
 import { Dispatcher } from './Dispatcher';
 import { getOSInfo } from '../client-details';
+import { VideoLayerSetting } from '../gen/video/sfu/event/events';
 
 const logger: Logger = getLogger(['Publisher']);
 
@@ -414,11 +415,11 @@ export class Publisher {
     });
   };
 
-  updateVideoPublishQuality = async (enabledRids: string[]) => {
+  updateVideoPublishQuality = async (enabledLayers: VideoLayerSetting[]) => {
     logger(
       'info',
-      'Update publish quality, requested rids by SFU:',
-      enabledRids,
+      'Update publish quality, requested layers by SFU:',
+      enabledLayers,
     );
 
     const videoSender = this.transceiverRegistry[TrackType.VIDEO]?.sender;
@@ -437,24 +438,73 @@ export class Publisher {
     }
 
     let changed = false;
+    let enabledRids = enabledLayers
+      .filter((ly) => ly.active)
+      .map((ly) => ly.name);
     params.encodings.forEach((enc) => {
       // flip 'active' flag only when necessary
       const shouldEnable = enabledRids.includes(enc.rid!);
       if (shouldEnable !== enc.active) {
         enc.active = shouldEnable;
         changed = true;
       }
+      if (shouldEnable) {
+        let layer = enabledLayers.find((vls) => vls.name === enc.rid);
+        if (layer !== undefined) {
+          if (
+            layer.scaleResolutionDownBy >= 1 &&
+            layer.scaleResolutionDownBy !== enc.scaleResolutionDownBy
+          ) {
+            logger(
+              'debug',
+              '[dynascale]: setting scaleResolutionDownBy from server',
+              'layer',
+              layer.name,
+              'scale-resolution-down-by',
+              layer.scaleResolutionDownBy,
+            );
+            enc.scaleResolutionDownBy = layer.scaleResolutionDownBy;
+            changed = true;
+          }
+
+          if (layer.maxBitrate > 0 && layer.maxBitrate !== enc.maxBitrate) {
+            logger(
+              'debug',
+              '[dynascale] setting max-bitrate from the server',
+              'layer',
+              layer.name,
+              'max-bitrate',
+              layer.maxBitrate,
+            );
+            enc.maxBitrate = layer.maxBitrate;
+            changed = true;
+          }
+
+          if (
+            layer.maxFramerate > 0 &&
+            layer.maxFramerate !== enc.maxFramerate
+          ) {
+            logger(
+              'debug',
+              '[dynascale]: setting maxFramerate from server',
+              'layer',
+              layer.name,
+              'max-framerate',
+              layer.maxFramerate,
+            );
+            enc.maxFramerate = layer.maxFramerate;
+            changed = true;
+          }
+        }
+      }
     });
 
-    const activeRids = params.encodings
-      .filter((e) => e.active)
-      .map((e) => e.rid)
-      .join(', ');
+    const activeLayers = params.encodings.filter((e) => e.active);
     if (changed) {
       await videoSender.setParameters(params);
-      logger('info', `Update publish quality, enabled rids: ${activeRids}`);
+      logger('info', `Update publish quality, enabled rids: `, activeLayers);
     } else {
-      logger('info', `Update publish quality, no change: ${activeRids}`);
+      logger('info', `Update publish quality, no change: `, activeLayers);
     }
   };