Implement manual quality selection (#1242)

GetStream · Dec 6, 2024 · 3d0dc1c · 3d0dc1c
1 parent 1bb5618
commit 3d0dc1c
Show file tree

Hide file tree

Showing 12 changed files with 449 additions and 109 deletions.
diff --git a/demo-app/src/main/kotlin/io/getstream/video/android/ui/call/CallScreen.kt b/demo-app/src/main/kotlin/io/getstream/video/android/ui/call/CallScreen.kt
@@ -94,6 +94,7 @@ import io.getstream.video.android.compose.ui.components.video.VideoScalingType
 import io.getstream.video.android.core.Call
 import io.getstream.video.android.core.RealtimeConnection
 import io.getstream.video.android.core.call.state.ChooseLayout
+import io.getstream.video.android.core.model.PreferredVideoResolution
 import io.getstream.video.android.core.utils.isEnabled
 import io.getstream.video.android.filters.video.BlurredBackgroundVideoFilter
 import io.getstream.video.android.filters.video.VirtualBackgroundVideoFilter
@@ -149,6 +150,10 @@ fun CallScreen(
     val messageScope = rememberCoroutineScope()
     var showingLandscapeControls by remember { mutableStateOf(false) }
     var preferredScaleType by remember { mutableStateOf(VideoScalingType.SCALE_ASPECT_FILL) }
+    var selectedIncomingVideoResolution by remember {
+        mutableStateOf<PreferredVideoResolution?>(null)
+    }
+    var isIncomingVideoEnabled by remember { mutableStateOf(true) }
 
     val connection by call.state.connection.collectAsStateWithLifecycle()
     val me by call.state.me.collectAsStateWithLifecycle()
@@ -501,6 +506,23 @@ fun CallScreen(
                 onNoiseCancellation = {
                     isNoiseCancellationEnabled = call.toggleAudioProcessing()
                 },
+                selectedIncomingVideoResolution = selectedIncomingVideoResolution,
+                onSelectIncomingVideoResolution = {
+                    call.setIncomingVideoEnabled(true)
+                    isIncomingVideoEnabled = true
+
+                    call.setPreferredIncomingVideoResolution(it)
+                    selectedIncomingVideoResolution = it
+
+                    isShowingSettingMenu = false
+                },
+                isIncomingVideoEnabled = isIncomingVideoEnabled,
+                onToggleIncomingVideoVisibility = {
+                    call.setIncomingVideoEnabled(it)
+                    isIncomingVideoEnabled = it
+
+                    isShowingSettingMenu = false
+                },
                 onSelectScaleType = {
                     preferredScaleType = it
                     isShowingSettingMenu = false

diff --git a/demo-app/src/main/kotlin/io/getstream/video/android/ui/menu/MenuDefinitions.kt b/demo-app/src/main/kotlin/io/getstream/video/android/ui/menu/MenuDefinitions.kt
@@ -41,8 +41,11 @@ import androidx.compose.material.icons.filled.SwitchLeft
 import androidx.compose.material.icons.filled.VideoFile
 import androidx.compose.material.icons.filled.VideoLibrary
 import androidx.compose.material.icons.filled.VideoSettings
+import androidx.compose.material.icons.filled.Videocam
+import androidx.compose.material.icons.filled.VideocamOff
 import io.getstream.video.android.compose.ui.components.video.VideoScalingType
 import io.getstream.video.android.core.audio.StreamAudioDevice
+import io.getstream.video.android.core.model.PreferredVideoResolution
 import io.getstream.video.android.ui.menu.base.ActionMenuItem
 import io.getstream.video.android.ui.menu.base.DynamicSubMenuItem
 import io.getstream.video.android.ui.menu.base.MenuItem
@@ -66,6 +69,10 @@ fun defaultStreamMenu(
     onSwitchSfuClick: () -> Unit,
     onShowFeedback: () -> Unit,
     onNoiseCancellation: () -> Unit,
+    selectedIncomingVideoResolution: PreferredVideoResolution?,
+    onSelectIncomingVideoResolution: (PreferredVideoResolution?) -> Unit,
+    isIncomingVideoEnabled: Boolean,
+    onToggleIncomingVideoEnabled: (Boolean) -> Unit,
     onDeviceSelected: (StreamAudioDevice) -> Unit,
     onSfuRejoinClick: () -> Unit,
     onSfuFastReconnectClick: () -> Unit,
@@ -130,6 +137,65 @@ fun defaultStreamMenu(
             ),
         )
     }
+    add(
+        SubMenuItem(
+            title = "Incoming video settings",
+            icon = Icons.Default.VideoSettings,
+            items = listOf(
+                ActionMenuItem(
+                    title = "Auto Quality",
+                    icon = Icons.Default.AspectRatio,
+                    highlight = selectedIncomingVideoResolution == null,
+                    action = { onSelectIncomingVideoResolution(null) },
+                ),
+                ActionMenuItem(
+                    title = "4K 2160p",
+                    icon = Icons.Default.AspectRatio,
+                    highlight = selectedIncomingVideoResolution == PreferredVideoResolution(3840, 2160),
+                    action = {
+                        onSelectIncomingVideoResolution(PreferredVideoResolution(3840, 2160))
+                    },
+                ),
+                ActionMenuItem(
+                    title = "Full HD 1080p",
+                    icon = Icons.Default.AspectRatio,
+                    highlight = selectedIncomingVideoResolution == PreferredVideoResolution(1920, 1080),
+                    action = {
+                        onSelectIncomingVideoResolution(PreferredVideoResolution(1920, 1080))
+                    },
+                ),
+                ActionMenuItem(
+                    title = "HD 720p",
+                    icon = Icons.Default.AspectRatio,
+                    highlight = selectedIncomingVideoResolution == PreferredVideoResolution(1280, 720),
+                    action = {
+                        onSelectIncomingVideoResolution(PreferredVideoResolution(1280, 720))
+                    },
+                ),
+                ActionMenuItem(
+                    title = "SD 480p",
+                    icon = Icons.Default.AspectRatio,
+                    highlight = selectedIncomingVideoResolution == PreferredVideoResolution(640, 480),
+                    action = {
+                        onSelectIncomingVideoResolution(PreferredVideoResolution(640, 480))
+                    },
+                ),
+                ActionMenuItem(
+                    title = "Data Saver 144p",
+                    icon = Icons.Default.AspectRatio,
+                    highlight = selectedIncomingVideoResolution == PreferredVideoResolution(256, 144),
+                    action = {
+                        onSelectIncomingVideoResolution(PreferredVideoResolution(256, 144))
+                    },
+                ),
+                ActionMenuItem(
+                    title = if (isIncomingVideoEnabled) "Disable incoming video" else "Enable incoming video",
+                    icon = if (isIncomingVideoEnabled) Icons.Default.VideocamOff else Icons.Default.Videocam,
+                    action = { onToggleIncomingVideoEnabled(!isIncomingVideoEnabled) },
+                ),
+            ),
+        ),
+    )
     if (showDebugOptions) {
         add(
             SubMenuItem(

diff --git a/demo-app/src/main/kotlin/io/getstream/video/android/ui/menu/SettingsMenu.kt b/demo-app/src/main/kotlin/io/getstream/video/android/ui/menu/SettingsMenu.kt
@@ -54,6 +54,7 @@ import io.getstream.video.android.compose.ui.components.video.VideoScalingType
 import io.getstream.video.android.core.Call
 import io.getstream.video.android.core.call.audio.InputAudioFilter
 import io.getstream.video.android.core.mapper.ReactionMapper
+import io.getstream.video.android.core.model.PreferredVideoResolution
 import io.getstream.video.android.tooling.extensions.toPx
 import io.getstream.video.android.ui.call.ReactionsMenu
 import io.getstream.video.android.ui.menu.base.ActionMenuItem
@@ -75,6 +76,10 @@ internal fun SettingsMenu(
     onSelectVideoFilter: (Int) -> Unit,
     onShowFeedback: () -> Unit,
     onNoiseCancellation: () -> Unit,
+    selectedIncomingVideoResolution: PreferredVideoResolution?,
+    onSelectIncomingVideoResolution: (PreferredVideoResolution?) -> Unit,
+    isIncomingVideoEnabled: Boolean,
+    onToggleIncomingVideoVisibility: (Boolean) -> Unit,
     onShowCallStats: () -> Unit,
     onSelectScaleType: (VideoScalingType) -> Unit,
 ) {
@@ -236,6 +241,10 @@ internal fun SettingsMenu(
                 onSwitchSfuClick = onSwitchSfuClick,
                 onShowCallStats = onShowCallStats,
                 onNoiseCancellation = onNoiseCancellation,
+                selectedIncomingVideoResolution = selectedIncomingVideoResolution,
+                onSelectIncomingVideoResolution = { onSelectIncomingVideoResolution(it) },
+                isIncomingVideoEnabled = isIncomingVideoEnabled,
+                onToggleIncomingVideoEnabled = { onToggleIncomingVideoVisibility(it) },
                 onSfuRejoinClick = onSfuRejoinClick,
                 onSfuFastReconnectClick = onSfuFastReconnectClick,
                 isScreenShareEnabled = isScreenSharing,
@@ -303,6 +312,10 @@ private fun SettingsMenuPreview() {
                 onShowFeedback = {},
                 onSelectScaleType = {},
                 onNoiseCancellation = {},
+                selectedIncomingVideoResolution = null,
+                onSelectIncomingVideoResolution = {},
+                isIncomingVideoEnabled = true,
+                onToggleIncomingVideoEnabled = {},
                 loadRecordings = { emptyList() },
             ),
         )

diff --git a/demo-app/src/main/kotlin/io/getstream/video/android/ui/menu/base/DynamicMenu.kt b/demo-app/src/main/kotlin/io/getstream/video/android/ui/menu/base/DynamicMenu.kt
@@ -225,6 +225,10 @@ private fun DynamicMenuPreview() {
                 onDeviceSelected = {},
                 onShowFeedback = {},
                 onNoiseCancellation = {},
+                selectedIncomingVideoResolution = null,
+                onSelectIncomingVideoResolution = {},
+                isIncomingVideoEnabled = true,
+                onToggleIncomingVideoEnabled = {},
                 onSelectScaleType = {},
                 loadRecordings = { emptyList() },
             ),
@@ -255,6 +259,10 @@ private fun DynamicMenuDebugOptionPreview() {
                 onShowFeedback = {},
                 onSelectScaleType = { },
                 onNoiseCancellation = {},
+                selectedIncomingVideoResolution = null,
+                onSelectIncomingVideoResolution = {},
+                isIncomingVideoEnabled = true,
+                onToggleIncomingVideoEnabled = {},
                 loadRecordings = { emptyList() },
             ),
         )

diff --git a/stream-video-android-core/api/stream-video-android-core.api b/stream-video-android-core/api/stream-video-android-core.api
@@ -68,6 +68,10 @@ public final class io/getstream/video/android/core/Call {
 	public static synthetic fun sendReaction$default (Lio/getstream/video/android/core/Call;Ljava/lang/String;Ljava/lang/String;Ljava/util/Map;Lkotlin/coroutines/Continuation;ILjava/lang/Object;)Ljava/lang/Object;
 	public final fun setAudioFilter (Lio/getstream/video/android/core/call/audio/InputAudioFilter;)V
 	public final fun setAudioProcessingEnabled (Z)V
+	public final fun setIncomingVideoEnabled (Ljava/lang/Boolean;Ljava/util/List;)V
+	public static synthetic fun setIncomingVideoEnabled$default (Lio/getstream/video/android/core/Call;Ljava/lang/Boolean;Ljava/util/List;ILjava/lang/Object;)V
+	public final fun setPreferredIncomingVideoResolution (Lio/getstream/video/android/core/model/PreferredVideoResolution;Ljava/util/List;)V
+	public static synthetic fun setPreferredIncomingVideoResolution$default (Lio/getstream/video/android/core/Call;Lio/getstream/video/android/core/model/PreferredVideoResolution;Ljava/util/List;ILjava/lang/Object;)V
 	public final fun setSessionId (Ljava/lang/String;)V
 	public final fun setVideoFilter (Lio/getstream/video/android/core/call/video/VideoFilter;)V
 	public final fun setVisibility (Ljava/lang/String;Lstream/video/sfu/models/TrackType;Z)V
@@ -149,6 +153,7 @@ public final class io/getstream/video/android/core/CallState {
 	public final fun getOwnCapabilities ()Lkotlinx/coroutines/flow/StateFlow;
 	public final fun getParticipantBySessionId (Ljava/lang/String;)Lio/getstream/video/android/core/ParticipantState;
 	public final fun getParticipantCounts ()Lkotlinx/coroutines/flow/StateFlow;
+	public final fun getParticipantVideoEnabledOverrides ()Lkotlinx/coroutines/flow/StateFlow;
 	public final fun getParticipants ()Lkotlinx/coroutines/flow/StateFlow;
 	public final fun getPermissionRequests ()Lkotlinx/coroutines/flow/StateFlow;
 	public final fun getPinnedParticipants ()Lkotlinx/coroutines/flow/StateFlow;
@@ -2857,6 +2862,10 @@ public final class io/getstream/video/android/core/call/stats/model/discriminato
 	public final fun fromAlias (Ljava/lang/String;)Lio/getstream/video/android/core/call/stats/model/discriminator/RtcReportType;
 }
 
+public final class io/getstream/video/android/core/call/utils/TrackOverridesHandlerKt {
+	public static final field ALL_PARTICIPANTS Ljava/lang/String;
+}
+
 public abstract class io/getstream/video/android/core/call/video/BitmapVideoFilter : io/getstream/video/android/core/call/video/VideoFilter {
 	public fun <init> ()V
 	public abstract fun applyFilter (Landroid/graphics/Bitmap;)V
@@ -3959,6 +3968,19 @@ public final class io/getstream/video/android/core/model/NetworkQuality$UnSpecif
 	public fun toString ()Ljava/lang/String;
 }
 
+public final class io/getstream/video/android/core/model/PreferredVideoResolution {
+	public fun <init> (II)V
+	public final fun component1 ()I
+	public final fun component2 ()I
+	public final fun copy (II)Lio/getstream/video/android/core/model/PreferredVideoResolution;
+	public static synthetic fun copy$default (Lio/getstream/video/android/core/model/PreferredVideoResolution;IIILjava/lang/Object;)Lio/getstream/video/android/core/model/PreferredVideoResolution;
+	public fun equals (Ljava/lang/Object;)Z
+	public final fun getHeight ()I
+	public final fun getWidth ()I
+	public fun hashCode ()I
+	public fun toString ()Ljava/lang/String;
+}
+
 public final class io/getstream/video/android/core/model/QueriedCalls {
 	public fun <init> (Ljava/util/List;Ljava/lang/String;Ljava/lang/String;)V
 	public final fun component1 ()Ljava/util/List;

diff --git a/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/Call.kt b/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/Call.kt
@@ -38,6 +38,7 @@ import io.getstream.video.android.core.events.VideoEventListener
 import io.getstream.video.android.core.internal.InternalStreamVideoApi
 import io.getstream.video.android.core.internal.network.NetworkStateProvider
 import io.getstream.video.android.core.model.MuteUsersData
+import io.getstream.video.android.core.model.PreferredVideoResolution
 import io.getstream.video.android.core.model.QueriedMembers
 import io.getstream.video.android.core.model.RejectReason
 import io.getstream.video.android.core.model.SortField
@@ -1289,6 +1290,34 @@ public class Call(
         return clientImpl.listTranscription(type, id)
     }
 
+    /**
+     * Sets the preferred incoming video resolution.
+     *
+     * @param resolution The preferred resolution. Set to `null` to switch back to auto.
+     * @param sessionIds The participant session IDs to apply the resolution to. If `null`, the resolution will be applied to all participants.
+     */
+    fun setPreferredIncomingVideoResolution(
+        resolution: PreferredVideoResolution?,
+        sessionIds: List<String>? = null,
+    ) {
+        session?.let { session ->
+            session.trackOverridesHandler.updateOverrides(
+                sessionIds = sessionIds,
+                dimensions = resolution?.let { VideoDimension(it.width, it.height) },
+            )
+        }
+    }
+
+    /**
+     * Enables/disables incoming video feed.
+     *
+     * @param enabled Whether the video feed should be enabled or disabled. Set to `null` to switch back to auto.
+     * @param sessionIds The participant session IDs to enable/disable the video feed for. If `null`, the setting will be applied to all participants.
+     */
+    fun setIncomingVideoEnabled(enabled: Boolean?, sessionIds: List<String>? = null) {
+        session?.trackOverridesHandler?.updateOverrides(sessionIds, visible = enabled)
+    }
+
     @InternalStreamVideoApi
     public val debug = Debug(this)
 

diff --git a/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/CallState.kt b/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/CallState.kt
@@ -59,6 +59,7 @@ import kotlinx.coroutines.flow.Flow
 import kotlinx.coroutines.flow.MutableStateFlow
 import kotlinx.coroutines.flow.SharingStarted
 import kotlinx.coroutines.flow.StateFlow
+import kotlinx.coroutines.flow.asStateFlow
 import kotlinx.coroutines.flow.channelFlow
 import kotlinx.coroutines.flow.collectLatest
 import kotlinx.coroutines.flow.combine
@@ -559,10 +560,13 @@ public class CallState(
     internal val _reactions = MutableStateFlow<List<ReactionResponse>>(emptyList())
     val reactions: StateFlow<List<ReactionResponse>> = _reactions
 
-    private val _errors: MutableStateFlow<List<ErrorEvent>> =
-        MutableStateFlow(emptyList())
+    private val _errors: MutableStateFlow<List<ErrorEvent>> = MutableStateFlow(emptyList())
     public val errors: StateFlow<List<ErrorEvent>> = _errors
 
+    internal val _participantVideoEnabledOverrides =
+        MutableStateFlow<Map<String, Boolean?>>(emptyMap())
+    val participantVideoEnabledOverrides = _participantVideoEnabledOverrides.asStateFlow()
+
     private var speakingWhileMutedResetJob: Job? = null
     private var autoJoiningCall: Job? = null
     private var ringingTimerJob: Job? = null

diff --git a/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/CallStats.kt b/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/CallStats.kt
@@ -195,8 +195,8 @@ public class CallStats(val call: Call, val callScope: CoroutineScope) {
                 val received = it.members["framesReceived"] as? Long
                 val duration = it.members["totalFramesDuration"] as? Long
                 if (participantId != null) {
-                    logger.i {
-                        "receiving video for $participantId at $frameWidth: ${it.members["frameWidth"]} and rendering it at ${visibleAt?.dimensions?.width} visible: ${visibleAt?.visible}"
+                    logger.v {
+                        "[stats] #manual-quality-selection; receiving video for $participantId at $frameWidth: ${it.members["frameWidth"]} and rendering it at ${visibleAt?.dimensions?.width} visible: ${visibleAt?.visible}"
                     }
                 }
             }