From 3d0dc1c79df5bf1db0775b1adcfcb8c4cfdaa9e5 Mon Sep 17 00:00:00 2001 From: Liviu Timar <65943217+liviu-timar@users.noreply.github.com> Date: Fri, 6 Dec 2024 14:13:55 +0200 Subject: [PATCH] Implement manual quality selection (#1242) --- .../video/android/ui/call/CallScreen.kt | 22 +++ .../video/android/ui/menu/MenuDefinitions.kt | 66 ++++++++ .../video/android/ui/menu/SettingsMenu.kt | 13 ++ .../video/android/ui/menu/base/DynamicMenu.kt | 8 + .../api/stream-video-android-core.api | 22 +++ .../io/getstream/video/android/core/Call.kt | 29 ++++ .../getstream/video/android/core/CallState.kt | 8 +- .../getstream/video/android/core/CallStats.kt | 4 +- .../video/android/core/call/RtcSession.kt | 109 ++++++------- .../core/call/utils/TrackOverridesHandler.kt | 153 ++++++++++++++++++ .../core/model/PreferredVideoResolution.kt | 22 +++ .../ui/components/video/VideoRenderer.kt | 102 ++++++------ 12 files changed, 449 insertions(+), 109 deletions(-) create mode 100644 stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/call/utils/TrackOverridesHandler.kt create mode 100644 stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/model/PreferredVideoResolution.kt diff --git a/demo-app/src/main/kotlin/io/getstream/video/android/ui/call/CallScreen.kt b/demo-app/src/main/kotlin/io/getstream/video/android/ui/call/CallScreen.kt index 9aad60cb25..52ac27132a 100644 --- a/demo-app/src/main/kotlin/io/getstream/video/android/ui/call/CallScreen.kt +++ b/demo-app/src/main/kotlin/io/getstream/video/android/ui/call/CallScreen.kt @@ -94,6 +94,7 @@ import io.getstream.video.android.compose.ui.components.video.VideoScalingType import io.getstream.video.android.core.Call import io.getstream.video.android.core.RealtimeConnection import io.getstream.video.android.core.call.state.ChooseLayout +import io.getstream.video.android.core.model.PreferredVideoResolution import io.getstream.video.android.core.utils.isEnabled import io.getstream.video.android.filters.video.BlurredBackgroundVideoFilter import io.getstream.video.android.filters.video.VirtualBackgroundVideoFilter @@ -149,6 +150,10 @@ fun CallScreen( val messageScope = rememberCoroutineScope() var showingLandscapeControls by remember { mutableStateOf(false) } var preferredScaleType by remember { mutableStateOf(VideoScalingType.SCALE_ASPECT_FILL) } + var selectedIncomingVideoResolution by remember { + mutableStateOf(null) + } + var isIncomingVideoEnabled by remember { mutableStateOf(true) } val connection by call.state.connection.collectAsStateWithLifecycle() val me by call.state.me.collectAsStateWithLifecycle() @@ -501,6 +506,23 @@ fun CallScreen( onNoiseCancellation = { isNoiseCancellationEnabled = call.toggleAudioProcessing() }, + selectedIncomingVideoResolution = selectedIncomingVideoResolution, + onSelectIncomingVideoResolution = { + call.setIncomingVideoEnabled(true) + isIncomingVideoEnabled = true + + call.setPreferredIncomingVideoResolution(it) + selectedIncomingVideoResolution = it + + isShowingSettingMenu = false + }, + isIncomingVideoEnabled = isIncomingVideoEnabled, + onToggleIncomingVideoVisibility = { + call.setIncomingVideoEnabled(it) + isIncomingVideoEnabled = it + + isShowingSettingMenu = false + }, onSelectScaleType = { preferredScaleType = it isShowingSettingMenu = false diff --git a/demo-app/src/main/kotlin/io/getstream/video/android/ui/menu/MenuDefinitions.kt b/demo-app/src/main/kotlin/io/getstream/video/android/ui/menu/MenuDefinitions.kt index df032d9791..9aa22b0473 100644 --- a/demo-app/src/main/kotlin/io/getstream/video/android/ui/menu/MenuDefinitions.kt +++ b/demo-app/src/main/kotlin/io/getstream/video/android/ui/menu/MenuDefinitions.kt @@ -41,8 +41,11 @@ import androidx.compose.material.icons.filled.SwitchLeft import androidx.compose.material.icons.filled.VideoFile import androidx.compose.material.icons.filled.VideoLibrary import androidx.compose.material.icons.filled.VideoSettings +import androidx.compose.material.icons.filled.Videocam +import androidx.compose.material.icons.filled.VideocamOff import io.getstream.video.android.compose.ui.components.video.VideoScalingType import io.getstream.video.android.core.audio.StreamAudioDevice +import io.getstream.video.android.core.model.PreferredVideoResolution import io.getstream.video.android.ui.menu.base.ActionMenuItem import io.getstream.video.android.ui.menu.base.DynamicSubMenuItem import io.getstream.video.android.ui.menu.base.MenuItem @@ -66,6 +69,10 @@ fun defaultStreamMenu( onSwitchSfuClick: () -> Unit, onShowFeedback: () -> Unit, onNoiseCancellation: () -> Unit, + selectedIncomingVideoResolution: PreferredVideoResolution?, + onSelectIncomingVideoResolution: (PreferredVideoResolution?) -> Unit, + isIncomingVideoEnabled: Boolean, + onToggleIncomingVideoEnabled: (Boolean) -> Unit, onDeviceSelected: (StreamAudioDevice) -> Unit, onSfuRejoinClick: () -> Unit, onSfuFastReconnectClick: () -> Unit, @@ -130,6 +137,65 @@ fun defaultStreamMenu( ), ) } + add( + SubMenuItem( + title = "Incoming video settings", + icon = Icons.Default.VideoSettings, + items = listOf( + ActionMenuItem( + title = "Auto Quality", + icon = Icons.Default.AspectRatio, + highlight = selectedIncomingVideoResolution == null, + action = { onSelectIncomingVideoResolution(null) }, + ), + ActionMenuItem( + title = "4K 2160p", + icon = Icons.Default.AspectRatio, + highlight = selectedIncomingVideoResolution == PreferredVideoResolution(3840, 2160), + action = { + onSelectIncomingVideoResolution(PreferredVideoResolution(3840, 2160)) + }, + ), + ActionMenuItem( + title = "Full HD 1080p", + icon = Icons.Default.AspectRatio, + highlight = selectedIncomingVideoResolution == PreferredVideoResolution(1920, 1080), + action = { + onSelectIncomingVideoResolution(PreferredVideoResolution(1920, 1080)) + }, + ), + ActionMenuItem( + title = "HD 720p", + icon = Icons.Default.AspectRatio, + highlight = selectedIncomingVideoResolution == PreferredVideoResolution(1280, 720), + action = { + onSelectIncomingVideoResolution(PreferredVideoResolution(1280, 720)) + }, + ), + ActionMenuItem( + title = "SD 480p", + icon = Icons.Default.AspectRatio, + highlight = selectedIncomingVideoResolution == PreferredVideoResolution(640, 480), + action = { + onSelectIncomingVideoResolution(PreferredVideoResolution(640, 480)) + }, + ), + ActionMenuItem( + title = "Data Saver 144p", + icon = Icons.Default.AspectRatio, + highlight = selectedIncomingVideoResolution == PreferredVideoResolution(256, 144), + action = { + onSelectIncomingVideoResolution(PreferredVideoResolution(256, 144)) + }, + ), + ActionMenuItem( + title = if (isIncomingVideoEnabled) "Disable incoming video" else "Enable incoming video", + icon = if (isIncomingVideoEnabled) Icons.Default.VideocamOff else Icons.Default.Videocam, + action = { onToggleIncomingVideoEnabled(!isIncomingVideoEnabled) }, + ), + ), + ), + ) if (showDebugOptions) { add( SubMenuItem( diff --git a/demo-app/src/main/kotlin/io/getstream/video/android/ui/menu/SettingsMenu.kt b/demo-app/src/main/kotlin/io/getstream/video/android/ui/menu/SettingsMenu.kt index 7429927535..53d4582c37 100644 --- a/demo-app/src/main/kotlin/io/getstream/video/android/ui/menu/SettingsMenu.kt +++ b/demo-app/src/main/kotlin/io/getstream/video/android/ui/menu/SettingsMenu.kt @@ -54,6 +54,7 @@ import io.getstream.video.android.compose.ui.components.video.VideoScalingType import io.getstream.video.android.core.Call import io.getstream.video.android.core.call.audio.InputAudioFilter import io.getstream.video.android.core.mapper.ReactionMapper +import io.getstream.video.android.core.model.PreferredVideoResolution import io.getstream.video.android.tooling.extensions.toPx import io.getstream.video.android.ui.call.ReactionsMenu import io.getstream.video.android.ui.menu.base.ActionMenuItem @@ -75,6 +76,10 @@ internal fun SettingsMenu( onSelectVideoFilter: (Int) -> Unit, onShowFeedback: () -> Unit, onNoiseCancellation: () -> Unit, + selectedIncomingVideoResolution: PreferredVideoResolution?, + onSelectIncomingVideoResolution: (PreferredVideoResolution?) -> Unit, + isIncomingVideoEnabled: Boolean, + onToggleIncomingVideoVisibility: (Boolean) -> Unit, onShowCallStats: () -> Unit, onSelectScaleType: (VideoScalingType) -> Unit, ) { @@ -236,6 +241,10 @@ internal fun SettingsMenu( onSwitchSfuClick = onSwitchSfuClick, onShowCallStats = onShowCallStats, onNoiseCancellation = onNoiseCancellation, + selectedIncomingVideoResolution = selectedIncomingVideoResolution, + onSelectIncomingVideoResolution = { onSelectIncomingVideoResolution(it) }, + isIncomingVideoEnabled = isIncomingVideoEnabled, + onToggleIncomingVideoEnabled = { onToggleIncomingVideoVisibility(it) }, onSfuRejoinClick = onSfuRejoinClick, onSfuFastReconnectClick = onSfuFastReconnectClick, isScreenShareEnabled = isScreenSharing, @@ -303,6 +312,10 @@ private fun SettingsMenuPreview() { onShowFeedback = {}, onSelectScaleType = {}, onNoiseCancellation = {}, + selectedIncomingVideoResolution = null, + onSelectIncomingVideoResolution = {}, + isIncomingVideoEnabled = true, + onToggleIncomingVideoEnabled = {}, loadRecordings = { emptyList() }, ), ) diff --git a/demo-app/src/main/kotlin/io/getstream/video/android/ui/menu/base/DynamicMenu.kt b/demo-app/src/main/kotlin/io/getstream/video/android/ui/menu/base/DynamicMenu.kt index 463ca2625a..5037a728de 100644 --- a/demo-app/src/main/kotlin/io/getstream/video/android/ui/menu/base/DynamicMenu.kt +++ b/demo-app/src/main/kotlin/io/getstream/video/android/ui/menu/base/DynamicMenu.kt @@ -225,6 +225,10 @@ private fun DynamicMenuPreview() { onDeviceSelected = {}, onShowFeedback = {}, onNoiseCancellation = {}, + selectedIncomingVideoResolution = null, + onSelectIncomingVideoResolution = {}, + isIncomingVideoEnabled = true, + onToggleIncomingVideoEnabled = {}, onSelectScaleType = {}, loadRecordings = { emptyList() }, ), @@ -255,6 +259,10 @@ private fun DynamicMenuDebugOptionPreview() { onShowFeedback = {}, onSelectScaleType = { }, onNoiseCancellation = {}, + selectedIncomingVideoResolution = null, + onSelectIncomingVideoResolution = {}, + isIncomingVideoEnabled = true, + onToggleIncomingVideoEnabled = {}, loadRecordings = { emptyList() }, ), ) diff --git a/stream-video-android-core/api/stream-video-android-core.api b/stream-video-android-core/api/stream-video-android-core.api index d40444a79e..5fec5db314 100644 --- a/stream-video-android-core/api/stream-video-android-core.api +++ b/stream-video-android-core/api/stream-video-android-core.api @@ -68,6 +68,10 @@ public final class io/getstream/video/android/core/Call { public static synthetic fun sendReaction$default (Lio/getstream/video/android/core/Call;Ljava/lang/String;Ljava/lang/String;Ljava/util/Map;Lkotlin/coroutines/Continuation;ILjava/lang/Object;)Ljava/lang/Object; public final fun setAudioFilter (Lio/getstream/video/android/core/call/audio/InputAudioFilter;)V public final fun setAudioProcessingEnabled (Z)V + public final fun setIncomingVideoEnabled (Ljava/lang/Boolean;Ljava/util/List;)V + public static synthetic fun setIncomingVideoEnabled$default (Lio/getstream/video/android/core/Call;Ljava/lang/Boolean;Ljava/util/List;ILjava/lang/Object;)V + public final fun setPreferredIncomingVideoResolution (Lio/getstream/video/android/core/model/PreferredVideoResolution;Ljava/util/List;)V + public static synthetic fun setPreferredIncomingVideoResolution$default (Lio/getstream/video/android/core/Call;Lio/getstream/video/android/core/model/PreferredVideoResolution;Ljava/util/List;ILjava/lang/Object;)V public final fun setSessionId (Ljava/lang/String;)V public final fun setVideoFilter (Lio/getstream/video/android/core/call/video/VideoFilter;)V public final fun setVisibility (Ljava/lang/String;Lstream/video/sfu/models/TrackType;Z)V @@ -149,6 +153,7 @@ public final class io/getstream/video/android/core/CallState { public final fun getOwnCapabilities ()Lkotlinx/coroutines/flow/StateFlow; public final fun getParticipantBySessionId (Ljava/lang/String;)Lio/getstream/video/android/core/ParticipantState; public final fun getParticipantCounts ()Lkotlinx/coroutines/flow/StateFlow; + public final fun getParticipantVideoEnabledOverrides ()Lkotlinx/coroutines/flow/StateFlow; public final fun getParticipants ()Lkotlinx/coroutines/flow/StateFlow; public final fun getPermissionRequests ()Lkotlinx/coroutines/flow/StateFlow; public final fun getPinnedParticipants ()Lkotlinx/coroutines/flow/StateFlow; @@ -2857,6 +2862,10 @@ public final class io/getstream/video/android/core/call/stats/model/discriminato public final fun fromAlias (Ljava/lang/String;)Lio/getstream/video/android/core/call/stats/model/discriminator/RtcReportType; } +public final class io/getstream/video/android/core/call/utils/TrackOverridesHandlerKt { + public static final field ALL_PARTICIPANTS Ljava/lang/String; +} + public abstract class io/getstream/video/android/core/call/video/BitmapVideoFilter : io/getstream/video/android/core/call/video/VideoFilter { public fun ()V public abstract fun applyFilter (Landroid/graphics/Bitmap;)V @@ -3959,6 +3968,19 @@ public final class io/getstream/video/android/core/model/NetworkQuality$UnSpecif public fun toString ()Ljava/lang/String; } +public final class io/getstream/video/android/core/model/PreferredVideoResolution { + public fun (II)V + public final fun component1 ()I + public final fun component2 ()I + public final fun copy (II)Lio/getstream/video/android/core/model/PreferredVideoResolution; + public static synthetic fun copy$default (Lio/getstream/video/android/core/model/PreferredVideoResolution;IIILjava/lang/Object;)Lio/getstream/video/android/core/model/PreferredVideoResolution; + public fun equals (Ljava/lang/Object;)Z + public final fun getHeight ()I + public final fun getWidth ()I + public fun hashCode ()I + public fun toString ()Ljava/lang/String; +} + public final class io/getstream/video/android/core/model/QueriedCalls { public fun (Ljava/util/List;Ljava/lang/String;Ljava/lang/String;)V public final fun component1 ()Ljava/util/List; diff --git a/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/Call.kt b/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/Call.kt index 1a07f13a93..0838f4d4ba 100644 --- a/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/Call.kt +++ b/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/Call.kt @@ -38,6 +38,7 @@ import io.getstream.video.android.core.events.VideoEventListener import io.getstream.video.android.core.internal.InternalStreamVideoApi import io.getstream.video.android.core.internal.network.NetworkStateProvider import io.getstream.video.android.core.model.MuteUsersData +import io.getstream.video.android.core.model.PreferredVideoResolution import io.getstream.video.android.core.model.QueriedMembers import io.getstream.video.android.core.model.RejectReason import io.getstream.video.android.core.model.SortField @@ -1289,6 +1290,34 @@ public class Call( return clientImpl.listTranscription(type, id) } + /** + * Sets the preferred incoming video resolution. + * + * @param resolution The preferred resolution. Set to `null` to switch back to auto. + * @param sessionIds The participant session IDs to apply the resolution to. If `null`, the resolution will be applied to all participants. + */ + fun setPreferredIncomingVideoResolution( + resolution: PreferredVideoResolution?, + sessionIds: List? = null, + ) { + session?.let { session -> + session.trackOverridesHandler.updateOverrides( + sessionIds = sessionIds, + dimensions = resolution?.let { VideoDimension(it.width, it.height) }, + ) + } + } + + /** + * Enables/disables incoming video feed. + * + * @param enabled Whether the video feed should be enabled or disabled. Set to `null` to switch back to auto. + * @param sessionIds The participant session IDs to enable/disable the video feed for. If `null`, the setting will be applied to all participants. + */ + fun setIncomingVideoEnabled(enabled: Boolean?, sessionIds: List? = null) { + session?.trackOverridesHandler?.updateOverrides(sessionIds, visible = enabled) + } + @InternalStreamVideoApi public val debug = Debug(this) diff --git a/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/CallState.kt b/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/CallState.kt index 43ae1ac3f0..25e4325f7f 100644 --- a/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/CallState.kt +++ b/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/CallState.kt @@ -59,6 +59,7 @@ import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.SharingStarted import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.asStateFlow import kotlinx.coroutines.flow.channelFlow import kotlinx.coroutines.flow.collectLatest import kotlinx.coroutines.flow.combine @@ -559,10 +560,13 @@ public class CallState( internal val _reactions = MutableStateFlow>(emptyList()) val reactions: StateFlow> = _reactions - private val _errors: MutableStateFlow> = - MutableStateFlow(emptyList()) + private val _errors: MutableStateFlow> = MutableStateFlow(emptyList()) public val errors: StateFlow> = _errors + internal val _participantVideoEnabledOverrides = + MutableStateFlow>(emptyMap()) + val participantVideoEnabledOverrides = _participantVideoEnabledOverrides.asStateFlow() + private var speakingWhileMutedResetJob: Job? = null private var autoJoiningCall: Job? = null private var ringingTimerJob: Job? = null diff --git a/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/CallStats.kt b/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/CallStats.kt index ef66bfa5de..08c466816b 100644 --- a/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/CallStats.kt +++ b/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/CallStats.kt @@ -195,8 +195,8 @@ public class CallStats(val call: Call, val callScope: CoroutineScope) { val received = it.members["framesReceived"] as? Long val duration = it.members["totalFramesDuration"] as? Long if (participantId != null) { - logger.i { - "receiving video for $participantId at $frameWidth: ${it.members["frameWidth"]} and rendering it at ${visibleAt?.dimensions?.width} visible: ${visibleAt?.visible}" + logger.v { + "[stats] #manual-quality-selection; receiving video for $participantId at $frameWidth: ${it.members["frameWidth"]} and rendering it at ${visibleAt?.dimensions?.width} visible: ${visibleAt?.visible}" } } } diff --git a/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/call/RtcSession.kt b/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/call/RtcSession.kt index 637a1dd666..b84854b00f 100644 --- a/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/call/RtcSession.kt +++ b/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/call/RtcSession.kt @@ -44,6 +44,7 @@ import io.getstream.video.android.core.StreamVideo import io.getstream.video.android.core.StreamVideoClient import io.getstream.video.android.core.call.connection.StreamPeerConnection import io.getstream.video.android.core.call.stats.model.RtcStatsReport +import io.getstream.video.android.core.call.utils.TrackOverridesHandler import io.getstream.video.android.core.call.utils.stringify import io.getstream.video.android.core.dispatchers.DispatcherProvider import io.getstream.video.android.core.errors.RtcException @@ -83,7 +84,6 @@ import kotlinx.coroutines.Job import kotlinx.coroutines.SupervisorJob import kotlinx.coroutines.async import kotlinx.coroutines.awaitAll -import kotlinx.coroutines.cancel import kotlinx.coroutines.delay import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.StateFlow @@ -109,7 +109,6 @@ import org.webrtc.MediaConstraints import org.webrtc.MediaStream import org.webrtc.MediaStreamTrack import org.webrtc.PeerConnection -import org.webrtc.PeerConnection.PeerConnectionState import org.webrtc.RTCStatsReport import org.webrtc.RtpParameters.Encoding import org.webrtc.RtpTransceiver @@ -235,16 +234,12 @@ public class RtcSession internal constructor( null, ) - val trackDimensions = - MutableStateFlow>>( - emptyMap(), - ) - val trackDimensionsDebounced = trackDimensions.debounce(100) - // run all calls on a supervisor job so we can easily cancel them private val supervisorJob = SupervisorJob() private val coroutineScope = CoroutineScope(clientImpl.scope.coroutineContext + supervisorJob) + internal val defaultVideoDimension = VideoDimension(1080, 2340) + // participants by session id -> participant state private val trackPrefixToSessionIdMap = call.state.participants.mapState { it.associate { it.trackLookupPrefix to it.sessionId } } @@ -252,6 +247,17 @@ public class RtcSession internal constructor( // We need to update tracks for all participants // It's cleaner to store here and have the participant state reference to it var tracks: MutableMap> = mutableMapOf() + val trackDimensions = MutableStateFlow>>( + emptyMap(), + ) + val trackDimensionsDebounced = trackDimensions.debounce(100) + internal val trackOverridesHandler = TrackOverridesHandler( + onOverridesUpdate = { + setVideoSubscriptions() + call.state._participantVideoEnabledOverrides.value = it.mapValues { it.value.visible } + }, + logger = logger, + ) private fun getTrack(sessionId: String, type: TrackType): MediaTrack? { if (!tracks.containsKey(sessionId)) { @@ -992,8 +998,6 @@ public class RtcSession internal constructor( } } - private val defaultVideoDimension = VideoDimension(1080, 2340) - /** * This is called when you are look at a different set of participants * or at a different size @@ -1061,68 +1065,55 @@ public class RtcSession internal constructor( * - it sends the resolutions we're displaying the video at so the SFU can decide which track to send * - when switching SFU we should repeat this info * - http calls failing here breaks the call. (since you won't receive the video) - * - we should retry continously until it works and after it continues to fail, raise an error that shuts down the call + * - we should retry continuously until it works and after it continues to fail, raise an error that shuts down the call * - we retry when: * -- error isn't permanent, SFU didn't change, the mute/publish state didn't change * -- we cap at 30 retries to prevent endless loops */ - private fun setVideoSubscriptions(useDefaults: Boolean = false) { + internal fun setVideoSubscriptions(useDefaults: Boolean = false) { logger.d { "[setVideoSubscriptions] #sfu; #track; useDefaults: $useDefaults" } - // default is to subscribe to the top 5 sorted participants var tracks = if (useDefaults) { + // default is to subscribe to the top 5 sorted participants defaultTracks() } else { // if we're not using the default, sub to visible tracks visibleTracks() - } + }.let(trackOverridesHandler::applyOverrides) - // TODO: - // This is a hotfix to help with performance. Most devices struggle even with H resolution - // if there are more than 2 remote participants and especially if there is a H264 participant. - // We just report a very small window so force SFU to deliver us Q resolution. This will - // be later less visible to the user once we make the participant grid smaller - if (tracks.size > 2) { - tracks = tracks.map { - it.copy(dimension = it.dimension?.copy(width = 200, height = 200)) - } - } - logger.v { "[setVideoSubscriptions] #sfu; #track; tracks.size: ${tracks.size}" } - - val new = tracks.toList() - subscriptions.value = new + subscriptions.value = tracks val currentSfu = sfuUrl subscriptionSyncJob?.cancel() - - if (new.isNotEmpty()) { - // start a new job - // this code is a bit more complicated due to the retry behaviour - subscriptionSyncJob = coroutineScope.launch { - flow { - val request = UpdateSubscriptionsRequest( - session_id = sessionId, - tracks = subscriptions.value, - ) - println("request $request") - val sessionToDimension = tracks.map { it.session_id to it.dimension } - dynascaleLogger.v { - "[setVideoSubscriptions] $useDefaults #sfu; #track; $sessionId subscribing to : $sessionToDimension" - } - val result = updateSubscriptions(request) - emit(result.getOrThrow()) - }.flowOn(DispatcherProvider.IO).retryWhen { cause, attempt -> - val sameValue = new == subscriptions.value - val sameSfu = currentSfu == sfuUrl - val isPermanent = isPermanentError(cause) - val willRetry = !isPermanent && sameValue && sameSfu && attempt < 30 - val delayInMs = if (attempt <= 1) 100L else if (attempt <= 3) 300L else 2500L - logger.w { - "updating subscriptions failed with error $cause, retry attempt: $attempt. will retry $willRetry in $delayInMs ms" - } - delay(delayInMs) - willRetry - }.collect() - } + subscriptionSyncJob = coroutineScope.launch { + flow { + val request = UpdateSubscriptionsRequest( + session_id = sessionId, + tracks = subscriptions.value, + ) + dynascaleLogger.d { + "[setVideoSubscriptions] #sfu; #track; #manual-quality-selection; UpdateSubscriptionsRequest: $request" + } + val sessionToDimension = tracks.map { it.session_id to it.dimension } + dynascaleLogger.v { + "[setVideoSubscriptions] #sfu; #track; #manual-quality-selection; Subscribing to: $sessionToDimension" + } + val result = updateSubscriptions(request) + dynascaleLogger.v { + "[setVideoSubscriptions] #sfu; #track; #manual-quality-selection; Result: $result" + } + emit(result.getOrThrow()) + }.flowOn(DispatcherProvider.IO).retryWhen { cause, attempt -> + val sameValue = tracks == subscriptions.value + val sameSfu = currentSfu == sfuUrl + val isPermanent = isPermanentError(cause) + val willRetry = !isPermanent && sameValue && sameSfu && attempt < 30 + val delayInMs = if (attempt <= 1) 100L else if (attempt <= 3) 300L else 2500L + logger.w { + "updating subscriptions failed with error $cause, retry attempt: $attempt. will retry $willRetry in $delayInMs ms" + } + delay(delayInMs) + willRetry + }.collect() } } @@ -1807,7 +1798,7 @@ public class RtcSession internal constructor( dimensions: VideoDimension = defaultVideoDimension, ) { logger.v { - "[updateTrackDimensions] #track; #sfu; sessionId: $sessionId, trackType: $trackType, visible: $visible, dimensions: $dimensions" + "[updateTrackDimensions] #track; #sfu; #manual-quality-selection; sessionId: $sessionId, trackType: $trackType, visible: $visible, dimensions: $dimensions" } // The map contains all track dimensions for all participants dynascaleLogger.d { "updating dimensions $sessionId $visible $dimensions" } diff --git a/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/call/utils/TrackOverridesHandler.kt b/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/call/utils/TrackOverridesHandler.kt new file mode 100644 index 0000000000..9bdd20ffd0 --- /dev/null +++ b/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/call/utils/TrackOverridesHandler.kt @@ -0,0 +1,153 @@ +/* + * Copyright (c) 2014-2024 Stream.io Inc. All rights reserved. + * + * Licensed under the Stream License; + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://github.com/GetStream/stream-video-android/blob/main/LICENSE + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.getstream.video.android.core.call.utils + +import io.getstream.log.TaggedLogger +import stream.video.sfu.models.VideoDimension +import stream.video.sfu.signal.TrackSubscriptionDetails + +/** + * Handles incoming video track overrides (resolution and visibility). + * + * @param onOverridesUpdate Lambda used to notify the caller when the overrides are updated. + * @param logger Logger to be used. + */ +internal class TrackOverridesHandler( + private val onOverridesUpdate: (overrides: Map) -> Unit, + private val logger: TaggedLogger? = null, +) { + + private val trackOverrides: MutableMap = mutableMapOf() + + data class TrackOverride( + val dimensions: VideoDimension? = null, + val visible: Boolean? = null, + ) + + /** + * Updates incoming video dimensions overrides. + * + * @param sessionIds List of session IDs to update. If `null`, the override will be applied to all participants. + * @param dimensions Video dimensions to set. Set to `null` to switch back to auto. + */ + fun updateOverrides( + sessionIds: List? = null, + dimensions: VideoDimension? = null, + ) { + val putDimensions = dimensions + + if (sessionIds == null) { + putOrRemoveDimensionsOverride( + sessionId = ALL_PARTICIPANTS, + putDimensions = putDimensions, + existingVisibility = trackOverrides[ALL_PARTICIPANTS]?.visible, + ) + } else { + sessionIds.forEach { sessionId -> + putOrRemoveDimensionsOverride( + sessionId = sessionId, + putDimensions = putDimensions, + existingVisibility = trackOverrides[sessionId]?.visible, + ) + } + } + + onOverridesUpdate(trackOverrides) + + logger?.d { "[updateOverrides] #manual-quality-selection; Overrides: $trackOverrides" } + } + + private fun putOrRemoveDimensionsOverride(sessionId: String, putDimensions: VideoDimension?, existingVisibility: Boolean?) { + if (putDimensions == null && (existingVisibility == true || existingVisibility == null)) { + trackOverrides.remove(sessionId) + } else { + trackOverrides.put( + sessionId, + TrackOverride(dimensions = putDimensions, visible = existingVisibility), + ) + } + } + + /** + * Updates incoming video visibility overrides. + * + * @param sessionIds List of session IDs to update. If `null`, the override will be applied to all participants. + * @param visible Video visibility to set. Set to `null` to switch back to auto. + */ + fun updateOverrides( + sessionIds: List? = null, + visible: Boolean? = null, + ) { + val putVisibility = visible + + if (sessionIds == null) { + putOrRemoveVisibilityOverride( + sessionId = ALL_PARTICIPANTS, + putVisibility = putVisibility, + existingDimensions = trackOverrides[ALL_PARTICIPANTS]?.dimensions, + ) + } else { + sessionIds.forEach { sessionId -> + putOrRemoveVisibilityOverride( + sessionId = sessionId, + putVisibility = putVisibility, + existingDimensions = trackOverrides[sessionId]?.dimensions, + ) + } + } + + onOverridesUpdate(trackOverrides) + + logger?.d { "[updateOverrides] #manual-quality-selection; Overrides: $trackOverrides" } + } + + private fun putOrRemoveVisibilityOverride(sessionId: String, putVisibility: Boolean?, existingDimensions: VideoDimension?) { + if (existingDimensions == null && (putVisibility == true || putVisibility == null)) { + trackOverrides.remove(sessionId) + } else { + trackOverrides.put( + sessionId, + TrackOverride(dimensions = existingDimensions, visible = putVisibility), + ) + } + } + + /** + * Applies overrides to given list of tracks. + * + * @return List of tracks with visibility and dimensions overrides applied. + */ + fun applyOverrides( + tracks: List, + ): List { + return tracks.mapNotNull { track -> + val override = trackOverrides[track.session_id] ?: trackOverrides[ALL_PARTICIPANTS] + + if (override == null) { + track + } else { + if (override.visible == false) { + null + } else { + override.dimensions?.let { track.copy(dimension = it) } ?: track + } + } + } + } +} + +const val ALL_PARTICIPANTS = "all" diff --git a/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/model/PreferredVideoResolution.kt b/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/model/PreferredVideoResolution.kt new file mode 100644 index 0000000000..d2dfb1c65e --- /dev/null +++ b/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/model/PreferredVideoResolution.kt @@ -0,0 +1,22 @@ +/* + * Copyright (c) 2014-2024 Stream.io Inc. All rights reserved. + * + * Licensed under the Stream License; + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://github.com/GetStream/stream-video-android/blob/main/LICENSE + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.getstream.video.android.core.model + +data class PreferredVideoResolution( + val width: Int, + val height: Int, +) diff --git a/stream-video-android-ui-compose/src/main/kotlin/io/getstream/video/android/compose/ui/components/video/VideoRenderer.kt b/stream-video-android-ui-compose/src/main/kotlin/io/getstream/video/android/compose/ui/components/video/VideoRenderer.kt index 62fa843d8b..1abe4828e4 100644 --- a/stream-video-android-ui-compose/src/main/kotlin/io/getstream/video/android/compose/ui/components/video/VideoRenderer.kt +++ b/stream-video-android-ui-compose/src/main/kotlin/io/getstream/video/android/compose/ui/components/video/VideoRenderer.kt @@ -45,6 +45,7 @@ import androidx.compose.ui.tooling.preview.Preview import androidx.compose.ui.unit.dp import androidx.compose.ui.unit.sp import androidx.compose.ui.viewinterop.AndroidView +import androidx.lifecycle.compose.collectAsStateWithLifecycle import io.getstream.log.StreamLog import io.getstream.video.android.compose.theme.VideoTheme import io.getstream.video.android.compose.ui.components.video.VideoScalingType.Companion.toCommonScalingType @@ -52,6 +53,7 @@ import io.getstream.video.android.compose.ui.components.video.config.VideoRender import io.getstream.video.android.compose.ui.components.video.config.videoRenderConfig import io.getstream.video.android.core.Call import io.getstream.video.android.core.ParticipantState +import io.getstream.video.android.core.call.utils.ALL_PARTICIPANTS import io.getstream.video.android.core.model.MediaTrack import io.getstream.video.android.core.model.VideoTrack import io.getstream.video.android.mock.StreamPreviewDataUtils @@ -90,60 +92,64 @@ public fun VideoRenderer( videoRendererConfig.fallbackContent.invoke(call) if (video?.enabled == true) { - val mediaTrack = video.track val sessionId = video.sessionId - val trackType = video.type + val videoEnabledOverrides by call.state.participantVideoEnabledOverrides.collectAsStateWithLifecycle() - var view: VideoTextureViewRenderer? by remember { mutableStateOf(null) } + if (isIncomingVideoEnabled(call, sessionId, videoEnabledOverrides)) { + val mediaTrack = video.track + val trackType = video.type - DisposableEffect(call, video) { - // inform the call that we want to render this video track. (this will trigger a subscription to the track) - call.setVisibility(sessionId, trackType, true) + var view: VideoTextureViewRenderer? by remember { mutableStateOf(null) } - onDispose { - cleanTrack(view, mediaTrack) - // inform the call that we no longer want to render this video track - call.setVisibility(sessionId, trackType, false) + DisposableEffect(call, video) { + // inform the call that we want to render this video track. (this will trigger a subscription to the track) + call.setVisibility(sessionId, trackType, true) + + onDispose { + cleanTrack(view, mediaTrack) + // inform the call that we no longer want to render this video track + call.setVisibility(sessionId, trackType, false) + } } - } - if (mediaTrack != null) { - Box( - modifier = videoRendererConfig.modifiers.containerModifier.invoke(this), - contentAlignment = Alignment.Center, - ) { - AndroidView( - factory = { context -> - StreamVideoTextureViewRenderer(context).apply { - call.initRenderer( - videoRenderer = this, - sessionId = sessionId, - trackType = trackType, - onRendered = onRendered, - ) - setMirror(videoRendererConfig.mirrorStream) - setScalingType( + if (mediaTrack != null) { + Box( + modifier = videoRendererConfig.modifiers.containerModifier.invoke(this), + contentAlignment = Alignment.Center, + ) { + AndroidView( + factory = { context -> + StreamVideoTextureViewRenderer(context).apply { + call.initRenderer( + videoRenderer = this, + sessionId = sessionId, + trackType = trackType, + onRendered = onRendered, + ) + setMirror(videoRendererConfig.mirrorStream) + setScalingType( + videoRendererConfig.scalingType.toCommonScalingType(), + ) + setupVideo(mediaTrack, this) + + view = this + } + }, + update = { v -> + v.setMirror(videoRendererConfig.mirrorStream) + v.setScalingType( videoRendererConfig.scalingType.toCommonScalingType(), ) - setupVideo(mediaTrack, this) - - view = this - } - }, - update = { v -> - v.setMirror(videoRendererConfig.mirrorStream) - v.setScalingType( - videoRendererConfig.scalingType.toCommonScalingType(), - ) - setupVideo(mediaTrack, v) - }, - modifier = videoRendererConfig - .modifiers - .componentModifier( - this, - ) - .testTag("video_renderer"), - ) + setupVideo(mediaTrack, v) + }, + modifier = videoRendererConfig + .modifiers + .componentModifier( + this, + ) + .testTag("video_renderer"), + ) + } } } } @@ -187,6 +193,10 @@ public fun VideoRenderer( ) } +private fun isIncomingVideoEnabled(call: Call, sessionId: String, videoEnabledOverrides: Map) = + (videoEnabledOverrides[sessionId] ?: videoEnabledOverrides[ALL_PARTICIPANTS]) != false || + call.state.me.value?.sessionId == sessionId + private fun cleanTrack( view: VideoTextureViewRenderer?, mediaTrack: MediaTrack?,