Merge pull request #17 from 100mslive/dev

using v0.9.15 of sdk - audio detection impl

app/build.gradle

@@ -15,8 +15,8 @@ android {
         applicationId "live.hms.android100ms"
         minSdkVersion 24
         targetSdkVersion 30
-        versionCode 16
-        versionName "1.0.14"
+        versionCode 20
+        versionName "1.0.18"
 
         testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner"
     }
@@ -58,7 +58,7 @@ dependencies {
 
     //100ms.live SDK
     implementation 'org.webrtc:google-webrtc:1.0.32006'
-    implementation 'com.github.100mslive:android-sdk:0.9.14.1'
+    implementation 'com.github.100mslive:android-sdk:0.9.15'
 
     // Navigation
     implementation "androidx.navigation:navigation-fragment-ktx:2.3.3"

app/src/main/java/live/hms/android100ms/ui/home/settings/SettingsFragment.kt

@@ -32,7 +32,7 @@ class SettingsFragment : Fragment() {
       "90p" to "90 x 90"
     )
 
-    private val CODECS = arrayOf("VP8")
+    private val CODECS = arrayOf("VP8", "H264")
 
     private val VIDEO_BITRATE = mapOf(
       "Lowest (100 kbps)" to 100,
@@ -130,6 +130,59 @@ class SettingsFragment : Fragment() {
     }
   }
 
+  private fun initNonEmptyEditTextWithRange(
+    defaultText: Long,
+    editText: TextInputEditText,
+    container: TextInputLayout,
+    name: String,
+    minValue: Long, maxValue: Long,
+    saveOnValid: (value: Long) -> Unit
+  ) {
+    editText.apply {
+      setText(defaultText.toString())
+      addTextChangedListener { text ->
+        if (text.toString().isEmpty()) {
+          container.error = "$name cannot be empty"
+        } else {
+          val value = text.toString().toLong()
+          if (value < minValue || value > maxValue) {
+            container.error = "$name value should be in range [$minValue, $maxValue] inclusive"
+          } else {
+            container.error = null
+            saveOnValid(value)
+          }
+        }
+      }
+    }
+  }
+
+
+  private fun initNonEmptyEditTextWithRange(
+    defaultText: Float,
+    editText: TextInputEditText,
+    container: TextInputLayout,
+    name: String,
+    minValue: Float, maxValue: Float,
+    saveOnValid: (value: Float) -> Unit
+  ) {
+    editText.apply {
+      setText(defaultText.toString())
+      addTextChangedListener { text ->
+        if (text.toString().isEmpty()) {
+          container.error = "$name cannot be empty"
+        } else {
+          val value = text.toString().toFloat()
+          if (value < minValue || value > maxValue) {
+            container.error = "$name value should be in range [$minValue, $maxValue] inclusive"
+          } else {
+            container.error = null
+            saveOnValid(value)
+          }
+        }
+      }
+    }
+  }
+
   private fun initEditTexts() {
     binding.apply {
       initNonEmptyEditText(
@@ -159,6 +212,21 @@ class SettingsFragment : Fragment() {
         1, 3,
       ) { commitHelper.setVideoGridColumns(it) }
 
+      initNonEmptyEditTextWithRange(
+        settings.audioPollInterval,
+        editTextAudioPollInterval, containerAudioPollInterval,
+        "Audio Poll Interval",
+        100, 10000
+      ) { commitHelper.setAudioPollInterval(it) }
+
+      initNonEmptyEditTextWithRange(
+        settings.silenceAudioLevelThreshold,
+        editTextSilenceAudioLevelThreshold,
+        containerSilenceAudioLevelThreshold,
+        "Silence Audio Level Threshold",
+        0.0f, 5.0f
+      ) { commitHelper.setSilenceAudioLevelThreshold(it) }
+
       initNonEmptyEditTextWithRange(
         settings.videoFrameRate,
         editTextVideoFramerate, containerVideoFramerate,
@@ -294,6 +362,11 @@ class SettingsFragment : Fragment() {
         switchToggleLeakCanary
       ) { handleLeakCanaryToggle(it) }
 
+      initSwitch(
+        settings.detectDominantSpeaker,
+        switchShowDominantSpeaker
+      ) { commitHelper.setDetectDominantSpeaker(it) }
+
       // Disable the switches not yet supported (TODO: Add support)
       switchMirrorVideo.isEnabled = false
       switchShowPreviewBeforeJoin.isEnabled = false

app/src/main/java/live/hms/android100ms/ui/home/settings/SettingsStore.kt

@@ -19,6 +19,10 @@ class SettingsStore(context: Context) {
     const val VIDEO_FRAME_RATE = "video-frame-rate"
     const val USERNAME = "username"
 
+    const val DETECT_DOMINANT_SPEAKER = "detect-dominant-speaker"
+    const val AUDIO_POLL_INTERVAL = "audio-poll-interval"
+    const val SILENCE_AUDIO_LEVEL_THRESHOLD = "silence-audio-level-threshold"
+
     const val LAST_USED_ROOM_ID = "last-used-room-id"
     const val ENVIRONMENT = "last-used-env"
 
@@ -40,6 +44,13 @@ class SettingsStore(context: Context) {
     }
   }
 
+  private fun putLong(key: String, value: Long) {
+    sharedPreferences.edit {
+      putLong(key, value)
+      commit()
+    }
+  }
+
   private fun putInt(key: String, value: Int) {
     sharedPreferences.edit {
       putInt(key, value)
@@ -54,6 +65,13 @@ class SettingsStore(context: Context) {
     }
   }
 
+  private fun putFloat(key: String, value: Float) {
+    sharedPreferences.edit {
+      putFloat(key, value)
+      commit()
+    }
+  }
+
   var publishVideo: Boolean
     get() = sharedPreferences.getBoolean(PUBLISH_VIDEO, true)
     set(value) = putBoolean(PUBLISH_VIDEO, value)
@@ -91,6 +109,17 @@ class SettingsStore(context: Context) {
     get() = sharedPreferences.getString(USERNAME, "Android User")!!
     set(value) = putString(USERNAME, value)
 
+  var detectDominantSpeaker: Boolean
+    get() = sharedPreferences.getBoolean(DETECT_DOMINANT_SPEAKER, true)
+    set(value) = putBoolean(DETECT_DOMINANT_SPEAKER, value)
+
+  var audioPollInterval: Long
+    get() = sharedPreferences.getLong(AUDIO_POLL_INTERVAL, 1000)
+    set(value) = putLong(AUDIO_POLL_INTERVAL, value)
+
+  var silenceAudioLevelThreshold: Float
+    get() = sharedPreferences.getFloat(SILENCE_AUDIO_LEVEL_THRESHOLD, 0.01f)
+    set(value) = putFloat(SILENCE_AUDIO_LEVEL_THRESHOLD, value)
 
   var lastUsedRoomId: String
     get() = sharedPreferences.getString(LAST_USED_ROOM_ID, "")!!
@@ -162,6 +191,21 @@ class SettingsStore(context: Context) {
       return this
     }
 
+    fun setDetectDominantSpeaker(value: Boolean): MultiCommitHelper {
+      editor.putBoolean(DETECT_DOMINANT_SPEAKER, value)
+      return this
+    }
+
+    fun setAudioPollInterval(value: Long): MultiCommitHelper {
+      editor.putLong(AUDIO_POLL_INTERVAL, value)
+      return this
+    }
+
+    fun setSilenceAudioLevelThreshold(value: Float): MultiCommitHelper {
+      editor.putFloat(SILENCE_AUDIO_LEVEL_THRESHOLD, value)
+      return this
+    }
+
     fun setLastUsedRoomId(value: String): MultiCommitHelper {
       editor.putString(LAST_USED_ROOM_ID, value)
       return this

app/src/main/java/live/hms/android100ms/ui/meeting/MeetingFragment.kt

@@ -1,6 +1,7 @@
 package live.hms.android100ms.ui.meeting
 
 import android.content.Intent
+import android.media.MediaCodecList
 import android.os.Bundle
 import android.util.Log
 import android.view.*
@@ -249,7 +250,7 @@ class MeetingFragment : Fragment() {
     meetingViewModel.broadcastsReceived.observe(viewLifecycleOwner) { data ->
       chatViewModel.receivedMessage(
         ChatMessage(
-          data.peer.uid,
+          data.peer.customerUserId,
           data.senderName,
           Date(),
           data.msg,

app/src/main/java/live/hms/android100ms/ui/meeting/MeetingTrack.kt

@@ -15,7 +15,7 @@ data class MeetingTrack(
 
   override fun equals(other: Any?): Boolean {
     if (other is MeetingTrack) {
-      return other.peer.uid == peer.uid &&
+      return other.peer.peerId == peer.peerId &&
           other.mediaId == mediaId
     }
 

app/src/main/java/live/hms/android100ms/ui/meeting/MeetingViewModel.kt

@@ -17,7 +17,6 @@ import live.hms.video.payload.HMSStreamInfo
 import live.hms.video.webrtc.HMSRTCAudioTrack
 import live.hms.video.webrtc.HMSRTCMediaStreamConstraints
 import live.hms.video.webrtc.HMSRTCVideoTrack
-import java.lang.Exception
 import java.util.*
 import kotlin.collections.ArrayList
 
@@ -65,6 +64,9 @@ class MeetingViewModel(
   // Live data to notify about broadcast data
   val broadcastsReceived = MutableLiveData<HMSPayloadData>()
 
+  // Dominant speaker
+  val dominantSpeaker = MutableLiveData<MeetingTrack?>(null)
+
   val peer = HMSPeer(roomDetails.username, roomDetails.authToken).apply {
     crashlytics.setUserId(customerUserId)
   }
@@ -194,7 +196,7 @@ class MeetingViewModel(
   private fun removeTrack(uid: String, mid: String) {
     synchronized(_tracks) {
       val trackToRemove = _tracks.find {
-        it.peer.uid == uid && it.mediaId == mid
+        it.peer.peerId == uid && it.mediaId == mid
       }
       _tracks.remove(trackToRemove)
 
@@ -235,16 +237,18 @@ class MeetingViewModel(
           crashlyticsLog(TAG, "Publish Success ${data.mid}")
 
           currentDeviceTrack = MeetingTrack(
-            data.mid,
+            localStream.streamId,
             peer,
             videoTrack,
             audioTrack,
             true
           ).apply {
-            state.postValue(MeetingState.Ongoing())
-
             addTrack(this)
             Log.v(TAG, "Adding user track $currentDeviceTrack to VideoGrid")
+
+            if (settings.detectDominantSpeaker) startDetectDominantSpeakerMonitor()
+
+            state.postValue(MeetingState.Ongoing())
           }
         }
 
@@ -330,6 +334,10 @@ class MeetingViewModel(
    * Resets all the values to default
    */
   private fun cleanup() {
+    /** NOTE: Calling [HMSClient.disconnect] stop's the audio-level monitor
+     * However, we can manually stop it using [HMSClient.stopAudioLevelMonitor]
+     */
+
     // NOTE: Make sure that we have stopped capturing whenever we disconnect/leave/handle failures
     if (settings.publishVideo) {
       try {
@@ -381,7 +389,7 @@ class MeetingViewModel(
   override fun onPeerJoin(peer: HMSPeer) {
     crashlyticsLog(
       TAG,
-      "onPeerJoin: uid=${peer.uid}, " +
+      "onPeerJoin: peerId=${peer.peerId}, " +
           "role=${peer.role}, " +
           "userId=${peer.customerUserId}, " +
           "peerId=${peer.peerId}"
@@ -391,7 +399,7 @@ class MeetingViewModel(
   override fun onPeerLeave(peer: HMSPeer) {
     crashlyticsLog(
       TAG,
-      "onPeerLeave: uid=${peer.uid}, " +
+      "onPeerLeave: peerId=${peer.peerId}, " +
           "role=${peer.role}, " +
           "userId=${peer.customerUserId}, " +
           "peerId=${peer.peerId}"
@@ -401,23 +409,23 @@ class MeetingViewModel(
   override fun onStreamAdd(peer: HMSPeer, info: HMSStreamInfo) {
     crashlyticsLog(
       TAG,
-      "onStreamAdd: peer-uid:${peer.uid} " +
+      "onStreamAdd: peerId:${peer.peerId} " +
           "name=${peer.userName}, " +
           "role=${peer.role} " +
           "userId=${peer.customerUserId} " +
           "mid=${info.mid} " +
-          "uid=${info.uid}"
+          "peerId=${info.peer.peerId}"
     )
 
     client.subscribe(info, room, object : HMSMediaRequestHandler {
       override fun onSuccess(stream: HMSRTCMediaStream) {
         crashlyticsLog(
           TAG,
           "Subscribe(" +
-              "uid=${info.uid}, " +
+              "uid=${info.peer.peerId}, " +
               "mid=${info.mid}, " +
               "userName=${info.userName}): " +
-              "peer-id=${peer.uid} -- onSuccess($stream)"
+              "peer-id=${peer.peerId} -- onSuccess($stream)"
         )
 
         var videoTrack: HMSRTCVideoTrack? = null
@@ -444,7 +452,7 @@ class MeetingViewModel(
       override fun onFailure(exception: HMSException) {
         crashlyticsLog(
           TAG,
-          "Subscribe($info): peer-id=${peer.uid} -- onFailure(${toString(exception)})"
+          "Subscribe($info): peer-id=${peer.peerId} -- onFailure(${toString(exception)})"
         )
         handleFailure(exception)
       }
@@ -456,21 +464,39 @@ class MeetingViewModel(
       TAG,
       "onStreamRemove: " +
           "name=${info.userName} " +
-          "uid=${info.uid} " +
+          "peerId=${info.peer.peerId} " +
           "mid=${info.mid}"
     )
 
-    removeTrack(info.uid, info.mid)
+    removeTrack(info.peer.peerId, info.mid)
   }
 
   override fun onBroadcast(data: HMSPayloadData) {
     crashlyticsLog(
       TAG,
-      "onBroadcast: customerId=${data.peer.customerUserId}, " +
+      "onBroadcast: peerId=${data.peer.peerId}, " +
           "userName=${data.peer.userName}, " +
           "msg=${data.msg}"
     )
 
     broadcastsReceived.postValue(data)
   }
+
+  private fun startDetectDominantSpeakerMonitor() {
+    client.startAudioLevelMonitor(settings.audioPollInterval) { audioInfo ->
+      Log.d(TAG, "startAudioLevelMonitor: $audioInfo")
+      // The audioInfo is sorted in descending order of audio-levels
+      // 1st item is dominant speaker
+      if (audioInfo.isNotEmpty()) {
+        if (audioInfo[0].audioLevel > settings.silenceAudioLevelThreshold) {
+          synchronized(_tracks) {
+            _tracks.find { it.mediaId == audioInfo[0].streamId }?.let {
+              dominantSpeaker.postValue(it)
+            }
+          }
+        } else dominantSpeaker.postValue(null)
+      }
+    }
+  }
+
 }