App/Vision: Add a vision example that the exploits MLAgent Service

This patch adds a draft of the vision example (i.e., an object classification scenario using MobileNet_v1) that exploits the MLAgent Service. Signed-off-by: Wook Song <[email protected]>
nnstreamer · Oct 4, 2024 · b257959 · b257959
1 parent 6508e86
commit b257959
Show file tree

Hide file tree

Showing 10 changed files with 341 additions and 11 deletions.
diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml
@@ -29,6 +29,8 @@ tensorflowLite = "2.16.1"
 tukaani-xz-plugin = "1.9"
 uiTestJunit4Android = "1.6.8"
 xyz-simple-git-plugin = "2.0.3"
+cameraCore = "1.3.4"
+firebaseCrashlyticsBuildtools = "3.0.2"
 
 [libraries]
 commons-compress = { group = "org.apache.commons", name = "commons-compress", version.ref = "commons-compress-lib" }
@@ -43,6 +45,12 @@ robolectric = { module = "org.robolectric:robolectric", version.ref = "robolectr
 tukaani-xz = { group = "org.tukaani", name = "xz", version.ref = "tukaani-xz-plugin" }
 androidx-activity-compose = { group = "androidx.activity", name = "activity-compose", version.ref = "activityCompose" }
 androidx-appcompat = { group = "androidx.appcompat", name = "appcompat", version.ref = "appcompat" }
+androidx-camera-camera2 = { module = "androidx.camera:camera-camera2", version.ref = "cameraCore" }
+androidx-camera-core = { module = "androidx.camera:camera-core", version.ref = "cameraCore" }
+androidx-camera-extensions = { module = "androidx.camera:camera-extensions", version.ref = "cameraCore" }
+androidx-camera-lifecycle = { module = "androidx.camera:camera-lifecycle", version.ref = "cameraCore" }
+androidx-camera-video = { module = "androidx.camera:camera-video", version.ref = "cameraCore" }
+androidx-camera-view = { module = "androidx.camera:camera-view", version.ref = "cameraCore" }
 androidx-compose-bom = { group = "androidx.compose", name = "compose-bom", version.ref = "composeBom" }
 androidx-core-ktx = { group = "androidx.core", name = "core-ktx", version.ref = "coreKtx" }
 androidx-datastore-core-android = { group = "androidx.datastore", name = "datastore-core-android", version.ref = "datastore" }
@@ -68,6 +76,7 @@ androidx-ui-test-manifest = { group = "androidx.compose.ui", name = "ui-test-man
 androidx-ui-test-junit4 = { group = "androidx.compose.ui", name = "ui-test-junit4" }
 androidx-ui-test-junit4-android = { group = "androidx.compose.ui", name = "ui-test-junit4-android", version.ref = "uiTestJunit4Android" }
 kotlinx-serialization-json = { module = "org.jetbrains.kotlinx:kotlinx-serialization-json", version.ref = "kotlinxSerializationJson" }
+firebase-crashlytics-buildtools = { group = "com.google.firebase", name = "firebase-crashlytics-buildtools", version.ref = "firebaseCrashlyticsBuildtools" }
 
 [plugins]
 androidApplication = { id = "com.android.application", version.ref = "agp" }

diff --git a/ml_inference_offloading/build.gradle.kts b/ml_inference_offloading/build.gradle.kts
@@ -87,6 +87,7 @@ dependencies {
     implementation(libs.androidx.core.ktx)
     implementation(libs.androidx.lifecycle.runtime.compose)
     implementation(libs.androidx.lifecycle.runtime.ktx)
+    implementation(libs.androidx.lifecycle.runtime.compose)
     implementation(libs.androidx.material.icons.core)
     implementation(libs.androidx.material.icons.extended)
     implementation(libs.androidx.material3)
@@ -99,6 +100,17 @@ dependencies {
     // Room
     implementation(libs.androidx.room.common)
     implementation(libs.androidx.room.runtime)
+
+    // Camera
+    implementation(libs.androidx.camera.view)
+    implementation(libs.androidx.camera.core)
+    implementation(libs.androidx.camera.camera2)
+    implementation(libs.androidx.camera.lifecycle)
+    implementation(libs.androidx.camera.video)
+    implementation(libs.androidx.camera.view)
+    implementation(libs.androidx.camera.extensions)
+    implementation(libs.firebase.crashlytics.buildtools)
+
     ksp(libs.androidx.room.compiler)
     implementation(libs.androidx.room.ktx)
 

diff --git a/ml_inference_offloading/src/main/AndroidManifest.xml b/ml_inference_offloading/src/main/AndroidManifest.xml
@@ -1,6 +1,11 @@
 <?xml version="1.0" encoding="utf-8"?>
 <manifest xmlns:android="http://schemas.android.com/apk/res/android">
 
+    <uses-feature
+            android:name="android.hardware.camera"
+            android:required="false" />
+
+    <uses-permission android:name="android.permission.CAMERA" />
     <uses-permission android:name="android.permission.FOREGROUND_SERVICE" />
     <uses-permission android:name="android.permission.FOREGROUND_SERVICE_SPECIAL_USE" />
     <uses-permission android:name="android.permission.POST_NOTIFICATIONS" />

diff --git a/ml_inference_offloading/src/main/java/ai/nnstreamer/ml/inference/offloading/MainActivity.kt b/ml_inference_offloading/src/main/java/ai/nnstreamer/ml/inference/offloading/MainActivity.kt
@@ -1,25 +1,35 @@
 package ai.nnstreamer.ml.inference.offloading
 
+import ai.nnstreamer.ml.inference.offloading.data.Classification
+import ai.nnstreamer.ml.inference.offloading.data.ImageAnalyzer
+import ai.nnstreamer.ml.inference.offloading.domain.MobilenetClassifier
 import ai.nnstreamer.ml.inference.offloading.ui.MainViewModel
 import ai.nnstreamer.ml.inference.offloading.ui.components.ButtonList
 import ai.nnstreamer.ml.inference.offloading.ui.components.ServiceList
 import ai.nnstreamer.ml.inference.offloading.ui.theme.NnstreamerandroidTheme
+import android.Manifest
 import android.annotation.SuppressLint
 import android.content.ComponentName
 import android.content.Context
 import android.content.Intent
 import android.content.ServiceConnection
+import android.content.pm.PackageManager
 import android.os.Bundle
 import android.os.IBinder
 import android.os.Message
 import android.os.Messenger
 import androidx.activity.ComponentActivity
 import androidx.activity.compose.setContent
+import androidx.camera.view.CameraController
+import androidx.camera.view.LifecycleCameraController
+import androidx.camera.view.PreviewView
+import androidx.compose.foundation.background
 import androidx.compose.foundation.layout.Arrangement
 import androidx.compose.foundation.layout.Box
 import androidx.compose.foundation.layout.Column
 import androidx.compose.foundation.layout.Spacer
 import androidx.compose.foundation.layout.fillMaxSize
+import androidx.compose.foundation.layout.fillMaxWidth
 import androidx.compose.foundation.layout.height
 import androidx.compose.foundation.layout.padding
 import androidx.compose.material.icons.Icons
@@ -44,16 +54,24 @@ import androidx.compose.material3.Surface
 import androidx.compose.material3.Text
 import androidx.compose.material3.TopAppBar
 import androidx.compose.material3.rememberDrawerState
-import androidx.compose.runtime.collectAsState
+import androidx.compose.runtime.Composable
 import androidx.compose.runtime.getValue
 import androidx.compose.runtime.mutableIntStateOf
+import androidx.compose.runtime.mutableStateOf
+import androidx.compose.runtime.remember
 import androidx.compose.runtime.rememberCoroutineScope
 import androidx.compose.runtime.saveable.rememberSaveable
 import androidx.compose.runtime.setValue
 import androidx.compose.ui.Alignment
 import androidx.compose.ui.Modifier
 import androidx.compose.ui.graphics.vector.ImageVector
+import androidx.compose.ui.text.style.TextAlign
 import androidx.compose.ui.unit.dp
+import androidx.compose.ui.unit.sp
+import androidx.compose.ui.viewinterop.AndroidView
+import androidx.core.app.ActivityCompat
+import androidx.core.content.ContextCompat
+import androidx.lifecycle.compose.LocalLifecycleOwner
 import androidx.lifecycle.compose.collectAsStateWithLifecycle
 import androidx.navigation.compose.NavHost
 import androidx.navigation.compose.composable
@@ -112,6 +130,14 @@ class MainActivity : ComponentActivity() {
         // Dependency Injection
         (application as App).appComponent.inject(this)
         super.onCreate(savedInstanceState)
+        if (!hasCameraPermission()) {
+            ActivityCompat.requestPermissions(
+                this,
+                arrayOf(Manifest.permission.CAMERA),
+                0
+            )
+        }
+
         setContent {
             NnstreamerandroidTheme {
                 val navController = rememberNavController()
@@ -268,17 +294,55 @@ class MainActivity : ComponentActivity() {
                                         }
 
                                         composable<ScreenVisionExample> {
-                                            Column(
-                                                modifier = Modifier.fillMaxSize(),
-                                                verticalArrangement = Arrangement.Center,
-                                                horizontalAlignment = Alignment.CenterHorizontally
-                                            ) {
-                                                Box(modifier = Modifier.fillMaxSize()) {
-                                                    Text(text = "Vision Examples")
+                                            var classifications by remember {
+                                                mutableStateOf(emptyList<Classification>())
+                                            }
+                                            val analyzer = remember {
+                                                ImageAnalyzer(
+                                                    MobilenetClassifier(
+                                                        mService,
+                                                        onResults = {
+                                                            classifications = it
+                                                        }),
+                                                )
+                                            }
+                                            val controller = remember {
+                                                LifecycleCameraController(applicationContext).apply {
+                                                    setEnabledUseCases(CameraController.IMAGE_ANALYSIS)
+                                                    setImageAnalysisAnalyzer(
+                                                        ContextCompat.getMainExecutor(
+                                                            applicationContext
+                                                        ),
+                                                        analyzer
+                                                    )
                                                 }
                                             }
+                                            Box(modifier = Modifier.fillMaxSize()) {
+                                                CameraPreview(
+                                                    controller,
+                                                    Modifier
+                                                        .fillMaxSize()
+                                                )
+                                                Column(
+                                                    modifier = Modifier
+                                                        .fillMaxSize()
+                                                        .align(Alignment.BottomCenter),
+                                                ) {
+                                                    classifications.forEach { classification ->
+                                                        Text(
+                                                            text = "${classification.label} (${classification.confidence}%)",
+                                                            modifier = Modifier
+                                                                .fillMaxWidth()
+                                                                .background(colorScheme.secondaryContainer)
+                                                                .padding(16.dp),
+                                                            textAlign = TextAlign.Center,
+                                                            fontSize = 20.sp,
+                                                            color = colorScheme.onSecondaryContainer,
+                                                        )
+                                                    }
+                                                } // Column
+                                            } // Box
                                         }
-
                                         composable<ScreenSettings> {
                                             val args = it.toRoute<ScreenSettings>()
                                             Column(
@@ -321,6 +385,28 @@ class MainActivity : ComponentActivity() {
         super.onStop()
         unbindService(connection)
     }
+
+    @Composable
+    fun CameraPreview(
+        controller: LifecycleCameraController,
+        modifier: Modifier = Modifier
+    ) {
+        val lifecycleOwner = LocalLifecycleOwner.current
+
+        AndroidView(
+            factory = {
+                PreviewView(it).apply {
+                    this.controller = controller
+                    controller.bindToLifecycle(lifecycleOwner)
+                }
+            },
+            modifier = modifier,
+        )
+    }
+
+    private fun hasCameraPermission() = ContextCompat.checkSelfPermission(
+        this, Manifest.permission.CAMERA
+    ) == PackageManager.PERMISSION_GRANTED
 }
 
 @Serializable

diff --git a/ml_inference_offloading/src/main/java/ai/nnstreamer/ml/inference/offloading/MainService.kt b/ml_inference_offloading/src/main/java/ai/nnstreamer/ml/inference/offloading/MainService.kt
@@ -19,6 +19,7 @@ import android.net.nsd.NsdManager
 import android.net.nsd.NsdManager.RegistrationListener
 import android.net.nsd.NsdServiceInfo
 import android.os.Build
+import android.os.Bundle
 import android.os.Handler
 import android.os.HandlerThread
 import android.os.IBinder
@@ -46,7 +47,8 @@ enum class MessageType(val value: Int) {
     LOAD_MODELS(0),
     START_MODEL(1),
     STOP_MODEL(2),
-    DESTROY_MODEL(3)
+    DESTROY_MODEL(3),
+    REQ_OBJ_CLASSIFICATION_FILTER(4),
 }
 
 /**
@@ -81,6 +83,41 @@ class MainService : Service() {
                 MessageType.DESTROY_MODEL.value ->
                     destroyService(msg.arg1)
 
+                // todo: Generalize the message handling for ML service requests
+                MessageType.REQ_OBJ_CLASSIFICATION_FILTER.value -> {
+                    val models = modelsRepository.getAllModelsStream()
+                    val bundle = Bundle()
+                    val replyMsg = Message()
+                    val labels = mutableListOf<String>()
+
+                    // todo: Remove hardcoded label file path
+                    val imagenetLabelPath =
+                        applicationContext.getExternalFilesDir("models")?.run {
+                            resolve("imagenet_labels.txt")
+                        }
+
+                    imagenetLabelPath?.bufferedReader()?.use { reader ->
+                        reader.lineSequence().forEach { line ->
+                            labels.add(line)
+                        }
+                    }
+
+                    bundle.putStringArray("labels", labels.toTypedArray())
+                    // fixme: This is dangerous. We should not use blocking calls in handler.
+                    runBlocking {
+                        models.collect {
+                            it.forEach { model ->
+                                if (model.name.contains("mobilenet")) {
+                                    bundle.putString("filter", model.getNNSFilterDesc())
+
+                                    replyMsg.data = bundle
+                                    msg.replyTo.send(replyMsg)
+                                }
+                            }
+                        }
+                    }
+                }
+
                 else -> super.handleMessage(msg)
             }
         }

diff --git a/...nce_offloading/src/main/java/ai/nnstreamer/ml/inference/offloading/data/Classification.kt b/...nce_offloading/src/main/java/ai/nnstreamer/ml/inference/offloading/data/Classification.kt
@@ -0,0 +1,3 @@
+package ai.nnstreamer.ml.inference.offloading.data
+
+data class Classification(val label: String, val confidence: Double)
diff --git a/...ence_offloading/src/main/java/ai/nnstreamer/ml/inference/offloading/data/ImageAnalyzer.kt b/...ence_offloading/src/main/java/ai/nnstreamer/ml/inference/offloading/data/ImageAnalyzer.kt
@@ -0,0 +1,18 @@
+package ai.nnstreamer.ml.inference.offloading.data
+
+import ai.nnstreamer.ml.inference.offloading.domain.MobilenetClassifier
+import androidx.camera.core.ImageAnalysis
+import androidx.camera.core.ImageProxy
+
+class ImageAnalyzer(
+    private val classifier: MobilenetClassifier,
+) : ImageAnalysis.Analyzer {
+    override fun analyze(imageProxy: ImageProxy) {
+        val rotationDegrees = imageProxy.imageInfo.rotationDegrees
+        val bitmap = imageProxy.toBitmap()
+
+        classifier.classify(bitmap, rotationDegrees)
+
+        imageProxy.close()
+    }
+}