From 28c25a6cbc715d446a809d15edf2cdcaa0ef44f3 Mon Sep 17 00:00:00 2001 From: Bene0202 <131483360+Bene0202@users.noreply.github.com> Date: Tue, 30 Jan 2024 17:53:48 +0100 Subject: [PATCH 1/5] Add files via upload --- local.properties | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 local.properties diff --git a/local.properties b/local.properties new file mode 100644 index 0000000..28b54d1 --- /dev/null +++ b/local.properties @@ -0,0 +1,9 @@ +## This file must *NOT* be checked into Version Control Systems, +# as it contains information specific to your local configuration. +# +# Location of the SDK. This is only used by Gradle. +# For customization when using a Version Control System, please read the +# header note. +#Tue Jan 30 17:44:56 CET 2024 +sdk.dir=C\:\\Users\\benek\\AppData\\Local\\Android\\Sdk +MAPS_API_KEY="empty" From 7f84f048469ae37885730db0186f83e62819404e Mon Sep 17 00:00:00 2001 From: Bene0202 <131483360+Bene0202@users.noreply.github.com> Date: Tue, 30 Jan 2024 17:55:08 +0100 Subject: [PATCH 2/5] Delete local.properties --- local.properties | 9 --------- 1 file changed, 9 deletions(-) delete mode 100644 local.properties diff --git a/local.properties b/local.properties deleted file mode 100644 index 28b54d1..0000000 --- a/local.properties +++ /dev/null @@ -1,9 +0,0 @@ -## This file must *NOT* be checked into Version Control Systems, -# as it contains information specific to your local configuration. -# -# Location of the SDK. This is only used by Gradle. -# For customization when using a Version Control System, please read the -# header note. -#Tue Jan 30 17:44:56 CET 2024 -sdk.dir=C\:\\Users\\benek\\AppData\\Local\\Android\\Sdk -MAPS_API_KEY="empty" From ce86045b4d25d8a82b0d26e000009a6495cb51e9 Mon Sep 17 00:00:00 2001 From: Bene0202 <131483360+Bene0202@users.noreply.github.com> Date: Tue, 30 Jan 2024 18:09:39 +0100 Subject: [PATCH 3/5] Update .gitignore --- .gitignore | 2 -- 1 file changed, 2 deletions(-) diff --git a/.gitignore b/.gitignore index a9eb6f4..b04fce1 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,4 @@ # Gradle files -.gradle/ -build/ # Local configuration file (sdk path, etc) local.properties From 9888b33289af5baf44906e2ef6137c60cd45ffe6 Mon Sep 17 00:00:00 2001 From: Bene0202 <131483360+Bene0202@users.noreply.github.com> Date: Tue, 30 Jan 2024 18:37:01 +0100 Subject: [PATCH 4/5] Your commit message here --- app/build.gradle | 34 ++- app/src/main/AndroidManifest.xml | 6 +- .../nlinterface/activities/MainActivity.kt | 23 +- .../viewmodels/BarcodeProductinfoViewModel.kt | 209 ++++++++++++++++++ build.gradle | 4 +- gradle/wrapper/gradle-wrapper.properties | 2 +- 6 files changed, 270 insertions(+), 8 deletions(-) create mode 100644 app/src/main/java/com/nlinterface/viewmodels/BarcodeProductinfoViewModel.kt diff --git a/app/build.gradle b/app/build.gradle index ffa8d2a..477d96c 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -8,12 +8,13 @@ plugins { android { namespace 'com.nlinterface' - compileSdk 33 + compileSdk 34 defaultConfig { applicationId "com.nlinterface" minSdk 24 - targetSdk 33 + //noinspection EditedTargetSdkVersion + targetSdk 34 versionCode 1 versionName '0.0.1' @@ -36,16 +37,24 @@ android { buildFeatures { viewBinding true } + packaging { + resources { + excludes += "/META-INF/{AL2.0,LGPL2.1}" + excludes += "META-INF/DEPENDENCIES" + excludes += "mozilla/public-suffix-list.txt" + } + } } dependencies { - implementation 'androidx.core:core-ktx:1.7.0' + implementation 'androidx.core:core-ktx:1.12.0' implementation 'androidx.appcompat:appcompat:1.6.1' implementation 'com.google.android.material:material:1.5.0-alpha04' implementation 'androidx.constraintlayout:constraintlayout:2.1.4' implementation 'androidx.navigation:navigation-fragment-ktx:2.5.3' implementation 'androidx.navigation:navigation-ui-ktx:2.5.3' + implementation 'androidx.lifecycle:lifecycle-process:2.7.0' testImplementation 'junit:junit:4.13.2' androidTestImplementation 'androidx.test.ext:junit:1.1.5' androidTestImplementation 'androidx.test.espresso:espresso-core:3.5.1' @@ -61,4 +70,23 @@ dependencies { implementation("org.jetbrains.kotlinx:kotlinx-coroutines-android:1.3.9") + //mlkit + implementation 'com.google.mlkit:barcode-scanning:17.2.0' + + //it.jsoup + implementation "org.jsoup:jsoup:1.14.3" + + + //androix.compose + implementation(platform('androidx.compose:compose-bom:2023.08.00')) + androidTestImplementation(platform("androidx.compose:compose-bom:2023.08.00")) + debugImplementation 'androidx.compose.ui:ui-tooling' + + + //camera + implementation ("androidx.camera:camera-camera2:1.3.1") + implementation("androidx.camera:camera-core:1.3.1") + implementation("androidx.camera:camera-lifecycle:1.3.1") + implementation("androidx.camera:camera-view:1.3.1") + } \ No newline at end of file diff --git a/app/src/main/AndroidManifest.xml b/app/src/main/AndroidManifest.xml index 40322fd..c2ff521 100644 --- a/app/src/main/AndroidManifest.xml +++ b/app/src/main/AndroidManifest.xml @@ -7,6 +7,8 @@ + + - + \ No newline at end of file diff --git a/app/src/main/java/com/nlinterface/activities/MainActivity.kt b/app/src/main/java/com/nlinterface/activities/MainActivity.kt index 8ae6fe6..bc1d7f9 100644 --- a/app/src/main/java/com/nlinterface/activities/MainActivity.kt +++ b/app/src/main/java/com/nlinterface/activities/MainActivity.kt @@ -17,6 +17,7 @@ import androidx.lifecycle.ViewModelProvider import com.nlinterface.R import com.nlinterface.databinding.ActivityMainBinding import com.nlinterface.utility.* +import com.nlinterface.viewmodels.ConstantScanning import com.nlinterface.viewmodels.MainViewModel /** @@ -64,6 +65,10 @@ class MainActivity : AppCompatActivity() { configureUI() configureTTS() configureSTT() + + verifyCameraPermissions() + val serviceIntent = Intent(this, ConstantScanning()::class.java) + startService(serviceIntent) } /** @@ -285,5 +290,21 @@ class MainActivity : AppCompatActivity() { viewModel.cancelListening() } } - + + /** + * Request the user to grant camera permissions, if not already granted. + * Will probably not be used further once other camera is included + */ + private fun verifyCameraPermissions() { + if (checkCallingOrSelfPermission( + Manifest.permission.CAMERA + ) != PackageManager.PERMISSION_GRANTED + ) { + ActivityCompat.requestPermissions( + this, + arrayOf(Manifest.permission.CAMERA), + STT_PERMISSION_REQUEST_CODE + ) + } + } } \ No newline at end of file diff --git a/app/src/main/java/com/nlinterface/viewmodels/BarcodeProductinfoViewModel.kt b/app/src/main/java/com/nlinterface/viewmodels/BarcodeProductinfoViewModel.kt new file mode 100644 index 0000000..d42ca20 --- /dev/null +++ b/app/src/main/java/com/nlinterface/viewmodels/BarcodeProductinfoViewModel.kt @@ -0,0 +1,209 @@ +package com.nlinterface.viewmodels + + +import android.app.Service +import android.content.Context +import androidx.camera.core.CameraSelector +import androidx.camera.core.ExperimentalGetImage +import androidx.camera.core.ImageAnalysis +import androidx.camera.core.ImageProxy +import androidx.camera.lifecycle.ProcessCameraProvider +import com.google.mlkit.vision.barcode.BarcodeScannerOptions +import com.google.mlkit.vision.barcode.BarcodeScanning +import com.google.mlkit.vision.barcode.common.Barcode +import com.google.mlkit.vision.common.InputImage +import androidx.core.content.ContextCompat +import android.content.Intent +import android.os.IBinder +import androidx.lifecycle.ProcessLifecycleOwner +import org.jsoup.Jsoup +import org.jsoup.nodes.Document + +/** + * Classes and functions managing a background Service, + + * which is constantly checking for barcodes + * + * TODO: Add voice command allowing to stop TTS + */ + +/** + * The Scanner is used to create a BarcodeScanner object, + * analyze the imageProxy and check for barcodes, + * and defines how to handle them. + * + * @param viewModel: viewModel allows the use of the say function + */ +class Scanner( + private val viewModel: MainViewModel +) : ImageAnalysis.Analyzer { + + /** + * BarcodeScanner object: Currently only scans EAN 13 barcodes + */ + private val options = BarcodeScannerOptions.Builder() + .setBarcodeFormats( + Barcode.FORMAT_EAN_13 + ) + .build() + private val barcodeScanner = BarcodeScanning.getClient(options) + + /** + * Analysis of the ImageProxy. Converts the camera output into analyzable format. + * Applies the function handleBarcodeResult to the first barcode scanned. + * This is embedded into a Thread to avoid to much calculations on the Main thread. + * + * @param image: uses the camera image without passing an argument directly + * + * TODO: potentially add sound feedback on scanned barcode, + * TODO: hence the scraping and verbal feedback is a bit delayed + * + */ + @ExperimentalGetImage + override fun analyze(image: ImageProxy) { + val mediaImage = image.image + if (mediaImage != null) { + val scannerInput = + InputImage.fromMediaImage(mediaImage, image.imageInfo.rotationDegrees) + barcodeScanner.process(scannerInput) + .addOnSuccessListener { barcodes -> + for (barcode in barcodes) { + val urlAddOn = barcode.rawValue ?: "default" + Thread { + handleBarcodeResult(urlAddOn) + }.start() + break + } + } + .addOnFailureListener { exception -> + exception.printStackTrace() + } + .addOnCompleteListener { image.close() } + } + } + + /** + * Function to handle the Barcode Result. + * It creates an object BrowserSearch to use its function searchUrl + * + * @param urlAddOn: the Barcode as a String to use for web-search + */ + private fun handleBarcodeResult(urlAddOn: String) { + // Handle the barcode result logic here + val eanSearch = BrowserSearch() + eanSearch.searchUrl(viewModel, urlAddOn) + + } +} + +/** + * The BrowserSearch class embeds the web-search. + * + */ +class BrowserSearch{ + + /** + * The function to do the web-search and scraping. + * Url: https://de.openfoodfacts.org/produkt/$barcode + * searches the website with regards to the Barcodes value + * Then scrapes the document for the specified elements and TTS the extracted text. + * + * @param viewModel: viewModel allows the use of the say function + * @param barcode: the barcode as String to add on to the Url + */ + fun searchUrl (viewModel:MainViewModel, barcode: String) { + try { + val searchUrl = "https://de.openfoodfacts.org/produkt/$barcode" + val document: Document = Jsoup.connect(searchUrl).get() + val name = document.select("h1.title-3").first()?.text() + val ingredients = + document.getElementById("panel_ingredients_content")?.text() + val allInfo = name.toString() + ingredients.toString() + viewModel.say(allInfo) + } catch (e: Exception) { + e.printStackTrace() + } + } +} +/** + * The Scanning process handles the camera and initiates the imageAnalysis. + * + */ + +class ScanningProcess{ + + /** + * Function starting the scanning process. + * Selects the camera to be used, initializes an ImageAnalysis object, + * from which to use the scanner method. + * Also binds the camera to a Lifecycle + * + * @param viewModel: passed on to allow using the say function + * @param context: context for the cameraProvider and the imageAnalyzes + * + * TODO: Change selector to other camera + */ + fun activateScanning(viewModel: MainViewModel, context: Context) { + + val selector = CameraSelector.Builder() + .requireLensFacing(CameraSelector.LENS_FACING_BACK) + .build() + val imageAnalysis = ImageAnalysis.Builder() + .setBackpressureStrategy(ImageAnalysis.STRATEGY_KEEP_ONLY_LATEST) + .build() + imageAnalysis.setAnalyzer( + ContextCompat.getMainExecutor(context), + Scanner(viewModel) + ) + val cameraProvider = ProcessCameraProvider.getInstance(context).get() + try { + cameraProvider.bindToLifecycle( + ProcessLifecycleOwner.get(), + selector, + imageAnalysis + ) + } catch (e: Exception) { + e.printStackTrace() + } + } +} + +/** + * ConstantScanning is a Service, + * meaning it is allowed to run in the background parallel to the other activity. + * Also it runs without an UI, which is wanted in the case of the Scanner + */ + +class ConstantScanning: Service() { + + /** + * This method is required by the Service architecture, + * but not needed because the scanning should be done constantly. + * Therefore it just return null + */ + override fun onBind(intent: Intent?): IBinder? { + return null + } + + /** + * Function that manages the Service, once initialized. + * It creates a viewModel object to be passed on, so the say method can be used later on. + * It also creates a Scanner object and starts the Scanning process. + * Returns Start-Sticky to ensure it will try to start again, + * if it is destroyed for whatever reason + * + * @param intent + * @param flags + * @param startId + */ + + override fun onStartCommand(intent: Intent?, flags: Int, startId: Int): Int { + val application = application + val viewModel = MainViewModel(application) + viewModel.initTTS() + val scanner = ScanningProcess() + scanner.activateScanning(viewModel, this) + + return START_STICKY + } + +} diff --git a/build.gradle b/build.gradle index e1af099..730e50b 100644 --- a/build.gradle +++ b/build.gradle @@ -6,7 +6,7 @@ buildscript { } plugins { - id 'com.android.application' version '8.1.2' apply false - id 'com.android.library' version '8.1.2' apply false + id 'com.android.application' version '8.2.0' apply false + id 'com.android.library' version '8.2.0' apply false id 'org.jetbrains.kotlin.android' version '1.7.10' apply false } \ No newline at end of file diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 2938e48..2fb278d 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,6 +1,6 @@ #Thu May 25 17:04:02 CEST 2023 distributionBase=GRADLE_USER_HOME -distributionUrl=https\://services.gradle.org/distributions/gradle-8.0-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.2-bin.zip distributionPath=wrapper/dists zipStorePath=wrapper/dists zipStoreBase=GRADLE_USER_HOME From c2b69500e1d1761bb1657c589d8d09ac79ea2e50 Mon Sep 17 00:00:00 2001 From: Bene0202 <131483360+Bene0202@users.noreply.github.com> Date: Fri, 9 Feb 2024 19:40:44 +0100 Subject: [PATCH 5/5] Changes apllied --- app/build.gradle | 10 +----- app/src/main/AndroidManifest.xml | 1 + .../nlinterface/activities/MainActivity.kt | 8 +++-- .../viewmodels/BarcodeProductinfoViewModel.kt | 31 ++++++++++++------- 4 files changed, 28 insertions(+), 22 deletions(-) diff --git a/app/build.gradle b/app/build.gradle index 477d96c..0094416 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -13,8 +13,7 @@ android { defaultConfig { applicationId "com.nlinterface" minSdk 24 - //noinspection EditedTargetSdkVersion - targetSdk 34 + targetSdk 33 versionCode 1 versionName '0.0.1' @@ -76,13 +75,6 @@ dependencies { //it.jsoup implementation "org.jsoup:jsoup:1.14.3" - - //androix.compose - implementation(platform('androidx.compose:compose-bom:2023.08.00')) - androidTestImplementation(platform("androidx.compose:compose-bom:2023.08.00")) - debugImplementation 'androidx.compose.ui:ui-tooling' - - //camera implementation ("androidx.camera:camera-camera2:1.3.1") implementation("androidx.camera:camera-core:1.3.1") diff --git a/app/src/main/AndroidManifest.xml b/app/src/main/AndroidManifest.xml index c2ff521..f2bdeb5 100644 --- a/app/src/main/AndroidManifest.xml +++ b/app/src/main/AndroidManifest.xml @@ -8,6 +8,7 @@ + for (barcode in barcodes) { + Log.println(Log.INFO, "Scanner", "Barcode " + barcode.rawValue + " was detected") val urlAddOn = barcode.rawValue ?: "default" Thread { handleBarcodeResult(urlAddOn) @@ -83,15 +84,16 @@ class Scanner( /** * Function to handle the Barcode Result. + * Initiates a vibration to give feedback to the user, that a barcode was scanned * It creates an object BrowserSearch to use its function searchUrl * * @param urlAddOn: the Barcode as a String to use for web-search */ private fun handleBarcodeResult(urlAddOn: String) { - // Handle the barcode result logic here + vibrator.vibrate(200) val eanSearch = BrowserSearch() + Log.println(Log.INFO, "Scraping", "Waiting for Scraping result") eanSearch.searchUrl(viewModel, urlAddOn) - } } @@ -142,7 +144,7 @@ class ScanningProcess{ * * TODO: Change selector to other camera */ - fun activateScanning(viewModel: MainViewModel, context: Context) { + fun activateScanning(viewModel: MainViewModel, vibrator: Vibrator, context: Context) { val selector = CameraSelector.Builder() .requireLensFacing(CameraSelector.LENS_FACING_BACK) @@ -152,7 +154,7 @@ class ScanningProcess{ .build() imageAnalysis.setAnalyzer( ContextCompat.getMainExecutor(context), - Scanner(viewModel) + Scanner(viewModel, vibrator) ) val cameraProvider = ProcessCameraProvider.getInstance(context).get() try { @@ -161,8 +163,10 @@ class ScanningProcess{ selector, imageAnalysis ) + Log.println(Log.INFO, "Camera", "Camera binding successful") + } catch (e: Exception) { - e.printStackTrace() + e.printStackTrace().toString() } } } @@ -187,7 +191,7 @@ class ConstantScanning: Service() { /** * Function that manages the Service, once initialized. * It creates a viewModel object to be passed on, so the say method can be used later on. - * It also creates a Scanner object and starts the Scanning process. + * It also creates a Vibrator and Scanner object and starts the Scanning process. * Returns Start-Sticky to ensure it will try to start again, * if it is destroyed for whatever reason * @@ -197,11 +201,16 @@ class ConstantScanning: Service() { */ override fun onStartCommand(intent: Intent?, flags: Int, startId: Int): Int { + + + val application = application val viewModel = MainViewModel(application) viewModel.initTTS() + val vibrator = getSystemService(Context.VIBRATOR_SERVICE) as Vibrator val scanner = ScanningProcess() - scanner.activateScanning(viewModel, this) + scanner.activateScanning(viewModel, vibrator, this) + Log.println(Log.INFO, "Scanner","Barcode Scanning Service is Active") return START_STICKY }