diff --git a/README.md b/README.md index c8a9b92..3f9a082 100644 --- a/README.md +++ b/README.md @@ -156,7 +156,9 @@ await imageClassifier.loadModel(); The `UltralyticsYoloCameraPreview` widget is used to display the camera preview and the results of the prediction. ```dart -final _controller = UltralyticsYoloCameraController(); +final _controller = UltralyticsYoloCameraController( + deferredProcessing: true, // deferred processing for better performance of android (Android only, default: false) +); UltralyticsYoloCameraPreview( predictor: predictor, // Your prediction model data controller: _controller, // Ultralytics camera controller diff --git a/android/src/main/java/com/ultralytics/ultralytics_yolo/CameraPreview.java b/android/src/main/java/com/ultralytics/ultralytics_yolo/CameraPreview.java index eae8dd7..6cc7f13 100644 --- a/android/src/main/java/com/ultralytics/ultralytics_yolo/CameraPreview.java +++ b/android/src/main/java/com/ultralytics/ultralytics_yolo/CameraPreview.java @@ -19,6 +19,9 @@ import com.ultralytics.ultralytics_yolo.predict.Predictor; import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.atomic.AtomicBoolean; public class CameraPreview { @@ -30,14 +33,16 @@ public class CameraPreview { private Activity activity; private PreviewView mPreviewView; private boolean busy = false; + private boolean deferredProcessing = false; public CameraPreview(Context context) { this.context = context; } - public void openCamera(int facing, Activity activity, PreviewView mPreviewView) { + public void openCamera(int facing, Activity activity, PreviewView mPreviewView, boolean deferredProcessing) { this.activity = activity; this.mPreviewView = mPreviewView; + this.deferredProcessing = deferredProcessing; final ListenableFuture cameraProviderFuture = ProcessCameraProvider.getInstance(context); cameraProviderFuture.addListener(() -> { @@ -55,6 +60,8 @@ private void bindPreview(int facing) { if (!busy) { busy = true; + final boolean isMirrored = (facing == CameraSelector.LENS_FACING_FRONT); + Preview cameraPreview = new Preview.Builder() .setTargetAspectRatio(AspectRatio.RATIO_4_3) .build(); @@ -65,15 +72,42 @@ private void bindPreview(int facing) { ImageAnalysis imageAnalysis = new ImageAnalysis.Builder() - .setBackpressureStrategy(ImageAnalysis.STRATEGY_KEEP_ONLY_LATEST) + .setBackpressureStrategy(deferredProcessing ? ImageAnalysis.STRATEGY_BLOCK_PRODUCER : ImageAnalysis.STRATEGY_KEEP_ONLY_LATEST) .setTargetAspectRatio(AspectRatio.RATIO_4_3) .build(); - imageAnalysis.setAnalyzer(Runnable::run, imageProxy -> { - predictor.predict(imageProxy, facing == CameraSelector.LENS_FACING_FRONT); - //clear stream for next image - imageProxy.close(); - }); + if (deferredProcessing) { + final ExecutorService executorService = Executors.newSingleThreadExecutor(); + final AtomicBoolean isPredicting = new AtomicBoolean(false); + + imageAnalysis.setAnalyzer(Runnable::run, imageProxy -> { + if (isPredicting.get()) { + imageProxy.close(); + return; + } + + isPredicting.set(true); + + executorService.submit(() -> { + try { + predictor.predict(imageProxy, isMirrored); + } catch (Exception e) { + e.printStackTrace(); + } finally { + //clear stream for next image + imageProxy.close(); + + isPredicting.set(false); + } + }); + }); + } else { + imageAnalysis.setAnalyzer(Runnable::run, imageProxy -> { + predictor.predict(imageProxy, isMirrored); + //clear stream for next image + imageProxy.close(); + }); + } // Unbind use cases before rebinding cameraProvider.unbindAll(); diff --git a/android/src/main/java/com/ultralytics/ultralytics_yolo/NativeView.java b/android/src/main/java/com/ultralytics/ultralytics_yolo/NativeView.java index b9325d0..c583ba6 100644 --- a/android/src/main/java/com/ultralytics/ultralytics_yolo/NativeView.java +++ b/android/src/main/java/com/ultralytics/ultralytics_yolo/NativeView.java @@ -25,11 +25,12 @@ class NativeView implements PlatformView { final int lensDirection = (int) creationParams.get("lensDirection"); // final String format = (String) creationParams.get("format"); + final boolean deferredProcessing = (boolean) creationParams.getOrDefault("deferredProcessing", false); // if (Objects.requireNonNull(format).equals("tflite")) { view = LayoutInflater.from(context).inflate(R.layout.activity_tflite_camera, null); mPreviewView = view.findViewById(R.id.previewView); - startTfliteCamera(lensDirection); + startTfliteCamera(lensDirection, deferredProcessing); // } } @@ -44,7 +45,7 @@ public View getView() { public void dispose() { } - private void startTfliteCamera(int facing) { - cameraPreview.openCamera(facing, activity, mPreviewView); + private void startTfliteCamera(int facing, boolean deferredProcessing) { + cameraPreview.openCamera(facing, activity, mPreviewView, deferredProcessing); } } diff --git a/lib/camera_preview/ultralytics_yolo_camera_controller.dart b/lib/camera_preview/ultralytics_yolo_camera_controller.dart index 009c9de..a965d5d 100644 --- a/lib/camera_preview/ultralytics_yolo_camera_controller.dart +++ b/lib/camera_preview/ultralytics_yolo_camera_controller.dart @@ -7,6 +7,7 @@ class UltralyticsYoloCameraValue { UltralyticsYoloCameraValue({ required this.lensDirection, required this.strokeWidth, + required this.deferredProcessing, }); /// The direction of the camera lens @@ -15,15 +16,20 @@ class UltralyticsYoloCameraValue { /// The width of the stroke used to draw the bounding boxes final double strokeWidth; + /// Whether the processing of the frames should be deferred (android only) + final bool deferredProcessing; + /// Creates a copy of this [UltralyticsYoloCameraValue] but with /// the given fields UltralyticsYoloCameraValue copyWith({ int? lensDirection, double? strokeWidth, + bool? deferredProcessing, }) => UltralyticsYoloCameraValue( lensDirection: lensDirection ?? this.lensDirection, strokeWidth: strokeWidth ?? this.strokeWidth, + deferredProcessing: deferredProcessing ?? this.deferredProcessing, ); } @@ -31,11 +37,12 @@ class UltralyticsYoloCameraValue { class UltralyticsYoloCameraController extends ValueNotifier { /// Constructor to create an instance of [UltralyticsYoloCameraController] - UltralyticsYoloCameraController() + UltralyticsYoloCameraController({bool deferredProcessing = false}) : super( UltralyticsYoloCameraValue( lensDirection: 1, strokeWidth: 2.5, + deferredProcessing: deferredProcessing, ), ); diff --git a/lib/camera_preview/ultralytics_yolo_camera_preview.dart b/lib/camera_preview/ultralytics_yolo_camera_preview.dart index edb7212..a765735 100644 --- a/lib/camera_preview/ultralytics_yolo_camera_preview.dart +++ b/lib/camera_preview/ultralytics_yolo_camera_preview.dart @@ -73,6 +73,8 @@ class _UltralyticsYoloCameraPreviewState final creationParams = { 'lensDirection': widget.controller.value.lensDirection, 'format': widget.predictor?.model.format.name, + 'deferredProcessing': + widget.controller.value.deferredProcessing, }; switch (defaultTargetPlatform) {