feat: replay orientation change (#2462)

* refactor: move replay integration to a separate class

* Add onBuild callback to ScreenshotWidget

* configure replay from dart to native (java)

* send replay config from dart to native

* fix: missing return in android plugin leading to duplicate response

* fix android replay start before the setting is present

* tmp log

* logging

* change screenshot dimensions to double

* refactor android replay integration and fix timing issues during orientation changes

* fix tests

* cleanup

* ktlint

* update test coverage

* linter issues

* chore: changelog

* enh: use ensureVisualUpdate instead of scheduleFrame

* avoid errors during initial "unconfigured" replay startup

* linter

CHANGELOG.md

@@ -4,6 +4,7 @@
 
 ### Features
 
+- Replay: device orientation change support & improve video size fit on Android ([#2462](https://github.com/getsentry/sentry-dart/pull/2462))
 - Support custom `Sentry.runZoneGuarded` zone creation ([#2088](https://github.com/getsentry/sentry-dart/pull/2088))
   - Sentry will not create a custom zone anymore if it is started within a custom one.
   - This fixes Zone miss-match errors when trying to initialize WidgetsBinding before Sentry on Flutter Web

flutter/android/src/main/kotlin/io/sentry/flutter/SentryFlutterPlugin.kt

@@ -2,6 +2,7 @@ package io.sentry.flutter
 
 import android.app.Activity
 import android.content.Context
+import android.content.res.Configuration
 import android.os.Build
 import android.os.Looper
 import android.util.Log
@@ -27,22 +28,40 @@ import io.sentry.android.core.SentryAndroidOptions
 import io.sentry.android.core.performance.AppStartMetrics
 import io.sentry.android.core.performance.TimeSpan
 import io.sentry.android.replay.ReplayIntegration
+import io.sentry.android.replay.ScreenshotRecorderConfig
 import io.sentry.protocol.DebugImage
 import io.sentry.protocol.SdkVersion
 import io.sentry.protocol.SentryId
 import io.sentry.protocol.User
 import io.sentry.transport.CurrentDateProvider
 import java.io.File
 import java.lang.ref.WeakReference
+import kotlin.math.roundToInt
 
 private const val APP_START_MAX_DURATION_MS = 60000
+public const val VIDEO_BLOCK_SIZE = 16
 
-class SentryFlutterPlugin : FlutterPlugin, MethodCallHandler, ActivityAware {
+class SentryFlutterPlugin :
+  FlutterPlugin,
+  MethodCallHandler,
+  ActivityAware {
   private lateinit var channel: MethodChannel
   private lateinit var context: Context
   private lateinit var sentryFlutter: SentryFlutter
   private lateinit var replay: ReplayIntegration
 
+  // Note: initial config because we don't yet have the numbers of the actual Flutter widget.
+  // See how SentryFlutterReplayRecorder.start() handles it. New settings will be set by setReplayConfig() method below.
+  private var replayConfig =
+    ScreenshotRecorderConfig(
+      recordingWidth = VIDEO_BLOCK_SIZE,
+      recordingHeight = VIDEO_BLOCK_SIZE,
+      scaleFactorX = 1.0f,
+      scaleFactorY = 1.0f,
+      frameRate = 1,
+      bitRate = 75000,
+    )
+
   private var activity: WeakReference<Activity>? = null
   private var framesTracker: ActivityFramesTracker? = null
   private var pluginRegistrationTime: Long? = null
@@ -86,6 +105,7 @@ class SentryFlutterPlugin : FlutterPlugin, MethodCallHandler, ActivityAware {
       "loadContexts" -> loadContexts(result)
       "displayRefreshRate" -> displayRefreshRate(result)
       "nativeCrash" -> crash()
+      "setReplayConfig" -> setReplayConfig(call, result)
       "addReplayScreenshot" -> addReplayScreenshot(call.argument("path"), call.argument("timestamp"), result)
       "captureReplay" -> captureReplay(call.argument("isCrash"), result)
       else -> result.notImplemented()
@@ -152,7 +172,18 @@ class SentryFlutterPlugin : FlutterPlugin, MethodCallHandler, ActivityAware {
             context,
             dateProvider = CurrentDateProvider.getInstance(),
             recorderProvider = { SentryFlutterReplayRecorder(channel, replay) },
-            recorderConfigProvider = null,
+            recorderConfigProvider = {
+              Log.i(
+                "Sentry",
+                "Replay configuration requested. Returning: %dx%d at %d FPS, %d BPS".format(
+                  replayConfig.recordingWidth,
+                  replayConfig.recordingHeight,
+                  replayConfig.frameRate,
+                  replayConfig.bitRate,
+                ),
+              )
+              replayConfig
+            },
             replayCacheProvider = null,
           )
         replay.breadcrumbConverter = SentryFlutterReplayBreadcrumbConverter()
@@ -181,6 +212,7 @@ class SentryFlutterPlugin : FlutterPlugin, MethodCallHandler, ActivityAware {
         "Invalid app start data: app not launched in foreground or app start took too long (>60s)",
       )
       result.success(null)
+      return
     }
 
     val appStartTimeSpan = appStartMetrics.appStartTimeSpan
@@ -546,6 +578,15 @@ class SentryFlutterPlugin : FlutterPlugin, MethodCallHandler, ActivityAware {
       mainThread.uncaughtExceptionHandler.uncaughtException(mainThread, exception)
       mainThread.join(NATIVE_CRASH_WAIT_TIME)
     }
+
+    private fun Double.adjustReplaySizeToBlockSize(): Double {
+      val remainder = this % VIDEO_BLOCK_SIZE
+      return if (remainder <= VIDEO_BLOCK_SIZE / 2) {
+        this - remainder
+      } else {
+        this + (VIDEO_BLOCK_SIZE - remainder)
+      }
+    }
   }
 
   private fun loadContexts(result: Result) {
@@ -577,6 +618,48 @@ class SentryFlutterPlugin : FlutterPlugin, MethodCallHandler, ActivityAware {
     result.success("")
   }
 
+  private fun setReplayConfig(
+    call: MethodCall,
+    result: Result,
+  ) {
+    // Since codec block size is 16, so we have to adjust the width and height to it,
+    // otherwise the codec might fail to configure on some devices, see
+    // https://cs.android.com/android/platform/superproject/+/master:frameworks/base/media/java/android/media/MediaCodecInfo.java;l=1999-2001
+    var width = call.argument("width") as? Double ?: 0.0
+    var height = call.argument("height") as? Double ?: 0.0
+    // First update the smaller dimension, as changing that will affect the screen ratio more.
+    if (width < height) {
+      val newWidth = width.adjustReplaySizeToBlockSize()
+      height = (height * (newWidth / width)).adjustReplaySizeToBlockSize()
+      width = newWidth
+    } else {
+      val newHeight = height.adjustReplaySizeToBlockSize()
+      width = (width * (newHeight / height)).adjustReplaySizeToBlockSize()
+      height = newHeight
+    }
+
+    replayConfig =
+      ScreenshotRecorderConfig(
+        recordingWidth = width.roundToInt(),
+        recordingHeight = height.roundToInt(),
+        scaleFactorX = 1.0f,
+        scaleFactorY = 1.0f,
+        frameRate = call.argument("frameRate") as? Int ?: 0,
+        bitRate = call.argument("bitRate") as? Int ?: 0,
+      )
+    Log.i(
+      "Sentry",
+      "Configuring replay: %dx%d at %d FPS, %d BPS".format(
+        replayConfig.recordingWidth,
+        replayConfig.recordingHeight,
+        replayConfig.frameRate,
+        replayConfig.bitRate,
+      ),
+    )
+    replay.onConfigurationChanged(Configuration())
+    result.success("")
+  }
+
   private fun captureReplay(
     isCrash: Boolean?,
     result: Result,

flutter/android/src/main/kotlin/io/sentry/flutter/SentryFlutterReplayRecorder.kt

@@ -13,6 +13,12 @@ internal class SentryFlutterReplayRecorder(
   private val integration: ReplayIntegration,
 ) : Recorder {
   override fun start(recorderConfig: ScreenshotRecorderConfig) {
+    // Ignore if this is the initial call before we actually got the configuration from Flutter.
+    // We'll get another call here when the configuration is changed according to the widget size.
+    if (recorderConfig.recordingHeight <= VIDEO_BLOCK_SIZE && recorderConfig.recordingWidth <= VIDEO_BLOCK_SIZE) {
+      return
+    }
+
     val cacheDirPath = integration.replayCacheDir?.absolutePath
     if (cacheDirPath == null) {
       Log.w("Sentry", "Replay cache directory is null, can't start replay recorder.")

flutter/example/android/app/src/main/kotlin/io/sentry/samples/flutter/MainActivity.kt

@@ -44,6 +44,7 @@ class MainActivity : FlutterActivity() {
   }
 
   private external fun crash()
+
   private external fun message()
 
   companion object {

flutter/lib/src/native/c/sentry_native.dart

@@ -8,6 +8,7 @@ import 'package:ffi/ffi.dart';
 import 'package:meta/meta.dart';
 
 import '../../../sentry_flutter.dart';
+import '../../replay/replay_config.dart';
 import '../native_app_start.dart';
 import '../native_frames.dart';
 import '../sentry_native_binding.dart';
@@ -268,6 +269,14 @@ class SentryNative with SentryNativeSafeInvoker implements SentryNativeBinding {
     Pointer.fromAddress(1).cast<Utf8>().toDartString();
   }
 
+  @override
+  bool get supportsReplay => false;
+
+  @override
+  FutureOr<void> setReplayConfig(ReplayConfig config) {
+    _logNotSupported('replay config');
+  }
+
   @override
   FutureOr<SentryId> captureReplay(bool isCrash) {
     _logNotSupported('capturing replay');

flutter/lib/src/native/cocoa/sentry_native_cocoa.dart

@@ -5,10 +5,8 @@ import 'dart:ui';
 import 'package:meta/meta.dart';
 
 import '../../../sentry_flutter.dart';
-import '../../event_processor/replay_event_processor.dart';
 import '../../screenshot/recorder.dart';
 import '../../screenshot/recorder_config.dart';
-import '../../replay/integration.dart';
 import '../sentry_native_channel.dart';
 import 'binding.dart' as cocoa;
 
@@ -20,19 +18,14 @@ class SentryNativeCocoa extends SentryNativeChannel {
 
   SentryNativeCocoa(super.options);
 
+  @override
+  bool get supportsReplay => options.platformChecker.platform.isIOS;
+
   @override
   Future<void> init(Hub hub) async {
     // We only need these when replay is enabled (session or error capture)
     // so let's set it up conditionally. This allows Dart to trim the code.
-    if (options.experimental.replay.isEnabled &&
-        options.platformChecker.platform.isIOS) {
-      options.sdk.addIntegration(replayIntegrationName);
-
-      // We only need the integration when error-replay capture is enabled.
-      if ((options.experimental.replay.onErrorSampleRate ?? 0) > 0) {
-        options.addEventProcessor(ReplayEventProcessor(hub, this));
-      }
-
+    if (options.experimental.replay.isEnabled) {
       channel.setMethodCallHandler((call) async {
         switch (call.method) {
           case 'captureReplayScreenshot':

flutter/lib/src/native/java/android_replay_recorder.dart

@@ -0,0 +1,48 @@
+import 'package:meta/meta.dart';
+
+import '../../../sentry_flutter.dart';
+import '../../replay/scheduled_recorder.dart';
+import '../sentry_safe_method_channel.dart';
+
+@internal
+class AndroidReplayRecorder extends ScheduledScreenshotRecorder {
+  final SentrySafeMethodChannel _channel;
+  final String _cacheDir;
+
+  AndroidReplayRecorder(
+      super.config, super.options, this._channel, this._cacheDir) {
+    super.callback = _addReplayScreenshot;
+  }
+
+  Future<void> _addReplayScreenshot(
+      ScreenshotPng screenshot, bool isNewlyCaptured) async {
+    final timestamp = DateTime.now().millisecondsSinceEpoch;
+    final filePath = "$_cacheDir/$timestamp.png";
+
+    options.logger(
+        SentryLevel.debug,
+        '$logName: saving ${isNewlyCaptured ? 'new' : 'repeated'} screenshot to'
+        ' $filePath (${screenshot.width}x${screenshot.height} pixels, '
+        '${screenshot.data.lengthInBytes} bytes)');
+    try {
+      await options.fileSystem
+          .file(filePath)
+          .writeAsBytes(screenshot.data.buffer.asUint8List(), flush: true);
+
+      await _channel.invokeMethod(
+        'addReplayScreenshot',
+        {'path': filePath, 'timestamp': timestamp},
+      );
+    } catch (error, stackTrace) {
+      options.logger(
+        SentryLevel.error,
+        '$logName: native call `addReplayScreenshot` failed',
+        exception: error,
+        stackTrace: stackTrace,
+      );
+      if (options.automatedTestMode) {
+        rethrow;
+      }
+    }
+  }
+}