From eb8b74a0392d2f40dfe5506f90095872cb7f2db1 Mon Sep 17 00:00:00 2001 From: Littlegnal <8847263+littleGnAl@users.noreply.github.com> Date: Wed, 10 Apr 2024 18:18:17 +0800 Subject: [PATCH] fix: Fix flutter texture rendering downscale on Android (#1696) --- .../main/cpp/iris_rtc_rendering_android.cc | 12 +- .../include/iris/iris_rtc_rendering_c.h | 2 +- .../agora/agora_rtc_ng/TextureRenderer.java | 61 +--- example/lib/components/config_override.dart | 4 + example/lib/examples/basic/index.dart | 8 + ...flutter_texture_android_internal_test.dart | 319 ++++++++++++++++++ lib/src/impl/agora_video_view_impl.dart | 65 ---- 7 files changed, 348 insertions(+), 123 deletions(-) create mode 100644 example/lib/examples/basic/join_channel_video/flutter_texture_android_internal_test.dart diff --git a/android/src/main/cpp/iris_rtc_rendering_android.cc b/android/src/main/cpp/iris_rtc_rendering_android.cc index d23671db8..9a5dd8e8d 100644 --- a/android/src/main/cpp/iris_rtc_rendering_android.cc +++ b/android/src/main/cpp/iris_rtc_rendering_android.cc @@ -303,6 +303,8 @@ class Texture2DRendering final : public RenderingOp { CHECK_GL_ERROR() glClearColor(0.0f, 0.0f, 0.0f, 1.0f); CHECK_GL_ERROR() + glViewport(0, 0, video_frame->width, video_frame->height); + CHECK_GL_ERROR() // Bind 2D texture glActiveTexture(GL_TEXTURE0); @@ -409,6 +411,8 @@ class OESTextureRendering final : public RenderingOp { CHECK_GL_ERROR() glClearColor(0.0f, 0.0f, 0.0f, 1.0f); CHECK_GL_ERROR() + glViewport(0, 0, video_frame->width, video_frame->height); + CHECK_GL_ERROR() // Bind external oes texture glActiveTexture(GL_TEXTURE0); @@ -528,6 +532,8 @@ class YUVRendering final : public RenderingOp { CHECK_GL_ERROR() glClearColor(0.0f, 0.0f, 0.0f, 1.0f); CHECK_GL_ERROR() + glViewport(0, 0, width, height); + CHECK_GL_ERROR() glEnableVertexAttribArray(aPositionLoc_); CHECK_GL_ERROR() @@ -538,8 +544,8 @@ class YUVRendering final : public RenderingOp { // Adjust the tex coords to avoid green edge issue float sFactor = 1.0f; - if (video_frame->width != video_frame->yStride) { - sFactor = (float) video_frame->width / (float) video_frame->yStride - 0.02f; + if (width != yStride) { + sFactor = (float) width / (float) yStride - 0.02f; } float fragment[] = {sFactor, 0.0f, 0.0f, 0.0f, sFactor, 1.0f, 0.0f, 1.0f}; @@ -704,6 +710,8 @@ class NativeTextureRenderer final strcpy(config.channelId, ""); } config.video_view_setup_mode = video_view_setup_mode; + config.observed_frame_position = agora::media::base::VIDEO_MODULE_POSITION::POSITION_POST_CAPTURER + | agora::media::base::VIDEO_MODULE_POSITION::POSITION_PRE_RENDERER; if (iris_rtc_rendering_) { delegate_id_ = diff --git a/android/src/main/cpp/third_party/include/iris/iris_rtc_rendering_c.h b/android/src/main/cpp/third_party/include/iris/iris_rtc_rendering_c.h index cf2997eb6..72590f448 100644 --- a/android/src/main/cpp/third_party/include/iris/iris_rtc_rendering_c.h +++ b/android/src/main/cpp/third_party/include/iris/iris_rtc_rendering_c.h @@ -46,7 +46,7 @@ typedef struct IrisRtcVideoFrameConfig { int video_view_setup_mode; /// int value of agora::media::base::VIDEO_MODULE_POSITION. - /// Default value is + /// Default value is /// `agora::media::base::VIDEO_MODULE_POSITION::POSITION_PRE_ENCODER | agora::media::base::VIDEO_MODULE_POSITION::POSITION_PRE_RENDERER` uint32_t observed_frame_position; } IrisRtcVideoFrameConfig; diff --git a/android/src/main/java/io/agora/agora_rtc_ng/TextureRenderer.java b/android/src/main/java/io/agora/agora_rtc_ng/TextureRenderer.java index 7c9a83401..f7f1d3329 100644 --- a/android/src/main/java/io/agora/agora_rtc_ng/TextureRenderer.java +++ b/android/src/main/java/io/agora/agora_rtc_ng/TextureRenderer.java @@ -5,12 +5,9 @@ import android.os.Looper; import android.view.Surface; -import androidx.annotation.NonNull; - import java.util.HashMap; import io.flutter.plugin.common.BinaryMessenger; -import io.flutter.plugin.common.MethodCall; import io.flutter.plugin.common.MethodChannel; import io.flutter.view.TextureRegistry; @@ -23,9 +20,6 @@ public class TextureRenderer { private SurfaceTexture flutterSurfaceTexture; private Surface renderSurface; - int width = 0; - int height = 0; - public TextureRenderer( TextureRegistry textureRegistry, BinaryMessenger binaryMessenger, @@ -42,33 +36,6 @@ public TextureRenderer( this.renderSurface = new Surface(this.flutterSurfaceTexture); this.methodChannel = new MethodChannel(binaryMessenger, "agora_rtc_engine/texture_render_" + flutterTexture.id()); - this.methodChannel.setMethodCallHandler(new MethodChannel.MethodCallHandler() { - @Override - public void onMethodCall(@NonNull MethodCall call, @NonNull MethodChannel.Result result) { - if (call.method.equals("setSizeNative")) { - if (call.arguments() == null) { - result.success(false); - return; - } - - int width = 0; - int height = 0; - if (call.hasArgument("width")) { - width = call.argument("width"); - } - if (call.hasArgument("height")) { - height = call.argument("height"); - } - - startRendering(width, height); - - result.success(true); - return; - } - - result.notImplemented(); - } - }); this.irisRenderer = new IrisRenderer( irisRtcRenderingHandle, @@ -79,6 +46,11 @@ public void onMethodCall(@NonNull MethodCall call, @NonNull MethodChannel.Result this.irisRenderer.setCallback(new IrisRenderer.Callback() { @Override public void onSizeChanged(int width, int height) { + final SurfaceTexture st = TextureRenderer.this.flutterSurfaceTexture; + if (null != st) { + st.setDefaultBufferSize(width, height); + } + handler.post(() -> { methodChannel.invokeMethod( "onSizeChanged", @@ -89,29 +61,8 @@ public void onSizeChanged(int width, int height) { }); } }); - } - - private void startRendering(int width, int height) { - if (width == 0 && height == 0) { - return; - } - - final SurfaceTexture st = TextureRenderer.this.flutterSurfaceTexture; - if (null == st) { - return; - } - - if (this.width != width || this.height != height) { - st.setDefaultBufferSize(width, height); - // Only call `irisRenderer.startRenderingToSurface` in the first time. - if (this.width == 0 && this.height == 0) { - this.irisRenderer.startRenderingToSurface(renderSurface); - } - - this.width = width; - this.height = height; - } + this.irisRenderer.startRenderingToSurface(renderSurface); } public long getTextureId() { diff --git a/example/lib/components/config_override.dart b/example/lib/components/config_override.dart index 9727c2a21..e57d0a112 100644 --- a/example/lib/components/config_override.dart +++ b/example/lib/components/config_override.dart @@ -45,4 +45,8 @@ class ExampleConfigOverride { void set(String name, String value) { _overridedConfig[name] = value; } + + /// Internal testing flag + bool get isInternalTesting => + const bool.fromEnvironment('INTERNAL_TESTING', defaultValue: false); } diff --git a/example/lib/examples/basic/index.dart b/example/lib/examples/basic/index.dart index 9f058bd65..205ba99c0 100644 --- a/example/lib/examples/basic/index.dart +++ b/example/lib/examples/basic/index.dart @@ -1,3 +1,5 @@ +import 'package:agora_rtc_engine_example/components/config_override.dart'; +import 'package:agora_rtc_engine_example/examples/basic/join_channel_video/flutter_texture_android_internal_test.dart'; import 'package:agora_rtc_engine_example/examples/basic/string_uid/string_uid.dart'; import 'join_channel_audio/join_channel_audio.dart'; @@ -9,5 +11,11 @@ final basic = [ {'name': 'Basic'}, {'name': 'JoinChannelAudio', 'widget': const JoinChannelAudio()}, {'name': 'JoinChannelVideo', 'widget': const JoinChannelVideo()}, + if (defaultTargetPlatform == TargetPlatform.android && + ExampleConfigOverride().isInternalTesting) + { + 'name': 'FlutterTextureAndroidTest', + 'widget': const FlutterTextureAndroidTest() + }, {'name': 'StringUid', 'widget': const StringUid()} ]; diff --git a/example/lib/examples/basic/join_channel_video/flutter_texture_android_internal_test.dart b/example/lib/examples/basic/join_channel_video/flutter_texture_android_internal_test.dart new file mode 100644 index 000000000..7c37981ea --- /dev/null +++ b/example/lib/examples/basic/join_channel_video/flutter_texture_android_internal_test.dart @@ -0,0 +1,319 @@ +import 'package:agora_rtc_engine/agora_rtc_engine.dart'; +import 'package:agora_rtc_engine_example/components/basic_video_configuration_widget.dart'; +import 'package:agora_rtc_engine_example/config/agora.config.dart' as config; +import 'package:agora_rtc_engine_example/components/example_actions_widget.dart'; +import 'package:agora_rtc_engine_example/components/log_sink.dart'; +import 'package:flutter/foundation.dart'; +import 'package:flutter/material.dart'; + +/// A case for internal testing only +class FlutterTextureAndroidTest extends StatefulWidget { + /// Construct the [FlutterTextureAndroidTest] + const FlutterTextureAndroidTest({Key? key}) : super(key: key); + + @override + State createState() => _State(); +} + +class _State extends State { + late final RtcEngine _engine; + + bool isJoined = false; + Set remoteUid = {}; + late TextEditingController _controller; + static const bool _isUseFlutterTexture = true; + bool _isAndroidTextureOes = false; + bool _isAndroidYuv = false; + bool _isAndroidTexture2D = false; + bool _isStartedPreview = false; + ChannelProfileType _channelProfileType = + ChannelProfileType.channelProfileLiveBroadcasting; + late final RtcEngineEventHandler _rtcEngineEventHandler; + + @override + void initState() { + super.initState(); + _controller = TextEditingController(text: config.channelId); + + _initEngine(); + } + + @override + void dispose() { + super.dispose(); + _dispose(); + } + + Future _dispose() async { + _engine.unregisterEventHandler(_rtcEngineEventHandler); + await _engine.leaveChannel(); + await _engine.release(); + } + + Future _initEngine() async { + _engine = createAgoraRtcEngine(); + await _engine.initialize(RtcEngineContext( + appId: config.appId, + )); + _rtcEngineEventHandler = RtcEngineEventHandler( + onError: (ErrorCodeType err, String msg) { + logSink.log('[onError] err: $err, msg: $msg'); + }, + onJoinChannelSuccess: (RtcConnection connection, int elapsed) { + logSink.log( + '[onJoinChannelSuccess] connection: ${connection.toJson()} elapsed: $elapsed'); + setState(() { + isJoined = true; + }); + }, + onUserJoined: (RtcConnection connection, int rUid, int elapsed) { + logSink.log( + '[onUserJoined] connection: ${connection.toJson()} remoteUid: $rUid elapsed: $elapsed'); + setState(() { + remoteUid.add(rUid); + }); + }, + onUserOffline: + (RtcConnection connection, int rUid, UserOfflineReasonType reason) { + logSink.log( + '[onUserOffline] connection: ${connection.toJson()} rUid: $rUid reason: $reason'); + setState(() { + remoteUid.removeWhere((element) => element == rUid); + }); + }, + onLeaveChannel: (RtcConnection connection, RtcStats stats) { + logSink.log( + '[onLeaveChannel] connection: ${connection.toJson()} stats: ${stats.toJson()}'); + setState(() { + isJoined = false; + remoteUid.clear(); + }); + }, + ); + + _engine.registerEventHandler(_rtcEngineEventHandler); + + await _engine.enableVideo(); + } + + Future _joinChannel() async { + await _engine.joinChannel( + token: config.token, + channelId: _controller.text, + uid: config.uid, + options: ChannelMediaOptions( + channelProfile: _channelProfileType, + clientRoleType: ClientRoleType.clientRoleBroadcaster, + ), + ); + } + + Future _leaveChannel() async { + await _engine.leaveChannel(); + } + + @override + Widget build(BuildContext context) { + return ExampleActionsWidget( + displayContentBuilder: (context, isLayoutHorizontal) { + return Stack( + children: [ + AgoraVideoView( + controller: VideoViewController( + rtcEngine: _engine, + canvas: const VideoCanvas(uid: 0), + useFlutterTexture: _isUseFlutterTexture, + ), + ), + Align( + alignment: Alignment.topLeft, + child: SingleChildScrollView( + scrollDirection: Axis.horizontal, + child: Row( + children: List.of(remoteUid.map( + (e) => SizedBox( + width: 120, + height: 120, + child: AgoraVideoView( + controller: VideoViewController.remote( + rtcEngine: _engine, + canvas: VideoCanvas(uid: e), + connection: + RtcConnection(channelId: _controller.text), + useFlutterTexture: _isUseFlutterTexture, + ), + ), + ), + )), + ), + ), + ) + ], + ); + }, + actionsBuilder: (context, isLayoutHorizontal) { + final channelProfileType = [ + ChannelProfileType.channelProfileLiveBroadcasting, + ChannelProfileType.channelProfileCommunication, + ]; + final items = channelProfileType + .map((e) => DropdownMenuItem( + child: Text( + e.toString().split('.')[1], + ), + value: e, + )) + .toList(); + + return Column( + mainAxisAlignment: MainAxisAlignment.start, + crossAxisAlignment: CrossAxisAlignment.start, + mainAxisSize: MainAxisSize.min, + children: [ + TextField( + controller: _controller, + decoration: const InputDecoration(hintText: 'Channel ID'), + ), + if (!kIsWeb && + (defaultTargetPlatform == TargetPlatform.android || + defaultTargetPlatform == TargetPlatform.iOS)) + Row( + mainAxisSize: MainAxisSize.min, + mainAxisAlignment: MainAxisAlignment.start, + children: [ + Column( + mainAxisAlignment: MainAxisAlignment.start, + crossAxisAlignment: CrossAxisAlignment.start, + mainAxisSize: MainAxisSize.min, + children: [ + Column( + mainAxisAlignment: MainAxisAlignment.start, + crossAxisAlignment: CrossAxisAlignment.start, + mainAxisSize: MainAxisSize.min, + children: [ + const Text('Video Format: textureoes'), + Switch( + value: _isAndroidTextureOes, + onChanged: isJoined + ? null + : (changed) { + setState(() { + _isAndroidTextureOes = changed; + }); + + if (_isAndroidTextureOes) { + _engine.setParameters( + '{"che.video.android_texture.copy_enable":false}'); + } + }, + ), + const Text('Video Format: textureo2d'), + Switch( + value: _isAndroidTexture2D, + onChanged: isJoined + ? null + : (changed) { + setState(() { + _isAndroidTexture2D = changed; + }); + if (_isAndroidTexture2D) { + _engine.setParameters( + '{"che.video.android_texture.copy_enable":true}'); + } + }, + ), + const Text('Video Format: yuv'), + Switch( + value: _isAndroidYuv, + onChanged: isJoined + ? null + : (changed) { + setState(() { + _isAndroidYuv = changed; + }); + if (_isAndroidYuv) { + _engine.setParameters( + '{"che.video.android_camera_output_type":0}'); + } + }, + ), + ], + ), + ], + ), + ], + ), + const SizedBox( + height: 20, + ), + const Text('Channel Profile: '), + DropdownButton( + items: items, + value: _channelProfileType, + onChanged: isJoined + ? null + : (v) { + setState(() { + _channelProfileType = v!; + }); + }, + ), + const SizedBox( + height: 20, + ), + BasicVideoConfigurationWidget( + rtcEngine: _engine, + title: 'Video Encoder Configuration', + setConfigButtonText: const Text( + 'setVideoEncoderConfiguration', + style: TextStyle(fontSize: 10), + ), + onConfigChanged: (width, height, frameRate, bitrate) { + _engine.setVideoEncoderConfiguration(VideoEncoderConfiguration( + dimensions: VideoDimensions(width: width, height: height), + frameRate: frameRate, + bitrate: bitrate, + )); + }, + ), + const SizedBox( + height: 20, + ), + Row( + children: [ + Expanded( + flex: 1, + child: ElevatedButton( + onPressed: () { + if (_isStartedPreview) { + _engine.stopPreview(); + } else { + _engine.startPreview(); + } + setState(() { + _isStartedPreview = !_isStartedPreview; + }); + }, + child: + Text('${_isStartedPreview ? 'Stop' : 'Start'} Preview'), + ), + ) + ], + ), + Row( + children: [ + Expanded( + flex: 1, + child: ElevatedButton( + onPressed: isJoined ? _leaveChannel : _joinChannel, + child: Text('${isJoined ? 'Leave' : 'Join'} channel'), + ), + ) + ], + ), + ], + ); + }, + ); + } +} diff --git a/lib/src/impl/agora_video_view_impl.dart b/lib/src/impl/agora_video_view_impl.dart index c4543eeda..034f9eea4 100644 --- a/lib/src/impl/agora_video_view_impl.dart +++ b/lib/src/impl/agora_video_view_impl.dart @@ -465,16 +465,6 @@ class _AgoraRtcRenderTextureState extends State return child; } - Future _setSizeNative(Size size, Offset position) async { - assert(defaultTargetPlatform == TargetPlatform.android); - // Call `SurfaceTexture.setDefaultBufferSize` on Android， or the video will be - // black screen - await methodChannel!.invokeMethod('setSizeNative', { - 'width': size.width.toInt(), - 'height': size.height.toInt(), - }); - } - @override Widget build(BuildContext context) { Widget result = const SizedBox.expand(); @@ -508,63 +498,8 @@ class _AgoraRtcRenderTextureState extends State result = _applyRenderMode(RenderModeType.renderModeFit, result); } } - - // Only need to size in native side on Android - if (!kIsWeb && defaultTargetPlatform == TargetPlatform.android) { - result = _SizeChangedAwareWidget( - onChange: (size) { - _setSizeNative(size, Offset.zero); - }, - child: result, - ); - } } return result; } } - -typedef _OnWidgetSizeChange = void Function(Size size); - -class _SizeChangedAwareRenderObject extends RenderProxyBox { - Size? oldSize; - _OnWidgetSizeChange onChange; - - _SizeChangedAwareRenderObject(this.onChange); - - @override - void performLayout() { - super.performLayout(); - - Size newSize = child!.size; - if (oldSize == newSize) return; - - oldSize = newSize; - // Compatible with Flutter SDK 2.10.x - // ignore: invalid_null_aware_operator - SchedulerBinding.instance?.addPostFrameCallback((_) { - onChange(newSize); - }); - } -} - -class _SizeChangedAwareWidget extends SingleChildRenderObjectWidget { - final _OnWidgetSizeChange onChange; - - const _SizeChangedAwareWidget({ - Key? key, - required this.onChange, - required Widget child, - }) : super(key: key, child: child); - - @override - RenderObject createRenderObject(BuildContext context) { - return _SizeChangedAwareRenderObject(onChange); - } - - @override - void updateRenderObject(BuildContext context, - covariant _SizeChangedAwareRenderObject renderObject) { - renderObject.onChange = onChange; - } -}