diff --git a/shared/darwin/TextureRenderer.mm b/shared/darwin/TextureRenderer.mm index 00e3a4bc8..efe8e6f61 100644 --- a/shared/darwin/TextureRenderer.mm +++ b/shared/darwin/TextureRenderer.mm @@ -1,8 +1,9 @@ -#import #import "TextureRenderer.h" -#import -#import + #import +#import +#import +#import #import @@ -12,62 +13,72 @@ @interface TextureRender () @property(nonatomic, weak) NSObject *textureRegistry; @property(nonatomic, strong) FlutterMethodChannel *channel; -@property(nonatomic) CVPixelBufferRef buffer_cache; -@property(nonatomic, strong) dispatch_semaphore_t lock; @property(nonatomic) agora::iris::IrisRtcRendering *irisRtcRendering; @property(nonatomic, assign) int delegateId; -@property(nonatomic, assign) BOOL isDirtyBuffer; + +/// Tracks the latest pixel buffer sent from AVFoundation's sample buffer +/// delegate callback. Used to deliver the latest pixel buffer to the flutter +/// engine via the `copyPixelBuffer` API. +@property(readwrite, nonatomic) CVPixelBufferRef latestPixelBuffer; +/// The queue on which `latestPixelBuffer` property is accessed. +@property(strong, nonatomic) dispatch_queue_t pixelBufferSynchronizationQueue; @end namespace { class RendererDelegate : public agora::iris::VideoFrameObserverDelegate { public: - RendererDelegate(void *renderer) : renderer_(renderer), pre_width_(0), pre_height_(0) { } - + RendererDelegate(void *renderer) + : renderer_(renderer), pre_width_(0), pre_height_(0) {} + void OnVideoFrameReceived(const void *videoFrame, - const IrisRtcVideoFrameConfig &config, bool resize) override { + const IrisRtcVideoFrameConfig &config, + bool resize) override { @autoreleasepool { - TextureRender *renderer = (__bridge TextureRender *)renderer_; - - agora::media::base::VideoFrame *vf = (agora::media::base::VideoFrame *)videoFrame; - - if (vf->width == 0 || vf->height == 0) { - return; - } - - int tmpWidth = vf->width; - int tmpHeight = vf->height; - bool is_resize = pre_width_ != tmpWidth || pre_height_ != tmpHeight; - pre_width_ = tmpWidth; - pre_height_ = tmpHeight; - - CVPixelBufferRef _Nullable pixelBuffer = reinterpret_cast(vf->pixelBuffer); - if (pixelBuffer) { - if (is_resize) { - dispatch_async(dispatch_get_main_queue(), ^{ - [renderer.channel invokeMethod:@"onSizeChanged" - arguments:@{@"width": @(tmpWidth), - @"height": @(tmpHeight)}]; - }); - } - - dispatch_semaphore_wait(renderer.lock, DISPATCH_TIME_FOREVER); - if (!renderer.isDirtyBuffer) { - // Ensure the previous retained `CVPixelBufferRef` be released. - if (renderer.buffer_cache) { - CVBufferRelease(renderer.buffer_cache); - } - - renderer.buffer_cache = CVPixelBufferRetain(pixelBuffer); - renderer.isDirtyBuffer = YES; - } - dispatch_semaphore_signal(renderer.lock); - - if (renderer.textureRegistry && renderer.isDirtyBuffer) { - [renderer.textureRegistry textureFrameAvailable:renderer.textureId]; - } - } + TextureRender *renderer = (__bridge TextureRender *)renderer_; + + agora::media::base::VideoFrame *vf = + (agora::media::base::VideoFrame *)videoFrame; + + if (vf->width == 0 || vf->height == 0) { + return; + } + + CVPixelBufferRef _Nullable pixelBuffer = + reinterpret_cast(vf->pixelBuffer); + if (!pixelBuffer) { + return; + } + + if (pre_width_ != vf->width || pre_height_ != vf->height) { + pre_width_ = vf->width; + pre_height_ = vf->height; + dispatch_sync(dispatch_get_main_queue(), ^{ + [renderer.channel invokeMethod:@"onSizeChanged" + arguments:@{ + @"width" : @(vf->width), + @"height" : @(vf->height) + }]; + }); + } + + __block CVPixelBufferRef previousPixelBuffer = nil; + // Use `dispatch_sync` to avoid unnecessary context switch under common + // non-contest scenarios; + // Under rare contest scenarios, it will not block for too long since + // the critical section is quite lightweight. + dispatch_sync(renderer.pixelBufferSynchronizationQueue, ^{ + previousPixelBuffer = renderer.latestPixelBuffer; + renderer.latestPixelBuffer = CVPixelBufferRetain(pixelBuffer); + }); + if (previousPixelBuffer) { + CFRelease(previousPixelBuffer); + } + + // notify new frame available on main thread + dispatch_sync(dispatch_get_main_queue(), ^{ + [renderer.textureRegistry textureFrameAvailable:renderer.textureId]; + }); } } @@ -76,7 +87,7 @@ void OnVideoFrameReceived(const void *videoFrame, int pre_width_; int pre_height_; }; -} +} // namespace @interface TextureRender () @@ -86,75 +97,87 @@ @interface TextureRender () @implementation TextureRender -- (instancetype) initWithTextureRegistry:(NSObject *)textureRegistry - messenger:(NSObject *)messenger - irisRtcRenderingHandle:(void *)irisRtcRenderingHandle { - self = [super init]; - if (self) { - self.textureRegistry = textureRegistry; - self.irisRtcRendering = (agora::iris::IrisRtcRendering *)irisRtcRenderingHandle; - self.textureId = [self.textureRegistry registerTexture:self]; - self.channel = [FlutterMethodChannel - methodChannelWithName: - [NSString stringWithFormat:@"agora_rtc_engine/texture_render_%lld", - self.textureId] - binaryMessenger:messenger]; - - self.lock = dispatch_semaphore_create(1); - - self.delegate = new ::RendererDelegate((__bridge void *)self); - self.buffer_cache = NULL; - self.isDirtyBuffer = YES; - } - return self; +- (instancetype) + initWithTextureRegistry:(NSObject *)textureRegistry + messenger:(NSObject *)messenger + irisRtcRenderingHandle:(void *)irisRtcRenderingHandle { + self = [super init]; + if (self) { + self.textureRegistry = textureRegistry; + self.irisRtcRendering = + (agora::iris::IrisRtcRendering *)irisRtcRenderingHandle; + self.textureId = [self.textureRegistry registerTexture:self]; + self.channel = [FlutterMethodChannel + methodChannelWithName: + [NSString stringWithFormat:@"agora_rtc_engine/texture_render_%lld", + self.textureId] + binaryMessenger:messenger]; + + self.delegate = new ::RendererDelegate((__bridge void *)self); + self.latestPixelBuffer = nil; + self.pixelBufferSynchronizationQueue = dispatch_queue_create( + [[NSString stringWithFormat:@"io.agora.flutter.render_%lld", _textureId] + UTF8String], + nil); + } + return self; } -- (void)updateData:(NSNumber *)uid channelId:(NSString *)channelId videoSourceType:(NSNumber *)videoSourceType videoViewSetupMode:(NSNumber *)videoViewSetupMode { - IrisRtcVideoFrameConfig config; - config.video_frame_format = agora::media::base::VIDEO_PIXEL_FORMAT::VIDEO_CVPIXEL_NV12; - config.uid = [uid unsignedIntValue]; - config.video_source_type = [videoSourceType intValue]; - if (channelId && (NSNull *)channelId != [NSNull null]) { - strcpy(config.channelId, [channelId UTF8String]); - } else { - strcpy(config.channelId, ""); - } - config.video_view_setup_mode = [videoViewSetupMode intValue]; - config.observed_frame_position = agora::media::base::VIDEO_MODULE_POSITION::POSITION_POST_CAPTURER | agora::media::base::VIDEO_MODULE_POSITION::POSITION_PRE_RENDERER; - - self.delegateId = self.irisRtcRendering->AddVideoFrameObserverDelegate(config, self.delegate); +- (void)updateData:(NSNumber *)uid + channelId:(NSString *)channelId + videoSourceType:(NSNumber *)videoSourceType + videoViewSetupMode:(NSNumber *)videoViewSetupMode { + IrisRtcVideoFrameConfig config; + config.video_frame_format = + agora::media::base::VIDEO_PIXEL_FORMAT::VIDEO_CVPIXEL_NV12; + config.uid = [uid unsignedIntValue]; + config.video_source_type = [videoSourceType intValue]; + if (channelId && (NSNull *)channelId != [NSNull null]) { + strcpy(config.channelId, [channelId UTF8String]); + } else { + strcpy(config.channelId, ""); + } + config.video_view_setup_mode = [videoViewSetupMode intValue]; + config.observed_frame_position = + agora::media::base::VIDEO_MODULE_POSITION::POSITION_POST_CAPTURER | + agora::media::base::VIDEO_MODULE_POSITION::POSITION_PRE_RENDERER; + + self.delegateId = self.irisRtcRendering->AddVideoFrameObserverDelegate( + config, self.delegate); +} + +- (CVPixelBufferRef _Nullable)copyPixelBuffer { + __block CVPixelBufferRef pixelBuffer = nil; + // Use `dispatch_sync` because `copyPixelBuffer` API requires synchronous + // return. + dispatch_sync(self.pixelBufferSynchronizationQueue, ^{ + // No need weak self because it's dispatch_sync. + pixelBuffer = self.latestPixelBuffer; + self.latestPixelBuffer = nil; + }); + return pixelBuffer; } - (void)dispose { if (self.irisRtcRendering) { - self.irisRtcRendering->RemoveVideoFrameObserverDelegate(self.delegateId); - self.irisRtcRendering = NULL; + self.irisRtcRendering->RemoveVideoFrameObserverDelegate(self.delegateId); + self.irisRtcRendering = nil; } if (self.delegate) { - delete self.delegate; - self.delegate = NULL; + delete self.delegate; + self.delegate = nil; } if (self.textureRegistry) { - [self.textureRegistry unregisterTexture:self.textureId]; - self.textureRegistry = NULL; - } - if (self.buffer_cache && self.isDirtyBuffer) { - CVPixelBufferRelease(self.buffer_cache); - self.buffer_cache = NULL; + [self.textureRegistry unregisterTexture:self.textureId]; + self.textureRegistry = nil; } } -- (CVPixelBufferRef _Nullable)copyPixelBuffer { - dispatch_semaphore_wait(self.lock, DISPATCH_TIME_FOREVER); - CVPixelBufferRef buffer_temp = CVPixelBufferRetain(self.buffer_cache); - self.isDirtyBuffer = NO; - dispatch_semaphore_signal(self.lock); - - return buffer_temp; -} - -- (void)onTextureUnregistered:(NSObject *)texture { +- (void)dealloc { + if (self.latestPixelBuffer) { + CVPixelBufferRelease(self.latestPixelBuffer); + self.latestPixelBuffer = nil; + } } @end -