Skip to content

Commit

Permalink
fix: crash in texture renderer for ios
Browse files Browse the repository at this point in the history
  • Loading branch information
@peilinok
peilinok committed Dec 31, 2024
1 parent 828fff3 commit 432baa2
Showing 1 changed file with 129 additions and 106 deletions.
235 changes: 129 additions & 106 deletions shared/darwin/TextureRenderer.mm
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,9 @@
#import <Foundation/Foundation.h>
#import "TextureRenderer.h"
#import <AgoraRtcWrapper/iris_rtc_rendering_cxx.h>
#import <AgoraRtcKit/IAgoraMediaEngine.h>

#import <AgoraRtcKit/AgoraMediaBase.h>
#import <AgoraRtcKit/IAgoraMediaEngine.h>
#import <AgoraRtcWrapper/iris_rtc_rendering_cxx.h>
#import <Foundation/Foundation.h>

#import <stdio.h>

Expand All @@ -12,62 +13,72 @@ @interface TextureRender ()

@property(nonatomic, weak) NSObject<FlutterTextureRegistry> *textureRegistry;
@property(nonatomic, strong) FlutterMethodChannel *channel;
@property(nonatomic) CVPixelBufferRef buffer_cache;
@property(nonatomic, strong) dispatch_semaphore_t lock;
@property(nonatomic) agora::iris::IrisRtcRendering *irisRtcRendering;
@property(nonatomic, assign) int delegateId;
@property(nonatomic, assign) BOOL isDirtyBuffer;

/// Tracks the latest pixel buffer sent from AVFoundation's sample buffer
/// delegate callback. Used to deliver the latest pixel buffer to the flutter
/// engine via the `copyPixelBuffer` API.
@property(readwrite, nonatomic) CVPixelBufferRef latestPixelBuffer;
/// The queue on which `latestPixelBuffer` property is accessed.
@property(strong, nonatomic) dispatch_queue_t pixelBufferSynchronizationQueue;

@end

namespace {
class RendererDelegate : public agora::iris::VideoFrameObserverDelegate {
public:
RendererDelegate(void *renderer) : renderer_(renderer), pre_width_(0), pre_height_(0) { }

RendererDelegate(void *renderer)
: renderer_(renderer), pre_width_(0), pre_height_(0) {}

void OnVideoFrameReceived(const void *videoFrame,
const IrisRtcVideoFrameConfig &config, bool resize) override {
const IrisRtcVideoFrameConfig &config,
bool resize) override {
@autoreleasepool {
TextureRender *renderer = (__bridge TextureRender *)renderer_;

agora::media::base::VideoFrame *vf = (agora::media::base::VideoFrame *)videoFrame;

if (vf->width == 0 || vf->height == 0) {
return;
}

int tmpWidth = vf->width;
int tmpHeight = vf->height;
bool is_resize = pre_width_ != tmpWidth || pre_height_ != tmpHeight;
pre_width_ = tmpWidth;
pre_height_ = tmpHeight;

CVPixelBufferRef _Nullable pixelBuffer = reinterpret_cast<CVPixelBufferRef>(vf->pixelBuffer);
if (pixelBuffer) {
if (is_resize) {
dispatch_async(dispatch_get_main_queue(), ^{
[renderer.channel invokeMethod:@"onSizeChanged"
arguments:@{@"width": @(tmpWidth),
@"height": @(tmpHeight)}];
});
}

dispatch_semaphore_wait(renderer.lock, DISPATCH_TIME_FOREVER);
if (!renderer.isDirtyBuffer) {
// Ensure the previous retained `CVPixelBufferRef` be released.
if (renderer.buffer_cache) {
CVBufferRelease(renderer.buffer_cache);
}

renderer.buffer_cache = CVPixelBufferRetain(pixelBuffer);
renderer.isDirtyBuffer = YES;
}
dispatch_semaphore_signal(renderer.lock);

if (renderer.textureRegistry && renderer.isDirtyBuffer) {
[renderer.textureRegistry textureFrameAvailable:renderer.textureId];
}
}
TextureRender *renderer = (__bridge TextureRender *)renderer_;

agora::media::base::VideoFrame *vf =
(agora::media::base::VideoFrame *)videoFrame;

if (vf->width == 0 || vf->height == 0) {
return;
}

CVPixelBufferRef _Nullable pixelBuffer =
reinterpret_cast<CVPixelBufferRef>(vf->pixelBuffer);
if (!pixelBuffer) {
return;
}

if (pre_width_ != vf->width || pre_height_ != vf->height) {
pre_width_ = vf->width;
pre_height_ = vf->height;
dispatch_sync(dispatch_get_main_queue(), ^{
[renderer.channel invokeMethod:@"onSizeChanged"
arguments:@{
@"width" : @(vf->width),
@"height" : @(vf->height)
}];
});
}

__block CVPixelBufferRef previousPixelBuffer = nil;
// Use `dispatch_sync` to avoid unnecessary context switch under common
// non-contest scenarios;
// Under rare contest scenarios, it will not block for too long since
// the critical section is quite lightweight.
dispatch_sync(renderer.pixelBufferSynchronizationQueue, ^{
previousPixelBuffer = renderer.latestPixelBuffer;
renderer.latestPixelBuffer = CVPixelBufferRetain(pixelBuffer);
});
if (previousPixelBuffer) {
CFRelease(previousPixelBuffer);
}

// notify new frame available on main thread
dispatch_sync(dispatch_get_main_queue(), ^{
[renderer.textureRegistry textureFrameAvailable:renderer.textureId];
});
}
}

Expand All @@ -76,7 +87,7 @@ void OnVideoFrameReceived(const void *videoFrame,
int pre_width_;
int pre_height_;
};
}
} // namespace

@interface TextureRender ()

Expand All @@ -86,75 +97,87 @@ @interface TextureRender ()

@implementation TextureRender

- (instancetype) initWithTextureRegistry:(NSObject<FlutterTextureRegistry> *)textureRegistry
messenger:(NSObject<FlutterBinaryMessenger> *)messenger
irisRtcRenderingHandle:(void *)irisRtcRenderingHandle {
self = [super init];
if (self) {
self.textureRegistry = textureRegistry;
self.irisRtcRendering = (agora::iris::IrisRtcRendering *)irisRtcRenderingHandle;
self.textureId = [self.textureRegistry registerTexture:self];
self.channel = [FlutterMethodChannel
methodChannelWithName:
[NSString stringWithFormat:@"agora_rtc_engine/texture_render_%lld",
self.textureId]
binaryMessenger:messenger];

self.lock = dispatch_semaphore_create(1);

self.delegate = new ::RendererDelegate((__bridge void *)self);
self.buffer_cache = NULL;
self.isDirtyBuffer = YES;
}
return self;
- (instancetype)
initWithTextureRegistry:(NSObject<FlutterTextureRegistry> *)textureRegistry
messenger:(NSObject<FlutterBinaryMessenger> *)messenger
irisRtcRenderingHandle:(void *)irisRtcRenderingHandle {
self = [super init];
if (self) {
self.textureRegistry = textureRegistry;
self.irisRtcRendering =
(agora::iris::IrisRtcRendering *)irisRtcRenderingHandle;
self.textureId = [self.textureRegistry registerTexture:self];
self.channel = [FlutterMethodChannel
methodChannelWithName:
[NSString stringWithFormat:@"agora_rtc_engine/texture_render_%lld",
self.textureId]
binaryMessenger:messenger];

self.delegate = new ::RendererDelegate((__bridge void *)self);
self.latestPixelBuffer = nil;
self.pixelBufferSynchronizationQueue = dispatch_queue_create(
[[NSString stringWithFormat:@"io.agora.flutter.render_%lld", _textureId]
UTF8String],
nil);
}
return self;
}

- (void)updateData:(NSNumber *)uid channelId:(NSString *)channelId videoSourceType:(NSNumber *)videoSourceType videoViewSetupMode:(NSNumber *)videoViewSetupMode {
IrisRtcVideoFrameConfig config;
config.video_frame_format = agora::media::base::VIDEO_PIXEL_FORMAT::VIDEO_CVPIXEL_NV12;
config.uid = [uid unsignedIntValue];
config.video_source_type = [videoSourceType intValue];
if (channelId && (NSNull *)channelId != [NSNull null]) {
strcpy(config.channelId, [channelId UTF8String]);
} else {
strcpy(config.channelId, "");
}
config.video_view_setup_mode = [videoViewSetupMode intValue];
config.observed_frame_position = agora::media::base::VIDEO_MODULE_POSITION::POSITION_POST_CAPTURER | agora::media::base::VIDEO_MODULE_POSITION::POSITION_PRE_RENDERER;

self.delegateId = self.irisRtcRendering->AddVideoFrameObserverDelegate(config, self.delegate);
- (void)updateData:(NSNumber *)uid
channelId:(NSString *)channelId
videoSourceType:(NSNumber *)videoSourceType
videoViewSetupMode:(NSNumber *)videoViewSetupMode {
IrisRtcVideoFrameConfig config;
config.video_frame_format =
agora::media::base::VIDEO_PIXEL_FORMAT::VIDEO_CVPIXEL_NV12;
config.uid = [uid unsignedIntValue];
config.video_source_type = [videoSourceType intValue];
if (channelId && (NSNull *)channelId != [NSNull null]) {
strcpy(config.channelId, [channelId UTF8String]);
} else {
strcpy(config.channelId, "");
}
config.video_view_setup_mode = [videoViewSetupMode intValue];
config.observed_frame_position =
agora::media::base::VIDEO_MODULE_POSITION::POSITION_POST_CAPTURER |
agora::media::base::VIDEO_MODULE_POSITION::POSITION_PRE_RENDERER;

self.delegateId = self.irisRtcRendering->AddVideoFrameObserverDelegate(
config, self.delegate);
}

- (CVPixelBufferRef _Nullable)copyPixelBuffer {
__block CVPixelBufferRef pixelBuffer = nil;
// Use `dispatch_sync` because `copyPixelBuffer` API requires synchronous
// return.
dispatch_sync(self.pixelBufferSynchronizationQueue, ^{
// No need weak self because it's dispatch_sync.
pixelBuffer = self.latestPixelBuffer;
self.latestPixelBuffer = nil;
});
return pixelBuffer;
}

- (void)dispose {
if (self.irisRtcRendering) {
self.irisRtcRendering->RemoveVideoFrameObserverDelegate(self.delegateId);
self.irisRtcRendering = NULL;
self.irisRtcRendering->RemoveVideoFrameObserverDelegate(self.delegateId);
self.irisRtcRendering = nil;
}
if (self.delegate) {
delete self.delegate;
self.delegate = NULL;
delete self.delegate;
self.delegate = nil;
}
if (self.textureRegistry) {
[self.textureRegistry unregisterTexture:self.textureId];
self.textureRegistry = NULL;
}
if (self.buffer_cache && self.isDirtyBuffer) {
CVPixelBufferRelease(self.buffer_cache);
self.buffer_cache = NULL;
[self.textureRegistry unregisterTexture:self.textureId];
self.textureRegistry = nil;
}
}

- (CVPixelBufferRef _Nullable)copyPixelBuffer {
dispatch_semaphore_wait(self.lock, DISPATCH_TIME_FOREVER);
CVPixelBufferRef buffer_temp = CVPixelBufferRetain(self.buffer_cache);
self.isDirtyBuffer = NO;
dispatch_semaphore_signal(self.lock);

return buffer_temp;
}

- (void)onTextureUnregistered:(NSObject<FlutterTexture> *)texture {
- (void)dealloc {
if (self.latestPixelBuffer) {
CVPixelBufferRelease(self.latestPixelBuffer);
self.latestPixelBuffer = nil;
}
}

@end

0 comments on commit 432baa2

Please sign in to comment.