diff --git a/shared/darwin/TextureRenderer.mm b/shared/darwin/TextureRenderer.mm index efe8e6f61..00e3a4bc8 100644 --- a/shared/darwin/TextureRenderer.mm +++ b/shared/darwin/TextureRenderer.mm @@ -1,9 +1,8 @@ +#import #import "TextureRenderer.h" - -#import -#import #import -#import +#import +#import #import @@ -13,72 +12,62 @@ @interface TextureRender () @property(nonatomic, weak) NSObject *textureRegistry; @property(nonatomic, strong) FlutterMethodChannel *channel; +@property(nonatomic) CVPixelBufferRef buffer_cache; +@property(nonatomic, strong) dispatch_semaphore_t lock; @property(nonatomic) agora::iris::IrisRtcRendering *irisRtcRendering; @property(nonatomic, assign) int delegateId; - -/// Tracks the latest pixel buffer sent from AVFoundation's sample buffer -/// delegate callback. Used to deliver the latest pixel buffer to the flutter -/// engine via the `copyPixelBuffer` API. -@property(readwrite, nonatomic) CVPixelBufferRef latestPixelBuffer; -/// The queue on which `latestPixelBuffer` property is accessed. -@property(strong, nonatomic) dispatch_queue_t pixelBufferSynchronizationQueue; +@property(nonatomic, assign) BOOL isDirtyBuffer; @end namespace { class RendererDelegate : public agora::iris::VideoFrameObserverDelegate { public: - RendererDelegate(void *renderer) - : renderer_(renderer), pre_width_(0), pre_height_(0) {} - + RendererDelegate(void *renderer) : renderer_(renderer), pre_width_(0), pre_height_(0) { } + void OnVideoFrameReceived(const void *videoFrame, - const IrisRtcVideoFrameConfig &config, - bool resize) override { + const IrisRtcVideoFrameConfig &config, bool resize) override { @autoreleasepool { - TextureRender *renderer = (__bridge TextureRender *)renderer_; - - agora::media::base::VideoFrame *vf = - (agora::media::base::VideoFrame *)videoFrame; - - if (vf->width == 0 || vf->height == 0) { - return; - } - - CVPixelBufferRef _Nullable pixelBuffer = - reinterpret_cast(vf->pixelBuffer); - if (!pixelBuffer) { - return; - } - - if (pre_width_ != vf->width || pre_height_ != vf->height) { - pre_width_ = vf->width; - pre_height_ = vf->height; - dispatch_sync(dispatch_get_main_queue(), ^{ - [renderer.channel invokeMethod:@"onSizeChanged" - arguments:@{ - @"width" : @(vf->width), - @"height" : @(vf->height) - }]; - }); - } - - __block CVPixelBufferRef previousPixelBuffer = nil; - // Use `dispatch_sync` to avoid unnecessary context switch under common - // non-contest scenarios; - // Under rare contest scenarios, it will not block for too long since - // the critical section is quite lightweight. - dispatch_sync(renderer.pixelBufferSynchronizationQueue, ^{ - previousPixelBuffer = renderer.latestPixelBuffer; - renderer.latestPixelBuffer = CVPixelBufferRetain(pixelBuffer); - }); - if (previousPixelBuffer) { - CFRelease(previousPixelBuffer); - } - - // notify new frame available on main thread - dispatch_sync(dispatch_get_main_queue(), ^{ - [renderer.textureRegistry textureFrameAvailable:renderer.textureId]; - }); + TextureRender *renderer = (__bridge TextureRender *)renderer_; + + agora::media::base::VideoFrame *vf = (agora::media::base::VideoFrame *)videoFrame; + + if (vf->width == 0 || vf->height == 0) { + return; + } + + int tmpWidth = vf->width; + int tmpHeight = vf->height; + bool is_resize = pre_width_ != tmpWidth || pre_height_ != tmpHeight; + pre_width_ = tmpWidth; + pre_height_ = tmpHeight; + + CVPixelBufferRef _Nullable pixelBuffer = reinterpret_cast(vf->pixelBuffer); + if (pixelBuffer) { + if (is_resize) { + dispatch_async(dispatch_get_main_queue(), ^{ + [renderer.channel invokeMethod:@"onSizeChanged" + arguments:@{@"width": @(tmpWidth), + @"height": @(tmpHeight)}]; + }); + } + + dispatch_semaphore_wait(renderer.lock, DISPATCH_TIME_FOREVER); + if (!renderer.isDirtyBuffer) { + // Ensure the previous retained `CVPixelBufferRef` be released. + if (renderer.buffer_cache) { + CVBufferRelease(renderer.buffer_cache); + } + + renderer.buffer_cache = CVPixelBufferRetain(pixelBuffer); + renderer.isDirtyBuffer = YES; + } + dispatch_semaphore_signal(renderer.lock); + + if (renderer.textureRegistry && renderer.isDirtyBuffer) { + [renderer.textureRegistry textureFrameAvailable:renderer.textureId]; + } + } } } @@ -87,7 +76,7 @@ void OnVideoFrameReceived(const void *videoFrame, int pre_width_; int pre_height_; }; -} // namespace +} @interface TextureRender () @@ -97,87 +86,75 @@ @interface TextureRender () @implementation TextureRender -- (instancetype) - initWithTextureRegistry:(NSObject *)textureRegistry - messenger:(NSObject *)messenger - irisRtcRenderingHandle:(void *)irisRtcRenderingHandle { - self = [super init]; - if (self) { - self.textureRegistry = textureRegistry; - self.irisRtcRendering = - (agora::iris::IrisRtcRendering *)irisRtcRenderingHandle; - self.textureId = [self.textureRegistry registerTexture:self]; - self.channel = [FlutterMethodChannel - methodChannelWithName: - [NSString stringWithFormat:@"agora_rtc_engine/texture_render_%lld", - self.textureId] - binaryMessenger:messenger]; - - self.delegate = new ::RendererDelegate((__bridge void *)self); - self.latestPixelBuffer = nil; - self.pixelBufferSynchronizationQueue = dispatch_queue_create( - [[NSString stringWithFormat:@"io.agora.flutter.render_%lld", _textureId] - UTF8String], - nil); - } - return self; -} - -- (void)updateData:(NSNumber *)uid - channelId:(NSString *)channelId - videoSourceType:(NSNumber *)videoSourceType - videoViewSetupMode:(NSNumber *)videoViewSetupMode { - IrisRtcVideoFrameConfig config; - config.video_frame_format = - agora::media::base::VIDEO_PIXEL_FORMAT::VIDEO_CVPIXEL_NV12; - config.uid = [uid unsignedIntValue]; - config.video_source_type = [videoSourceType intValue]; - if (channelId && (NSNull *)channelId != [NSNull null]) { - strcpy(config.channelId, [channelId UTF8String]); - } else { - strcpy(config.channelId, ""); - } - config.video_view_setup_mode = [videoViewSetupMode intValue]; - config.observed_frame_position = - agora::media::base::VIDEO_MODULE_POSITION::POSITION_POST_CAPTURER | - agora::media::base::VIDEO_MODULE_POSITION::POSITION_PRE_RENDERER; - - self.delegateId = self.irisRtcRendering->AddVideoFrameObserverDelegate( - config, self.delegate); +- (instancetype) initWithTextureRegistry:(NSObject *)textureRegistry + messenger:(NSObject *)messenger + irisRtcRenderingHandle:(void *)irisRtcRenderingHandle { + self = [super init]; + if (self) { + self.textureRegistry = textureRegistry; + self.irisRtcRendering = (agora::iris::IrisRtcRendering *)irisRtcRenderingHandle; + self.textureId = [self.textureRegistry registerTexture:self]; + self.channel = [FlutterMethodChannel + methodChannelWithName: + [NSString stringWithFormat:@"agora_rtc_engine/texture_render_%lld", + self.textureId] + binaryMessenger:messenger]; + + self.lock = dispatch_semaphore_create(1); + + self.delegate = new ::RendererDelegate((__bridge void *)self); + self.buffer_cache = NULL; + self.isDirtyBuffer = YES; + } + return self; } -- (CVPixelBufferRef _Nullable)copyPixelBuffer { - __block CVPixelBufferRef pixelBuffer = nil; - // Use `dispatch_sync` because `copyPixelBuffer` API requires synchronous - // return. - dispatch_sync(self.pixelBufferSynchronizationQueue, ^{ - // No need weak self because it's dispatch_sync. - pixelBuffer = self.latestPixelBuffer; - self.latestPixelBuffer = nil; - }); - return pixelBuffer; +- (void)updateData:(NSNumber *)uid channelId:(NSString *)channelId videoSourceType:(NSNumber *)videoSourceType videoViewSetupMode:(NSNumber *)videoViewSetupMode { + IrisRtcVideoFrameConfig config; + config.video_frame_format = agora::media::base::VIDEO_PIXEL_FORMAT::VIDEO_CVPIXEL_NV12; + config.uid = [uid unsignedIntValue]; + config.video_source_type = [videoSourceType intValue]; + if (channelId && (NSNull *)channelId != [NSNull null]) { + strcpy(config.channelId, [channelId UTF8String]); + } else { + strcpy(config.channelId, ""); + } + config.video_view_setup_mode = [videoViewSetupMode intValue]; + config.observed_frame_position = agora::media::base::VIDEO_MODULE_POSITION::POSITION_POST_CAPTURER | agora::media::base::VIDEO_MODULE_POSITION::POSITION_PRE_RENDERER; + + self.delegateId = self.irisRtcRendering->AddVideoFrameObserverDelegate(config, self.delegate); } - (void)dispose { if (self.irisRtcRendering) { - self.irisRtcRendering->RemoveVideoFrameObserverDelegate(self.delegateId); - self.irisRtcRendering = nil; + self.irisRtcRendering->RemoveVideoFrameObserverDelegate(self.delegateId); + self.irisRtcRendering = NULL; } if (self.delegate) { - delete self.delegate; - self.delegate = nil; + delete self.delegate; + self.delegate = NULL; } if (self.textureRegistry) { - [self.textureRegistry unregisterTexture:self.textureId]; - self.textureRegistry = nil; + [self.textureRegistry unregisterTexture:self.textureId]; + self.textureRegistry = NULL; + } + if (self.buffer_cache && self.isDirtyBuffer) { + CVPixelBufferRelease(self.buffer_cache); + self.buffer_cache = NULL; } } -- (void)dealloc { - if (self.latestPixelBuffer) { - CVPixelBufferRelease(self.latestPixelBuffer); - self.latestPixelBuffer = nil; - } +- (CVPixelBufferRef _Nullable)copyPixelBuffer { + dispatch_semaphore_wait(self.lock, DISPATCH_TIME_FOREVER); + CVPixelBufferRef buffer_temp = CVPixelBufferRetain(self.buffer_cache); + self.isDirtyBuffer = NO; + dispatch_semaphore_signal(self.lock); + + return buffer_temp; +} + +- (void)onTextureUnregistered:(NSObject *)texture { } @end +