390 lines
12 KiB
Plaintext
390 lines
12 KiB
Plaintext
#import "VideoMetalView.h"
|
|
|
|
#import <Metal/Metal.h>
|
|
#import <MetalKit/MetalKit.h>
|
|
|
|
#import "base/RTCLogging.h"
|
|
#import "base/RTCVideoFrame.h"
|
|
#import "base/RTCVideoFrameBuffer.h"
|
|
#import "TGRTCCVPixelBuffer.h"
|
|
#include "sdk/objc/native/api/video_frame.h"
|
|
#include "sdk/objc/native/src/objc_frame_buffer.h"
|
|
|
|
#import "api/video/video_sink_interface.h"
|
|
#import "api/media_stream_interface.h"
|
|
#import "rtc_base/time_utils.h"
|
|
|
|
#import "RTCMTLI420Renderer.h"
|
|
#import "RTCMTLNV12Renderer.h"
|
|
#import "RTCMTLRGBRenderer.h"
|
|
|
|
#define MTKViewClass NSClassFromString(@"MTKView")
|
|
#define RTCMTLNV12RendererClass NSClassFromString(@"RTCMTLNV12Renderer")
|
|
#define RTCMTLI420RendererClass NSClassFromString(@"RTCMTLI420Renderer")
|
|
#define RTCMTLRGBRendererClass NSClassFromString(@"RTCMTLRGBRenderer")
|
|
|
|
namespace {
|
|
|
|
static RTCVideoFrame *customToObjCVideoFrame(const webrtc::VideoFrame &frame, RTCVideoRotation &rotation) {
|
|
rotation = RTCVideoRotation(frame.rotation());
|
|
RTCVideoFrame *videoFrame =
|
|
[[RTCVideoFrame alloc] initWithBuffer:webrtc::ToObjCVideoFrameBuffer(frame.video_frame_buffer())
|
|
rotation:rotation
|
|
timeStampNs:frame.timestamp_us() * rtc::kNumNanosecsPerMicrosec];
|
|
videoFrame.timeStamp = frame.timestamp();
|
|
|
|
return videoFrame;
|
|
}
|
|
|
|
class VideoRendererAdapterImpl : public rtc::VideoSinkInterface<webrtc::VideoFrame> {
|
|
public:
|
|
VideoRendererAdapterImpl(void (^frameReceived)(CGSize, RTCVideoFrame *, RTCVideoRotation)) {
|
|
_frameReceived = [frameReceived copy];
|
|
}
|
|
|
|
void OnFrame(const webrtc::VideoFrame& nativeVideoFrame) override {
|
|
RTCVideoRotation rotation = RTCVideoRotation_90;
|
|
RTCVideoFrame* videoFrame = customToObjCVideoFrame(nativeVideoFrame, rotation);
|
|
|
|
CGSize currentSize = (videoFrame.rotation % 180 == 0) ? CGSizeMake(videoFrame.width, videoFrame.height) : CGSizeMake(videoFrame.height, videoFrame.width);
|
|
|
|
if (_frameReceived) {
|
|
_frameReceived(currentSize, videoFrame, rotation);
|
|
}
|
|
}
|
|
|
|
private:
|
|
void (^_frameReceived)(CGSize, RTCVideoFrame *, RTCVideoRotation);
|
|
};
|
|
|
|
}
|
|
|
|
@interface VideoMetalView () <MTKViewDelegate> {
|
|
RTCMTLI420Renderer *_rendererI420;
|
|
RTCMTLNV12Renderer *_rendererNV12;
|
|
RTCMTLRGBRenderer *_rendererRGB;
|
|
MTKView *_metalView;
|
|
RTCVideoFrame *_videoFrame;
|
|
CGSize _videoFrameSize;
|
|
int64_t _lastFrameTimeNs;
|
|
|
|
CGSize _currentSize;
|
|
std::shared_ptr<VideoRendererAdapterImpl> _sink;
|
|
|
|
void (^_onFirstFrameReceived)();
|
|
bool _firstFrameReceivedReported;
|
|
|
|
void (^_onOrientationUpdated)(int);
|
|
|
|
void (^_onIsMirroredUpdated)(bool);
|
|
|
|
bool _didSetShouldBeMirrored;
|
|
bool _shouldBeMirrored;
|
|
}
|
|
|
|
@end
|
|
|
|
@implementation VideoMetalView
|
|
|
|
+ (bool)isSupported {
|
|
static bool value;
|
|
static dispatch_once_t onceToken;
|
|
dispatch_once(&onceToken, ^{
|
|
id<MTLDevice> device = MTLCreateSystemDefaultDevice();
|
|
value = device != nil;
|
|
});
|
|
return value;
|
|
}
|
|
|
|
- (instancetype)initWithFrame:(CGRect)frameRect {
|
|
self = [super initWithFrame:frameRect];
|
|
if (self) {
|
|
[self configure];
|
|
|
|
_currentSize = CGSizeZero;
|
|
//_rotationOverride = @(RTCVideoRotation_90);
|
|
|
|
__weak VideoMetalView *weakSelf = self;
|
|
_sink.reset(new VideoRendererAdapterImpl(^(CGSize size, RTCVideoFrame *videoFrame, RTCVideoRotation rotation) {
|
|
dispatch_async(dispatch_get_main_queue(), ^{
|
|
__strong VideoMetalView *strongSelf = weakSelf;
|
|
if (strongSelf == nil) {
|
|
return;
|
|
}
|
|
if (!CGSizeEqualToSize(size, strongSelf->_currentSize)) {
|
|
strongSelf->_currentSize = size;
|
|
[strongSelf setSize:size];
|
|
}
|
|
|
|
int mappedValue = 0;
|
|
switch (rotation) {
|
|
case RTCVideoRotation_90:
|
|
mappedValue = 0;
|
|
break;
|
|
case RTCVideoRotation_180:
|
|
mappedValue = 1;
|
|
break;
|
|
case RTCVideoRotation_270:
|
|
mappedValue = 2;
|
|
break;
|
|
default:
|
|
mappedValue = 0;
|
|
break;
|
|
}
|
|
[strongSelf setInternalOrientation:mappedValue];
|
|
|
|
[strongSelf renderFrame:videoFrame];
|
|
});
|
|
}));
|
|
}
|
|
return self;
|
|
}
|
|
|
|
- (BOOL)isEnabled {
|
|
return !_metalView.paused;
|
|
}
|
|
|
|
- (void)setEnabled:(BOOL)enabled {
|
|
_metalView.paused = !enabled;
|
|
}
|
|
|
|
- (UIViewContentMode)videoContentMode {
|
|
return _metalView.contentMode;
|
|
}
|
|
|
|
- (void)setVideoContentMode:(UIViewContentMode)mode {
|
|
_metalView.contentMode = mode;
|
|
}
|
|
|
|
#pragma mark - Private
|
|
|
|
+ (BOOL)isMetalAvailable {
|
|
return MTLCreateSystemDefaultDevice() != nil;
|
|
}
|
|
|
|
+ (MTKView *)createMetalView:(CGRect)frame {
|
|
return [[MTKViewClass alloc] initWithFrame:frame];
|
|
}
|
|
|
|
+ (RTCMTLNV12Renderer *)createNV12Renderer {
|
|
return [[RTCMTLNV12RendererClass alloc] init];
|
|
}
|
|
|
|
+ (RTCMTLI420Renderer *)createI420Renderer {
|
|
return [[RTCMTLI420RendererClass alloc] init];
|
|
}
|
|
|
|
+ (RTCMTLRGBRenderer *)createRGBRenderer {
|
|
return [[RTCMTLRGBRenderer alloc] init];
|
|
}
|
|
|
|
- (void)configure {
|
|
NSAssert([VideoMetalView isMetalAvailable], @"Metal not availiable on this device");
|
|
|
|
_metalView = [VideoMetalView createMetalView:self.bounds];
|
|
_metalView.delegate = self;
|
|
_metalView.contentMode = UIViewContentModeScaleAspectFill;
|
|
[self addSubview:_metalView];
|
|
_videoFrameSize = CGSizeZero;
|
|
}
|
|
|
|
- (void)setMultipleTouchEnabled:(BOOL)multipleTouchEnabled {
|
|
[super setMultipleTouchEnabled:multipleTouchEnabled];
|
|
_metalView.multipleTouchEnabled = multipleTouchEnabled;
|
|
}
|
|
|
|
- (void)layoutSubviews {
|
|
[super layoutSubviews];
|
|
|
|
CGRect bounds = self.bounds;
|
|
_metalView.frame = bounds;
|
|
if (!CGSizeEqualToSize(_videoFrameSize, CGSizeZero)) {
|
|
_metalView.drawableSize = [self drawableSize];
|
|
} else {
|
|
_metalView.drawableSize = bounds.size;
|
|
}
|
|
}
|
|
|
|
#pragma mark - MTKViewDelegate methods
|
|
|
|
- (void)drawInMTKView:(nonnull MTKView *)view {
|
|
NSAssert(view == _metalView, @"Receiving draw callbacks from foreign instance.");
|
|
RTCVideoFrame *videoFrame = _videoFrame;
|
|
// Skip rendering if we've already rendered this frame.
|
|
if (!videoFrame || videoFrame.timeStampNs == _lastFrameTimeNs) {
|
|
return;
|
|
}
|
|
|
|
if (CGRectIsEmpty(view.bounds)) {
|
|
return;
|
|
}
|
|
|
|
RTCMTLRenderer *renderer;
|
|
if ([videoFrame.buffer isKindOfClass:[RTCCVPixelBuffer class]]) {
|
|
RTCCVPixelBuffer *buffer = (RTCCVPixelBuffer*)videoFrame.buffer;
|
|
|
|
if ([buffer isKindOfClass:[TGRTCCVPixelBuffer class]]) {
|
|
bool shouldBeMirrored = ((TGRTCCVPixelBuffer *)buffer).shouldBeMirrored;
|
|
if (shouldBeMirrored != _shouldBeMirrored) {
|
|
_shouldBeMirrored = shouldBeMirrored;
|
|
if (_shouldBeMirrored) {
|
|
_metalView.transform = CGAffineTransformMakeScale(-1.0f, 1.0f);
|
|
} else {
|
|
_metalView.transform = CGAffineTransformIdentity;
|
|
}
|
|
|
|
if (_didSetShouldBeMirrored) {
|
|
if (_onIsMirroredUpdated) {
|
|
_onIsMirroredUpdated(_shouldBeMirrored);
|
|
}
|
|
} else {
|
|
_didSetShouldBeMirrored = true;
|
|
}
|
|
}
|
|
}
|
|
|
|
const OSType pixelFormat = CVPixelBufferGetPixelFormatType(buffer.pixelBuffer);
|
|
if (pixelFormat == kCVPixelFormatType_32BGRA || pixelFormat == kCVPixelFormatType_32ARGB) {
|
|
if (!_rendererRGB) {
|
|
_rendererRGB = [VideoMetalView createRGBRenderer];
|
|
if (![_rendererRGB addRenderingDestination:_metalView]) {
|
|
_rendererRGB = nil;
|
|
RTCLogError(@"Failed to create RGB renderer");
|
|
return;
|
|
}
|
|
}
|
|
renderer = _rendererRGB;
|
|
} else {
|
|
if (!_rendererNV12) {
|
|
_rendererNV12 = [VideoMetalView createNV12Renderer];
|
|
if (![_rendererNV12 addRenderingDestination:_metalView]) {
|
|
_rendererNV12 = nil;
|
|
RTCLogError(@"Failed to create NV12 renderer");
|
|
return;
|
|
}
|
|
}
|
|
renderer = _rendererNV12;
|
|
}
|
|
} else {
|
|
if (!_rendererI420) {
|
|
_rendererI420 = [VideoMetalView createI420Renderer];
|
|
if (![_rendererI420 addRenderingDestination:_metalView]) {
|
|
_rendererI420 = nil;
|
|
RTCLogError(@"Failed to create I420 renderer");
|
|
return;
|
|
}
|
|
}
|
|
renderer = _rendererI420;
|
|
}
|
|
|
|
renderer.rotationOverride = _rotationOverride;
|
|
|
|
[renderer drawFrame:videoFrame];
|
|
_lastFrameTimeNs = videoFrame.timeStampNs;
|
|
}
|
|
|
|
- (void)mtkView:(MTKView *)view drawableSizeWillChange:(CGSize)size {
|
|
}
|
|
|
|
#pragma mark -
|
|
|
|
- (void)setRotationOverride:(NSValue *)rotationOverride {
|
|
_rotationOverride = rotationOverride;
|
|
|
|
_metalView.drawableSize = [self drawableSize];
|
|
[self setNeedsLayout];
|
|
}
|
|
|
|
- (RTCVideoRotation)frameRotation {
|
|
if (_rotationOverride) {
|
|
RTCVideoRotation rotation;
|
|
if (@available(iOS 11, *)) {
|
|
[_rotationOverride getValue:&rotation size:sizeof(rotation)];
|
|
} else {
|
|
[_rotationOverride getValue:&rotation];
|
|
}
|
|
return rotation;
|
|
}
|
|
|
|
return _videoFrame.rotation;
|
|
}
|
|
|
|
- (CGSize)drawableSize {
|
|
// Flip width/height if the rotations are not the same.
|
|
CGSize videoFrameSize = _videoFrameSize;
|
|
RTCVideoRotation frameRotation = [self frameRotation];
|
|
|
|
BOOL useLandscape =
|
|
(frameRotation == RTCVideoRotation_0) || (frameRotation == RTCVideoRotation_180);
|
|
BOOL sizeIsLandscape = (_videoFrame.rotation == RTCVideoRotation_0) ||
|
|
(_videoFrame.rotation == RTCVideoRotation_180);
|
|
|
|
if (useLandscape == sizeIsLandscape) {
|
|
return videoFrameSize;
|
|
} else {
|
|
return CGSizeMake(videoFrameSize.height, videoFrameSize.width);
|
|
}
|
|
}
|
|
|
|
#pragma mark - RTCVideoRenderer
|
|
|
|
- (void)setSize:(CGSize)size {
|
|
assert([NSThread isMainThread]);
|
|
|
|
_videoFrameSize = size;
|
|
CGSize drawableSize = [self drawableSize];
|
|
|
|
_metalView.drawableSize = drawableSize;
|
|
[self setNeedsLayout];
|
|
//[strongSelf.delegate videoView:self didChangeVideoSize:size];
|
|
}
|
|
|
|
- (void)renderFrame:(nullable RTCVideoFrame *)frame {
|
|
assert([NSThread isMainThread]);
|
|
|
|
if (!_firstFrameReceivedReported && _onFirstFrameReceived) {
|
|
_firstFrameReceivedReported = true;
|
|
_onFirstFrameReceived();
|
|
}
|
|
|
|
if (!self.isEnabled) {
|
|
return;
|
|
}
|
|
|
|
if (frame == nil) {
|
|
RTCLogInfo(@"Incoming frame is nil. Exiting render callback.");
|
|
return;
|
|
}
|
|
_videoFrame = frame;
|
|
}
|
|
|
|
- (std::shared_ptr<rtc::VideoSinkInterface<webrtc::VideoFrame>>)getSink {
|
|
assert([NSThread isMainThread]);
|
|
|
|
return _sink;
|
|
}
|
|
|
|
- (void)setOnFirstFrameReceived:(void (^ _Nullable)())onFirstFrameReceived {
|
|
_onFirstFrameReceived = [onFirstFrameReceived copy];
|
|
_firstFrameReceivedReported = false;
|
|
}
|
|
|
|
- (void)setInternalOrientation:(int)internalOrientation {
|
|
if (_internalOrientation != internalOrientation) {
|
|
_internalOrientation = internalOrientation;
|
|
if (_onOrientationUpdated) {
|
|
_onOrientationUpdated(internalOrientation);
|
|
}
|
|
}
|
|
}
|
|
|
|
- (void)internalSetOnOrientationUpdated:(void (^ _Nullable)(int))onOrientationUpdated {
|
|
_onOrientationUpdated = [onOrientationUpdated copy];
|
|
}
|
|
|
|
- (void)internalSetOnIsMirroredUpdated:(void (^ _Nullable)(bool))onIsMirroredUpdated {
|
|
_onIsMirroredUpdated = [onIsMirroredUpdated copy];
|
|
}
|
|
|
|
@end
|