8000 Frame capture support for MacOS (#1160) · ashishcors/flutter-webrtc@22e2bab · GitHub
[go: up one dir, main page]
Skip to content

Commit 22e2bab

Browse files< 8000 /span>
rostopirarostopira
authored
Frame capture support for MacOS (flutter-webrtc#1160)
1 parent 02bd91c commit 22e2bab

File tree

2 files changed

+127
-85
lines changed

2 files changed

+127
-85
lines changed

‎common/darwin/Classes/FlutterRTCFrameCapturer.h

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,4 @@
99

1010
- (instancetype)initWithTrack:(RTCVideoTrack *) track toPath:(NSString *) path result:(FlutterResult)result;
1111

12-
#if TARGET_OS_IPHONE
13-
+ (UIImage *)convertFrameToUIImage:(RTCVideoFrame *)frame;
14-
#endif
15-
1612
@end

‎common/darwin/Classes/FlutterRTCFrameCapturer.m

Lines changed: 127 additions & 81 deletions
Original file line numberDiff line numberDiff line change
@@ -4,11 +4,8 @@
44
#import <FlutterMacOS/FlutterMacOS.h>
55
#endif
66

7-
87
#import "FlutterRTCFrameCapturer.h"
98

10-
#define clamp(a) (a>255?255:(a<0?0:a))
11-
129
@import CoreImage;
1310
@import CoreVideo;
1411

@@ -36,99 +33,148 @@ - (void)setSize:(CGSize)size
3633
{
3734
}
3835

39-
#if TARGET_OS_IPHONE
40-
// Thanks Juan Giorello https://groups.google.com/g/discuss-webrtc/c/ULGIodbbLvM
41-
+ (UIImage *)convertFrameToUIImage:(RTCVideoFrame *)frame {
42-
// https://chromium.googlesource.com/external/webrtc/+/refs/heads/main/sdk/objc/base/RTCVideoFrame.h
43-
// RTCVideoFrameBuffer *rtcFrameBuffer = (RTCVideoFrameBuffer *)frame.buffer;
44-
// RTCI420Buffer *buffer = [rtcFrameBuffer toI420];
45-
46-
// https://chromium.googlesource.com/external/webrtc/+/refs/heads/main/sdk/objc/base/RTCVideoFrameBuffer.h
47-
// This guarantees the buffer will be RTCI420Buffer
48-
RTCI420Buffer *buffer = [frame.buffer toI420];
49-
50-
51-
int width = buffer.width;
52-
int height = buffer.height;
53-
int bytesPerPixel = 4;
54-
uint8_t *rgbBuffer = malloc(width * height * bytesPerPixel);
55-
56-
for(int row = 0; row < height; row++) {
57-
const uint8_t *yLine = &buffer.dataY[row * buffer.strideY];
58-
const uint8_t *uLine = &buffer.dataU[(row >> 1) * buffer.strideU];
59-
const uint8_t *vLine = &buffer.dataV[(row >> 1) * buffer.strideV];
60-
61-
for(int x = 0; x < width; x++) {
62-
int16_t y = yLine[x];
63-
int16_t u = uLine[x >> 1] - 128;
64-
int16_t v = vLine[x >> 1] - 128;
65-
66-
int16_t r = roundf(y + v * 1.4);
67-
int16_t g = roundf(y + u * -0.343 + v * -0.711);
68-
int16_t b = roundf(y + u * 1.765);
69-
70-
uint8_t *rgb = &rgbBuffer[(row * width + x) * bytesPerPixel];
71-
rgb[0] = 0xff;
72-
rgb[1] = clamp(b);
73-
rgb[2] = clamp(g);
74-
rgb[3] = clamp(r);
36+
- (void)renderFrame:(nullable RTCVideoFrame *)frame
37+
{
 8000
38+
if (_gotFrame || frame == nil)
39+
return;
40+
_gotFrame = true;
41+
id <RTCVideoFrameBuffer> buffer = frame.buffer;
42+
CVPixelBufferRef pixelBufferRef;
43+
bool shouldRelease;
44+
if (![buffer isKindOfClass:[RTCCVPixelBuffer class]]) {
45+
pixelBufferRef = [self convertToCVPixelBuffer:frame];
46+
shouldRelease = true;
47+
} else {
48+
pixelBufferRef = ((RTCCVPixelBuffer *) buffer).pixelBuffer;
49+
shouldRelease = false;
50+
}
51+
CIImage *ciImage = [CIImage imageWithCVPixelBuffer:pixelBufferRef];
52+
CGRect outputSize;
53+
if (@available(iOS 11, macOS 10.13, *)) {
54+
switch (frame.rotation) {
55+
case RTCVideoRotation_90:
56+
ciImage = [ciImage imageByApplyingCGOrientation:kCGImagePropertyOrientationRight];
57+
outputSize = CGRectMake(0, 0, frame.height, frame.width);
58+
break;
59+
case RTCVideoRotation_180:
60+
ciImage = [ciImage imageByApplyingCGOrientation:kCGImagePropertyOrientationDown];
61+
outputSize = CGRectMake(0, 0, frame.width, frame.height);
62+
break;
63+
case RTCVideoRotation_270:
64+
ciImage = [ciImage imageByApplyingCGOrientation:kCGImagePropertyOrientationLeft];
65+
outputSize = CGRectMake(0, 0, frame.height, frame.width);
66+
break;
67+
default:
68+
outputSize = CGRectMake(0, 0, frame.width, frame.height);
69+
break;
7570
}
71+
} else {
72+
outputSize = CGRectMake(0, 0, frame.width, frame.height);
7673
}
77-
78-
CGColorSpaceRef colorSpace = CGColorSpaceCreateDeviceRGB();
79-
CGContextRef context = CGBitmapContextCreate(rgbBuffer, width, height, 8, width * bytesPerPixel, colorSpace, kCGBitmapByteOrder32Little | kCGImageAlphaNoneSkipLast);
80-
81-
CGImageRef cgImage = CGBitmapContextCreateImage(context);
82-
CGContextRelease(context);
83-
CGColorSpaceRelease(colorSpace);
84-
free(rgbBuffer);
85-
86-
UIImageOrientation orientation;
87-
switch (frame.rotation) {
88-
case RTCVideoRotation_90:
89-
orientation = UIImageOrientationRight;
90-
break;
91-
case RTCVideoRotation_180:
92-
orientation = UIImageOrientationDown;
93-
break;
94-
case RTCVideoRotation_270:
95-
orientation = UIImageOrientationLeft;
96-
default:
97-
orientation = UIImageOrientationUp;
98-
break;
74+
CIContext *tempContext = [CIContext contextWithOptions:nil];
75+
CGImageRef cgImage = [tempContext
76+
createCGImage:ciImage
77+
fromRect:outputSize];
78+
NSData *imageData;
79+
#if TARGET_OS_IPHONE
80+
UIImage *uiImage = [UIImage imageWithCGImage:cgImage];
81+
if ([[_path pathExtension] isEqualToString:@"jpg"]) {
82+
imageData = UIImageJPEGRepresentation(uiImage, 1.0f);
83+
} else {
84+
imageData = UIImagePNGRepresentation(uiImage);
9985
}
100-
101-
UIImage *image = [UIImage imageWithCGImage:cgImage scale:1 orientation:orientation];
86+
#else
87+
NSBitmapImageRep *newRep = [[NSBitmapImageRep alloc] initWithCGImage:cgImage];
88+
[newRep setSize:NSSizeToCGSize(outputSize.size)];
89+
NSDictionary<NSBitmapImageRepPropertyKey, id>* quality = @{
90+
NSImageCompressionFactor: @1.0f
91+
};
92+
if ([[_path pathExtension] isEqualToString:@"jpg"]) {
93+
imageData = [newRep representationUsingType:NSJPEGFileType properties:quality];
94+
} else {
95+
imageData = [newRep representationUsingType:NSPNGFileType properties:quality];
96+
}
97+
#endif
10298
CGImageRelease(cgImage);
103-
104-
return image;
105-
}
106-
#endif
107-
108-
- (void)renderFrame:(nullable RTCVideoFrame *)frame
109-
{
110-
#if TARGET_OS_IPHONE
111-
if (_gotFrame || frame == nil) return;
112-
_gotFrame = true;
113-
114-
UIImage *uiImage = [FlutterRTCFrameCapturer convertFrameToUIImage:frame];
115-
NSData *jpgData = UIImageJPEGRepresentation(uiImage, 0.9f);
116-
117-
if ([jpgData writeToFile:_path atomically:NO]) {
99+
if (shouldRelease)
100+
CVPixelBufferRelease(pixelBufferRef);
101+
if (imageData && [imageData writeToFile:_path atomically:NO]) {
118102
NSLog(@"File writed successfully to %@", _path);
119103
_result(nil);
120104
} else {
121105
NSLog(@"Failed to write to file");
122106
_result([FlutterError errorWithCode:@"CaptureFrameFailed"
123-
message:@"Failed to write JPEG data to file"
107+
message:@"Failed to write image data to file"
124108
details:nil]);
125109
}
126-
127110
dispatch_async(dispatch_get_main_queue(), ^{
128111
[self->_track removeRenderer:self];
129112
self->_track = nil;
130113
});
131-
#endif
114+
}
115+
116+
-(CVPixelBufferRef)convertToCVPixelBuffer:(RTCVideoFrame *) frame
117+
{
118+
id<RTCI420Buffer> i420Buffer = [frame.buffer toI420];
119+
CVPixelBufferRef outputPixelBuffer;
120+
size_t w = (size_t) roundf(i420Buffer.width);
121+
size_t h = (size_t) roundf(i420Buffer.height);
122+
NSDictionary *pixelAttributes = @{(id)kCVPixelBufferIOSurfacePropertiesKey : @{}};
123+
CVPixelBufferCreate(kCFAllocatorDefault, w, h, kCVPixelFormatType_32BGRA, (__bridge CFDictionaryRef)(pixelAttributes), &outputPixelBuffer);
124+
CVPixelBufferLockBaseAddress(outputPixelBuffer, 0);
125+
const OSType pixelFormat = CVPixelBufferGetPixelFormatType(outputPixelBuffer);
126+
if (pixelFormat == kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange ||
127+
pixelFormat == kCVPixelFormatType_420YpCbCr8BiPlanarFullRange) {
128+
// NV12
129+
uint8_t* dstY = CVPixelBufferGetBaseAddressOfPlane(outputPixelBuffer, 0);
130+
const size_t dstYStride = CVPixelBufferGetBytesPerRowOfPlane(outputPixelBuffer, 0);
131+
uint8_t* dstUV = CVPixelBufferGetBaseAddressOfPlane(outputPixelBuffer, 1);
132+
const size_t dstUVStride = CVPixelBufferGetBytesPerRowOfPlane(outputPixelBuffer, 1);
133+
134+
[RTCYUVHelper I420ToNV12:i420Buffer.dataY
135+
srcStrideY:i420Buffer.strideY
136+
srcU:i420Buffer.dataU
137+
srcStrideU:i420Buffer.strideU
138+
srcV:i420Buffer.dataV
139+
srcStrideV:i420Buffer.strideV
140+
dstY:dstY
141+
dstStrideY:(int)dstYStride
142+
dstUV:dstUV
143+
dstStrideUV:(int)dstUVStride
144+
width:i420Buffer.width
145+
width:i420Buffer.height];
146+
} else {
147+
uint8_t* dst = CVPixelBufferGetBaseAddress(outputPixelBuffer);
148+
const size_t bytesPerRow = CVPixelBufferGetBytesPerRow(outputPixelBuffer);
149+
150+
if (pixelFormat == kCVPixelFormatType_32BGRA) {
151+
// Corresponds to libyuv::FOURCC_ARGB
152+
[RTCYUVHelper I420ToARGB:i420Buffer.dataY
153+
srcStrideY:i420Buffer.strideY
154+
srcU:i420Buffer.dataU
155+
srcStrideU:i420Buffer.strideU
156+
srcV:i420Buffer.dataV
157+
srcStrideV:i420Buffer.strideV
158+
dstARGB:dst
159+
dstStrideARGB:(int)bytesPerRow
160+
width:i420Buffer.width
161+
height:i420Buffer.height];
162+
} else if (pixelFormat == kCVPixelFormatType_32ARGB) {
163+
// Corresponds to libyuv::FOURCC_BGRA
164+
[RTCYUVHelper I420ToBGRA:i420Buffer.dataY
165+
srcStrideY:i420Buffer.strideY
166+
srcU:i420Buffer.dataU
167+
srcStrideU:i420Buffer.strideU
168+
srcV:i420Buffer.dataV
169+
srcStrideV:i420Buffer.strideV
170+
dstBGRA:dst
171+
dstStrideBGRA:(int)bytesPerRow
172+
width:i420Buffer.width
173+
height:i420Buffer.height];
174+
}
175+
}
176+
CVPixelBufferUnlockBaseAddress(outputPixelBuffer, 0);
177+
return outputPixelBuffer;
132178
}
133179

134180
@end

0 commit comments

Comments
 (0)
0