8000 Report actual sizes for camera media stream track (#1636) · The-Funk/flutter-webrtc@b416aec · GitHub
[go: up one dir, main page]
Skip to content

Commit b416aec

Browse files
davidliudavidliu
authored
Report actual sizes for camera media stream track (flutter-webrtc#1636)
Also remove adaptOutputFormat due it causing extreme downscaling
1 parent fc249b6 commit b416aec

File tree

4 files changed

+155
-34
lines changed

4 files changed

+155
-34
lines changed

‎android/src/main/java/com/cloudwebrtc/webrtc/GetUserMediaImpl.java

Lines changed: 43 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -29,8 +29,8 @@
2929
import android.os.Looper;
3030
import android.os.ResultReceiver;
3131
import android.provider.MediaStore;
32-
import android.util.DisplayMetrics;
3332
import android.util.Log;
33+
import android.util.Pair;
3434
import android.util.Range;
3535
import android.util.SparseArray;
3636
import android.view.Display;
@@ -58,15 +58,18 @@
5858
import org.webrtc.AudioTrack;
5959
import org.webrtc.Camera1Capturer;
6060
import org.webrtc.Camera1Enumerator;
61+
import org.webrtc.Camera1Helper;
6162
import org.webrtc.Camera2Capturer;
6263
import org.webrtc.Camera2Enumerator;
64+
import org.webrtc.Camera2Helper;
6365
import org.webrtc.CameraEnumerationAndroid.CaptureFormat;
6466
import org.webrtc.CameraEnumerator;
6567
import org.webrtc.CameraVideoCapturer;
6668
import org.webrtc.MediaConstraints;
6769
import org.webrtc.MediaStream;
6870
import org.webrtc.MediaStreamTrack;
6971
import org.webrtc.PeerConnectionFactory;
72+
import org.webrtc.Size;
7073
import org.webrtc.SurfaceTextureHelper;
7174
import org.webrtc.VideoCapturer;
7275
import org.webrtc.VideoSource;
@@ -266,13 +269,12 @@ private void addDefaultAudioConstraints(MediaConstraints audioConstraints) {
266269
* @param isFacing 'user' mapped with 'front' is true (default) 'environment' mapped with 'back'
267270
* is false
268271
* @param sourceId (String) use this sourceId and ignore facing mode if specified.
269-
* @return VideoCapturer can invoke with <tt>startCapture</tt>/<tt>stopCapture</tt> <tt>null</tt>
272+
* @return Pair of deviceName to VideoCapturer. Can invoke with <tt>startCapture</tt>/<tt>stopCapture</tt> <tt>null</tt>
270273
* if not matched camera with specified facing mode.
271274
*/
272-
private Map<String, VideoCapturer> createVideoCapturer(
275+
private Pair<String, VideoCapturer> createVideoCapturer(
273276
CameraEnumerator enumerator, boolean isFacing, String sourceId, CameraEventsHandler cameraEventsHandler) {
274-
VideoCapturer videoCapturer = null;
275-
Map<String, VideoCapturer> result = new HashMap<String, VideoCapturer>();
277+
VideoCapturer videoCapturer;
276278
// if sourceId given, use specified sourceId first
277279
final String[] deviceNames = enumerator.getDeviceNames();
278280
if (sourceId != null && !sourceId.equals("")) {
@@ -281,8 +283,7 @@ private Map<String, VideoCapturer> createVideoCapturer(
281283
videoCapturer = enumerator.createCapturer(name, cameraEventsHandler);
282284
if (videoCapturer != null) {
283285
Log.d(TAG, "create user specified camera " + name + " succeeded");
284-
result.put(name, videoCapturer);
285-
return result;
286+
return new Pair<>(name, videoCapturer);
286287
} else {
287288
Log.d(TAG, "create user specified camera " + name + " failed");
288289
break; // fallback to facing mode
@@ -298,22 +299,22 @@ private Map<String, VideoCapturer> createVideoCapturer(
298299
videoCapturer = enumerator.createCapturer(name, cameraEventsHandler);
299300
if (videoCapturer != null) {
300301
Log.d(TAG, "Create " + facingStr + " camera " + name + " succeeded");
301-
result.put(name, videoCapturer);
302-
return result;
302+
303+
return new Pair<>(name, videoCapturer);
303304
} else {
304305
Log.e(TAG, "Create " + facingStr + " camera " + name + " failed");
305306
}
306307
}
307308
}
308309

309310
// falling back to the first available camera
310-
if (videoCapturer == null && deviceNames.length > 0) {
311+
if (deviceNames.length > 0) {
311312
videoCapturer = enumerator.createCapturer(deviceNames[0], cameraEventsHandler);
312313
Log.d(TAG, "Falling back to the first available camera");
313-
result.put(deviceNames[0], videoCapturer);
314+
return new Pair<>(deviceNames[0], videoCapturer);
314315
}
315316

316-
return result;
317+
return null;
317318
}
318319

319320
/**
@@ -741,21 +742,14 @@ private ConstraintsMap getUserVideo(ConstraintsMap constraints, MediaStream medi
741742
isFacing = facingMode == null || !facingMode.equals("environment");
742743
String deviceId = getSourceIdConstraint(videoConstraintsMap);
743744
CameraEventsHandler cameraEventsHandler = new CameraEventsHandler();
744-
Map<String, VideoCapturer> result = createVideoCapturer(cameraEnumerator, isFacing, deviceId, cameraEventsHandler);
745+
Pair<String, VideoCapturer> result = createVideoCapturer(cameraEnumerator, isFacing, deviceId, cameraEventsHandler);
745746

746747
if (result == null) {
747748
return null;
748749
}
749750

750-
if (deviceId == null) {
751-
if(!result.keySet().isEmpty()) {
752-
deviceId = result.keySet().iterator().next();
753-
} else {
754-
return null;
755-
}
756-
}
757-
758-
VideoCapturer videoCapturer = result.get(deviceId);
751+
deviceId = result.first;
752+
VideoCapturer videoCapturer = result.second;
759753

760754
PeerConnectionFactory pcFactory = stateProvider.getPeerConnectionFactory();
761755
VideoSource videoSource = pcFactory.createVideoSource(false);
@@ -768,36 +762,56 @@ private ConstraintsMap getUserVideo(ConstraintsMap constraints, MediaStream medi
768762
VideoCapturerInfo info = new VideoCapturerInfo();
769763

770764
Integer videoWidth = getConstrainInt(videoConstraintsMap, "width");
771-
info.width = videoWidth != null
765+
int targetWidth = videoWidth != null
772766
? videoWidth
773767
: videoConstraintsMandatory != null && videoConstraintsMandatory.hasKey("minWidth")
774768
? videoConstraintsMandatory.getInt("minWidth")
775769
: DEFAULT_WIDTH;
776770

777771
Integer videoHeight = getConstrainInt(videoConstraintsMap, "height");
778-
info.height = videoHeight != null
772+
int targetHeight = videoHeight != null
779773
? videoHeight
780774
: videoConstraintsMandatory != null && videoConstraintsMandatory.hasKey("minHeight")
781775
? videoConstraintsMandatory.getInt("minHeight")
782776
: DEFAULT_HEIGHT;
783777

784778
Integer videoFrameRate = getConstrainInt(videoConstraintsMap, "frameRate");
785-
info.fps = videoFrameRate != null
779+
int targetFps = videoFrameRate != null
786780
? videoFrameRate
787781
: videoConstraintsMandatory != null && videoConstraintsMandatory.hasKey("minFrameRate")
788782
? videoConstraintsMandatory.getInt("minFrameRate")
789783
: DEFAULT_FPS;
784+
785+
info.width = targetWidth;
786+
info.height = targetHeight;
787+
info.fps = targetFps;
790788
info.capturer = videoCapturer;
789+
790+
// Find actual capture format.
791+
Size actualSize = null;
792+
if (videoCapturer instanceof Camera1Capturer) {
793+
int cameraId = Camera1Helper.getCameraId(deviceId);
794+
actualSize = Camera1Helper.findClosestCaptureFormat(cameraId, targetWidth, targetHeight);
795+
} else if (videoCapturer instanceof Camera2Capturer) {
796+
CameraManager cameraManager = (CameraManager) applicationContext.getSystemService(Context.CAMERA_SERVICE);
797+
actualSize = Camera2Helper.findClosestCaptureFormat(cameraManager, deviceId, targetWidth, targetHeight);
798+
}
799+
800+
if (actualSize != null) {
801+
info.width = actualSize.width;
802+
info.height = actualSize.height;
803+
}
804+
791805
info.cameraEventsHandler = cameraEventsHandler;
792-
videoCapturer.startCapture(info.width, info.height, info.fps);
806+
videoCapturer.startCapture(targetWidth, targetHeight, targetFps);
793807

794808
cameraEventsHandler.waitForCameraOpen();
795809

796810
String trackId = stateProvider.getNextTrackUUID();
797811
mVideoCapturers.put(trackId, info);
798812
mSurfaceTextureHelpers.put(trackId, surfaceTextureHelper);
799-
Log.d(TAG, "changeCaptureFormat: " + info.width + "x" + info.height + "@" + info.fps);
800-
videoSource.adaptOutputFormat(info.width, info.height, info.fps);
813+
814+
Log.d(TAG, "Target: " + targetWidth + "x" + targetHeight + "@" + targetFps + ", Actual: " + info.width + "x" + info.height + "@" + info.fps);
801815

802816
VideoTrack track = pcFactory.createVideoTrack(trackId, videoSource);
803817
mediaStream.addTrack(track);
@@ -1284,7 +1298,7 @@ public interface IsCameraEnabled {
12841298
boolean isEnabled(String id);
12851299
}
12861300

1287-
public class VideoCapturerInfo {
1301+
public static class VideoCapturerInfo {
12881302
VideoCapturer capturer;
12891303
int width;
12901304
int height;

‎android/src/main/java/org/webrtc/Camera1Helper.java

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
/*
2+
* Copyright 2023-2024 LiveKit, Inc.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package org.webrtc;
18+
19+
import androidx.annotation.Nullable;
20+
21+
import java.util.ArrayList;
22+
import java.util.List;
23+
24+
/**
25+
* A helper to access package-protected methods used in [Camera2Session]
26+
* <p>
27+
* Note: cameraId as used in the Camera1XXX classes refers to the index within the list of cameras.
28+
*
29+
* @suppress
30+
*/
31+
32+
public class Camera1Helper {
33+
34+
public static int getCameraId(String deviceName) {
35+
return Camera1Enumerator.getCameraIndex(deviceName);
36+
}
37+
38+
@Nullable
39+
public static List<CameraEnumerationAndroid.CaptureFormat> getSupportedFormats(int cameraId) {
40+
return Camera1Enumerator.getSupportedFormats(cameraId);
41+
}
42+
43+
public static Size findClosestCaptureFormat(int cameraId, int width, int height) {
44+
List<CameraEnumerationAndroid.CaptureFormat> formats = getSupportedFormats(cameraId);
45+
46+
List<Size> sizes = new ArrayList<>();
47+
if (formats != null) {
48+
for (CameraEnumerationAndroid.CaptureFormat format : formats) {
49+
sizes.add(new Size(format.width, format.height));
50+
}
51+
}
52+
53+
return CameraEnumerationAndroid.getClosestSupportedSize(sizes, width, height);
54+
}
55+
}

‎android/src/main/java/org/webrtc/Camera2Helper.java

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
/*
2+
* Copyright 2023-2024 LiveKit, Inc.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package org.webrtc;
18+
19+
import android.hardware.camera2.CameraManager;
20+
21+
import androidx.annotation.Nullable;
22+
23+
import java.util.ArrayList;
24+
import java.util.List;
25+
26+
/**
27+
* A helper to access package-protected methods used in [Camera2Session]
28+
* <p>
29+
* Note: cameraId as used in the Camera2XXX classes refers to the id returned
30+
* by [CameraManager.getCameraIdList].
31+
*/
32+
public class Camera2Helper {
33+
34+
@Nullable
35+
public static List<CameraEnumerationAndroid.CaptureFormat> getSupportedFormats(CameraManager cameraManager, @Nullable String cameraId) {
36+
return Camera2Enumerator.getSupportedFormats(cameraManager, cameraId);
37+
}
38+
39+
public static Size findClosestCaptureFormat(CameraManager cameraManager, @Nullable String cameraId, int width, int height) {
40+
List<CameraEnumerationAndroid.CaptureFormat> formats = getSupportedFormats(cameraManager, cameraId);
41+
42+
List<Size> sizes = new ArrayList<>();
43+
if (formats != null) {
44+
for (CameraEnumerationAndroid.CaptureFormat format : formats) {
45+
sizes.add(new Size(format.width, format.height));
46+
}
47+
}
48+
49+
return CameraEnumerationAndroid.getClosestSupportedSize(sizes, width, height);
50+
}
51+
}

‎common/darwin/Classes/FlutterRTCMediaStream.m

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -429,13 +429,16 @@ - (void)getUserVideo:(NSDictionary*)constraints
429429
targetWidth:targetWidth
430430
targetHeight:targetHeight];
431431

432+
CMVideoDimensions selectedDimension = CMVideoFormatDescriptionGetDimensions(selectedFormat.formatDescription);
433+
NSInteger selectedWidth = (NSInteger) selectedDimension.width;
434+
NSInteger selectedHeight = (NSInteger) selectedDimension.height;
432435
NSInteger selectedFps = [self selectFpsForFormat:selectedFormat targetFps:targetFps];
433436

434437
self._lastTargetFps = selectedFps;
435438
self._lastTargetWidth = targetWidth;
436439
self._lastTargetHeight = targetHeight;
437440

438-
NSLog(@"target format %ldx%ld, targetFps: %ld, seledted fps %ld", targetWidth, targetHeight, targetFps, selectedFps);
441+
NSLog(@"target format %ldx%ld, targetFps: %ld, selected format: %ldx%ld, selected fps %ld", targetWidth, targetHeight, targetFps, selectedWidth, selectedHeight, selectedFps);
439442

440443
if ([videoDevice lockForConfiguration:NULL]) {
441444
@try {
@@ -475,14 +478,12 @@ - (void)getUserVideo:(NSDictionary*)constraints
475478
: videoDevice.position == AVCaptureDevicePositionFront ? @"user"
476479
: @"unspecified";
477480
}
478-
479-
[videoSource adaptOutputFormatToWidth:(int)targetWidth height:(int)targetHeight fps:(int)targetFps];
480481

481482
videoTrack.settings = @{
482483
@"deviceId" : videoDeviceId,
483484
@"kind" : @"videoinput",
484-
@"width" : [NSNumber numberWithInteger:targetWidth ],
485-
@"height" : [NSNumber numberWithInteger:targetHeight],
485+
@"width" : [NSNumber numberWithInteger:selectedWidth],
486+
@"height" : [NSNumber numberWithInteger:selectedHeight],
486487
@"frameRate" : [NSNumber numberWithInteger:selectedFps],
487488
@"facingMode" : facingMode,
488489
};

0 commit comments

Comments
 (0)
0