From dd8eb0f50c1811dd095d5e01b0052042c708d9eb Mon Sep 17 00:00:00 2001 From: David Liu Date: Wed, 1 Dec 2021 20:29:25 +0900 Subject: [PATCH 1/7] clean up unused variables --- .../cloudwebrtc/webrtc/GetUserMediaImpl.java | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/android/src/main/java/com/cloudwebrtc/webrtc/GetUserMediaImpl.java b/android/src/main/java/com/cloudwebrtc/webrtc/GetUserMediaImpl.java index 5db44056df..9eb087a90b 100755 --- a/android/src/main/java/com/cloudwebrtc/webrtc/GetUserMediaImpl.java +++ b/android/src/main/java/com/cloudwebrtc/webrtc/GetUserMediaImpl.java @@ -88,7 +88,7 @@ class GetUserMediaImpl { private static final String PERMISSION_AUDIO = Manifest.permission.RECORD_AUDIO; private static final String PERMISSION_VIDEO = Manifest.permission.CAMERA; private static final String PERMISSION_SCREEN = "android.permission.MediaProjection"; - private static int CAPTURE_PERMISSION_REQUEST_CODE = 1; + private static final int CAPTURE_PERMISSION_REQUEST_CODE = 1; private static final String GRANT_RESULTS = "GRANT_RESULT"; private static final String PERMISSIONS = "PERMISSION"; private static final String PROJECTION_DATA = "PROJECTION_DATA"; @@ -103,8 +103,6 @@ class GetUserMediaImpl { private final Context applicationContext; static final int minAPILevel = Build.VERSION_CODES.LOLLIPOP; - private MediaProjectionManager mProjectionManager = null; - private static MediaProjection sMediaProjection = null; final AudioSamplesInterceptor inputSamplesInterceptor = new AudioSamplesInterceptor(); private OutputAudioSamplesInterceptor outputSamplesInterceptor = null; @@ -357,17 +355,6 @@ void getUserMedia( // should change `parseConstraints()` according // see: https://www.w3.org/TR/mediacapture-streams/#idl-def-MediaTrackConstraints - ConstraintsMap videoConstraintsMap = null; - ConstraintsMap videoConstraintsMandatory = null; - - if (constraints.getType("video") == ObjectType.Map) { - videoConstraintsMap = constraints.getMap("video"); - if (videoConstraintsMap.hasKey("mandatory") - && videoConstraintsMap.getType("mandatory") == ObjectType.Map) { - videoConstraintsMandatory = videoConstraintsMap.getMap("mandatory"); - } - } - final ArrayList requestPermissions = new ArrayList<>(); if (constraints.hasKey("audio")) { @@ -452,8 +439,6 @@ void getDisplayMedia( } } - final ConstraintsMap videoConstraintsMandatory2 = videoConstraintsMandatory; - screenRequestPremissions( new ResultReceiver(new Handler(Looper.getMainLooper())) { @Override From 0544541619d1dda3af2239bda53aa10691498398 Mon Sep 17 00:00:00 2001 From: David Liu Date: Wed, 1 Dec 2021 21:00:35 +0900 Subject: [PATCH 2/7] more unused variables --- .../java/com/cloudwebrtc/webrtc/GetUserMediaImpl.java | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/android/src/main/java/com/cloudwebrtc/webrtc/GetUserMediaImpl.java b/android/src/main/java/com/cloudwebrtc/webrtc/GetUserMediaImpl.java index 9eb087a90b..43f90a9ede 100755 --- a/android/src/main/java/com/cloudwebrtc/webrtc/GetUserMediaImpl.java +++ b/android/src/main/java/com/cloudwebrtc/webrtc/GetUserMediaImpl.java @@ -428,16 +428,6 @@ public void invoke(Object... args) { void getDisplayMedia( final ConstraintsMap constraints, final Result result, final MediaStream mediaStream) { - ConstraintsMap videoConstraintsMap = null; - ConstraintsMap videoConstraintsMandatory = null; - - if (constraints.getType("video") == ObjectType.Map) { - videoConstraintsMap = constraints.getMap("video"); - if (videoConstraintsMap.hasKey("mandatory") - && videoConstraintsMap.getType("mandatory") == ObjectType.Map) { - videoConstraintsMandatory = videoConstraintsMap.getMap("mandatory"); - } - } screenRequestPremissions( new ResultReceiver(new Handler(Looper.getMainLooper())) { From 1696f496b95a55b8e46d5ccdbf712852a4985ec8 Mon Sep 17 00:00:00 2001 From: David Liu Date: Wed, 1 Dec 2021 21:41:11 +0900 Subject: [PATCH 3/7] use new constraints syntax for video --- .../cloudwebrtc/webrtc/GetUserMediaImpl.java | 56 +++++++++++++++---- 1 file changed, 44 insertions(+), 12 deletions(-) diff --git a/android/src/main/java/com/cloudwebrtc/webrtc/GetUserMediaImpl.java b/android/src/main/java/com/cloudwebrtc/webrtc/GetUserMediaImpl.java index 43f90a9ede..6a5c8b0493 100755 --- a/android/src/main/java/com/cloudwebrtc/webrtc/GetUserMediaImpl.java +++ b/android/src/main/java/com/cloudwebrtc/webrtc/GetUserMediaImpl.java @@ -349,12 +349,6 @@ private AudioTrack getUserAudio(ConstraintsMap constraints) { void getUserMedia( final ConstraintsMap constraints, final Result result, final MediaStream mediaStream) { - // TODO: change getUserMedia constraints format to support new syntax - // constraint format seems changed, and there is no mandatory any more. - // and has a new syntax/attrs to specify resolution - // should change `parseConstraints()` according - // see: https://www.w3.org/TR/mediacapture-streams/#idl-def-MediaTrackConstraints - final ArrayList requestPermissions = new ArrayList<>(); if (constraints.hasKey("audio")) { @@ -615,6 +609,34 @@ private void getUserMedia( private boolean isFacing = true; + /** + * @return Returns the integer at the key, or the `ideal` property if it is a map. + */ + @Nullable + private Integer getConstrainInt(@Nullable ConstraintsMap constraintsMap, String key) { + if(constraintsMap == null){ + return null; + } + + if (constraintsMap.getType(key) == ObjectType.Number) { + try { + return constraintsMap.getInt(key); + } catch (Exception e) { + // Could be a double instead + return (int) Math.round(constraintsMap.getDouble(key)); + } + } + + if (constraintsMap.getType(key) == ObjectType.Map) { + ConstraintsMap innerMap = constraintsMap.getMap(key); + if (constraintsMap.getType("ideal") == ObjectType.Number) { + return innerMap.getInt("ideal"); + } + } + + return null; + } + private VideoTrack getUserVideo(ConstraintsMap constraints) { ConstraintsMap videoConstraintsMap = null; ConstraintsMap videoConstraintsMandatory = null; @@ -626,6 +648,7 @@ private VideoTrack getUserVideo(ConstraintsMap constraints) { } } + Log.i(TAG, "getUserMedia(video): " + videoConstraintsMap); // NOTE: to support Camera2, the device should: @@ -663,16 +686,25 @@ private VideoTrack getUserVideo(ConstraintsMap constraints) { surfaceTextureHelper, applicationContext, videoSource.getCapturerObserver()); VideoCapturerInfo info = new VideoCapturerInfo(); - info.width = - videoConstraintsMandatory != null && videoConstraintsMandatory.hasKey("minWidth") + + Integer videoWidth = getConstrainInt(videoConstraintsMap, "width"); + info.width = videoWidth != null + ? videoWidth + : videoConstraintsMandatory != null && videoConstraintsMandatory.hasKey("minWidth") ? videoConstraintsMandatory.getInt("minWidth") : DEFAULT_WIDTH; - info.height = - videoConstraintsMandatory != null && videoConstraintsMandatory.hasKey("minHeight") + + Integer videoHeight = getConstrainInt(videoConstraintsMap, "height"); + info.height = videoHeight != null + ? videoHeight + : videoConstraintsMandatory != null && videoConstraintsMandatory.hasKey("minHeight") ? videoConstraintsMandatory.getInt("minHeight") : DEFAULT_HEIGHT; - info.fps = - videoConstraintsMandatory != null && videoConstraintsMandatory.hasKey("minFrameRate") + + Integer videoFrameRate = getConstrainInt(videoConstraintsMap, "frameRate"); + info.fps = videoFrameRate != null + ? videoFrameRate + : videoConstraintsMandatory != null && videoConstraintsMandatory.hasKey("minFrameRate") ? videoConstraintsMandatory.getInt("minFrameRate") : DEFAULT_FPS; info.capturer = videoCapturer; From 57b73478ae1fae01b70c64e40df64dee890bdad3 Mon Sep 17 00:00:00 2001 From: David Liu Date: Wed, 1 Dec 2021 22:15:09 +0900 Subject: [PATCH 4/7] constraints map toString --- .../java/com/cloudwebrtc/webrtc/utils/ConstraintsMap.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/android/src/main/java/com/cloudwebrtc/webrtc/utils/ConstraintsMap.java b/android/src/main/java/com/cloudwebrtc/webrtc/utils/ConstraintsMap.java index 08291dcc50..f447e76607 100644 --- a/android/src/main/java/com/cloudwebrtc/webrtc/utils/ConstraintsMap.java +++ b/android/src/main/java/com/cloudwebrtc/webrtc/utils/ConstraintsMap.java @@ -128,4 +128,11 @@ public ConstraintsArray getArray(String name){ public ArrayList getListArray(String name){ return (ArrayList) mMap.get(name); } + + @Override + public String toString() { + return "ConstraintsMap{" + + "mMap=" + mMap + + '}'; + } } From e5ae1f55d75039016ecd1cf00732b152289bcab7 Mon Sep 17 00:00:00 2001 From: David Liu Date: Wed, 1 Dec 2021 22:34:07 +0900 Subject: [PATCH 5/7] constraint comment todo --- .../com/cloudwebrtc/webrtc/utils/MediaConstraintsUtils.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/android/src/main/java/com/cloudwebrtc/webrtc/utils/MediaConstraintsUtils.java b/android/src/main/java/com/cloudwebrtc/webrtc/utils/MediaConstraintsUtils.java index ce41031a8c..3ba4ae9824 100644 --- a/android/src/main/java/com/cloudwebrtc/webrtc/utils/MediaConstraintsUtils.java +++ b/android/src/main/java/com/cloudwebrtc/webrtc/utils/MediaConstraintsUtils.java @@ -24,6 +24,11 @@ public class MediaConstraintsUtils { public static MediaConstraints parseMediaConstraints(ConstraintsMap constraints) { MediaConstraints mediaConstraints = new MediaConstraints(); + // TODO: change getUserMedia constraints format to support new syntax + // constraint format seems changed, and there is no mandatory any more. + // and has a new syntax/attrs to specify resolution + // should change `parseConstraints()` according + // see: https://www.w3.org/TR/mediacapture-streams/#idl-def-MediaTrackConstraints if (constraints.hasKey("mandatory") && constraints.getType("mandatory") == ObjectType.Map) { parseConstraints(constraints.getMap("mandatory"), From cb2a42c1dd29b2a589dc880c87e2c7b39ba32fde Mon Sep 17 00:00:00 2001 From: David Liu Date: Thu, 2 Dec 2021 14:08:16 +0900 Subject: [PATCH 6/7] darwin version of video constraints syntax --- common/darwin/Classes/FlutterRTCMediaStream.m | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/common/darwin/Classes/FlutterRTCMediaStream.m b/common/darwin/Classes/FlutterRTCMediaStream.m index afda726304..9cf717b5e2 100755 --- a/common/darwin/Classes/FlutterRTCMediaStream.m +++ b/common/darwin/Classes/FlutterRTCMediaStream.m @@ -195,6 +195,34 @@ - (void)getUserMedia:(NSDictionary *)constraints successCallback(mediaStream); } +- (int)getConstrainInt:(NSDictionary *)constraints + forKey:(NSString *)key { + + if (![constraints isKindOfClass:[NSDictionary class]]) { + return 0; + } + + id constraint = constraints[key]; + if ([constraint isKindOfClass:[NSNumber class]]) { + return [constraint intValue]; + } else if ([constraint isKindOfClass:[NSString class]]) { + int possibleValue = [constraint intValue]; + if (possibleValue != 0) { + return possibleValue; + } + } else if ([constraint isKindOfClass:[NSDictionary class]]) { + id idealConstraint = constraint[@"ideal"]; + if([idealConstraint isKindOfClass: [NSString class]]) { + int possibleValue = [idealConstraint intValue]; + if(possibleValue != 0){ + return possibleValue; + } + } + } + + return 0; +} + /** * Initializes a new {@link RTCVideoTrack} which satisfies specific constraints, * adds it to a specific {@link RTCMediaStream}, and reports success to a @@ -301,6 +329,21 @@ - (void)getUserVideo:(NSDictionary *)constraints } } + int possibleWidth = [self getConstrainInt:videoConstraints forKey:@"width"]; + if(possibleWidth != 0){ + self._targetWidth = possibleWidth; + } + + int possibleHeight = [self getConstrainInt:videoConstraints forKey:@"height"]; + if(possibleHeight != 0){ + self._targetHeight = possibleHeight; + } + + int possibleFps = [self getConstrainInt:videoConstraints forKey:@"frameRate"]; + if(possibleFps != 0){ + self._targetFps = possibleFps; + } + if (videoDevice) { RTCVideoSource *videoSource = [self.peerConnectionFactory videoSource]; if (self.videoCapturer) { From e5fc1095d73599dd032b94ed1bf9761c82bd7851 Mon Sep 17 00:00:00 2001 From: David Liu Date: Fri, 3 Dec 2021 00:25:44 +0900 Subject: [PATCH 7/7] cpp version of video constraints --- common/cpp/src/flutter_media_stream.cc | 40 ++++++++++++++++++++++---- 1 file changed, 34 insertions(+), 6 deletions(-) diff --git a/common/cpp/src/flutter_media_stream.cc b/common/cpp/src/flutter_media_stream.cc index 58ec4f6d6e..ada3358f38 100644 --- a/common/cpp/src/flutter_media_stream.cc +++ b/common/cpp/src/flutter_media_stream.cc @@ -133,6 +133,26 @@ std::string getSourceIdConstraint(const EncodableMap& mediaConstraints) { return ""; } +EncodableValue getConstrainInt(const EncodableMap& constraints, const std::string& key) { + EncodableValue value; + auto it = constraints.find(EncodableValue(key)); + if(it != constraints.end()) { + if(TypeIs(it->second)) { + return it->second; + } + + if(TypeIs(it->second)) { + EncodableMap innerMap = GetValue(it->second); + auto it2 = innerMap.find(EncodableValue("ideal")); + if(it2 != constraints.end() && TypeIs(it2->second)){ + return it2->second; + } + } + } + + return EncodableValue(); +} + void FlutterMediaStream::GetUserVideo(const EncodableMap& constraints, scoped_refptr stream, EncodableMap& params) { @@ -140,10 +160,10 @@ void FlutterMediaStream::GetUserVideo(const EncodableMap& constraints, EncodableMap video_mandatory; auto it = constraints.find(EncodableValue("video")); if (it != constraints.end() && TypeIs(it->second)) { - EncodableMap video_map = GetValue(it->second); - if (video_map.find(EncodableValue("mandatory")) != video_map.end()) { + video_constraints = GetValue(it->second); + if (video_constraints.find(EncodableValue("mandatory")) != video_constraints.end()) { video_mandatory = - GetValue(video_map.find(EncodableValue("mandatory"))->second); + GetValue(video_constraints.find(EncodableValue("mandatory"))->second); } } @@ -151,18 +171,26 @@ void FlutterMediaStream::GetUserVideo(const EncodableMap& constraints, //bool isFacing = facing_mode == "" || facing_mode != "environment"; std::string sourceId = getSourceIdConstraint(video_constraints); - EncodableValue widthValue = findEncodableValue(video_mandatory, "minWidth"); + EncodableValue widthValue = getConstrainInt(video_constraints, "width"); + + if (widthValue == EncodableValue()) + widthValue = findEncodableValue(video_mandatory, "minWidth"); if (widthValue == EncodableValue()) widthValue = findEncodableValue(video_mandatory, "width"); - EncodableValue heightValue = findEncodableValue(video_mandatory, "minHeight"); + EncodableValue heightValue = getConstrainInt(video_constraints, "height"); + + if(heightValue == EncodableValue()) + heightValue = findEncodableValue(video_mandatory, "minHeight"); if (heightValue == EncodableValue()) heightValue = findEncodableValue(video_mandatory, "height"); + EncodableValue fpsValue = getConstrainInt(video_constraints, "frameRate"); - EncodableValue fpsValue = findEncodableValue(video_mandatory, "minFrameRate"); + if(fpsValue == EncodableValue()) + fpsValue = findEncodableValue(video_mandatory, "minFrameRate"); if (fpsValue == EncodableValue()) fpsValue = findEncodableValue(video_mandatory, "frameRate");