flutter-webrtc · cloudwebrtc · Oct 20, 2021 · Oct 19, 2021 · Oct 19, 2021 · Oct 19, 2021
diff --git a/NOTICE b/NOTICE
@@ -0,0 +1,24 @@
+###################################################################################
+
+The following modifications follow Apache License 2.0 from shiguredo.
+
+SimulcastVideoEncoderFactoryWrapper.kt
+
+Apache License 2.0
+
+Copyright 2017, Lyo Kato <lyo.kato at gmail.com> (Original Author)
+Copyright 2017-2021, Shiguredo Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+#####################################################################################
diff --git a/android/build.gradle b/android/build.gradle
@@ -2,13 +2,15 @@ group 'com.cloudwebrtc.webrtc'
 version '1.0-SNAPSHOT'
 
 buildscript {
+    ext.kotlin_version = '1.3.50'
     repositories {
         google()
         jcenter()
     }
 
     dependencies {
         classpath 'com.android.tools.build:gradle:3.6.3'
+        classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:$kotlin_version"
     }
 }
 
@@ -21,13 +23,14 @@ rootProject.allprojects {
 }
 
 apply plugin: 'com.android.library'
+apply plugin: 'kotlin-android'
 
 android {
     compileSdkVersion 30
 
     defaultConfig {
         minSdkVersion 21
-        testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner"
+        testInstrumentationRunner 'androidx.test.runner.AndroidJUnitRunner'
         consumerProguardFiles 'proguard-rules.pro'
     }
 
@@ -39,9 +42,14 @@ android {
         sourceCompatibility JavaVersion.VERSION_1_8
         targetCompatibility JavaVersion.VERSION_1_8
     }
+
+    kotlinOptions {
+        jvmTarget = '1.8'
+    }
 }
 
 dependencies {
     implementation 'com.github.webrtc-sdk:android:92.4515.01'
-    implementation "androidx.annotation:annotation:1.1.0"
+    implementation 'androidx.annotation:annotation:1.1.0'
+    implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk7:$kotlin_version"
 }
diff --git a/android/src/main/java/com/cloudwebrtc/webrtc/MethodCallHandlerImpl.java b/android/src/main/java/com/cloudwebrtc/webrtc/MethodCallHandlerImpl.java
@@ -18,6 +18,7 @@
 import com.cloudwebrtc.webrtc.utils.ConstraintsMap;
 import com.cloudwebrtc.webrtc.utils.EglUtils;
 import com.cloudwebrtc.webrtc.utils.ObjectType;
+import com.cloudwebrtc.webrtc.SimulcastVideoEncoderFactoryWrapper;
 
 import org.webrtc.AudioTrack;
 import org.webrtc.CryptoOptions;
@@ -155,7 +156,7 @@ private void ensureInitialized() {
 
     mFactory = PeerConnectionFactory.builder()
             .setOptions(new Options())
-            .setVideoEncoderFactory(new DefaultVideoEncoderFactory(eglContext, false, true))
+            .setVideoEncoderFactory(new SimulcastVideoEncoderFactoryWrapper(eglContext, true, false))
             .setVideoDecoderFactory(new DefaultVideoDecoderFactory(eglContext))
             .setAudioDeviceModule(audioDeviceModule)
             .createPeerConnectionFactory();

diff --git a/android/src/main/java/com/cloudwebrtc/webrtc/SimulcastVideoEncoderFactoryWrapper.kt b/android/src/main/java/com/cloudwebrtc/webrtc/SimulcastVideoEncoderFactoryWrapper.kt
@@ -0,0 +1,207 @@
+package com.cloudwebrtc.webrtc
+
+import android.util.Log
+import org.webrtc.*
+import java.util.concurrent.*
+
+/*
+Copyright 2017, Lyo Kato <lyo.kato at gmail.com> (Original Author)
+Copyright 2017-2021, Shiguredo Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+ */
+internal class SimulcastVideoEncoderFactoryWrapper(
+    sharedContext: EglBase.Context?,
+    enableIntelVp8Encoder: Boolean,
+    enableH264HighProfile: Boolean
+) : VideoEncoderFactory {
+
+    /**
+     * Factory that prioritizes software encoder.
+     *
+     * When the selected codec can't be handled by the software encoder,
+     * it uses the hardware encoder as a fallback. However, this class is
+     * primarily used to address an issue in libwebrtc, and does not have
+     * purposeful usecase itself.
+     *
+     * To use simulcast in libwebrtc, SimulcastEncoderAdapter is used.
+     * SimulcastEncoderAdapter takes in a primary and fallback encoder.
+     * If HardwareVideoEncoderFactory and SoftwareVideoEncoderFactory are
+     * passed in directly as primary and fallback, when H.264 is used,
+     * libwebrtc will crash.
+     *
+     * This is because SoftwareVideoEncoderFactory does not handle H.264,
+     * so [SoftwareVideoEncoderFactory.createEncoder] returns null, and
+     * the libwebrtc side does not handle nulls, regardless of whether the
+     * fallback is actually used or not.
+     *
+     * To avoid nulls, we simply pass responsibility over to the HardwareVideoEncoderFactory.
+     * This results in HardwareVideoEncoderFactory being both the primary and fallback,
+     * but there aren't any specific problems in doing so.
+     */
+    private class Fallback(private val hardwareVideoEncoderFactory: VideoEncoderFactory) :
+        VideoEncoderFactory {
+
+        private val softwareVideoEncoderFactory: VideoEncoderFactory = SoftwareVideoEncoderFactory()
+
+        override fun createEncoder(info: VideoCodecInfo): VideoEncoder? {
+            val softwareEncoder = softwareVideoEncoderFactory.createEncoder(info)
+            val hardwareEncoder = hardwareVideoEncoderFactory.createEncoder(info)
+            return if (hardwareEncoder != null && softwareEncoder != null) {
+                VideoEncoderFallback(hardwareEncoder, softwareEncoder)
+            } else {
+                softwareEn
E29B
coder ?: hardwareEncoder
+            }
+        }
+
+        override fun getSupportedCodecs(): Array<VideoCodecInfo> {
+            val supportedCodecInfos: MutableList<VideoCodecInfo> = mutableListOf()
+            supportedCodecInfos.addAll(softwareVideoEncoderFactory.supportedCodecs)
+            supportedCodecInfos.addAll(hardwareVideoEncoderFactory.supportedCodecs)
+            return supportedCodecInfos.toTypedArray()
+        }
+
+    }
+
+    /**
+     * Wraps each stream encoder and performs the following:
+     * - Starts up a single thread
+     * - When the width/height from [initEncode] doesn't match the frame buffer's,
+     *   scales the frame prior to encoding.
+     * - Always calls the encoder on the thread.
+     */
+    private class StreamEncoderWrapper(private val encoder: VideoEncoder) : VideoEncoder {
+
+        val executor: ExecutorService = Executors.newSingleThreadExecutor()
+        var streamSettings: VideoEncoder.Settings? = null
+
+        override fun initEncode(
+            settings: VideoEncoder.Settings,
+            callback: VideoEncoder.Callback?
+        ): VideoCodecStatus {
+            streamSettings = settings
+            val future = executor.submit(Callable {
+            //     LKLog.i {
+            //         """initEncode() thread=${Thread.currentThread().name} [${Thread.currentThread().id}]
+            //     |  streamSettings:
+            //     |    numberOfCores=${settings.numberOfCores}
+            //     |    width=${settings.width}
+            //     |    height=${settings.height}
+            //     |    startBitrate=${settings.startBitrate}
+            //     |    maxFramerate=${settings.maxFramerate}
+            //     |    automaticResizeOn=${settings.automaticResizeOn}
+            //     |    numberOfSimulcastStreams=${settings.numberOfSimulcastStreams}
+            //     |    lossNotification=${settings.capabilities.lossNotification}
+            // """.trimMargin()
+            //     }
+                return@Callable encoder.initEncode(settings, callback)
+            })
+            return future.get()
+        }
+
+        override fun release(): VideoCodecStatus {
+            val future = executor.submit(Callable { return@Callable encoder.release() })
+            return future.get()
+        }
+
+        override fun encode(
+            frame: VideoFrame,
+            encodeInfo: VideoEncoder.EncodeInfo?
+        ): VideoCodecStatus {
+            val future = executor.submit(Callable {
+                //LKLog.d { "encode() buffer=${frame.buffer}, thread=${Thread.currentThread().name} " +
+                //        "[${Thread.currentThread().id}]" }
+                if (streamSettings == null) {
+                    return@Callable encoder.encode(frame, encodeInfo)
+                } else if (frame.buffer.width == streamSettings!!.width) {
+                    return@Callable encoder.encode(frame, encodeInfo)
+                } else {
+                    // The incoming buffer is different than the streamSettings received in initEncode()
+                    // Need to scale.
+                    val originalBuffer = frame.buffer
+                    // TODO: Do we need to handle when the scale factor is weird?
+                    val adaptedBuffer = originalBuffer.cropAndScale(
+                        0, 0, originalBuffer.width, originalBuffer.height,
+                        streamSettings!!.width, streamSettings!!.height
+                    )
+                    val adaptedFrame = VideoFrame(adaptedBuffer, frame.rotation, frame.timestampNs)
+                    val result = encoder.encode(adaptedFrame, encodeInfo)
+                    adaptedBuffer.release()
+                    return@Callable result
+                }
+            })
+            return future.get()
+        }
+
+        override fun setRateAllocation(
+            allocation: VideoEncoder.BitrateAllocation?,
+            frameRate: Int
+        ): VideoCodecStatus {
+            val future = executor.submit(Callable {
+                return@Callable encoder.setRateAllocation(
+                    allocation,
+                    frameRate
+                )
+            })
+            return future.get()
+        }
+
+        override fun getScalingSettings(): VideoEncoder.ScalingSettings {
+            val future = executor.submit(Callable { return@Callable encoder.scalingSettings })
+            return future.get()
+        }
+
+        override fun getImplementationName(): String {
+            val future = executor.submit(Callable { return@Callable encoder.implementationName })
+            return future.get()
+        }
+    }
+
+    private class StreamEncoderWrapperFactory(private val factory: VideoEncoderFactory) :
+        VideoEncoderFactory {
+        override fun createEncoder(videoCodecInfo: VideoCodecInfo?): VideoEncoder? {
+            val encoder = factory.createEncoder(videoCodecInfo)
+            if (encoder == null) {
+                return null
+            }
+            return StreamEncoderWrapper(encoder)
+        }
+
+        override fun getSupportedCodecs(): Array<VideoCodecInfo> {
+            return factory.supportedCodecs
+        }
+    }
+
+
+    private val primary: VideoEncoderFactory
+    private val fallback: VideoEncoderFactory
+    private val native: SimulcastVideoEncoderFactory
+
+    init {
+        val hardwareVideoEncoderFactory = HardwareVideoEncoderFactory(
+            sharedContext, enableIntelVp8Encoder, enableH264HighProfile
+        )
+        primary = StreamEncoderWrapperFactory(hardwareVideoEncoderFactory)
+        fallback = StreamEncoderWrapperFactory(Fallback(primary))
+        native = SimulcastVideoEncoderFactory(primary, fallback)
+    }
+
+    override fun createEncoder(info: VideoCodecInfo?): VideoEncoder? {
+        return native.createEncoder(info)
+    }
+
+    override fun getSupportedCodecs(): Array<VideoCodecInfo> {
+        return native.supportedCodecs
+    }
+
+}