From 5a9f4b444631c95b01136e91555a70c258428cc9 Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Wed, 20 Oct 2021 01:25:57 +0900 Subject: [PATCH 1/2] enable simulcast --- .../webrtc/MethodCallHandlerImpl.java | 3 +- ...SimulcastVideoEncoderFactoryWrapper.kt.txt | 207 ++++++++++++++++++ 2 files changed, 209 insertions(+), 1 deletion(-) create mode 100644 android/src/main/java/com/cloudwebrtc/webrtc/SimulcastVideoEncoderFactoryWrapper.kt.txt diff --git a/android/src/main/java/com/cloudwebrtc/webrtc/MethodCallHandlerImpl.java b/android/src/main/java/com/cloudwebrtc/webrtc/MethodCallHandlerImpl.java index 3d0b0001d8..a976cde81a 100644 --- a/android/src/main/java/com/cloudwebrtc/webrtc/MethodCallHandlerImpl.java +++ b/android/src/main/java/com/cloudwebrtc/webrtc/MethodCallHandlerImpl.java @@ -23,6 +23,7 @@ import org.webrtc.CryptoOptions; import org.webrtc.DefaultVideoDecoderFactory; import org.webrtc.DefaultVideoEncoderFactory; +import org.webrtc.SimulcastVideoEncoderFactoryWrapper; import org.webrtc.DtmfSender; import org.webrtc.EglBase; import org.webrtc.IceCandidate; @@ -155,7 +156,7 @@ private void ensureInitialized() { mFactory = PeerConnectionFactory.builder() .setOptions(new Options()) - .setVideoEncoderFactory(new DefaultVideoEncoderFactory(eglContext, false, true)) + .setVideoEncoderFactory(new SimulcastVideoEncoderFactoryWrapper(eglContext, true, false)) .setVideoDecoderFactory(new DefaultVideoDecoderFactory(eglContext)) .setAudioDeviceModule(audioDeviceModule) .createPeerConnectionFactory(); diff --git a/android/src/main/java/com/cloudwebrtc/webrtc/SimulcastVideoEncoderFactoryWrapper.kt.txt b/android/src/main/java/com/cloudwebrtc/webrtc/SimulcastVideoEncoderFactoryWrapper.kt.txt new file mode 100644 index 0000000000..b1139a9855 --- /dev/null +++ b/android/src/main/java/com/cloudwebrtc/webrtc/SimulcastVideoEncoderFactoryWrapper.kt.txt @@ -0,0 +1,207 @@ +package io.livekit.android.webrtc + +import io.livekit.android.util.LKLog +import org.webrtc.* +import java.util.concurrent.* + +/* +Copyright 2017, Lyo Kato (Original Author) +Copyright 2017-2021, Shiguredo Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + */ +internal class SimulcastVideoEncoderFactoryWrapper( + sharedContext: EglBase.Context?, + enableIntelVp8Encoder: Boolean, + enableH264HighProfile: Boolean +) : VideoEncoderFactory { + + /** + * Factory that prioritizes software encoder. + * + * When the selected codec can't be handled by the software encoder, + * it uses the hardware encoder as a fallback. However, this class is + * primarily used to address an issue in libwebrtc, and does not have + * purposeful usecase itself. + * + * To use simulcast in libwebrtc, SimulcastEncoderAdapter is used. + * SimulcastEncoderAdapter takes in a primary and fallback encoder. + * If HardwareVideoEncoderFactory and SoftwareVideoEncoderFactory are + * passed in directly as primary and fallback, when H.264 is used, + * libwebrtc will crash. + * + * This is because SoftwareVideoEncoderFactory does not handle H.264, + * so [SoftwareVideoEncoderFactory.createEncoder] returns null, and + * the libwebrtc side does not handle nulls, regardless of whether the + * fallback is actually used or not. + * + * To avoid nulls, we simply pass responsibility over to the HardwareVideoEncoderFactory. + * This results in HardwareVideoEncoderFactory being both the primary and fallback, + * but there aren't any specific problems in doing so. + */ + private class Fallback(private val hardwareVideoEncoderFactory: VideoEncoderFactory) : + VideoEncoderFactory { + + private val softwareVideoEncoderFactory: VideoEncoderFactory = SoftwareVideoEncoderFactory() + + override fun createEncoder(info: VideoCodecInfo): VideoEncoder? { + val softwareEncoder = softwareVideoEncoderFactory.createEncoder(info) + val hardwareEncoder = hardwareVideoEncoderFactory.createEncoder(info) + return if (hardwareEncoder != null && softwareEncoder != null) { + VideoEncoderFallback(hardwareEncoder, softwareEncoder) + } else { + softwareEncoder ?: hardwareEncoder + } + } + + override fun getSupportedCodecs(): Array { + val supportedCodecInfos: MutableList = mutableListOf() + supportedCodecInfos.addAll(softwareVideoEncoderFactory.supportedCodecs) + supportedCodecInfos.addAll(hardwareVideoEncoderFactory.supportedCodecs) + return supportedCodecInfos.toTypedArray() + } + + } + + /** + * Wraps each stream encoder and performs the following: + * - Starts up a single thread + * - When the width/height from [initEncode] doesn't match the frame buffer's, + * scales the frame prior to encoding. + * - Always calls the encoder on the thread. + */ + private class StreamEncoderWrapper(private val encoder: VideoEncoder) : VideoEncoder { + + val executor: ExecutorService = Executors.newSingleThreadExecutor() + var streamSettings: VideoEncoder.Settings? = null + + override fun initEncode( + settings: VideoEncoder.Settings, + callback: VideoEncoder.Callback? + ): VideoCodecStatus { + streamSettings = settings + val future = executor.submit(Callable { + LKLog.i { + """initEncode() thread=${Thread.currentThread().name} [${Thread.currentThread().id}] + | streamSettings: + | numberOfCores=${settings.numberOfCores} + | width=${settings.width} + | height=${settings.height} + | startBitrate=${settings.startBitrate} + | maxFramerate=${settings.maxFramerate} + | automaticResizeOn=${settings.automaticResizeOn} + | numberOfSimulcastStreams=${settings.numberOfSimulcastStreams} + | lossNotification=${settings.capabilities.lossNotification} + """.trimMargin() + } + return@Callable encoder.initEncode(settings, callback) + }) + return future.get() + } + + override fun release(): VideoCodecStatus { + val future = executor.submit(Callable { return@Callable encoder.release() }) + return future.get() + } + + override fun encode( + frame: VideoFrame, + encodeInfo: VideoEncoder.EncodeInfo? + ): VideoCodecStatus { + val future = executor.submit(Callable { + //LKLog.d { "encode() buffer=${frame.buffer}, thread=${Thread.currentThread().name} " + + // "[${Thread.currentThread().id}]" } + if (streamSettings == null) { + return@Callable encoder.encode(frame, encodeInfo) + } else if (frame.buffer.width == streamSettings!!.width) { + return@Callable encoder.encode(frame, encodeInfo) + } else { + // The incoming buffer is different than the streamSettings received in initEncode() + // Need to scale. + val originalBuffer = frame.buffer + // TODO: Do we need to handle when the scale factor is weird? + val adaptedBuffer = originalBuffer.cropAndScale( + 0, 0, originalBuffer.width, originalBuffer.height, + streamSettings!!.width, streamSettings!!.height + ) + val adaptedFrame = VideoFrame(adaptedBuffer, frame.rotation, frame.timestampNs) + val result = encoder.encode(adaptedFrame, encodeInfo) + adaptedBuffer.release() + return@Callable result + } + }) + return future.get() + } + + override fun setRateAllocation( + allocation: VideoEncoder.BitrateAllocation?, + frameRate: Int + ): VideoCodecStatus { + val future = executor.submit(Callable { + return@Callable encoder.setRateAllocation( + allocation, + frameRate + ) + }) + return future.get() + } + + override fun getScalingSettings(): VideoEncoder.ScalingSettings { + val future = executor.submit(Callable { return@Callable encoder.scalingSettings }) + return future.get() + } + + override fun getImplementationName(): String { + val future = executor.submit(Callable { return@Callable encoder.implementationName }) + return future.get() + } + } + + private class StreamEncoderWrapperFactory(private val factory: VideoEncoderFactory) : + VideoEncoderFactory { + override fun createEncoder(videoCodecInfo: VideoCodecInfo?): VideoEncoder? { + val encoder = factory.createEncoder(videoCodecInfo) + if (encoder == null) { + return null + } + return StreamEncoderWrapper(encoder) + } + + override fun getSupportedCodecs(): Array { + return factory.supportedCodecs + } + } + + + private val primary: VideoEncoderFactory + private val fallback: VideoEncoderFactory + private val native: SimulcastVideoEncoderFactory + + init { + val hardwareVideoEncoderFactory = HardwareVideoEncoderFactory( + sharedContext, enableIntelVp8Encoder, enableH264HighProfile + ) + primary = StreamEncoderWrapperFactory(hardwareVideoEncoderFactory) + fallback = StreamEncoderWrapperFactory(Fallback(primary)) + native = SimulcastVideoEncoderFactory(primary, fallback) + } + + override fun createEncoder(info: VideoCodecInfo?): VideoEncoder? { + return native.createEncoder(info) + } + + override fun getSupportedCodecs(): Array { + return native.supportedCodecs + } + +} \ No newline at end of file From 8d98bc83d9ea5ca1aae404eaa91f2c8726fb3de7 Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Wed, 20 Oct 2021 03:12:24 +0900 Subject: [PATCH 2/2] turn on kotlin compiler --- NOTICE | 24 +++++++++++++++ android/build.gradle | 12 ++++++-- .../webrtc/MethodCallHandlerImpl.java | 2 +- ...=> SimulcastVideoEncoderFactoryWrapper.kt} | 30 +++++++++---------- 4 files changed, 50 insertions(+), 18 deletions(-) create mode 100644 NOTICE rename android/src/main/java/com/cloudwebrtc/webrtc/{SimulcastVideoEncoderFactoryWrapper.kt.txt => SimulcastVideoEncoderFactoryWrapper.kt} (90%) diff --git a/NOTICE b/NOTICE new file mode 100644 index 0000000000..05e94cd7b1 --- /dev/null +++ b/NOTICE @@ -0,0 +1,24 @@ +################################################################################### + +The following modifications follow Apache License 2.0 from shiguredo. + +SimulcastVideoEncoderFactoryWrapper.kt + +Apache License 2.0 + +Copyright 2017, Lyo Kato (Original Author) +Copyright 2017-2021, Shiguredo Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +##################################################################################### diff --git a/android/build.gradle b/android/build.gradle index 4444ba2e7a..d9a0715ec1 100644 --- a/android/build.gradle +++ b/android/build.gradle @@ -2,6 +2,7 @@ group 'com.cloudwebrtc.webrtc' version '1.0-SNAPSHOT' buildscript { + ext.kotlin_version = '1.3.50' repositories { google() jcenter() @@ -9,6 +10,7 @@ buildscript { dependencies { classpath 'com.android.tools.build:gradle:3.6.3' + classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:$kotlin_version" } } @@ -21,13 +23,14 @@ rootProject.allprojects { } apply plugin: 'com.android.library' +apply plugin: 'kotlin-android' android { compileSdkVersion 30 defaultConfig { minSdkVersion 21 - testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner" + testInstrumentationRunner 'androidx.test.runner.AndroidJUnitRunner' consumerProguardFiles 'proguard-rules.pro' } @@ -39,9 +42,14 @@ android { sourceCompatibility JavaVersion.VERSION_1_8 targetCompatibility JavaVersion.VERSION_1_8 } + + kotlinOptions { + jvmTarget = '1.8' + } } dependencies { implementation 'com.github.webrtc-sdk:android:92.4515.01' - implementation "androidx.annotation:annotation:1.1.0" + implementation 'androidx.annotation:annotation:1.1.0' + implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk7:$kotlin_version" } diff --git a/android/src/main/java/com/cloudwebrtc/webrtc/MethodCallHandlerImpl.java b/android/src/main/java/com/cloudwebrtc/webrtc/MethodCallHandlerImpl.java index 8011b133e4..0004ecbe56 100644 --- a/android/src/main/java/com/cloudwebrtc/webrtc/MethodCallHandlerImpl.java +++ b/android/src/main/java/com/cloudwebrtc/webrtc/MethodCallHandlerImpl.java @@ -18,12 +18,12 @@ import com.cloudwebrtc.webrtc.utils.ConstraintsMap; import com.cloudwebrtc.webrtc.utils.EglUtils; import com.cloudwebrtc.webrtc.utils.ObjectType; +import com.cloudwebrtc.webrtc.SimulcastVideoEncoderFactoryWrapper; import org.webrtc.AudioTrack; import org.webrtc.CryptoOptions; import org.webrtc.DefaultVideoDecoderFactory; import org.webrtc.DefaultVideoEncoderFactory; -import org.webrtc.SimulcastVideoEncoderFactoryWrapper; import org.webrtc.DtmfSender; import org.webrtc.EglBase; import org.webrtc.IceCandidate; diff --git a/android/src/main/java/com/cloudwebrtc/webrtc/SimulcastVideoEncoderFactoryWrapper.kt.txt b/android/src/main/java/com/cloudwebrtc/webrtc/SimulcastVideoEncoderFactoryWrapper.kt similarity index 90% rename from android/src/main/java/com/cloudwebrtc/webrtc/SimulcastVideoEncoderFactoryWrapper.kt.txt rename to android/src/main/java/com/cloudwebrtc/webrtc/SimulcastVideoEncoderFactoryWrapper.kt index b1139a9855..85d99a2d94 100644 --- a/android/src/main/java/com/cloudwebrtc/webrtc/SimulcastVideoEncoderFactoryWrapper.kt.txt +++ b/android/src/main/java/com/cloudwebrtc/webrtc/SimulcastVideoEncoderFactoryWrapper.kt @@ -1,6 +1,6 @@ -package io.livekit.android.webrtc +package com.cloudwebrtc.webrtc -import io.livekit.android.util.LKLog +import android.util.Log import org.webrtc.* import java.util.concurrent.* @@ -91,19 +91,19 @@ internal class SimulcastVideoEncoderFactoryWrapper( ): VideoCodecStatus { streamSettings = settings val future = executor.submit(Callable { - LKLog.i { - """initEncode() thread=${Thread.currentThread().name} [${Thread.currentThread().id}] - | streamSettings: - | numberOfCores=${settings.numberOfCores} - | width=${settings.width} - | height=${settings.height} - | startBitrate=${settings.startBitrate} - | maxFramerate=${settings.maxFramerate} - | automaticResizeOn=${settings.automaticResizeOn} - | numberOfSimulcastStreams=${settings.numberOfSimulcastStreams} - | lossNotification=${settings.capabilities.lossNotification} - """.trimMargin() - } + // LKLog.i { + // """initEncode() thread=${Thread.currentThread().name} [${Thread.currentThread().id}] + // | streamSettings: + // | numberOfCores=${settings.numberOfCores} + // | width=${settings.width} + // | height=${settings.height} + // | startBitrate=${settings.startBitrate} + // | maxFramerate=${settings.maxFramerate} + // | automaticResizeOn=${settings.automaticResizeOn} + // | numberOfSimulcastStreams=${settings.numberOfSimulcastStreams} + // | lossNotification=${settings.capabilities.lossNotification} + // """.trimMargin() + // } return@Callable encoder.initEncode(settings, callback) }) return future.get()