fix: hw decoder rendering delay after frame resize

.gitignore

@@ -17,4 +17,12 @@ examples/GumTestApp_macOS/package-lock.json
 *.zip
 lib/
 src/*.js
-
+# Android/IntelliJ
+#
+build/
+.idea
+.gradle
+local.properties
+*.iml
+*.hprof
+.cxx/

android/build.gradle

@@ -1,15 +1,48 @@
 import java.nio.file.Paths
 
-apply plugin: 'com.android.library'
+buildscript {
+  ext.getExtOrDefault = {name, fallback ->
+    return rootProject.ext.has(name) ? rootProject.ext.get(name) : fallback
+  }
+
+  repositories {
+    google()
+    mavenCentral()
+    maven { url 'https://central.sonatype.com/repository/maven-snapshots/' }
+  }
+
+  dependencies {
+    classpath("com.android.tools.build:gradle:7.3.1")
+    // noinspection DifferentKotlinGradleVersion
+    classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:${getExtOrDefault('kotlinVersion', '1.8.10')}"
+  }
+}
+
+apply plugin: "com.android.library"
+apply plugin: "kotlin-android"
 
 def safeExtGet(prop, fallback) {
     rootProject.ext.has(prop) ? rootProject.ext.get(prop) : fallback
 }
 
+def supportsNamespace() {
+  def parsed = com.android.Version.ANDROID_GRADLE_PLUGIN_VERSION.tokenize('.')
+  def major = parsed[0].toInteger()
+  def minor = parsed[1].toInteger()
+
+  // Namespace support was added in 7.3.0
+  return (major == 7 && minor >= 3) || major >= 8
+}
+
 android {
-    def agpVersion = com.android.Version.ANDROID_GRADLE_PLUGIN_VERSION
-    if (agpVersion.tokenize('.')[0].toInteger() >= 7) {
-      namespace "com.oney.WebRTCModule"
+    if (supportsNamespace()) {
+        namespace "com.oney.WebRTCModule"
+
+        sourceSets {
+            main {
+                manifest.srcFile "src/main/AndroidManifestNew.xml"
+            }
+        }
     }
 
     compileSdkVersion safeExtGet('compileSdkVersion', 24)
@@ -31,7 +64,8 @@ android {
 }
 
 dependencies {
-    api 'io.getstream:stream-webrtc-android:1.3.8'
+    api 'io.getstream:stream-webrtc-android:1.3.9'
     implementation 'com.facebook.react:react-native:+'
+    implementation "org.jetbrains.kotlin:kotlin-stdlib:${getExtOrDefault('kotlinVersion', '1.8.10')}"
     implementation "androidx.core:core:1.7.0"
 }

android/src/main/AndroidManifestNew.xml

@@ -0,0 +1,10 @@
+<manifest xmlns:android="http://schemas.android.com/apk/res/android"
+          xmlns:tools="http://schemas.android.com/tools"
+    >
+    <application>
+        <service
+                android:name=".MediaProjectionService"
+                android:foregroundServiceType="mediaProjection">
+        </service>
+    </application>
+</manifest>

android/src/main/java/com/oney/WebRTCModule/EglUtils.java

@@ -1,16 +1,11 @@
 package com.oney.WebRTCModule;
-
-import android.os.Build.VERSION;
-import android.util.Log;
-
 import org.webrtc.EglBase;
 
 public class EglUtils {
     /**
      * The root {@link EglBase} instance shared by the entire application for
      * the sake of reducing the utilization of system resources (such as EGL
-     * contexts). It selects between {@link EglBase10} and {@link EglBase14}
-     * by performing a runtime check.
+     * contexts).
      */
     private static EglBase rootEglBase;
 
@@ -20,47 +15,13 @@ public class EglUtils {
      */
     public static synchronized EglBase getRootEglBase() {
         if (rootEglBase == null) {
-            // XXX EglBase14 will report that isEGL14Supported() but its
-            // getEglConfig() will fail with a RuntimeException with message
-            // "Unable to find any matching EGL config". Fall back to EglBase10
-            // in the described scenario.
-            EglBase eglBase = null;
-            int[] configAttributes = EglBase.CONFIG_PLAIN;
-            RuntimeException cause = null;
-
-            try {
-                // WebRTC internally does this check in isEGL14Supported, but it's no longer exposed
-                // in the public API
-                if (VERSION.SDK_INT >= 18) {
-                    eglBase = EglBase.createEgl14(configAttributes);
-                }
-            } catch (RuntimeException ex) {
-                // Fall back to EglBase10.
-                cause = ex;
-            }
-
-            if (eglBase == null) {
-                try {
-                    eglBase = EglBase.createEgl10(configAttributes);
-                } catch (RuntimeException ex) {
-                    // Neither EglBase14, nor EglBase10 succeeded to initialize.
-                    cause = ex;
-                }
-            }
-
-            if (cause != null) {
-                Log.e(EglUtils.class.getName(), "Failed to create EglBase", cause);
-            } else {
-                rootEglBase = eglBase;
-            }
+            rootEglBase = EglBase.create();
         }
-
         return rootEglBase;
     }
 
     public static EglBase.Context getRootEglBaseContext() {
         EglBase eglBase = getRootEglBase();
-
         return eglBase == null ? null : eglBase.getEglBaseContext();
     }
 }

android/src/main/java/com/oney/WebRTCModule/WebRTCModule.java

@@ -23,14 +23,14 @@
 import com.facebook.react.modules.core.DeviceEventManagerModule;
 import com.oney.WebRTCModule.audio.AudioProcessingFactoryProvider;
 import com.oney.WebRTCModule.audio.AudioProcessingController;
-import com.oney.WebRTCModule.webrtcutils.H264AndSoftwareVideoDecoderFactory;
-import com.oney.WebRTCModule.webrtcutils.H264AndSoftwareVideoEncoderFactory;
+import com.oney.WebRTCModule.webrtcutils.SelectiveVideoDecoderFactory;
 
 import org.webrtc.*;
 import org.webrtc.audio.AudioDeviceModule;
 import org.webrtc.audio.JavaAudioDeviceModule;
 
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -83,14 +83,9 @@ public WebRTCModule(ReactApplicationContext reactContext) {
         if (encoderFactory == null || decoderFactory == null) {
             // Initialize EGL context required for HW acceleration.
             EglBase.Context eglContext = EglUtils.getRootEglBaseContext();
+            encoderFactory = new SimulcastAlignedVideoEncoderFactory(eglContext, true, true, ResolutionAdjustment.MULTIPLE_OF_16);
+            decoderFactory = new SelectiveVideoDecoderFactory(eglContext, false, Arrays.asList("VP9", "AV1"));
 
-            if (eglContext != null) {
-                encoderFactory = new H264AndSoftwareVideoEncoderFactory(eglContext);
-                decoderFactory = new H264AndSoftwareVideoDecoderFactory(eglContext);
-            } else {
-                encoderFactory = new SoftwareVideoEncoderFactory();
-                decoderFactory = new SoftwareVideoDecoderFactory();
-            }
         }
 
         if (adm == null) {
@@ -134,9 +129,11 @@ private PeerConnection getPeerConnection(int id) {
     }
 
     void sendEvent(String eventName, @Nullable ReadableMap params) {
-        getReactApplicationContext()
-                .getJSModule(DeviceEventManagerModule.RCTDeviceEventEmitter.class)
-                .emit(eventName, params);
+        if (getReactApplicationContext().hasActiveReactInstance()) {
+            getReactApplicationContext()
+                    .getJSModule(DeviceEventManagerModule.RCTDeviceEventEmitter.class)
+                    .emit(eventName, params);
+        }
     }
 
     private PeerConnection.IceServer createIceServer(String url) {

android/src/main/java/com/oney/WebRTCModule/webrtcutils/SelectiveVideoDecoderFactory.kt

@@ -0,0 +1,62 @@
+/*
+ * Copyright (c) 2014-2024 Stream.io Inc. All rights reserved.
+ *
+ * Licensed under the Stream License;
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    https://github.com/GetStream/stream-video-android/blob/main/LICENSE
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.oney.WebRTCModule.webrtcutils
+
+import org.webrtc.EglBase
+import org.webrtc.SoftwareVideoDecoderFactory
+import org.webrtc.VideoCodecInfo
+import org.webrtc.VideoDecoder
+import org.webrtc.VideoDecoderFactory
+
+internal class SelectiveVideoDecoderFactory(
+    sharedContext: EglBase.Context?,
+    private var forceSWCodec: Boolean = false,
+    private var forceSWCodecs: List<String> = listOf("VP9", "AV1"),
+) : VideoDecoderFactory {
+    private val softwareVideoDecoderFactory = SoftwareVideoDecoderFactory()
+    private val wrappedVideoDecoderFactory = WrappedVideoDecoderFactory(sharedContext, forceSWCodec)
+
+    /**
+     * Set to true to force software codecs.
+     */
+    fun setForceSWCodec(forceSWCodec: Boolean) {
+        this.forceSWCodec = forceSWCodec
+    }
+
+    /**
+     * Set a list of codecs for which to use software codecs.
+     */
+    fun setForceSWCodecList(forceSWCodecs: List<String>) {
+        this.forceSWCodecs = forceSWCodecs
+    }
+
+    override fun createDecoder(videoCodecInfo: VideoCodecInfo): VideoDecoder? {
+        if (forceSWCodecs.isNotEmpty()) {
+            if (forceSWCodecs.contains(videoCodecInfo.name)) {
+                return softwareVideoDecoderFactory.createDecoder(videoCodecInfo)
+            }
+        }
+        if (forceSWCodec) {
+            return wrappedVideoDecoderFactory.createDecoder(videoCodecInfo)
+        }
+        return wrappedVideoDecoderFactory.createDecoder(videoCodecInfo)
+    }
+
+    override fun getSupportedCodecs(): Array<VideoCodecInfo> {
+        return wrappedVideoDecoderFactory.supportedCodecs
+    }
+}

android/src/main/java/com/oney/WebRTCModule/webrtcutils/SelectiveVideoEncoderFactory.kt

@@ -0,0 +1,75 @@
+/*
+ * Copyright (c) 2014-2024 Stream.io Inc. All rights reserved.
+ *
+ * Licensed under the Stream License;
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    https://github.com/GetStream/stream-video-android/blob/main/LICENSE
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.oney.WebRTCModule.webrtcutils
+
+import org.webrtc.EglBase
+import org.webrtc.ResolutionAdjustment
+import org.webrtc.SimulcastAlignedVideoEncoderFactory
+import org.webrtc.SoftwareVideoEncoderFactory
+import org.webrtc.VideoCodecInfo
+import org.webrtc.VideoEncoder
+import org.webrtc.VideoEncoderFactory
+
+internal class SelectiveVideoEncoderFactory(
+    sharedContext: EglBase.Context?,
+    enableIntelVp8Encoder: Boolean,
+    enableH264HighProfile: Boolean,
+    private var forceSWCodec: Boolean = false,
+    private var forceSWCodecs: List<String> = listOf("VP9", "AV1"),
+) : VideoEncoderFactory {
+    private val softwareVideoEncoderFactory = SoftwareVideoEncoderFactory()
+    private val simulcastVideoEncoderFactoryWrapper: SimulcastAlignedVideoEncoderFactory
+
+    init {
+        simulcastVideoEncoderFactoryWrapper =
+            SimulcastAlignedVideoEncoderFactory(sharedContext, enableIntelVp8Encoder, enableH264HighProfile, ResolutionAdjustment.NONE)
+    }
+
+    /**
+     * Set to true to force software codecs.
+     */
+    fun setForceSWCodec(forceSWCodec: Boolean) {
+        this.forceSWCodec = forceSWCodec
+    }
+
+    /**
+     * Set a list of codecs for which to use software codecs.
+     */
+    fun setForceSWCodecList(forceSWCodecs: List<String>) {
+        this.forceSWCodecs = forceSWCodecs
+    }
+
+    override fun createEncoder(videoCodecInfo: VideoCodecInfo): VideoEncoder? {
+        if (forceSWCodec) {
+            return softwareVideoEncoderFactory.createEncoder(videoCodecInfo)
+        }
+        if (forceSWCodecs.isNotEmpty()) {
+            if (forceSWCodecs.contains(videoCodecInfo.name)) {
+                return softwareVideoEncoderFactory.createEncoder(videoCodecInfo)
+            }
+        }
+        return simulcastVideoEncoderFactoryWrapper.createEncoder(videoCodecInfo)
+    }
+
+    override fun getSupportedCodecs(): Array<VideoCodecInfo> {
+        return if (forceSWCodec && forceSWCodecs.isEmpty()) {
+            softwareVideoEncoderFactory.supportedCodecs
+        } else {
+            simulcastVideoEncoderFactoryWrapper.supportedCodecs
+        }
+    }
+}