Multiple video frame processors (+ implementation for ios) (react-native-webrtc#1331)

wjaykim · 8BallBomBom · santhoshvai · commit 3e80e4fa8018 · 2024-06-06T15:52:09.000+02:00
* feat(ios): add video frame processor

* fix(ios): add missing capturer property

* feat(android): support multiple VideoFrameProcessors

* Update MediaStreamTrack.ts

Quick lint fix.

---------

Co-authored-by: Johnathon Weaver &lt;weaver123_johnathon@hotmail.com&gt;
diff --git a/android/src/main/java/com/oney/WebRTCModule/GetUserMediaImpl.java b/android/src/main/java/com/oney/WebRTCModule/GetUserMediaImpl.java
@@ -25,8 +25,11 @@
 
 import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.UUID;
+import java.util.stream.Collectors;
 
 /**
  * The implementation of {@code getUserMedia} extracted into a separate file in
@@ -388,28 +391,33 @@ VideoTrack createVideoTrack(AbstractVideoCaptureController videoCaptureControlle
     }
 
     /**
-     * Set video effect to the TrackPrivate corresponding to the trackId with the help of VideoEffectProcessor
-     * corresponding to the name.
+     * Set video effects to the TrackPrivate corresponding to the trackId with the help of VideoEffectProcessor
+     * corresponding to the names.
      * @param trackId TrackPrivate id
-     * @param name VideoEffectProcessor name
+     * @param names VideoEffectProcessor names
      */
-    void setVideoEffect(String trackId, String name) {
+    void setVideoEffects(String trackId, ReadableArray names) {
         TrackPrivate track = tracks.get(trackId);
 
         if (track != null && track.videoCaptureController instanceof CameraCaptureController) {
             VideoSource videoSource = (VideoSource) track.mediaSource;
             SurfaceTextureHelper surfaceTextureHelper = track.surfaceTextureHelper;
 
-            if (name != null) {
-                VideoFrameProcessor videoFrameProcessor = ProcessorProvider.getProcessor(name);
-
-                if (videoFrameProcessor == null) {
-                    Log.e(TAG, "no videoFrameProcessor associated with this name");
-                    return;
-                }
+            if (names != null) {
+                List<VideoFrameProcessor> processors = names.toArrayList().stream()
+                    .filter(name -> name instanceof String)
+                    .map(name -> {
+                        VideoFrameProcessor videoFrameProcessor = ProcessorProvider.getProcessor((String) name);
+                        if (videoFrameProcessor == null) {
+                            Log.e(TAG, "no videoFrameProcessor associated with this name: " + name);
+                        }
+                        return videoFrameProcessor;
+                    })
+                    .filter(Objects::nonNull)
+                    .collect(Collectors.toList());
 
                 VideoEffectProcessor videoEffectProcessor =
-                        new VideoEffectProcessor(videoFrameProcessor, surfaceTextureHelper);
+                        new VideoEffectProcessor(processors, surfaceTextureHelper);
                 videoSource.setVideoProcessor(videoEffectProcessor);
 
             } else {
diff --git a/android/src/main/java/com/oney/WebRTCModule/WebRTCModule.java b/android/src/main/java/com/oney/WebRTCModule/WebRTCModule.java
@@ -942,9 +942,12 @@ private ReadableArray getTransceiversInfo(PeerConnection peerConnection) {
         return transceiverUpdates;
     }
 
+
     @ReactMethod
-    public void mediaStreamTrackSetVideoEffect(String id, String name) {
-        ThreadUtils.runOnExecutor(() -> { getUserMediaImpl.setVideoEffect(id, name); });
+    public void mediaStreamTrackSetVideoEffects(String id, ReadableArray names) {
+        ThreadUtils.runOnExecutor(() -> {
+            getUserMediaImpl.setVideoEffects(id, names);
+        });
     }
 
     @ReactMethod
diff --git a/android/src/main/java/com/oney/WebRTCModule/videoEffects/VideoEffectProcessor.java b/android/src/main/java/com/oney/WebRTCModule/videoEffects/VideoEffectProcessor.java
@@ -5,18 +5,20 @@
 import org.webrtc.VideoProcessor;
 import org.webrtc.VideoSink;
 
+import java.util.List;
+
 /**
  * Lightweight abstraction for an object that can receive video frames, process and add effects in
  * them, and pass them on to another object.
  */
 public class VideoEffectProcessor implements VideoProcessor {
     private VideoSink mSink;
     final private SurfaceTextureHelper textureHelper;
-    final private VideoFrameProcessor videoFrameProcessor;
+    final private List<VideoFrameProcessor> videoFrameProcessors;
 
-    public VideoEffectProcessor(VideoFrameProcessor processor, SurfaceTextureHelper textureHelper) {
+    public VideoEffectProcessor(List<VideoFrameProcessor> processors, SurfaceTextureHelper textureHelper) {
         this.textureHelper = textureHelper;
-        this.videoFrameProcessor = processor;
+        this.videoFrameProcessors = processors;
     }
 
     @Override
@@ -39,15 +41,12 @@ public void setSink(VideoSink sink) {
     @Override
     public void onFrameCaptured(VideoFrame frame) {
         frame.retain();
-        VideoFrame outputFrame = videoFrameProcessor.process(frame, textureHelper);
-
-        if (outputFrame == null) {
-            mSink.onFrame(frame);
-            frame.release();
-            return;
+        VideoFrame outputFrame = frame;
+        for (VideoFrameProcessor processor : this.videoFrameProcessors) {
+            outputFrame = processor.process(outputFrame, textureHelper);
         }
+
         mSink.onFrame(outputFrame);
         outputFrame.release();
-        frame.release();
     }
 }
diff --git a/ios/RCTWebRTC/VideoCaptureController.h b/ios/RCTWebRTC/VideoCaptureController.h
@@ -6,8 +6,10 @@
 #import "CaptureController.h"
 
 @interface VideoCaptureController : CaptureController
-@property(nonatomic, readonly, strong) AVCaptureDeviceFormat *selectedFormat;
-@property(nonatomic, readonly, assign) int frameRate;
+
+@property (nonatomic, readonly, strong) RTCCameraVideoCapturer *capturer;
+@property (nonatomic, readonly, strong) AVCaptureDeviceFormat *selectedFormat;
+@property (nonatomic, readonly, assign) int frameRate;
 
 - (instancetype)initWithCapturer:(RTCCameraVideoCapturer *)capturer andConstraints:(NSDictionary *)constraints;
 - (void)startCapture;
diff --git a/ios/RCTWebRTC/WebRTCModule+RTCMediaStream.h b/ios/RCTWebRTC/WebRTCModule+RTCMediaStream.h
@@ -1,8 +1,13 @@
 #import "CaptureController.h"
 #import "WebRTCModule.h"
+#import "VideoEffectProcessor.h"
 
 @interface WebRTCModule (RTCMediaStream)
+
+@property (nonatomic, strong) VideoEffectProcessor *videoEffectProcessor;
+
 - (RTCVideoTrack *)createVideoTrackWithCaptureController:
     (CaptureController * (^)(RTCVideoSource *))captureControllerCreator;
 - (NSArray *)createMediaStream:(NSArray<RTCMediaStreamTrack *> *)tracks;
+
 @end
diff --git a/ios/RCTWebRTC/WebRTCModule+RTCMediaStream.m b/ios/RCTWebRTC/WebRTCModule+RTCMediaStream.m
@@ -9,13 +9,24 @@
 #import "WebRTCModule+RTCMediaStream.h"
 #import "WebRTCModule+RTCPeerConnection.h"
 
+#import "ProcessorProvider.h"
 #import "ScreenCaptureController.h"
 #import "ScreenCapturer.h"
 #import "TrackCapturerEventsEmitter.h"
 #import "VideoCaptureController.h"
 
 @implementation WebRTCModule (RTCMediaStream)
 
+- (VideoEffectProcessor *)videoEffectProcessor
+{
+  return objc_getAssociatedObject(self, _cmd);
+}
+
+- (void)setVideoEffectProcessor:(VideoEffectProcessor *)videoEffectProcessor
+{
+  objc_setAssociatedObject(self, @selector(videoEffectProcessor), videoEffectProcessor, OBJC_ASSOCIATION_RETAIN_NONATOMIC);
+}
+
 #pragma mark - getUserMedia
 
 /**
@@ -411,6 +422,31 @@ - (RTCVideoTrack *)createScreenCaptureVideoTrack {
     }
 }
 
+RCT_EXPORT_METHOD(mediaStreamTrackSetVideoEffects:(nonnull NSString *)trackID names:(nonnull NSArray<NSString *> *)names)
+{
+  RTCMediaStreamTrack *track = self.localTracks[trackID];
+  if (track) {
+    RTCVideoTrack *videoTrack = (RTCVideoTrack *)track;
+    RTCVideoSource *videoSource = videoTrack.source;
+    
+    NSMutableArray *processors = [[NSMutableArray alloc] init];
+    for (NSString *name in names) {
+      NSObject<VideoFrameProcessorDelegate> *processor = [ProcessorProvider getProcessor:name];
+      if (processor != nil) {
+        [processors addObject:processor];
+      }
+    }
+    
+    self.videoEffectProcessor = [[VideoEffectProcessor alloc] initWithProcessors:processors
+                                                                     videoSource:videoSource];
+    
+    VideoCaptureController *vcc = (VideoCaptureController *)videoTrack.captureController;
+    RTCVideoCapturer *capturer = vcc.capturer;
+    
+    capturer.delegate = self.videoEffectProcessor;
+  }
+}
+
 #pragma mark - Helpers
 
 - (RTCMediaStreamTrack *)trackForId:(nonnull NSString *)trackId pcId:(nonnull NSNumber *)pcId {
diff --git a/ios/RCTWebRTC/videoEffects/ProcessorProvider.h b/ios/RCTWebRTC/videoEffects/ProcessorProvider.h
@@ -0,0 +1,10 @@
+#import "VideoFrameProcessor.h"
+
+@interface ProcessorProvider : NSObject
+
++ (NSObject<VideoFrameProcessorDelegate> *)getProcessor:(NSString *)name;
++ (void)addProcessor:(NSObject<VideoFrameProcessorDelegate> *)processor
+             forName:(NSString *)name;
++ (void)removeProcessor:(NSString *)name;
+
+@end
diff --git a/ios/RCTWebRTC/videoEffects/ProcessorProvider.m b/ios/RCTWebRTC/videoEffects/ProcessorProvider.m
@@ -0,0 +1,24 @@
+#import "ProcessorProvider.h"
+
+@implementation ProcessorProvider
+
+static NSMutableDictionary<NSString *, NSObject<VideoFrameProcessorDelegate> *> *processorMap;
+
++ (void)initialize {
+  processorMap = [[NSMutableDictionary alloc] init];
+}
+
++ (NSObject<VideoFrameProcessorDelegate> *)getProcessor:(NSString *)name {
+  return [processorMap objectForKey:name];
+}
+
++ (void)addProcessor:(NSObject<VideoFrameProcessorDelegate> *)processor
+            forName:(NSString *)name {
+  [processorMap setObject:processor forKey:name];
+}
+
++ (void)removeProcessor:(NSString *)name {
+  [processorMap removeObjectForKey:name];
+}
+
+@end
diff --git a/ios/RCTWebRTC/videoEffects/VideoEffectProcessor.h b/ios/RCTWebRTC/videoEffects/VideoEffectProcessor.h
@@ -0,0 +1,13 @@
+#import <WebRTC/RTCVideoSource.h>
+
+#import "VideoFrameProcessor.h"
+
+@interface VideoEffectProcessor : NSObject<RTCVideoCapturerDelegate>
+
+@property (nonatomic, strong) NSArray<NSObject<VideoFrameProcessorDelegate> *> *videoFrameProcessors;
+@property (nonatomic, strong) RTCVideoSource *videoSource;
+
+- (instancetype)initWithProcessors:(NSArray<NSObject<VideoFrameProcessorDelegate> *> *)videoFrameProcessors
+                       videoSource:(RTCVideoSource *)videoSource;
+
+@end
diff --git a/ios/RCTWebRTC/videoEffects/VideoEffectProcessor.m b/ios/RCTWebRTC/videoEffects/VideoEffectProcessor.m
@@ -0,0 +1,23 @@
+#import <WebRTC/RTCVideoCapturer.h>
+
+#import "VideoEffectProcessor.h"
+
+@implementation VideoEffectProcessor
+
+- (instancetype)initWithProcessors:(NSArray<NSObject<VideoFrameProcessorDelegate> *> *)videoFrameProcessors
+                       videoSource:(RTCVideoSource *)videoSource {
+  self = [super init];
+  _videoFrameProcessors = videoFrameProcessors;
+  _videoSource = videoSource;
+  return self;
+}
+
+- (void)capturer:(nonnull RTCVideoCapturer *)capturer didCaptureVideoFrame:(nonnull RTCVideoFrame *)frame {
+  RTCVideoFrame *processedFrame = frame;
+  for (NSObject<VideoFrameProcessorDelegate> *processor in _videoFrameProcessors) {
+    processedFrame = [processor capturer:capturer didCaptureVideoFrame:processedFrame];
+  }
+  [self.videoSource capturer:capturer didCaptureVideoFrame:processedFrame];
+}
+
+@end
diff --git a/ios/RCTWebRTC/videoEffects/VideoFrameProcessor.h b/ios/RCTWebRTC/videoEffects/VideoFrameProcessor.h
@@ -0,0 +1,9 @@
+#import <WebRTC/RTCVideoCapturer.h>
+#import <WebRTC/RTCVideoFrame.h>
+
+@protocol VideoFrameProcessorDelegate
+
+- (RTCVideoFrame *)capturer:(RTCVideoCapturer *)capturer
+      didCaptureVideoFrame:(RTCVideoFrame *)frame;
+
+@end
diff --git a/src/MediaStreamTrack.ts b/src/MediaStreamTrack.ts
@@ -110,7 +110,7 @@ export default class MediaStreamTrack extends EventTarget<MediaStreamTrackEventM
         WebRTCModule.mediaStreamTrackSwitchCamera(this.id);
     }
 
-    _setVideoEffect(name:string) {
+    _setVideoEffects(names: string[]) {
         if (this.remote) {
             throw new Error('Not implemented for remote tracks');
         }
@@ -119,7 +119,11 @@ export default class MediaStreamTrack extends EventTarget<MediaStreamTrackEventM
             throw new Error('Only implemented for video tracks');
         }
 
-        WebRTCModule.mediaStreamTrackSetVideoEffect(this.id, name);
+        WebRTCModule.mediaStreamTrackSetVideoEffects(this.id, names);
+    }
+
+    _setVideoEffect(name: string) {
+        this._setVideoEffects([ name ]);
     }
 
     /**

Original file line number	Diff line number	Diff line change
`@@ -942,9 +942,12 @@ private ReadableArray getTransceiversInfo(PeerConnection peerConnection) {`
`942`	`942`	`return transceiverUpdates;`
`943`	`943`	`}`
`944`	`944`
	`945`	`+`
`945`	`946`	`@ReactMethod`
`946`		`- public void mediaStreamTrackSetVideoEffect(String id, String name) {`
`947`		`- ThreadUtils.runOnExecutor(() -> { getUserMediaImpl.setVideoEffect(id, name); });`
	`947`	`+ public void mediaStreamTrackSetVideoEffects(String id, ReadableArray names) {`
	`948`	`+ ThreadUtils.runOnExecutor(() -> {`
	`949`	`+ getUserMediaImpl.setVideoEffects(id, names);`
	`950`	`+ });`
`948`	`951`	`}`
`949`	`952`
`950`	`953`	`@ReactMethod`