diff --git a/android/src/main/java/com/cloudwebrtc/webrtc/GetUserMediaImpl.java b/android/src/main/java/com/cloudwebrtc/webrtc/GetUserMediaImpl.java index 0ea7a9663b..6fdbfd4393 100755 --- a/android/src/main/java/com/cloudwebrtc/webrtc/GetUserMediaImpl.java +++ b/android/src/main/java/com/cloudwebrtc/webrtc/GetUserMediaImpl.java @@ -53,6 +53,9 @@ import com.cloudwebrtc.webrtc.utils.MediaConstraintsUtils; import com.cloudwebrtc.webrtc.utils.ObjectType; import com.cloudwebrtc.webrtc.utils.PermissionUtils; +import com.cloudwebrtc.webrtc.videoEffects.VideoFrameProcessor; +import com.cloudwebrtc.webrtc.videoEffects.VideoEffectProcessor; +import com.cloudwebrtc.webrtc.videoEffects.ProcessorProvider; import org.webrtc.AudioSource; import org.webrtc.AudioTrack; @@ -82,6 +85,8 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; import io.flutter.plugin.common.MethodChannel.Result; @@ -108,6 +113,8 @@ class GetUserMediaImpl { private final Map mVideoCapturers = new HashMap<>(); private final Map mSurfaceTextureHelpers = new HashMap<>(); + private final Map mVideoSources = new HashMap<>(); + private final StateProvider stateProvider; private final Context applicationContext; @@ -810,6 +817,7 @@ private ConstraintsMap getUserVideo(ConstraintsMap constraints, MediaStream medi String trackId = stateProvider.getNextTrackUUID(); mVideoCapturers.put(trackId, info); mSurfaceTextureHelpers.put(trackId, surfaceTextureHelper); + mVideoSources.put(trackId, videoSource); Log.d(TAG, "Target: " + targetWidth + "x" + targetHeight + "@" + targetFps + ", Actual: " + info.width + "x" + info.height + "@" + info.fps); @@ -857,11 +865,38 @@ void removeVideoCapturer(String id) { helper.stopListening(); helper.dispose(); mSurfaceTextureHelpers.remove(id); + mVideoSources.remove(id); } } } } + void setVideoEffect(String trackId, List names) { + VideoSource videoSource = mVideoSources.get(trackId); + SurfaceTextureHelper surfaceTextureHelper = mSurfaceTextureHelpers.get(trackId); + + if (names != null && !names.isEmpty()) { + List processors = names.stream() + .filter(name -> name instanceof String) + .map(name -> { + VideoFrameProcessor videoFrameProcessor = ProcessorProvider.getProcessor((String) name); + if (videoFrameProcessor == null) { + Log.e(TAG, "no videoFrameProcessor associated with this name: " + name); + } + return videoFrameProcessor; + }) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + + + VideoEffectProcessor videoEffectProcessor = new VideoEffectProcessor(processors, surfaceTextureHelper); + videoSource.setVideoProcessor(videoEffectProcessor); + + } else { + videoSource.setVideoProcessor(null); + } + } + @RequiresApi(api = VERSION_CODES.M) private void requestPermissions( final ArrayList permissions, diff --git a/android/src/main/java/com/cloudwebrtc/webrtc/MethodCallHandlerImpl.java b/android/src/main/java/com/cloudwebrtc/webrtc/MethodCallHandlerImpl.java index f67f105363..a9319e1e09 100644 --- a/android/src/main/java/com/cloudwebrtc/webrtc/MethodCallHandlerImpl.java +++ b/android/src/main/java/com/cloudwebrtc/webrtc/MethodCallHandlerImpl.java @@ -295,6 +295,14 @@ public void onMethodCall(MethodCall call, @NonNull Result notSafeResult) { result.success(null); break; } + case "setVideoEffects": { + String trackId = call.argument("trackId"); + List names = call.argument("names"); + + getUserMediaImpl.setVideoEffect(trackId, names); + result.success(null); + break; + } case "createPeerConnection": { Map constraints = call.argument("constraints"); Map configuration = call.argument("configuration"); diff --git a/android/src/main/java/com/cloudwebrtc/webrtc/videoEffects/ProcessorProvider.java b/android/src/main/java/com/cloudwebrtc/webrtc/videoEffects/ProcessorProvider.java new file mode 100644 index 0000000000..d5055b5991 --- /dev/null +++ b/android/src/main/java/com/cloudwebrtc/webrtc/videoEffects/ProcessorProvider.java @@ -0,0 +1,37 @@ +package com.cloudwebrtc.webrtc.videoEffects; + +import java.util.HashMap; +import java.util.Map; + +/** + * Manages VideoFrameProcessorFactoryInterfaces corresponding to name using hashmap, and provides + * get, add and remove functionality. + */ +public class ProcessorProvider { + private static Map methodMap = new HashMap(); + + public static VideoFrameProcessor getProcessor(String name) { + if (methodMap.containsKey(name)) { + return methodMap.get(name).build(); + } else{ + return null; + } + } + + public static void addProcessor(String name, + VideoFrameProcessorFactoryInterface videoFrameProcessorFactoryInterface) { + if (name != null && videoFrameProcessorFactoryInterface != null) { + methodMap.put(name, videoFrameProcessorFactoryInterface); + } else{ + throw new NullPointerException("Name or VideoFrameProcessorFactry can not be null"); + } + } + + public static void removeProcessor(String name) { + if (name != null && methodMap.containsKey(name)) { + methodMap.remove(name); + } else{ + throw new RuntimeException("VideoFrameProcessorFactry with " + name + " does not exist"); + } + } +} \ No newline at end of file diff --git a/android/src/main/java/com/cloudwebrtc/webrtc/videoEffects/VideoEffectProcessor.java b/android/src/main/java/com/cloudwebrtc/webrtc/videoEffects/VideoEffectProcessor.java new file mode 100644 index 0000000000..e9c8bde375 --- /dev/null +++ b/android/src/main/java/com/cloudwebrtc/webrtc/videoEffects/VideoEffectProcessor.java @@ -0,0 +1,58 @@ +package com.cloudwebrtc.webrtc.videoEffects; + +import org.webrtc.SurfaceTextureHelper; +import org.webrtc.VideoFrame; +import org.webrtc.VideoProcessor; +import org.webrtc.VideoSink; + +import java.util.List; + +/** + * Lightweight abstraction for an object that can receive video frames, process and add effects in + * them, and pass them on to another object. + */ +public class VideoEffectProcessor implements VideoProcessor { + private VideoSink mSink; + final private SurfaceTextureHelper textureHelper; + final private List videoFrameProcessors; + + public VideoEffectProcessor(List processors, SurfaceTextureHelper textureHelper) { + this.textureHelper = textureHelper; + this.videoFrameProcessors = processors; + } + + @Override + public void onCapturerStarted(boolean success) { + + } + + @Override + public void onCapturerStopped() { + + } + + @Override + public void setSink(VideoSink sink) { + mSink = sink; + } + + /** + * Called just after the frame is captured. + * Will process the VideoFrame with the help of VideoFrameProcessor and send the processed + * VideoFrame back to webrtc using onFrame method in VideoSink. + * @param frame raw VideoFrame received from webrtc. + */ + @Override + public void onFrameCaptured(VideoFrame frame) { + frame.retain(); + VideoFrame outputFrame = frame; + + for (VideoFrameProcessor processor : this.videoFrameProcessors) { + outputFrame = processor.process(outputFrame, textureHelper); + } + + mSink.onFrame(outputFrame); + outputFrame.release(); + frame.release(); + } +} \ No newline at end of file diff --git a/android/src/main/java/com/cloudwebrtc/webrtc/videoEffects/VideoFrameProcessor.java b/android/src/main/java/com/cloudwebrtc/webrtc/videoEffects/VideoFrameProcessor.java new file mode 100644 index 0000000000..c2c276b79c --- /dev/null +++ b/android/src/main/java/com/cloudwebrtc/webrtc/videoEffects/VideoFrameProcessor.java @@ -0,0 +1,19 @@ +package com.cloudwebrtc.webrtc.videoEffects; + +import org.webrtc.SurfaceTextureHelper; +import org.webrtc.VideoFrame; + +/** + * Interface contains process method to process VideoFrame. + * The caller takes ownership of the object. + */ +public interface VideoFrameProcessor { + /** + * Applies the image processing algorithms to the frame. Returns the processed frame. + * The caller is responsible for releasing the returned frame. + * @param frame raw videoframe which need to be processed + * @param textureHelper + * @return processed videoframe which will rendered + */ + public VideoFrame process(VideoFrame frame, SurfaceTextureHelper textureHelper); +} \ No newline at end of file diff --git a/android/src/main/java/com/cloudwebrtc/webrtc/videoEffects/VideoFrameProcessorFactoryInterface.java b/android/src/main/java/com/cloudwebrtc/webrtc/videoEffects/VideoFrameProcessorFactoryInterface.java new file mode 100644 index 0000000000..1d0a255cd3 --- /dev/null +++ b/android/src/main/java/com/cloudwebrtc/webrtc/videoEffects/VideoFrameProcessorFactoryInterface.java @@ -0,0 +1,13 @@ +package com.cloudwebrtc.webrtc.videoEffects; + + /** + * Factory for creating VideoFrameProcessor instances. + */ + public interface VideoFrameProcessorFactoryInterface { + + /** + * Dynamically allocates a VideoFrameProcessor instance and returns a pointer to it. + * The caller takes ownership of the object. + */ + public VideoFrameProcessor build(); + } \ No newline at end of file diff --git a/common/darwin/Classes/FlutterWebRTCPlugin.h b/common/darwin/Classes/FlutterWebRTCPlugin.h index 4b156b3d2e..ac99f7a546 100644 --- a/common/darwin/Classes/FlutterWebRTCPlugin.h +++ b/common/darwin/Classes/FlutterWebRTCPlugin.h @@ -7,6 +7,7 @@ #import #import +@class VideoEffectProcessor; @class FlutterRTCVideoRenderer; @class FlutterRTCFrameCapturer; @@ -46,12 +47,15 @@ typedef void (^CapturerStopHandler)(CompletionHandler _Nonnull handler); @property(nonatomic, strong) RTCCameraVideoCapturer* _Nullable videoCapturer; @property(nonatomic, strong) FlutterRTCFrameCapturer* _Nullable frameCapturer; @property(nonatomic, strong) AVAudioSessionPort _Nullable preferredInput; +@property (nonatomic, strong) VideoEffectProcessor* videoEffectProcessor; @property(nonatomic) BOOL _usingFrontCamera; @property(nonatomic) NSInteger _lastTargetWidth; @property(nonatomic) NSInteger _lastTargetHeight; @property(nonatomic) NSInteger _lastTargetFps; +- (void)mediaStreamTrackSetVideoEffects:(nonnull NSString *)trackId + names:(nonnull NSArray *)names; - (RTCMediaStream* _Nullable)streamForId:(NSString* _Nonnull)streamId peerConnectionId:(NSString* _Nullable)peerConnectionId; - (RTCMediaStreamTrack* _Nullable)trackForId:(NSString* _Nonnull)trackId peerConnectionId:(NSString* _Nullable)peerConnectionId; - (RTCRtpTransceiver* _Nullable)getRtpTransceiverById:(RTCPeerConnection* _Nonnull)peerConnection Id:(NSString* _Nullable)Id; diff --git a/common/darwin/Classes/FlutterWebRTCPlugin.m b/common/darwin/Classes/FlutterWebRTCPlugin.m index 45c3fe42b4..74410ba3b3 100644 --- a/common/darwin/Classes/FlutterWebRTCPlugin.m +++ b/common/darwin/Classes/FlutterWebRTCPlugin.m @@ -6,6 +6,9 @@ #import "FlutterRTCPeerConnection.h" #import "FlutterRTCVideoRenderer.h" #import "FlutterRTCFrameCryptor.h" +#import "VideoEffectProcessor.h" +#import "ProcessorProvider.h" +#import "VideoFrameProcessor.h" #if TARGET_OS_IPHONE #import "FlutterRTCVideoPlatformViewFactory.h" #import "FlutterRTCVideoPlatformViewController.h" @@ -173,6 +176,7 @@ - (instancetype)initWithChannel:(FlutterMethodChannel*)channel self.frameCryptors = [NSMutableDictionary new]; self.keyProviders = [NSMutableDictionary new]; self.videoCapturerStopHandlers = [NSMutableDictionary new]; + #if TARGET_OS_IPHONE AVAudioSession* session = [AVAudioSession sharedInstance]; [[NSNotificationCenter defaultCenter] addObserver:self @@ -294,6 +298,12 @@ - (void)handleMethodCall:(FlutterMethodCall*)call result:(FlutterResult)result { } [self initialize:networkIgnoreMask bypassVoiceProcessing:enableBypassVoiceProcessing]; result(@""); + } else if([@"setVideoEffects" isEqualToString:call.method]) { + NSDictionary* argsMap = call.arguments; + NSString* trackId = argsMap[@"trackId"]; + NSArray* names = argsMap[@"names"]; + + [self mediaStreamTrackSetVideoEffects:trackId names:names]; } else if ([@"createPeerConnection" isEqualToString:call.method]) { NSDictionary* argsMap = call.arguments; NSDictionary* configuration = argsMap[@"configuration"]; @@ -1452,6 +1462,36 @@ - (void)deactiveRtcAudioSession { #endif } +- (void)mediaStreamTrackSetVideoEffects:(nonnull NSString *)trackId names:(nonnull NSArray *)names +{ + RTCMediaStreamTrack *track = [self trackForId:trackId peerConnectionId: nil]; + + if (track) { + NSLog(@"mediaStreamTrackSetVideoEffects: track found"); + + RTCVideoTrack *videoTrack = (RTCVideoTrack *)track; + RTCVideoSource *videoSource = videoTrack.source; + + NSMutableArray *processors = [[NSMutableArray alloc] init]; + for (NSString *name in names) { + NSObject *processor = [ProcessorProvider getProcessor:name]; + if (processor != nil) { + [processors addObject:processor]; + } + } + + self.videoEffectProcessor = [[VideoEffectProcessor alloc] initWithProcessors:processors + videoSource:videoSource]; + + + self.videoCapturer.delegate = self.videoEffectProcessor; + } else { + NSLog(@"mediaStreamTrackSetVideoEffects: track not found"); + } + + +} + - (void)mediaStreamGetTracks:(NSString*)streamId result:(FlutterResult)result { RTCMediaStream* stream = [self streamForId:streamId peerConnectionId:@""]; if (stream) { diff --git a/common/darwin/Classes/ProcessorProvider.h b/common/darwin/Classes/ProcessorProvider.h new file mode 100644 index 0000000000..702ccc5e87 --- /dev/null +++ b/common/darwin/Classes/ProcessorProvider.h @@ -0,0 +1,10 @@ +#import "VideoFrameProcessor.h" + +@interface ProcessorProvider : NSObject + ++ (NSObject *)getProcessor:(NSString *)name; ++ (void)addProcessor:(NSObject *)processor + forName:(NSString *)name; ++ (void)removeProcessor:(NSString *)name; + +@end \ No newline at end of file diff --git a/common/darwin/Classes/ProcessorProvider.m b/common/darwin/Classes/ProcessorProvider.m new file mode 100644 index 0000000000..9f496cb890 --- /dev/null +++ b/common/darwin/Classes/ProcessorProvider.m @@ -0,0 +1,24 @@ +#import "ProcessorProvider.h" + +@implementation ProcessorProvider + +static NSMutableDictionary *> *processorMap; + ++ (void)initialize { + processorMap = [[NSMutableDictionary alloc] init]; +} + ++ (NSObject *)getProcessor:(NSString *)name { + return [processorMap objectForKey:name]; +} + ++ (void)addProcessor:(NSObject *)processor + forName:(NSString *)name { + [processorMap setObject:processor forKey:name]; +} + ++ (void)removeProcessor:(NSString *)name { + [processorMap removeObjectForKey:name]; +} + +@end \ No newline at end of file diff --git a/common/darwin/Classes/VideoEffectProcessor.h b/common/darwin/Classes/VideoEffectProcessor.h new file mode 100644 index 0000000000..7b59cbfb69 --- /dev/null +++ b/common/darwin/Classes/VideoEffectProcessor.h @@ -0,0 +1,13 @@ +#import + +#import "VideoFrameProcessor.h" + +@interface VideoEffectProcessor : NSObject + +@property (nonatomic, strong) NSArray *> *videoFrameProcessors; +@property (nonatomic, strong) RTCVideoSource *videoSource; + +- (instancetype)initWithProcessors:(NSArray *> *)videoFrameProcessors + videoSource:(RTCVideoSource *)videoSource; + +@end \ No newline at end of file diff --git a/common/darwin/Classes/VideoEffectProcessor.m b/common/darwin/Classes/VideoEffectProcessor.m new file mode 100644 index 0000000000..627de2c67d --- /dev/null +++ b/common/darwin/Classes/VideoEffectProcessor.m @@ -0,0 +1,19 @@ +#import +#import "VideoEffectProcessor.h" + +@implementation VideoEffectProcessor +- (instancetype)initWithProcessors:(NSArray *> *)videoFrameProcessors + videoSource:(RTCVideoSource *)videoSource { + self = [super init]; + _videoFrameProcessors = videoFrameProcessors; + _videoSource = videoSource; + return self; +} +- (void)capturer:(nonnull RTCVideoCapturer *)capturer didCaptureVideoFrame:(nonnull RTCVideoFrame *)frame { + RTCVideoFrame *processedFrame = frame; + for (NSObject *processor in _videoFrameProcessors) { + processedFrame = [processor capturer:capturer didCaptureVideoFrame:processedFrame]; + } + [self.videoSource capturer:capturer didCaptureVideoFrame:processedFrame]; +} +@end \ No newline at end of file diff --git a/common/darwin/Classes/VideoFrameProcessor.h b/common/darwin/Classes/VideoFrameProcessor.h new file mode 100644 index 0000000000..0ca5ad82cb --- /dev/null +++ b/common/darwin/Classes/VideoFrameProcessor.h @@ -0,0 +1,7 @@ +#import +#import + +@protocol VideoFrameProcessorDelegate +- (RTCVideoFrame *)capturer:(RTCVideoCapturer *)capturer + didCaptureVideoFrame:(RTCVideoFrame *)frame; +@end \ No newline at end of file diff --git a/ios/Classes/ProcessorProvider.h b/ios/Classes/ProcessorProvider.h new file mode 120000 index 0000000000..6212492e97 --- /dev/null +++ b/ios/Classes/ProcessorProvider.h @@ -0,0 +1 @@ +../../common/darwin/Classes/ProcessorProvider.h \ No newline at end of file diff --git a/ios/Classes/ProcessorProvider.m b/ios/Classes/ProcessorProvider.m new file mode 120000 index 0000000000..724d8ea942 --- /dev/null +++ b/ios/Classes/ProcessorProvider.m @@ -0,0 +1 @@ +../../common/darwin/Classes/ProcessorProvider.m \ No newline at end of file diff --git a/ios/Classes/VideoEffectProcessor.h b/ios/Classes/VideoEffectProcessor.h new file mode 120000 index 0000000000..231b99672b --- /dev/null +++ b/ios/Classes/VideoEffectProcessor.h @@ -0,0 +1 @@ +../../common/darwin/Classes/VideoEffectProcessor.h \ No newline at end of file diff --git a/ios/Classes/VideoEffectProcessor.m b/ios/Classes/VideoEffectProcessor.m new file mode 120000 index 0000000000..14d99a7e07 --- /dev/null +++ b/ios/Classes/VideoEffectProcessor.m @@ -0,0 +1 @@ +../../common/darwin/Classes/VideoEffectProcessor.m \ No newline at end of file diff --git a/ios/Classes/VideoFrameProcessor.h b/ios/Classes/VideoFrameProcessor.h new file mode 120000 index 0000000000..3e244a030d --- /dev/null +++ b/ios/Classes/VideoFrameProcessor.h @@ -0,0 +1 @@ +../../common/darwin/Classes/VideoFrameProcessor.h \ No newline at end of file diff --git a/lib/src/native/factory_impl.dart b/lib/src/native/factory_impl.dart index 676e8c67da..13e705e4f5 100644 --- a/lib/src/native/factory_impl.dart +++ b/lib/src/native/factory_impl.dart @@ -18,6 +18,14 @@ class RTCFactoryNative extends RTCFactory { static final RTCFactory instance = RTCFactoryNative._internal(); + @override + Future setVideoEffects(String trackId, List names) async { + await WebRTC.invokeMethod('setVideoEffects', { + 'trackId': trackId, + 'names': names, + }); + } + @override Future createLocalMediaStream(String label) async { final response = await WebRTC.invokeMethod('createLocalMediaStream'); @@ -90,6 +98,13 @@ class RTCFactoryNative extends RTCFactory { } } +Future setVideoEffects( + String trackId, { + required List names, +}) async { + return RTCFactoryNative.instance.setVideoEffects(trackId, names); +} + Future createPeerConnection( Map configuration, [Map constraints = const {}]) async { diff --git a/macos/Classes/ProcessorProvider.h b/macos/Classes/ProcessorProvider.h new file mode 120000 index 0000000000..6212492e97 --- /dev/null +++ b/macos/Classes/ProcessorProvider.h @@ -0,0 +1 @@ +../../common/darwin/Classes/ProcessorProvider.h \ No newline at end of file diff --git a/macos/Classes/ProcessorProvider.m b/macos/Classes/ProcessorProvider.m new file mode 120000 index 0000000000..724d8ea942 --- /dev/null +++ b/macos/Classes/ProcessorProvider.m @@ -0,0 +1 @@ +../../common/darwin/Classes/ProcessorProvider.m \ No newline at end of file diff --git a/macos/Classes/VideoEffectProcessor.h b/macos/Classes/VideoEffectProcessor.h new file mode 120000 index 0000000000..231b99672b --- /dev/null +++ b/macos/Classes/VideoEffectProcessor.h @@ -0,0 +1 @@ +../../common/darwin/Classes/VideoEffectProcessor.h \ No newline at end of file diff --git a/macos/Classes/VideoEffectProcessor.m b/macos/Classes/VideoEffectProcessor.m new file mode 120000 index 0000000000..14d99a7e07 --- /dev/null +++ b/macos/Classes/VideoEffectProcessor.m @@ -0,0 +1 @@ +../../common/darwin/Classes/VideoEffectProcessor.m \ No newline at end of file diff --git a/macos/Classes/VideoFrameProcessor.h b/macos/Classes/VideoFrameProcessor.h new file mode 120000 index 0000000000..3e244a030d --- /dev/null +++ b/macos/Classes/VideoFrameProcessor.h @@ -0,0 +1 @@ +../../common/darwin/Classes/VideoFrameProcessor.h \ No newline at end of file