feat: Skia for V4! 🥳

.github/ISSUE_TEMPLATE/BUILD_ERROR.yml

@@ -32,10 +32,10 @@ body:
       render: json
       placeholder: >
         "dependencies": {
-          "react-native": "^0.72.3",
-          "react-native-reanimated": "^3.4.2",
-          "react-native-vision-camera": "^3.0.0",
-          "react-native-worklets-core": "^0.2.0",
+          "react-native": "^0.73.4",
+          "react-native-reanimated": "^3.9.0",
+          "react-native-vision-camera": "^4.0.0",
+          "react-native-worklets-core": "^1.0.0",
           ...
         },
     validations:

.github/workflows/build-android.yml

@@ -96,8 +96,8 @@ jobs:
         run: yarn install --frozen-lockfile
       - name: Install node_modules for example/
         run: yarn install --frozen-lockfile --cwd example
-      - name: Remove react-native-worklets-core
-        run: yarn remove react-native-worklets-core --cwd example
+      - name: Remove worklets, skia and reanimated
+        run: yarn remove react-native-worklets-core @shopify/react-native-skia react-native-reanimated --cwd ..
 
       - name: Restore Gradle cache
         uses: actions/cache@v4

.github/workflows/build-ios.yml

@@ -98,8 +98,8 @@ jobs:
             ${{ runner.os }}-yarn-
       - name: Install node_modules for example/
         run: yarn install --frozen-lockfile --cwd ..
-      - name: Remove react-native-worklets-core
-        run: yarn remove react-native-worklets-core --cwd ..
+      - name: Remove worklets, skia and reanimated
+        run: yarn remove react-native-worklets-core @shopify/react-native-skia react-native-reanimated --cwd ..
 
       - name: Restore buildcache
         uses: mikehardy/buildcache-action@v2

docs/docs/guides/FRAME_PROCESSORS.mdx

@@ -54,7 +54,7 @@ Because they are written in JS, Frame Processors are simple, powerful, extensibl
 
 ### react-native-worklets-core
 
-Frame Processors require [react-native-worklets-core](https://github.com/margelo/react-native-worklets-core) 0.2.0 or higher. Install it:
+Frame Processors require [react-native-worklets-core](https://github.com/margelo/react-native-worklets-core) 1.0.0 or higher. Install it:
 
 ```sh
 npm i react-native-worklets-core
@@ -147,10 +147,10 @@ const onDraw = useDrawCallback((canvas) => {
 
 ### Call functions
 
-And you can also call back to the React-JS thread by using `createRunInJsFn(...)`:
+And you can also call back to the React-JS thread by using `createRunOnJS(...)`:
 
 ```tsx
-const onFaceDetected = Worklets.createRunInJsFn((face: Face) => {
+const onFaceDetected = Worklets.createRunOnJS((face: Face) => {
   navigation.push("FiltersPage", { face: face })
 })
 

docs/docs/guides/FRAME_PROCESSORS_CREATE_OVERVIEW.mdx

@@ -54,7 +54,7 @@ Return values will automatically be converted to JS values, assuming they are re
 ```java
 @Nullable
 @Override
-public Object callback(@NonNull Frame frame, @Nullable Map<String, Object> arguments) {
+public Object callback(@NonNull Frame frame, @Nullable Map<String, Object> arguments) throws Throwable {
   return "cat";
 }
 ```
@@ -74,7 +74,7 @@ You can also manipulate the buffer and return it (or a copy of it) by returning
 ```java
 @Nullable
 @Override
-public Object callback(@NonNull Frame frame, @Nullable Map<String, Object> arguments) {
+public Object callback(@NonNull Frame frame, @Nullable Map<String, Object> arguments) throws Throwable {
   Frame resizedFrame = new Frame(/* ... */);
   return resizedFrame;
 }
@@ -112,11 +112,11 @@ To let the user know that something went wrong you can use Exceptions:
 ```java
 @Nullable
 @Override
-public Object callback(@NonNull Frame frame, @Nullable Map<String, Object> arguments) {
+public Object callback(@NonNull Frame frame, @Nullable Map<String, Object> arguments) throws Throwable {
   if (arguments != null && arguments.get("codeType") instanceof String) {
     // ...
   } else {
-    throw new Exception("codeType property has to be a string!");
+    throw new RuntimeException("codeType property has to be a string!");
   }
 }
 ```
@@ -149,7 +149,7 @@ For example, a realtime video chat application might use WebRTC to send the fram
 ```java
 @Nullable
 @Override
-public Object callback(@NonNull Frame frame, @Nullable Map<String, Object> arguments) {
+public Object callback(@NonNull Frame frame, @Nullable Map<String, Object> arguments) throws Throwable {
   if (arguments == null) {
     return null;
   }

docs/docs/guides/FRAME_PROCESSORS_TIPS.mdx

@@ -40,7 +40,7 @@ If you use native Frame Processor Plugins, make sure they are optimized for real
 
 ## ESLint react-hooks plugin
 
-If you are using the [react-hooks ESLint plugin](https://www.npmjs.com/package/eslint-plugin-react-hooks), make sure to add `useFrameProcessor` to `additionalHooks` inside your ESLint config so dependencies are detected properly. (See ["advanced configuration"](https://www.npmjs.com/package/eslint-plugin-react-hooks#advanced-configuration))
+If you are using the [react-hooks ESLint plugin](https://www.npmjs.com/package/eslint-plugin-react-hooks), make sure to add `useFrameProcessor` and `useSkiaFrameProcessor` to `additionalHooks` inside your ESLint config so dependencies are detected properly. (See ["advanced configuration"](https://www.npmjs.com/package/eslint-plugin-react-hooks#advanced-configuration))
 
 ## Technical
 

docs/docs/guides/FRAME_PROCESSOR_CREATE_PLUGIN_ANDROID.mdx

@@ -67,7 +67,7 @@ public class FaceDetectorFrameProcessorPlugin extends FrameProcessorPlugin {
 
   @Nullable
   @Override
-  public Object callback(@NonNull Frame frame, @Nullable Map<String, Object> arguments) {
+  public Object callback(@NonNull Frame frame, @Nullable Map<String, Object> arguments) throws Throwable {
 // highlight-next-line
     // code goes here
     return null;

docs/docs/guides/SETUP.mdx

@@ -44,7 +44,7 @@ npx expo install react-native-vision-camera
 
 VisionCamera requires **iOS 12 or higher**, and **Android-SDK version 26 or higher**. See [Troubleshooting](/docs/guides/troubleshooting) if you're having installation issues.
 
-> **(Optional)** If you want to use [Frame Processors](/docs/guides/frame-processors), you need to install [react-native-worklets-core](https://github.com/margelo/react-native-worklets-core) 0.2.0 or higher.
+> **(Optional)** If you want to use [Frame Processors](/docs/guides/frame-processors), you need to install [react-native-worklets-core](https://github.com/margelo/react-native-worklets-core) 1.0.0 or higher.
 
 ## Updating manifests
 

package/.eslintrc.js

@@ -73,12 +73,13 @@ module.exports = {
     ],
     '@typescript-eslint/no-non-null-assertion': 'error',
     '@typescript-eslint/no-unnecessary-condition': 'error',
+    '@typescript-eslint/consistent-type-imports': 'warn',
 
     // react hooks
     'react-hooks/exhaustive-deps': [
       'error',
       {
-        additionalHooks: '(useDerivedValue|useAnimatedStyle|useAnimatedProps|useWorkletCallback|useFrameProcessor)',
+        additionalHooks: '(useDerivedValue|useAnimatedStyle|useAnimatedProps|useWorkletCallback|useFrameProcessor|useSkiaFrameProcessor)',
       },
     ],
   },

package/VisionCamera.podspec

@@ -28,7 +28,7 @@ Pod::Spec.new do |s|
   s.source       = { :git => "https://github.com/mrousavy/react-native-vision-camera.git", :tag => "#{s.version}" }
 
   s.pod_target_xcconfig = {
-    "GCC_PREPROCESSOR_DEFINITIONS" => "$(inherited) SK_METAL=1 SK_GANESH=1 VISION_CAMERA_ENABLE_FRAME_PROCESSORS=#{hasWorklets}",
+    "GCC_PREPROCESSOR_DEFINITIONS" => "$(inherited) VISION_CAMERA_ENABLE_FRAME_PROCESSORS=#{hasWorklets}",
     "OTHER_SWIFT_FLAGS" => "$(inherited) #{hasWorklets ? "-D VISION_CAMERA_ENABLE_FRAME_PROCESSORS" : ""}",
     "CLANG_CXX_LANGUAGE_STANDARD" => "c++17",
     "HEADER_SEARCH_PATHS" => "\"$(PODS_TARGET_SRCROOT)/cpp/\"/** "

package/android/src/main/cpp/frameprocessor/FrameHostObject.cpp

@@ -50,6 +50,7 @@ std::vector<jsi::PropNameID> FrameHostObject::getPropertyNames(jsi::Runtime& rt)
     // Conversion
     result.push_back(jsi::PropNameID::forUtf8(rt, std::string("toString")));
     result.push_back(jsi::PropNameID::forUtf8(rt, std::string("toArrayBuffer")));
+    result.push_back(jsi::PropNameID::forUtf8(rt, std::string("getNativeBuffer")));
   }
 
   return result;
@@ -60,6 +61,40 @@ std::vector<jsi::PropNameID> FrameHostObject::getPropertyNames(jsi::Runtime& rt)
 jsi::Value FrameHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& propName) {
   auto name = propName.utf8(runtime);
 
+  // Properties
+  if (name == "isValid") {
+    return jsi::Value(this->frame && this->frame->getIsValid());
+  }
+  if (name == "width") {
+    return jsi::Value(this->frame->getWidth());
+  }
+  if (name == "height") {
+    return jsi::Value(this->frame->getHeight());
+  }
+  if (name == "isMirrored") {
+    return jsi::Value(this->frame->getIsMirrored());
+  }
+  if (name == "orientation") {
+    auto orientation = this->frame->getOrientation();
+    auto string = orientation->getUnionValue();
+    return jsi::String::createFromUtf8(runtime, string->toStdString());
+  }
+  if (name == "pixelFormat") {
+    auto pixelFormat = this->frame->getPixelFormat();
+    auto string = pixelFormat->getUnionValue();
+    return jsi::String::createFromUtf8(runtime, string->toStdString());
+  }
+  if (name == "timestamp") {
+    return jsi::Value(static_cast<double>(this->frame->getTimestamp()));
+  }
+  if (name == "bytesPerRow") {
+    return jsi::Value(this->frame->getBytesPerRow());
+  }
+  if (name == "planesCount") {
+    return jsi::Value(this->frame->getPlanesCount());
+  }
+
+  // Internal Methods
   if (name == "incrementRefCount") {
     jsi::HostFunctionType incrementRefCount = JSI_FUNC {
       // Increment retain count by one.
@@ -76,19 +111,34 @@ jsi::Value FrameHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& pr
     };
     return jsi::Function::createFromHostFunction(runtime, jsi::PropNameID::forUtf8(runtime, "decrementRefCount"), 0, decrementRefCount);
   }
-  if (name == "toString") {
-    jsi::HostFunctionType toString = JSI_FUNC {
+
+  // Conversion methods
+  if (name == "getNativeBuffer") {
+    jsi::HostFunctionType getNativeBuffer = JSI_FUNC {
       if (!this->frame) {
-        return jsi::String::createFromUtf8(runtime, "[closed frame]");
+        throw jsi::JSError(runtime, "Cannot get Platform Buffer - this Frame is already closed!");
       }
-      auto width = this->frame->getWidth();
-      auto height = this->frame->getHeight();
-      auto format = this->frame->getPixelFormat();
-      auto formatString = format->getUnionValue();
-      auto str = std::to_string(width) + " x " + std::to_string(height) + " " + formatString->toString() + " Frame";
-      return jsi::String::createFromUtf8(runtime, str);
+#if __ANDROID_API__ >= 26
+      AHardwareBuffer* hardwareBuffer = this->frame->getHardwareBuffer();
+      AHardwareBuffer_acquire(hardwareBuffer);
+      uintptr_t pointer = reinterpret_cast<uintptr_t>(hardwareBuffer);
+      jsi::HostFunctionType deleteFunc = [=](jsi::Runtime& runtime, const jsi::Value& thisArg, const jsi::Value* args,
+                                             size_t count) -> jsi::Value {
+        AHardwareBuffer_release(hardwareBuffer);
+        return jsi::Value::undefined();
+      };
+
+      jsi::Object buffer(runtime);
+      buffer.setProperty(runtime, "pointer", jsi::BigInt::fromUint64(runtime, pointer));
+      buffer.setProperty(runtime, "delete",
+                         jsi::Function::createFromHostFunction(runtime, jsi::PropNameID::forUtf8(runtime, "delete"), 0, deleteFunc));
+      return buffer;
+#else
+      throw jsi::JSError(runtime, "Cannot get Platform Buffer - getNativeBuffer() requires HardwareBuffers, which are "
+                                  "only available on Android API 26 or above. Set your app's minSdk version to 26 and try again.");
+#endif
     };
-    return jsi::Function::createFromHostFunction(runtime, jsi::PropNameID::forUtf8(runtime, "toString"), 0, toString);
+    return jsi::Function::createFromHostFunction(runtime, jsi::PropNameID::forUtf8(runtime, "getNativeBuffer"), 0, getNativeBuffer);
   }
   if (name == "toArrayBuffer") {
     jsi::HostFunctionType toArrayBuffer = JSI_FUNC {
@@ -144,37 +194,19 @@ jsi::Value FrameHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& pr
     };
     return jsi::Function::createFromHostFunction(runtime, jsi::PropNameID::forUtf8(runtime, "toArrayBuffer"), 0, toArrayBuffer);
   }
-
-  if (name == "isValid") {
-    return jsi::Value(this->frame && this->frame->getIsValid());
-  }
-  if (name == "width") {
-    return jsi::Value(this->frame->getWidth());
-  }
-  if (name == "height") {
-    return jsi::Value(this->frame->getHeight());
-  }
-  if (name == "isMirrored") {
-    return jsi::Value(this->frame->getIsMirrored());
-  }
-  if (name == "orientation") {
-    auto orientation = this->frame->getOrientation();
-    auto string = orientation->getUnionValue();
-    return jsi::String::createFromUtf8(runtime, string->toStdString());
-  }
-  if (name == "pixelFormat") {
-    auto pixelFormat = this->frame->getPixelFormat();
-    auto string = pixelFormat->getUnionValue();
-    return jsi::String::createFromUtf8(runtime, string->toStdString());
-  }
-  if (name == "timestamp") {
-    return jsi::Value(static_cast<double>(this->frame->getTimestamp()));
-  }
-  if (name == "bytesPerRow") {
-    return jsi::Value(this->frame->getBytesPerRow());
-  }
-  if (name == "planesCount") {
-    return jsi::Value(this->frame->getPlanesCount());
+  if (name == "toString") {
+    jsi::HostFunctionType toString = JSI_FUNC {
+      if (!this->frame) {
+        return jsi::String::createFromUtf8(runtime, "[closed frame]");
+      }
+      auto width = this->frame->getWidth();
+      auto height = this->frame->getHeight();
+      auto format = this->frame->getPixelFormat();
+      auto formatString = format->getUnionValue();
+      auto str = std::to_string(width) + " x " + std::to_string(height) + " " + formatString->toString() + " Frame";
+      return jsi::String::createFromUtf8(runtime, str);
+    };
+    return jsi::Function::createFromHostFunction(runtime, jsi::PropNameID::forUtf8(runtime, "toString"), 0, toString);
   }
 
   // fallback to base implementation

package/android/src/main/cpp/frameprocessor/FrameProcessorPluginHostObject.cpp

@@ -36,7 +36,16 @@ jsi::Value FrameProcessorPluginHostObject::get(jsi::Runtime& runtime, const jsi:
         runtime, jsi::PropNameID::forUtf8(runtime, "call"), 2,
         [=](jsi::Runtime& runtime, const jsi::Value& thisValue, const jsi::Value* arguments, size_t count) -> jsi::Value {
           // Frame is first argument
-          auto frameHostObject = arguments[0].asObject(runtime).asHostObject<FrameHostObject>(runtime);
+          auto frameHolder = arguments[0].asObject(runtime);
+          std::shared_ptr<FrameHostObject> frameHostObject;
+          if (frameHolder.isHostObject<FrameHostObject>(runtime)) {
+            // User directly passed FrameHostObject
+            frameHostObject = frameHolder.getHostObject<FrameHostObject>(runtime);
+          } else {
+            // User passed a wrapper, e.g. DrawableFrame which contains the FrameHostObject as a hidden property
+            jsi::Object actualFrame = frameHolder.getPropertyAsObject(runtime, "__frame");
+            frameHostObject = actualFrame.asHostObject<FrameHostObject>(runtime);
+          }
           auto frame = frameHostObject->frame;
 
           // Options are second argument (possibly undefined)

package/android/src/main/cpp/frameprocessor/VisionCameraProxy.cpp

@@ -45,8 +45,8 @@ std::vector<jsi::PropNameID> VisionCameraProxy::getPropertyNames(jsi::Runtime& r
   return result;
 }
 
-void VisionCameraProxy::setFrameProcessor(int viewTag, jsi::Runtime& runtime, const jsi::Object& object) {
-  _javaProxy->cthis()->setFrameProcessor(viewTag, runtime, object);
+void VisionCameraProxy::setFrameProcessor(int viewTag, jsi::Runtime& runtime, const std::shared_ptr<jsi::Function>& function) {
+  _javaProxy->cthis()->setFrameProcessor(viewTag, runtime, function);
 }
 
 void VisionCameraProxy::removeFrameProcessor(int viewTag) {
@@ -70,8 +70,9 @@ jsi::Value VisionCameraProxy::get(jsi::Runtime& runtime, const jsi::PropNameID&
         runtime, jsi::PropNameID::forUtf8(runtime, "setFrameProcessor"), 1,
         [this](jsi::Runtime& runtime, const jsi::Value& thisValue, const jsi::Value* arguments, size_t count) -> jsi::Value {
           auto viewTag = arguments[0].asNumber();
-          auto object = arguments[1].asObject(runtime);
-          this->setFrameProcessor(static_cast<int>(viewTag), runtime, object);
+          auto frameProcessor = arguments[1].asObject(runtime).asFunction(runtime);
+          auto sharedFunction = std::make_shared<jsi::Function>(std::move(frameProcessor));
+          this->setFrameProcessor(static_cast<int>(viewTag), runtime, sharedFunction);
           return jsi::Value::undefined();
         });
   }

package/android/src/main/cpp/frameprocessor/VisionCameraProxy.h

@@ -26,7 +26,7 @@ class VisionCameraProxy : public jsi::HostObject {
   jsi::Value get(jsi::Runtime& runtime, const jsi::PropNameID& name) override;
 
 private:
-  void setFrameProcessor(int viewTag, jsi::Runtime& runtime, const jsi::Object& frameProcessor);
+  void setFrameProcessor(int viewTag, jsi::Runtime& runtime, const std::shared_ptr<jsi::Function>& frameProcessor);
   void removeFrameProcessor(int viewTag);
   jsi::Value initFrameProcessorPlugin(jsi::Runtime& runtime, const std::string& name, const jsi::Object& options);
 

package/android/src/main/cpp/frameprocessor/java-bindings/JVisionCameraProxy.cpp

@@ -54,17 +54,10 @@ JVisionCameraProxy::~JVisionCameraProxy() {
 #endif
 }
 
-void JVisionCameraProxy::setFrameProcessor(int viewTag, jsi::Runtime& runtime, const jsi::Object& frameProcessorObject) {
+void JVisionCameraProxy::setFrameProcessor(int viewTag, jsi::Runtime& runtime, const std::shared_ptr<jsi::Function>& function) {
 #if VISION_CAMERA_ENABLE_FRAME_PROCESSORS
-  auto frameProcessorType = frameProcessorObject.getProperty(runtime, "type").asString(runtime).utf8(runtime);
-  auto worklet = std::make_shared<RNWorklet::JsiWorklet>(runtime, frameProcessorObject.getProperty(runtime, "frameProcessor"));
-
-  jni::local_ref<JFrameProcessor::javaobject> frameProcessor;
-  if (frameProcessorType == "frame-processor") {
-    frameProcessor = JFrameProcessor::create(worklet, _workletContext);
-  } else {
-    throw std::runtime_error("Unknown FrameProcessor.type passed! Received: " + frameProcessorType);
-  }
+  auto worklet = std::make_shared<RNWorklet::JsiWorklet>(runtime, function);
+  jni::local_ref<JFrameProcessor::javaobject> frameProcessor = JFrameProcessor::create(worklet, _workletContext);
 
   auto setFrameProcessorMethod = javaClassLocal()->getMethod<void(int, alias_ref<JFrameProcessor::javaobject>)>("setFrameProcessor");
   setFrameProcessorMethod(_javaPart, viewTag, frameProcessor);

package/android/src/main/cpp/frameprocessor/java-bindings/JVisionCameraProxy.h

@@ -28,7 +28,7 @@ class JVisionCameraProxy : public jni::HybridClass<JVisionCameraProxy> {
   ~JVisionCameraProxy();
   static void registerNatives();
 
-  void setFrameProcessor(int viewTag, jsi::Runtime& runtime, const jsi::Object& frameProcessor);
+  void setFrameProcessor(int viewTag, jsi::Runtime& runtime, const std::shared_ptr<jsi::Function>& frameProcessor);
   void removeFrameProcessor(int viewTag);
   jni::local_ref<JFrameProcessorPlugin::javaobject> initFrameProcessorPlugin(const std::string& name,
                                                                              jni::local_ref<JMap<jstring, jobject>> options);

package/android/src/main/java/com/mrousavy/camera/CameraView+Focus.kt

@@ -1,12 +1,14 @@
 package com.mrousavy.camera
 
 import com.facebook.react.bridge.ReadableMap
+import com.mrousavy.camera.core.FocusRequiresPreviewError
 import com.mrousavy.camera.extensions.px
 import com.mrousavy.camera.utils.runOnUiThreadAndWait
 
 suspend fun CameraView.focus(pointMap: ReadableMap) {
   val x = pointMap.getDouble("x")
   val y = pointMap.getDouble("y")
+  val previewView = previewView ?: throw FocusRequiresPreviewError()
 
   val point = runOnUiThreadAndWait {
     previewView.meteringPointFactory.createPoint(x.toFloat().px, y.toFloat().px)