feat: Basic read-only Frame Processors (ImageAnalysis Use-Case) (#2664

) * feat: Basic read-only Frame Processors (`ImageAnalysis` Use-Case) * feat: Set target resolution * Update CameraSession.kt * Refactor * fix: Fix Frame Processor not being unbound
mrousavy · Mar 19, 2024 · ca6518f · ca6518f
1 parent 12d72c9
commit ca6518f
Show file tree

Hide file tree

Showing 5 changed files with 64 additions and 280 deletions.
diff --git a/package/android/src/main/java/com/mrousavy/camera/CameraView.kt b/package/android/src/main/java/com/mrousavy/camera/CameraView.kt
@@ -173,20 +173,20 @@ class CameraView(context: Context) :
           config.photo = CameraConfiguration.Output.Disabled.create()
         }
 
-        // Video/Frame Processor
+        // Video
         if (video || enableFrameProcessor) {
-          config.video = CameraConfiguration.Output.Enabled.create(
-            CameraConfiguration.Video(
-              videoHdr,
-              pixelFormat,
-              enableFrameProcessor,
-              enableGpuBuffers
-            )
-          )
+          config.video = CameraConfiguration.Output.Enabled.create(CameraConfiguration.Video(videoHdr, pixelFormat))
         } else {
           config.video = CameraConfiguration.Output.Disabled.create()
         }
 
+        // Frame Processor
+        if (enableFrameProcessor) {
+          config.frameProcessor = CameraConfiguration.Output.Enabled.create(CameraConfiguration.FrameProcessor(Unit))
+        } else {
+          config.frameProcessor = CameraConfiguration.Output.Disabled.create()
+        }
+
         // Audio
         if (audio) {
           config.audio = CameraConfiguration.Output.Enabled.create(CameraConfiguration.Audio(Unit))

diff --git a/package/android/src/main/java/com/mrousavy/camera/core/CameraConfiguration.kt b/package/android/src/main/java/com/mrousavy/camera/core/CameraConfiguration.kt
@@ -17,6 +17,7 @@ data class CameraConfiguration(
   var preview: Output<Preview> = Output.Disabled.create(),
   var photo: Output<Photo> = Output.Disabled.create(),
   var video: Output<Video> = Output.Disabled.create(),
+  var frameProcessor: Output<FrameProcessor> = Output.Disabled.create(),
   var codeScanner: Output<CodeScanner> = Output.Disabled.create(),
 
   // Orientation
@@ -44,7 +45,8 @@ data class CameraConfiguration(
   // Output<T> types, those need to be comparable
   data class CodeScanner(val codeTypes: List<CodeType>)
   data class Photo(val enableHdr: Boolean, val photoQualityBalance: QualityBalance)
-  data class Video(val enableHdr: Boolean, val pixelFormat: PixelFormat, val enableFrameProcessor: Boolean, val enableGpuBuffers: Boolean)
+  data class Video(val enableHdr: Boolean, val pixelFormat: PixelFormat)
+  data class FrameProcessor(val nothing: Unit)
   data class Audio(val nothing: Unit)
   data class Preview(val surfaceProvider: SurfaceProvider)
 
@@ -92,6 +94,7 @@ data class CameraConfiguration(
         left.video != right.video ||
         left.enableLowLightBoost != right.enableLowLightBoost ||
         left.videoStabilizationMode != right.videoStabilizationMode ||
+        left.frameProcessor != right.frameProcessor ||
         left.codeScanner != right.codeScanner ||
         left.preview != right.preview ||
         left.format != right.format ||

diff --git a/package/android/src/main/java/com/mrousavy/camera/core/CameraSession.kt b/package/android/src/main/java/com/mrousavy/camera/core/CameraSession.kt
@@ -15,14 +15,15 @@ import androidx.camera.core.CameraControl
 import androidx.camera.core.CameraSelector
 import androidx.camera.core.CameraState
 import androidx.camera.core.DynamicRange
+import androidx.camera.core.ExperimentalGetImage
 import androidx.camera.core.FocusMeteringAction
 import androidx.camera.core.ImageAnalysis
 import androidx.camera.core.ImageCapture
 import androidx.camera.core.MeteringPoint
 import androidx.camera.core.MirrorMode
 import androidx.camera.core.Preview
 import androidx.camera.core.TorchState
-import androidx.camera.core.UseCaseGroup
+import androidx.camera.core.UseCase
 import androidx.camera.core.resolutionselector.ResolutionSelector
 import androidx.camera.extensions.ExtensionMode
 import androidx.camera.lifecycle.ProcessCameraProvider
@@ -80,11 +81,13 @@ class CameraSession(private val context: Context, private val callback: Callback
   private var previewOutput: Preview? = null
   private var photoOutput: ImageCapture? = null
   private var videoOutput: VideoCapture<Recorder>? = null
+  private var frameProcessorOutput: ImageAnalysis? = null
   private var codeScannerOutput: ImageAnalysis? = null
+  private val useCases: List<UseCase>
+    get() = listOfNotNull(previewOutput, photoOutput, videoOutput, frameProcessorOutput, codeScannerOutput)
 
   // Camera Outputs State
   private var recorderOutput: Recorder? = null
-  private var frameProcessorEffect: FrameProcessorEffect? = null
 
   // Camera State
   private val mutex = Mutex()
@@ -212,6 +215,7 @@ class CameraSession(private val context: Context, private val callback: Callback
     }
   }
 
+  @OptIn(ExperimentalGetImage::class)
   @SuppressLint("RestrictedApi")
   @Suppress("LiftReturnOrAssignment")
   private fun configureOutputs(configuration: CameraConfiguration) {
@@ -221,8 +225,6 @@ class CameraSession(private val context: Context, private val callback: Callback
 
     Log.i(TAG, "Using FPS Range: $fpsRange")
 
-    // TODO: Check if all of the values we set are supported with Video/Photo/Preview Capabilities from CameraInfo.
-
     // 1. Preview
     val previewConfig = configuration.preview as? CameraConfiguration.Output.Enabled<CameraConfiguration.Preview>
     if (previewConfig != null) {
@@ -321,15 +323,27 @@ class CameraSession(private val context: Context, private val callback: Callback
       recorderOutput = null
     }
 
-    // 3.5 Frame Processor (middleman)
-    if (videoConfig != null && videoConfig.config.enableFrameProcessor) {
-      // The FrameProcessorEffect is a middle-man between the Camera stream and the output surfaces.
-      frameProcessorEffect = FrameProcessorEffect(videoConfig.config.pixelFormat, videoConfig.config.enableGpuBuffers, callback)
+    // 4. Frame Processor
+    val frameProcessorConfig = configuration.frameProcessor as? CameraConfiguration.Output.Enabled<CameraConfiguration.FrameProcessor>
+    if (frameProcessorConfig != null) {
+      Log.i(TAG, "Creating Frame Processor output...")
+      val analyzer = ImageAnalysis.Builder().also { analysis ->
+        analysis.setBackpressureStrategy(ImageAnalysis.STRATEGY_BLOCK_PRODUCER)
+        analysis.setOutputImageFormat(ImageAnalysis.OUTPUT_IMAGE_FORMAT_YUV_420_888)
+        if (format != null) {
+          Log.i(TAG, "Frame Processor size: ${format.videoSize}")
+          val resolutionSelector = ResolutionSelector.Builder().forSize(format.videoSize)
+          analysis.setResolutionSelector(resolutionSelector.build())
+        }
+      }.build()
+      val pipeline = FrameProcessorPipeline(callback)
+      analyzer.setAnalyzer(CameraQueues.videoQueue.executor, pipeline)
+      frameProcessorOutput = analyzer
     } else {
-      frameProcessorEffect = null
+      frameProcessorOutput = null
     }
 
-    // 4. Code Scanner
+    // 5. Code Scanner
     val codeScannerConfig = configuration.codeScanner as? CameraConfiguration.Output.Enabled<CameraConfiguration.CodeScanner>
     if (codeScannerConfig != null) {
       Log.i(TAG, "Creating CodeScanner output...")
@@ -344,7 +358,6 @@ class CameraSession(private val context: Context, private val callback: Callback
   }
 
   private fun closeCurrentOutputs(provider: ProcessCameraProvider) {
-    val useCases = listOfNotNull(previewOutput, photoOutput, videoOutput, codeScannerOutput)
     if (useCases.isEmpty()) {
       return
     }
@@ -353,13 +366,11 @@ class CameraSession(private val context: Context, private val callback: Callback
   }
 
   @SuppressLint("RestrictedApi")
-  @Suppress("LiftReturnOrAssignment")
   private suspend fun configureCamera(provider: ProcessCameraProvider, configuration: CameraConfiguration) {
     Log.i(TAG, "Binding Camera #${configuration.cameraId}...")
     checkCameraPermission()
 
     // Outputs
-    val useCases = listOfNotNull(previewOutput, photoOutput, videoOutput, codeScannerOutput)
     if (useCases.isEmpty()) {
       throw NoOutputsError()
     }
@@ -394,19 +405,9 @@ class CameraSession(private val context: Context, private val callback: Callback
       cameraSelector = cameraSelector.withExtension(context, provider, needsImageAnalysis, ExtensionMode.NIGHT, "NIGHT")
     }
 
-    // Frame Processor is a CameraEffect (Surface middleman)
-    val frameProcessorEffect = frameProcessorEffect
-    if (frameProcessorEffect != null) {
-      val useCaseGroup = UseCaseGroup.Builder()
-      useCases.forEach { useCase -> useCaseGroup.addUseCase(useCase) }
-      useCaseGroup.addEffect(frameProcessorEffect)
-
-      // Bind it all together (must be on UI Thread)
-      camera = provider.bindToLifecycle(this, cameraSelector, useCaseGroup.build())
-    } else {
-      // Bind it all together (must be on UI Thread)
-      camera = provider.bindToLifecycle(this, cameraSelector, *useCases.toTypedArray())
-    }
+    // Bind it all together (must be on UI Thread)
+    Log.i(TAG, "Binding ${useCases.size} use-cases...")
+    camera = provider.bindToLifecycle(this, cameraSelector, *useCases.toTypedArray())
 
     // Listen to Camera events
     var lastState = CameraState.Type.OPENING