prepare work for the improvement of face swap

jbilcke-hf · Aug 25, 2024 · 616e708 · 616e708
1 parent 154f3b7
commit 616e708
Show file tree

Hide file tree

Showing 3 changed files with 67 additions and 40 deletions.
diff --git a/packages/app/src/app/api/resolve/providers/falai/index.ts b/packages/app/src/app/api/resolve/providers/falai/index.ts
@@ -12,6 +12,7 @@ import {
 import { getWorkflowInputValues } from '../getWorkflowInputValues'
 import { sampleVoice } from '@/lib/core/constants'
 import { getWorkflowLora } from '@/services/editors/workflow-editor/workflows/common/loras/getWorkflowLora'
+import { runImageFaceSwap } from './runImageFaceSwap'
 
 export async function resolveSegment(
   request: ResolveRequest
@@ -171,45 +172,10 @@ export async function resolveSegment(
 
     segment.assetUrl = result.images[0]?.url || ''
 
-    const imageFaceswapWorkflowModel =
-      request.settings.imageFaceswapWorkflow.data || ''
-
-    if (!isUsingIntegratedFaceId && imageFaceswapWorkflowModel) {
-      try {
-        const faceSwapResult = (await fal.run(imageFaceswapWorkflowModel, {
-          input: {
-            base_image_url: segment.assetUrl,
-            swap_image_url: request.prompts.image.identity,
-
-            sync_mode: true,
-            num_images: 1,
-            enable_safety_checker:
-              request.settings.censorNotForAllAudiencesContent,
-          },
-        })) as FalAiImageResponse
-
-        // note how it is
-        const imageResult = faceSwapResult.image?.url || ''
-
-        if (!imageResult) {
-          throw new Error(`the generate image is empty`)
-        }
-
-        if (request.settings.censorNotForAllAudiencesContent) {
-          if (
-            Array.isArray(result.has_nsfw_concepts) &&
-            result.has_nsfw_concepts.includes(true)
-          ) {
-            throw new Error(
-              `The generated content has been filtered according to your safety settings`
-            )
-          }
-        }
-
-        segment.assetUrl = imageResult
-      } catch (err) {
-        console.error(`failed to run a face-swap using Fal.ai:`, err)
-      }
+    // TODO move this to the router, so that we can use the Fal.ai
+    // face swap with other image providers
+    if (!isUsingIntegratedFaceId) {
+      await runImageFaceSwap(request)
     }
   } else if (request.segment.category === ClapSegmentCategory.VIDEO) {
     model = request.settings.videoGenerationWorkflow.data || ''

diff --git a/packages/app/src/app/api/resolve/providers/falai/runImageFaceSwap.ts b/packages/app/src/app/api/resolve/providers/falai/runImageFaceSwap.ts
@@ -0,0 +1,61 @@
+import * as fal from '@fal-ai/serverless-client'
+import { TimelineSegment } from '@aitube/timeline'
+import { ResolveRequest } from '@aitube/clapper-services'
+import { FalAiImageResponse } from './types'
+
+export async function runImageFaceSwap(
+  request: ResolveRequest
+): Promise<TimelineSegment> {
+  if (!request.settings.falAiApiKey) {
+    throw new Error(`Missing API key for "Fal.ai"`)
+  }
+
+  fal.config({
+    credentials: request.settings.falAiApiKey,
+  })
+
+  const segment: TimelineSegment = request.segment
+
+  const imageFaceswapWorkflowModel =
+    request.settings.imageFaceswapWorkflow.data || ''
+
+  if (imageFaceswapWorkflowModel) {
+    try {
+      const faceSwapResult = (await fal.run(imageFaceswapWorkflowModel, {
+        input: {
+          base_image_url: segment.assetUrl,
+          swap_image_url: request.prompts.image.identity,
+
+          sync_mode: true,
+          num_images: 1,
+          enable_safety_checker:
+            request.settings.censorNotForAllAudiencesContent,
+        },
+      })) as FalAiImageResponse
+
+      // note how it is
+      const imageResult = faceSwapResult.image?.url || ''
+
+      if (!imageResult) {
+        throw new Error(`the generate image is empty`)
+      }
+
+      if (request.settings.censorNotForAllAudiencesContent) {
+        if (
+          Array.isArray(faceSwapResult.has_nsfw_concepts) &&
+          faceSwapResult.has_nsfw_concepts.includes(true)
+        ) {
+          throw new Error(
+            `The generated content has been filtered according to your safety settings`
+          )
+        }
+      }
+
+      segment.assetUrl = imageResult
+    } catch (err) {
+      console.error(`failed to run a face-swap using Fal.ai:`, err)
+    }
+  }
+
+  return segment
+}
diff --git a/packages/app/src/app/api/resolve/route.ts b/packages/app/src/app/api/resolve/route.ts
@@ -48,7 +48,7 @@ export async function POST(req: NextRequest) {
   const { workflow, provider, engine } =
     getSegmentWorkflowProviderAndEngine(request)
 
-    /*
+  /*
   console.log(`Resolving a ${request.segment.category} segment using:`, {
     workflow,
     provider,