Skip to content

Commit

Permalink
type fixes
Browse files Browse the repository at this point in the history
  • Loading branch information
@jbilcke-hf
jbilcke-hf committed Aug 18, 2024
1 parent 8c10101 commit d0e414a
Show file tree
Hide file tree
Showing 38 changed files with 251 additions and 119 deletions.
2 changes: 1 addition & 1 deletion packages/app/src/app/api/resolve/providers/bigmodel/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ export async function resolveSegment(

// https://bigmodel.cn/dev/api#cogvideox
const result = await callCogVideoX(request.settings.bigModelApiKey, {
model: request.settings.videoGenerationWorkflow,
model: request.settings.videoGenerationWorkflow.data,
image_url: request.prompts.video.image,
})

Expand Down
35 changes: 17 additions & 18 deletions packages/app/src/app/api/resolve/providers/replicate/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@ export async function resolveSegment(
const segment = request.segment

if (request.segment.category == ClapSegmentCategory.STORYBOARD) {

const { workflowValues } = getWorkflowInputValues(
request.settings.imageGenerationWorkflow
)
Expand All @@ -31,11 +30,11 @@ export async function resolveSegment(
}

const aspectRatio =
request.meta.orientation === ClapMediaOrientation.SQUARE
? "1:1"
: request.meta.orientation === ClapMediaOrientation.PORTRAIT
? "9:16"
: "16:9"
request.meta.orientation === ClapMediaOrientation.SQUARE
? '1:1'
: request.meta.orientation === ClapMediaOrientation.PORTRAIT
? '9:16'
: '16:9'

if (
request.settings.imageGenerationWorkflow.data === 'fofr/pulid-lightning'
Expand All @@ -47,9 +46,10 @@ export async function resolveSegment(
} else if (
request.settings.imageGenerationWorkflow.data === 'lucataco/flux-dev-lora'
) {

// note: this isn't the right place to do this, because maybe the LoRAs are dynamic
const loraModel = getWorkflowLora(request.settings.imageGenerationWorkflow)
const loraModel = getWorkflowLora(
request.settings.imageGenerationWorkflow
)

params = {
// for some reason this model doesn't support arbitrary width and height,
Expand All @@ -58,14 +58,13 @@ export async function resolveSegment(

hf_lora: workflowValues['hf_lora'] || '',

prompt: [
loraModel?.trigger,
request.prompts.image.positive
].filter(x => x).join(' '),
prompt: [loraModel?.trigger, request.prompts.image.positive]
.filter((x) => x)
.join(' '),

disable_safety_checker: !request.settings.censorNotForAllAudiencesContent,
disable_safety_checker:
!request.settings.censorNotForAllAudiencesContent,
}

} else if (
request.settings.imageGenerationWorkflow.data === 'zsxkib/pulid'
) {
Expand All @@ -86,17 +85,16 @@ export async function resolveSegment(
{ input: params }
)) as any


segment.assetUrl = `${response[0] || ''}`

} else if (request.segment.category === ClapSegmentCategory.DIALOGUE) {
const response = (await replicate.run(
request.settings.voiceGenerationWorkflow.data as any,
{
input: {
text: request.prompts.voice.positive,
audio: request.prompts.voice.identity,
disable_safety_checker: !request.settings.censorNotForAllAudiencesContent,
disable_safety_checker:
!request.settings.censorNotForAllAudiencesContent,
},
}
)) as any
Expand All @@ -107,7 +105,8 @@ export async function resolveSegment(
{
input: {
image: request.prompts.video.image,
disable_safety_checker: !request.settings.censorNotForAllAudiencesContent,
disable_safety_checker:
!request.settings.censorNotForAllAudiencesContent,
},
}
)) as any
Expand Down
2 changes: 0 additions & 2 deletions packages/app/src/app/api/resolve/route.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -132,8 +132,6 @@ export async function POST(req: NextRequest) {
segment.outputType === ClapOutputType.AUDIO ||
segment.outputType === ClapOutputType.VIDEO
) {


// TODO this should be down in the browser side, so that we can scale better
const { durationInMs, hasAudio } = await getMediaInfo(segment.assetUrl)
segment.assetDurationInMs = durationInMs
Expand Down
1 change: 0 additions & 1 deletion packages/app/src/components/toolbars/top-menu/lists/ImageGenerationWorkflows.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,6 @@ export function ImageGenerationWorkflows() {
workflow={w}
currentLora={workflowLora}
onChange={(newLora?: Lora) => {

console.log(`onChange:`, {
w,
newLora,
Expand Down
4 changes: 3 additions & 1 deletion packages/app/src/components/toolbars/top-menu/lists/LoraModelList.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,9 @@ export function LoraModelList({
<MenubarSubTrigger>
<div className="pl-6">
<span>{workflow.label}</span>
<span className="ml-1 opacity-70">{currentLora ? `(${currentLora.label})` : `(no lora selected)`}</span>
<span className="ml-1 opacity-70">
{currentLora ? `(${currentLora.label})` : `(no lora selected)`}
</span>
</div>
</MenubarSubTrigger>
<MenubarSubContent>
Expand Down
1 change: 0 additions & 1 deletion packages/app/src/lib/utils/decodeOutput.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@ export async function decodeOutput(input: any): Promise<string> {
? urlOrBase64
: await fetchContentToBase64(urlOrBase64)


if (base64Url.startsWith('data:image/')) {
if (
base64Url.startsWith('data:image/jpeg') ||
Expand Down
37 changes: 23 additions & 14 deletions packages/app/src/lib/utils/fetchContentToBase64.ts
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
export async function fetchContentToBase64(url: string) {

const predictedFormat = url.split(".").pop()?.trim().toLowerCase()
const predictedFormat = url.split('.').pop()?.trim().toLowerCase()

const res = await fetch(url, {
method: 'GET',
Expand All @@ -15,18 +14,28 @@ export async function fetchContentToBase64(url: string) {
const buffer = Buffer.from(await blob.arrayBuffer())

// some providers such as Replicate return a generic octet-stream type in the headers
const type = blob.type === "application/octet-stream"
? (predictedFormat === "webp" ? "image/webp" :
predictedFormat === "jpeg" ? "image/jpeg" :
predictedFormat === "jpg" ? "image/jpeg" :
predictedFormat === "png" ? "image/png" :
predictedFormat === "avif" ? "image/avif" :
predictedFormat === "heic" ? "image/heic" :
predictedFormat === "mp4" ? "video/mp4" :
predictedFormat === "mp3" ? "audio/mp3" :
predictedFormat === "wav" ? "audio/wav" :
"application/octet-stream"
) : blob.type
const type =
blob.type === 'application/octet-stream'
? predictedFormat === 'webp'
? 'image/webp'
: predictedFormat === 'jpeg'
? 'image/jpeg'
: predictedFormat === 'jpg'
? 'image/jpeg'
: predictedFormat === 'png'
? 'image/png'
: predictedFormat === 'avif'
? 'image/avif'
: predictedFormat === 'heic'
? 'image/heic'
: predictedFormat === 'mp4'
? 'video/mp4'
: predictedFormat === 'mp3'
? 'audio/mp3'
: predictedFormat === 'wav'
? 'audio/wav'
: 'application/octet-stream'
: blob.type

return 'data:' + type + ';base64,' + buffer.toString('base64')
}
2 changes: 1 addition & 1 deletion packages/app/src/services/autocomplete/useAutocomplete.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ export const useAutocomplete = create<AutocompleteStore>((set, get) => ({
set({ isRunning: true })

try {
const storyboards = filterSegments(
const storyboards = filterSegments<TimelineSegment>(
ClapSegmentFilteringMode.ANY,
range,
timeline.segments,
Expand Down
6 changes: 6 additions & 0 deletions packages/app/src/services/editors/workflow-editor/workflows/_documentation_/demo_examples.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
// a workflow is a core concept within the OpenClap format itself
// (a .clap file can contain a workflow)
import {
ClapInputCategory,
ClapWorkflow,
ClapWorkflowCategory,
ClapWorkflowEngine,
Expand Down Expand Up @@ -48,6 +49,8 @@ const justAnExample_DoNotUseThis: ClapWorkflow[] = [
// a screenshot preview of the thumbnail
thumbnailUrl: '',

nonCommercial: false,

// the engine used for this workflow
engine: ClapWorkflowEngine.COMFYUI_WORKFLOW,

Expand All @@ -64,6 +67,8 @@ const justAnExample_DoNotUseThis: ClapWorkflow[] = [
// containing the workflow, serialized so it can fit in a string
data: '{"5":{"inputs":{"width":1024,"height":1024,"batch_size":1},"class_type":"EmptyLatentImage"},"6":{"inputs":{"clip":["11",0],"text":"detailed cinematic dof render of an old dusty detailed CRT monitor on a wooden desk in a dim room with items around, messy dirty room. On the screen are the letters "FLUX Schnell" glowing softly. High detail hard surface render"},"class_type":"CLIPTextEncode"},"8":{"inputs":{"vae":["10",0],"samples":["13",0]},"class_type":"VAEDecode"},"9":{"inputs":{"images":["8",0],"filename_prefix":"ComfyUI"},"class_type":"SaveImage"},"10":{"inputs":{"vae_name":"flux1-ae.safetensors"},"class_type":"VAELoader"},"11":{"inputs":{"type":"flux","clip_name1":"t5xxl_fp16.safetensors","clip_name2":"clip_l.safetensors"},"class_type":"DualCLIPLoader"},"12":{"inputs":{"unet_name":"flux1-schnell.safetensors","weight_dtype":"default"},"class_type":"UNETLoader"},"13":{"inputs":{"noise":["25",0],"guider":["22",0],"sigmas":["17",0],"sampler":["16",0],"latent_image":["5",0]},"class_type":"SamplerCustomAdvanced"},"16":{"inputs":{"sampler_name":"euler"},"class_type":"KSamplerSelect"},"17":{"inputs":{"model":["12",0],"steps":4,"denoise":1,"scheduler":"simple"},"class_type":"BasicScheduler"},"22":{"inputs":{"model":["12",0],"conditioning":["6",0]},"class_type":"BasicGuider"},"25":{"inputs":{"noise_seed":24016052484185},"class_type":"RandomNoise"}}',

schema: '',

// this describe the parameters of the workflow
// this should be as close as possible to the actual parameters
// of the workflow (eg. this could come from some kind of registry, community website API etc)
Expand All @@ -73,6 +78,7 @@ const justAnExample_DoNotUseThis: ClapWorkflow[] = [
id: 'custom_field',
label: 'Custom field',
description: 'A field unique to this workflow',
category: ClapInputCategory.UNKNOWN,
type: 'number',
minValue: 0,
maxValue: 1,
Expand Down
2 changes: 0 additions & 2 deletions packages/app/src/services/editors/workflow-editor/workflows/aitube/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,6 @@ export const aitubeWorkflows: ClapWorkflow[] = [
author: 'AiTube.at',
thumbnailUrl: '',
nonCommercial: false,
canSupportLora: false,
engine: ClapWorkflowEngine.OPENCLAP,
category: ClapWorkflowCategory.IMAGE_GENERATION,
provider: ClapWorkflowProvider.AITUBE,
Expand All @@ -46,7 +45,6 @@ export const aitubeWorkflows: ClapWorkflow[] = [
author: 'AiTube.at',
thumbnailUrl: '',
nonCommercial: false,
canSupportLora: false,
engine: ClapWorkflowEngine.OPENCLAP,
category: ClapWorkflowCategory.MUSIC_GENERATION,
provider: ClapWorkflowProvider.AITUBE,
Expand Down
3 changes: 0 additions & 3 deletions packages/app/src/services/editors/workflow-editor/workflows/anthropic/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,6 @@ export const anthropicWorkflows: ClapWorkflow[] = [
author: 'Anthropic',
thumbnailUrl: '',
nonCommercial: false,
canSupportLora: false,
engine: ClapWorkflowEngine.REST_API,
category: ClapWorkflowCategory.ASSISTANT,
provider: ClapWorkflowProvider.ANTHROPIC,
Expand All @@ -44,7 +43,6 @@ export const anthropicWorkflows: ClapWorkflow[] = [
author: 'Anthropic',
thumbnailUrl: '',
nonCommercial: false,
canSupportLora: false,
engine: ClapWorkflowEngine.REST_API,
category: ClapWorkflowCategory.ASSISTANT,
provider: ClapWorkflowProvider.ANTHROPIC,
Expand All @@ -63,7 +61,6 @@ export const anthropicWorkflows: ClapWorkflow[] = [
author: 'Anthropic',
thumbnailUrl: '',
nonCommercial: false,
canSupportLora: false,
engine: ClapWorkflowEngine.REST_API,
category: ClapWorkflowCategory.ASSISTANT,
provider: ClapWorkflowProvider.ANTHROPIC,
Expand Down
1 change: 0 additions & 1 deletion packages/app/src/services/editors/workflow-editor/workflows/bigmodel/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@ export const bigModelWorkflows: ClapWorkflow[] = [
author: '',
thumbnailUrl: '',
nonCommercial: false,
canSupportLora: false,
engine: ClapWorkflowEngine.REST_API,
provider: ClapWorkflowProvider.BIGMODEL,
category: ClapWorkflowCategory.VIDEO_GENERATION,
Expand Down
8 changes: 4 additions & 4 deletions packages/app/src/services/editors/workflow-editor/workflows/cohere/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,10 +23,12 @@ export const cohereWorkflows: ClapWorkflow[] = [
tags: ['command-r'],
author: 'Cohere',
thumbnailUrl: '',
nonCommercial: false,
engine: ClapWorkflowEngine.REST_API,
category: ClapWorkflowCategory.ASSISTANT,
provider: ClapWorkflowProvider.COHERE,
data: 'command-r-plus',
schema: '',
inputFields: [genericPrompt],
inputValues: {
prompt: genericPrompt.defaultValue,
Expand All @@ -39,10 +41,12 @@ export const cohereWorkflows: ClapWorkflow[] = [
tags: ['command-r'],
author: 'Cohere',
thumbnailUrl: '',
nonCommercial: false,
engine: ClapWorkflowEngine.REST_API,
category: ClapWorkflowCategory.ASSISTANT,
provider: ClapWorkflowProvider.COHERE,
data: 'command-r',
schema: '',
inputFields: [genericPrompt],
inputValues: {
prompt: genericPrompt.defaultValue,
Expand All @@ -56,7 +60,6 @@ export const cohereWorkflows: ClapWorkflow[] = [
author: 'Cohere',
thumbnailUrl: '',
nonCommercial: false,
canSupportLora: false,
engine: ClapWorkflowEngine.REST_API,
category: ClapWorkflowCategory.ASSISTANT,
provider: ClapWorkflowProvider.COHERE,
Expand All @@ -75,7 +78,6 @@ export const cohereWorkflows: ClapWorkflow[] = [
author: 'Cohere',
thumbnailUrl: '',
nonCommercial: false,
canSupportLora: false,
engine: ClapWorkflowEngine.REST_API,
category: ClapWorkflowCategory.ASSISTANT,
provider: ClapWorkflowProvider.COHERE,
Expand All @@ -94,7 +96,6 @@ export const cohereWorkflows: ClapWorkflow[] = [
author: 'Cohere',
thumbnailUrl: '',
nonCommercial: false,
canSupportLora: false,
engine: ClapWorkflowEngine.REST_API,
category: ClapWorkflowCategory.ASSISTANT,
provider: ClapWorkflowProvider.COHERE,
Expand All @@ -113,7 +114,6 @@ export const cohereWorkflows: ClapWorkflow[] = [
author: 'Cohere',
thumbnailUrl: '',
nonCommercial: false,
canSupportLora: false,
engine: ClapWorkflowEngine.REST_API,
category: ClapWorkflowCategory.ASSISTANT,
provider: ClapWorkflowProvider.COHERE,
Expand Down
1 change: 0 additions & 1 deletion packages/app/src/services/editors/workflow-editor/workflows/comfyicu/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,6 @@ export const comfyicuWorkflows: ClapWorkflow[] = [
author: 'BFL (https://BlackForestLabs.ai)',
thumbnailUrl: '',
nonCommercial: false,
canSupportLora: false,
engine: ClapWorkflowEngine.COMFYUI_WORKFLOW,
provider: ClapWorkflowProvider.COMFYICU,
category: ClapWorkflowCategory.IMAGE_GENERATION,
Expand Down
6 changes: 0 additions & 6 deletions packages/app/src/services/editors/workflow-editor/workflows/comfyui/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@ export const comfyuiWorkflows: ClapWorkflow[] = [
author: '',
thumbnailUrl: '',
nonCommercial: false,
canSupportLora: false,
engine: ClapWorkflowEngine.COMFYUI_WORKFLOW,
provider: ClapWorkflowProvider.COMFYUI,
category: ClapWorkflowCategory.IMAGE_GENERATION,
Expand Down Expand Up @@ -53,7 +52,6 @@ export async function getDynamicComfyuiWorkflows(): Promise<ClapWorkflow[]> {
author: 'You',
thumbnailUrl: '',
nonCommercial: false,
canSupportLora: false,
engine: ClapWorkflowEngine.COMFYUI_WORKFLOW,
provider: ClapWorkflowProvider.COMFYUI,
category: ClapWorkflowCategory.IMAGE_GENERATION,
Expand All @@ -72,7 +70,6 @@ export async function getDynamicComfyuiWorkflows(): Promise<ClapWorkflow[]> {
author: 'You',
thumbnailUrl: '',
nonCommercial: false,
canSupportLora: false,
engine: ClapWorkflowEngine.COMFYUI_WORKFLOW,
provider: ClapWorkflowProvider.COMFYUI,
category: ClapWorkflowCategory.VIDEO_GENERATION,
Expand All @@ -91,7 +88,6 @@ export async function getDynamicComfyuiWorkflows(): Promise<ClapWorkflow[]> {
author: 'You',
thumbnailUrl: '',
nonCommercial: false,
canSupportLora: false,
engine: ClapWorkflowEngine.COMFYUI_WORKFLOW,
provider: ClapWorkflowProvider.COMFYUI,
category: ClapWorkflowCategory.VOICE_GENERATION,
Expand All @@ -110,7 +106,6 @@ export async function getDynamicComfyuiWorkflows(): Promise<ClapWorkflow[]> {
author: 'You',
thumbnailUrl: '',
nonCommercial: false,
canSupportLora: false,
engine: ClapWorkflowEngine.COMFYUI_WORKFLOW,
provider: ClapWorkflowProvider.COMFYUI,
category: ClapWorkflowCategory.MUSIC_GENERATION,
Expand All @@ -129,7 +124,6 @@ export async function getDynamicComfyuiWorkflows(): Promise<ClapWorkflow[]> {
author: 'You',
thumbnailUrl: '',
nonCommercial: false,
canSupportLora: false,
engine: ClapWorkflowEngine.COMFYUI_WORKFLOW,
provider: ClapWorkflowProvider.COMFYUI,
category: ClapWorkflowCategory.SOUND_GENERATION,
Expand Down
Loading

0 comments on commit d0e414a

Please sign in to comment.