From e98d7a52d49deca19bc852dcdfc14d34f72bb062 Mon Sep 17 00:00:00 2001 From: blessedcoolant <54517381+blessedcoolant@users.noreply.github.com> Date: Wed, 16 Aug 2023 17:25:55 +1200 Subject: [PATCH 1/4] feat: Add Seam Painting Options --- invokeai/frontend/web/public/locales/en.json | 6 +- .../Canvas/SeamPainting/ParamSeamBlur.tsx | 36 ++++++ .../ParamSeamPaintingCollapse.tsx | 27 ++++ .../Canvas/SeamPainting/ParamSeamSize.tsx | 36 ++++++ .../Canvas/SeamPainting/ParamSeamSteps.tsx | 36 ++++++ .../Canvas/SeamPainting/ParamSeamStrength.tsx | 36 ++++++ .../SeamPainting/ParamSeamThreshold.tsx | 121 ++++++++++++++++++ .../parameters/store/generationSlice.ts | 36 ++++++ .../SDXLUnifiedCanvasTabParameters.tsx | 2 + .../UnifiedCanvas/UnifiedCanvasParameters.tsx | 2 + 10 files changed, 337 insertions(+), 1 deletion(-) create mode 100644 invokeai/frontend/web/src/features/parameters/components/Parameters/Canvas/SeamPainting/ParamSeamBlur.tsx create mode 100644 invokeai/frontend/web/src/features/parameters/components/Parameters/Canvas/SeamPainting/ParamSeamPaintingCollapse.tsx create mode 100644 invokeai/frontend/web/src/features/parameters/components/Parameters/Canvas/SeamPainting/ParamSeamSize.tsx create mode 100644 invokeai/frontend/web/src/features/parameters/components/Parameters/Canvas/SeamPainting/ParamSeamSteps.tsx create mode 100644 invokeai/frontend/web/src/features/parameters/components/Parameters/Canvas/SeamPainting/ParamSeamStrength.tsx create mode 100644 invokeai/frontend/web/src/features/parameters/components/Parameters/Canvas/SeamPainting/ParamSeamThreshold.tsx diff --git a/invokeai/frontend/web/public/locales/en.json b/invokeai/frontend/web/public/locales/en.json index fbae5b4a303..f41da82e079 100644 --- a/invokeai/frontend/web/public/locales/en.json +++ b/invokeai/frontend/web/public/locales/en.json @@ -506,10 +506,14 @@ "maskAdjustmentsHeader": "Mask Adjustments", "maskBlur": "Mask Blur", "maskBlurMethod": "Mask Blur Method", + "seamPaintingHeader": "Seam Painting", "seamSize": "Seam Size", "seamBlur": "Seam Blur", - "seamStrength": "Seam Strength", "seamSteps": "Seam Steps", + "seamStrength": "Seam Strength", + "seamThreshold": "Seam Threshold", + "seamLowThreshold": "Low", + "seamHighThreshold": "High", "scaleBeforeProcessing": "Scale Before Processing", "scaledWidth": "Scaled W", "scaledHeight": "Scaled H", diff --git a/invokeai/frontend/web/src/features/parameters/components/Parameters/Canvas/SeamPainting/ParamSeamBlur.tsx b/invokeai/frontend/web/src/features/parameters/components/Parameters/Canvas/SeamPainting/ParamSeamBlur.tsx new file mode 100644 index 00000000000..2ab048ce721 --- /dev/null +++ b/invokeai/frontend/web/src/features/parameters/components/Parameters/Canvas/SeamPainting/ParamSeamBlur.tsx @@ -0,0 +1,36 @@ +import type { RootState } from 'app/store/store'; +import { useAppDispatch, useAppSelector } from 'app/store/storeHooks'; +import IAISlider from 'common/components/IAISlider'; +import { setSeamBlur } from 'features/parameters/store/generationSlice'; +import { memo } from 'react'; +import { useTranslation } from 'react-i18next'; + +const ParamSeamBlur = () => { + const dispatch = useAppDispatch(); + const seamBlur = useAppSelector( + (state: RootState) => state.generation.seamBlur + ); + const { t } = useTranslation(); + + return ( + { + dispatch(setSeamBlur(v)); + }} + withInput + withSliderMarks + withReset + handleReset={() => { + dispatch(setSeamBlur(8)); + }} + /> + ); +}; + +export default memo(ParamSeamBlur); diff --git a/invokeai/frontend/web/src/features/parameters/components/Parameters/Canvas/SeamPainting/ParamSeamPaintingCollapse.tsx b/invokeai/frontend/web/src/features/parameters/components/Parameters/Canvas/SeamPainting/ParamSeamPaintingCollapse.tsx new file mode 100644 index 00000000000..23e06797e5a --- /dev/null +++ b/invokeai/frontend/web/src/features/parameters/components/Parameters/Canvas/SeamPainting/ParamSeamPaintingCollapse.tsx @@ -0,0 +1,27 @@ +import { Flex } from '@chakra-ui/react'; +import IAICollapse from 'common/components/IAICollapse'; +import { memo } from 'react'; +import { useTranslation } from 'react-i18next'; +import ParamSeamBlur from './ParamSeamBlur'; +import ParamSeamSize from './ParamSeamSize'; +import ParamSeamSteps from './ParamSeamSteps'; +import ParamSeamStrength from './ParamSeamStrength'; +import ParamSeamThreshold from './ParamSeamThreshold'; + +const ParamSeamPaintingCollapse = () => { + const { t } = useTranslation(); + + return ( + + + + + + + + + + ); +}; + +export default memo(ParamSeamPaintingCollapse); diff --git a/invokeai/frontend/web/src/features/parameters/components/Parameters/Canvas/SeamPainting/ParamSeamSize.tsx b/invokeai/frontend/web/src/features/parameters/components/Parameters/Canvas/SeamPainting/ParamSeamSize.tsx new file mode 100644 index 00000000000..841e9555fdf --- /dev/null +++ b/invokeai/frontend/web/src/features/parameters/components/Parameters/Canvas/SeamPainting/ParamSeamSize.tsx @@ -0,0 +1,36 @@ +import type { RootState } from 'app/store/store'; +import { useAppDispatch, useAppSelector } from 'app/store/storeHooks'; +import IAISlider from 'common/components/IAISlider'; +import { setSeamSize } from 'features/parameters/store/generationSlice'; +import { memo } from 'react'; +import { useTranslation } from 'react-i18next'; + +const ParamSeamSize = () => { + const dispatch = useAppDispatch(); + const seamSize = useAppSelector( + (state: RootState) => state.generation.seamSize + ); + const { t } = useTranslation(); + + return ( + { + dispatch(setSeamSize(v)); + }} + withInput + withSliderMarks + withReset + handleReset={() => { + dispatch(setSeamSize(16)); + }} + /> + ); +}; + +export default memo(ParamSeamSize); diff --git a/invokeai/frontend/web/src/features/parameters/components/Parameters/Canvas/SeamPainting/ParamSeamSteps.tsx b/invokeai/frontend/web/src/features/parameters/components/Parameters/Canvas/SeamPainting/ParamSeamSteps.tsx new file mode 100644 index 00000000000..e69339dbfec --- /dev/null +++ b/invokeai/frontend/web/src/features/parameters/components/Parameters/Canvas/SeamPainting/ParamSeamSteps.tsx @@ -0,0 +1,36 @@ +import type { RootState } from 'app/store/store'; +import { useAppDispatch, useAppSelector } from 'app/store/storeHooks'; +import IAISlider from 'common/components/IAISlider'; +import { setSeamSteps } from 'features/parameters/store/generationSlice'; +import { memo } from 'react'; +import { useTranslation } from 'react-i18next'; + +const ParamSeamSteps = () => { + const dispatch = useAppDispatch(); + const seamSteps = useAppSelector( + (state: RootState) => state.generation.seamSteps + ); + const { t } = useTranslation(); + + return ( + { + dispatch(setSeamSteps(v)); + }} + withInput + withSliderMarks + withReset + handleReset={() => { + dispatch(setSeamSteps(20)); + }} + /> + ); +}; + +export default memo(ParamSeamSteps); diff --git a/invokeai/frontend/web/src/features/parameters/components/Parameters/Canvas/SeamPainting/ParamSeamStrength.tsx b/invokeai/frontend/web/src/features/parameters/components/Parameters/Canvas/SeamPainting/ParamSeamStrength.tsx new file mode 100644 index 00000000000..3f0fa01fcb9 --- /dev/null +++ b/invokeai/frontend/web/src/features/parameters/components/Parameters/Canvas/SeamPainting/ParamSeamStrength.tsx @@ -0,0 +1,36 @@ +import type { RootState } from 'app/store/store'; +import { useAppDispatch, useAppSelector } from 'app/store/storeHooks'; +import IAISlider from 'common/components/IAISlider'; +import { setSeamStrength } from 'features/parameters/store/generationSlice'; +import { memo } from 'react'; +import { useTranslation } from 'react-i18next'; + +const ParamSeamStrength = () => { + const dispatch = useAppDispatch(); + const seamStrength = useAppSelector( + (state: RootState) => state.generation.seamStrength + ); + const { t } = useTranslation(); + + return ( + { + dispatch(setSeamStrength(v)); + }} + withInput + withSliderMarks + withReset + handleReset={() => { + dispatch(setSeamStrength(0.7)); + }} + /> + ); +}; + +export default memo(ParamSeamStrength); diff --git a/invokeai/frontend/web/src/features/parameters/components/Parameters/Canvas/SeamPainting/ParamSeamThreshold.tsx b/invokeai/frontend/web/src/features/parameters/components/Parameters/Canvas/SeamPainting/ParamSeamThreshold.tsx new file mode 100644 index 00000000000..f40491db982 --- /dev/null +++ b/invokeai/frontend/web/src/features/parameters/components/Parameters/Canvas/SeamPainting/ParamSeamThreshold.tsx @@ -0,0 +1,121 @@ +import { + FormControl, + FormLabel, + HStack, + RangeSlider, + RangeSliderFilledTrack, + RangeSliderMark, + RangeSliderThumb, + RangeSliderTrack, + Tooltip, +} from '@chakra-ui/react'; +import type { RootState } from 'app/store/store'; +import { useAppDispatch, useAppSelector } from 'app/store/storeHooks'; +import IAIIconButton from 'common/components/IAIIconButton'; +import { + setSeamHighThreshold, + setSeamLowThreshold, +} from 'features/parameters/store/generationSlice'; +import { memo, useCallback } from 'react'; +import { useTranslation } from 'react-i18next'; +import { BiReset } from 'react-icons/bi'; + +const ParamSeamThreshold = () => { + const dispatch = useAppDispatch(); + const seamLowThreshold = useAppSelector( + (state: RootState) => state.generation.seamLowThreshold + ); + + const seamHighThreshold = useAppSelector( + (state: RootState) => state.generation.seamHighThreshold + ); + const { t } = useTranslation(); + + const handleSeamThresholdChange = useCallback( + (v: number[]) => { + dispatch(setSeamLowThreshold(v[0] as number)); + dispatch(setSeamHighThreshold(v[1] as number)); + }, + [dispatch] + ); + + const handleSeamThresholdReset = () => { + dispatch(setSeamLowThreshold(100)); + dispatch(setSeamHighThreshold(200)); + }; + + return ( + + {t('parameters.seamThreshold')} + + + + + + + + + + + + + 0 + + + 100 + + + 200 + + + 255 + + + } + onClick={handleSeamThresholdReset} + /> + + + ); +}; + +export default memo(ParamSeamThreshold); diff --git a/invokeai/frontend/web/src/features/parameters/store/generationSlice.ts b/invokeai/frontend/web/src/features/parameters/store/generationSlice.ts index 0173391833c..d8495c57518 100644 --- a/invokeai/frontend/web/src/features/parameters/store/generationSlice.ts +++ b/invokeai/frontend/web/src/features/parameters/store/generationSlice.ts @@ -37,6 +37,12 @@ export interface GenerationState { scheduler: SchedulerParam; maskBlur: number; maskBlurMethod: MaskBlurMethodParam; + seamSize: number; + seamBlur: number; + seamSteps: number; + seamStrength: StrengthParam; + seamLowThreshold: number; + seamHighThreshold: number; seed: SeedParam; seedWeights: string; shouldFitToWidthHeight: boolean; @@ -74,6 +80,12 @@ export const initialGenerationState: GenerationState = { scheduler: 'euler', maskBlur: 16, maskBlurMethod: 'box', + seamSize: 16, + seamBlur: 8, + seamSteps: 20, + seamStrength: 0.7, + seamLowThreshold: 100, + seamHighThreshold: 200, seed: 0, seedWeights: '', shouldFitToWidthHeight: true, @@ -200,6 +212,24 @@ export const generationSlice = createSlice({ setMaskBlurMethod: (state, action: PayloadAction) => { state.maskBlurMethod = action.payload; }, + setSeamSize: (state, action: PayloadAction) => { + state.seamSize = action.payload; + }, + setSeamBlur: (state, action: PayloadAction) => { + state.seamBlur = action.payload; + }, + setSeamSteps: (state, action: PayloadAction) => { + state.seamSteps = action.payload; + }, + setSeamStrength: (state, action: PayloadAction) => { + state.seamStrength = action.payload; + }, + setSeamLowThreshold: (state, action: PayloadAction) => { + state.seamLowThreshold = action.payload; + }, + setSeamHighThreshold: (state, action: PayloadAction) => { + state.seamHighThreshold = action.payload; + }, setTileSize: (state, action: PayloadAction) => { state.tileSize = action.payload; }, @@ -306,6 +336,12 @@ export const { setScheduler, setMaskBlur, setMaskBlurMethod, + setSeamSize, + setSeamBlur, + setSeamSteps, + setSeamStrength, + setSeamLowThreshold, + setSeamHighThreshold, setSeed, setSeedWeights, setShouldFitToWidthHeight, diff --git a/invokeai/frontend/web/src/features/sdxl/components/SDXLUnifiedCanvasTabParameters.tsx b/invokeai/frontend/web/src/features/sdxl/components/SDXLUnifiedCanvasTabParameters.tsx index c6af754ad91..74833ebd70d 100644 --- a/invokeai/frontend/web/src/features/sdxl/components/SDXLUnifiedCanvasTabParameters.tsx +++ b/invokeai/frontend/web/src/features/sdxl/components/SDXLUnifiedCanvasTabParameters.tsx @@ -2,6 +2,7 @@ import ParamDynamicPromptsCollapse from 'features/dynamicPrompts/components/Para import ParamLoraCollapse from 'features/lora/components/ParamLoraCollapse'; import ParamInfillAndScalingCollapse from 'features/parameters/components/Parameters/Canvas/InfillAndScaling/ParamInfillAndScalingCollapse'; import ParamMaskAdjustmentCollapse from 'features/parameters/components/Parameters/Canvas/MaskAdjustment/ParamMaskAdjustmentCollapse'; +import ParamSeamPaintingCollapse from 'features/parameters/components/Parameters/Canvas/SeamPainting/ParamSeamPaintingCollapse'; import ParamControlNetCollapse from 'features/parameters/components/Parameters/ControlNet/ParamControlNetCollapse'; import ParamNoiseCollapse from 'features/parameters/components/Parameters/Noise/ParamNoiseCollapse'; import ProcessButtons from 'features/parameters/components/ProcessButtons/ProcessButtons'; @@ -22,6 +23,7 @@ export default function SDXLUnifiedCanvasTabParameters() { + ); } diff --git a/invokeai/frontend/web/src/features/ui/components/tabs/UnifiedCanvas/UnifiedCanvasParameters.tsx b/invokeai/frontend/web/src/features/ui/components/tabs/UnifiedCanvas/UnifiedCanvasParameters.tsx index fcfffee48bf..9e6dc8fef83 100644 --- a/invokeai/frontend/web/src/features/ui/components/tabs/UnifiedCanvas/UnifiedCanvasParameters.tsx +++ b/invokeai/frontend/web/src/features/ui/components/tabs/UnifiedCanvas/UnifiedCanvasParameters.tsx @@ -6,6 +6,7 @@ import ParamControlNetCollapse from 'features/parameters/components/Parameters/C import ParamSymmetryCollapse from 'features/parameters/components/Parameters/Symmetry/ParamSymmetryCollapse'; // import ParamVariationCollapse from 'features/parameters/components/Parameters/Variations/ParamVariationCollapse'; import ParamMaskAdjustmentCollapse from 'features/parameters/components/Parameters/Canvas/MaskAdjustment/ParamMaskAdjustmentCollapse'; +import ParamSeamPaintingCollapse from 'features/parameters/components/Parameters/Canvas/SeamPainting/ParamSeamPaintingCollapse'; import ParamPromptArea from 'features/parameters/components/Parameters/Prompt/ParamPromptArea'; import ProcessButtons from 'features/parameters/components/ProcessButtons/ProcessButtons'; import UnifiedCanvasCoreParameters from './UnifiedCanvasCoreParameters'; @@ -23,6 +24,7 @@ const UnifiedCanvasParameters = () => { + ); From f2ee8a3da8ff078b32fb209ea9af216405faf482 Mon Sep 17 00:00:00 2001 From: blessedcoolant <54517381+blessedcoolant@users.noreply.github.com> Date: Wed, 16 Aug 2023 17:26:23 +1200 Subject: [PATCH 2/4] wip: Basic Seam Painting (only normal models) (no scale) --- .../graphBuilders/buildCanvasOutpaintGraph.ts | 133 +++++++++++++++++- .../nodes/util/graphBuilders/constants.ts | 4 +- 2 files changed, 130 insertions(+), 7 deletions(-) diff --git a/invokeai/frontend/web/src/features/nodes/util/graphBuilders/buildCanvasOutpaintGraph.ts b/invokeai/frontend/web/src/features/nodes/util/graphBuilders/buildCanvasOutpaintGraph.ts index b9c3d5e28e3..587ede0350c 100644 --- a/invokeai/frontend/web/src/features/nodes/util/graphBuilders/buildCanvasOutpaintGraph.ts +++ b/invokeai/frontend/web/src/features/nodes/util/graphBuilders/buildCanvasOutpaintGraph.ts @@ -32,6 +32,7 @@ import { MAIN_MODEL_LOADER, MASK_BLUR, MASK_COMBINE, + MASK_EDGE, MASK_FROM_ALPHA, MASK_RESIZE_DOWN, MASK_RESIZE_UP, @@ -40,6 +41,8 @@ import { POSITIVE_CONDITIONING, RANDOM_INT, RANGE_OF_SIZE, + SEAM_FIX_DENOISE_LATENTS, + SEAM_MASK_COMBINE, } from './constants'; /** @@ -67,6 +70,12 @@ export const buildCanvasOutpaintGraph = ( shouldUseCpuNoise, maskBlur, maskBlurMethod, + seamSize, + seamBlur, + seamSteps, + seamStrength, + seamLowThreshold, + seamHighThreshold, tileSize, infillMethod, clipSkip, @@ -130,6 +139,11 @@ export const buildCanvasOutpaintGraph = ( is_intermediate: true, mask2: canvasMaskImage, }, + [SEAM_MASK_COMBINE]: { + type: 'mask_combine', + id: MASK_COMBINE, + is_intermediate: true, + }, [MASK_BLUR]: { type: 'img_blur', id: MASK_BLUR, @@ -165,6 +179,25 @@ export const buildCanvasOutpaintGraph = ( denoising_start: 1 - strength, denoising_end: 1, }, + [MASK_EDGE]: { + type: 'mask_edge', + id: MASK_EDGE, + is_intermediate: true, + edge_size: seamSize, + edge_blur: seamBlur, + low_threshold: seamLowThreshold, + high_threshold: seamHighThreshold, + }, + [SEAM_FIX_DENOISE_LATENTS]: { + type: 'denoise_latents', + id: SEAM_FIX_DENOISE_LATENTS, + is_intermediate: true, + steps: seamSteps, + cfg_scale: cfg_scale, + scheduler: scheduler, + denoising_start: 1 - seamStrength, + denoising_end: 1, + }, [LATENTS_TO_IMAGE]: { type: 'l2i', id: LATENTS_TO_IMAGE, @@ -333,12 +366,63 @@ export const buildCanvasOutpaintGraph = ( field: 'seed', }, }, - // Decode the result from Inpaint + // Seam Paint + { + source: { + node_id: MAIN_MODEL_LOADER, + field: 'unet', + }, + destination: { + node_id: SEAM_FIX_DENOISE_LATENTS, + field: 'unet', + }, + }, + { + source: { + node_id: POSITIVE_CONDITIONING, + field: 'conditioning', + }, + destination: { + node_id: SEAM_FIX_DENOISE_LATENTS, + field: 'positive_conditioning', + }, + }, + { + source: { + node_id: NEGATIVE_CONDITIONING, + field: 'conditioning', + }, + destination: { + node_id: SEAM_FIX_DENOISE_LATENTS, + field: 'negative_conditioning', + }, + }, + { + source: { + node_id: NOISE, + field: 'noise', + }, + destination: { + node_id: SEAM_FIX_DENOISE_LATENTS, + field: 'noise', + }, + }, { source: { node_id: DENOISE_LATENTS, field: 'latents', }, + destination: { + node_id: SEAM_FIX_DENOISE_LATENTS, + field: 'latents', + }, + }, + // Decode the result from Inpaint + { + source: { + node_id: SEAM_FIX_DENOISE_LATENTS, + field: 'latents', + }, destination: { node_id: LATENTS_TO_IMAGE, field: 'latents', @@ -348,7 +432,6 @@ export const buildCanvasOutpaintGraph = ( }; // Add Infill Nodes - if (infillMethod === 'patchmatch') { graph.nodes[INPAINT_INFILL] = { type: 'infill_patchmatch', @@ -553,7 +636,6 @@ export const buildCanvasOutpaintGraph = ( }; graph.nodes[MASK_BLUR] = { ...(graph.nodes[MASK_BLUR] as ImageBlurInvocation), - image: canvasMaskImage, }; graph.edges.push( @@ -568,6 +650,47 @@ export const buildCanvasOutpaintGraph = ( field: 'image', }, }, + // Seam Paint Mask + { + source: { + node_id: MASK_FROM_ALPHA, + field: 'image', + }, + destination: { + node_id: MASK_EDGE, + field: 'image', + }, + }, + { + source: { + node_id: MASK_EDGE, + field: 'image', + }, + destination: { + node_id: SEAM_FIX_DENOISE_LATENTS, + field: 'mask', + }, + }, + { + source: { + node_id: MASK_FROM_ALPHA, + field: 'image', + }, + destination: { + node_id: SEAM_MASK_COMBINE, + field: 'mask1', + }, + }, + { + source: { + node_id: MASK_EDGE, + field: 'image', + }, + destination: { + node_id: SEAM_MASK_COMBINE, + field: 'mask2', + }, + }, // Color Correct The Inpainted Result { source: { @@ -591,7 +714,7 @@ export const buildCanvasOutpaintGraph = ( }, { source: { - node_id: MASK_BLUR, + node_id: SEAM_MASK_COMBINE, field: 'image', }, destination: { @@ -622,7 +745,7 @@ export const buildCanvasOutpaintGraph = ( }, { source: { - node_id: MASK_BLUR, + node_id: SEAM_MASK_COMBINE, field: 'image', }, destination: { diff --git a/invokeai/frontend/web/src/features/nodes/util/graphBuilders/constants.ts b/invokeai/frontend/web/src/features/nodes/util/graphBuilders/constants.ts index 3e213120b31..48d759a6e38 100644 --- a/invokeai/frontend/web/src/features/nodes/util/graphBuilders/constants.ts +++ b/invokeai/frontend/web/src/features/nodes/util/graphBuilders/constants.ts @@ -18,8 +18,6 @@ export const IMAGE_TO_LATENTS = 'image_to_latents'; export const LATENTS_TO_LATENTS = 'latents_to_latents'; export const RESIZE = 'resize_image'; export const CANVAS_OUTPUT = 'canvas_output'; -export const INPAINT = 'inpaint'; -export const INPAINT_SEAM_FIX = 'inpaint_seam_fix'; export const INPAINT_IMAGE = 'inpaint_image'; export const SCALED_INPAINT_IMAGE = 'scaled_inpaint_image'; export const INPAINT_IMAGE_RESIZE_UP = 'inpaint_image_resize_up'; @@ -27,10 +25,12 @@ export const INPAINT_IMAGE_RESIZE_DOWN = 'inpaint_image_resize_down'; export const INPAINT_INFILL = 'inpaint_infill'; export const INPAINT_INFILL_RESIZE_DOWN = 'inpaint_infill_resize_down'; export const INPAINT_FINAL_IMAGE = 'inpaint_final_image'; +export const SEAM_FIX_DENOISE_LATENTS = 'seam_fix_denoise_latents'; export const MASK_FROM_ALPHA = 'tomask'; export const MASK_EDGE = 'mask_edge'; export const MASK_BLUR = 'mask_blur'; export const MASK_COMBINE = 'mask_combine'; +export const SEAM_MASK_COMBINE = 'seam_mask_combine'; export const MASK_RESIZE_UP = 'mask_resize_up'; export const MASK_RESIZE_DOWN = 'mask_resize_down'; export const COLOR_CORRECT = 'color_correct'; From cc7c6e5d4161136a0faf13c1c29833cfb35abf96 Mon Sep 17 00:00:00 2001 From: blessedcoolant <54517381+blessedcoolant@users.noreply.github.com> Date: Wed, 16 Aug 2023 19:35:03 +1200 Subject: [PATCH 3/4] feat: Add Seam Painting with Scale Before --- .../graphBuilders/buildCanvasOutpaintGraph.ts | 83 ++++++++++++++++++- .../nodes/util/graphBuilders/constants.ts | 2 + 2 files changed, 82 insertions(+), 3 deletions(-) diff --git a/invokeai/frontend/web/src/features/nodes/util/graphBuilders/buildCanvasOutpaintGraph.ts b/invokeai/frontend/web/src/features/nodes/util/graphBuilders/buildCanvasOutpaintGraph.ts index 587ede0350c..a949c88e5f2 100644 --- a/invokeai/frontend/web/src/features/nodes/util/graphBuilders/buildCanvasOutpaintGraph.ts +++ b/invokeai/frontend/web/src/features/nodes/util/graphBuilders/buildCanvasOutpaintGraph.ts @@ -43,6 +43,8 @@ import { RANGE_OF_SIZE, SEAM_FIX_DENOISE_LATENTS, SEAM_MASK_COMBINE, + SEAM_MASK_RESIZE_DOWN, + SEAM_MASK_RESIZE_UP, } from './constants'; /** @@ -461,6 +463,13 @@ export const buildCanvasOutpaintGraph = ( width: scaledWidth, height: scaledHeight, }; + graph.nodes[SEAM_MASK_RESIZE_UP] = { + type: 'img_resize', + id: SEAM_MASK_RESIZE_UP, + is_intermediate: true, + width: scaledWidth, + height: scaledHeight, + }; graph.nodes[INPAINT_IMAGE_RESIZE_DOWN] = { type: 'img_resize', id: INPAINT_IMAGE_RESIZE_DOWN, @@ -482,6 +491,13 @@ export const buildCanvasOutpaintGraph = ( width: width, height: height, }; + graph.nodes[SEAM_MASK_RESIZE_DOWN] = { + type: 'img_resize', + id: SEAM_MASK_RESIZE_DOWN, + is_intermediate: true, + width: width, + height: height, + }; graph.nodes[NOISE] = { ...(graph.nodes[NOISE] as NoiseInvocation), @@ -523,6 +539,57 @@ export const buildCanvasOutpaintGraph = ( field: 'image', }, }, + // Seam Paint Mask + { + source: { + node_id: MASK_FROM_ALPHA, + field: 'image', + }, + destination: { + node_id: MASK_EDGE, + field: 'image', + }, + }, + { + source: { + node_id: MASK_EDGE, + field: 'image', + }, + destination: { + node_id: SEAM_MASK_RESIZE_UP, + field: 'image', + }, + }, + { + source: { + node_id: SEAM_MASK_RESIZE_UP, + field: 'image', + }, + destination: { + node_id: SEAM_FIX_DENOISE_LATENTS, + field: 'mask', + }, + }, + { + source: { + node_id: MASK_BLUR, + field: 'image', + }, + destination: { + node_id: SEAM_MASK_COMBINE, + field: 'mask1', + }, + }, + { + source: { + node_id: SEAM_MASK_RESIZE_UP, + field: 'image', + }, + destination: { + node_id: SEAM_MASK_COMBINE, + field: 'mask2', + }, + }, // Resize Results Down { source: { @@ -536,7 +603,7 @@ export const buildCanvasOutpaintGraph = ( }, { source: { - node_id: MASK_BLUR, + node_id: MASK_RESIZE_UP, field: 'image', }, destination: { @@ -544,6 +611,16 @@ export const buildCanvasOutpaintGraph = ( field: 'image', }, }, + { + source: { + node_id: SEAM_MASK_COMBINE, + field: 'image', + }, + destination: { + node_id: SEAM_MASK_RESIZE_DOWN, + field: 'image', + }, + }, { source: { node_id: INPAINT_INFILL, @@ -577,7 +654,7 @@ export const buildCanvasOutpaintGraph = ( }, { source: { - node_id: MASK_RESIZE_DOWN, + node_id: SEAM_MASK_RESIZE_DOWN, field: 'image', }, destination: { @@ -608,7 +685,7 @@ export const buildCanvasOutpaintGraph = ( }, { source: { - node_id: MASK_RESIZE_DOWN, + node_id: SEAM_MASK_RESIZE_DOWN, field: 'image', }, destination: { diff --git a/invokeai/frontend/web/src/features/nodes/util/graphBuilders/constants.ts b/invokeai/frontend/web/src/features/nodes/util/graphBuilders/constants.ts index 48d759a6e38..1f6acd4e264 100644 --- a/invokeai/frontend/web/src/features/nodes/util/graphBuilders/constants.ts +++ b/invokeai/frontend/web/src/features/nodes/util/graphBuilders/constants.ts @@ -31,6 +31,8 @@ export const MASK_EDGE = 'mask_edge'; export const MASK_BLUR = 'mask_blur'; export const MASK_COMBINE = 'mask_combine'; export const SEAM_MASK_COMBINE = 'seam_mask_combine'; +export const SEAM_MASK_RESIZE_UP = 'seam_mask_resize_up'; +export const SEAM_MASK_RESIZE_DOWN = 'seam_mask_resize_down'; export const MASK_RESIZE_UP = 'mask_resize_up'; export const MASK_RESIZE_DOWN = 'mask_resize_down'; export const COLOR_CORRECT = 'color_correct'; From 9cbaefaa8139352da24dc017e6d0b437126c4e66 Mon Sep 17 00:00:00 2001 From: blessedcoolant <54517381+blessedcoolant@users.noreply.github.com> Date: Wed, 16 Aug 2023 19:46:48 +1200 Subject: [PATCH 4/4] feat: Add Seam Painting to SDXL --- .../buildCanvasSDXLOutpaintGraph.ts | 217 +++++++++++++++++- 1 file changed, 209 insertions(+), 8 deletions(-) diff --git a/invokeai/frontend/web/src/features/nodes/util/graphBuilders/buildCanvasSDXLOutpaintGraph.ts b/invokeai/frontend/web/src/features/nodes/util/graphBuilders/buildCanvasSDXLOutpaintGraph.ts index 4d098f959fe..1cc268c03d7 100644 --- a/invokeai/frontend/web/src/features/nodes/util/graphBuilders/buildCanvasSDXLOutpaintGraph.ts +++ b/invokeai/frontend/web/src/features/nodes/util/graphBuilders/buildCanvasSDXLOutpaintGraph.ts @@ -29,6 +29,7 @@ import { LATENTS_TO_IMAGE, MASK_BLUR, MASK_COMBINE, + MASK_EDGE, MASK_FROM_ALPHA, MASK_RESIZE_DOWN, MASK_RESIZE_UP, @@ -40,6 +41,10 @@ import { SDXL_CANVAS_OUTPAINT_GRAPH, SDXL_DENOISE_LATENTS, SDXL_MODEL_LOADER, + SEAM_FIX_DENOISE_LATENTS, + SEAM_MASK_COMBINE, + SEAM_MASK_RESIZE_DOWN, + SEAM_MASK_RESIZE_UP, } from './constants'; import { craftSDXLStylePrompt } from './helpers/craftSDXLStylePrompt'; @@ -67,6 +72,12 @@ export const buildCanvasSDXLOutpaintGraph = ( shouldUseCpuNoise, maskBlur, maskBlurMethod, + seamSize, + seamBlur, + seamSteps, + seamStrength, + seamLowThreshold, + seamHighThreshold, tileSize, infillMethod, } = state.generation; @@ -133,6 +144,11 @@ export const buildCanvasSDXLOutpaintGraph = ( is_intermediate: true, mask2: canvasMaskImage, }, + [SEAM_MASK_COMBINE]: { + type: 'mask_combine', + id: MASK_COMBINE, + is_intermediate: true, + }, [MASK_BLUR]: { type: 'img_blur', id: MASK_BLUR, @@ -170,6 +186,25 @@ export const buildCanvasSDXLOutpaintGraph = ( : 1 - strength, denoising_end: shouldUseSDXLRefiner ? refinerStart : 1, }, + [MASK_EDGE]: { + type: 'mask_edge', + id: MASK_EDGE, + is_intermediate: true, + edge_size: seamSize, + edge_blur: seamBlur, + low_threshold: seamLowThreshold, + high_threshold: seamHighThreshold, + }, + [SEAM_FIX_DENOISE_LATENTS]: { + type: 'denoise_latents', + id: SEAM_FIX_DENOISE_LATENTS, + is_intermediate: true, + steps: seamSteps, + cfg_scale: cfg_scale, + scheduler: scheduler, + denoising_start: 1 - seamStrength, + denoising_end: 1, + }, [LATENTS_TO_IMAGE]: { type: 'l2i', id: LATENTS_TO_IMAGE, @@ -347,12 +382,63 @@ export const buildCanvasSDXLOutpaintGraph = ( field: 'seed', }, }, - // Decode inpainted latents to image + // Seam Paint + { + source: { + node_id: SDXL_MODEL_LOADER, + field: 'unet', + }, + destination: { + node_id: SEAM_FIX_DENOISE_LATENTS, + field: 'unet', + }, + }, + { + source: { + node_id: POSITIVE_CONDITIONING, + field: 'conditioning', + }, + destination: { + node_id: SEAM_FIX_DENOISE_LATENTS, + field: 'positive_conditioning', + }, + }, + { + source: { + node_id: NEGATIVE_CONDITIONING, + field: 'conditioning', + }, + destination: { + node_id: SEAM_FIX_DENOISE_LATENTS, + field: 'negative_conditioning', + }, + }, + { + source: { + node_id: NOISE, + field: 'noise', + }, + destination: { + node_id: SEAM_FIX_DENOISE_LATENTS, + field: 'noise', + }, + }, { source: { node_id: SDXL_DENOISE_LATENTS, field: 'latents', }, + destination: { + node_id: SEAM_FIX_DENOISE_LATENTS, + field: 'latents', + }, + }, + // Decode inpainted latents to image + { + source: { + node_id: SEAM_FIX_DENOISE_LATENTS, + field: 'latents', + }, destination: { node_id: LATENTS_TO_IMAGE, field: 'latents', @@ -392,6 +478,13 @@ export const buildCanvasSDXLOutpaintGraph = ( width: scaledWidth, height: scaledHeight, }; + graph.nodes[SEAM_MASK_RESIZE_UP] = { + type: 'img_resize', + id: SEAM_MASK_RESIZE_UP, + is_intermediate: true, + width: scaledWidth, + height: scaledHeight, + }; graph.nodes[INPAINT_IMAGE_RESIZE_DOWN] = { type: 'img_resize', id: INPAINT_IMAGE_RESIZE_DOWN, @@ -413,6 +506,13 @@ export const buildCanvasSDXLOutpaintGraph = ( width: width, height: height, }; + graph.nodes[SEAM_MASK_RESIZE_DOWN] = { + type: 'img_resize', + id: SEAM_MASK_RESIZE_DOWN, + is_intermediate: true, + width: width, + height: height, + }; graph.nodes[NOISE] = { ...(graph.nodes[NOISE] as NoiseInvocation), @@ -454,6 +554,57 @@ export const buildCanvasSDXLOutpaintGraph = ( field: 'image', }, }, + // Seam Paint Mask + { + source: { + node_id: MASK_FROM_ALPHA, + field: 'image', + }, + destination: { + node_id: MASK_EDGE, + field: 'image', + }, + }, + { + source: { + node_id: MASK_EDGE, + field: 'image', + }, + destination: { + node_id: SEAM_MASK_RESIZE_UP, + field: 'image', + }, + }, + { + source: { + node_id: SEAM_MASK_RESIZE_UP, + field: 'image', + }, + destination: { + node_id: SEAM_FIX_DENOISE_LATENTS, + field: 'mask', + }, + }, + { + source: { + node_id: MASK_BLUR, + field: 'image', + }, + destination: { + node_id: SEAM_MASK_COMBINE, + field: 'mask1', + }, + }, + { + source: { + node_id: SEAM_MASK_RESIZE_UP, + field: 'image', + }, + destination: { + node_id: SEAM_MASK_COMBINE, + field: 'mask2', + }, + }, // Resize Results Down { source: { @@ -467,7 +618,7 @@ export const buildCanvasSDXLOutpaintGraph = ( }, { source: { - node_id: MASK_BLUR, + node_id: MASK_RESIZE_UP, field: 'image', }, destination: { @@ -475,6 +626,16 @@ export const buildCanvasSDXLOutpaintGraph = ( field: 'image', }, }, + { + source: { + node_id: SEAM_MASK_COMBINE, + field: 'image', + }, + destination: { + node_id: SEAM_MASK_RESIZE_DOWN, + field: 'image', + }, + }, { source: { node_id: INPAINT_INFILL, @@ -508,7 +669,7 @@ export const buildCanvasSDXLOutpaintGraph = ( }, { source: { - node_id: MASK_RESIZE_DOWN, + node_id: SEAM_MASK_RESIZE_DOWN, field: 'image', }, destination: { @@ -539,7 +700,7 @@ export const buildCanvasSDXLOutpaintGraph = ( }, { source: { - node_id: MASK_RESIZE_DOWN, + node_id: SEAM_MASK_RESIZE_DOWN, field: 'image', }, destination: { @@ -567,7 +728,6 @@ export const buildCanvasSDXLOutpaintGraph = ( }; graph.nodes[MASK_BLUR] = { ...(graph.nodes[MASK_BLUR] as ImageBlurInvocation), - image: canvasMaskImage, }; graph.edges.push( @@ -582,6 +742,47 @@ export const buildCanvasSDXLOutpaintGraph = ( field: 'image', }, }, + // Seam Paint Mask + { + source: { + node_id: MASK_FROM_ALPHA, + field: 'image', + }, + destination: { + node_id: MASK_EDGE, + field: 'image', + }, + }, + { + source: { + node_id: MASK_EDGE, + field: 'image', + }, + destination: { + node_id: SEAM_FIX_DENOISE_LATENTS, + field: 'mask', + }, + }, + { + source: { + node_id: MASK_FROM_ALPHA, + field: 'image', + }, + destination: { + node_id: SEAM_MASK_COMBINE, + field: 'mask1', + }, + }, + { + source: { + node_id: MASK_EDGE, + field: 'image', + }, + destination: { + node_id: SEAM_MASK_COMBINE, + field: 'mask2', + }, + }, // Color Correct The Inpainted Result { source: { @@ -605,7 +806,7 @@ export const buildCanvasSDXLOutpaintGraph = ( }, { source: { - node_id: MASK_BLUR, + node_id: SEAM_MASK_COMBINE, field: 'image', }, destination: { @@ -636,7 +837,7 @@ export const buildCanvasSDXLOutpaintGraph = ( }, { source: { - node_id: MASK_BLUR, + node_id: SEAM_MASK_COMBINE, field: 'image', }, destination: { @@ -669,7 +870,7 @@ export const buildCanvasSDXLOutpaintGraph = ( // Add Refiner if enabled if (shouldUseSDXLRefiner) { - addSDXLRefinerToGraph(state, graph, SDXL_DENOISE_LATENTS); + addSDXLRefinerToGraph(state, graph, SEAM_FIX_DENOISE_LATENTS); } // optionally add custom VAE