feat(ui): add concept for editable image state

Author: psychedelicious

invokeai/frontend/web/src/features/controlLayers/store/types.ts

@@ -37,6 +37,24 @@ export const zImageWithDims = z.object({
 });
 export type ImageWithDims = z.infer<typeof zImageWithDims>;
 
+const zCropBox = z.object({
+  x: z.number().int().min(0),
+  y: z.number().int().min(0),
+  width: z.number().int().positive(),
+  height: z.number().int().positive(),
+});
+export const zCroppableImage = z.object({
+  original: zImageWithDims,
+  crop: z
+    .object({
+      box: zCropBox,
+      ratio: z.number().gt(0).nullable(),
+      image: zImageWithDims,
+    })
+    .optional(),
+});
+export type CroppableImageWithDims = z.infer<typeof zCroppableImage>;
+
 const zImageWithDimsDataURL = z.object({
   dataURL: z.string(),
   width: z.number().int().positive(),

invokeai/frontend/web/src/features/controlLayers/store/util.ts

@@ -10,6 +10,7 @@ import type {
   ChatGPT4oReferenceImageConfig,
   ControlLoRAConfig,
   ControlNetConfig,
+  CroppableImageWithDims,
   FluxKontextReferenceImageConfig,
   FLUXReduxConfig,
   Gemini2_5ReferenceImageConfig,
@@ -45,6 +46,21 @@ export const imageDTOToImageWithDims = ({ image_name, width, height }: ImageDTO)
   height,
 });
 
+export const imageDTOToCroppableImage = (
+  originalImageDTO: ImageDTO,
+  crop?: CroppableImageWithDims['crop']
+): CroppableImageWithDims => {
+  const { image_name, width, height } = originalImageDTO;
+  const val: CroppableImageWithDims = {
+    original: { image_name, width, height },
+  };
+  if (crop) {
+    val.crop = deepClone(crop);
+  }
+
+  return val;
+};
+
 export const imageDTOToImageField = ({ image_name }: ImageDTO): ImageField => ({ image_name });
 
 const DEFAULT_RG_MASK_FILL_COLORS: RgbColor[] = [

invokeai/frontend/web/src/features/dnd/DndImage.tsx

@@ -9,8 +9,10 @@ import { singleImageDndSource } from 'features/dnd/dnd';
 import type { DndDragPreviewSingleImageState } from 'features/dnd/DndDragPreviewSingleImage';
 import { createSingleImageDragPreview, setSingleImageDragPreview } from 'features/dnd/DndDragPreviewSingleImage';
 import { firefoxDndFix } from 'features/dnd/util';
+import { Editor } from 'features/editImageModal/lib/editor';
+import { openEditImageModal } from 'features/editImageModal/store';
 import { useImageContextMenu } from 'features/gallery/components/ContextMenu/ImageContextMenu';
-import { forwardRef, memo, useEffect, useImperativeHandle, useRef, useState } from 'react';
+import { forwardRef, memo, useCallback, useEffect, useImperativeHandle, useRef, useState } from 'react';
 import type { ImageDTO } from 'services/api/types';
 
 const sx = {
@@ -26,12 +28,14 @@ const sx = {
 type Props = {
   imageDTO: ImageDTO;
   asThumbnail?: boolean;
+  editable?: boolean;
 } & ImageProps;
 
 export const DndImage = memo(
-  forwardRef(({ imageDTO, asThumbnail, ...rest }: Props, forwardedRef) => {
+  forwardRef(({ imageDTO, asThumbnail, editable, ...rest }: Props, forwardedRef) => {
     const store = useAppStore();
     const crossOrigin = useStore($crossOrigin);
+    const [previewDataURL, setPreviewDataURl] = useState<string | null>(null);
 
     const [isDragging, setIsDragging] = useState(false);
     const ref = useRef<HTMLImageElement>(null);
@@ -69,18 +73,32 @@ export const DndImage = memo(
 
     useImageContextMenu(imageDTO, ref);
 
+    const edit = useCallback(() => {
+      if (!editable) {
+        return;
+      }
+
+      const editor = new Editor();
+      editor.onCropApply(async () => {
+        const previewDataURL = await editor.exportImage('dataURL', { withCropOverlay: true });
+        setPreviewDataURl(previewDataURL);
+      });
+      openEditImageModal(imageDTO.image_name, editor);
+    }, [editable, imageDTO.image_name]);
+
     return (
       <>
         <Image
           role="button"
           ref={ref}
-          src={asThumbnail ? imageDTO.thumbnail_url : imageDTO.image_url}
+          src={previewDataURL ?? (asThumbnail ? imageDTO.thumbnail_url : imageDTO.image_url)}
           fallbackSrc={asThumbnail ? undefined : imageDTO.thumbnail_url}
           width={imageDTO.width}
           height={imageDTO.height}
           sx={sx}
           data-is-dragging={isDragging}
           crossOrigin={!asThumbnail ? crossOrigin : undefined}
+          onClick={edit}
           {...rest}
         />
         {dragPreviewState?.type === 'single-image' ? createSingleImageDragPreview(dragPreviewState) : null}

invokeai/frontend/web/src/features/dnd/dnd.ts

@@ -4,7 +4,7 @@ import { getDefaultRefImageConfig } from 'features/controlLayers/hooks/addLayerH
 import { getPrefixedId } from 'features/controlLayers/konva/util';
 import { refImageAdded } from 'features/controlLayers/store/refImagesSlice';
 import type { CanvasEntityIdentifier, CanvasEntityType } from 'features/controlLayers/store/types';
-import { imageDTOToImageWithDims } from 'features/controlLayers/store/util';
+import { imageDTOToCroppableImage, imageDTOToImageWithDims } from 'features/controlLayers/store/util';
 import { selectComparisonImages } from 'features/gallery/components/ImageViewer/common';
 import type { BoardId } from 'features/gallery/store/types';
 import {
@@ -641,7 +641,7 @@ export const videoFrameFromImageDndTarget: DndTarget<VideoFrameFromImageDndTarge
   },
   handler: ({ sourceData, dispatch }) => {
     const { imageDTO } = sourceData.payload;
-    dispatch(startingFrameImageChanged(imageDTOToImageWithDims(imageDTO)));
+    dispatch(startingFrameImageChanged(imageDTOToCroppableImage(imageDTO)));
   },
 };
 //#endregion

invokeai/frontend/web/src/features/editImageModal/components/EditorContainer.tsx

@@ -12,6 +12,29 @@ type Props = {
   imageName: string;
 };
 
+const CROP_ASPECT_RATIO_MAP: Record<string, number> = {
+  '16:9': 16 / 9,
+  '3:2': 3 / 2,
+  '4:3': 4 / 3,
+  '1:1': 1,
+  '3:4': 3 / 4,
+  '2:3': 2 / 3,
+  '9:16': 9 / 16,
+};
+
+export const getAspectRatioString = (ratio: number | null) => {
+  if (!ratio) {
+    return 'free';
+  }
+  const entries = Object.entries(CROP_ASPECT_RATIO_MAP);
+  for (const [key, value] of entries) {
+    if (value === ratio) {
+      return key;
+    }
+  }
+  return 'free';
+};
+
 export const EditorContainer = ({ editor, imageName }: Props) => {
   const containerRef = useRef<HTMLDivElement>(null);
   const [zoom, setZoom] = useState(100);
@@ -26,51 +49,44 @@ export const EditorContainer = ({ editor, imageName }: Props) => {
 
   const setup = useCallback(
     async (imageDTO: ImageDTO, container: HTMLDivElement) => {
+      console.log('Setting up editor');
       editor.init(container);
-      editor.setCallbacks({
-        onZoomChange: (zoom) => {
-          setZoom(zoom);
-        },
-        onCropStart: () => {
-          setCropInProgress(true);
-          setCropBox(null);
-        },
-        onCropBoxChange: (crop) => {
-          setCropBox(crop);
-        },
-        onCropApply: () => {
-          setCropApplied(true);
-          setCropInProgress(false);
-          setCropBox(null);
-        },
-        onCropReset: () => {
-          setCropApplied(true);
-          setCropInProgress(false);
-          setCropBox(null);
-        },
-        onCropCancel: () => {
-          setCropInProgress(false);
-          setCropBox(null);
-        },
-        onImageLoad: () => {
-          // setCropInfo('');
-          // setIsCropping(false);
-          // setHasCropBbox(false);
-        },
+      editor.onZoomChange((zoom) => {
+        setZoom(zoom);
+      });
+      editor.onCropStart(() => {
+        setCropInProgress(true);
+        setCropBox(null);
+      });
+      editor.onCropBoxChange((crop) => {
+        setCropBox(crop);
+      });
+      editor.onCropApply(() => {
+        setCropApplied(true);
+        setCropInProgress(false);
+        setCropBox(null);
+      });
+      editor.onCropReset(() => {
+        setCropApplied(true);
+        setCropInProgress(false);
+        setCropBox(null);
+      });
+      editor.onCropCancel(() => {
+        setCropInProgress(false);
+        setCropBox(null);
+      });
+      editor.onImageLoad(() => {
+        // setCropInfo('');
+        // setIsCropping(false);
+        // setHasCropBbox(false);
       });
       const blob = await convertImageUrlToBlob(imageDTO.image_url);
       if (!blob) {
         console.error('Failed to convert image to blob');
         return;
       }
-
+      setAspectRatio(getAspectRatioString(editor.getCropAspectRatio()));
       await editor.loadImage(imageDTO.image_url);
-      editor.startCrop({
-        x: 0,
-        y: 0,
-        width: imageDTO.width,
-        height: imageDTO.height,
-      });
       editor.fitToContainer();
     },
     [editor]
@@ -98,15 +114,7 @@ export const EditorContainer = ({ editor, imageName }: Props) => {
     editor.startCrop();
     // Apply current aspect ratio if not free
     if (aspectRatio !== 'free') {
-      const ratios: Record<string, number> = {
-        '1:1': 1,
-        '4:3': 4 / 3,
-        '16:9': 16 / 9,
-        '3:2': 3 / 2,
-        '2:3': 2 / 3,
-        '9:16': 9 / 16,
-      };
-      editor.setCropAspectRatio(ratios[aspectRatio]);
+      editor.setCropAspectRatio(CROP_ASPECT_RATIO_MAP[aspectRatio] ?? null);
     }
   }, [aspectRatio, editor]);
 
@@ -116,17 +124,9 @@ export const EditorContainer = ({ editor, imageName }: Props) => {
       setAspectRatio(newRatio);
 
       if (newRatio === 'free') {
-        editor.setCropAspectRatio(undefined);
+        editor.setCropAspectRatio(null);
       } else {
-        const ratios: Record<string, number> = {
-          '1:1': 1,
-          '4:3': 4 / 3,
-          '16:9': 16 / 9,
-          '3:2': 3 / 2,
-          '2:3': 2 / 3,
-          '9:16': 9 / 16,
-        };
-        editor.setCropAspectRatio(ratios[newRatio]);
+        editor.setCropAspectRatio(CROP_ASPECT_RATIO_MAP[newRatio] ?? null);
       }
     },
     [editor]
@@ -146,7 +146,7 @@ export const EditorContainer = ({ editor, imageName }: Props) => {
 
   const handleExport = useCallback(async () => {
     try {
-      const blob = await editor.exportImage('blob');
+      const blob = await editor.exportImage('blob', { withCropOverlay: true });
       const file = new File([blob], 'image.png', { type: 'image/png' });
 
       await uploadImage({