diff --git a/README.md b/README.md index e4394f4a..f7ae745f 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,8 @@ ### 已支持 - [x] 原`ChatGPT-Next-Web`所有功能 -- [x] Midjourney `Imgine` `Upscale` `Variation` `Zoom` `Vary` `Pan` `Reroll` `Describe` `Blend` 等众多操作，完美支持 Midjourney 图像生成之后的任何操作（暂除Vary Region以外） +- [x] Midjourney `Imgine` `Upscale` `Variation` `Zoom` `Vary` `Pan` `Reroll` `Describe` `Blend` 等众多操作，完美支持 Midjourney 图像生成之后的任何操作 +- [x] Midjourney 区域重绘（Vary Region）支持 - [x] Midjourney 参考图片 - [x] 绘图进度百分比、实时图像显示 - [x] 支持 Stable Image Ultra @@ -81,6 +82,8 @@ npm run start // #或者开发模式启动： npm run dev ## 截图 ### Midjourney生成主界面 ![step-1](./docs/images/step-2.png) +### Midjourney区域重绘 +![step-1](./docs/images/step-5.png) ### StabilityAI生成主界面 ![step-1](./docs/images/step-3.png) ### 自定义配置接口 diff --git a/README_EN.md b/README_EN.md index ad5f10d2..747cdcfa 100644 --- a/README_EN.md +++ b/README_EN.md @@ -19,7 +19,8 @@ One-click to own your own ChatGPT+StabilityAI+Midjourney web service (based on [ ### Already supported - [x] All functions of the original `ChatGPT-Next-Web` -- [x] Midjourney `Imgine` `Upscale` `Variation` `Zoom` `Vary` `Pan` `Reroll` `Describe` `Blend` and many other operations, perfect support for any operation after Midjourney image generation (except Vary Region for the time being) +- [x] Midjourney `Imgine` `Upscale` `Variation` `Zoom` `Vary` `Pan` `Reroll` `Describe` `Blend` and many other operations, perfect support for any operation after Midjourney image generation +- [x] Midjourney Vary Region Support - [x] Midjourney reference image - [x] Drawing progress percentage, real-time image display - [x] Support Stable Image Ultra @@ -81,6 +82,8 @@ After deployment, click the painting in the upper left corner and select the pai ## Screenshots ### Midjourney generates the main interface ![step-1](./docs/images/step-2-en.png) +### Midjourney Vary Region +![step-1](./docs/images/step-5-en.png) ### StabilityAI generates the main interface ![step-1](./docs/images/step-3-en.png) ### Custom configuration interface diff --git a/app/components/mj/mask-editor/maskEditor.scss b/app/components/mj/mask-editor/maskEditor.scss new file mode 100644 index 00000000..ee690d3d --- /dev/null +++ b/app/components/mj/mask-editor/maskEditor.scss @@ -0,0 +1,37 @@ +$grid-size: 10px; + +.react-mask-editor-outer { + display: flex; + flex-direction: column; + justify-content: flex-start; + align-items: stretch; + + .react-mask-editor-inner { + display: flex; + flex-direction: column; + justify-content: flex-start; + align-items: flex-start; + position: relative; + overflow: auto; + flex: 1 1 auto; + + .all-canvases { + position: relative; + } + } + + canvas { + position: absolute; + top: 0; + left: 0; + } + + .mask-canvas { + z-index: 10; + } + + .cursor-canvas { + z-index: 20; + background-color: transparent; + } +} diff --git a/app/components/mj/mask-editor/maskEditor.tsx b/app/components/mj/mask-editor/maskEditor.tsx new file mode 100644 index 00000000..93a0cebd --- /dev/null +++ b/app/components/mj/mask-editor/maskEditor.tsx @@ -0,0 +1,279 @@ +/* +Github: https://github.com/la-voliere/react-mask-editor +LICENSE: MIT + */ +import * as React from "react"; +import "./maskEditor.scss"; +import { hexToRgb } from "./utils"; + +export interface MaskEditorProps { + src: string; + canvasRef?: React.MutableRefObject; + cursorSize?: number; + onCursorSizeChange?: (size: number) => void; + maskOpacity?: number; + maskColor?: string; + boxSize: { x: number; y: number }; + maskBlendMode?: + | "normal" + | "multiply" + | "screen" + | "overlay" + | "darken" + | "lighten" + | "color-dodge" + | "color-burn" + | "hard-light" + | "soft-light" + | "difference" + | "exclusion" + | "hue" + | "saturation" + | "color" + | "luminosity"; +} + +export const MaskEditorDefaults = { + cursorSize: 10, + maskOpacity: 0.75, + maskColor: "#23272d", + maskBlendMode: "normal", +}; + +export const MaskEditor: React.FC = ( + props: MaskEditorProps, +) => { + const src = props.src; + const cursorSize = props.cursorSize ?? MaskEditorDefaults.cursorSize; + const maskColor = props.maskColor ?? MaskEditorDefaults.maskColor; + const maskBlendMode = props.maskBlendMode ?? MaskEditorDefaults.maskBlendMode; + const maskOpacity = props.maskOpacity ?? MaskEditorDefaults.maskOpacity; + + const canvas = React.useRef(null); + const maskCanvas = React.useRef(null); + const cursorCanvas = React.useRef(null); + const [context, setContext] = React.useState( + null, + ); + const [maskContext, setMaskContext] = + React.useState(null); + const [cursorContext, setCursorContext] = + React.useState(null); + const [size, setSize] = React.useState<{ x: number; y: number }>({ + x: 256, + y: 256, + }); + + React.useLayoutEffect(() => { + if (canvas.current && !context) { + const ctx = (canvas.current as HTMLCanvasElement).getContext("2d"); + setContext(ctx); + } + }, [canvas]); + + React.useLayoutEffect(() => { + if (maskCanvas.current && !context) { + const ctx = (maskCanvas.current as HTMLCanvasElement).getContext("2d"); + if (ctx) { + ctx.fillStyle = "#ffffff"; + ctx.fillRect(0, 0, props.boxSize.x, props.boxSize.y); + } + setMaskContext(ctx); + } + }, [maskCanvas]); + + React.useLayoutEffect(() => { + if (cursorCanvas.current && !context) { + const ctx = (cursorCanvas.current as HTMLCanvasElement).getContext("2d"); + setCursorContext(ctx); + } + }, [cursorCanvas]); + + const [image, setImage] = React.useState(); + React.useEffect(() => { + const img = new Image(); + img.onload = (evt) => { + const canvasWidth = props.boxSize.x; + const canvasHeight = props.boxSize.y; + + const imgWidth = img.width; + const imgHeight = img.height; + + // 计算比例，使图片填满canvas，而不是挤压变形。 + const ratio = Math.max(canvasWidth / imgWidth, canvasHeight / imgHeight); + + // 求出图片放大后必须裁剪的起点位置（为了居中）。 + const newWidth = imgWidth * ratio; + const newHeight = imgHeight * ratio; + const x = (canvasWidth - newWidth) / 2; + const y = (canvasHeight - newHeight) / 2; + setSize({ x: img.width, y: img.height }); + context?.drawImage(img, x, y, newWidth, newHeight); + }; + img.src = src; + setImage(img); + }, [src]); + + React.useEffect(() => { + if (!image) return; + const canvasWidth = props.boxSize.x; + const canvasHeight = props.boxSize.y; + + const imgWidth = image.width; + const imgHeight = image.height; + + // 计算比例，使图片填满canvas，而不是挤压变形。 + const ratio = Math.max(canvasWidth / imgWidth, canvasHeight / imgHeight); + + // 新计算的图片尺寸 + const newWidth = imgWidth * ratio; + const newHeight = imgHeight * ratio; + + // 求出图片放大后需要进行的裁剪区域 + const srcX = (newWidth - canvasWidth) / 2 / ratio; + const srcY = (newHeight - canvasHeight) / 2 / ratio; + const srcWidth = canvasWidth / ratio; + const srcHeight = canvasHeight / ratio; + + // 将源图片裁剪部分绘制到Canvas + context?.drawImage( + image, + srcX, + srcY, + srcWidth, + srcHeight, + 0, + 0, + canvasWidth, + canvasHeight, + ); + }, [size, image]); + + // Pass mask canvas up + React.useLayoutEffect(() => { + if (props.canvasRef && maskCanvas.current) { + props.canvasRef.current = maskCanvas.current; + } + }, [maskCanvas]); + + React.useEffect(() => { + const listener = (evt: MouseEvent) => { + if (cursorContext) { + cursorContext.clearRect(0, 0, size.x, size.y); + + cursorContext.beginPath(); + cursorContext.fillStyle = `${maskColor}88`; + cursorContext.strokeStyle = maskColor; + cursorContext.arc(evt.offsetX, evt.offsetY, cursorSize, 0, 360); + cursorContext.fill(); + cursorContext.stroke(); + } + if (maskContext && evt.buttons > 0) { + maskContext.beginPath(); + maskContext.fillStyle = + evt.buttons > 1 || evt.shiftKey ? "#ffffff" : maskColor; + maskContext.arc(evt.offsetX, evt.offsetY, cursorSize, 0, 360); + maskContext.fill(); + } + }; + const scrollListener = (evt: WheelEvent) => { + if (cursorContext) { + props.onCursorSizeChange?.( + Math.max(0, cursorSize + (evt.deltaY > 0 ? 1 : -1)), + ); + + cursorContext.clearRect(0, 0, size.x, size.y); + + cursorContext.beginPath(); + cursorContext.fillStyle = `${maskColor}88`; + cursorContext.strokeStyle = maskColor; + cursorContext.arc(evt.offsetX, evt.offsetY, cursorSize, 0, 360); + cursorContext.fill(); + cursorContext.stroke(); + + evt.stopPropagation(); + evt.preventDefault(); + } + }; + + cursorCanvas.current?.addEventListener("mousemove", listener); + if (props.onCursorSizeChange) { + cursorCanvas.current?.addEventListener("wheel", scrollListener); + } + return () => { + cursorCanvas.current?.removeEventListener("mousemove", listener); + if (props.onCursorSizeChange) { + cursorCanvas.current?.removeEventListener("wheel", scrollListener); + } + }; + }, [cursorContext, maskContext, cursorCanvas, cursorSize, maskColor, size]); + + const replaceMaskColor = React.useCallback( + (hexColor: string, invert: boolean) => { + const imageData = maskContext?.getImageData(0, 0, size.x, size.y); + const color = hexToRgb(hexColor); + if (imageData) { + for (var i = 0; i < imageData?.data.length; i += 4) { + const pixelColor = + (imageData.data[i] === 255) != invert ? [255, 255, 255] : color; + if (pixelColor) { + imageData.data[i] = pixelColor[0]; + imageData.data[i + 1] = pixelColor[1]; + imageData.data[i + 2] = pixelColor[2]; + } + imageData.data[i + 3] = imageData.data[i + 3]; + } + maskContext?.putImageData(imageData, 0, 0); + } + }, + [maskContext], + ); + React.useEffect(() => replaceMaskColor(maskColor, false), [maskColor]); + + const useSize = props.boxSize ? props.boxSize : size; + + return ( +
+
+ + + +
+
+ ); +}; diff --git a/app/components/mj/mask-editor/utils.ts b/app/components/mj/mask-editor/utils.ts new file mode 100644 index 00000000..3120aca7 --- /dev/null +++ b/app/components/mj/mask-editor/utils.ts @@ -0,0 +1,36 @@ +export const toMask = (canvas: HTMLCanvasElement) => { + const ctx = canvas.getContext("2d"); + const size = { + x: canvas.width, + y: canvas.height, + }; + const imageData = ctx?.getImageData(0, 0, size.x, size.y); + // @ts-ignore + const origData = Uint8ClampedArray.from(imageData.data); + if (imageData) { + for (var i = 0; i < imageData?.data.length; i += 4) { + const pixelColor = + imageData.data[i] === 255 ? [255, 255, 255] : [0, 0, 0]; + imageData.data[i] = pixelColor[0]; + imageData.data[i + 1] = pixelColor[1]; + imageData.data[i + 2] = pixelColor[2]; + imageData.data[i + 3] = 255; + } + ctx?.putImageData(imageData, 0, 0); + } + + const dataUrl = canvas.toDataURL(); + for (var i = 0; i < (imageData?.data?.length || 0); i++) { + // @ts-ignore + imageData.data[i] = origData[i]; + } + // @ts-ignore + ctx?.putImageData(imageData, 0, 0); + + return dataUrl; +}; + +export const hexToRgb = (color: string) => { + var parts = color.replace("#", "").match(/.{1,2}/g); + return parts?.map((part) => parseInt(part, 16)); +}; diff --git a/app/components/mj/mj.tsx b/app/components/mj/mj.tsx index 531edb1c..0e1db4b3 100644 --- a/app/components/mj/mj.tsx +++ b/app/components/mj/mj.tsx @@ -1,6 +1,7 @@ import chatStyles from "@/app/components/chat.module.scss"; import styles from "@/app/components/mj/mj.module.scss"; import homeStyles from "@/app/components/home.module.scss"; +import "react-mask-editor/dist/style.css"; import { IconButton } from "@/app/components/button"; import ReturnIcon from "@/app/icons/return.svg"; @@ -27,6 +28,7 @@ import LoadingIcon from "@/app/icons/three-dots.svg"; import ErrorIcon from "@/app/icons/delete.svg"; import { Property } from "csstype"; import { + Modal, showConfirm, showImageModal, showModal, @@ -34,6 +36,10 @@ import { import { removeImage } from "@/app/utils/chat"; import { SideBar } from "./mj-sidebar"; import { WindowContent } from "@/app/components/home"; +import { MaskEditor } from "./mask-editor/maskEditor"; +import { toMask } from "./mask-editor/utils"; +import Locales from "@/app/locales"; +import { createRoot } from "react-dom/client"; function getSdTaskStatus(item: any) { let s: string; @@ -102,6 +108,8 @@ export function Mj() { const mjDataStore = useMjDataStore(); const [sdImages, setSdImages] = useState(mjStore.draw); const isMj = location.pathname === Path.Mj; + //@ts-ignore + const canvasRef: React.MutableRefObject = useRef(null); useEffect(() => { setSdImages(mjStore.draw); @@ -315,9 +323,7 @@ export function Mj() {
{item.buttons .filter((btn: any) => { - return !["Vary (Region)", "❤️"].includes( - btn.label || btn.emoji, - ); + return !["❤️"].includes(btn.label || btn.emoji); }) .map((btn: any) => { return ( @@ -331,16 +337,114 @@ export function Mj() { - mjStore.sendTask({ - action: "action", - model: item.model, - data: { - customId: btn.customId, - taskId: item.taskId, - }, - }) - } + onClick={() => { + if ( + btn.customId.startsWith( + "MJ::Inpaint::1", + ) + ) { + let inputText = ""; + const div = + document.createElement("div"); + div.className = "modal-mask"; + document.body.appendChild(div); + const root = createRoot(div); + const closeModal = () => { + root.unmount(); + div.remove(); + }; + root.render( + + { + mjStore.sendTask( + { + action: "action", + model: item.model, + data: { + customId: + btn.customId, + taskId: item.taskId, + }, + opt: { + modalData: { + maskBase64: + toMask( + canvasRef.current, + ), + prompt: inputText, + }, + }, + }, + () => { + closeModal(); + }, + ); + }} + > + + } + > +
+
+ + + (inputText = + e.currentTarget.value) + } + /> +
+
+ , + ); + } else { + mjStore.sendTask({ + action: "action", + model: item.model, + data: { + customId: btn.customId, + taskId: item.taskId, + }, + }); + } + }} />
); diff --git a/app/locales/cn.ts b/app/locales/cn.ts index 1f667ca7..27cde2a7 100644 --- a/app/locales/cn.ts +++ b/app/locales/cn.ts @@ -650,6 +650,8 @@ const cn = { Imagine: "生图", Blend: "混图", Describe: "识图", + VaryRegion: "区域重绘", + SubmitRegion: "提交区域重绘", ImagineParam: { IwImage: "参考图", IwImageTip: "最多2张，选择图片后点击图片即可移除", diff --git a/app/locales/en.ts b/app/locales/en.ts index 5fc32c2a..da6e5530 100644 --- a/app/locales/en.ts +++ b/app/locales/en.ts @@ -656,6 +656,8 @@ const en: LocaleType = { Imagine: "Imagine", Blend: "Blend", Describe: "Describe", + VaryRegion: "Vary Region", + SubmitRegion: "Submit Vary Region", ImagineParam: { IwImage: "Reference image", IwImageTip: diff --git a/app/store/mj.ts b/app/store/mj.ts index a325d054..4908cd31 100644 --- a/app/store/mj.ts +++ b/app/store/mj.ts @@ -84,6 +84,7 @@ export const useMjStore = createPersistStore< async sendTask(data: any, call?: (err: any) => void) { const id = nanoid(); const requestData = deepcopy(data.data); + const optData = deepcopy(data.opt); const saveData: any = {}; for (let dataKey in data.data) { if (!["base64Array", "base64"].includes(dataKey)) { @@ -125,6 +126,7 @@ export const useMjStore = createPersistStore< MJProxy.ActionPath, requestData, call, + optData, ); break; } @@ -134,6 +136,10 @@ export const useMjStore = createPersistStore< path: string, data: any, call?: (err: any) => void, + opt?: { + modalData?: any; + dontFetchStatus?: boolean; + }, ) { await this.MjRequestCall("POST", path, data, (resData, err) => { if (err) { @@ -152,7 +158,10 @@ export const useMjStore = createPersistStore< taskId: taskId, }); call && call(null); - this.intervalFetchStatus(id, taskId); + if (!opt?.dontFetchStatus) { + console.log("fetch status", opt); + this.intervalFetchStatus(id, taskId, opt); + } } else { this.updateDraw({ id, @@ -163,11 +172,19 @@ export const useMjStore = createPersistStore< } }); }, - intervalFetchStatus(id: string, taskId: string) { + intervalFetchStatus( + id: string, + taskId: string, + opt?: { + modalData?: any; + }, + ) { if (fetchTasks[taskId]) { return; } setTimeout(async () => { + const task = this.getDraw(id); + if (!task) return; await this.MjRequestCall( "GET", MJProxy.GetTaskById.replace("{id}", taskId), @@ -196,6 +213,35 @@ export const useMjStore = createPersistStore< imageUrl: resData.imageUrl, }); break; + case "MODAL": + if (!task.modalSubmit) { + this.commonPostReqSend( + id, + MJProxy.ModalPath, + { + ...opt?.modalData, + taskId, + }, + (err: any) => { + if (err) { + this.updateDraw({ + id, + status: "error", + error: err?.message || err, + }); + } else { + this.updateDraw({ + id, + modalSubmit: true, + }); + } + }, + { + dontFetchStatus: true, + }, + ); + } + break; } this.intervalFetchStatus(id, taskId); } else { @@ -282,6 +328,9 @@ export const useMjStore = createPersistStore< } }); }, + getDraw(id: string) { + return _get().draw.find((item) => item.id === id); + }, }; return methods; diff --git a/docs/images/step-5-en.png b/docs/images/step-5-en.png new file mode 100644 index 00000000..0c4a37f7 Binary files /dev/null and b/docs/images/step-5-en.png differ diff --git a/docs/images/step-5.png b/docs/images/step-5.png new file mode 100644 index 00000000..0050b22c Binary files /dev/null and b/docs/images/step-5.png differ diff --git a/package.json b/package.json index af8b738e..19b691ba 100644 --- a/package.json +++ b/package.json @@ -36,6 +36,7 @@ "react": "^18.2.0", "react-dom": "^18.2.0", "react-markdown": "^8.0.7", + "react-mask-editor": "^0.0.2", "react-router-dom": "^6.15.0", "rehype-highlight": "^6.0.0", "rehype-katex": "^6.0.3", diff --git a/yarn.lock b/yarn.lock index df713820..cc82c01c 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5316,6 +5316,11 @@ react-markdown@^8.0.7: unist-util-visit "^4.0.0" vfile "^5.0.0" +react-mask-editor@^0.0.2: + version "0.0.2" + resolved "https://registry.npmmirror.com/react-mask-editor/-/react-mask-editor-0.0.2.tgz#d5fd95de9a77a518b8581aa2176a33c596d1e455" + integrity sha512-YXSkdvzt7avOnPymDdQ7d3OO3TLFmkyYNRgBPt70mzL4+B13x9SOqzHIcVTILZlMJr4ntwhoBhRf3nNyinNnKA== + react-redux@^8.1.3: version "8.1.3" resolved "https://registry.yarnpkg.com/react-redux/-/react-redux-8.1.3.tgz#4fdc0462d0acb59af29a13c27ffef6f49ab4df46"