Feature: Mask Generation

README.md

@@ -104,8 +104,18 @@ The project is structured as follows:
 - Create a new story for the task in `QuickInput.stories.js`
 - If the user is able to upload files for this task, add the task to `UppyFileTypeCheckerPlugin`
 - Add the new task to `ModelDetailPage` in `getSampleInputs` and `getInputType`
+- If the task requires new input types, you will need to add those
+  - Add the new input type to `TaskInputTypes` and `QuickInputType`
+  - Update `SampleInputsTab` as necessary
+  - Update `UploadInputsTab` as necessary
+  - Update `URLInputsTab` as necessary
+    - Be sure to read the `IMPORTANT` comment before editing the inputs
+- ...to be continued
 - Additional Notes:
-  - To test the upload dashboard, open `useUploadInputControl` and find the text `UNCOMMENT THIS BEFORE COMMITTING` and comment it out.
+  - To test the upload dashboard in storybook, open `useUploadInputControl` and:
+    - Find the text `COMMENT THIS OUT BEFORE COMMITTING` and uncomment it.
+    - Find the text `UNCOMMENT THIS BEFORE COMMITTING` and comment it out.
+    - Be sure to reset these when you are finished. These changes are necessary to avoid making a call to S3 (which will not work) as well as to make a fake `uploadURL` so that our code will continue to execute as if the S3 call was successful.
   - To see what your currently-selected inputs, and the current state of the data that will be sent to the API (prior to clicking the "Run Model" button), go to `useQuickInputControl.js` and uncomment the useEffect with `console.log`s in it.
   - To check the array of inputs that you are submitting to the API, add an `onRunModelClicked` function to your task in `QuickInput.stories.js` (it takes the inputs as a param). Because of how Storybook works, the component you're building won't be passed the real method, but you can make a mock in the stories file to test behavior.
     - See `TextConversationOutput.stories.js` for an example of how to test api requests in this way

src/components/Experiment/QuickInput/QuickAudioInput.js

@@ -17,6 +17,7 @@ export default function QuickAudioInput(props) {
     selectTab,
     selectInput,
     runModel,
+    submitButtonIsDisabled
   } = useQuickInputControl(props);
   const { getBlock, getElement } = useBEMNaming("quick-audio-input");
 
@@ -61,7 +62,7 @@ export default function QuickAudioInput(props) {
       </div>
       <button
         className={getElement("run-model")}
-        disabled={selectedInputs.length === 0 || selectedInputs[0] === ""}
+        disabled={submitButtonIsDisabled()}
         onClick={() => runModel()}
       >
         Run model and see results

src/components/Experiment/QuickInput/QuickImageInput.js

@@ -17,6 +17,7 @@ export default function QuickImageInput(props) {
     selectTab,
     selectInput,
     runModel,
+    submitButtonIsDisabled
   } = useQuickInputControl(props);
   const { getBlock, getElement } = useBEMNaming("quick-image-input");
 
@@ -60,7 +61,7 @@ export default function QuickImageInput(props) {
       </div>
       <button
         className={getElement("run-model")}
-        disabled={selectedInputs.length === 0 || selectedInputs[0] === ""}
+        disabled={submitButtonIsDisabled()}
         onClick={() => runModel()}
       >
         Run model and see results

src/components/Experiment/QuickInput/QuickInput.js

@@ -13,15 +13,18 @@ import QuickMultiInput from "./QuickMultiInput";
 
 export default function QuickInput(props) {
   const task = Task.getStaticTask(props.model.output.type);
+
   const [URLValidity, setURLValidity] = useState(false);
   const [selectedInputSrc, setSelectedInputSrc] = useState("");
+
   const inputPreviewProps = {
     URLValidity,
     setURLValidity,
     selectedInputSrc,
     setSelectedInputSrc,
   };
   props = { ...props, inputPreviewProps };
+
   if (task.useMultiInput) {
     // TODO: At some point this should replace the switch statement below
     return <QuickMultiInput {...props} />;
@@ -35,6 +38,7 @@ export default function QuickInput(props) {
     case TaskInputTypes.Video:
       return <QuickVideoInput {...props} />;
     case TaskInputTypes.Image:
+    case TaskInputTypes.ImageCanvas:
     default:
       return <QuickImageInput {...props} />;
   }

src/components/Experiment/QuickInput/QuickInput.stories.js

@@ -23,6 +23,7 @@ import {
   audioToAudio,
   audioClassification,
   videoClassification,
+  maskGeneration,
 } from "../../../helpers/TaskIDs";
 import {
   SampleImageClassificationInputs,
@@ -42,6 +43,7 @@ import {
   SampleAudioToAudioInputs,
   SampleAudioClassificationInputs,
   SampleVideoClassificationInputs,
+  SampleMaskGenerationInputs,
 } from "../../../helpers/sampleImages";
 import { QuickInputType } from "./quickInputType";
 import { TaskInputTypes } from "../../../helpers/TaskInputTypes";
@@ -116,6 +118,16 @@ ImageTo3D.args = {
   }
 };
 
+export const MaskGeneration = Template.bind({});
+MaskGeneration.args = {
+  sampleInputs: SampleMaskGenerationInputs,
+  model: {
+    output: {
+      type: maskGeneration,
+    },
+  },
+};
+
 export const Text = Template.bind({});
 Text.args = {
   sampleInputs: [

src/components/Experiment/QuickInput/QuickMultiInput.js

@@ -17,7 +17,9 @@ export default function QuickMultiInput(props) {
     selectTab,
     selectInput,
     runModel,
+    submitButtonIsDisabled
   } = useQuickInputControl(props);
+
   const { getBlock, getElement } = useBEMNaming("quick-image-input");
 
   const task = Task.getStaticTask(props.model.output.type);
@@ -76,7 +78,7 @@ export default function QuickMultiInput(props) {
       </div>
       <button
         className={getElement("run-model")}
-        disabled={selectedInputs.length < task.inputs.length || selectedInputs[0] === ""}
+        disabled={submitButtonIsDisabled()}
         onClick={() => runModel()}
       >
         Run model and see results

src/components/Experiment/QuickInput/QuickMultiInputTabContent.js

@@ -25,6 +25,8 @@ export function QuickMultiInputTabContent(props) {
             values={props.selectedInputs} {...tab.props} 
             inputIndex={props.inputIndex}
             input={props.input}
+            inputPreviewProps={props?.inputPreviewProps}
+            {...props}
         />
     </div>
   )

src/components/Experiment/QuickInput/QuickTextInput.js

@@ -18,6 +18,7 @@ export default function QuickTextInput(props) {
         selectInput,
         runModel,
         hideUpload,
+        submitButtonIsDisabled
     } = useQuickInputControl(props);
     const {getBlock, getElement} = useBEMNaming("quick-text-input");
     const task = Task.getStaticTask(props.model.output.type);
@@ -62,7 +63,7 @@ export default function QuickTextInput(props) {
 
 
             <button
-                disabled={selectedInputs.length === 0 || selectedInputs[0] === ""}
+                disabled={submitButtonIsDisabled()}
                 onClick={() => runModel()}
                 className={getElement("submit-button")}
             >

src/components/Experiment/QuickInput/Tabs/CanvasInput/CanvasInput.js

@@ -0,0 +1,146 @@
+import {useEffect, useRef, useState} from 'react';
+import useBEMNaming from "../../../../../common/useBEMNaming";
+
+import './CanvasInput.scss';
+
+
+const loadImage = (setImageDimensions, imageUrl) => {
+    const img = new Image();
+    img.src = imageUrl;
+
+    img.onload = () => {
+      setImageDimensions({
+        height: img.height,
+        width: img.width
+      });
+    };
+    img.onerror = (err) => {
+      console.log("img error");
+      console.error(err);
+    };
+};
+
+const CanvasInput = (props) => {
+    const { getBlock, getElement } = useBEMNaming("canvas-input");
+
+    const inputIndex = props.index;
+    const imageUrl = props.url.src ?? props.url;
+
+    const imageRef = useRef(null);
+    const canvasRef = useRef(null);
+    const contextRef = useRef(null);
+
+    const [isDrawing, setIsDrawing] = useState(false);
+
+    const canvasOffSetX = useRef(null);
+    const canvasOffSetY = useRef(null);
+    const startX = useRef(null);
+    const startY = useRef(null);
+
+    const [rectangleWidth, setRectangleWidth] = useState(0);
+    const [rectangleHeight, setRectangleHeight] = useState(0);
+
+    const [imageDimensions, setImageDimensions] = useState({});  // Prob could change this to a boolean?
+
+    useEffect(() => {
+        loadImage(setImageDimensions, imageUrl); 
+      }, []
+    );
+
+    useEffect(() => {
+        // This useEffect fires once, when imageDimensions is set, then we use imageRef to get the
+        // size of the image as-displayed (rather than )
+        const image = imageRef.current;
+
+        const canvas = canvasRef.current;
+        canvas.width = image.width;
+        canvas.height = image.height;   
+
+        const context = canvas.getContext("2d");
+        context.lineCap = "round";
+        context.strokeStyle = "#5FA9FF";
+        context.lineWidth = 5;
+        contextRef.current = context;
+
+        const canvasOffSet = canvas.getBoundingClientRect();
+        canvasOffSetX.current = canvasOffSet.top;
+        canvasOffSetY.current = canvasOffSet.left;
+    }, [imageDimensions]);
+
+    const startDrawingRectangle = ({nativeEvent}) => {
+        nativeEvent.preventDefault();
+        nativeEvent.stopPropagation();
+
+        startX.current = nativeEvent.offsetX;
+        startY.current = nativeEvent.offsetY;        
+
+        setIsDrawing(true);
+    };
+
+    const drawRectangle = ({nativeEvent}) => {
+        if (!isDrawing) {
+            return;
+        }
+
+        nativeEvent.preventDefault();
+        nativeEvent.stopPropagation();
+
+        const newMouseX = nativeEvent.offsetX;
+        const newMouseY = nativeEvent.offsetY;        
+
+        const rectWidth = newMouseX - startX.current;
+        const rectHeight = newMouseY - startY.current;
+        setRectangleWidth(rectWidth)
+        setRectangleHeight(rectHeight)
+
+        contextRef.current.clearRect(0, 0, canvasRef.current.width, canvasRef.current.height);
+        contextRef.current.strokeRect(startX.current, startY.current, rectWidth, rectHeight);
+    };
+
+    const stopDrawingRectangle = () => {
+        setIsDrawing(false);
+
+        const dimensions = {
+            xmin: startX.current,
+            xmax: rectangleWidth,
+            ymin: startY.current,
+            ymax: rectangleHeight
+        };
+
+        props.selectInput(imageUrl, inputIndex, dimensions);
+    };
+
+    return (
+        <div className={getBlock()}>
+            {
+                props.tab.id !== "sample-input" && (
+                    <p className={getElement("help-text")}>
+                        Click and drag to draw a rectangle around the object you wish to identify.
+                    </p>
+                )
+            }
+
+            <div className={getElement("canvas-container")}>
+                <img 
+                    className={getElement("canvas-background")}
+                    src={imageUrl}
+                    alt="canvas background" 
+                    ref={imageRef}
+                />
+                <canvas className={getElement("canvas-element")}
+                    ref={canvasRef}
+                    onMouseDown={startDrawingRectangle}
+                    onMouseMove={drawRectangle}
+                    onMouseUp={stopDrawingRectangle}
+                    onMouseLeave={stopDrawingRectangle} 
+                />
+            </div>
+        </div>
+
+    )
+}
+
+export default CanvasInput;
+
+// Note: Based off of this example: 
+// https://coolboi567.medium.com/dynamically-get-image-dimensions-from-image-url-in-react-d7e216887b68

src/components/Experiment/QuickInput/Tabs/CanvasInput/CanvasInput.scss

@@ -0,0 +1,30 @@
+@import "src/App";
+
+.canvas-input {
+    &__help-text {
+        font-size: 18px;
+        font-weight: normal;
+        line-height: 24px;
+        color: $charcoalLightest;
+        margin-top: 20px;
+        margin-bottom: 20px;        
+    }
+
+    &__canvas-container {
+        position: relative;
+        top: 0;
+        left: 0;
+    }
+
+    &__canvas-background {
+        position: relative;
+        top: 0;
+        left: 0;
+    }
+
+    &__canvas-element {
+        position: absolute;
+        top: 0px;
+        left: 0px;  
+    }
+}

src/components/Experiment/QuickInput/Tabs/SampleInput/SampleInputsTab.js

@@ -8,12 +8,14 @@ import { ReactComponent as DocumentIcon } from "../../../../../resources/icons/i
 import { imageTo3D } from '../../../../../helpers/TaskIDs';
 import URLInputPreview from '../URLInput/URLInputPreview';
 import { TaskInputTypes } from '../../../../../helpers/TaskInputTypes';
+import CanvasInput from '../CanvasInput/CanvasInput';
 
 
 export default function SampleInputsTab(props) {
     // Note: This is the content for the Sample Input Tab, below the header
     const { getBlock, getElement } = useBEMNaming("sample-inputs");
-    const { isUnselected, isSelected, selectInput, type, sampleInputType } = useSampleInputControl(props);
+    const { isUnselected, isSelected, selectInput, type,  sampleInputType } = useSampleInputControl(props);
+
     const task = Task.getStaticTask(props.task);
 
     const getInputClassName = (url) => {
@@ -44,6 +46,8 @@ export default function SampleInputsTab(props) {
                 return makeSampleDocumentInput(url, index);
             case QuickInputType.Video:
                 return makeSampleVideoInput(url, index);
+            case QuickInputType.ImageCanvas:
+                return makeSampleImageCanvasInput(url, index);
             default:
                 return makeDefaultErrorInput();
         }
@@ -91,11 +95,18 @@ export default function SampleInputsTab(props) {
         );
     }
 
+    function makeSampleImageCanvasInput(url, index) {
+        return (
+            <div key={index} className={getElement(getInputClassName(url))}>
+                <CanvasInput selectInput={selectInput} index={index} url={url} {...props} />
+            </div>
+        );
+    }    
+
     function makeSampleVideoInput(url, index) {
         return (
             <button onClick={() => onSampleInputClickPreview(index, url)} key={index} className={getElement(getInputClassName(url))}>
                 <video src={url.src} alt={url.alt} autoPlay muted={true} loop className={getElement("sample-video-content")} />
-
             </button>
         );
     }
@@ -134,6 +145,8 @@ export default function SampleInputsTab(props) {
                 return "Select a document";
             case QuickInputType.Video:
                 return "Select a video";
+            case QuickInputType.ImageCanvas:
+                return "Draw a rectangle";
             default:
                 return "Error: no input type set";
         }