diff --git a/.gitattributes b/.gitattributes index f41e91d85c22..7c0b591b072c 100644 --- a/.gitattributes +++ b/.gitattributes @@ -15,7 +15,6 @@ LICENSE text *.conf text *.mimetypes text *.sh text eol=lf -components/openvino/eula.cfg text eol=lf *.avi binary *.bmp binary diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 4b7ae8b1ded5..32af66a215ee 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -23,7 +23,7 @@ /datumaro/ @zhiltsov-max /cvat/apps/dataset_manager/ @zhiltsov-max -# Advanced components (e.g. OpenVINO) +# Advanced components (e.g. analytics) /components/ @azhavoro # Infrastructure diff --git a/.gitignore b/.gitignore index c25a121364a1..cbe91f69a9c3 100644 --- a/.gitignore +++ b/.gitignore @@ -7,7 +7,6 @@ /.env /keys /logs -/components/openvino/*.tgz /profiles /ssh/* !/ssh/README.md diff --git a/.pylintrc b/.pylintrc index effa67355168..677a395622ab 100644 --- a/.pylintrc +++ b/.pylintrc @@ -20,7 +20,7 @@ persistent=yes # List of plugins (as comma separated values of python modules names) to load, # usually to register additional checkers. -load-plugins= +load-plugins=pylint_django # Use multiple processes to speed up Pylint. jobs=1 @@ -66,8 +66,8 @@ enable= E0001,E0100,E0101,E0102,E0103,E0104,E0105,E0106,E0107,E0110, W0122,W0124,W0150,W0199,W0221,W0222,W0233,W0404,W0410,W0601, W0602,W0604,W0611,W0612,W0622,W0623,W0702,W0705,W0711,W1300, W1301,W1302,W1303,,W1305,W1306,W1307 - R0102,R0201,R0202,R0203 - + R0102,R0202,R0203 + # Disable the message, report, category or checker with the given id(s). You # can either give multiple identifiers separated by comma (,) or put this diff --git a/.vscode/launch.json b/.vscode/launch.json index 1fefab7fee5b..48bc45835ce0 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -27,7 +27,7 @@ "request": "launch", "stopOnEntry": false, "justMyCode": false, - "pythonPath": "${config:python.pythonPath}", + "pythonPath": "${command:python.interpreterPath}", "program": "${workspaceRoot}/manage.py", "args": [ "runserver", @@ -58,7 +58,7 @@ "request": "launch", "stopOnEntry": false, "justMyCode": false, - "pythonPath": "${config:python.pythonPath}", + "pythonPath": "${command:python.interpreterPath}", "program": "${workspaceRoot}/manage.py", "args": [ "rqworker", @@ -77,7 +77,7 @@ "request": "launch", "stopOnEntry": false, "justMyCode": false, - "pythonPath": "${config:python.pythonPath}", + "pythonPath": "${command:python.interpreterPath}", "program": "${workspaceRoot}/manage.py", "args": [ "rqscheduler", @@ -93,7 +93,7 @@ "request": "launch", "justMyCode": false, "stopOnEntry": false, - "pythonPath": "${config:python.pythonPath}", + "pythonPath":"${command:python.interpreterPath}", "program": "${workspaceRoot}/manage.py", "args": [ "rqworker", @@ -112,7 +112,7 @@ "request": "launch", "justMyCode": false, "stopOnEntry": false, - "pythonPath": "${config:python.pythonPath}", + "pythonPath": "${command:python.interpreterPath}", "program": "${workspaceRoot}/manage.py", "args": [ "update_git_states" @@ -128,7 +128,7 @@ "request": "launch", "justMyCode": false, "stopOnEntry": false, - "pythonPath": "${config:python.pythonPath}", + "pythonPath": "${command:python.interpreterPath}", "program": "${workspaceRoot}/manage.py", "args": [ "migrate" @@ -144,7 +144,7 @@ "request": "launch", "justMyCode": false, "stopOnEntry": false, - "pythonPath": "${config:python.pythonPath}", + "pythonPath": "${command:python.interpreterPath}", "program": "${workspaceRoot}/manage.py", "args": [ "test", diff --git a/CHANGELOG.md b/CHANGELOG.md index b297e626e5ec..768d146aec6a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [1.1.0-beta] - Unreleased ### Added +- DL models as serverless functions () - Source type support for tags, shapes and tracks () - Source type support for CVAT Dumper/Loader () - Intelligent polygon editing () @@ -13,13 +14,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed - Smaller object details () +- It is impossible to submit a DL model in OpenVINO format using UI. Now you can deploy new models on the server using serverless functions () - Files and folders under share path are now alphabetically sorted ### Deprecated - ### Removed -- +- Removed OpenVINO and CUDA components because they are not necessary anymore () ### Fixed - Some objects aren't shown on canvas sometimes. For example after propagation on of objects is invisible () @@ -81,12 +83,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Colorized object items in the side panel () - [Datumaro] Annotation-less files are not generated anymore in COCO format, unless tasks explicitly requested () -### Deprecated -- - -### Removed -- - ### Fixed - Problem with exported frame stepped image task () - Fixed dataset filter item representation for imageless dataset items () diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1c7c7ce5d6ef..6fa845c86ace 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -101,71 +101,134 @@ You develop CVAT under WSL (Windows subsystem for Linux) following next steps. - Run all commands from this isntallation guide in WSL Ubuntu shell. ## Setup additional components in development environment -### Automatic annotation -- Install OpenVINO on your host machine according to instructions from -[OpenVINO website](https://docs.openvinotoolkit.org/latest/index.html) - -- Add some environment variables (copy code below to the end of ``.env/bin/activate`` file): -```sh - source /opt/intel/openvino/bin/setupvars.sh +### DL models as serverless functions + +Install [nuclio platform](https://github.com/nuclio/nuclio): +- You have to install `nuctl` command line tool to build and deploy serverless +functions. Download [the latest release]( +https://github.com/nuclio/nuclio/blob/development/docs/reference/nuctl/nuctl.md#download). +- The simplest way to explore Nuclio is to run its graphical user interface (GUI) +of the Nuclio dashboard. All you need in order to run the dashboard is Docker. See +[nuclio documentation](https://github.com/nuclio/nuclio#quick-start-steps) +for more details. +- Create `cvat` project inside nuclio dashboard where you will deploy new +serverless functions and deploy a couple of DL models. + +```bash +nuctl create project cvat +``` - export OPENVINO_TOOLKIT="yes" - export IE_PLUGINS_PATH="/opt/intel/openvino/deployment_tools/inference_engine/lib/intel64" - export OpenCV_DIR="/usr/local/lib/cmake/opencv4" - export LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/opt/intel/openvino/inference_engine/lib/intel64" +```bash +nuctl deploy --project-name cvat \ + --path serverless/openvino/dextr/nuclio \ + --volume `pwd`/serverless/openvino/common:/opt/nuclio/common ``` -Notice 1: be sure that these paths actually exist. Some of them can differ in different OpenVINO versions. +
+ +```bash +20.07.17 12:02:23.247 nuctl (I) Deploying function {"name": ""} +20.07.17 12:02:23.248 nuctl (I) Building {"versionInfo": "Label: 1.4.8, Git commit: 238d4539ac7783896d6c414535d0462b5f4cbcf1, OS: darwin, Arch: amd64, Go version: go1.14.3", "name": ""} +20.07.17 12:02:23.447 nuctl (I) Cleaning up before deployment +20.07.17 12:02:23.535 nuctl (I) Function already exists, deleting +20.07.17 12:02:25.877 nuctl (I) Staging files and preparing base images +20.07.17 12:02:25.891 nuctl (I) Building processor image {"imageName": "cvat/openvino.dextr:latest"} +20.07.17 12:02:25.891 nuctl.platform.docker (I) Pulling image {"imageName": "quay.io/nuclio/handler-builder-python-onbuild:1.4.8-amd64"} +20.07.17 12:02:29.270 nuctl.platform.docker (I) Pulling image {"imageName": "quay.io/nuclio/uhttpc:0.0.1-amd64"} +20.07.17 12:02:33.208 nuctl.platform (I) Building docker image {"image": "cvat/openvino.dextr:latest"} +20.07.17 12:02:34.464 nuctl.platform (I) Pushing docker image into registry {"image": "cvat/openvino.dextr:latest", "registry": ""} +20.07.17 12:02:34.464 nuctl.platform (I) Docker image was successfully built and pushed into docker registry {"image": "cvat/openvino.dextr:latest"} +20.07.17 12:02:34.464 nuctl (I) Build complete {"result": {"Image":"cvat/openvino.dextr:latest","UpdatedFunctionConfig":{"metadata":{"name":"openvino.dextr","namespace":"nuclio","labels":{"nuclio.io/project-name":"cvat"},"annotations":{"framework":"openvino","spec":"","type":"interactor"}},"spec":{"description":"Deep Extreme Cut","handler":"main:handler","runtime":"python:3.6","env":[{"name":"NUCLIO_PYTHON_EXE_PATH","value":"/opt/nuclio/python3"}],"resources":{},"image":"cvat/openvino.dextr:latest","targetCPU":75,"triggers":{"myHttpTrigger":{"class":"","kind":"http","name":"","maxWorkers":2,"workerAvailabilityTimeoutMilliseconds":10000,"attributes":{"maxRequestBodySize":33554432}}},"volumes":[{"volume":{"name":"volume-1","hostPath":{"path":"/Users/nmanovic/Workspace/cvat/serverless/openvino/common"}},"volumeMount":{"name":"volume-1","mountPath":"/opt/nuclio/common"}}],"build":{"image":"cvat/openvino.dextr","baseImage":"openvino/ubuntu18_runtime:2020.2","directives":{"postCopy":[{"kind":"RUN","value":"curl -O https://download.01.org/openvinotoolkit/models_contrib/cvat/dextr_model_v1.zip"},{"kind":"RUN","value":"unzip dextr_model_v1.zip"},{"kind":"RUN","value":"pip3 install Pillow"},{"kind":"USER","value":"openvino"}],"preCopy":[{"kind":"USER","value":"root"},{"kind":"WORKDIR","value":"/opt/nuclio"},{"kind":"RUN","value":"ln -s /usr/bin/pip3 /usr/bin/pip"}]},"codeEntryType":"image"},"platform":{},"readinessTimeoutSeconds":60,"eventTimeout":"30s"}}}} +20.07.17 12:02:35.012 nuctl.platform (I) Waiting for function to be ready {"timeout": 60} +20.07.17 12:02:37.448 nuctl (I) Function deploy complete {"httpPort": 55274} +``` -Notice 2: you need to deactivate, activate again and restart vs code -to changes in ``.env/bin/activate`` file are active. +
-### ReID algorithm -- Perform all steps in the automatic annotation section -- Download ReID model and save it somewhere: -```sh - curl https://download.01.org/openvinotoolkit/2018_R5/open_model_zoo/person-reidentification-retail-0079/FP32/person-reidentification-retail-0079.xml -o reid.xml - curl https://download.01.org/openvinotoolkit/2018_R5/open_model_zoo/person-reidentification-retail-0079/FP32/person-reidentification-retail-0079.bin -o reid.bin -``` -- Add next line to ``.env/bin/activate``: -```sh - export REID_MODEL_DIR="/path/to/dir" # dir must contain .xml and .bin files +```bash +nuctl deploy --project-name cvat \ + --path serverless/openvino/omz/public/yolo-v3-tf/nuclio \ + --volume `pwd`/serverless/openvino/common:/opt/nuclio/common ``` -### Deep Extreme Cut -- Perform all steps in the automatic annotation section -- Download Deep Extreme Cut model, unpack it, and save somewhere: -```sh -curl https://download.01.org/openvinotoolkit/models_contrib/cvat/dextr_model_v1.zip -o dextr.zip -unzip dextr.zip -``` -- Add next lines to ``.env/bin/activate``: -```sh - export WITH_DEXTR="yes" - export DEXTR_MODEL_DIR="/path/to/dir" # dir must contain .xml and .bin files +
+ +```bash +20.07.17 12:05:23.377 nuctl (I) Deploying function {"name": ""} +20.07.17 12:05:23.378 nuctl (I) Building {"versionInfo": "Label: 1.4.8, Git commit: 238d4539ac7783896d6c414535d0462b5f4cbcf1, OS: darwin, Arch: amd64, Go version: go1.14.3", "name": ""} +20.07.17 12:05:23.590 nuctl (I) Cleaning up before deployment +20.07.17 12:05:23.694 nuctl (I) Function already exists, deleting +20.07.17 12:05:24.271 nuctl (I) Staging files and preparing base images +20.07.17 12:05:24.274 nuctl (I) Building processor image {"imageName": "cvat/openvino.omz.public.yolo-v3-tf:latest"} +20.07.17 12:05:24.274 nuctl.platform.docker (I) Pulling image {"imageName": "quay.io/nuclio/handler-builder-python-onbuild:1.4.8-amd64"} +20.07.17 12:05:27.432 nuctl.platform.docker (I) Pulling image {"imageName": "quay.io/nuclio/uhttpc:0.0.1-amd64"} +20.07.17 12:05:31.462 nuctl.platform (I) Building docker image {"image": "cvat/openvino.omz.public.yolo-v3-tf:latest"} +20.07.17 12:05:32.798 nuctl.platform (I) Pushing docker image into registry {"image": "cvat/openvino.omz.public.yolo-v3-tf:latest", "registry": ""} +20.07.17 12:05:32.798 nuctl.platform (I) Docker image was successfully built and pushed into docker registry {"image": "cvat/openvino.omz.public.yolo-v3-tf:latest"} +20.07.17 12:05:32.798 nuctl (I) Build complete {"result": {"Image":"cvat/openvino.omz.public.yolo-v3-tf:latest","UpdatedFunctionConfig":{"metadata":{"name":"openvino.omz.public.yolo-v3-tf","namespace":"nuclio","labels":{"nuclio.io/project-name":"cvat"},"annotations":{"framework":"openvino","name":"YOLO v3","spec":"[\n { \"id\": 0, \"name\": \"person\" },\n { \"id\": 1, \"name\": \"bicycle\" },\n { \"id\": 2, \"name\": \"car\" },\n { \"id\": 3, \"name\": \"motorbike\" },\n { \"id\": 4, \"name\": \"aeroplane\" },\n { \"id\": 5, \"name\": \"bus\" },\n { \"id\": 6, \"name\": \"train\" },\n { \"id\": 7, \"name\": \"truck\" },\n { \"id\": 8, \"name\": \"boat\" },\n { \"id\": 9, \"name\": \"traffic light\" },\n { \"id\": 10, \"name\": \"fire hydrant\" },\n { \"id\": 11, \"name\": \"stop sign\" },\n { \"id\": 12, \"name\": \"parking meter\" },\n { \"id\": 13, \"name\": \"bench\" },\n { \"id\": 14, \"name\": \"bird\" },\n { \"id\": 15, \"name\": \"cat\" },\n { \"id\": 16, \"name\": \"dog\" },\n { \"id\": 17, \"name\": \"horse\" },\n { \"id\": 18, \"name\": \"sheep\" },\n { \"id\": 19, \"name\": \"cow\" },\n { \"id\": 20, \"name\": \"elephant\" },\n { \"id\": 21, \"name\": \"bear\" },\n { \"id\": 22, \"name\": \"zebra\" },\n { \"id\": 23, \"name\": \"giraffe\" },\n { \"id\": 24, \"name\": \"backpack\" },\n { \"id\": 25, \"name\": \"umbrella\" },\n { \"id\": 26, \"name\": \"handbag\" },\n { \"id\": 27, \"name\": \"tie\" },\n { \"id\": 28, \"name\": \"suitcase\" },\n { \"id\": 29, \"name\": \"frisbee\" },\n { \"id\": 30, \"name\": \"skis\" },\n { \"id\": 31, \"name\": \"snowboard\" },\n { \"id\": 32, \"name\": \"sports ball\" },\n { \"id\": 33, \"name\": \"kite\" },\n { \"id\": 34, \"name\": \"baseball bat\" },\n { \"id\": 35, \"name\": \"baseball glove\" },\n { \"id\": 36, \"name\": \"skateboard\" },\n { \"id\": 37, \"name\": \"surfboard\" },\n { \"id\": 38, \"name\": \"tennis racket\" },\n { \"id\": 39, \"name\": \"bottle\" },\n { \"id\": 40, \"name\": \"wine glass\" },\n { \"id\": 41, \"name\": \"cup\" },\n { \"id\": 42, \"name\": \"fork\" },\n { \"id\": 43, \"name\": \"knife\" },\n { \"id\": 44, \"name\": \"spoon\" },\n { \"id\": 45, \"name\": \"bowl\" },\n { \"id\": 46, \"name\": \"banana\" },\n { \"id\": 47, \"name\": \"apple\" },\n { \"id\": 48, \"name\": \"sandwich\" },\n { \"id\": 49, \"name\": \"orange\" },\n { \"id\": 50, \"name\": \"broccoli\" },\n { \"id\": 51, \"name\": \"carrot\" },\n { \"id\": 52, \"name\": \"hot dog\" },\n { \"id\": 53, \"name\": \"pizza\" },\n { \"id\": 54, \"name\": \"donut\" },\n { \"id\": 55, \"name\": \"cake\" },\n { \"id\": 56, \"name\": \"chair\" },\n { \"id\": 57, \"name\": \"sofa\" },\n { \"id\": 58, \"name\": \"pottedplant\" },\n { \"id\": 59, \"name\": \"bed\" },\n { \"id\": 60, \"name\": \"diningtable\" },\n { \"id\": 61, \"name\": \"toilet\" },\n { \"id\": 62, \"name\": \"tvmonitor\" },\n { \"id\": 63, \"name\": \"laptop\" },\n { \"id\": 64, \"name\": \"mouse\" },\n { \"id\": 65, \"name\": \"remote\" },\n { \"id\": 66, \"name\": \"keyboard\" },\n { \"id\": 67, \"name\": \"cell phone\" },\n { \"id\": 68, \"name\": \"microwave\" },\n { \"id\": 69, \"name\": \"oven\" },\n { \"id\": 70, \"name\": \"toaster\" },\n { \"id\": 71, \"name\": \"sink\" },\n { \"id\": 72, \"name\": \"refrigerator\" },\n { \"id\": 73, \"name\": \"book\" },\n { \"id\": 74, \"name\": \"clock\" },\n { \"id\": 75, \"name\": \"vase\" },\n { \"id\": 76, \"name\": \"scissors\" },\n { \"id\": 77, \"name\": \"teddy bear\" },\n { \"id\": 78, \"name\": \"hair drier\" },\n { \"id\": 79, \"name\": \"toothbrush\" }\n]\n","type":"detector"}},"spec":{"description":"YOLO v3 via Intel OpenVINO","handler":"main:handler","runtime":"python:3.6","env":[{"name":"NUCLIO_PYTHON_EXE_PATH","value":"/opt/nuclio/common/python3"}],"resources":{},"image":"cvat/openvino.omz.public.yolo-v3-tf:latest","targetCPU":75,"triggers":{"myHttpTrigger":{"class":"","kind":"http","name":"","maxWorkers":2,"workerAvailabilityTimeoutMilliseconds":10000,"attributes":{"maxRequestBodySize":33554432}}},"volumes":[{"volume":{"name":"volume-1","hostPath":{"path":"/Users/nmanovic/Workspace/cvat/serverless/openvino/common"}},"volumeMount":{"name":"volume-1","mountPath":"/opt/nuclio/common"}}],"build":{"image":"cvat/openvino.omz.public.yolo-v3-tf","baseImage":"openvino/ubuntu18_dev:2020.2","directives":{"postCopy":[{"kind":"USER","value":"openvino"}],"preCopy":[{"kind":"USER","value":"root"},{"kind":"WORKDIR","value":"/opt/nuclio"},{"kind":"RUN","value":"ln -s /usr/bin/pip3 /usr/bin/pip"},{"kind":"RUN","value":"/opt/intel/openvino/deployment_tools/open_model_zoo/tools/downloader/downloader.py --name yolo-v3-tf -o /opt/nuclio/open_model_zoo"},{"kind":"RUN","value":"/opt/intel/openvino/deployment_tools/open_model_zoo/tools/downloader/converter.py --name yolo-v3-tf --precisions FP32 -d /opt/nuclio/open_model_zoo -o /opt/nuclio/open_model_zoo"}]},"codeEntryType":"image"},"platform":{},"readinessTimeoutSeconds":60,"eventTimeout":"30s"}}}} +20.07.17 12:05:33.285 nuctl.platform (I) Waiting for function to be ready {"timeout": 60} +20.07.17 12:05:35.452 nuctl (I) Function deploy complete {"httpPort": 57308} ``` -### Tensorflow RCNN -- Download RCNN model, unpack it, and save it somewhere: -```sh -curl http://download.tensorflow.org/models/object_detection/faster_rcnn_inception_resnet_v2_atrous_coco_2018_01_28.tar.gz -o model.tar.gz && \ -tar -xzf model.tar.gz +
+ +- Display a list of running serverless functions using `nuctl` command or see them +in nuclio dashboard: + +```bash +nuctl get function ``` -- Add next lines to ``.env/bin/activate``: -```sh - export TF_ANNOTATION="yes" - export TF_ANNOTATION_MODEL_PATH="/path/to/the/model/graph" # truncate .pb extension + +
+ +```bash + NAMESPACE | NAME | PROJECT | STATE | NODE PORT | REPLICAS + nuclio | openvino.dextr | cvat | ready | 55274 | 1/1 + nuclio | openvino.omz.public.yolo-v3-tf | cvat | ready | 57308 | 1/1 ``` -### Tensorflow Mask RCNN -- Download Mask RCNN model, and save it somewhere: -```sh -curl https://github.com/matterport/Mask_RCNN/releases/download/v2.0/mask_rcnn_coco.h5 -o mask_rcnn_coco.h5 +
+ +- Test your deployed DL model as a serverless function. The command below +should work on Linux and Mac OS. + +```bash +image=$(curl https://upload.wikimedia.org/wikipedia/en/7/7d/Lenna_%28test_image%29.png --output - | base64 | tr -d '\n') +cat << EOF > /tmp/input.json +{"image": "$image"} +EOF +cat /tmp/input.json | nuctl invoke openvino.omz.public.yolo-v3-tf -c 'application/json' ``` -- Add next lines to ``.env/bin/activate``: -```sh - export AUTO_SEGMENTATION="yes" - export AUTO_SEGMENTATION_PATH="/path/to/dir" # dir must contain mask_rcnn_coco.h5 file + +
+ +```bash +20.07.17 12:07:44.519 nuctl.platform.invoker (I) Executing function {"method": "POST", "url": "http://:57308", "headers": {"Content-Type":["application/json"],"X-Nuclio-Log-Level":["info"],"X-Nuclio-Target":["openvino.omz.public.yolo-v3-tf"]}} +20.07.17 12:07:45.275 nuctl.platform.invoker (I) Got response {"status": "200 OK"} +20.07.17 12:07:45.275 nuctl (I) >>> Start of function logs +20.07.17 12:07:45.275 ino.omz.public.yolo-v3-tf (I) Run yolo-v3-tf model {"worker_id": "0", "time": 1594976864570.9353} +20.07.17 12:07:45.275 nuctl (I) <<< End of function logs + +> Response headers: +Date = Fri, 17 Jul 2020 09:07:45 GMT +Content-Type = application/json +Content-Length = 100 +Server = nuclio + +> Response body: +[ + { + "confidence": "0.9992254", + "label": "person", + "points": [ + 39, + 124, + 408, + 512 + ], + "type": "rectangle" + } +] ``` ### Run Cypress tests diff --git a/Dockerfile b/Dockerfile index c1c48502ee38..c70baef85344 100644 --- a/Dockerfile +++ b/Dockerfile @@ -76,33 +76,6 @@ RUN adduser --shell /bin/bash --disabled-password --gecos "" ${USER} && \ COPY components /tmp/components -# OpenVINO toolkit support -ARG OPENVINO_TOOLKIT -ENV OPENVINO_TOOLKIT=${OPENVINO_TOOLKIT} -ENV REID_MODEL_DIR=${HOME}/reid -RUN if [ "$OPENVINO_TOOLKIT" = "yes" ]; then \ - /tmp/components/openvino/install.sh && \ - mkdir ${REID_MODEL_DIR} && \ - curl https://download.01.org/openvinotoolkit/2018_R5/open_model_zoo/person-reidentification-retail-0079/FP32/person-reidentification-retail-0079.xml -o reid/reid.xml && \ - curl https://download.01.org/openvinotoolkit/2018_R5/open_model_zoo/person-reidentification-retail-0079/FP32/person-reidentification-retail-0079.bin -o reid/reid.bin; \ - fi - -# Tensorflow annotation support -ARG TF_ANNOTATION -ENV TF_ANNOTATION=${TF_ANNOTATION} -ENV TF_ANNOTATION_MODEL_PATH=${HOME}/rcnn/inference_graph -RUN if [ "$TF_ANNOTATION" = "yes" ]; then \ - bash -i /tmp/components/tf_annotation/install.sh; \ - fi - -# Auto segmentation support. by Mohammad -ARG AUTO_SEGMENTATION -ENV AUTO_SEGMENTATION=${AUTO_SEGMENTATION} -ENV AUTO_SEGMENTATION_PATH=${HOME}/Mask_RCNN -RUN if [ "$AUTO_SEGMENTATION" = "yes" ]; then \ - bash -i /tmp/components/auto_segmentation/install.sh; \ - fi - # Install and initialize CVAT, copy all necessary files COPY cvat/requirements/ /tmp/requirements/ COPY supervisord.conf mod_wsgi.conf wait-for-it.sh manage.py ${HOME}/ @@ -110,24 +83,6 @@ RUN python3 -m pip install --no-cache-dir -r /tmp/requirements/${DJANGO_CONFIGUR # pycocotools package is impossible to install with its dependencies by one pip install command RUN python3 -m pip install --no-cache-dir pycocotools==2.0.0 - -# CUDA support -ARG CUDA_SUPPORT -ENV CUDA_SUPPORT=${CUDA_SUPPORT} -RUN if [ "$CUDA_SUPPORT" = "yes" ]; then \ - /tmp/components/cuda/install.sh; \ - fi - -# TODO: CHANGE URL -ARG WITH_DEXTR -ENV WITH_DEXTR=${WITH_DEXTR} -ENV DEXTR_MODEL_DIR=${HOME}/dextr -RUN if [ "$WITH_DEXTR" = "yes" ]; then \ - mkdir ${DEXTR_MODEL_DIR} -p && \ - curl https://download.01.org/openvinotoolkit/models_contrib/cvat/dextr_model_v1.zip -o ${DEXTR_MODEL_DIR}/dextr.zip && \ - 7z e ${DEXTR_MODEL_DIR}/dextr.zip -o${DEXTR_MODEL_DIR} && rm ${DEXTR_MODEL_DIR}/dextr.zip; \ - fi - ARG CLAM_AV ENV CLAM_AV=${CLAM_AV} RUN if [ "$CLAM_AV" = "yes" ]; then \ diff --git a/Dockerfile.ci b/Dockerfile.ci index b4c24738d5e9..d3d068741d76 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -1,4 +1,4 @@ -FROM cvat +FROM cvat/server ENV DJANGO_CONFIGURATION=testing USER root diff --git a/README.md b/README.md index bf1186b60ca5..fb77e0cae8fd 100644 --- a/README.md +++ b/README.md @@ -71,7 +71,6 @@ are visible to users. Disabled features: - [Analytics: management and monitoring of data annotation team](/components/analytics/README.md) -- [Support for NVIDIA GPUs](/components/cuda/README.md) Limitations: - No more than 10 tasks per user diff --git a/components/auto_segmentation/README.md b/components/auto_segmentation/README.md deleted file mode 100644 index b456857ec4c2..000000000000 --- a/components/auto_segmentation/README.md +++ /dev/null @@ -1,38 +0,0 @@ -## [Keras+Tensorflow Mask R-CNN Segmentation](https://github.com/matterport/Mask_RCNN) - -### What is it? -- This application allows you automatically to segment many various objects on images. -- It's based on Feature Pyramid Network (FPN) and a ResNet101 backbone. - -- It uses a pre-trained model on MS COCO dataset -- It supports next classes (use them in "labels" row): -```python -'BG', 'person', 'bicycle', 'car', 'motorcycle', 'airplane', -'bus', 'train', 'truck', 'boat', 'traffic light', -'fire hydrant', 'stop sign', 'parking meter', 'bench', 'bird', -'cat', 'dog', 'horse', 'sheep', 'cow', 'elephant', 'bear', -'zebra', 'giraffe', 'backpack', 'umbrella', 'handbag', 'tie', -'suitcase', 'frisbee', 'skis', 'snowboard', 'sports ball', -'kite', 'baseball bat', 'baseball glove', 'skateboard', -'surfboard', 'tennis racket', 'bottle', 'wine glass', 'cup', -'fork', 'knife', 'spoon', 'bowl', 'banana', 'apple', -'sandwich', 'orange', 'broccoli', 'carrot', 'hot dog', 'pizza', -'donut', 'cake', 'chair', 'couch', 'potted plant', 'bed', -'dining table', 'toilet', 'tv', 'laptop', 'mouse', 'remote', -'keyboard', 'cell phone', 'microwave', 'oven', 'toaster', -'sink', 'refrigerator', 'book', 'clock', 'vase', 'scissors', -'teddy bear', 'hair drier', 'toothbrush'. -``` -- Component adds "Run Auto Segmentation" button into dashboard. - -### Build docker image -```bash -# From project root directory -docker-compose -f docker-compose.yml -f components/auto_segmentation/docker-compose.auto_segmentation.yml build -``` - -### Run docker container -```bash -# From project root directory -docker-compose -f docker-compose.yml -f components/auto_segmentation/docker-compose.auto_segmentation.yml up -d -``` diff --git a/components/auto_segmentation/docker-compose.auto_segmentation.yml b/components/auto_segmentation/docker-compose.auto_segmentation.yml deleted file mode 100644 index 1d9763cdf6cd..000000000000 --- a/components/auto_segmentation/docker-compose.auto_segmentation.yml +++ /dev/null @@ -1,13 +0,0 @@ -# -# Copyright (C) 2018 Intel Corporation -# -# SPDX-License-Identifier: MIT -# -version: "2.3" - -services: - cvat: - build: - context: . - args: - AUTO_SEGMENTATION: "yes" diff --git a/components/auto_segmentation/install.sh b/components/auto_segmentation/install.sh deleted file mode 100755 index d6881a68150d..000000000000 --- a/components/auto_segmentation/install.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash -# - -set -e - -MASK_RCNN_URL=https://github.com/matterport/Mask_RCNN - -cd ${HOME} && \ -git clone ${MASK_RCNN_URL}.git && \ -curl -L ${MASK_RCNN_URL}/releases/download/v2.0/mask_rcnn_coco.h5 -o Mask_RCNN/mask_rcnn_coco.h5 - -# TODO remove useless files -# tensorflow and Keras are installed globally diff --git a/components/cuda/README.md b/components/cuda/README.md deleted file mode 100644 index a6ecbfefba1b..000000000000 --- a/components/cuda/README.md +++ /dev/null @@ -1,41 +0,0 @@ -## [NVIDIA CUDA Toolkit](https://developer.nvidia.com/cuda-toolkit) - -### Requirements - -* NVIDIA GPU with a compute capability [3.0 - 7.2] -* Latest GPU driver - -### Installation - -#### Install the latest driver for your graphics card - -```bash -sudo add-apt-repository ppa:graphics-drivers/ppa -sudo apt-get update -sudo apt-cache search nvidia-* # find latest nvidia driver -sudo apt-get --no-install-recommends install nvidia-* # install the nvidia driver -sudo apt-get --no-install-recommends install mesa-common-dev -sudo apt-get --no-install-recommends install freeglut3-dev -sudo apt-get --no-install-recommends install nvidia-modprobe -``` - -#### Reboot your PC and verify installation by `nvidia-smi` command. - -#### Install [Nvidia-Docker](https://github.com/NVIDIA/nvidia-docker) - -Please be sure that installation was successful. -```bash -docker info | grep 'Runtimes' # output should contains 'nvidia' -``` - -### Build docker image -```bash -# From project root directory -docker-compose -f docker-compose.yml -f components/cuda/docker-compose.cuda.yml build -``` - -### Run docker container -```bash -# From project root directory -docker-compose -f docker-compose.yml -f components/cuda/docker-compose.cuda.yml up -d -``` diff --git a/components/cuda/docker-compose.cuda.yml b/components/cuda/docker-compose.cuda.yml deleted file mode 100644 index 41d325f3f2bf..000000000000 --- a/components/cuda/docker-compose.cuda.yml +++ /dev/null @@ -1,23 +0,0 @@ -# -# Copyright (C) 2018 Intel Corporation -# -# SPDX-License-Identifier: MIT -# -version: "2.3" - -services: - cvat: - build: - context: . - args: - CUDA_SUPPORT: "yes" - runtime: "nvidia" - environment: - NVIDIA_VISIBLE_DEVICES: all - NVIDIA_DRIVER_CAPABILITIES: compute,utility - # That environment variable is used by the Nvidia Container Runtime. - # The Nvidia Container Runtime parses this as: - # :space:: logical OR - # ,: Logical AND - # https://gitlab.com/nvidia/container-images/cuda/issues/31#note_149432780 - NVIDIA_REQUIRE_CUDA: "cuda>=10.0 brand=tesla,driver>=384,driver<385 brand=tesla,driver>=410,driver<411" diff --git a/components/cuda/install.sh b/components/cuda/install.sh deleted file mode 100755 index 58f99acff509..000000000000 --- a/components/cuda/install.sh +++ /dev/null @@ -1,38 +0,0 @@ -#!/bin/bash -# -# Copyright (C) 2018 Intel Corporation -# -# SPDX-License-Identifier: MIT -# -set -e - -NVIDIA_GPGKEY_SUM=d1be581509378368edeec8c1eb2958702feedf3bc3d17011adbf24efacce4ab5 && \ -NVIDIA_GPGKEY_FPR=ae09fe4bbd223a84b2ccfce3f60f4b3d7fa2af80 && \ -apt-key adv --fetch-keys http://developer.download.nvidia.com/compute/cuda/repos/ubuntu1604/x86_64/7fa2af80.pub && \ -apt-key adv --export --no-emit-version -a $NVIDIA_GPGKEY_FPR | tail -n +5 > cudasign.pub && \ -echo "$NVIDIA_GPGKEY_SUM cudasign.pub" | sha256sum -c --strict - && rm cudasign.pub && \ -echo "deb http://developer.download.nvidia.com/compute/cuda/repos/ubuntu1604/x86_64 /" > /etc/apt/sources.list.d/cuda.list && \ -echo "deb http://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1604/x86_64 /" > /etc/apt/sources.list.d/nvidia-ml.list - -CUDA_VERSION=10.0.130 -NCCL_VERSION=2.5.6 -CUDNN_VERSION=7.6.5.32 -CUDA_PKG_VERSION="10-0=$CUDA_VERSION-1" -echo 'export PATH=/usr/local/nvidia/bin:/usr/local/cuda/bin:${PATH}' >> ${HOME}/.bashrc -echo 'export LD_LIBRARY_PATH=/usr/local/nvidia/lib:/usr/local/nvidia/lib64:${LD_LIBRARY_PATH}' >> ${HOME}/.bashrc - -apt-get update && apt-get install -y --no-install-recommends --allow-unauthenticated \ - cuda-cudart-$CUDA_PKG_VERSION \ - cuda-compat-10-0 \ - cuda-libraries-$CUDA_PKG_VERSION \ - cuda-nvtx-$CUDA_PKG_VERSION \ - libnccl2=$NCCL_VERSION-1+cuda10.0 \ - libcudnn7=$CUDNN_VERSION-1+cuda10.0 && \ - ln -s cuda-10.0 /usr/local/cuda && \ - apt-mark hold libnccl2 libcudnn7 && \ - rm -rf /var/lib/apt/lists/* \ - /etc/apt/sources.list.d/nvidia-ml.list /etc/apt/sources.list.d/cuda.list - -python3 -m pip uninstall -y tensorflow -python3 -m pip install --no-cache-dir tensorflow-gpu==1.15.2 - diff --git a/components/openvino/README.md b/components/openvino/README.md deleted file mode 100644 index 3b3a123b304b..000000000000 --- a/components/openvino/README.md +++ /dev/null @@ -1,45 +0,0 @@ -## [Intel OpenVINO toolkit](https://software.intel.com/en-us/openvino-toolkit) - -### Requirements - -* Intel Core with 6th generation and higher or Intel Xeon CPUs. - -### Preparation - -- Download the latest [OpenVINO toolkit](https://software.intel.com/en-us/openvino-toolkit) .tgz installer -(offline or online) for Ubuntu platforms. Note that OpenVINO does not maintain forward compatability between -Intermediate Representations (IRs), so the version of OpenVINO in CVAT and the version used to translate the -models needs to be the same. -- Put downloaded file into ```cvat/components/openvino```. -- Accept EULA in the `cvat/components/openvino/eula.cfg` file. - -### Build docker image -```bash -# From project root directory -docker-compose -f docker-compose.yml -f components/openvino/docker-compose.openvino.yml build -``` - -### Run docker container -```bash -# From project root directory -docker-compose -f docker-compose.yml -f components/openvino/docker-compose.openvino.yml up -d -``` - -You should be able to login and see the web interface for CVAT now, complete with the new "Model Manager" button. - -### OpenVINO Models - -Clone the [Open Model Zoo](https://github.com/opencv/open_model_zoo). `$ git clone https://github.com/opencv/open_model_zoo.git` - -Install the appropriate libraries. Currently that command would be `$ pip install -r open_model_zoo/tools/downloader/requirements.in` - -Download the models using `downloader.py` file in `open_model_zoo/tools/downloader/`. -The `--name` command can be used to specify specific models. -The `--print_all` command can print all the available models. -Specific models that are already integrated into Cvat can be found [here](https://github.com/opencv/cvat/tree/develop/utils/open_model_zoo). - -From the web user interface in CVAT, upload the models using the model manager. -You'll need to include the xml and bin file from the model downloader. -You'll need to include the python and JSON files from scratch or by using the ones in the CVAT libary. -See [here](https://github.com/opencv/cvat/tree/develop/cvat/apps/auto_annotation) for instructions for creating custom -python and JSON files. diff --git a/components/openvino/docker-compose.openvino.yml b/components/openvino/docker-compose.openvino.yml deleted file mode 100644 index 3805f1b810df..000000000000 --- a/components/openvino/docker-compose.openvino.yml +++ /dev/null @@ -1,13 +0,0 @@ -# -# Copyright (C) 2018 Intel Corporation -# -# SPDX-License-Identifier: MIT -# -version: "2.3" - -services: - cvat: - build: - context: . - args: - OPENVINO_TOOLKIT: "yes" diff --git a/components/openvino/eula.cfg b/components/openvino/eula.cfg deleted file mode 100644 index 7c34e8fe1d62..000000000000 --- a/components/openvino/eula.cfg +++ /dev/null @@ -1,3 +0,0 @@ -# Accept actual EULA from openvino installation archive. Valid values are: {accept, decline} -ACCEPT_EULA=accept - diff --git a/components/openvino/install.sh b/components/openvino/install.sh deleted file mode 100755 index 159ff32d1b43..000000000000 --- a/components/openvino/install.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash -# -# Copyright (C) 2018 Intel Corporation -# -# SPDX-License-Identifier: MIT -# -set -e - -if [[ `lscpu | grep -o "GenuineIntel"` != "GenuineIntel" ]]; then - echo "OpenVINO supports only Intel CPUs" - exit 1 -fi - -if [[ `lscpu | grep -o "sse4" | head -1` != "sse4" ]] && [[ `lscpu | grep -o "avx2" | head -1` != "avx2" ]]; then - echo "OpenVINO expects your CPU to support SSE4 or AVX2 instructions" - exit 1 -fi - - -cd /tmp/components/openvino - -tar -xzf `ls | grep "openvino_toolkit"` -cd `ls -d */ | grep "openvino_toolkit"` - -apt-get update && apt-get --no-install-recommends install -y sudo cpio && \ - if [ -f "install_cv_sdk_dependencies.sh" ]; then ./install_cv_sdk_dependencies.sh; \ - else ./install_openvino_dependencies.sh; fi && SUDO_FORCE_REMOVE=yes apt-get remove -y sudo - - -cat ../eula.cfg >> silent.cfg -./install.sh -s silent.cfg - -cd /tmp/components && rm openvino -r - -if [ -f "/opt/intel/computer_vision_sdk/bin/setupvars.sh" ]; then - echo "source /opt/intel/computer_vision_sdk/bin/setupvars.sh" >> ${HOME}/.bashrc; - echo -e '\nexport IE_PLUGINS_PATH=${IE_PLUGINS_PATH}' >> /opt/intel/computer_vision_sdk/bin/setupvars.sh; -else - echo "source /opt/intel/openvino/bin/setupvars.sh" >> ${HOME}/.bashrc; - echo -e '\nexport IE_PLUGINS_PATH=${IE_PLUGINS_PATH}' >> /opt/intel/openvino/bin/setupvars.sh; -fi diff --git a/components/tf_annotation/README.md b/components/tf_annotation/README.md deleted file mode 100644 index 5a9a2c10de43..000000000000 --- a/components/tf_annotation/README.md +++ /dev/null @@ -1,41 +0,0 @@ -## [Tensorflow Object Detector](https://github.com/tensorflow/models/tree/master/research/object_detection) - -### What is it? -* This application allows you automatically to annotate many various objects on images. -* It uses [Faster RCNN Inception Resnet v2 Atrous Coco Model](http://download.tensorflow.org/models/object_detection/faster_rcnn_inception_resnet_v2_atrous_coco_2018_01_28.tar.gz) from [tensorflow detection model zoo](https://github.com/tensorflow/models/blob/master/research/object_detection/g3doc/detection_model_zoo.md) -* It can work on CPU (with Tensorflow or OpenVINO) or GPU (with Tensorflow GPU). -* It supports next classes (just specify them in "labels" row): -``` -'surfboard', 'car', 'skateboard', 'boat', 'clock', -'cat', 'cow', 'knife', 'apple', 'cup', 'tv', -'baseball_bat', 'book', 'suitcase', 'tennis_racket', -'stop_sign', 'couch', 'cell_phone', 'keyboard', -'cake', 'tie', 'frisbee', 'truck', 'fire_hydrant', -'snowboard', 'bed', 'vase', 'teddy_bear', -'toaster', 'wine_glass', 'traffic_light', -'broccoli', 'backpack', 'carrot', 'potted_plant', -'donut', 'umbrella', 'parking_meter', 'bottle', -'sandwich', 'motorcycle', 'bear', 'banana', -'person', 'scissors', 'elephant', 'dining_table', -'toothbrush', 'toilet', 'skis', 'bowl', 'sheep', -'refrigerator', 'oven', 'microwave', 'train', -'orange', 'mouse', 'laptop', 'bench', 'bicycle', -'fork', 'kite', 'zebra', 'baseball_glove', 'bus', -'spoon', 'horse', 'handbag', 'pizza', 'sports_ball', -'airplane', 'hair_drier', 'hot_dog', 'remote', -'sink', 'dog', 'bird', 'giraffe', 'chair'. -``` -* Component adds "Run TF Annotation" button into dashboard. - - -### Build docker image -```bash -# From project root directory -docker-compose -f docker-compose.yml -f components/tf_annotation/docker-compose.tf_annotation.yml build -``` - -### Run docker container -```bash -# From project root directory -docker-compose -f docker-compose.yml -f components/tf_annotation/docker-compose.tf_annotation.yml up -d -``` diff --git a/components/tf_annotation/docker-compose.tf_annotation.yml b/components/tf_annotation/docker-compose.tf_annotation.yml deleted file mode 100644 index 89fd7844f4b5..000000000000 --- a/components/tf_annotation/docker-compose.tf_annotation.yml +++ /dev/null @@ -1,13 +0,0 @@ -# -# Copyright (C) 2018 Intel Corporation -# -# SPDX-License-Identifier: MIT -# -version: "2.3" - -services: - cvat: - build: - context: . - args: - TF_ANNOTATION: "yes" diff --git a/components/tf_annotation/install.sh b/components/tf_annotation/install.sh deleted file mode 100755 index 8dc034832a2a..000000000000 --- a/components/tf_annotation/install.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash -# -# Copyright (C) 2018 Intel Corporation -# -# SPDX-License-Identifier: MIT -# -set -e - -cd ${HOME} && \ -curl http://download.tensorflow.org/models/object_detection/faster_rcnn_inception_resnet_v2_atrous_coco_2018_01_28.tar.gz -o model.tar.gz && \ -tar -xzf model.tar.gz && rm model.tar.gz && \ -mv faster_rcnn_inception_resnet_v2_atrous_coco_2018_01_28 ${HOME}/rcnn && cd ${HOME} && \ -mv rcnn/frozen_inference_graph.pb rcnn/inference_graph.pb - -# tensorflow is installed globally diff --git a/cvat-core/package.json b/cvat-core/package.json index 63ef41369095..5f7d2fcd6833 100644 --- a/cvat-core/package.json +++ b/cvat-core/package.json @@ -1,6 +1,6 @@ { "name": "cvat-core", - "version": "3.1.2", + "version": "3.3.0", "description": "Part of Computer Vision Tool which presents an interface for client-side integration", "main": "babel.config.js", "scripts": { @@ -27,7 +27,7 @@ "eslint-plugin-security": "^1.4.0", "jest": "^24.8.0", "jest-junit": "^6.4.0", - "jsdoc": "^3.6.2", + "jsdoc": "^3.6.4", "webpack": "^4.31.0", "webpack-cli": "^3.3.2" }, diff --git a/cvat-core/src/api-implementation.js b/cvat-core/src/api-implementation.js index da83fcac47e2..7372ed3b6d48 100644 --- a/cvat-core/src/api-implementation.js +++ b/cvat-core/src/api-implementation.js @@ -1,5 +1,5 @@ /* -* Copyright (C) 2019 Intel Corporation +* Copyright (C) 2019-2020 Intel Corporation * SPDX-License-Identifier: MIT */ @@ -12,6 +12,7 @@ (() => { const PluginRegistry = require('./plugins'); const serverProxy = require('./server-proxy'); + const lambdaManager = require('./lambda-manager'); const { isBoolean, isInteger, @@ -20,10 +21,7 @@ checkFilter, } = require('./common'); - const { - TaskStatus, - TaskMode, - } = require('./enums'); + const { TaskStatus, TaskMode } = require('./enums'); const User = require('./user'); const { AnnotationFormats } = require('./annotation-formats.js'); @@ -54,6 +52,13 @@ cvat.plugins.list.implementation = PluginRegistry.list; cvat.plugins.register.implementation = PluginRegistry.register.bind(cvat); + cvat.lambda.list.implementation = lambdaManager.list.bind(lambdaManager); + cvat.lambda.run.implementation = lambdaManager.run.bind(lambdaManager); + cvat.lambda.call.implementation = lambdaManager.call.bind(lambdaManager); + cvat.lambda.cancel.implementation = lambdaManager.cancel.bind(lambdaManager); + cvat.lambda.listen.implementation = lambdaManager.listen.bind(lambdaManager); + cvat.lambda.requests.implementation = lambdaManager.requests.bind(lambdaManager); + cvat.server.about.implementation = async () => { const result = await serverProxy.server.about(); return result; diff --git a/cvat-core/src/api.js b/cvat-core/src/api.js index 41a7007ad62b..53c30d0d5058 100644 --- a/cvat-core/src/api.js +++ b/cvat-core/src/api.js @@ -20,6 +20,7 @@ function build() { const Statistics = require('./statistics'); const { Job, Task } = require('./session'); const { Attribute, Label } = require('./labels'); + const MLModel = require('./ml-model'); const { ShareFileType, @@ -30,6 +31,7 @@ function build() { ObjectShape, LogType, HistoryActions, + RQStatus, colors, Source, } = require('./enums'); @@ -128,10 +130,10 @@ function build() { * @throws {module:API.cvat.exceptions.PluginError} * @throws {module:API.cvat.exceptions.ServerError} */ - async userAgreements() { - const result = await PluginRegistry - .apiWrapper(cvat.server.userAgreements); - return result; + async userAgreements() { + const result = await PluginRegistry + .apiWrapper(cvat.server.userAgreements); + return result; }, /** @@ -149,7 +151,15 @@ function build() { * @throws {module:API.cvat.exceptions.PluginError} * @throws {module:API.cvat.exceptions.ServerError} */ - async register(username, firstName, lastName, email, password1, password2, userConfirmations) { + async register( + username, + firstName, + lastName, + email, + password1, + password2, + userConfirmations, + ) { const result = await PluginRegistry .apiWrapper(cvat.server.register, username, firstName, lastName, email, password1, password2, userConfirmations); @@ -424,6 +434,119 @@ function build() { return result; }, }, + /** + * Namespace is used for serverless functions management (mainly related with DL models) + * @namespace lambda + * @memberof module:API.cvat + */ + lambda: { + /** + * Method returns list of available serverless models + * @method list + * @async + * @memberof module:API.cvat.lambda + * @returns {module:API.cvat.classes.MLModel[]} + * @throws {module:API.cvat.exceptions.ServerError} + * @throws {module:API.cvat.exceptions.PluginError} + */ + async list() { + const result = await PluginRegistry + .apiWrapper(cvat.lambda.list); + return result; + }, + + /** + * Run long-time request for a function on a specific task + * @method run + * @async + * @memberof module:API.cvat.lambda + * @param {module:API.cvat.classes.Task} task task to be annotated + * @param {module:API.cvat.classes.MLModel} model model used to get annotation + * @param {object} [args] extra arguments + * @returns {string} requestID + * @throws {module:API.cvat.exceptions.ServerError} + * @throws {module:API.cvat.exceptions.PluginError} + * @throws {module:API.cvat.exceptions.ArgumentError} + */ + async run(task, model, args) { + const result = await PluginRegistry + .apiWrapper(cvat.lambda.run, task, model, args); + return result; + }, + + /** + * Run short-time request for a function on a specific task + * @method call + * @async + * @memberof module:API.cvat.lambda + * @param {module:API.cvat.classes.Task} task task to be annotated + * @param {module:API.cvat.classes.MLModel} model model used to get annotation + * @param {object} [args] extra arguments + * @returns {string} requestID + * @throws {module:API.cvat.exceptions.ServerError} + * @throws {module:API.cvat.exceptions.PluginError} + * @throws {module:API.cvat.exceptions.ArgumentError} + */ + async call(task, model, args) { + const result = await PluginRegistry + .apiWrapper(cvat.lambda.call, task, model, args); + return result; + }, + + /** + * Cancel running of a serverless function for a specific task + * @method cancel + * @async + * @memberof module:API.cvat.lambda + * @param {string} requestID + * @throws {module:API.cvat.exceptions.ServerError} + * @throws {module:API.cvat.exceptions.PluginError} + * @throws {module:API.cvat.exceptions.ArgumentError} + */ + async cancel(requestID) { + const result = await PluginRegistry + .apiWrapper(cvat.lambda.cancel, requestID); + return result; + }, + + /** + * @callback onRequestStatusChange + * @param {string} status + * @param {number} progress + * @param {string} [message] + * @global + */ + /** + * Listen for a specific request + * @method listen + * @async + * @memberof module:API.cvat.lambda + * @param {string} requestID + * @param {onRequestStatusChange} onChange + * @throws {module:API.cvat.exceptions.ArgumentError} + * @throws {module:API.cvat.exceptions.ServerError} + * @throws {module:API.cvat.exceptions.PluginError} + */ + async listen(requestID, onChange) { + const result = await PluginRegistry + .apiWrapper(cvat.lambda.listen, requestID, onChange); + return result; + }, + + /** + * Get active lambda requests + * @method requests + * @async + * @memberof module:API.cvat.lambda + * @throws {module:API.cvat.exceptions.ServerError} + * @throws {module:API.cvat.exceptions.PluginError} + */ + async requests() { + const result = await PluginRegistry + .apiWrapper(cvat.lambda.requests); + return result; + }, + }, /** * Namespace to working with logs * @namespace logger @@ -531,6 +654,7 @@ function build() { ObjectShape, LogType, HistoryActions, + RQStatus, colors, Source, }, @@ -561,6 +685,7 @@ function build() { Label, Statistics, ObjectState, + MLModel, }, }; @@ -569,6 +694,7 @@ function build() { cvat.jobs = Object.freeze(cvat.jobs); cvat.users = Object.freeze(cvat.users); cvat.plugins = Object.freeze(cvat.plugins); + cvat.lambda = Object.freeze(cvat.lambda); cvat.client = Object.freeze(cvat.client); cvat.enums = Object.freeze(cvat.enums); diff --git a/cvat-core/src/enums.js b/cvat-core/src/enums.js index a1a5918fd59d..c7b6c5952fcb 100644 --- a/cvat-core/src/enums.js +++ b/cvat-core/src/enums.js @@ -34,6 +34,26 @@ COMPLETED: 'completed', }); + /** + * List of RQ statuses + * @enum {string} + * @name RQStatus + * @memberof module:API.cvat.enums + * @property {string} QUEUED 'queued' + * @property {string} STARTED 'started' + * @property {string} FINISHED 'finished' + * @property {string} FAILED 'failed' + * @property {string} UNKNOWN 'unknown' + * @readonly + */ + const RQStatus = Object.freeze({ + QUEUED: 'queued', + STARTED: 'started', + FINISHED: 'finished', + FAILED: 'failed', + UNKNOWN: 'unknown', + }); + /** * Task modes * @enum {string} @@ -232,6 +252,18 @@ REMOVED_OBJECT: 'Removed object', }); + /** + * Enum string values. + * @name ModelType + * @memberof module:API.cvat.enums + * @enum {string} + */ + const ModelType = { + DETECTOR: 'detector', + INTERACTOR: 'interactor', + TRACKER: 'tracker', + }; + /** * Array of hex colors * @name colors @@ -255,7 +287,9 @@ ObjectType, ObjectShape, LogType, + ModelType, HistoryActions, + RQStatus, colors, Source, }; diff --git a/cvat-core/src/lambda-manager.js b/cvat-core/src/lambda-manager.js new file mode 100644 index 000000000000..4b936cd21307 --- /dev/null +++ b/cvat-core/src/lambda-manager.js @@ -0,0 +1,126 @@ +/* +* Copyright (C) 2020 Intel Corporation +* SPDX-License-Identifier: MIT +*/ + +/* global + require:false +*/ + +const serverProxy = require('./server-proxy'); +const { ArgumentError } = require('./exceptions'); +const { Task } = require('./session'); +const MLModel = require('./ml-model'); +const { RQStatus } = require('./enums'); + +class LambdaManager { + constructor() { + this.listening = {}; + this.cachedList = null; + } + + async list() { + if (Array.isArray(this.cachedList)) { + return [...this.cachedList]; + } + + const result = await serverProxy.lambda.list(); + const models = []; + + for (const model of result) { + models.push(new MLModel({ + id: model.id, + name: model.name, + description: model.description, + framework: model.framework, + labels: [...model.labels], + type: model.kind, + })); + } + + this.cachedList = models; + return models; + } + + async run(task, model, args) { + if (!(task instanceof Task)) { + throw new ArgumentError( + `Argument task is expected to be an instance of Task class, but got ${typeof (task)}`, + ); + } + + if (!(model instanceof MLModel)) { + throw new ArgumentError( + `Argument model is expected to be an instance of MLModel class, but got ${typeof (model)}`, + ); + } + + if (args && typeof (args) !== 'object') { + throw new ArgumentError( + `Argument args is expected to be an object, but got ${typeof (model)}`, + ); + } + + const body = args; + body.task = task.id; + body.function = model.id; + + const result = await serverProxy.lambda.run(body); + return result.id; + } + + async call(task, model, args) { + const body = args; + body.task = task.id; + const result = await serverProxy.lambda.call(model.id, body); + return result; + } + + async requests() { + const result = await serverProxy.lambda.requests(); + return result.filter((request) => ['queued', 'started'].includes(request.status)); + } + + async cancel(requestID) { + if (typeof (requestID) !== 'string') { + throw new ArgumentError(`Request id argument is required to be a string. But got ${requestID}`); + } + + if (this.listening[requestID]) { + clearTimeout(this.listening[requestID].timeout); + delete this.listening[requestID]; + } + await serverProxy.lambda.cancel(requestID); + } + + async listen(requestID, onUpdate) { + const timeoutCallback = async () => { + try { + this.listening[requestID].timeout = null; + const response = await serverProxy.lambda.status(requestID); + + if (response.status === RQStatus.QUEUED || response.status === RQStatus.STARTED) { + onUpdate(response.status, response.progress || 0); + this.listening[requestID].timeout = setTimeout(timeoutCallback, 2000); + } else { + if (response.status === RQStatus.FINISHED) { + onUpdate(response.status, response.progress || 100); + } else { + onUpdate(response.status, response.progress || 0, response.exc_info || ''); + } + + delete this.listening[requestID]; + } + } catch (error) { + onUpdate(RQStatus.UNKNOWN, 0, `Could not get a status of the request ${requestID}. ${error.toString()}`); + } + }; + + this.listening[requestID] = { + onUpdate, + timeout: setTimeout(timeoutCallback, 2000), + }; + } +} + +module.exports = new LambdaManager(); diff --git a/cvat-core/src/ml-model.js b/cvat-core/src/ml-model.js new file mode 100644 index 000000000000..4169be8e2066 --- /dev/null +++ b/cvat-core/src/ml-model.js @@ -0,0 +1,73 @@ +/* +* Copyright (C) 2019-2020 Intel Corporation +* SPDX-License-Identifier: MIT +*/ + +/** + * Class representing a machine learning model + * @memberof module:API.cvat.classes +*/ +class MLModel { + constructor(data) { + this._id = data.id; + this._name = data.name; + this._labels = data.labels; + this._framework = data.framework; + this._description = data.description; + this._type = data.type; + } + + /** + * @returns {string} + * @readonly + */ + get id() { + return this._id; + } + + /** + * @returns {string} + * @readonly + */ + get name() { + return this._name; + } + + /** + * @returns {string[]} + * @readonly + */ + get labels() { + if (Array.isArray(this._labels)) { + return [...this._labels]; + } + + return []; + } + + /** + * @returns {string} + * @readonly + */ + get framework() { + return this._framework; + } + + /** + * @returns {string} + * @readonly + */ + get description() { + return this._description; + } + + /** + * @returns {module:API.cvat.enums.ModelType} + * @readonly + */ + get type() { + return this._type; + } +} + +module.exports = MLModel; diff --git a/cvat-core/src/server-proxy.js b/cvat-core/src/server-proxy.js index 5b685114e3a7..00f06d77c41d 100644 --- a/cvat-core/src/server-proxy.js +++ b/cvat-core/src/server-proxy.js @@ -162,7 +162,6 @@ response = await Axios.get(`${backendAPI}/restrictions/user-agreements`, { proxy: config.proxy, }); - } catch (errorData) { throw generateError(errorData); } @@ -170,7 +169,15 @@ return response.data; } - async function register(username, firstName, lastName, email, password1, password2, confirmations) { + async function register( + username, + firstName, + lastName, + email, + password1, + password2, + confirmations, + ) { let response = null; try { const data = JSON.stringify({ @@ -662,6 +669,96 @@ } } + async function getLambdaFunctions() { + const { backendAPI } = config; + + try { + const response = await Axios.get(`${backendAPI}/lambda/functions`, { + proxy: config.proxy, + }); + return response.data; + } catch (errorData) { + throw generateError(errorData); + } + } + + async function runLambdaRequest(body) { + const { backendAPI } = config; + + try { + const response = await Axios.post(`${backendAPI}/lambda/requests`, + JSON.stringify(body), { + proxy: config.proxy, + headers: { + 'Content-Type': 'application/json', + }, + }); + + return response.data; + } catch (errorData) { + throw generateError(errorData); + } + } + + async function callLambdaFunction(funId, body) { + const { backendAPI } = config; + + try { + const response = await Axios.post(`${backendAPI}/lambda/functions/${funId}`, + JSON.stringify(body), { + proxy: config.proxy, + headers: { + 'Content-Type': 'application/json', + }, + }); + + return response.data; + } catch (errorData) { + throw generateError(errorData); + } + } + + async function getLambdaRequests() { + const { backendAPI } = config; + + try { + const response = await Axios.get(`${backendAPI}/lambda/requests`, { + proxy: config.proxy, + }); + + return response.data; + } catch (errorData) { + throw generateError(errorData); + } + } + + async function getRequestStatus(requestID) { + const { backendAPI } = config; + + try { + const response = await Axios.get(`${backendAPI}/lambda/requests/${requestID}`, { + proxy: config.proxy, + }); + return response.data; + } catch (errorData) { + throw generateError(errorData); + } + } + + async function cancelLambdaRequest(requestId) { + const { backendAPI } = config; + + try { + await Axios.delete( + `${backendAPI}/lambda/requests/${requestId}`, { + method: 'DELETE', + }, + ); + } catch (errorData) { + throw generateError(errorData); + } + } + Object.defineProperties(this, Object.freeze({ server: { value: Object.freeze({ @@ -731,6 +828,18 @@ }), writable: false, }, + + lambda: { + value: Object.freeze({ + list: getLambdaFunctions, + status: getRequestStatus, + requests: getLambdaRequests, + run: runLambdaRequest, + call: callLambdaFunction, + cancel: cancelLambdaRequest, + }), + writable: false, + }, })); } } diff --git a/cvat-ui/src/actions/models-actions.ts b/cvat-ui/src/actions/models-actions.ts index eebacbaedd1c..1f2afe142a45 100644 --- a/cvat-ui/src/actions/models-actions.ts +++ b/cvat-ui/src/actions/models-actions.ts @@ -3,27 +3,14 @@ // SPDX-License-Identifier: MIT import { ActionUnion, createAction, ThunkAction } from 'utils/redux'; -import { - Model, - ModelType, - ModelFiles, - ActiveInference, - CombinedState, -} from 'reducers/interfaces'; +import { Model, ActiveInference, RQStatus } from 'reducers/interfaces'; import getCore from 'cvat-core-wrapper'; -export enum PreinstalledModels { - RCNN = 'RCNN Object Detector', - MaskRCNN = 'Mask RCNN Object Detector', -} - export enum ModelsActionTypes { GET_MODELS = 'GET_MODELS', GET_MODELS_SUCCESS = 'GET_MODELS_SUCCESS', GET_MODELS_FAILED = 'GET_MODELS_FAILED', DELETE_MODEL = 'DELETE_MODEL', - DELETE_MODEL_SUCCESS = 'DELETE_MODEL_SUCCESS', - DELETE_MODEL_FAILED = 'DELETE_MODEL_FAILED', CREATE_MODEL = 'CREATE_MODEL', CREATE_MODEL_SUCCESS = 'CREATE_MODEL_SUCCESS', CREATE_MODEL_FAILED = 'CREATE_MODEL_FAILED', @@ -50,28 +37,6 @@ export const modelsActions = { error, }, ), - deleteModelSuccess: (id: number) => createAction( - ModelsActionTypes.DELETE_MODEL_SUCCESS, { - id, - }, - ), - deleteModelFailed: (id: number, error: any) => createAction( - ModelsActionTypes.DELETE_MODEL_FAILED, { - error, id, - }, - ), - createModel: () => createAction(ModelsActionTypes.CREATE_MODEL), - createModelSuccess: () => createAction(ModelsActionTypes.CREATE_MODEL_SUCCESS), - createModelFailed: (error: any) => createAction( - ModelsActionTypes.CREATE_MODEL_FAILED, { - error, - }, - ), - createModelUpdateStatus: (status: string) => createAction( - ModelsActionTypes.CREATE_MODEL_STATUS_UPDATED, { - status, - }, - ), fetchMetaFailed: (error: any) => createAction(ModelsActionTypes.FETCH_META_FAILED, { error }), getInferenceStatusSuccess: (taskID: number, activeInference: ActiveInference) => createAction( ModelsActionTypes.GET_INFERENCE_STATUS_SUCCESS, { @@ -96,7 +61,7 @@ export const modelsActions = { taskID, }, ), - cancelInferenceFaild: (taskID: number, error: any) => createAction( + cancelInferenceFailed: (taskID: number, error: any) => createAction( ModelsActionTypes.CANCEL_INFERENCE_FAILED, { taskID, error, @@ -113,361 +78,76 @@ export const modelsActions = { export type ModelsActions = ActionUnion; const core = getCore(); -const baseURL = core.config.backendAPI.slice(0, -7); export function getModelsAsync(): ThunkAction { - return async (dispatch, getState): Promise => { - const state: CombinedState = getState(); - const OpenVINO = state.plugins.list.AUTO_ANNOTATION; - const RCNN = state.plugins.list.TF_ANNOTATION; - const MaskRCNN = state.plugins.list.TF_SEGMENTATION; - + return async (dispatch): Promise => { dispatch(modelsActions.getModels()); - const models: Model[] = []; try { - if (OpenVINO) { - const response = await core.server.request( - `${baseURL}/auto_annotation/meta/get`, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - }, - data: JSON.stringify([]), - }, - ); - - - for (const model of response.models) { - models.push({ - id: model.id, - ownerID: model.owner, - primary: model.primary, - name: model.name, - uploadDate: model.uploadDate, - updateDate: model.updateDate, - labels: [...model.labels], - }); - } - } - - if (RCNN) { - models.push({ - id: null, - ownerID: null, - primary: true, - name: PreinstalledModels.RCNN, - uploadDate: '', - updateDate: '', - labels: ['surfboard', 'car', 'skateboard', 'boat', 'clock', - 'cat', 'cow', 'knife', 'apple', 'cup', 'tv', - 'baseball_bat', 'book', 'suitcase', 'tennis_racket', - 'stop_sign', 'couch', 'cell_phone', 'keyboard', - 'cake', 'tie', 'frisbee', 'truck', 'fire_hydrant', - 'snowboard', 'bed', 'vase', 'teddy_bear', - 'toaster', 'wine_glass', 'traffic_light', - 'broccoli', 'backpack', 'carrot', 'potted_plant', - 'donut', 'umbrella', 'parking_meter', 'bottle', - 'sandwich', 'motorcycle', 'bear', 'banana', - 'person', 'scissors', 'elephant', 'dining_table', - 'toothbrush', 'toilet', 'skis', 'bowl', 'sheep', - 'refrigerator', 'oven', 'microwave', 'train', - 'orange', 'mouse', 'laptop', 'bench', 'bicycle', - 'fork', 'kite', 'zebra', 'baseball_glove', 'bus', - 'spoon', 'horse', 'handbag', 'pizza', 'sports_ball', - 'airplane', 'hair_drier', 'hot_dog', 'remote', - 'sink', 'dog', 'bird', 'giraffe', 'chair', - ], - }); - } - - if (MaskRCNN) { - models.push({ - id: null, - ownerID: null, - primary: true, - name: PreinstalledModels.MaskRCNN, - uploadDate: '', - updateDate: '', - labels: ['BG', 'person', 'bicycle', 'car', 'motorcycle', 'airplane', - 'bus', 'train', 'truck', 'boat', 'traffic light', - 'fire hydrant', 'stop sign', 'parking meter', 'bench', 'bird', - 'cat', 'dog', 'horse', 'sheep', 'cow', 'elephant', 'bear', - 'zebra', 'giraffe', 'backpack', 'umbrella', 'handbag', 'tie', - 'suitcase', 'frisbee', 'skis', 'snowboard', 'sports ball', - 'kite', 'baseball bat', 'baseball glove', 'skateboard', - 'surfboard', 'tennis racket', 'bottle', 'wine glass', 'cup', - 'fork', 'knife', 'spoon', 'bowl', 'banana', 'apple', - 'sandwich', 'orange', 'broccoli', 'carrot', 'hot dog', 'pizza', - 'donut', 'cake', 'chair', 'couch', 'potted plant', 'bed', - 'dining table', 'toilet', 'tv', 'laptop', 'mouse', 'remote', - 'keyboard', 'cell phone', 'microwave', 'oven', 'toaster', - 'sink', 'refrigerator', 'book', 'clock', 'vase', 'scissors', - 'teddy bear', 'hair drier', 'toothbrush', - ], - }); - } + const models = (await core.lambda.list()) + .filter((model: Model) => ['detector', 'reid'].includes(model.type)); + dispatch(modelsActions.getModelsSuccess(models)); } catch (error) { dispatch(modelsActions.getModelsFailed(error)); - return; } - - dispatch(modelsActions.getModelsSuccess(models)); }; } -export function deleteModelAsync(id: number): ThunkAction { - return async (dispatch): Promise => { - try { - await core.server.request(`${baseURL}/auto_annotation/delete/${id}`, { - method: 'DELETE', - }); - } catch (error) { - dispatch(modelsActions.deleteModelFailed(id, error)); - return; - } - - dispatch(modelsActions.deleteModelSuccess(id)); - }; -} - -export function createModelAsync(name: string, files: ModelFiles, global: boolean): ThunkAction { - return async (dispatch): Promise => { - async function checkCallback(id: string): Promise { - try { - const data = await core.server.request( - `${baseURL}/auto_annotation/check/${id}`, { - method: 'GET', - }, - ); - - switch (data.status) { - case 'failed': - dispatch(modelsActions.createModelFailed( - `Checking request has returned the "${data.status}" status. Message: ${data.error}`, - )); - break; - case 'unknown': - dispatch(modelsActions.createModelFailed( - `Checking request has returned the "${data.status}" status.`, - )); - break; - case 'finished': - dispatch(modelsActions.createModelSuccess()); - break; - default: - if ('progress' in data) { - modelsActions.createModelUpdateStatus(data.progress); - } - setTimeout(checkCallback.bind(null, id), 1000); - } - } catch (error) { - dispatch(modelsActions.createModelFailed(error)); - } - } - - dispatch(modelsActions.createModel()); - const data = new FormData(); - data.append('name', name); - data.append('storage', typeof files.bin === 'string' ? 'shared' : 'local'); - data.append('shared', global.toString()); - Object.keys(files).reduce((acc, key: string): FormData => { - acc.append(key, files[key]); - return acc; - }, data); - - try { - dispatch(modelsActions.createModelUpdateStatus('Request is beign sent..')); - const response = await core.server.request( - `${baseURL}/auto_annotation/create`, { - method: 'POST', - data, - contentType: false, - processData: false, - }, - ); - - dispatch(modelsActions.createModelUpdateStatus('Request is being processed..')); - setTimeout(checkCallback.bind(null, response.id), 1000); - } catch (error) { - dispatch(modelsActions.createModelFailed(error)); - } - }; -} interface InferenceMeta { - active: boolean; taskID: number; requestID: string; - modelType: ModelType; } -const timers: any = {}; - -async function timeoutCallback( - url: string, - taskID: number, - modelType: ModelType, +function listen( + inferenceMeta: InferenceMeta, dispatch: (action: ModelsActions) => void, -): Promise { - try { - delete timers[taskID]; - - const response = await core.server.request(url, { - method: 'GET', - }); - - const activeInference: ActiveInference = { - status: response.status, - progress: +response.progress || 0, - error: response.error || response.stderr || '', - modelType, - }; - - - if (activeInference.status === 'unknown') { - dispatch(modelsActions.getInferenceStatusFailed( - taskID, - new Error( - `Inference status for the task ${taskID} is unknown.`, - ), - )); - - return; - } - - if (activeInference.status === 'failed') { +): void { + const { taskID, requestID } = inferenceMeta; + core.lambda.listen(requestID, (status: RQStatus, progress: number, message: string) => { + if (status === RQStatus.failed || status === RQStatus.unknown) { dispatch(modelsActions.getInferenceStatusFailed( taskID, new Error( - `Inference status for the task ${taskID} is failed. ${activeInference.error}`, + `Inference status for the task ${taskID} is ${status}. ${message}`, ), )); return; } - if (activeInference.status !== 'finished') { - timers[taskID] = setTimeout( - timeoutCallback.bind( - null, - url, - taskID, - modelType, - dispatch, - ), 3000, - ); - } - - dispatch(modelsActions.getInferenceStatusSuccess(taskID, activeInference)); - } catch (error) { - dispatch(modelsActions.getInferenceStatusFailed(taskID, new Error( - `Server request for the task ${taskID} was failed`, - ))); - } -} - -function subscribe( - inferenceMeta: InferenceMeta, - dispatch: (action: ModelsActions) => void, -): void { - if (!(inferenceMeta.taskID in timers)) { - let requestURL = `${baseURL}`; - if (inferenceMeta.modelType === ModelType.OPENVINO) { - requestURL = `${requestURL}/auto_annotation/check`; - } else if (inferenceMeta.modelType === ModelType.RCNN) { - requestURL = `${requestURL}/tensorflow/annotation/check/task`; - } else if (inferenceMeta.modelType === ModelType.MASK_RCNN) { - requestURL = `${requestURL}/tensorflow/segmentation/check/task`; - } - requestURL = `${requestURL}/${inferenceMeta.requestID}`; - timers[inferenceMeta.taskID] = setTimeout( - timeoutCallback.bind( - null, - requestURL, - inferenceMeta.taskID, - inferenceMeta.modelType, - dispatch, - ), - ); - } + dispatch(modelsActions.getInferenceStatusSuccess(taskID, { + status, + progress, + error: message, + id: requestID, + })); + }).catch((error: Error) => { + dispatch(modelsActions.getInferenceStatusFailed(taskID, { + status: 'unknown', + progress: 0, + error: error.toString(), + id: requestID, + })); + }); } -export function getInferenceStatusAsync(tasks: number[]): ThunkAction { - return async (dispatch, getState): Promise => { - function parse(response: any, modelType: ModelType): InferenceMeta[] { - return Object.keys(response).map((key: string): InferenceMeta => ({ - taskID: +key, - requestID: response[key].rq_id || key, - active: typeof (response[key].active) === 'undefined' ? ['queued', 'started'] - .includes(response[key].status.toLowerCase()) : response[key].active, - modelType, - })); - } - - const state: CombinedState = getState(); - const OpenVINO = state.plugins.list.AUTO_ANNOTATION; - const RCNN = state.plugins.list.TF_ANNOTATION; - const MaskRCNN = state.plugins.list.TF_SEGMENTATION; - +export function getInferenceStatusAsync(): ThunkAction { + return async (dispatch): Promise => { const dispatchCallback = (action: ModelsActions): void => { dispatch(action); }; try { - if (OpenVINO) { - const response = await core.server.request( - `${baseURL}/auto_annotation/meta/get`, { - method: 'POST', - data: JSON.stringify(tasks), - headers: { - 'Content-Type': 'application/json', - }, - }, - ); - - parse(response.run, ModelType.OPENVINO) - .filter((inferenceMeta: InferenceMeta): boolean => inferenceMeta.active) - .forEach((inferenceMeta: InferenceMeta): void => { - subscribe(inferenceMeta, dispatchCallback); - }); - } - - if (RCNN) { - const response = await core.server.request( - `${baseURL}/tensorflow/annotation/meta/get`, { - method: 'POST', - data: JSON.stringify(tasks), - headers: { - 'Content-Type': 'application/json', - }, - }, - ); - - parse(response, ModelType.RCNN) - .filter((inferenceMeta: InferenceMeta): boolean => inferenceMeta.active) - .forEach((inferenceMeta: InferenceMeta): void => { - subscribe(inferenceMeta, dispatchCallback); - }); - } - - if (MaskRCNN) { - const response = await core.server.request( - `${baseURL}/tensorflow/segmentation/meta/get`, { - method: 'POST', - data: JSON.stringify(tasks), - headers: { - 'Content-Type': 'application/json', - }, - }, - ); - - parse(response, ModelType.MASK_RCNN) - .filter((inferenceMeta: InferenceMeta): boolean => inferenceMeta.active) - .forEach((inferenceMeta: InferenceMeta): void => { - subscribe(inferenceMeta, dispatchCallback); - }); - } + const requests = await core.lambda.requests(); + requests + .map((request: any): object => ({ + taskID: +request.function.task, + requestID: request.id, + })) + .forEach((inferenceMeta: InferenceMeta): void => { + listen(inferenceMeta, dispatchCallback); + }); } catch (error) { dispatch(modelsActions.fetchMetaFailed(error)); } @@ -477,37 +157,20 @@ export function getInferenceStatusAsync(tasks: number[]): ThunkAction { export function startInferenceAsync( taskInstance: any, model: Model, - mapping: { - [index: string]: string; - }, - cleanOut: boolean, + body: object, ): ThunkAction { return async (dispatch): Promise => { try { - if (model.name === PreinstalledModels.RCNN) { - await core.server.request( - `${baseURL}/tensorflow/annotation/create/task/${taskInstance.id}`, - ); - } else if (model.name === PreinstalledModels.MaskRCNN) { - await core.server.request( - `${baseURL}/tensorflow/segmentation/create/task/${taskInstance.id}`, - ); - } else { - await core.server.request( - `${baseURL}/auto_annotation/start/${model.id}/${taskInstance.id}`, { - method: 'POST', - data: JSON.stringify({ - reset: cleanOut, - labels: mapping, - }), - headers: { - 'Content-Type': 'application/json', - }, - }, - ); - } + const requestID: string = await core.lambda.run(taskInstance, model, body); + + const dispatchCallback = (action: ModelsActions): void => { + dispatch(action); + }; - dispatch(getInferenceStatusAsync([taskInstance.id])); + listen({ + taskID: taskInstance.id, + requestID, + }, dispatchCallback); } catch (error) { dispatch(modelsActions.startInferenceFailed(taskInstance.id, error)); } @@ -518,30 +181,10 @@ export function cancelInferenceAsync(taskID: number): ThunkAction { return async (dispatch, getState): Promise => { try { const inference = getState().models.inferences[taskID]; - if (inference) { - if (inference.modelType === ModelType.OPENVINO) { - await core.server.request( - `${baseURL}/auto_annotation/cancel/${taskID}`, - ); - } else if (inference.modelType === ModelType.RCNN) { - await core.server.request( - `${baseURL}/tensorflow/annotation/cancel/task/${taskID}`, - ); - } else if (inference.modelType === ModelType.MASK_RCNN) { - await core.server.request( - `${baseURL}/tensorflow/segmentation/cancel/task/${taskID}`, - ); - } - - if (timers[taskID]) { - clearTimeout(timers[taskID]); - delete timers[taskID]; - } - } - + await core.lambda.cancel(inference.id); dispatch(modelsActions.cancelInferenceSuccess(taskID)); } catch (error) { - dispatch(modelsActions.cancelInferenceFaild(taskID, error)); + dispatch(modelsActions.cancelInferenceFailed(taskID, error)); } }; } diff --git a/cvat-ui/src/actions/plugins-actions.ts b/cvat-ui/src/actions/plugins-actions.ts index 510f6590e827..21b3fed5e6f9 100644 --- a/cvat-ui/src/actions/plugins-actions.ts +++ b/cvat-ui/src/actions/plugins-actions.ts @@ -29,27 +29,19 @@ export function checkPluginsAsync(): ThunkAction { dispatch(pluginActions.checkPlugins()); const plugins: PluginObjects = { ANALYTICS: false, - AUTO_ANNOTATION: false, GIT_INTEGRATION: false, - TF_ANNOTATION: false, - TF_SEGMENTATION: false, - REID: false, DEXTR_SEGMENTATION: false, }; const promises: Promise[] = [ PluginChecker.check(SupportedPlugins.ANALYTICS), - PluginChecker.check(SupportedPlugins.AUTO_ANNOTATION), PluginChecker.check(SupportedPlugins.GIT_INTEGRATION), - PluginChecker.check(SupportedPlugins.TF_ANNOTATION), - PluginChecker.check(SupportedPlugins.TF_SEGMENTATION), PluginChecker.check(SupportedPlugins.DEXTR_SEGMENTATION), - PluginChecker.check(SupportedPlugins.REID), ]; const values = await Promise.all(promises); - [plugins.ANALYTICS, plugins.AUTO_ANNOTATION, plugins.GIT_INTEGRATION, plugins.TF_ANNOTATION, - plugins.TF_SEGMENTATION, plugins.DEXTR_SEGMENTATION, plugins.REID] = values; + [plugins.ANALYTICS, plugins.GIT_INTEGRATION, + plugins.DEXTR_SEGMENTATION] = values; dispatch(pluginActions.checkedAllPlugins(plugins)); }; } diff --git a/cvat-ui/src/actions/tasks-actions.ts b/cvat-ui/src/actions/tasks-actions.ts index fd1bfbc48ff1..4097f0198e85 100644 --- a/cvat-ui/src/actions/tasks-actions.ts +++ b/cvat-ui/src/actions/tasks-actions.ts @@ -102,13 +102,7 @@ ThunkAction, {}, {}, AnyAction> { const promises = array .map((task): string => (task as any).frames.preview()); - dispatch( - getInferenceStatusAsync( - array.map( - (task: any): number => task.id, - ), - ), - ); + dispatch(getInferenceStatusAsync()); for (const promise of promises) { try { diff --git a/cvat-ui/src/components/actions-menu/actions-menu.tsx b/cvat-ui/src/components/actions-menu/actions-menu.tsx index 5a1070d05a01..d51560e2bf79 100644 --- a/cvat-ui/src/components/actions-menu/actions-menu.tsx +++ b/cvat-ui/src/components/actions-menu/actions-menu.tsx @@ -15,16 +15,11 @@ interface Props { taskID: number; taskMode: string; bugTracker: string; - loaders: any[]; dumpers: any[]; loadActivity: string | null; dumpActivities: string[] | null; exportActivities: string[] | null; - - installedTFAnnotation: boolean; - installedTFSegmentation: boolean; - installedAutoAnnotation: boolean; inferenceIsActive: boolean; onClickMenu: (params: ClickParam, file?: File) => void; @@ -44,12 +39,7 @@ export default function ActionsMenuComponent(props: Props): JSX.Element { taskID, taskMode, bugTracker, - - installedAutoAnnotation, - installedTFAnnotation, - installedTFSegmentation, inferenceIsActive, - dumpers, loaders, onClickMenu, @@ -58,9 +48,6 @@ export default function ActionsMenuComponent(props: Props): JSX.Element { loadActivity, } = props; - const renderModelRunner = installedAutoAnnotation - || installedTFAnnotation || installedTFSegmentation; - let latestParams: ClickParam | null = null; function onClickMenuWrapper(params: ClickParam | null, file?: File): void { const copyParams = params || latestParams; @@ -137,17 +124,12 @@ export default function ActionsMenuComponent(props: Props): JSX.Element { }) } {!!bugTracker && Open bug tracker} - { - renderModelRunner - && ( - - Automatic annotation - - ) - } + + Automatic annotation +
Delete diff --git a/cvat-ui/src/components/annotation-page/top-bar/annotation-menu.tsx b/cvat-ui/src/components/annotation-page/top-bar/annotation-menu.tsx index 92da2240d251..02d3d74849b7 100644 --- a/cvat-ui/src/components/annotation-page/top-bar/annotation-menu.tsx +++ b/cvat-ui/src/components/annotation-page/top-bar/annotation-menu.tsx @@ -9,7 +9,6 @@ import Modal from 'antd/lib/modal'; import DumpSubmenu from 'components/actions-menu/dump-submenu'; import LoadSubmenu from 'components/actions-menu/load-submenu'; import ExportSubmenu from 'components/actions-menu/export-submenu'; -import ReIDPlugin from './reid-plugin'; interface Props { taskMode: string; @@ -18,7 +17,6 @@ interface Props { loadActivity: string | null; dumpActivities: string[] | null; exportActivities: string[] | null; - installedReID: boolean; taskID: number; onClickMenu(params: ClickParam, file?: File): void; } @@ -40,7 +38,6 @@ export default function AnnotationMenuComponent(props: Props): JSX.Element { loadActivity, dumpActivities, exportActivities, - installedReID, taskID, } = props; @@ -125,7 +122,6 @@ export default function AnnotationMenuComponent(props: Props): JSX.Element { Open the task - { installedReID && } ); } diff --git a/cvat-ui/src/components/annotation-page/top-bar/reid-plugin.tsx b/cvat-ui/src/components/annotation-page/top-bar/reid-plugin.tsx deleted file mode 100644 index 25e418838953..000000000000 --- a/cvat-ui/src/components/annotation-page/top-bar/reid-plugin.tsx +++ /dev/null @@ -1,229 +0,0 @@ -// Copyright (C) 2020 Intel Corporation -// -// SPDX-License-Identifier: MIT - -import ReactDOM from 'react-dom'; -import React, { useState, useEffect } from 'react'; -import { Row, Col } from 'antd/lib/grid'; -import Modal from 'antd/lib/modal'; -import Menu from 'antd/lib/menu'; -import Text from 'antd/lib/typography/Text'; -import InputNumber from 'antd/lib/input-number'; -import Tooltip from 'antd/lib/tooltip'; - -import { clamp } from 'utils/math'; -import { run, cancel } from 'utils/reid-utils'; -import { connect } from 'react-redux'; -import { CombinedState } from 'reducers/interfaces'; -import { fetchAnnotationsAsync } from 'actions/annotation-actions'; - -interface InputModalProps { - visible: boolean; - onCancel(): void; - onSubmit(threshold: number, distance: number): void; -} - -function InputModal(props: InputModalProps): JSX.Element { - const { visible, onCancel, onSubmit } = props; - const [threshold, setThreshold] = useState(0.5); - const [distance, setDistance] = useState(50); - - const [thresholdMin, thresholdMax] = [0.05, 0.95]; - const [distanceMin, distanceMax] = [1, 1000]; - return ( - onSubmit(threshold, distance)} - okText='Merge' - > - - - - Similarity threshold: - - - - { - if (typeof (value) === 'number') { - setThreshold(clamp(value, thresholdMin, thresholdMax)); - } - }} - /> - - - - - - Max pixel distance: - - - - { - if (typeof (value) === 'number') { - setDistance(clamp(value, distanceMin, distanceMax)); - } - }} - /> - - - - ); -} - -interface InProgressDialogProps { - visible: boolean; - progress: number; - onCancel(): void; -} - -function InProgressDialog(props: InProgressDialogProps): JSX.Element { - const { visible, onCancel, progress } = props; - return ( - - {`Merging is in progress ${progress}%`} - - ); -} - -const reidContainer = window.document.createElement('div'); -reidContainer.setAttribute('id', 'cvat-reid-wrapper'); -window.document.body.appendChild(reidContainer); - - -interface StateToProps { - jobInstance: any | null; -} - -interface DispatchToProps { - updateAnnotations(): void; -} - -function mapStateToProps(state: CombinedState): StateToProps { - const { - annotation: { - job: { - instance: jobInstance, - }, - }, - } = state; - - return { - jobInstance, - }; -} - -function mapDispatchToProps(dispatch: any): DispatchToProps { - return { - updateAnnotations(): void { - dispatch(fetchAnnotationsAsync()); - }, - }; -} - - -function ReIDPlugin(props: StateToProps & DispatchToProps): JSX.Element { - const { jobInstance, updateAnnotations, ...rest } = props; - const [showInputDialog, setShowInputDialog] = useState(false); - const [showInProgressDialog, setShowInProgressDialog] = useState(false); - const [progress, setProgress] = useState(0); - - useEffect(() => { - ReactDOM.render(( - <> - { - cancel(jobInstance.id); - }} - /> - setShowInputDialog(false)} - onSubmit={async (threshold: number, distance: number) => { - setProgress(0); - setShowInputDialog(false); - setShowInProgressDialog(true); - - const onUpdatePercentage = (percent: number): void => { - setProgress(percent); - }; - - try { - const annotations = await jobInstance.annotations.export(); - const merged = await run({ - threshold, - distance, - onUpdatePercentage, - jobID: jobInstance.id, - annotations, - }); - await jobInstance.annotations.clear(); - updateAnnotations(); // one more call to do not confuse canvas - // False positive of no-unsanitized/method - // eslint-disable-next-line no-unsanitized/method - await jobInstance.annotations.import(merged); - updateAnnotations(); - } catch (error) { - Modal.error({ - title: 'Could not merge annotations', - content: error.toString(), - }); - } finally { - setShowInProgressDialog(false); - } - }} - /> - - ), reidContainer); - }); - - return ( - { - if (jobInstance) { - setShowInputDialog(true); - } - }} - > - Run ReID merge - - ); -} - -export default connect( - mapStateToProps, - mapDispatchToProps, -)(ReIDPlugin); diff --git a/cvat-ui/src/components/create-model-page/create-model-content.tsx b/cvat-ui/src/components/create-model-page/create-model-content.tsx deleted file mode 100644 index a562bfce3277..000000000000 --- a/cvat-ui/src/components/create-model-page/create-model-content.tsx +++ /dev/null @@ -1,160 +0,0 @@ -// Copyright (C) 2020 Intel Corporation -// -// SPDX-License-Identifier: MIT - -import React from 'react'; -import { Row, Col } from 'antd/lib/grid'; -import Icon from 'antd/lib/icon'; -import Alert from 'antd/lib/alert'; -import Button from 'antd/lib/button'; -import Tooltip from 'antd/lib/tooltip'; -import message from 'antd/lib/message'; -import notification from 'antd/lib/notification'; -import Text from 'antd/lib/typography/Text'; - -import consts from 'consts'; -import ConnectedFileManager, { - FileManagerContainer, -} from 'containers/file-manager/file-manager'; -import { ModelFiles } from 'reducers/interfaces'; - -import WrappedCreateModelForm, { - CreateModelForm, -} from './create-model-form'; - -interface Props { - createModel(name: string, files: ModelFiles, global: boolean): void; - isAdmin: boolean; - modelCreatingStatus: string; -} - -export default class CreateModelContent extends React.PureComponent { - private modelForm: CreateModelForm; - private fileManagerContainer: FileManagerContainer; - - public constructor(props: Props) { - super(props); - this.modelForm = null as any as CreateModelForm; - this.fileManagerContainer = null as any as FileManagerContainer; - } - - public componentDidUpdate(prevProps: Props): void { - const { modelCreatingStatus } = this.props; - - if (prevProps.modelCreatingStatus !== 'CREATED' - && modelCreatingStatus === 'CREATED') { - message.success('The model has been uploaded'); - this.modelForm.resetFields(); - this.fileManagerContainer.reset(); - } - } - - private handleSubmitClick = (): void => { - const { createModel } = this.props; - this.modelForm.submit() - .then((data) => { - const { - local, - share, - } = this.fileManagerContainer.getFiles(); - - const files = local.length ? local : share; - const grouppedFiles: ModelFiles = { - xml: '', - bin: '', - py: '', - json: '', - }; - - (files as any).reduce((acc: ModelFiles, value: File | string): ModelFiles => { - const name = typeof value === 'string' ? value : value.name; - const [extension] = name.split('.').reverse(); - if (extension in acc) { - acc[extension] = value; - } - - return acc; - }, grouppedFiles); - - if (Object.keys(grouppedFiles) - .map((key: string) => grouppedFiles[key]) - .filter((val) => !!val).length !== 4) { - notification.error({ - message: 'Could not upload a model', - description: 'Please, specify correct files', - }); - } else { - createModel(data.name, grouppedFiles, data.global); - } - }).catch(() => { - notification.error({ - message: 'Could not upload a model', - description: 'Please, check input fields', - }); - }); - }; - - public render(): JSX.Element { - const { - modelCreatingStatus, - } = this.props; - const loading = !!modelCreatingStatus - && modelCreatingStatus !== 'CREATED'; - const status = modelCreatingStatus - && modelCreatingStatus !== 'CREATED' ? modelCreatingStatus : ''; - - const { AUTO_ANNOTATION_GUIDE_URL } = consts; - return ( - - - - { - // false positive - // eslint-disable-next-line - window.open(AUTO_ANNOTATION_GUIDE_URL, '_blank'); - }} - type='question-circle' - /> - - - - { - this.modelForm = ref; - } - } - /> - - - * - Select files: - - - { - this.fileManagerContainer = container; - } - } - withRemote={false} - /> - - - {status && } - - - - - - ); - } -} diff --git a/cvat-ui/src/components/create-model-page/create-model-form.tsx b/cvat-ui/src/components/create-model-page/create-model-form.tsx deleted file mode 100644 index 004e96458d2a..000000000000 --- a/cvat-ui/src/components/create-model-page/create-model-form.tsx +++ /dev/null @@ -1,80 +0,0 @@ -// Copyright (C) 2020 Intel Corporation -// -// SPDX-License-Identifier: MIT - -import React from 'react'; -import { Row, Col } from 'antd/lib/grid'; -import Form, { FormComponentProps } from 'antd/lib/form/Form'; -import Input from 'antd/lib/input'; -import Tooltip from 'antd/lib/tooltip'; -import Checkbox from 'antd/lib/checkbox'; -import Text from 'antd/lib/typography/Text'; - -type Props = FormComponentProps; - -export class CreateModelForm extends React.PureComponent { - public submit(): Promise<{name: string; global: boolean}> { - const { form } = this.props; - return new Promise((resolve, reject) => { - form.validateFields((errors, values): void => { - if (!errors) { - resolve({ - name: values.name, - global: values.global, - }); - } else { - reject(errors); - } - }); - }); - } - - public resetFields(): void { - const { form } = this.props; - form.resetFields(); - } - - public render(): JSX.Element { - const { form } = this.props; - const { getFieldDecorator } = form; - - return ( -
e.preventDefault()}> - - - * - Name: - - - - { getFieldDecorator('name', { - rules: [{ - required: true, - message: 'Please, specify a model name', - }], - })()} - - - - - - { getFieldDecorator('global', { - initialValue: false, - valuePropName: 'checked', - })( - - - Load globally - - , - )} - - - - -
- ); - } -} - -export default Form.create()(CreateModelForm); diff --git a/cvat-ui/src/components/create-model-page/create-model-page.tsx b/cvat-ui/src/components/create-model-page/create-model-page.tsx deleted file mode 100644 index 7c81aca0f878..000000000000 --- a/cvat-ui/src/components/create-model-page/create-model-page.tsx +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright (C) 2020 Intel Corporation -// -// SPDX-License-Identifier: MIT - -import './styles.scss'; -import React from 'react'; -import { Row, Col } from 'antd/lib/grid'; -import Text from 'antd/lib/typography/Text'; - -import { ModelFiles } from 'reducers/interfaces'; -import CreateModelContent from './create-model-content'; - -interface Props { - createModel(name: string, files: ModelFiles, global: boolean): void; - isAdmin: boolean; - modelCreatingStatus: string; -} - -export default function CreateModelPageComponent(props: Props): JSX.Element { - const { - isAdmin, - modelCreatingStatus, - createModel, - } = props; - - return ( - - - Upload a new model - - - - ); -} diff --git a/cvat-ui/src/components/create-model-page/styles.scss b/cvat-ui/src/components/create-model-page/styles.scss deleted file mode 100644 index 65df22b23d33..000000000000 --- a/cvat-ui/src/components/create-model-page/styles.scss +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright (C) 2020 Intel Corporation -// -// SPDX-License-Identifier: MIT - -@import '../../base.scss'; - -.cvat-create-model-form-wrapper { - text-align: center; - margin-top: 40px; - overflow-y: auto; - height: 90%; - - > div > span { - font-size: 36px; - } - - .cvat-create-model-content { - margin-top: 20px; - width: 100%; - height: auto; - border: 1px solid $border-color-1; - border-radius: 3px; - padding: 20px; - background: $background-color-1; - text-align: initial; - - > div:nth-child(1) > i { - float: right; - font-size: 20px; - color: $danger-icon-color; - } - - > div:nth-child(4) { - margin-top: 10px; - } - - > div:nth-child(6) > button { - margin-top: 10px; - float: right; - width: 120px; - } - } -} diff --git a/cvat-ui/src/components/cvat-app.tsx b/cvat-ui/src/components/cvat-app.tsx index 22dccb88ec6e..19c26f66a072 100644 --- a/cvat-ui/src/components/cvat-app.tsx +++ b/cvat-ui/src/components/cvat-app.tsx @@ -18,7 +18,6 @@ import TasksPageContainer from 'containers/tasks-page/tasks-page'; import CreateTaskPageContainer from 'containers/create-task-page/create-task-page'; import TaskPageContainer from 'containers/task-page/task-page'; import ModelsPageContainer from 'containers/models-page/models-page'; -import CreateModelPageContainer from 'containers/create-model-page/create-model-page'; import AnnotationPageContainer from 'containers/annotation-page/annotation-page'; import LoginPageContainer from 'containers/login-page/login-page'; import RegisterPageContainer from 'containers/register-page/register-page'; @@ -50,9 +49,6 @@ interface CVATAppProps { usersFetching: boolean; aboutInitialized: boolean; aboutFetching: boolean; - installedAutoAnnotation: boolean; - installedTFAnnotation: boolean; - installedTFSegmentation: boolean; userAgreementsFetching: boolean; userAgreementsInitialized: boolean; notifications: NotificationsState; @@ -222,9 +218,6 @@ class CVATApplication extends React.PureComponent - {withModels - && } - {installedAutoAnnotation - && } + diff --git a/cvat-ui/src/components/header/header.tsx b/cvat-ui/src/components/header/header.tsx index dc01678b80b1..9d3e5fee4302 100644 --- a/cvat-ui/src/components/header/header.tsx +++ b/cvat-ui/src/components/header/header.tsx @@ -24,9 +24,6 @@ interface HeaderContainerProps { switchSettingsDialog: (show: boolean) => void; logoutFetching: boolean; installedAnalytics: boolean; - installedAutoAnnotation: boolean; - installedTFAnnotation: boolean; - installedTFSegmentation: boolean; serverHost: string; username: string; toolName: string; @@ -43,9 +40,6 @@ type Props = HeaderContainerProps & RouteComponentProps; function HeaderContainer(props: Props): JSX.Element { const { - installedTFSegmentation, - installedAutoAnnotation, - installedTFAnnotation, installedAnalytics, username, toolName, @@ -62,10 +56,6 @@ function HeaderContainer(props: Props): JSX.Element { switchSettingsDialog, } = props; - const renderModels = installedAutoAnnotation - || installedTFAnnotation - || installedTFSegmentation; - const { CHANGELOG_URL, LICENSE_URL, @@ -172,19 +162,16 @@ function HeaderContainer(props: Props): JSX.Element { > Tasks - { renderModels - && ( - - )} + { installedAnalytics && ( - )} - ); } - -export default withRouter(TopBarComponent); diff --git a/cvat-ui/src/components/models-page/uploaded-model-item.tsx b/cvat-ui/src/components/models-page/uploaded-model-item.tsx deleted file mode 100644 index b108ec8ddf18..000000000000 --- a/cvat-ui/src/components/models-page/uploaded-model-item.tsx +++ /dev/null @@ -1,89 +0,0 @@ -// Copyright (C) 2020 Intel Corporation -// -// SPDX-License-Identifier: MIT - -import React from 'react'; -import { Row, Col } from 'antd/lib/grid'; -import Tag from 'antd/lib/tag'; -import Select from 'antd/lib/select'; -import Icon from 'antd/lib/icon'; -import Menu from 'antd/lib/menu'; -import Dropdown from 'antd/lib/dropdown'; -import Text from 'antd/lib/typography/Text'; -import moment from 'moment'; - -import { MenuIcon } from 'icons'; -import { Model } from 'reducers/interfaces'; - -interface Props { - model: Model; - owner: any; - onDelete(): void; -} - -export default function UploadedModelItem(props: Props): JSX.Element { - const { - model, - owner, - onDelete, - } = props; - - return ( - - - OpenVINO - - - - {model.name} - - - - - {owner ? owner.username : 'undefined'} - - - - - {moment(model.uploadDate).format('MMMM Do YYYY')} - - - - - - - Actions - - { - onDelete(); - }} - key='delete' - > - Delete - - - ) - } - > - - - - - ); -} diff --git a/cvat-ui/src/components/models-page/uploaded-models-list.tsx b/cvat-ui/src/components/models-page/uploaded-models-list.tsx deleted file mode 100644 index be5a035461b9..000000000000 --- a/cvat-ui/src/components/models-page/uploaded-models-list.tsx +++ /dev/null @@ -1,70 +0,0 @@ -// Copyright (C) 2020 Intel Corporation -// -// SPDX-License-Identifier: MIT - -import React from 'react'; -import { Row, Col } from 'antd/lib/grid'; -import Text from 'antd/lib/typography/Text'; - -import { Model } from 'reducers/interfaces'; -import UploadedModelItem from './uploaded-model-item'; - - -interface Props { - registeredUsers: any[]; - models: Model[]; - deleteModel(id: number): void; -} - -export default function UploadedModelsListComponent(props: Props): JSX.Element { - const { - models, - registeredUsers, - deleteModel, - } = props; - - const items = models.map((model): JSX.Element => { - const owner = registeredUsers.filter((user) => user.id === model.ownerID)[0]; - return ( - deleteModel(model.id as number)} - /> - ); - }); - - return ( - <> - - - Uploaded by a user - - - - - - - Framework - - - Name - - - Owner - - - Uploaded - - - Labels - - - - { items } - - - - ); -} diff --git a/cvat-ui/src/consts.ts b/cvat-ui/src/consts.ts index e0a94454e8cf..e5df33c7ff1e 100644 --- a/cvat-ui/src/consts.ts +++ b/cvat-ui/src/consts.ts @@ -11,8 +11,8 @@ const GITTER_PUBLIC_URL = 'https://gitter.im/opencv-cvat/public'; const FORUM_URL = 'https://software.intel.com/en-us/forums/intel-distribution-of-openvino-toolkit'; const GITHUB_URL = 'https://github.com/opencv/cvat'; const GITHUB_IMAGE_URL = 'https://raw.githubusercontent.com/opencv/cvat/develop/cvat/apps/documentation/static/documentation/images/cvat.jpg'; -const AUTO_ANNOTATION_GUIDE_URL = 'https://github.com/opencv/cvat/blob/develop/cvat/apps/auto_annotation/README.md'; const SHARE_MOUNT_GUIDE_URL = 'https://github.com/opencv/cvat/blob/master/cvat/apps/documentation/installation.md#share-path'; +const NUCLIO_GUIDE = 'https://github.com/opencv/cvat/blob/develop/cvat/apps/documentation/installation.md#semi-automatic-and-automatic-annotation'; const CANVAS_BACKGROUND_COLORS = ['#ffffff', '#f1f1f1', '#e5e5e5', '#d8d8d8', '#CCCCCC', '#B3B3B3', '#999999']; export default { @@ -25,7 +25,7 @@ export default { FORUM_URL, GITHUB_URL, GITHUB_IMAGE_URL, - AUTO_ANNOTATION_GUIDE_URL, SHARE_MOUNT_GUIDE_URL, CANVAS_BACKGROUND_COLORS, + NUCLIO_GUIDE, }; diff --git a/cvat-ui/src/containers/actions-menu/actions-menu.tsx b/cvat-ui/src/containers/actions-menu/actions-menu.tsx index a92a42bb1fff..206201f7f68a 100644 --- a/cvat-ui/src/containers/actions-menu/actions-menu.tsx +++ b/cvat-ui/src/containers/actions-menu/actions-menu.tsx @@ -28,9 +28,6 @@ interface StateToProps { loadActivity: string | null; dumpActivities: string[] | null; exportActivities: string[] | null; - installedTFAnnotation: boolean; - installedTFSegmentation: boolean; - installedAutoAnnotation: boolean; inferenceIsActive: boolean; } @@ -53,13 +50,6 @@ function mapStateToProps(state: CombinedState, own: OwnProps): StateToProps { formats: { annotationFormats, }, - plugins: { - list: { - TF_ANNOTATION: installedTFAnnotation, - TF_SEGMENTATION: installedTFSegmentation, - AUTO_ANNOTATION: installedAutoAnnotation, - }, - }, tasks: { activities: { dumps, @@ -70,9 +60,6 @@ function mapStateToProps(state: CombinedState, own: OwnProps): StateToProps { } = state; return { - installedTFAnnotation, - installedTFSegmentation, - installedAutoAnnotation, dumpActivities: tid in dumps ? dumps[tid] : null, exportActivities: tid in activeExports ? activeExports[tid] : null, loadActivity: tid in loads ? loads[tid] : null, @@ -112,9 +99,6 @@ function ActionsMenuContainer(props: OwnProps & StateToProps & DispatchToProps): dumpActivities, exportActivities, inferenceIsActive, - installedAutoAnnotation, - installedTFAnnotation, - installedTFSegmentation, loadAnnotations, dumpAnnotations, @@ -172,9 +156,6 @@ function ActionsMenuContainer(props: OwnProps & StateToProps & DispatchToProps): dumpActivities={dumpActivities} exportActivities={exportActivities} inferenceIsActive={inferenceIsActive} - installedAutoAnnotation={installedAutoAnnotation} - installedTFAnnotation={installedTFAnnotation} - installedTFSegmentation={installedTFSegmentation} onClickMenu={onClickMenu} /> ); diff --git a/cvat-ui/src/containers/annotation-page/top-bar/annotation-menu.tsx b/cvat-ui/src/containers/annotation-page/top-bar/annotation-menu.tsx index 139168a8f293..74edb4c28062 100644 --- a/cvat-ui/src/containers/annotation-page/top-bar/annotation-menu.tsx +++ b/cvat-ui/src/containers/annotation-page/top-bar/annotation-menu.tsx @@ -26,7 +26,6 @@ interface StateToProps { loadActivity: string | null; dumpActivities: string[] | null; exportActivities: string[] | null; - installedReID: boolean; } interface DispatchToProps { @@ -56,9 +55,6 @@ function mapStateToProps(state: CombinedState): StateToProps { exports: activeExports, }, }, - plugins: { - list, - }, } = state; const taskID = jobInstance.task.id; @@ -71,7 +67,6 @@ function mapStateToProps(state: CombinedState): StateToProps { ? loads[taskID] || jobLoads[jobID] : null, jobInstance, annotationFormats, - installedReID: list.REID, }; } @@ -109,7 +104,6 @@ function AnnotationMenuContainer(props: Props): JSX.Element { loadActivity, dumpActivities, exportActivities, - installedReID, } = props; const onClickMenu = (params: ClickParam, file?: File): void => { @@ -155,7 +149,6 @@ function AnnotationMenuContainer(props: Props): JSX.Element { loadActivity={loadActivity} dumpActivities={dumpActivities} exportActivities={exportActivities} - installedReID={installedReID} onClickMenu={onClickMenu} taskID={jobInstance.task.id} /> diff --git a/cvat-ui/src/containers/create-model-page/create-model-page.tsx b/cvat-ui/src/containers/create-model-page/create-model-page.tsx deleted file mode 100644 index 74cd406ea871..000000000000 --- a/cvat-ui/src/containers/create-model-page/create-model-page.tsx +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright (C) 2020 Intel Corporation -// -// SPDX-License-Identifier: MIT - -import { connect } from 'react-redux'; - -import CreateModelPageComponent from 'components/create-model-page/create-model-page'; -import { createModelAsync } from 'actions/models-actions'; -import { - ModelFiles, - CombinedState, -} from 'reducers/interfaces'; - -interface StateToProps { - isAdmin: boolean; - modelCreatingStatus: string; -} - -interface DispatchToProps { - createModel(name: string, files: ModelFiles, global: boolean): void; -} - -function mapStateToProps(state: CombinedState): StateToProps { - const { models } = state; - - return { - isAdmin: state.auth.user.isAdmin, - modelCreatingStatus: models.creatingStatus, - }; -} - -function mapDispatchToProps(dispatch: any): DispatchToProps { - return { - createModel(name: string, files: ModelFiles, global: boolean): void { - dispatch(createModelAsync(name, files, global)); - }, - }; -} - -export default connect( - mapStateToProps, - mapDispatchToProps, -)(CreateModelPageComponent); diff --git a/cvat-ui/src/containers/header/header.tsx b/cvat-ui/src/containers/header/header.tsx index 185226a7959a..abda8b465cd7 100644 --- a/cvat-ui/src/containers/header/header.tsx +++ b/cvat-ui/src/containers/header/header.tsx @@ -15,9 +15,6 @@ const core = getCore(); interface StateToProps { logoutFetching: boolean; installedAnalytics: boolean; - installedAutoAnnotation: boolean; - installedTFSegmentation: boolean; - installedTFAnnotation: boolean; username: string; toolName: string; serverHost: string; @@ -61,9 +58,6 @@ function mapStateToProps(state: CombinedState): StateToProps { return { logoutFetching, installedAnalytics: list[SupportedPlugins.ANALYTICS], - installedAutoAnnotation: list[SupportedPlugins.AUTO_ANNOTATION], - installedTFSegmentation: list[SupportedPlugins.TF_SEGMENTATION], - installedTFAnnotation: list[SupportedPlugins.TF_ANNOTATION], username, toolName: server.name as string, serverHost: core.config.backendAPI.slice(0, -7), diff --git a/cvat-ui/src/containers/model-runner-dialog/model-runner-dialog.tsx b/cvat-ui/src/containers/model-runner-dialog/model-runner-dialog.tsx index 4855f62d16a5..9ec5b670c3a9 100644 --- a/cvat-ui/src/containers/model-runner-dialog/model-runner-dialog.tsx +++ b/cvat-ui/src/containers/model-runner-dialog/model-runner-dialog.tsx @@ -31,10 +31,7 @@ interface DispatchToProps { runInference( taskInstance: any, model: Model, - mapping: { - [index: string]: string; - }, - cleanOut: boolean, + body: object, ): void; getModels(): void; closeDialog(): void; @@ -58,12 +55,9 @@ function mapDispatchToProps(dispatch: any): DispatchToProps { runInference( taskInstance: any, model: Model, - mapping: { - [index: string]: string; - }, - cleanOut: boolean, + body: object, ): void { - dispatch(startInferenceAsync(taskInstance, model, mapping, cleanOut)); + dispatch(startInferenceAsync(taskInstance, model, body)); }, getModels(): void { dispatch(getModelsAsync()); diff --git a/cvat-ui/src/containers/models-page/models-page.tsx b/cvat-ui/src/containers/models-page/models-page.tsx index 6e2150a8ba7f..27eb9df99a3e 100644 --- a/cvat-ui/src/containers/models-page/models-page.tsx +++ b/cvat-ui/src/containers/models-page/models-page.tsx @@ -2,7 +2,6 @@ // // SPDX-License-Identifier: MIT -import React from 'react'; import { connect } from 'react-redux'; import ModelsPageComponent from 'components/models-page/models-page'; @@ -10,38 +9,25 @@ import { Model, CombinedState, } from 'reducers/interfaces'; -import { - getModelsAsync, - deleteModelAsync, -} from 'actions/models-actions'; +import { getModelsAsync } from 'actions/models-actions'; interface StateToProps { - installedAutoAnnotation: boolean; - installedTFAnnotation: boolean; - installedTFSegmentation: boolean; modelsInitialized: boolean; modelsFetching: boolean; - models: Model[]; - registeredUsers: any[]; + deployedModels: Model[]; } interface DispatchToProps { getModels(): void; - deleteModel(id: number): void; } function mapStateToProps(state: CombinedState): StateToProps { - const { list } = state.plugins; const { models } = state; return { - installedAutoAnnotation: list.AUTO_ANNOTATION, - installedTFAnnotation: list.TF_ANNOTATION, - installedTFSegmentation: list.TF_SEGMENTATION, modelsInitialized: models.initialized, modelsFetching: models.fetching, - models: models.models, - registeredUsers: state.users.users, + deployedModels: models.models, }; } @@ -50,29 +36,10 @@ function mapDispatchToProps(dispatch: any): DispatchToProps { getModels(): void { dispatch(getModelsAsync()); }, - deleteModel(id: number): void { - dispatch(deleteModelAsync(id)); - }, }; } -function ModelsPageContainer(props: DispatchToProps & StateToProps): JSX.Element | null { - const { - installedAutoAnnotation, - installedTFSegmentation, - installedTFAnnotation, - } = props; - - const render = installedAutoAnnotation - || installedTFAnnotation - || installedTFSegmentation; - - return ( - render ? : null - ); -} - export default connect( mapStateToProps, mapDispatchToProps, -)(ModelsPageContainer); +)(ModelsPageComponent); diff --git a/cvat-ui/src/index.tsx b/cvat-ui/src/index.tsx index 439ba3959324..6d6148372c56 100644 --- a/cvat-ui/src/index.tsx +++ b/cvat-ui/src/index.tsx @@ -48,9 +48,6 @@ interface StateToProps { formatsFetching: boolean; userAgreementsInitialized: boolean; userAgreementsFetching: boolean; - installedAutoAnnotation: boolean; - installedTFSegmentation: boolean; - installedTFAnnotation: boolean; notifications: NotificationsState; user: any; keyMap: Record; @@ -66,7 +63,7 @@ interface DispatchToProps { resetMessages: () => void; switchShortcutsDialog: () => void; loadUserAgreements: () => void; - switchSettingsDialog: (show: boolean) => void; + switchSettingsDialog: () => void; } function mapStateToProps(state: CombinedState): StateToProps { @@ -91,9 +88,6 @@ function mapStateToProps(state: CombinedState): StateToProps { formatsFetching: formats.fetching, userAgreementsInitialized: userAgreements.initialized, userAgreementsFetching: userAgreements.fetching, - installedAutoAnnotation: plugins.list.AUTO_ANNOTATION, - installedTFSegmentation: plugins.list.TF_SEGMENTATION, - installedTFAnnotation: plugins.list.TF_ANNOTATION, notifications: state.notifications, user: auth.user, keyMap: shortcuts.keyMap, diff --git a/cvat-ui/src/reducers/interfaces.ts b/cvat-ui/src/reducers/interfaces.ts index 5af1ef99b9a9..065b0dd854e1 100644 --- a/cvat-ui/src/reducers/interfaces.ts +++ b/cvat-ui/src/reducers/interfaces.ts @@ -71,12 +71,8 @@ export interface FormatsState { // eslint-disable-next-line import/prefer-default-export export enum SupportedPlugins { GIT_INTEGRATION = 'GIT_INTEGRATION', - AUTO_ANNOTATION = 'AUTO_ANNOTATION', - TF_ANNOTATION = 'TF_ANNOTATION', - TF_SEGMENTATION = 'TF_SEGMENTATION', DEXTR_SEGMENTATION = 'DEXTR_SEGMENTATION', ANALYTICS = 'ANALYTICS', - REID = 'REID', } export interface PluginsState { @@ -133,13 +129,12 @@ export interface ShareState { } export interface Model { - id: number | null; // null for preinstalled models - ownerID: number | null; // null for preinstalled models + id: string; name: string; - primary: boolean; - uploadDate: string; - updateDate: string; labels: string[]; + framework: string; + description: string; + type: string; } export enum RQStatus { @@ -150,17 +145,11 @@ export enum RQStatus { failed = 'failed', } -export enum ModelType { - OPENVINO = 'openvino', - RCNN = 'rcnn', - MASK_RCNN = 'mask_rcnn', -} - export interface ActiveInference { status: RQStatus; progress: number; error: string; - modelType: ModelType; + id: string; } export interface ModelsState { @@ -175,14 +164,6 @@ export interface ModelsState { activeRunTask: any; } -export interface ModelFiles { - [key: string]: string | File; - xml: string | File; - bin: string | File; - py: string | File; - json: string | File; -} - export interface ErrorState { message: string; reason: string; diff --git a/cvat-ui/src/reducers/models-reducer.ts b/cvat-ui/src/reducers/models-reducer.ts index cfe788b31ccf..3e369584b5fe 100644 --- a/cvat-ui/src/reducers/models-reducer.ts +++ b/cvat-ui/src/reducers/models-reducer.ts @@ -44,39 +44,6 @@ export default function ( fetching: false, }; } - case ModelsActionTypes.DELETE_MODEL_SUCCESS: { - return { - ...state, - models: state.models.filter( - (model): boolean => model.id !== action.payload.id, - ), - }; - } - case ModelsActionTypes.CREATE_MODEL: { - return { - ...state, - creatingStatus: '', - }; - } - case ModelsActionTypes.CREATE_MODEL_STATUS_UPDATED: { - return { - ...state, - creatingStatus: action.payload.status, - }; - } - case ModelsActionTypes.CREATE_MODEL_FAILED: { - return { - ...state, - creatingStatus: '', - }; - } - case ModelsActionTypes.CREATE_MODEL_SUCCESS: { - return { - ...state, - initialized: false, - creatingStatus: 'CREATED', - }; - } case ModelsActionTypes.SHOW_RUN_MODEL_DIALOG: { return { ...state, diff --git a/cvat-ui/src/reducers/notifications-reducer.ts b/cvat-ui/src/reducers/notifications-reducer.ts index f7d4b5db9518..68a65ca54798 100644 --- a/cvat-ui/src/reducers/notifications-reducer.ts +++ b/cvat-ui/src/reducers/notifications-reducer.ts @@ -361,21 +361,6 @@ export default function (state = defaultState, action: AnyAction): Notifications }, }; } - case ModelsActionTypes.DELETE_MODEL_FAILED: { - return { - ...state, - errors: { - ...state.errors, - models: { - ...state.errors.models, - deleting: { - message: 'Could not delete the model', - reason: action.payload.error.toString(), - }, - }, - }, - }; - } case ModelsActionTypes.GET_INFERENCE_STATUS_SUCCESS: { if (action.payload.activeInference.status === 'finished') { const { taskID } = action.payload; @@ -420,7 +405,7 @@ export default function (state = defaultState, action: AnyAction): Notifications models: { ...state.errors.models, inferenceStatusFetching: { - message: 'Could not fetch inference status for the ' + message: 'Fetching inference status for the ' + `task ${taskID}`, reason: action.payload.error.toString(), }, diff --git a/cvat-ui/src/reducers/plugins-reducer.ts b/cvat-ui/src/reducers/plugins-reducer.ts index 0bdc6fd2fe0a..71cfc3aba2a9 100644 --- a/cvat-ui/src/reducers/plugins-reducer.ts +++ b/cvat-ui/src/reducers/plugins-reducer.ts @@ -5,21 +5,15 @@ import { PluginsActionTypes, PluginActions } from 'actions/plugins-actions'; import { registerGitPlugin } from 'utils/git-utils'; import { registerDEXTRPlugin } from 'utils/dextr-utils'; -import { - PluginsState, -} from './interfaces'; +import { PluginsState } from './interfaces'; const defaultState: PluginsState = { fetching: false, initialized: false, list: { GIT_INTEGRATION: false, - AUTO_ANNOTATION: false, - TF_ANNOTATION: false, - TF_SEGMENTATION: false, DEXTR_SEGMENTATION: false, ANALYTICS: false, - REID: false, }, }; diff --git a/cvat-ui/src/utils/dextr-utils.ts b/cvat-ui/src/utils/dextr-utils.ts index 03280cface55..a9d92abfffd2 100644 --- a/cvat-ui/src/utils/dextr-utils.ts +++ b/cvat-ui/src/utils/dextr-utils.ts @@ -4,11 +4,10 @@ import getCore from 'cvat-core-wrapper'; import { Canvas } from 'cvat-canvas-wrapper'; -import { ShapeType, RQStatus, CombinedState } from 'reducers/interfaces'; +import { ShapeType, CombinedState } from 'reducers/interfaces'; import { getCVATStore } from 'cvat-store'; const core = getCore(); -const baseURL = core.config.backendAPI.slice(0, -7); interface DEXTRPlugin { name: string; @@ -71,79 +70,33 @@ antModal.append(antModalContent); antModalWrap.append(antModal); antModalRoot.append(antModalMask, antModalWrap); - -function serverRequest( - plugin: DEXTRPlugin, - jid: number, +async function serverRequest( + taskInstance: any, frame: number, points: number[], ): Promise { - return new Promise((resolve, reject) => { - const reducer = (acc: Point[], _: number, index: number, array: number[]): Point[] => { - if (!(index % 2)) { // 0, 2, 4 - acc.push({ - x: array[index], - y: array[index + 1], - }); - } - - return acc; - }; - - const reducedPoints = points.reduce(reducer, []); - core.server.request( - `${baseURL}/dextr/create/${jid}`, { - method: 'POST', - data: JSON.stringify({ - frame, - points: reducedPoints, - }), - headers: { - 'Content-Type': 'application/json', - }, - }, - ).then(() => { - const timeoutCallback = (): void => { - core.server.request( - `${baseURL}/dextr/check/${jid}`, { - method: 'GET', - }, - ).then((response: any) => { - const { status } = response; - if (status === RQStatus.finished) { - resolve(response.result.split(/\s|,/).map((coord: string) => +coord)); - } else if (status === RQStatus.failed) { - reject(new Error(response.stderr)); - } else if (status === RQStatus.unknown) { - reject(new Error('Unknown DEXTR status has been received')); - } else { - if (status === RQStatus.queued) { - antModalButton.disabled = false; - } - if (!plugin.data.canceled) { - setTimeout(timeoutCallback, 1000); - } else { - core.server.request( - `${baseURL}/dextr/cancel/${jid}`, { - method: 'GET', - }, - ).then(() => { - resolve(points); - }).catch((error: Error) => { - reject(error); - }); - } - } - }).catch((error: Error) => { - reject(error); - }); - }; + const reducer = (acc: number[][], + _: number, index: number, + array: number[]): number[][] => { + if (!(index % 2)) { // 0, 2, 4 + acc.push([ + array[index], + array[index + 1], + ]); + } + return acc; + }; - setTimeout(timeoutCallback, 1000); - }).catch((error: Error) => { - reject(error); - }); + const reducedPoints = points.reduce(reducer, []); + const models = await core.lambda.list(); + const model = models.filter((func: any): boolean => func.id === 'openvino.dextr')[0]; + const result = await core.lambda.call(taskInstance, model, { + task: taskInstance, + frame, + points: reducedPoints, }); + + return result.flat(); } async function enter(this: any, self: DEXTRPlugin, objects: any[]): Promise { @@ -159,8 +112,7 @@ async function enter(this: any, self: DEXTRPlugin, objects: any[]): Promise= 8) { promises[i] = serverRequest( - self, - this.id, + this.task, objects[i].frame, objects[i].points, ); diff --git a/cvat-ui/src/utils/plugin-checker.ts b/cvat-ui/src/utils/plugin-checker.ts index 54113f9a52aa..eb9f7a5d5737 100644 --- a/cvat-ui/src/utils/plugin-checker.ts +++ b/cvat-ui/src/utils/plugin-checker.ts @@ -26,24 +26,13 @@ class PluginChecker { case SupportedPlugins.GIT_INTEGRATION: { return isReachable(`${serverHost}/git/repository/meta/get`, 'OPTIONS'); } - case SupportedPlugins.AUTO_ANNOTATION: { - return isReachable(`${serverHost}/auto_annotation/meta/get`, 'OPTIONS'); - } - case SupportedPlugins.TF_ANNOTATION: { - return isReachable(`${serverHost}/tensorflow/annotation/meta/get`, 'OPTIONS'); - } - case SupportedPlugins.TF_SEGMENTATION: { - return isReachable(`${serverHost}/tensorflow/segmentation/meta/get`, 'OPTIONS'); - } case SupportedPlugins.DEXTR_SEGMENTATION: { - return isReachable(`${serverHost}/dextr/enabled`, 'GET'); + const list = await core.lambda.list(); + return list.map((func: any): boolean => func.id).includes('openvino.dextr'); } case SupportedPlugins.ANALYTICS: { return isReachable(`${serverHost}/analytics/app/kibana`, 'GET'); } - case SupportedPlugins.REID: { - return isReachable(`${serverHost}/reid/enabled`, 'GET'); - } default: return false; } diff --git a/cvat-ui/src/utils/reid-utils.ts b/cvat-ui/src/utils/reid-utils.ts deleted file mode 100644 index f628b95faa76..000000000000 --- a/cvat-ui/src/utils/reid-utils.ts +++ /dev/null @@ -1,96 +0,0 @@ -// Copyright (C) 2020 Intel Corporation -// -// SPDX-License-Identifier: MIT - -import getCore from 'cvat-core-wrapper'; -import { ShapeType, RQStatus } from 'reducers/interfaces'; - - -const core = getCore(); -const baseURL = core.config.backendAPI.slice(0, -7); - -type Params = { - threshold: number; - distance: number; - onUpdatePercentage(percentage: number): void; - jobID: number; - annotations: any; -}; - -export function run(params: Params): Promise { - return new Promise((resolve, reject) => { - const { - threshold, - distance, - onUpdatePercentage, - jobID, - annotations, - } = params; - const { shapes, ...rest } = annotations; - - const boxes = shapes.filter((shape: any): boolean => shape.type === ShapeType.RECTANGLE); - const others = shapes.filter((shape: any): boolean => shape.type !== ShapeType.RECTANGLE); - - core.server.request( - `${baseURL}/reid/start/job/${params.jobID}`, { - method: 'POST', - data: JSON.stringify({ - threshold, - maxDistance: distance, - boxes, - }), - headers: { - 'Content-Type': 'application/json', - }, - }, - ).then(() => { - const timeoutCallback = (): void => { - core.server.request( - `${baseURL}/reid/check/${jobID}`, { - method: 'GET', - }, - ).then((response: any) => { - const { status } = response; - if (status === RQStatus.finished) { - if (!response.result) { - // cancelled - resolve(annotations); - } - - const result = JSON.parse(response.result); - const collection = rest; - Array.prototype.push.apply(collection.tracks, result); - collection.shapes = others; - resolve(collection); - } else if (status === RQStatus.started) { - const { progress } = response; - if (typeof (progress) === 'number') { - onUpdatePercentage(+progress.toFixed(2)); - } - setTimeout(timeoutCallback, 1000); - } else if (status === RQStatus.failed) { - reject(new Error(response.stderr)); - } else if (status === RQStatus.unknown) { - reject(new Error('Unknown REID status has been received')); - } else { - setTimeout(timeoutCallback, 1000); - } - }).catch((error: Error) => { - reject(error); - }); - }; - - setTimeout(timeoutCallback, 1000); - }).catch((error: Error) => { - reject(error); - }); - }); -} - -export function cancel(jobID: number): void { - core.server.request( - `${baseURL}/reid/cancel/${jobID}`, { - method: 'GET', - }, - ); -} diff --git a/cvat/apps/auto_annotation/README.md b/cvat/apps/auto_annotation/README.md deleted file mode 100644 index 27fecdf80721..000000000000 --- a/cvat/apps/auto_annotation/README.md +++ /dev/null @@ -1,372 +0,0 @@ -## Auto annotation - -- [Description](#description) -- [Installation](#installation) -- [Usage](#usage) -- [Testing script](#testing) -- [Examples](#examples) - - [Person-vehicle-bike-detection-crossroad-0078](#person-vehicle-bike-detection-crossroad-0078-openvino-toolkit) - - [Landmarks-regression-retail-0009](#landmarks-regression-retail-0009-openvino-toolkit) - - [Semantic Segmentation](#semantic-segmentation) -- [Available interpretation scripts](#available-interpretation-scripts) - -### Description - -The application will be enabled automatically if -[OpenVINO™ component](../../../components/openvino) -is installed. It allows to use custom models for auto annotation. Only models in -OpenVINO™ toolkit format are supported. If you would like to annotate a -task with a custom model please convert it to the intermediate representation -(IR) format via the model optimizer tool. See [OpenVINO documentation](https://software.intel.com/en-us/articles/OpenVINO-InferEngine) for details. - -### Installation - -See the installation instructions for [the OpenVINO component](../../../components/openvino) - -### Usage - -To annotate a task with a custom model you need to prepare 4 files: -1. __Model config__ (*.xml) - a text file with network configuration. -1. __Model weights__ (*.bin) - a binary file with trained weights. -1. __Label map__ (*.json) - a simple json file with `label_map` dictionary like -object with string values for label numbers. - Example: - ```json - { - "label_map": { - "0": "background", - "1": "aeroplane", - "2": "bicycle", - "3": "bird", - "4": "boat", - "5": "bottle", - "6": "bus", - "7": "car", - "8": "cat", - "9": "chair", - "10": "cow", - "11": "diningtable", - "12": "dog", - "13": "horse", - "14": "motorbike", - "15": "person", - "16": "pottedplant", - "17": "sheep", - "18": "sofa", - "19": "train", - "20": "tvmonitor" - } - } - ``` -1. __Interpretation script__ (*.py) - a file used to convert net output layer -to a predefined structure which can be processed by CVAT. This code will be run -inside a restricted python's environment, but it's possible to use some -builtin functions like __str, int, float, max, min, range__. - - Also two variables are available in the scope: - - - __detections__ - a list of dictionaries with detections for each frame: - * __frame_id__ - frame number - * __frame_height__ - frame height - * __frame_width__ - frame width - * __detections__ - output np.ndarray (See [ExecutableNetwork.infer](https://software.intel.com/en-us/articles/OpenVINO-InferEngine#inpage-nav-11-6-3) for details). - - - __results__ - an instance of python class with converted results. - Following methods should be used to add shapes: - ```python - # xtl, ytl, xbr, ybr - expected values are float or int - # label - expected value is int - # frame_number - expected value is int - # attributes - dictionary of attribute_name: attribute_value pairs, for example {"confidence": "0.83"} - add_box(self, xtl, ytl, xbr, ybr, label, frame_number, attributes=None) - - # points - list of (x, y) pairs of float or int, for example [(57.3, 100), (67, 102.7)] - # label - expected value is int - # frame_number - expected value is int - # attributes - dictionary of attribute_name: attribute_value pairs, for example {"confidence": "0.83"} - add_points(self, points, label, frame_number, attributes=None) - add_polygon(self, points, label, frame_number, attributes=None) - add_polyline(self, points, label, frame_number, attributes=None) - ``` - -### Testing script - -CVAT comes prepackaged with a small command line helper script to help develop interpretation scripts. - -It includes a small user interface which allows users to feed in images and see the results using -the user interfaces provided by OpenCV. - -See the script and the documentation in the -[auto_annotation directory](https://github.com/opencv/cvat/tree/develop/utils/auto_annotation). - -When using the Auto Annotation runner, it is often helpful to drop into a REPL prompt to interact with the variables -directly. You can do this using the `interact` method from the `code` module. - -```python -# Import the interact method from the `code` module -from code import interact - - -for frame_results in detections: - frame_height = frame_results["frame_height"] - frame_width = frame_results["frame_width"] - frame_number = frame_results["frame_id"] - # Unsure what other data members are in the `frame_results`? Use the `interact method! - interact(local=locals()) -``` - -```bash -$ python cvat/utils/auto_annotation/run_models.py --py /path/to/myfile.py --json /path/to/mapping.json --xml /path/to/inference.xml --bin /path/to/inference.bin -Python 3.6.6 (default, Sep 26 2018, 15:10:10) -[GCC 4.2.1 Compatible Apple LLVM 10.0.0 (clang-1000.10.44.2)] on darwin -Type "help", "copyright", "credits" or "license" for more information. ->>> dir() -['__builtins__', 'frame_results', 'detections', 'frame_number', 'frame_height', 'interact', 'results', 'frame_width'] ->>> type(frame_results) - ->>> frame_results.keys() -dict_keys(['frame_id', 'frame_height', 'frame_width', 'detections']) -``` - -When using the `interact` method, make sure you are running using the _testing script_, and ensure that you _remove it_ - before submitting to the server! If you don't remove it from the server, the code runners will hang during execution, - and you'll have to restart the server to fix them. - -Another useful development method is visualizing the results using OpenCV. This will be discussed more in the -[Semantic Segmentation](#segmentation) section. - -### Examples - -#### [Person-vehicle-bike-detection-crossroad-0078](https://github.com/opencv/open_model_zoo/blob/2018/intel_models/person-vehicle-bike-detection-crossroad-0078/description/person-vehicle-bike-detection-crossroad-0078.md) (OpenVINO toolkit) - -__Links__ -- [person-vehicle-bike-detection-crossroad-0078.xml](https://download.01.org/openvinotoolkit/2018_R5/open_model_zoo/person-vehicle-bike-detection-crossroad-0078/FP32/person-vehicle-bike-detection-crossroad-0078.xml) -- [person-vehicle-bike-detection-crossroad-0078.bin](https://download.01.org/openvinotoolkit/2018_R5/open_model_zoo/person-vehicle-bike-detection-crossroad-0078/FP32/person-vehicle-bike-detection-crossroad-0078.bin) - -__Task labels__: person vehicle non-vehicle - -__label_map.json__: -```json -{ -"label_map": { - "1": "person", - "2": "vehicle", - "3": "non-vehicle" - } -} -``` -__Interpretation script for SSD based networks__: -```python -def clip(value): - return max(min(1.0, value), 0.0) - -for frame_results in detections: - frame_height = frame_results["frame_height"] - frame_width = frame_results["frame_width"] - frame_number = frame_results["frame_id"] - - for i in range(frame_results["detections"].shape[2]): - confidence = frame_results["detections"][0, 0, i, 2] - if confidence < 0.5: - continue - - results.add_box( - xtl=clip(frame_results["detections"][0, 0, i, 3]) * frame_width, - ytl=clip(frame_results["detections"][0, 0, i, 4]) * frame_height, - xbr=clip(frame_results["detections"][0, 0, i, 5]) * frame_width, - ybr=clip(frame_results["detections"][0, 0, i, 6]) * frame_height, - label=int(frame_results["detections"][0, 0, i, 1]), - frame_number=frame_number, - attributes={ - "confidence": "{:.2f}".format(confidence), - }, - ) -``` - -#### [Landmarks-regression-retail-0009](https://github.com/opencv/open_model_zoo/blob/2018/intel_models/landmarks-regression-retail-0009/description/landmarks-regression-retail-0009.md) (OpenVINO toolkit) - -__Links__ -- [landmarks-regression-retail-0009.xml](https://download.01.org/openvinotoolkit/2018_R5/open_model_zoo/landmarks-regression-retail-0009/FP32/landmarks-regression-retail-0009.xml) -- [landmarks-regression-retail-0009.bin](https://download.01.org/openvinotoolkit/2018_R5/open_model_zoo/landmarks-regression-retail-0009/FP32/landmarks-regression-retail-0009.bin) - -__Task labels__: left_eye right_eye tip_of_nose left_lip_corner right_lip_corner - -__label_map.json__: -```json -{ - "label_map": { - "0": "left_eye", - "1": "right_eye", - "2": "tip_of_nose", - "3": "left_lip_corner", - "4": "right_lip_corner" - } -} -``` -__Interpretation script__: -```python -def clip(value): - return max(min(1.0, value), 0.0) - -for frame_results in detections: - frame_height = frame_results["frame_height"] - frame_width = frame_results["frame_width"] - frame_number = frame_results["frame_id"] - - for i in range(0, frame_results["detections"].shape[1], 2): - x = frame_results["detections"][0, i, 0, 0] - y = frame_results["detections"][0, i + 1, 0, 0] - - results.add_points( - points=[(clip(x) * frame_width, clip(y) * frame_height)], - label=i // 2, # see label map and model output specification, - frame_number=frame_number, - ) -``` - -#### Semantic Segmentation - -__Links__ -- [masck_rcnn_resnet50_atrous_coco][1] (OpenvVINO toolkit) -- [CVAT Implemenation][2] - -__label_map.json__: -```json -{ -"label_map": { - "1": "person", - "2": "bicycle", - "3": "car", - } -} -``` - -Note that the above labels are not all the labels in the model! See [here](https://github.com/opencv/cvat/blob/develop/utils/open_model_zoo/mask_rcnn_inception_resnet_v2_atrous_coco/mapping.json). - -**Interpretation script for a semantic segmentation network**: -```python -import numpy as np -import cv2 -from skimage.measure import approximate_polygon, find_contours - - -for frame_results in detections: - frame_height = frame_results['frame_height'] - frame_width = frame_results['frame_width'] - frame_number = frame_results['frame_id'] - detection = frame_results['detections'] - - # The keys for the below two members will vary based on the model - masks = frame_results['masks'] - boxes = frame_results['reshape_do_2d'] - - for box_index, box in enumerate(boxes): - # Again, these indexes specific to this model - class_label = int(box[1]) - box_class_probability = box[2] - - if box_class_probability > 0.2: - xmin = box[3] * frame_width - ymin = box[4] * frame_height - xmax = box[5] * frame_width - ymax = box[6] * frame_width - - box_width = int(xmax - xmin) - box_height = int(ymin - ymax) - - # use the box index and class label index to find the appropriate mask - # note that we need to convert the class label to a zero indexed array by subtracting `1` - class_mask = masks[box_index][class_label - 1] - - # Class mask is a 33 x 33 matrix - # resize it to the bounding box - resized_mask = cv2.resize(class_mask, dsize(box_height, box_width), interpolation=cv2.INTER_CUBIC) - - # Each pixel is a probability, select every pixel above the probability threshold, 0.5 - # Do this using the boolean `>` method - boolean_mask = (resized_mask > 0.5) - - # Convert the boolean values to uint8 - uint8_mask = boolean_mask.astype(np.uint8) * 255 - - # Change the x and y coordinates into integers - xmin = int(round(xmin)) - ymin = int(round(ymin)) - xmax = xmin + box_width - ymax = ymin + box_height - - # Create an empty blank frame, so that we can get the mask polygon in frame coordinates - mask_frame = np.zeros((frame_height, frame_width), dtype=np.uint8) - - # Put the uint8_mask on the mask frame using the integer coordinates - mask_frame[xmin:xmax, ymin:ymax] = uint8_mask - - mask_probability_threshold = 0.5 - # find the contours - contours = find_contours(mask_frame, mask_probability_threshold) - # every bounding box should only have a single contour - contour = contours[0] - contour = np.flip(contour, axis=1) - - # reduce the precision on the polygon - polygon_mask = approximate_polygon(contour, tolerance=2.5) - polygon_mask = polygon_mask.tolist() - - results.add_polygon(polygon_mask, class_label, frame_number) -``` - -Note that it is sometimes hard to see or understand what is happening in a script. -Use of the computer vision module can help you visualize what is happening. - -```python -import cv2 - - -for frame_results in detections: - frame_height = frame_results['frame_height'] - frame_width = frame_results['frame_width'] - detection = frame_results['detections'] - - masks = frame_results['masks'] - boxes = frame_results['reshape_do_2d'] - - for box_index, box in enumerate(boxes): - class_label = int(box[1]) - box_class_probability = box[2] - - if box_class_probability > 0.2: - xmin = box[3] * frame_width - ymin = box[4] * frame_height - xmax = box[5] * frame_width - ymax = box[6] * frame_width - - box_width = int(xmax - xmin) - box_height = int(ymin - ymax) - - class_mask = masks[box_index][class_label - 1] - # Visualize the class mask! - cv2.imshow('class mask', class_mask) - # wait until user presses keys - cv2.waitKeys() - - boolean_mask = (resized_mask > 0.5) - uint8_mask = boolean_mask.astype(np.uint8) * 255 - - # Visualize the class mask after it's been resized! - cv2.imshow('class mask', uint8_mask) - cv2.waitKeys() -``` - -Note that you should _only_ use the above commands while running the [Auto Annotation Model Runner][3]. -Running on the server will likely require a server restart to fix. -The method `cv2.destroyAllWindows()` or `cv2.destroyWindow('your-name-here')` might be required depending on your - implementation. - -### Available interpretation scripts - -CVAT comes prepackaged with several out of the box interpretation scripts. -See them in the [open model zoo directory](https://github.com/opencv/cvat/tree/develop/utils/open_model_zoo) - -[1]: https://github.com/opencv/open_model_zoo/blob/master/models/public/mask_rcnn_resnet50_atrous_coco/model.yml -[2]: https://github.com/opencv/cvat/tree/develop/utils/open_model_zoo/mask_rcnn_inception_resnet_v2_atrous_coco -[3]: https://github.com/opencv/cvat/tree/develop/utils/auto_annotation diff --git a/cvat/apps/auto_annotation/__init__.py b/cvat/apps/auto_annotation/__init__.py deleted file mode 100644 index c929093f7019..000000000000 --- a/cvat/apps/auto_annotation/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ - -# Copyright (C) 2018-2019 Intel Corporation -# -# SPDX-License-Identifier: MIT - -default_app_config = 'cvat.apps.auto_annotation.apps.AutoAnnotationConfig' diff --git a/cvat/apps/auto_annotation/admin.py b/cvat/apps/auto_annotation/admin.py deleted file mode 100644 index da1eabb4cbb3..000000000000 --- a/cvat/apps/auto_annotation/admin.py +++ /dev/null @@ -1,15 +0,0 @@ - -# Copyright (C) 2018 Intel Corporation -# -# SPDX-License-Identifier: MIT - -from django.contrib import admin -from .models import AnnotationModel - -@admin.register(AnnotationModel) -class AnnotationModelAdmin(admin.ModelAdmin): - list_display = ('name', 'owner', 'created_date', 'updated_date', - 'shared', 'primary', 'framework') - - def has_add_permission(self, request): - return False diff --git a/cvat/apps/auto_annotation/apps.py b/cvat/apps/auto_annotation/apps.py deleted file mode 100644 index cea75abfa48c..000000000000 --- a/cvat/apps/auto_annotation/apps.py +++ /dev/null @@ -1,15 +0,0 @@ - -# Copyright (C) 2018 Intel Corporation -# -# SPDX-License-Identifier: MIT - -from django.apps import AppConfig - - -class AutoAnnotationConfig(AppConfig): - name = "cvat.apps.auto_annotation" - - def ready(self): - from .permissions import setup_permissions - - setup_permissions() diff --git a/cvat/apps/auto_annotation/image_loader.py b/cvat/apps/auto_annotation/image_loader.py deleted file mode 100644 index 17335beba13e..000000000000 --- a/cvat/apps/auto_annotation/image_loader.py +++ /dev/null @@ -1,22 +0,0 @@ - -# Copyright (C) 2018 Intel Corporation -# -# SPDX-License-Identifier: MIT - -import cv2 -import numpy as np - -class ImageLoader(): - def __init__(self, frame_provider): - self._frame_provider = frame_provider - - def __iter__(self): - for frame, _ in self._frame_provider.get_frames(self._frame_provider.Quality.ORIGINAL): - yield self._load_image(frame) - - def __len__(self): - return len(self._frame_provider) - - @staticmethod - def _load_image(image): - return cv2.imdecode(np.fromstring(image.read(), np.uint8), cv2.IMREAD_COLOR) diff --git a/cvat/apps/auto_annotation/inference.py b/cvat/apps/auto_annotation/inference.py deleted file mode 100644 index b51cc10e8fc8..000000000000 --- a/cvat/apps/auto_annotation/inference.py +++ /dev/null @@ -1,163 +0,0 @@ -import itertools -from .model_loader import ModelLoader -from cvat.apps.engine.utils import import_modules, execute_python_code - -def _process_detections(detections, path_to_conv_script, restricted=True): - results = Results() - local_vars = { - "detections": detections, - "results": results, - } - source_code = open(path_to_conv_script).read() - - if restricted: - global_vars = { - "__builtins__": { - "str": str, - "int": int, - "float": float, - "max": max, - "min": min, - "range": range, - }, - } - else: - global_vars = globals() - imports = import_modules(source_code) - global_vars.update(imports) - - - execute_python_code(source_code, global_vars, local_vars) - - return results - -def _process_attributes(shape_attributes, label_attr_spec): - attributes = [] - for attr_text, attr_value in shape_attributes.items(): - if attr_text in label_attr_spec: - attributes.append({ - "spec_id": label_attr_spec[attr_text], - "value": attr_value, - }) - - return attributes - -class Results(): - def __init__(self): - self._results = { - "shapes": [], - "tracks": [] - } - - # https://stackoverflow.com/a/50928627/2701402 - def add_box(self, xtl: float, ytl: float, xbr: float, ybr: float, label: int, frame_number: int, attributes: dict=None): - """ - xtl - x coordinate, top left - ytl - y coordinate, top left - xbr - x coordinate, bottom right - ybr - y coordinate, bottom right - """ - self.get_shapes().append({ - "label": label, - "frame": frame_number, - "points": [xtl, ytl, xbr, ybr], - "type": "rectangle", - "attributes": attributes or {}, - }) - - def add_points(self, points: list, label: int, frame_number: int, attributes: dict=None): - points = self._create_polyshape(points, label, frame_number, attributes) - points["type"] = "points" - self.get_shapes().append(points) - - def add_polygon(self, points: list, label: int, frame_number: int, attributes: dict=None): - polygon = self._create_polyshape(points, label, frame_number, attributes) - polygon["type"] = "polygon" - self.get_shapes().append(polygon) - - def add_polyline(self, points: list, label: int, frame_number: int, attributes: dict=None): - polyline = self._create_polyshape(points, label, frame_number, attributes) - polyline["type"] = "polyline" - self.get_shapes().append(polyline) - - def get_shapes(self): - return self._results["shapes"] - - def get_tracks(self): - return self._results["tracks"] - - @staticmethod - def _create_polyshape(points: list, label: int, frame_number: int, attributes: dict=None): - return { - "label": label, - "frame": frame_number, - "points": list(itertools.chain.from_iterable(points)), - "attributes": attributes or {}, - } - -class InferenceAnnotationRunner: - def __init__(self, data, model_file, weights_file, labels_mapping, - attribute_spec, convertation_file): - self.data = iter(data) - self.data_len = len(data) - self.model = ModelLoader(model=model_file, weights=weights_file) - self.frame_counter = 0 - self.attribute_spec = attribute_spec - self.convertation_file = convertation_file - self.iteration_size = 128 - self.labels_mapping = labels_mapping - - - def run(self, job=None, update_progress=None, restricted=True): - result = { - "shapes": [], - "tracks": [], - "tags": [], - "version": 0 - } - - detections = [] - for _ in range(self.iteration_size): - try: - frame = next(self.data) - except StopIteration: - break - - orig_rows, orig_cols = frame.shape[:2] - - detections.append({ - "frame_id": self.frame_counter, - "frame_height": orig_rows, - "frame_width": orig_cols, - "detections": self.model.infer(frame), - }) - - self.frame_counter += 1 - if job and update_progress and not update_progress(job, self.frame_counter * 100 / self.data_len): - return None, False - - processed_detections = _process_detections(detections, self.convertation_file, restricted=restricted) - - self._add_shapes(processed_detections.get_shapes(), result["shapes"]) - - more_items = self.frame_counter != self.data_len - - return result, more_items - - def _add_shapes(self, shapes, target_container): - for shape in shapes: - if shape["label"] not in self.labels_mapping: - continue - - db_label = self.labels_mapping[shape["label"]] - label_attr_spec = self.attribute_spec.get(db_label) - target_container.append({ - "label_id": db_label, - "frame": shape["frame"], - "points": shape["points"], - "type": shape["type"], - "z_order": 0, - "group": None, - "occluded": False, - "attributes": _process_attributes(shape["attributes"], label_attr_spec), - }) diff --git a/cvat/apps/auto_annotation/inference_engine.py b/cvat/apps/auto_annotation/inference_engine.py deleted file mode 100644 index 766c0eb0cc77..000000000000 --- a/cvat/apps/auto_annotation/inference_engine.py +++ /dev/null @@ -1,52 +0,0 @@ -# Copyright (C) 2018 Intel Corporation -# -# SPDX-License-Identifier: MIT - -from openvino.inference_engine import IENetwork, IEPlugin, IECore, get_version - -import subprocess -import os -import platform - -_IE_PLUGINS_PATH = os.getenv("IE_PLUGINS_PATH", None) - -def _check_instruction(instruction): - return instruction == str.strip( - subprocess.check_output( - 'lscpu | grep -o "{}" | head -1'.format(instruction), shell=True - ).decode('utf-8') - ) - - -def make_plugin_or_core(): - version = get_version() - use_core_openvino = False - try: - major, minor, reference = [int(x) for x in version.split('.')] - if major >= 2 and minor >= 1: - use_core_openvino = True - except Exception: - pass - - if use_core_openvino: - ie = IECore() - return ie - - if _IE_PLUGINS_PATH is None: - raise OSError('Inference engine plugin path env not found in the system.') - - plugin = IEPlugin(device='CPU', plugin_dirs=[_IE_PLUGINS_PATH]) - if (_check_instruction('avx2')): - plugin.add_cpu_extension(os.path.join(_IE_PLUGINS_PATH, 'libcpu_extension_avx2.so')) - elif (_check_instruction('sse4')): - plugin.add_cpu_extension(os.path.join(_IE_PLUGINS_PATH, 'libcpu_extension_sse4.so')) - elif platform.system() == 'Darwin': - plugin.add_cpu_extension(os.path.join(_IE_PLUGINS_PATH, 'libcpu_extension.dylib')) - else: - raise Exception('Inference engine requires a support of avx2 or sse4.') - - return plugin - - -def make_network(model, weights): - return IENetwork(model = model, weights = weights) diff --git a/cvat/apps/auto_annotation/migrations/0001_initial.py b/cvat/apps/auto_annotation/migrations/0001_initial.py deleted file mode 100644 index ebd8a6e1a24e..000000000000 --- a/cvat/apps/auto_annotation/migrations/0001_initial.py +++ /dev/null @@ -1,39 +0,0 @@ -# Generated by Django 2.1.3 on 2019-01-24 14:05 - -import cvat.apps.auto_annotation.models -from django.conf import settings -import django.core.files.storage -from django.db import migrations, models -import django.db.models.deletion - - -class Migration(migrations.Migration): - - initial = True - - dependencies = [ - migrations.swappable_dependency(settings.AUTH_USER_MODEL), - ] - - operations = [ - migrations.CreateModel( - name='AnnotationModel', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('name', cvat.apps.auto_annotation.models.SafeCharField(max_length=256)), - ('created_date', models.DateTimeField(auto_now_add=True)), - ('updated_date', models.DateTimeField(auto_now_add=True)), - ('model_file', models.FileField(storage=django.core.files.storage.FileSystemStorage(), upload_to=cvat.apps.auto_annotation.models.upload_path_handler)), - ('weights_file', models.FileField(storage=django.core.files.storage.FileSystemStorage(), upload_to=cvat.apps.auto_annotation.models.upload_path_handler)), - ('labelmap_file', models.FileField(storage=django.core.files.storage.FileSystemStorage(), upload_to=cvat.apps.auto_annotation.models.upload_path_handler)), - ('interpretation_file', models.FileField(storage=django.core.files.storage.FileSystemStorage(), upload_to=cvat.apps.auto_annotation.models.upload_path_handler)), - ('shared', models.BooleanField(default=False)), - ('primary', models.BooleanField(default=False)), - ('framework', models.CharField(default=cvat.apps.auto_annotation.models.FrameworkChoice('openvino'), max_length=32)), - ('owner', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to=settings.AUTH_USER_MODEL)), - ], - options={ - 'default_permissions': (), - }, - ), - ] diff --git a/cvat/apps/auto_annotation/migrations/__init__.py b/cvat/apps/auto_annotation/migrations/__init__.py deleted file mode 100644 index d8e62e54b356..000000000000 --- a/cvat/apps/auto_annotation/migrations/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ - -# Copyright (C) 2018 Intel Corporation -# -# SPDX-License-Identifier: MIT - diff --git a/cvat/apps/auto_annotation/model_loader.py b/cvat/apps/auto_annotation/model_loader.py deleted file mode 100644 index e48d5c8e4d1a..000000000000 --- a/cvat/apps/auto_annotation/model_loader.py +++ /dev/null @@ -1,76 +0,0 @@ - -# Copyright (C) 2018-2019 Intel Corporation -# -# SPDX-License-Identifier: MIT - -import json -import cv2 -import os -import numpy as np - -from cvat.apps.auto_annotation.inference_engine import make_plugin_or_core, make_network - -class ModelLoader(): - def __init__(self, model, weights): - self._model = model - self._weights = weights - - core_or_plugin = make_plugin_or_core() - network = make_network(self._model, self._weights) - - if getattr(core_or_plugin, 'get_supported_layers', False): - supported_layers = core_or_plugin.get_supported_layers(network) - not_supported_layers = [l for l in network.layers.keys() if l not in supported_layers] - if len(not_supported_layers) != 0: - raise Exception("Following layers are not supported by the plugin for specified device {}:\n {}". - format(core_or_plugin.device, ", ".join(not_supported_layers))) - - iter_inputs = iter(network.inputs) - self._input_blob_name = next(iter_inputs) - self._input_info_name = '' - self._output_blob_name = next(iter(network.outputs)) - - self._require_image_info = False - - info_names = ('image_info', 'im_info') - - # NOTE: handeling for the inclusion of `image_info` in OpenVino2019 - if any(s in network.inputs for s in info_names): - self._require_image_info = True - self._input_info_name = set(network.inputs).intersection(info_names) - self._input_info_name = self._input_info_name.pop() - if self._input_blob_name in info_names: - self._input_blob_name = next(iter_inputs) - - if getattr(core_or_plugin, 'load_network', False): - self._net = core_or_plugin.load_network(network, - "CPU", - num_requests=2) - else: - self._net = core_or_plugin.load(network=network, num_requests=2) - input_type = network.inputs[self._input_blob_name] - self._input_layout = input_type if isinstance(input_type, list) else input_type.shape - - def infer(self, image): - _, _, h, w = self._input_layout - in_frame = image if image.shape[:-1] == (h, w) else cv2.resize(image, (w, h)) - in_frame = in_frame.transpose((2, 0, 1)) # Change data layout from HWC to CHW - inputs = {self._input_blob_name: in_frame} - if self._require_image_info: - info = np.zeros([1, 3]) - info[0, 0] = h - info[0, 1] = w - # frame number - info[0, 2] = 1 - inputs[self._input_info_name] = info - - results = self._net.infer(inputs) - if len(results) == 1: - return results[self._output_blob_name].copy() - else: - return results.copy() - - -def load_labelmap(labels_path): - with open(labels_path, "r") as f: - return json.load(f)["label_map"] diff --git a/cvat/apps/auto_annotation/model_manager.py b/cvat/apps/auto_annotation/model_manager.py deleted file mode 100644 index 7bac221a0e69..000000000000 --- a/cvat/apps/auto_annotation/model_manager.py +++ /dev/null @@ -1,276 +0,0 @@ -# Copyright (C) 2018-2020 Intel Corporation -# -# SPDX-License-Identifier: MIT - -import django_rq -import numpy as np -import os -import rq -import shutil -import tempfile - -from django.db import transaction -from django.utils import timezone -from django.conf import settings - -from cvat.apps.engine.log import slogger -from cvat.apps.engine.models import Task as TaskModel -from cvat.apps.authentication.auth import has_admin_role -from cvat.apps.engine.serializers import LabeledDataSerializer -from cvat.apps.dataset_manager.task import put_task_data, patch_task_data -from cvat.apps.engine.frame_provider import FrameProvider -from cvat.apps.engine.utils import av_scan_paths - -from .models import AnnotationModel, FrameworkChoice -from .model_loader import load_labelmap -from .image_loader import ImageLoader -from .inference import InferenceAnnotationRunner - - -def _remove_old_file(model_file_field): - if model_file_field and os.path.exists(model_file_field.name): - os.remove(model_file_field.name) - -def _update_dl_model_thread(dl_model_id, name, is_shared, model_file, weights_file, labelmap_file, - interpretation_file, run_tests, is_local_storage, delete_if_test_fails, restricted=True): - def _get_file_content(filename): - return os.path.basename(filename), open(filename, "rb") - - def _delete_source_files(): - for f in [model_file, weights_file, labelmap_file, interpretation_file]: - if f: - os.remove(f) - - def _run_test(model_file, weights_file, labelmap_file, interpretation_file): - test_image = np.ones((1024, 1980, 3), np.uint8) * 255 - try: - dummy_labelmap = {key: key for key in load_labelmap(labelmap_file).keys()} - runner = InferenceAnnotationRunner( - data=[test_image,], - model_file=model_file, - weights_file=weights_file, - labels_mapping=dummy_labelmap, - attribute_spec={}, - convertation_file=interpretation_file) - - runner.run(restricted=restricted) - except Exception as e: - return False, str(e) - - return True, "" - - job = rq.get_current_job() - job.meta["progress"] = "Saving data" - job.save_meta() - - with transaction.atomic(): - dl_model = AnnotationModel.objects.select_for_update().get(pk=dl_model_id) - - test_res = True - message = "" - if run_tests: - job.meta["progress"] = "Test started" - job.save_meta() - - test_res, message = _run_test( - model_file=model_file or dl_model.model_file.name, - weights_file=weights_file or dl_model.weights_file.name, - labelmap_file=labelmap_file or dl_model.labelmap_file.name, - interpretation_file=interpretation_file or dl_model.interpretation_file.name, - ) - - if not test_res: - job.meta["progress"] = "Test failed" - if delete_if_test_fails: - shutil.rmtree(dl_model.get_dirname(), ignore_errors=True) - dl_model.delete() - else: - job.meta["progress"] = "Test passed" - job.save_meta() - - # update DL model - if test_res: - if model_file: - _remove_old_file(dl_model.model_file) - dl_model.model_file.save(*_get_file_content(model_file)) - if weights_file: - _remove_old_file(dl_model.weights_file) - dl_model.weights_file.save(*_get_file_content(weights_file)) - if labelmap_file: - _remove_old_file(dl_model.labelmap_file) - dl_model.labelmap_file.save(*_get_file_content(labelmap_file)) - if interpretation_file: - _remove_old_file(dl_model.interpretation_file) - dl_model.interpretation_file.save(*_get_file_content(interpretation_file)) - - if name: - dl_model.name = name - - if is_shared != None: - dl_model.shared = is_shared - - dl_model.updated_date = timezone.now() - dl_model.save() - - if is_local_storage: - _delete_source_files() - - if not test_res: - raise Exception("Model was not properly created/updated. Test failed: {}".format(message)) - -def create_or_update(dl_model_id, name, model_file, weights_file, labelmap_file, interpretation_file, owner, storage, is_shared): - def get_abs_path(share_path): - if not share_path: - return share_path - share_root = settings.SHARE_ROOT - relpath = os.path.normpath(share_path).lstrip('/') - if '..' in relpath.split(os.path.sep): - raise Exception('Permission denied') - abspath = os.path.abspath(os.path.join(share_root, relpath)) - if os.path.commonprefix([share_root, abspath]) != share_root: - raise Exception('Bad file path on share: ' + abspath) - return abspath - - def save_file_as_tmp(data): - if not data: - return None - fd, filename = tempfile.mkstemp() - with open(filename, 'wb') as tmp_file: - for chunk in data.chunks(): - tmp_file.write(chunk) - os.close(fd) - return filename - - is_create_request = dl_model_id is None - if is_create_request: - dl_model_id = create_empty(owner=owner) - - run_tests = bool(model_file or weights_file or labelmap_file or interpretation_file) - if storage != "local": - model_file = get_abs_path(model_file) - weights_file = get_abs_path(weights_file) - labelmap_file = get_abs_path(labelmap_file) - interpretation_file = get_abs_path(interpretation_file) - else: - model_file = save_file_as_tmp(model_file) - weights_file = save_file_as_tmp(weights_file) - labelmap_file = save_file_as_tmp(labelmap_file) - interpretation_file = save_file_as_tmp(interpretation_file) - - files_to_scan = [] - if model_file: - files_to_scan.append(model_file) - if weights_file: - files_to_scan.append(weights_file) - if labelmap_file: - files_to_scan.append(labelmap_file) - if interpretation_file: - files_to_scan.append(interpretation_file) - av_scan_paths(*files_to_scan) - - if owner: - restricted = not has_admin_role(owner) - else: - restricted = not has_admin_role(AnnotationModel.objects.get(pk=dl_model_id).owner) - - rq_id = "auto_annotation.create.{}".format(dl_model_id) - queue = django_rq.get_queue("default") - queue.enqueue_call( - func=_update_dl_model_thread, - args=( - dl_model_id, - name, - is_shared, - model_file, - weights_file, - labelmap_file, - interpretation_file, - run_tests, - storage == "local", - is_create_request, - restricted - ), - job_id=rq_id - ) - - return rq_id - -@transaction.atomic -def create_empty(owner, framework=FrameworkChoice.OPENVINO): - db_model = AnnotationModel( - owner=owner, - ) - db_model.save() - - model_path = db_model.get_dirname() - if os.path.isdir(model_path): - shutil.rmtree(model_path) - os.mkdir(model_path) - - return db_model.id - -@transaction.atomic -def delete(dl_model_id): - dl_model = AnnotationModel.objects.select_for_update().get(pk=dl_model_id) - if dl_model: - if dl_model.primary: - raise Exception("Can not delete primary model {}".format(dl_model_id)) - - shutil.rmtree(dl_model.get_dirname(), ignore_errors=True) - dl_model.delete() - else: - raise Exception("Requested DL model {} doesn't exist".format(dl_model_id)) - -def run_inference_thread(tid, model_file, weights_file, labels_mapping, attributes, convertation_file, reset, user, restricted=True): - def update_progress(job, progress): - job.refresh() - if "cancel" in job.meta: - del job.meta["cancel"] - job.save() - return False - job.meta["progress"] = progress - job.save_meta() - return True - - try: - job = rq.get_current_job() - job.meta["progress"] = 0 - job.save_meta() - db_task = TaskModel.objects.get(pk=tid) - - result = None - slogger.glob.info("auto annotation with openvino toolkit for task {}".format(tid)) - more_data = True - runner = InferenceAnnotationRunner( - data=ImageLoader(FrameProvider(db_task.data)), - model_file=model_file, - weights_file=weights_file, - labels_mapping=labels_mapping, - attribute_spec=attributes, - convertation_file= convertation_file) - while more_data: - result, more_data = runner.run( - job=job, - update_progress=update_progress, - restricted=restricted) - - if result is None: - slogger.glob.info("auto annotation for task {} canceled by user".format(tid)) - return - - serializer = LabeledDataSerializer(data = result) - if serializer.is_valid(raise_exception=True): - if reset: - put_task_data(tid, result) - else: - patch_task_data(tid, result, "create") - - slogger.glob.info("auto annotation for task {} done".format(tid)) - except Exception as e: - try: - slogger.task[tid].exception("exception was occurred during auto annotation of the task", exc_info=True) - except Exception as ex: - slogger.glob.exception("exception was occurred during auto annotation of the task {}: {}".format(tid, str(ex)), exc_info=True) - raise ex - - raise e diff --git a/cvat/apps/auto_annotation/models.py b/cvat/apps/auto_annotation/models.py deleted file mode 100644 index 467997e0f9b2..000000000000 --- a/cvat/apps/auto_annotation/models.py +++ /dev/null @@ -1,56 +0,0 @@ - -# Copyright (C) 2018 Intel Corporation -# -# SPDX-License-Identifier: MIT - -import os -from enum import Enum - -from django.db import models -from django.conf import settings -from django.contrib.auth.models import User -from django.core.files.storage import FileSystemStorage - -fs = FileSystemStorage() - -def upload_path_handler(instance, filename): - return os.path.join(settings.MODELS_ROOT, str(instance.id), filename) - -class FrameworkChoice(Enum): - OPENVINO = 'openvino' - TENSORFLOW = 'tensorflow' - PYTORCH = 'pytorch' - - def __str__(self): - return self.value - - -class SafeCharField(models.CharField): - def get_prep_value(self, value): - value = super().get_prep_value(value) - if value: - return value[:self.max_length] - return value - -class AnnotationModel(models.Model): - name = SafeCharField(max_length=256) - owner = models.ForeignKey(User, null=True, blank=True, - on_delete=models.SET_NULL) - created_date = models.DateTimeField(auto_now_add=True) - updated_date = models.DateTimeField(auto_now_add=True) - model_file = models.FileField(upload_to=upload_path_handler, storage=fs) - weights_file = models.FileField(upload_to=upload_path_handler, storage=fs) - labelmap_file = models.FileField(upload_to=upload_path_handler, storage=fs) - interpretation_file = models.FileField(upload_to=upload_path_handler, storage=fs) - shared = models.BooleanField(default=False) - primary = models.BooleanField(default=False) - framework = models.CharField(max_length=32, default=FrameworkChoice.OPENVINO) - - class Meta: - default_permissions = () - - def get_dirname(self): - return "{models_root}/{id}".format(models_root=settings.MODELS_ROOT, id=self.id) - - def __str__(self): - return self.name diff --git a/cvat/apps/auto_annotation/permissions.py b/cvat/apps/auto_annotation/permissions.py deleted file mode 100644 index ede8d611248a..000000000000 --- a/cvat/apps/auto_annotation/permissions.py +++ /dev/null @@ -1,29 +0,0 @@ -# Copyright (C) 2018 Intel Corporation -# -# SPDX-License-Identifier: MIT - -import rules - -from cvat.apps.authentication.auth import has_admin_role, has_user_role - -@rules.predicate -def is_model_owner(db_user, db_dl_model): - return db_dl_model.owner == db_user - -@rules.predicate -def is_shared_model(_, db_dl_model): - return db_dl_model.shared - -@rules.predicate -def is_primary_model(_, db_dl_model): - return db_dl_model.primary - -def setup_permissions(): - rules.add_perm('auto_annotation.model.create', has_admin_role | has_user_role) - - rules.add_perm('auto_annotation.model.update', (has_admin_role | is_model_owner) & ~is_primary_model) - - rules.add_perm('auto_annotation.model.delete', (has_admin_role | is_model_owner) & ~is_primary_model) - - rules.add_perm('auto_annotation.model.access', has_admin_role | is_model_owner | - is_shared_model | is_primary_model) diff --git a/cvat/apps/auto_annotation/tests.py b/cvat/apps/auto_annotation/tests.py deleted file mode 100644 index a59acdef3783..000000000000 --- a/cvat/apps/auto_annotation/tests.py +++ /dev/null @@ -1,4 +0,0 @@ - -# Copyright (C) 2018 Intel Corporation -# -# SPDX-License-Identifier: MIT diff --git a/cvat/apps/auto_annotation/urls.py b/cvat/apps/auto_annotation/urls.py deleted file mode 100644 index 2aa75c5e664a..000000000000 --- a/cvat/apps/auto_annotation/urls.py +++ /dev/null @@ -1,19 +0,0 @@ - -# Copyright (C) 2018 Intel Corporation -# -# SPDX-License-Identifier: MIT - -from django.urls import path -from . import views - -urlpatterns = [ - path("create", views.create_model), - path("update/", views.update_model), - path("delete/", views.delete_model), - - path("start//", views.start_annotation), - path("check/", views.check), - path("cancel/", views.cancel), - - path("meta/get", views.get_meta_info), -] diff --git a/cvat/apps/auto_annotation/views.py b/cvat/apps/auto_annotation/views.py deleted file mode 100644 index c521424dfde9..000000000000 --- a/cvat/apps/auto_annotation/views.py +++ /dev/null @@ -1,265 +0,0 @@ -# Copyright (C) 2018 Intel Corporation -# -# SPDX-License-Identifier: MIT - -import django_rq -import json -import os - -from django.http import HttpResponse, JsonResponse, HttpResponseBadRequest -from rest_framework.decorators import api_view -from django.db.models import Q -from rules.contrib.views import permission_required, objectgetter - -from cvat.apps.authentication.decorators import login_required -from cvat.apps.engine.models import Task as TaskModel -from cvat.apps.authentication.auth import has_admin_role -from cvat.apps.engine.log import slogger - -from .model_loader import load_labelmap -from . import model_manager -from .models import AnnotationModel - -@login_required -@permission_required(perm=["engine.task.change"], - fn=objectgetter(TaskModel, "tid"), raise_exception=True) -def cancel(request, tid): - try: - queue = django_rq.get_queue("low") - job = queue.fetch_job("auto_annotation.run.{}".format(tid)) - if job is None or job.is_finished or job.is_failed: - raise Exception("Task is not being annotated currently") - elif "cancel" not in job.meta: - job.meta["cancel"] = True - job.save() - - except Exception as ex: - try: - slogger.task[tid].exception("cannot cancel auto annotation for task #{}".format(tid), exc_info=True) - except Exception as logger_ex: - slogger.glob.exception("exception was occured during cancel auto annotation request for task {}: {}".format(tid, str(logger_ex)), exc_info=True) - return HttpResponseBadRequest(str(ex)) - - return HttpResponse() - -@login_required -@permission_required(perm=["auto_annotation.model.create"], raise_exception=True) -def create_model(request): - if request.method != 'POST': - return HttpResponseBadRequest("Only POST requests are accepted") - - try: - params = request.POST - storage = params["storage"] - name = params["name"] - is_shared = params["shared"].lower() == "true" - if is_shared and not has_admin_role(request.user): - raise Exception("Only admin can create shared models") - - files = request.FILES if storage == "local" else params - model = files["xml"] - weights = files["bin"] - labelmap = files["json"] - interpretation_script = files["py"] - owner = request.user - - rq_id = model_manager.create_or_update( - dl_model_id=None, - name=name, - model_file=model, - weights_file=weights, - labelmap_file=labelmap, - interpretation_file=interpretation_script, - owner=owner, - storage=storage, - is_shared=is_shared, - ) - - return JsonResponse({"id": rq_id}) - except Exception as e: - return HttpResponseBadRequest(str(e)) - -@login_required -@permission_required(perm=["auto_annotation.model.update"], - fn=objectgetter(AnnotationModel, "mid"), raise_exception=True) -def update_model(request, mid): - if request.method != 'POST': - return HttpResponseBadRequest("Only POST requests are accepted") - - try: - params = request.POST - storage = params["storage"] - name = params.get("name") - is_shared = params.get("shared") - is_shared = is_shared.lower() == "true" if is_shared else None - if is_shared and not has_admin_role(request.user): - raise Exception("Only admin can create shared models") - files = request.FILES - model = files.get("xml") - weights = files.get("bin") - labelmap = files.get("json") - interpretation_script = files.get("py") - - rq_id = model_manager.create_or_update( - dl_model_id=mid, - name=name, - model_file=model, - weights_file=weights, - labelmap_file=labelmap, - interpretation_file=interpretation_script, - owner=None, - storage=storage, - is_shared=is_shared, - ) - - return JsonResponse({"id": rq_id}) - except Exception as e: - return HttpResponseBadRequest(str(e)) - -@login_required -@permission_required(perm=["auto_annotation.model.delete"], - fn=objectgetter(AnnotationModel, "mid"), raise_exception=True) -def delete_model(request, mid): - if request.method != 'DELETE': - return HttpResponseBadRequest("Only DELETE requests are accepted") - model_manager.delete(mid) - return HttpResponse() - -@api_view(['POST']) -@login_required -def get_meta_info(request): - try: - tids = request.data - response = { - "admin": has_admin_role(request.user), - "models": [], - "run": {}, - } - dl_model_list = list(AnnotationModel.objects.filter(Q(owner=request.user) | Q(primary=True) | Q(shared=True)).order_by('-created_date')) - for dl_model in dl_model_list: - labels = [] - if dl_model.labelmap_file and os.path.exists(dl_model.labelmap_file.name): - with dl_model.labelmap_file.open('r') as f: - labels = list(json.load(f)["label_map"].values()) - - response["models"].append({ - "id": dl_model.id, - "name": dl_model.name, - "primary": dl_model.primary, - "uploadDate": dl_model.created_date, - "updateDate": dl_model.updated_date, - "labels": labels, - "owner": dl_model.owner.id, - }) - - queue = django_rq.get_queue("low") - for tid in tids: - rq_id = "auto_annotation.run.{}".format(tid) - job = queue.fetch_job(rq_id) - if job is not None: - response["run"][tid] = { - "status": job.get_status(), - "rq_id": rq_id, - } - - return JsonResponse(response) - except Exception as e: - return HttpResponseBadRequest(str(e)) - -@login_required -@permission_required(perm=["engine.task.change"], - fn=objectgetter(TaskModel, "tid"), raise_exception=True) -@permission_required(perm=["auto_annotation.model.access"], - fn=objectgetter(AnnotationModel, "mid"), raise_exception=True) -def start_annotation(request, mid, tid): - slogger.glob.info("auto annotation create request for task {} via DL model {}".format(tid, mid)) - try: - db_task = TaskModel.objects.get(pk=tid) - queue = django_rq.get_queue("low") - job = queue.fetch_job("auto_annotation.run.{}".format(tid)) - if job is not None and (job.is_started or job.is_queued): - raise Exception("The process is already running") - - data = json.loads(request.body.decode('utf-8')) - - should_reset = data["reset"] - user_defined_labels_mapping = data["labels"] - - dl_model = AnnotationModel.objects.get(pk=mid) - - model_file_path = dl_model.model_file.name - weights_file_path = dl_model.weights_file.name - labelmap_file = dl_model.labelmap_file.name - convertation_file_path = dl_model.interpretation_file.name - restricted = not has_admin_role(dl_model.owner) - - db_labels = db_task.label_set.prefetch_related("attributespec_set").all() - db_attributes = {db_label.id: - {db_attr.name: db_attr.id for db_attr in db_label.attributespec_set.all()} for db_label in db_labels} - db_labels = {db_label.name:db_label.id for db_label in db_labels} - - model_labels = {value: key for key, value in load_labelmap(labelmap_file).items()} - - labels_mapping = {} - for user_model_label, user_db_label in user_defined_labels_mapping.items(): - if user_model_label in model_labels and user_db_label in db_labels: - labels_mapping[int(model_labels[user_model_label])] = db_labels[user_db_label] - - if not labels_mapping: - raise Exception("No labels found for annotation") - - rq_id="auto_annotation.run.{}".format(tid) - queue.enqueue_call(func=model_manager.run_inference_thread, - args=( - tid, - model_file_path, - weights_file_path, - labels_mapping, - db_attributes, - convertation_file_path, - should_reset, - request.user, - restricted, - ), - job_id = rq_id, - timeout=604800) # 7 days - - slogger.task[tid].info("auto annotation job enqueued") - - except Exception as ex: - try: - slogger.task[tid].exception("exception was occurred during annotation request", exc_info=True) - except Exception as logger_ex: - slogger.glob.exception("exception was occurred during create auto annotation request for task {}: {}".format(tid, str(logger_ex)), exc_info=True) - return HttpResponseBadRequest(str(ex)) - - return JsonResponse({"id": rq_id}) - -@login_required -def check(request, rq_id): - try: - target_queue = "low" if "auto_annotation.run" in rq_id else "default" - queue = django_rq.get_queue(target_queue) - job = queue.fetch_job(rq_id) - if job is not None and "cancel" in job.meta: - return JsonResponse({"status": "finished"}) - data = {} - if job is None: - data["status"] = "unknown" - elif job.is_queued: - data["status"] = "queued" - elif job.is_started: - data["status"] = "started" - data["progress"] = job.meta["progress"] if "progress" in job.meta else "" - elif job.is_finished: - data["status"] = "finished" - job.delete() - else: - data["status"] = "failed" - data["error"] = job.exc_info - job.delete() - - except Exception: - data["status"] = "unknown" - - return JsonResponse(data) diff --git a/cvat/apps/auto_segmentation/__init__.py b/cvat/apps/auto_segmentation/__init__.py deleted file mode 100644 index a0fca4cb39ea..000000000000 --- a/cvat/apps/auto_segmentation/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ - -# Copyright (C) 2018-2019 Intel Corporation -# -# SPDX-License-Identifier: MIT diff --git a/cvat/apps/auto_segmentation/admin.py b/cvat/apps/auto_segmentation/admin.py deleted file mode 100644 index 3c40ebdfe118..000000000000 --- a/cvat/apps/auto_segmentation/admin.py +++ /dev/null @@ -1,8 +0,0 @@ - -# Copyright (C) 2018 Intel Corporation -# -# SPDX-License-Identifier: MIT - - -# Register your models here. - diff --git a/cvat/apps/auto_segmentation/apps.py b/cvat/apps/auto_segmentation/apps.py deleted file mode 100644 index 03322710f457..000000000000 --- a/cvat/apps/auto_segmentation/apps.py +++ /dev/null @@ -1,11 +0,0 @@ - -# Copyright (C) 2018 Intel Corporation -# -# SPDX-License-Identifier: MIT - -from django.apps import AppConfig - - -class AutoSegmentationConfig(AppConfig): - name = 'auto_segmentation' - diff --git a/cvat/apps/auto_segmentation/migrations/__init__.py b/cvat/apps/auto_segmentation/migrations/__init__.py deleted file mode 100644 index d8e62e54b356..000000000000 --- a/cvat/apps/auto_segmentation/migrations/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ - -# Copyright (C) 2018 Intel Corporation -# -# SPDX-License-Identifier: MIT - diff --git a/cvat/apps/auto_segmentation/models.py b/cvat/apps/auto_segmentation/models.py deleted file mode 100644 index 37401bdd2207..000000000000 --- a/cvat/apps/auto_segmentation/models.py +++ /dev/null @@ -1,8 +0,0 @@ - -# Copyright (C) 2018 Intel Corporation -# -# SPDX-License-Identifier: MIT - - -# Create your models here. - diff --git a/cvat/apps/auto_segmentation/tests.py b/cvat/apps/auto_segmentation/tests.py deleted file mode 100644 index d20a46ab6a66..000000000000 --- a/cvat/apps/auto_segmentation/tests.py +++ /dev/null @@ -1,8 +0,0 @@ - -# Copyright (C) 2018 Intel Corporation -# -# SPDX-License-Identifier: MIT - - -# Create your tests here. - diff --git a/cvat/apps/auto_segmentation/urls.py b/cvat/apps/auto_segmentation/urls.py deleted file mode 100644 index f84019be9693..000000000000 --- a/cvat/apps/auto_segmentation/urls.py +++ /dev/null @@ -1,14 +0,0 @@ - -# Copyright (C) 2018 Intel Corporation -# -# SPDX-License-Identifier: MIT - -from django.urls import path -from . import views - -urlpatterns = [ - path('create/task/', views.create), - path('check/task/', views.check), - path('cancel/task/', views.cancel), - path('meta/get', views.get_meta_info), -] diff --git a/cvat/apps/auto_segmentation/views.py b/cvat/apps/auto_segmentation/views.py deleted file mode 100644 index 4b15b094f29d..000000000000 --- a/cvat/apps/auto_segmentation/views.py +++ /dev/null @@ -1,310 +0,0 @@ - -# Copyright (C) 2018-2020 Intel Corporation -# -# SPDX-License-Identifier: MIT - - -from django.http import HttpResponse, JsonResponse, HttpResponseBadRequest -from rest_framework.decorators import api_view -from rules.contrib.views import permission_required, objectgetter -from cvat.apps.authentication.decorators import login_required -from cvat.apps.dataset_manager.task import put_task_data -from cvat.apps.engine.models import Task as TaskModel -from cvat.apps.engine.serializers import LabeledDataSerializer -from cvat.apps.engine.frame_provider import FrameProvider - -import django_rq -import os -import rq - -import numpy as np - -from cvat.apps.engine.log import slogger - -import sys -import skimage.io -from skimage.measure import find_contours, approximate_polygon - -def run_tensorflow_auto_segmentation(frame_provider, labels_mapping, treshold): - def _convert_to_int(boolean_mask): - return boolean_mask.astype(np.uint8) - - def _convert_to_segmentation(mask): - contours = find_contours(mask, 0.5) - # only one contour exist in our case - contour = contours[0] - contour = np.flip(contour, axis=1) - # Approximate the contour and reduce the number of points - contour = approximate_polygon(contour, tolerance=2.5) - segmentation = contour.ravel().tolist() - return segmentation - - ## INITIALIZATION - - # workarround for tf.placeholder() is not compatible with eager execution - # https://github.com/tensorflow/tensorflow/issues/18165 - import tensorflow as tf - tf.compat.v1.disable_eager_execution() - - # Root directory of the project - ROOT_DIR = os.environ.get('AUTO_SEGMENTATION_PATH') - # Import Mask RCNN - sys.path.append(ROOT_DIR) # To find local version of the library - import mrcnn.model as modellib - - # Import COCO config - sys.path.append(os.path.join(ROOT_DIR, "samples/coco/")) # To find local version - import coco - - # Directory to save logs and trained model - MODEL_DIR = os.path.join(ROOT_DIR, "logs") - - # Local path to trained weights file - COCO_MODEL_PATH = os.path.join(ROOT_DIR, "mask_rcnn_coco.h5") - if COCO_MODEL_PATH is None: - raise OSError('Model path env not found in the system.') - job = rq.get_current_job() - - ## CONFIGURATION - - class InferenceConfig(coco.CocoConfig): - # Set batch size to 1 since we'll be running inference on - # one image at a time. Batch size = GPU_COUNT * IMAGES_PER_GPU - GPU_COUNT = 1 - IMAGES_PER_GPU = 1 - - # Print config details - config = InferenceConfig() - config.display() - - ## CREATE MODEL AND LOAD TRAINED WEIGHTS - - # Create model object in inference mode. - model = modellib.MaskRCNN(mode="inference", model_dir=MODEL_DIR, config=config) - # Load weights trained on MS-COCO - model.load_weights(COCO_MODEL_PATH, by_name=True) - - ## RUN OBJECT DETECTION - result = {} - frames = frame_provider.get_frames(frame_provider.Quality.ORIGINAL) - for image_num, (image_bytes, _) in enumerate(frames): - job.refresh() - if 'cancel' in job.meta: - del job.meta['cancel'] - job.save() - return None - job.meta['progress'] = image_num * 100 / len(frame_provider) - job.save_meta() - - image = skimage.io.imread(image_bytes) - - # for multiple image detection, "batch size" must be equal to number of images - r = model.detect([image], verbose=1) - - r = r[0] - # "r['rois'][index]" gives bounding box around the object - for index, c_id in enumerate(r['class_ids']): - if c_id in labels_mapping.keys(): - if r['scores'][index] >= treshold: - mask = _convert_to_int(r['masks'][:,:,index]) - segmentation = _convert_to_segmentation(mask) - label = labels_mapping[c_id] - if label not in result: - result[label] = [] - result[label].append( - [image_num, segmentation]) - - return result - -def convert_to_cvat_format(data): - result = { - "tracks": [], - "shapes": [], - "tags": [], - "version": 0, - } - - for label in data: - segments = data[label] - for segment in segments: - result['shapes'].append({ - "type": "polygon", - "label_id": label, - "frame": segment[0], - "points": segment[1], - "z_order": 0, - "group": None, - "occluded": False, - "attributes": [], - }) - - return result - -def create_thread(tid, labels_mapping, user): - try: - # If detected object accuracy bigger than threshold it will returend - TRESHOLD = 0.5 - # Init rq job - job = rq.get_current_job() - job.meta['progress'] = 0 - job.save_meta() - # Get job indexes and segment length - db_task = TaskModel.objects.get(pk=tid) - # Get image list - frame_provider = FrameProvider(db_task.data) - - # Run auto segmentation by tf - result = None - slogger.glob.info("auto segmentation with tensorflow framework for task {}".format(tid)) - result = run_tensorflow_auto_segmentation(frame_provider, labels_mapping, TRESHOLD) - - if result is None: - slogger.glob.info('auto segmentation for task {} canceled by user'.format(tid)) - return - - # Modify data format and save - result = convert_to_cvat_format(result) - serializer = LabeledDataSerializer(data = result) - if serializer.is_valid(raise_exception=True): - put_task_data(tid, result) - slogger.glob.info('auto segmentation for task {} done'.format(tid)) - except Exception as ex: - try: - slogger.task[tid].exception('exception was occured during auto segmentation of the task', exc_info=True) - except Exception: - slogger.glob.exception('exception was occured during auto segmentation of the task {}'.format(tid), exc_info=True) - raise ex - -@api_view(['POST']) -@login_required -def get_meta_info(request): - try: - queue = django_rq.get_queue('low') - tids = request.data - result = {} - for tid in tids: - job = queue.fetch_job('auto_segmentation.create/{}'.format(tid)) - if job is not None: - result[tid] = { - "active": job.is_queued or job.is_started, - "success": not job.is_failed - } - - return JsonResponse(result) - except Exception as ex: - slogger.glob.exception('exception was occured during tf meta request', exc_info=True) - return HttpResponseBadRequest(str(ex)) - - -@login_required -@permission_required(perm=['engine.task.change'], - fn=objectgetter(TaskModel, 'tid'), raise_exception=True) -def create(request, tid): - slogger.glob.info('auto segmentation create request for task {}'.format(tid)) - try: - db_task = TaskModel.objects.get(pk=tid) - queue = django_rq.get_queue('low') - job = queue.fetch_job('auto_segmentation.create/{}'.format(tid)) - if job is not None and (job.is_started or job.is_queued): - raise Exception("The process is already running") - - db_labels = db_task.label_set.prefetch_related('attributespec_set').all() - db_labels = {db_label.id:db_label.name for db_label in db_labels} - - # COCO Labels - auto_segmentation_labels = { "BG": 0, - "person": 1, "bicycle": 2, "car": 3, "motorcycle": 4, "airplane": 5, - "bus": 6, "train": 7, "truck": 8, "boat": 9, "traffic_light": 10, - "fire_hydrant": 11, "stop_sign": 12, "parking_meter": 13, "bench": 14, - "bird": 15, "cat": 16, "dog": 17, "horse": 18, "sheep": 19, "cow": 20, - "elephant": 21, "bear": 22, "zebra": 23, "giraffe": 24, "backpack": 25, - "umbrella": 26, "handbag": 27, "tie": 28, "suitcase": 29, "frisbee": 30, - "skis": 31, "snowboard": 32, "sports_ball": 33, "kite": 34, "baseball_bat": 35, - "baseball_glove": 36, "skateboard": 37, "surfboard": 38, "tennis_racket": 39, - "bottle": 40, "wine_glass": 41, "cup": 42, "fork": 43, "knife": 44, "spoon": 45, - "bowl": 46, "banana": 47, "apple": 48, "sandwich": 49, "orange": 50, "broccoli": 51, - "carrot": 52, "hot_dog": 53, "pizza": 54, "donut": 55, "cake": 56, "chair": 57, - "couch": 58, "potted_plant": 59, "bed": 60, "dining_table": 61, "toilet": 62, - "tv": 63, "laptop": 64, "mouse": 65, "remote": 66, "keyboard": 67, "cell_phone": 68, - "microwave": 69, "oven": 70, "toaster": 71, "sink": 72, "refrigerator": 73, - "book": 74, "clock": 75, "vase": 76, "scissors": 77, "teddy_bear": 78, "hair_drier": 79, - "toothbrush": 80 - } - - labels_mapping = {} - for key, labels in db_labels.items(): - if labels in auto_segmentation_labels.keys(): - labels_mapping[auto_segmentation_labels[labels]] = key - - if not len(labels_mapping.values()): - raise Exception('No labels found for auto segmentation') - - # Run auto segmentation job - queue.enqueue_call(func=create_thread, - args=(tid, labels_mapping, request.user), - job_id='auto_segmentation.create/{}'.format(tid), - timeout=604800) # 7 days - - slogger.task[tid].info('tensorflow segmentation job enqueued with labels {}'.format(labels_mapping)) - - except Exception as ex: - try: - slogger.task[tid].exception("exception was occured during tensorflow segmentation request", exc_info=True) - except Exception: - pass - return HttpResponseBadRequest(str(ex)) - - return HttpResponse() - -@login_required -@permission_required(perm=['engine.task.access'], - fn=objectgetter(TaskModel, 'tid'), raise_exception=True) -def check(request, tid): - try: - queue = django_rq.get_queue('low') - job = queue.fetch_job('auto_segmentation.create/{}'.format(tid)) - if job is not None and 'cancel' in job.meta: - return JsonResponse({'status': 'finished'}) - data = {} - if job is None: - data['status'] = 'unknown' - elif job.is_queued: - data['status'] = 'queued' - elif job.is_started: - data['status'] = 'started' - data['progress'] = job.meta['progress'] - elif job.is_finished: - data['status'] = 'finished' - job.delete() - else: - data['status'] = 'failed' - data['stderr'] = job.exc_info - job.delete() - - except Exception: - data['status'] = 'unknown' - - return JsonResponse(data) - - -@login_required -@permission_required(perm=['engine.task.change'], - fn=objectgetter(TaskModel, 'tid'), raise_exception=True) -def cancel(request, tid): - try: - queue = django_rq.get_queue('low') - job = queue.fetch_job('auto_segmentation.create/{}'.format(tid)) - if job is None or job.is_finished or job.is_failed: - raise Exception('Task is not being segmented currently') - elif 'cancel' not in job.meta: - job.meta['cancel'] = True - job.save() - - except Exception as ex: - try: - slogger.task[tid].exception("cannot cancel tensorflow segmentation for task #{}".format(tid), exc_info=True) - except Exception: - pass - return HttpResponseBadRequest(str(ex)) - - return HttpResponse() diff --git a/cvat/apps/dataset_manager/annotation.py b/cvat/apps/dataset_manager/annotation.py index 040788bc1bfe..30393dd56f14 100644 --- a/cvat/apps/dataset_manager/annotation.py +++ b/cvat/apps/dataset_manager/annotation.py @@ -316,7 +316,10 @@ def _get_cost_threshold(): def _calc_objects_similarity(obj0, obj1, start_frame, overlap): def _calc_polygons_similarity(p0, p1): overlap_area = p0.intersection(p1).area - return overlap_area / (p0.area + p1.area - overlap_area) + if p0.area == 0 or p1.area == 0: # a line with many points + return 0 + else: + return overlap_area / (p0.area + p1.area - overlap_area) has_same_type = obj0["type"] == obj1["type"] has_same_label = obj0.get("label_id") == obj1.get("label_id") @@ -328,7 +331,7 @@ def _calc_polygons_similarity(p0, p1): return _calc_polygons_similarity(p0, p1) elif obj0["type"] == ShapeType.POLYGON: p0 = geometry.Polygon(pairwise(obj0["points"])) - p1 = geometry.Polygon(pairwise(obj0["points"])) + p1 = geometry.Polygon(pairwise(obj1["points"])) return _calc_polygons_similarity(p0, p1) else: diff --git a/cvat/apps/dextr_segmentation/README.md b/cvat/apps/dextr_segmentation/README.md deleted file mode 100644 index 2382a380861d..000000000000 --- a/cvat/apps/dextr_segmentation/README.md +++ /dev/null @@ -1,31 +0,0 @@ -# Semi-Automatic Segmentation with [Deep Extreme Cut](http://www.vision.ee.ethz.ch/~cvlsegmentation/dextr/) - -## About the application - -The application allows to use deep learning models for semi-automatic semantic and instance segmentation. -You can get a segmentation polygon from four (or more) extreme points of an object. -This application uses the pre-trained DEXTR model which has been converted to Inference Engine format. - -We are grateful to K.K. Maninis, S. Caelles, J. Pont-Tuset, and L. Van Gool who permitted using their models in our tool - -## Build docker image -```bash -# OpenVINO component is also needed -docker-compose -f docker-compose.yml -f components/openvino/docker-compose.openvino.yml -f cvat/apps/dextr_segmentation/docker-compose.dextr.yml build -``` - -## Run docker container -```bash -docker-compose -f docker-compose.yml -f components/openvino/docker-compose.openvino.yml -f cvat/apps/dextr_segmentation/docker-compose.dextr.yml up -d -``` - -## Using - -1. Open a job -2. Select "Auto Segmentation" in the list of shapes -3. Run the draw mode as usually (by press the "Create Shape" button or by "N" shortcut) -4. Click four-six (or more if it's need) extreme points of an object -5. Close the draw mode as usually (by shortcut or pressing the button "Stop Creation") -6. Wait a moment and you will get a class agnostic annotation polygon -7. You can close an annotation request if it is too long -(in case if it is queued to rq worker and all workers are busy) diff --git a/cvat/apps/dextr_segmentation/__init__.py b/cvat/apps/dextr_segmentation/__init__.py deleted file mode 100644 index 472c2ac3c42f..000000000000 --- a/cvat/apps/dextr_segmentation/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -# Copyright (C) 2018 Intel Corporation -# -# SPDX-License-Identifier: MIT - -from cvat.settings.base import JS_3RDPARTY - -JS_3RDPARTY['engine'] = JS_3RDPARTY.get('engine', []) + ['dextr_segmentation/js/enginePlugin.js'] diff --git a/cvat/apps/dextr_segmentation/apps.py b/cvat/apps/dextr_segmentation/apps.py deleted file mode 100644 index d5d43a88e1c0..000000000000 --- a/cvat/apps/dextr_segmentation/apps.py +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright (C) 2018 Intel Corporation -# -# SPDX-License-Identifier: MIT - -from django.apps import AppConfig - -class DextrSegmentationConfig(AppConfig): - name = 'dextr_segmentation' diff --git a/cvat/apps/dextr_segmentation/dextr.py b/cvat/apps/dextr_segmentation/dextr.py deleted file mode 100644 index d6eb20022248..000000000000 --- a/cvat/apps/dextr_segmentation/dextr.py +++ /dev/null @@ -1,119 +0,0 @@ - -# Copyright (C) 2018-2020 Intel Corporation -# -# SPDX-License-Identifier: MIT - -from cvat.apps.auto_annotation.inference_engine import make_plugin_or_core, make_network -from cvat.apps.engine.frame_provider import FrameProvider - -import os -import cv2 -import PIL -import numpy as np - -_IE_CPU_EXTENSION = os.getenv("IE_CPU_EXTENSION", "libcpu_extension_avx2.so") -_IE_PLUGINS_PATH = os.getenv("IE_PLUGINS_PATH", None) - -_DEXTR_MODEL_DIR = os.getenv("DEXTR_MODEL_DIR", None) -_DEXTR_PADDING = 50 -_DEXTR_TRESHOLD = 0.9 -_DEXTR_SIZE = 512 - -class DEXTR_HANDLER: - def __init__(self): - self._plugin = None - self._network = None - self._exec_network = None - self._input_blob = None - self._output_blob = None - if not _DEXTR_MODEL_DIR: - raise Exception("DEXTR_MODEL_DIR is not defined") - - - def handle(self, db_data, frame, points): - # Lazy initialization - if not self._plugin: - self._plugin = make_plugin_or_core() - self._network = make_network(os.path.join(_DEXTR_MODEL_DIR, 'dextr.xml'), - os.path.join(_DEXTR_MODEL_DIR, 'dextr.bin')) - self._input_blob = next(iter(self._network.inputs)) - self._output_blob = next(iter(self._network.outputs)) - if getattr(self._plugin, 'load_network', False): - self._exec_network = self._plugin.load_network(self._network, 'CPU') - else: - self._exec_network = self._plugin.load(network=self._network) - - frame_provider = FrameProvider(db_data) - image = frame_provider.get_frame(frame, frame_provider.Quality.ORIGINAL) - image = PIL.Image.open(image[0]) - numpy_image = np.array(image) - points = np.asarray([[int(p["x"]), int(p["y"])] for p in points], dtype=int) - - # Padding mustn't be more than the closest distance to an edge of an image - [height, width] = numpy_image.shape[:2] - x_values = points[:, 0] - y_values = points[:, 1] - [min_x, max_x] = [np.min(x_values), np.max(x_values)] - [min_y, max_y] = [np.min(y_values), np.max(y_values)] - padding = min(min_x, min_y, width - max_x, height - max_y, _DEXTR_PADDING) - bounding_box = ( - max(min(points[:, 0]) - padding, 0), - max(min(points[:, 1]) - padding, 0), - min(max(points[:, 0]) + padding, width - 1), - min(max(points[:, 1]) + padding, height - 1) - ) - - # Prepare an image - numpy_cropped = np.array(image.crop(bounding_box)) - resized = cv2.resize(numpy_cropped, (_DEXTR_SIZE, _DEXTR_SIZE), - interpolation = cv2.INTER_CUBIC).astype(np.float32) - - # Make a heatmap - points = points - [min(points[:, 0]), min(points[:, 1])] + [padding, padding] - points = (points * [_DEXTR_SIZE / numpy_cropped.shape[1], _DEXTR_SIZE / numpy_cropped.shape[0]]).astype(int) - heatmap = np.zeros(shape=resized.shape[:2], dtype=np.float64) - for point in points: - gaussian_x_axis = np.arange(0, _DEXTR_SIZE, 1, float) - point[0] - gaussian_y_axis = np.arange(0, _DEXTR_SIZE, 1, float)[:, np.newaxis] - point[1] - gaussian = np.exp(-4 * np.log(2) * ((gaussian_x_axis ** 2 + gaussian_y_axis ** 2) / 100)).astype(np.float64) - heatmap = np.maximum(heatmap, gaussian) - cv2.normalize(heatmap, heatmap, 0, 255, cv2.NORM_MINMAX) - - # Concat an image and a heatmap - input_dextr = np.concatenate((resized, heatmap[:, :, np.newaxis].astype(resized.dtype)), axis=2) - input_dextr = input_dextr.transpose((2,0,1)) - - pred = self._exec_network.infer(inputs={self._input_blob: input_dextr[np.newaxis, ...]})[self._output_blob][0, 0, :, :] - pred = cv2.resize(pred, tuple(reversed(numpy_cropped.shape[:2])), interpolation = cv2.INTER_CUBIC) - result = np.zeros(numpy_image.shape[:2]) - result[bounding_box[1]:bounding_box[1] + pred.shape[0], bounding_box[0]:bounding_box[0] + pred.shape[1]] = pred > _DEXTR_TRESHOLD - - # Convert a mask to a polygon - result = np.array(result, dtype=np.uint8) - cv2.normalize(result,result,0,255,cv2.NORM_MINMAX) - contours = None - if int(cv2.__version__.split('.')[0]) > 3: - contours = cv2.findContours(result, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_TC89_KCOS)[0] - else: - contours = cv2.findContours(result, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_TC89_KCOS)[1] - - contours = max(contours, key=lambda arr: arr.size) - if contours.shape.count(1): - contours = np.squeeze(contours) - if contours.size < 3 * 2: - raise Exception('Less then three point have been detected. Can not build a polygon.') - - result = "" - for point in contours: - result += "{},{} ".format(int(point[0]), int(point[1])) - result = result[:-1] - - return result - - def __del__(self): - if self._exec_network: - del self._exec_network - if self._network: - del self._network - if self._plugin: - del self._plugin diff --git a/cvat/apps/dextr_segmentation/docker-compose.dextr.yml b/cvat/apps/dextr_segmentation/docker-compose.dextr.yml deleted file mode 100644 index 468523b8a05a..000000000000 --- a/cvat/apps/dextr_segmentation/docker-compose.dextr.yml +++ /dev/null @@ -1,14 +0,0 @@ -# -# Copyright (C) 2018 Intel Corporation -# -# SPDX-License-Identifier: MIT -# - -version: "2.3" - -services: - cvat: - build: - context: . - args: - WITH_DEXTR: "yes" diff --git a/cvat/apps/dextr_segmentation/static/dextr_segmentation/js/enginePlugin.js b/cvat/apps/dextr_segmentation/static/dextr_segmentation/js/enginePlugin.js deleted file mode 100644 index 0869ba8df048..000000000000 --- a/cvat/apps/dextr_segmentation/static/dextr_segmentation/js/enginePlugin.js +++ /dev/null @@ -1,214 +0,0 @@ -/* - * Copyright (C) 2018 Intel Corporation - * - * SPDX-License-Identifier: MIT - */ - -/* global - AREA_TRESHOLD:false - PolyShapeModel:false - ShapeCreatorModel:true - ShapeCreatorView:true - showMessage:false -*/ - -/* eslint no-underscore-dangle: 0 */ - -window.addEventListener('DOMContentLoaded', () => { - $('').appendTo('#shapeTypeSelector'); - - const dextrCancelButtonId = 'dextrCancelButton'; - const dextrOverlay = $(` - `).appendTo('body'); - - const dextrCancelButton = $(`#${dextrCancelButtonId}`); - dextrCancelButton.on('click', () => { - dextrCancelButton.prop('disabled', true); - $.ajax({ - url: `/dextr/cancel/${window.cvat.job.id}`, - type: 'GET', - error: (errorData) => { - const message = `Can not cancel segmentation. Code: ${errorData.status}. - Message: ${errorData.responseText || errorData.statusText}`; - showMessage(message); - }, - complete: () => { - dextrCancelButton.prop('disabled', false); - }, - }); - }); - - function ShapeCreatorModelWrapper(OriginalClass) { - // Constructor will patch some properties for a created instance - function constructorDecorator(...args) { - const instance = new OriginalClass(...args); - - // Decorator for the defaultType property - Object.defineProperty(instance, 'defaultType', { - get: () => instance._defaultType, - set: (type) => { - if (!['box', 'box_by_4_points', 'points', 'polygon', - 'polyline', 'auto_segmentation', 'cuboid'].includes(type)) { - throw Error(`Unknown shape type found ${type}`); - } - instance._defaultType = type; - }, - }); - - // Decorator for finish method. - const decoratedFinish = instance.finish; - instance.finish = (result) => { - if (instance._defaultType === 'auto_segmentation') { - try { - instance._defaultType = 'polygon'; - decoratedFinish.call(instance, result); - } finally { - instance._defaultType = 'auto_segmentation'; - } - } else { - decoratedFinish.call(instance, result); - } - }; - - return instance; - } - - constructorDecorator.prototype = OriginalClass.prototype; - constructorDecorator.prototype.constructor = constructorDecorator; - return constructorDecorator; - } - - - function ShapeCreatorViewWrapper(OriginalClass) { - // Constructor will patch some properties for each instance - function constructorDecorator(...args) { - const instance = new OriginalClass(...args); - - // Decorator for the _create() method. - // We save the decorated _create() and we will use it if type != 'auto_segmentation' - const decoratedCreate = instance._create; - instance._create = () => { - if (instance._type !== 'auto_segmentation') { - decoratedCreate.call(instance); - return; - } - - instance._drawInstance = instance._frameContent.polyline().draw({ snapToGrid: 0.1 }).addClass('shapeCreation').attr({ - 'stroke-width': 0, - z_order: Number.MAX_SAFE_INTEGER, - }); - instance._createPolyEvents(); - - /* the _createPolyEvents method have added "drawdone" - * event handler which invalid for this case - * because of that reason we remove the handler and - * create the valid handler instead - */ - instance._drawInstance.off('drawdone').on('drawdone', (e) => { - let actualPoints = window.cvat.translate.points.canvasToActual(e.target.getAttribute('points')); - actualPoints = PolyShapeModel.convertStringToNumberArray(actualPoints); - - if (actualPoints.length < 4) { - showMessage('It is need to specify minimum four extreme points for an object'); - instance._controller.switchCreateMode(true); - return; - } - - const { frameWidth } = window.cvat.player.geometry; - const { frameHeight } = window.cvat.player.geometry; - for (let idx = 0; idx < actualPoints.length; idx += 1) { - const point = actualPoints[idx]; - point.x = Math.clamp(point.x, 0, frameWidth); - point.y = Math.clamp(point.y, 0, frameHeight); - } - - e.target.setAttribute('points', - window.cvat.translate.points.actualToCanvas( - PolyShapeModel.convertNumberArrayToString(actualPoints), - )); - - const polybox = e.target.getBBox(); - const area = polybox.width * polybox.height; - - if (area > AREA_TRESHOLD) { - $.ajax({ - url: `/dextr/create/${window.cvat.job.id}`, - type: 'POST', - data: JSON.stringify({ - frame: window.cvat.player.frames.current, - points: actualPoints, - }), - contentType: 'application/json', - success: () => { - function intervalCallback() { - $.ajax({ - url: `/dextr/check/${window.cvat.job.id}`, - type: 'GET', - success: (jobData) => { - if (['queued', 'started'].includes(jobData.status)) { - if (jobData.status === 'queued') { - dextrCancelButton.prop('disabled', false); - } - setTimeout(intervalCallback, 1000); - } else { - dextrOverlay.addClass('hidden'); - if (jobData.status === 'finished') { - if (jobData.result) { - instance._controller.finish({ points: jobData.result }, 'polygon'); - } - } else if (jobData.status === 'failed') { - const message = `Segmentation has fallen. Error: '${jobData.stderr}'`; - showMessage(message); - } else { - let message = `Check segmentation request returned "${jobData.status}" status.`; - if (jobData.stderr) { - message += ` Error: ${jobData.stderr}`; - } - showMessage(message); - } - } - }, - error: (errorData) => { - dextrOverlay.addClass('hidden'); - const message = `Can not check segmentation. Code: ${errorData.status}.` - + ` Message: ${errorData.responseText || errorData.statusText}`; - showMessage(message); - }, - }); - } - - dextrCancelButton.prop('disabled', true); - dextrOverlay.removeClass('hidden'); - setTimeout(intervalCallback, 1000); - }, - error: (errorData) => { - const message = `Can not cancel ReID process. Code: ${errorData.status}.` - + ` Message: ${errorData.responseText || errorData.statusText}`; - showMessage(message); - }, - }); - } - - instance._controller.switchCreateMode(true); - }); // end of "drawdone" handler - }; // end of _create() method - - return instance; - } // end of constructorDecorator() - - constructorDecorator.prototype = OriginalClass.prototype; - constructorDecorator.prototype.constructor = constructorDecorator; - return constructorDecorator; - } // end of ShapeCreatorViewWrapper - - // Apply patch for classes - ShapeCreatorModel = ShapeCreatorModelWrapper(ShapeCreatorModel); - ShapeCreatorView = ShapeCreatorViewWrapper(ShapeCreatorView); -}); diff --git a/cvat/apps/dextr_segmentation/urls.py b/cvat/apps/dextr_segmentation/urls.py deleted file mode 100644 index 6b3120b67939..000000000000 --- a/cvat/apps/dextr_segmentation/urls.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright (C) 2018-2020 Intel Corporation -# -# SPDX-License-Identifier: MIT - -from django.urls import path -from . import views - -urlpatterns = [ - path('create/', views.create), - path('cancel/', views.cancel), - path('check/', views.check), - path('enabled', views.enabled) -] diff --git a/cvat/apps/dextr_segmentation/views.py b/cvat/apps/dextr_segmentation/views.py deleted file mode 100644 index dd78a2b01e42..000000000000 --- a/cvat/apps/dextr_segmentation/views.py +++ /dev/null @@ -1,128 +0,0 @@ -# Copyright (C) 2018 Intel Corporation -# -# SPDX-License-Identifier: MIT - -from django.http import HttpResponse, HttpResponseBadRequest, JsonResponse -from cvat.apps.authentication.decorators import login_required -from rules.contrib.views import permission_required, objectgetter - -from cvat.apps.engine.models import Job -from cvat.apps.engine.log import slogger -from cvat.apps.dextr_segmentation.dextr import DEXTR_HANDLER - -import django_rq -import json -import rq - -__RQ_QUEUE_NAME = "default" -__DEXTR_HANDLER = DEXTR_HANDLER() - -def _dextr_thread(db_data, frame, points): - job = rq.get_current_job() - job.meta["result"] = __DEXTR_HANDLER.handle(db_data, frame, points) - job.save_meta() - - -@login_required -@permission_required(perm=["engine.job.change"], - fn=objectgetter(Job, "jid"), raise_exception=True) -def create(request, jid): - try: - data = json.loads(request.body.decode("utf-8")) - - points = data["points"] - frame = int(data["frame"]) - username = request.user.username - - slogger.job[jid].info("create dextr request for the JOB: {} ".format(jid) - + "by the USER: {} on the FRAME: {}".format(username, frame)) - - db_data = Job.objects.select_related("segment__task__data").get(id=jid).segment.task.data - - queue = django_rq.get_queue(__RQ_QUEUE_NAME) - rq_id = "dextr.create/{}/{}".format(jid, username) - job = queue.fetch_job(rq_id) - - if job is not None and (job.is_started or job.is_queued): - if "cancel" not in job.meta: - raise Exception("Segmentation process has been already run for the " + - "JOB: {} and the USER: {}".format(jid, username)) - else: - job.delete() - - queue.enqueue_call(func=_dextr_thread, - args=(db_data, frame, points), - job_id=rq_id, - timeout=15, - ttl=30) - - return HttpResponse() - except Exception as ex: - slogger.job[jid].error("can't create a dextr request for the job {}".format(jid), exc_info=True) - return HttpResponseBadRequest(str(ex)) - - -@login_required -@permission_required(perm=["engine.job.change"], - fn=objectgetter(Job, "jid"), raise_exception=True) -def cancel(request, jid): - try: - username = request.user.username - slogger.job[jid].info("cancel dextr request for the JOB: {} ".format(jid) - + "by the USER: {}".format(username)) - - queue = django_rq.get_queue(__RQ_QUEUE_NAME) - rq_id = "dextr.create/{}/{}".format(jid, username) - job = queue.fetch_job(rq_id) - - if job is None or job.is_finished or job.is_failed: - raise Exception("Segmentation isn't running now") - elif "cancel" not in job.meta: - job.meta["cancel"] = True - job.save_meta() - - return HttpResponse() - except Exception as ex: - slogger.job[jid].error("can't cancel a dextr request for the job {}".format(jid), exc_info=True) - return HttpResponseBadRequest(str(ex)) - - -@login_required -@permission_required(perm=["engine.job.change"], - fn=objectgetter(Job, "jid"), raise_exception=True) -def check(request, jid): - try: - username = request.user.username - slogger.job[jid].info("check dextr request for the JOB: {} ".format(jid) - + "by the USER: {}".format(username)) - - queue = django_rq.get_queue(__RQ_QUEUE_NAME) - rq_id = "dextr.create/{}/{}".format(jid, username) - job = queue.fetch_job(rq_id) - data = {} - - if job is None: - data["status"] = "unknown" - else: - if "cancel" in job.meta: - data["status"] = "finished" - elif job.is_queued: - data["status"] = "queued" - elif job.is_started: - data["status"] = "started" - elif job.is_finished: - data["status"] = "finished" - data["result"] = job.meta["result"] - job.delete() - else: - data["status"] = "failed" - data["stderr"] = job.exc_info - job.delete() - - return JsonResponse(data) - except Exception as ex: - slogger.job[jid].error("can't check a dextr request for the job {}".format(jid), exc_info=True) - return HttpResponseBadRequest(str(ex)) - -def enabled(request): - return HttpResponse() diff --git a/cvat/apps/documentation/faq.md b/cvat/apps/documentation/faq.md index 4271e193c85b..d846a0933958 100644 --- a/cvat/apps/documentation/faq.md +++ b/cvat/apps/documentation/faq.md @@ -4,14 +4,11 @@ - [How to change default CVAT hostname or port](#how-to-change-default-cvat-hostname-or-port) - [How to configure connected share folder on Windows](#how-to-configure-connected-share-folder-on-windows) - [How to make unassigned tasks not visible to all users](#how-to-make-unassigned-tasks-not-visible-to-all-users) -- [Can Nvidia GPU be used to run inference with my own model](#can-nvidia-gpu-be-used-to-run-inference-with-my-own-model) -- [What versions of OpenVINO toolkit are supported](#what-versions-of-openvino-toolkit-are-supported) - [Where are uploaded images/videos stored](#where-are-uploaded-imagesvideos-stored) - [Where are annotations stored](#where-are-annotations-stored) - [How to mark job/task as completed](#how-to-mark-jobtask-as-completed) - [How to install CVAT on Windows 10 Home](#how-to-install-cvat-on-windows-10-home) - ## How to update CVAT Before upgrading, please follow the official docker [manual](https://docs.docker.com/storage/volumes/#backup-restore-or-migrate-data-volumes) and backup all CVAT volumes. @@ -81,14 +78,6 @@ volumes: ## How to make unassigned tasks not visible to all users Set [reduce_task_visibility](../../settings/base.py#L424) variable to `True`. -## Can Nvidia GPU be used to run inference with my own model -Nvidia GPU can be used to accelerate inference of [tf_annotation](../../../components/tf_annotation/README.md) and [auto_segmentation](../../../components/auto_segmentation/README.md) models. - -OpenVino doesn't support Nvidia cards, so you can run your own models only on CPU. - -## What versions of OpenVINO toolkit are supported -These versions are supported: `2019 R3`, `2019 R3.1`, `2020 1`, `2020 2` - ## Where are uploaded images/videos stored The uploaded data is stored in the `cvat_data` docker volume: ```yml diff --git a/cvat/apps/documentation/installation.md b/cvat/apps/documentation/installation.md index 0f5ed35c094f..286c73928dec 100644 --- a/cvat/apps/documentation/installation.md +++ b/cvat/apps/documentation/installation.md @@ -4,6 +4,7 @@ - [Mac OS Mojave](#mac-os-mojave) - [Advanced topics](#advanced-topics) - [Additional components](#additional-components) + - [Semi-automatic and automatic annotation](#semi-automatic-and-automatic-annotation) - [Stop all containers](#stop-all-containers) - [Advanced settings](#advanced-settings) - [Share path](#share-path) @@ -246,22 +247,52 @@ server. Proxy is an advanced topic and it is not covered by the guide. ### Additional components -- [Auto annotation using DL models in OpenVINO toolkit format](/cvat/apps/auto_annotation/README.md) - [Analytics: management and monitoring of data annotation team](/components/analytics/README.md) -- [TF Object Detection API: auto annotation](/components/tf_annotation/README.md) -- [Support for NVIDIA GPUs](/components/cuda/README.md) -- [Semi-automatic segmentation with Deep Extreme Cut](/cvat/apps/dextr_segmentation/README.md) -- [Auto segmentation: Keras+Tensorflow Mask R-CNN Segmentation](/components/auto_segmentation/README.md) ```bash -# Build and run containers with CUDA and OpenVINO support -# IMPORTANT: need to download OpenVINO package before running the command -docker-compose -f docker-compose.yml -f components/cuda/docker-compose.cuda.yml -f components/openvino/docker-compose.openvino.yml up -d --build - # Build and run containers with Analytics component support: docker-compose -f docker-compose.yml -f components/analytics/docker-compose.analytics.yml up -d --build ``` +### Semi-automatic and automatic annotation + +- You have to install `nuctl` command line tool to build and deploy serverless +functions. Download [the latest release](https://github.com/nuclio/nuclio/releases). +- Create `cvat` project inside nuclio dashboard where you will deploy new +serverless functions and deploy a couple of DL models. Commands below should +be run only after CVAT has been installed using docker-compose because it +runs nuclio dashboard which manages all serverless functions. + +```bash +nuctl create project cvat +``` + +```bash +nuctl deploy --project-name cvat \ + --path serverless/openvino/dextr/nuclio \ + --volume `pwd`/serverless/openvino/common:/opt/nuclio/common +``` + +```bash +nuctl deploy --project-name cvat \ + --path serverless/openvino/omz/public/yolo-v3-tf/nuclio \ + --volume `pwd`/serverless/openvino/common:/opt/nuclio/common +``` + +Note: see [deploy.sh](/serverless/deploy.sh) script for more examples. + +List of DL models as serverless functions: + +- [Deep Extreme Cut (OpenVINO)](/serverless/openvino/dextr/nuclio) +- [Faster RCNN (TensorFlow)](/serverless/tensorflow/faster_rcnn_inception_v2_coco/nuclio) +- [Mask RCNN (OpenVINO)](/serverless/openvino/omz/public/mask_rcnn_inception_resnet_v2_atrous_coco/nuclio) +- [YOLO v3 (OpenVINO)](/serverless/openvino/omz/public/yolo-v3-tf/nuclio) +- [Faster RCNN (OpenVINO)](/serverless/openvino/omz/public/faster_rcnn_inception_v2_coco/nuclio) +- [Text detection v4 (OpenVINO)](/serverless/openvino/omz/intel/text-detection-0004/nuclio) +- [Semantic segmentation for ADAS (OpenVINO)](/serverless/openvino/omz/intel/semantic-segmentation-adas-0001/nuclio) +- [Mask RCNN (TensorFlow)](/serverless/tensorflow/matterport/mask_rcnn/nuclio) +- [Person ReID (OpenVINO)](/serverless/openvino/omz/intel/person-reidentification-retail-300/nuclio) + ### Stop all containers The command below stops and removes containers, networks, volumes, and images @@ -327,9 +358,9 @@ our server connection. We assume that -- you have sudo access on your server machine, -- you have an IP address to use for remote access, and -- that the local CVAT installation works on your server. +- you have sudo access on your server machine, +- you have an IP address to use for remote access, and +- that the local CVAT installation works on your server. If this is not the case, please complete the steps in the installation manual first. @@ -525,7 +556,7 @@ server { proxy_set_header Host $http_host; proxy_pass_header Set-Cookie; - location ~* /api/.*|git/.*|tensorflow/.*|auto_annotation/.*|analytics/.*|static/.*|admin|admin/.*|documentation/.*|dextr/.*|reid/.* { + location ~* /api/.*|git/.*|analytics/.*|static/.*|admin|admin/.*|documentation/.* { proxy_pass http://cvat:8080; } @@ -550,6 +581,6 @@ server { Start cvat_proxy container with https enabled. -``` +```bash docker start cvat_proxy -``` \ No newline at end of file +``` diff --git a/cvat/apps/engine/models.py b/cvat/apps/engine/models.py index d4c46eb3e264..d410ef7c823f 100644 --- a/cvat/apps/engine/models.py +++ b/cvat/apps/engine/models.py @@ -302,7 +302,7 @@ def choices(cls): def __str__(self): return self.value -class source(str, Enum): +class SourceType(str, Enum): AUTO = 'auto' MANUAL = 'manual' @@ -319,7 +319,8 @@ class Annotation(models.Model): label = models.ForeignKey(Label, on_delete=models.CASCADE) frame = models.PositiveIntegerField() group = models.PositiveIntegerField(null=True) - source = models.CharField(max_length=16, choices=source.choices(), default="manual", null=True) + source = models.CharField(max_length=16, choices=SourceType.choices(), + default=str(SourceType.MANUAL), null=True) class Meta: abstract = True @@ -392,20 +393,3 @@ class TrackedShape(Shape): class TrackedShapeAttributeVal(AttributeVal): shape = models.ForeignKey(TrackedShape, on_delete=models.CASCADE) - -class Plugin(models.Model): - name = models.SlugField(max_length=32, primary_key=True) - description = SafeCharField(max_length=8192) - maintainer = models.ForeignKey(User, null=True, blank=True, - on_delete=models.SET_NULL, related_name="maintainers") - created_at = models.DateTimeField(auto_now_add=True) - updated_at = models.DateTimeField(auto_now_add=True) - - # Extend default permission model - class Meta: - default_permissions = () - -class PluginOption(models.Model): - plugin = models.ForeignKey(Plugin, on_delete=models.CASCADE) - name = SafeCharField(max_length=32) - value = SafeCharField(max_length=1024) diff --git a/cvat/apps/engine/serializers.py b/cvat/apps/engine/serializers.py index 90e24714c7d5..9edac559d784 100644 --- a/cvat/apps/engine/serializers.py +++ b/cvat/apps/engine/serializers.py @@ -451,12 +451,6 @@ class FileInfoSerializer(serializers.Serializer): name = serializers.CharField(max_length=1024) type = serializers.ChoiceField(choices=["REG", "DIR"]) -class PluginSerializer(serializers.ModelSerializer): - class Meta: - model = models.Plugin - fields = ('name', 'description', 'maintainer', 'created_at', - 'updated_at') - class LogEventSerializer(serializers.Serializer): job_id = serializers.IntegerField(required=False) task_id = serializers.IntegerField(required=False) diff --git a/cvat/apps/engine/urls.py b/cvat/apps/engine/urls.py index 6c608a00f9ed..bb5477530089 100644 --- a/cvat/apps/engine/urls.py +++ b/cvat/apps/engine/urls.py @@ -30,7 +30,6 @@ router.register('jobs', views.JobViewSet) router.register('users', views.UserViewSet) router.register('server', views.ServerViewSet, basename='server') -router.register('plugins', views.PluginViewSet) router.register('restrictions', RestrictionsViewSet, basename='restrictions') urlpatterns = [ diff --git a/cvat/apps/engine/views.py b/cvat/apps/engine/views.py index 120125ea1605..c9d6e5d05eeb 100644 --- a/cvat/apps/engine/views.py +++ b/cvat/apps/engine/views.py @@ -37,13 +37,13 @@ from cvat.apps.authentication.decorators import login_required from cvat.apps.dataset_manager.serializers import DatasetFormatsSerializer from cvat.apps.engine.frame_provider import FrameProvider -from cvat.apps.engine.models import Job, Plugin, StatusChoice, Task +from cvat.apps.engine.models import Job, StatusChoice, Task from cvat.apps.engine.serializers import ( AboutSerializer, AnnotationFileSerializer, BasicUserSerializer, DataMetaSerializer, DataSerializer, ExceptionSerializer, FileInfoSerializer, JobSerializer, LabeledDataSerializer, - LogEventSerializer, PluginSerializer, ProjectSerializer, - RqStatusSerializer, TaskSerializer, UserSerializer) + LogEventSerializer, ProjectSerializer, RqStatusSerializer, + TaskSerializer, UserSerializer) from cvat.settings.base import CSS_3RDPARTY, JS_3RDPARTY from cvat.apps.engine.utils import av_scan_paths @@ -757,33 +757,6 @@ def self(self, request): serializer = serializer_class(request.user, context={ "request": request }) return Response(serializer.data) -class PluginViewSet(viewsets.ModelViewSet): - queryset = Plugin.objects.all() - serializer_class = PluginSerializer - - # @action(detail=True, methods=['GET', 'PATCH', 'PUT'], serializer_class=None) - # def config(self, request, name): - # pass - - # @action(detail=True, methods=['GET', 'POST'], serializer_class=None) - # def data(self, request, name): - # pass - - # @action(detail=True, methods=['GET', 'DELETE', 'PATCH', 'PUT'], - # serializer_class=None, url_path='data/(?P\d+)') - # def data_detail(self, request, name, id): - # pass - - - @action(detail=True, methods=['GET', 'POST'], serializer_class=RqStatusSerializer) - def requests(self, request, name): - pass - - @action(detail=True, methods=['GET', 'DELETE'], - serializer_class=RqStatusSerializer, url_path='requests/(?P\d+)') - def request_detail(self, request, name, rq_id): - pass - def rq_handler(job, exc_type, exc_value, tb): job.exc_info = "".join( traceback.format_exception_only(exc_type, exc_value)) diff --git a/cvat/apps/lambda_manager/__init__.py b/cvat/apps/lambda_manager/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/cvat/apps/lambda_manager/apps.py b/cvat/apps/lambda_manager/apps.py new file mode 100644 index 000000000000..eda3a97180c6 --- /dev/null +++ b/cvat/apps/lambda_manager/apps.py @@ -0,0 +1,5 @@ +from django.apps import AppConfig + + +class LambdaManagerConfig(AppConfig): + name = 'lambda_manager' diff --git a/cvat/apps/lambda_manager/urls.py b/cvat/apps/lambda_manager/urls.py new file mode 100644 index 000000000000..0225db53731a --- /dev/null +++ b/cvat/apps/lambda_manager/urls.py @@ -0,0 +1,29 @@ +# Copyright (C) 2020 Intel Corporation +# +# SPDX-License-Identifier: MIT + +from django.urls import path +from rest_framework import routers +from django.urls import include +from . import views + +router = routers.DefaultRouter(trailing_slash=False) +# https://github.com/encode/django-rest-framework/issues/6645 +# I want to "call" my functions. To do that need to map my call method to +# POST (like get HTTP method is mapped to list(...)). One way is to implement +# own CustomRouter. But it is simpler just patch the router instance here. +router.routes[2].mapping.update({'post': 'call'}) +router.register('functions', views.FunctionViewSet, basename='function') +router.register('requests', views.RequestViewSet, basename='request') + +# GET /api/v1/lambda/functions - get list of functions +# GET /api/v1/lambda/functions/ - get information about the function +# POST /api/v1/labmda/requests - call a function +# { "function": "", "mode": "online|offline", "job": "", "frame": "", +# "points": [...], } +# GET /api/v1/lambda/requests - get list of requests +# GET /api/v1/lambda/requests/ - get status of the request +# DEL /api/v1/lambda/requests/ - cancel a request (don't delete) +urlpatterns = [ + path('api/v1/lambda/', include((router.urls, 'cvat'), namespace='v1')) +] \ No newline at end of file diff --git a/cvat/apps/lambda_manager/views.py b/cvat/apps/lambda_manager/views.py new file mode 100644 index 000000000000..43ec2e1893d8 --- /dev/null +++ b/cvat/apps/lambda_manager/views.py @@ -0,0 +1,598 @@ +import base64 +import json +from functools import wraps +from enum import Enum + +import django_rq +import requests +import rq +from django.conf import settings +from django.core.exceptions import ObjectDoesNotExist, ValidationError +from rest_framework import status, viewsets +from rest_framework.response import Response + +from cvat.apps.authentication import auth +import cvat.apps.dataset_manager as dm +from cvat.apps.engine.frame_provider import FrameProvider +from cvat.apps.engine.models import Task as TaskModel +from cvat.apps.engine.serializers import LabeledDataSerializer +from rest_framework.permissions import IsAuthenticated +from cvat.apps.engine.models import ShapeType, SourceType + +class LambdaType(Enum): + DETECTOR = "detector" + INTERACTOR = "interactor" + REID = "reid" + TRACKER = "tracker" + UNKNOWN = "unknown" + + def __str__(self): + return self.value + +class LambdaGateway: + NUCLIO_ROOT_URL = '/api/functions' + + def _http(self, method="get", scheme=None, host=None, port=None, + url=None, headers=None, data=None): + NUCLIO_GATEWAY = '{}://{}:{}'.format( + scheme or settings.NUCLIO['SCHEME'], + host or settings.NUCLIO['HOST'], + port or settings.NUCLIO['PORT']) + extra_headers = { + 'x-nuclio-project-name': 'cvat', + 'x-nuclio-function-namespace': 'nuclio', + } + if headers: + extra_headers.update(headers) + NUCLIO_TIMEOUT = settings.NUCLIO['DEFAULT_TIMEOUT'] + + if url: + url = "{}{}".format(NUCLIO_GATEWAY, url) + else: + url = NUCLIO_GATEWAY + + reply = getattr(requests, method)(url, headers=extra_headers, + timeout=NUCLIO_TIMEOUT, json=data) + reply.raise_for_status() + response = reply.json() + + return response + + def list(self): + data = self._http(url=self.NUCLIO_ROOT_URL) + response = [LambdaFunction(self, item) for item in data.values()] + return response + + def get(self, func_id): + data = self._http(url=self.NUCLIO_ROOT_URL + '/' + func_id) + response = LambdaFunction(self, data) + return response + + def invoke(self, func, payload): + # NOTE: it is overhead to invoke a function using nuclio + # dashboard REST API. Better to call host.docker.internal: + # Look at https://github.com/docker/for-linux/issues/264. + # host.docker.internal isn't supported by docker on Linux. + # There are many workarounds but let's try to use the + # simple solution. + return self._http(method="post", url='/api/function_invocations', + data=payload, headers={ + 'x-nuclio-function-name': func.id, + 'x-nuclio-path': '/' + }) + +class LambdaFunction: + def __init__(self, gateway, data): + # ID of the function (e.g. omz.public.yolo-v3) + self.id = data['metadata']['name'] + # type of the function (e.g. detector, interactor) + kind = data['metadata']['annotations'].get('type') + try: + self.kind = LambdaType(kind) + except ValueError: + self.kind = LambdaType.UNKNOWN + # dictionary of labels for the function (e.g. car, person) + spec = json.loads(data['metadata']['annotations'].get('spec') or '[]') + labels = [item['name'] for item in spec] + if len(labels) != len(set(labels)): + raise ValidationError( + "`{}` lambda function has non-unique labels".format(self.id), + code=status.HTTP_404_NOT_FOUND) + self.labels = labels + # state of the function + self.state = data['status']['state'] + # description of the function + self.description = data['spec']['description'] + # http port to access the serverless function + self.port = data["status"].get("httpPort") + # framework which is used for the function (e.g. tensorflow, openvino) + self.framework = data['metadata']['annotations'].get('framework') + # display name for the function + self.name = data['metadata']['annotations'].get('name', self.id) + self.gateway = gateway + + def to_dict(self): + response = { + 'id': self.id, + 'kind': str(self.kind), + 'labels': self.labels, + 'state': self.state, + 'description': self.description, + 'framework': self.framework, + 'name': self.name, + } + + return response + + def invoke(self, db_task, data): + try: + payload = {} + threshold = data.get("threshold") + if threshold: + payload.update({ + "threshold": threshold, + }) + quality = data.get("quality") + mapping = data.get("mapping") + if self.kind == LambdaType.DETECTOR: + payload.update({ + "image": self._get_image(db_task, data["frame"], quality) + }) + elif self.kind == LambdaType.INTERACTOR: + payload.update({ + "image": self._get_image(db_task, data["frame"], quality), + "points": data["points"], + }) + elif self.kind == LambdaType.REID: + payload.update({ + "image0": self._get_image(db_task, data["frame0"], quality), + "image1": self._get_image(db_task, data["frame1"], quality), + "boxes0": data["boxes0"], + "boxes1": data["boxes1"] + }) + max_distance = data.get("max_distance") + if max_distance: + payload.update({ + "max_distance": max_distance + }) + else: + raise ValidationError( + '`{}` lambda function has incorrect type: {}' + .format(self.id, self.kind), + code=status.HTTP_500_INTERNAL_SERVER_ERROR) + except KeyError as err: + raise ValidationError( + "`{}` lambda function was called without mandatory argument: {}" + .format(self.id, str(err)), + code=status.HTTP_400_BAD_REQUEST) + + response = self.gateway.invoke(self, payload) + if self.kind == LambdaType.DETECTOR: + if mapping: + for item in response: + item["label"] = mapping.get(item["label"]) + response = [item for item in response if item["label"]] + + return response + + def _get_image(self, db_task, frame, quality): + if quality is None or quality == "original": + quality = FrameProvider.Quality.ORIGINAL + elif quality == "compressed": + quality = FrameProvider.Quality.COMPRESSED + else: + raise ValidationError( + '`{}` lambda function was run '.format(self.id) + + 'with wrong arguments (quality={})'.format(quality), + code=status.HTTP_400_BAD_REQUEST) + + frame_provider = FrameProvider(db_task.data) + image = frame_provider.get_frame(frame, quality=quality) + + return base64.b64encode(image[0].getvalue()).decode('utf-8') + + +class LambdaQueue: + def _get_queue(self): + QUEUE_NAME = "low" + return django_rq.get_queue(QUEUE_NAME) + + def get_jobs(self): + queue = self._get_queue() + # Only failed jobs are not included in the list below. + job_ids = set(queue.get_job_ids() + + queue.started_job_registry.get_job_ids() + + queue.finished_job_registry.get_job_ids() + + queue.scheduled_job_registry.get_job_ids() + + queue.deferred_job_registry.get_job_ids()) + jobs = queue.job_class.fetch_many(job_ids, queue.connection) + + return [LambdaJob(job) for job in jobs if job.meta.get("lambda")] + + def enqueue(self, lambda_func, threshold, task, quality, mapping, cleanup): + jobs = self.get_jobs() + # It is still possible to run several concurrent jobs for the same task. + # But the race isn't critical. The filtration is just a light-weight + # protection. + if list(filter(lambda job: job.get_task() == task and not job.is_finished, jobs)): + raise ValidationError( + "Only one running request is allowed for the same task #{}".format(task), + code=status.HTTP_409_CONFLICT) + + queue = self._get_queue() + # LambdaJob(None) is a workaround for python-rq. It has multiple issues + # with invocation of non-trivial functions. For example, it cannot run + # staticmethod, it cannot run a callable class. Thus I provide an object + # which has __call__ function. + job = queue.create_job(LambdaJob(None), + meta = { "lambda": True }, + kwargs = { + "function": lambda_func, + "threshold": threshold, + "task": task, + "quality": quality, + "cleanup": cleanup, + "mapping": mapping + }) + + queue.enqueue_job(job) + + return LambdaJob(job) + + def fetch_job(self, pk): + queue = self._get_queue() + job = queue.fetch_job(pk) + if job == None or not job.meta.get("lambda"): + raise ValidationError("{} lambda job is not found".format(pk), + code=status.HTTP_404_NOT_FOUND) + + return LambdaJob(job) + + +class LambdaJob: + def __init__(self, job): + self.job = job + + def to_dict(self): + lambda_func = self.job.kwargs.get("function") + return { + "id": self.job.id, + "function": { + "id": lambda_func.id if lambda_func else None, + "threshold": self.job.kwargs.get("threshold"), + "task": self.job.kwargs.get("task") + }, + "status": self.job.get_status(), + "progress": self.job.meta.get('progress', 0), + "enqueued": self.job.enqueued_at, + "started": self.job.started_at, + "ended": self.job.ended_at, + "exc_info": self.job.exc_info + } + + def get_task(self): + return self.job.kwargs.get("task") + + def get_status(self): + return self.job.get_status() + + @property + def is_finished(self): + return self.get_status() == rq.job.JobStatus.FINISHED + + @property + def is_queued(self): + return self.get_status() == rq.job.JobStatus.QUEUED + + @property + def is_failed(self): + return self.get_status() == rq.job.JobStatus.FAILED + + @property + def is_started(self): + return self.get_status() == rq.job.JobStatus.STARTED + + @property + def is_deferred(self): + return self.get_status() == rq.job.JobStatus.DEFERRED + + @property + def is_scheduled(self): + return self.get_status() == rq.job.JobStatus.SCHEDULED + + def delete(self): + self.job.delete() + + @staticmethod + def _call_detector(function, db_task, labels, quality, threshold, mapping): + class Results: + def __init__(self, task_id): + self.task_id = task_id + self.reset() + + def append_shape(self, shape): + self.data["shapes"].append(shape) + + def submit(self): + if not self.is_empty(): + serializer = LabeledDataSerializer(data=self.data) + if serializer.is_valid(raise_exception=True): + dm.task.patch_task_data(self.task_id, serializer.data, "create") + self.reset() + + def is_empty(self): + return not (self.data["tags"] or self.data["shapes"] or self.data["tracks"]) + + def reset(self): + # TODO: need to make "tags" and "tracks" are optional + # FIXME: need to provide the correct version here + self.data = {"version": 0, "tags": [], "shapes": [], "tracks": []} + + results = Results(db_task.id) + + for frame in range(db_task.data.size): + annotations = function.invoke(db_task, data={ + "frame": frame, "quality": quality, "mapping": mapping, + "threshold": threshold}) + progress = (frame + 1) / db_task.data.size + if not LambdaJob._update_progress(progress): + break + + for anno in annotations: + label_id = labels.get(anno["label"]) + if label_id is not None: + results.append_shape({ + "frame": frame, + "label_id": label_id, + "type": anno["type"], + "occluded": False, + "points": anno["points"], + "z_order": 0, + "group": None, + "attributes": [], + "source": "auto" + }) + + # Accumulate data during 100 frames before sumbitting results. + # It is optimization to make fewer calls to our server. Also + # it isn't possible to keep all results in memory. + if frame % 100 == 0: + results.submit() + + results.submit() + + @staticmethod + # progress is in [0, 1] range + def _update_progress(progress): + job = rq.get_current_job() + # If the job has been deleted, get_status will return None. Thus it will + # exist the loop. + job.meta["progress"] = int(progress * 100) + job.save_meta() + + return job.get_status() + + + @staticmethod + def _call_reid(function, db_task, quality, threshold, max_distance): + data = dm.task.get_task_data(db_task.id) + boxes_by_frame = [[] for _ in range(db_task.data.size)] + shapes_without_boxes = [] + for shape in data["shapes"]: + if shape["type"] == str(ShapeType.RECTANGLE): + boxes_by_frame[shape["frame"]].append(shape) + else: + shapes_without_boxes.append(shape) + + paths = {} + for frame in range(db_task.data.size - 1): + boxes0 = boxes_by_frame[frame] + for box in boxes0: + if "path_id" not in box: + path_id = len(paths) + paths[path_id] = [box] + box["path_id"] = path_id + + boxes1 = boxes_by_frame[frame + 1] + if boxes0 and boxes1: + matching = function.invoke(db_task, data={ + "frame0": frame, "frame1": frame + 1, "quality": quality, + "boxes0": boxes0, "boxes1": boxes1, "threshold": threshold, + "max_distance": max_distance}) + + for idx0, idx1 in enumerate(matching): + if idx1 >= 0: + path_id = boxes0[idx0]["path_id"] + boxes1[idx1]["path_id"] = path_id + paths[path_id].append(boxes1[idx1]) + + progress = (frame + 2) / db_task.data.size + if not LambdaJob._update_progress(progress): + break + + + for box in boxes_by_frame[db_task.data.size - 1]: + if "path_id" not in box: + path_id = len(paths) + paths[path_id] = [box] + box["path_id"] = path_id + + tracks = [] + for path_id in paths: + box0 = paths[path_id][0] + tracks.append({ + "label_id": box0["label_id"], + "group": None, + "attributes": [], + "frame": box0["frame"], + "shapes": paths[path_id], + "source": str(SourceType.AUTO) + }) + + for box in tracks[-1]["shapes"]: + box.pop("id", None) + box.pop("path_id") + box.pop("group") + box.pop("label_id") + box.pop("source") + box["outside"] = False + box["attributes"] = [] + + for track in tracks: + if track["shapes"][-1]["frame"] != db_task.data.size - 1: + box = track["shapes"][-1].copy() + box["outside"] = True + box["frame"] += 1 + track["shapes"].append(box) + + if tracks: + data["shapes"] = shapes_without_boxes + data["tracks"].extend(tracks) + + serializer = LabeledDataSerializer(data=data) + if serializer.is_valid(raise_exception=True): + dm.task.put_task_data(db_task.id, serializer.data) + + @staticmethod + def __call__(function, task, quality, cleanup, **kwargs): + # TODO: need logging + db_task = TaskModel.objects.get(pk=task) + if cleanup: + dm.task.delete_task_data(db_task.id) + db_labels = db_task.label_set.prefetch_related("attributespec_set").all() + labels = {db_label.name:db_label.id for db_label in db_labels} + + if function.kind == LambdaType.DETECTOR: + LambdaJob._call_detector(function, db_task, labels, quality, + kwargs.get("threshold"), kwargs.get("mapping")) + elif function.kind == LambdaType.REID: + LambdaJob._call_reid(function, db_task, quality, + kwargs.get("threshold"), kwargs.get("max_distance")) + + +def return_response(success_code=status.HTTP_200_OK): + def wrap_response(func): + @wraps(func) + def func_wrapper(*args, **kwargs): + data = None + status_code = success_code + try: + data = func(*args, **kwargs) + except requests.ConnectionError as err: + status_code = status.HTTP_503_SERVICE_UNAVAILABLE + data = str(err) + except requests.HTTPError as err: + status_code = err.response.status_code + data = str(err) + except requests.Timeout as err: + status_code = status.HTTP_504_GATEWAY_TIMEOUT + data = str(err) + except requests.RequestException as err: + status_code = status.HTTP_500_INTERNAL_SERVER_ERROR + data = str(err) + except ValidationError as err: + status_code = err.code + data = err.message + + return Response(data=data, status=status_code) + + return func_wrapper + return wrap_response + +class FunctionViewSet(viewsets.ViewSet): + lookup_value_regex = '[a-zA-Z0-9_.-]+' + lookup_field = 'func_id' + + def get_permissions(self): + http_method = self.request.method + permissions = [IsAuthenticated] + + if http_method in ["POST"]: + permissions.append(auth.TaskAccessPermission) + + return [perm() for perm in permissions] + + @return_response() + def list(self, request): + gateway = LambdaGateway() + return [f.to_dict() for f in gateway.list()] + + @return_response() + def retrieve(self, request, func_id): + gateway = LambdaGateway() + return gateway.get(func_id).to_dict() + + @return_response() + def call(self, request, func_id): + try: + task_id = request.data['task'] + db_task = TaskModel.objects.get(pk=task_id) + # Check that the user has enough permissions to read + # data from the task. + self.check_object_permissions(self.request, db_task) + except (KeyError, ObjectDoesNotExist) as err: + raise ValidationError( + '`{}` lambda function was run '.format(func_id) + + 'with wrong arguments ({})'.format(str(err)), + code=status.HTTP_400_BAD_REQUEST) + + gateway = LambdaGateway() + lambda_func = gateway.get(func_id) + + return lambda_func.invoke(db_task, request.data) + +class RequestViewSet(viewsets.ViewSet): + def get_permissions(self): + http_method = self.request.method + permissions = [IsAuthenticated] + + if http_method in ["POST", "DELETE"]: + permissions.append(auth.TaskChangePermission) + + return [perm() for perm in permissions] + + @return_response() + def list(self, request): + queue = LambdaQueue() + return [job.to_dict() for job in queue.get_jobs()] + + @return_response() + def create(self, request): + try: + function = request.data['function'] + threshold = request.data.get('threshold') + task = request.data['task'] + quality = request.data.get("quality") + cleanup = request.data.get('cleanup', False) + mapping = request.data.get('mapping') + + db_task = TaskModel.objects.get(pk=task) + # Check that the user has enough permissions to modify + # the task. + self.check_object_permissions(self.request, db_task) + except (KeyError, ObjectDoesNotExist) as err: + raise ValidationError( + '`{}` lambda function was run '.format(function) + + 'with wrong arguments ({})'.format(str(err)), + code=status.HTTP_400_BAD_REQUEST) + + gateway = LambdaGateway() + queue = LambdaQueue() + lambda_func = gateway.get(function) + job = queue.enqueue(lambda_func, threshold, task, quality, + mapping, cleanup) + + return job.to_dict() + + @return_response() + def retrieve(self, request, pk): + queue = LambdaQueue() + job = queue.fetch_job(pk) + + return job.to_dict() + + @return_response(status.HTTP_204_NO_CONTENT) + def delete(self, request, pk): + queue = LambdaQueue() + job = queue.fetch_job(pk) + job.delete() diff --git a/cvat/apps/reid/README.md b/cvat/apps/reid/README.md deleted file mode 100644 index 8cf29c503781..000000000000 --- a/cvat/apps/reid/README.md +++ /dev/null @@ -1,22 +0,0 @@ -# Re-Identification Application - -## About the application - -The ReID application uses deep learning model to perform an automatic bbox merging between neighbor frames. -You can use "Merge" and "Split" functionality to edit automatically generated annotation. - -## Installation - -This application will be installed automatically with the [OpenVINO](https://github.com/opencv/cvat/blob/develop/components/openvino/README.md) component. - -## Running - -For starting the ReID merge process: - -- Open an annotation job -- Open the menu -- Click the "Run ReID Merge" button -- Click the "Submit" button. Also here you can experiment with values of model threshold or maximum distance. - - Model threshold is maximum cosine distance between objects embeddings. - - Maximum distance defines a maximum radius that an object can diverge between neightbor frames. -- The process will be run. You can cancel it in the menu. diff --git a/cvat/apps/reid/__init__.py b/cvat/apps/reid/__init__.py deleted file mode 100644 index 4a9759cc68f5..000000000000 --- a/cvat/apps/reid/__init__.py +++ /dev/null @@ -1,9 +0,0 @@ -# Copyright (C) 2018 Intel Corporation -# -# SPDX-License-Identifier: MIT - -from cvat.settings.base import JS_3RDPARTY - -default_app_config = 'cvat.apps.reid.apps.ReidConfig' - -JS_3RDPARTY['engine'] = JS_3RDPARTY.get('engine', []) + ['reid/js/enginePlugin.js'] diff --git a/cvat/apps/reid/apps.py b/cvat/apps/reid/apps.py deleted file mode 100644 index f6aa66e1a5f9..000000000000 --- a/cvat/apps/reid/apps.py +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright (C) 2018 Intel Corporation -# -# SPDX-License-Identifier: MIT - -from django.apps import AppConfig - -class ReidConfig(AppConfig): - name = 'cvat.apps.reid' diff --git a/cvat/apps/reid/reid.py b/cvat/apps/reid/reid.py deleted file mode 100644 index bf79bac9a238..000000000000 --- a/cvat/apps/reid/reid.py +++ /dev/null @@ -1,227 +0,0 @@ -# Copyright (C) 2018 Intel Corporation -# -# SPDX-License-Identifier: MIT - -import os -import rq -import cv2 -import math -import numpy -import itertools - -from openvino.inference_engine import IENetwork, IEPlugin -from scipy.optimize import linear_sum_assignment -from scipy.spatial.distance import euclidean, cosine - -from cvat.apps.engine.models import Job -from cvat.apps.engine.frame_provider import FrameProvider - - -class ReID: - __threshold = None - __max_distance = None - __frame_urls = None - __frame_boxes = None - __stop_frame = None - __plugin = None - __executable_network = None - __input_blob_name = None - __output_blob_name = None - __input_height = None - __input_width = None - - - def __init__(self, jid, data): - self.__threshold = data["threshold"] - self.__max_distance = data["maxDistance"] - - self.__frame_boxes = {} - - db_job = Job.objects.select_related('segment__task').get(pk = jid) - db_segment = db_job.segment - db_task = db_segment.task - self.__frame_iter = itertools.islice( - FrameProvider(db_task.data).get_frames(FrameProvider.Quality.ORIGINAL), - db_segment.start_frame, - db_segment.stop_frame + 1, - ) - - self.__stop_frame = db_segment.stop_frame - for frame in range(db_segment.start_frame, db_segment.stop_frame + 1): - self.__frame_boxes[frame] = [box for box in data["boxes"] if box["frame"] == frame] - - IE_PLUGINS_PATH = os.getenv('IE_PLUGINS_PATH', None) - REID_MODEL_DIR = os.getenv('REID_MODEL_DIR', None) - - if not IE_PLUGINS_PATH: - raise Exception("Environment variable 'IE_PLUGINS_PATH' isn't defined") - if not REID_MODEL_DIR: - raise Exception("Environment variable 'REID_MODEL_DIR' isn't defined") - - REID_XML = os.path.join(REID_MODEL_DIR, "reid.xml") - REID_BIN = os.path.join(REID_MODEL_DIR, "reid.bin") - - self.__plugin = IEPlugin(device="CPU", plugin_dirs=[IE_PLUGINS_PATH]) - network = IENetwork.from_ir(model=REID_XML, weights=REID_BIN) - self.__input_blob_name = next(iter(network.inputs)) - self.__output_blob_name = next(iter(network.outputs)) - self.__input_height, self.__input_width = network.inputs[self.__input_blob_name].shape[-2:] - self.__executable_network = self.__plugin.load(network=network) - del network - - - def __del__(self): - if self.__executable_network: - del self.__executable_network - self.__executable_network = None - - if self.__plugin: - del self.__plugin - self.__plugin = None - - - def __boxes_are_compatible(self, cur_box, next_box): - cur_c_x = (cur_box["points"][0] + cur_box["points"][2]) / 2 - cur_c_y = (cur_box["points"][1] + cur_box["points"][3]) / 2 - next_c_x = (next_box["points"][0] + next_box["points"][2]) / 2 - next_c_y = (next_box["points"][1] + next_box["points"][3]) / 2 - compatible_distance = euclidean([cur_c_x, cur_c_y], [next_c_x, next_c_y]) <= self.__max_distance - compatible_label = cur_box["label_id"] == next_box["label_id"] - return compatible_distance and compatible_label and "path_id" not in next_box - - - def __compute_difference(self, image_1, image_2): - image_1 = cv2.resize(image_1, (self.__input_width, self.__input_height)).transpose((2,0,1)) - image_2 = cv2.resize(image_2, (self.__input_width, self.__input_height)).transpose((2,0,1)) - - input_1 = { - self.__input_blob_name: image_1[numpy.newaxis, ...] - } - - input_2 = { - self.__input_blob_name: image_2[numpy.newaxis, ...] - } - - embedding_1 = self.__executable_network.infer(inputs = input_1)[self.__output_blob_name] - embedding_2 = self.__executable_network.infer(inputs = input_2)[self.__output_blob_name] - - embedding_1 = embedding_1.reshape(embedding_1.size) - embedding_2 = embedding_2.reshape(embedding_2.size) - - return cosine(embedding_1, embedding_2) - - - def __compute_difference_matrix(self, cur_boxes, next_boxes, cur_image, next_image): - def _int(number, upper): - return math.floor(numpy.clip(number, 0, upper - 1)) - - default_mat_value = 1000.0 - - matrix = numpy.full([len(cur_boxes), len(next_boxes)], default_mat_value, dtype=float) - for row, cur_box in enumerate(cur_boxes): - cur_width = cur_image.shape[1] - cur_height = cur_image.shape[0] - cur_xtl, cur_xbr, cur_ytl, cur_ybr = ( - _int(cur_box["points"][0], cur_width), _int(cur_box["points"][2], cur_width), - _int(cur_box["points"][1], cur_height), _int(cur_box["points"][3], cur_height) - ) - - for col, next_box in enumerate(next_boxes): - next_box = next_boxes[col] - next_width = next_image.shape[1] - next_height = next_image.shape[0] - next_xtl, next_xbr, next_ytl, next_ybr = ( - _int(next_box["points"][0], next_width), _int(next_box["points"][2], next_width), - _int(next_box["points"][1], next_height), _int(next_box["points"][3], next_height) - ) - - if not self.__boxes_are_compatible(cur_box, next_box): - continue - - crop_1 = cur_image[cur_ytl:cur_ybr, cur_xtl:cur_xbr] - crop_2 = next_image[next_ytl:next_ybr, next_xtl:next_xbr] - matrix[row][col] = self.__compute_difference(crop_1, crop_2) - - return matrix - - - def __apply_matching(self): - frames = sorted(list(self.__frame_boxes.keys())) - job = rq.get_current_job() - box_tracks = {} - - next_image = cv2.imdecode(numpy.fromstring((next(self.__frame_iter)[0]).read(), numpy.uint8), cv2.IMREAD_COLOR) - for idx, (cur_frame, next_frame) in enumerate(list(zip(frames[:-1], frames[1:]))): - job.refresh() - if "cancel" in job.meta: - return None - - job.meta["progress"] = idx * 100.0 / len(frames) - job.save_meta() - - cur_boxes = self.__frame_boxes[cur_frame] - next_boxes = self.__frame_boxes[next_frame] - - for box in cur_boxes: - if "path_id" not in box: - path_id = len(box_tracks) - box_tracks[path_id] = [box] - box["path_id"] = path_id - - if not (len(cur_boxes) and len(next_boxes)): - continue - - cur_image = next_image - next_image = cv2.imdecode(numpy.fromstring((next(self.__frame_iter)[0]).read(), numpy.uint8), cv2.IMREAD_COLOR) - difference_matrix = self.__compute_difference_matrix(cur_boxes, next_boxes, cur_image, next_image) - cur_idxs, next_idxs = linear_sum_assignment(difference_matrix) - for idx, cur_idx in enumerate(cur_idxs): - if (difference_matrix[cur_idx][next_idxs[idx]]) <= self.__threshold: - cur_box = cur_boxes[cur_idx] - next_box = next_boxes[next_idxs[idx]] - next_box["path_id"] = cur_box["path_id"] - box_tracks[cur_box["path_id"]].append(next_box) - - for box in self.__frame_boxes[frames[-1]]: - if "path_id" not in box: - path_id = len(box_tracks) - box["path_id"] = path_id - box_tracks[path_id] = [box] - - return box_tracks - - - def run(self): - box_tracks = self.__apply_matching() - output = [] - - # ReID process has been canceled - if box_tracks is None: - return - - for path_id in box_tracks: - output.append({ - "label_id": box_tracks[path_id][0]["label_id"], - "group": None, - "attributes": [], - "frame": box_tracks[path_id][0]["frame"], - "shapes": box_tracks[path_id] - }) - - for box in output[-1]["shapes"]: - if "id" in box: - del box["id"] - del box["path_id"] - del box["group"] - del box["label_id"] - box["outside"] = False - box["attributes"] = [] - - for path in output: - if path["shapes"][-1]["frame"] != self.__stop_frame: - copy = path["shapes"][-1].copy() - copy["outside"] = True - copy["frame"] += 1 - path["shapes"].append(copy) - - return output diff --git a/cvat/apps/reid/static/reid/js/enginePlugin.js b/cvat/apps/reid/static/reid/js/enginePlugin.js deleted file mode 100644 index 28fabb82319c..000000000000 --- a/cvat/apps/reid/static/reid/js/enginePlugin.js +++ /dev/null @@ -1,174 +0,0 @@ -/* - * Copyright (C) 2018 Intel Corporation - * - * SPDX-License-Identifier: MIT - */ - -/* global showMessage userConfirm */ - - -document.addEventListener('DOMContentLoaded', () => { - async function run(overlay, cancelButton, thresholdInput, distanceInput) { - const collection = window.cvat.data.get(); - const data = { - threshold: +thresholdInput.prop('value'), - maxDistance: +distanceInput.prop('value'), - boxes: collection.shapes.filter(el => el.type === 'rectangle'), - }; - - overlay.removeClass('hidden'); - cancelButton.prop('disabled', true); - - async function checkCallback() { - let jobData = null; - try { - jobData = await $.get(`/reid/check/${window.cvat.job.id}`); - } catch (errorData) { - overlay.addClass('hidden'); - const message = `Can not check ReID merge. Code: ${errorData.status}. ` - + `Message: ${errorData.responseText || errorData.statusText}`; - showMessage(message); - } - - if (jobData.progress) { - cancelButton.text(`Cancel ReID Merge (${jobData.progress.toString().slice(0, 4)}%)`); - } - - if (['queued', 'started'].includes(jobData.status)) { - setTimeout(checkCallback, 1000); - } else { - overlay.addClass('hidden'); - - if (jobData.status === 'finished') { - if (jobData.result) { - const result = JSON.parse(jobData.result); - collection.shapes = collection.shapes - .filter(el => el.type !== 'rectangle'); - collection.tracks = collection.tracks - .concat(result); - - window.cvat.data.clear(); - window.cvat.data.set(collection); - - showMessage('ReID merge has done.'); - } else { - showMessage('ReID merge been canceled.'); - } - } else if (jobData.status === 'failed') { - const message = `ReID merge has fallen. Error: '${jobData.stderr}'`; - showMessage(message); - } else { - let message = `Check request returned "${jobData.status}" status.`; - if (jobData.stderr) { - message += ` Error: ${jobData.stderr}`; - } - showMessage(message); - } - } - } - - try { - await $.ajax({ - url: `/reid/start/job/${window.cvat.job.id}`, - type: 'POST', - data: JSON.stringify(data), - contentType: 'application/json', - }); - - setTimeout(checkCallback, 1000); - } catch (errorData) { - overlay.addClass('hidden'); - const message = `Can not start ReID merge. Code: ${errorData.status}. ` - + `Message: ${errorData.responseText || errorData.statusText}`; - showMessage(message); - } finally { - cancelButton.prop('disabled', false); - } - } - - async function cancel(overlay, cancelButton) { - cancelButton.prop('disabled', true); - try { - await $.get(`/reid/cancel/${window.cvat.job.id}`); - overlay.addClass('hidden'); - cancelButton.text('Cancel ReID Merge (0%)'); - } catch (errorData) { - const message = `Can not cancel ReID process. Code: ${errorData.status}. Message: ${errorData.responseText || errorData.statusText}`; - showMessage(message); - } finally { - cancelButton.prop('disabled', false); - } - } - - const buttonsUI = $('#engineMenuButtons'); - const reidWindowId = 'reidSubmitWindow'; - const reidThresholdValueId = 'reidThresholdValue'; - const reidDistanceValueId = 'reidDistanceValue'; - const reidCancelMergeId = 'reidCancelMerge'; - const reidSubmitMergeId = 'reidSubmitMerge'; - const reidCancelButtonId = 'reidCancelReID'; - const reidOverlay = 'reidOverlay'; - - $('').on('click', () => { - $('#annotationMenu').addClass('hidden'); - $(`#${reidWindowId}`).removeClass('hidden'); - }).addClass('menuButton semiBold h2').prependTo(buttonsUI); - - $(` - - `).appendTo('body'); - - $(` - - `).appendTo('body'); - - $(`#${reidCancelMergeId}`).on('click', () => { - $(`#${reidWindowId}`).addClass('hidden'); - }); - - $(`#${reidCancelButtonId}`).on('click', () => { - userConfirm('ReID process will be canceld. Are you sure?', () => { - cancel($(`#${reidOverlay}`), $(`#${reidCancelButtonId}`)); - }); - }); - - $(`#${reidSubmitMergeId}`).on('click', () => { - $(`#${reidWindowId}`).addClass('hidden'); - run($(`#${reidOverlay}`), $(`#${reidCancelButtonId}`), - $(`#${reidThresholdValueId}`), $(`#${reidDistanceValueId}`)) - .catch((error) => { - setTimeout(() => { - throw error; - }); - }); - }); -}); diff --git a/cvat/apps/reid/urls.py b/cvat/apps/reid/urls.py deleted file mode 100644 index 431b11192ec7..000000000000 --- a/cvat/apps/reid/urls.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright (C) 2018-2020 Intel Corporation -# -# SPDX-License-Identifier: MIT - -from django.urls import path -from . import views - -urlpatterns = [ - path('start/job/', views.start), - path('cancel/', views.cancel), - path('check/', views.check), - path('enabled', views.enabled), -] diff --git a/cvat/apps/reid/views.py b/cvat/apps/reid/views.py deleted file mode 100644 index 100151ee9d1b..000000000000 --- a/cvat/apps/reid/views.py +++ /dev/null @@ -1,99 +0,0 @@ -# Copyright (C) 2018 Intel Corporation -# -# SPDX-License-Identifier: MIT - -from django.http import HttpResponse, HttpResponseBadRequest, JsonResponse -from cvat.apps.authentication.decorators import login_required -from rules.contrib.views import permission_required, objectgetter - -from cvat.apps.engine.models import Job -from cvat.apps.reid.reid import ReID - -import django_rq -import json -import rq - - -def _create_thread(jid, data): - job = rq.get_current_job() - reid_obj = ReID(jid, data) - job.meta["result"] = json.dumps(reid_obj.run()) - job.save_meta() - - -@login_required -@permission_required(perm=["engine.job.change"], - fn=objectgetter(Job, 'jid'), raise_exception=True) -def start(request, jid): - try: - data = json.loads(request.body.decode('utf-8')) - queue = django_rq.get_queue("low") - job_id = "reid.create.{}".format(jid) - job = queue.fetch_job(job_id) - if job is not None and (job.is_started or job.is_queued): - raise Exception('ReID process has been already started') - queue.enqueue_call(func=_create_thread, args=(jid, data), job_id=job_id, timeout=7200) - job = queue.fetch_job(job_id) - job.meta = {} - job.save_meta() - except Exception as e: - return HttpResponseBadRequest(str(e)) - - return HttpResponse() - - -@login_required -@permission_required(perm=["engine.job.change"], - fn=objectgetter(Job, 'jid'), raise_exception=True) -def check(request, jid): - try: - queue = django_rq.get_queue("low") - rq_id = "reid.create.{}".format(jid) - job = queue.fetch_job(rq_id) - if job is not None and "cancel" in job.meta: - return JsonResponse({"status": "finished"}) - data = {} - if job is None: - data["status"] = "unknown" - elif job.is_queued: - data["status"] = "queued" - elif job.is_started: - data["status"] = "started" - if "progress" in job.meta: - data["progress"] = job.meta["progress"] - elif job.is_finished: - data["status"] = "finished" - data["result"] = job.meta["result"] - job.delete() - else: - data["status"] = "failed" - data["stderr"] = job.exc_info - job.delete() - - except Exception as ex: - data["stderr"] = str(ex) - data["status"] = "unknown" - - return JsonResponse(data) - - -@login_required -@permission_required(perm=["engine.job.change"], - fn=objectgetter(Job, 'jid'), raise_exception=True) -def cancel(request, jid): - try: - queue = django_rq.get_queue("low") - rq_id = "reid.create.{}".format(jid) - job = queue.fetch_job(rq_id) - if job is None or job.is_finished or job.is_failed: - raise Exception("Task is not being annotated currently") - elif "cancel" not in job.meta: - job.meta["cancel"] = True - job.save_meta() - except Exception as e: - return HttpResponseBadRequest(str(e)) - - return HttpResponse() - -def enabled(request): - return HttpResponse() diff --git a/cvat/apps/tf_annotation/__init__.py b/cvat/apps/tf_annotation/__init__.py deleted file mode 100644 index a0fca4cb39ea..000000000000 --- a/cvat/apps/tf_annotation/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ - -# Copyright (C) 2018-2019 Intel Corporation -# -# SPDX-License-Identifier: MIT diff --git a/cvat/apps/tf_annotation/admin.py b/cvat/apps/tf_annotation/admin.py deleted file mode 100644 index af8dfc47525b..000000000000 --- a/cvat/apps/tf_annotation/admin.py +++ /dev/null @@ -1,9 +0,0 @@ - -# Copyright (C) 2018 Intel Corporation -# -# SPDX-License-Identifier: MIT - -from django.contrib import admin - -# Register your models here. - diff --git a/cvat/apps/tf_annotation/apps.py b/cvat/apps/tf_annotation/apps.py deleted file mode 100644 index d07a37b1e932..000000000000 --- a/cvat/apps/tf_annotation/apps.py +++ /dev/null @@ -1,11 +0,0 @@ - -# Copyright (C) 2018 Intel Corporation -# -# SPDX-License-Identifier: MIT - -from django.apps import AppConfig - - -class TFAnnotationConfig(AppConfig): - name = 'tf_annotation' - diff --git a/cvat/apps/tf_annotation/migrations/__init__.py b/cvat/apps/tf_annotation/migrations/__init__.py deleted file mode 100644 index d8e62e54b356..000000000000 --- a/cvat/apps/tf_annotation/migrations/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ - -# Copyright (C) 2018 Intel Corporation -# -# SPDX-License-Identifier: MIT - diff --git a/cvat/apps/tf_annotation/models.py b/cvat/apps/tf_annotation/models.py deleted file mode 100644 index cdf3b0827bf1..000000000000 --- a/cvat/apps/tf_annotation/models.py +++ /dev/null @@ -1,9 +0,0 @@ - -# Copyright (C) 2018 Intel Corporation -# -# SPDX-License-Identifier: MIT - -from django.db import models - -# Create your models here. - diff --git a/cvat/apps/tf_annotation/tests.py b/cvat/apps/tf_annotation/tests.py deleted file mode 100644 index 53bc3b7adb85..000000000000 --- a/cvat/apps/tf_annotation/tests.py +++ /dev/null @@ -1,9 +0,0 @@ - -# Copyright (C) 2018 Intel Corporation -# -# SPDX-License-Identifier: MIT - -from django.test import TestCase - -# Create your tests here. - diff --git a/cvat/apps/tf_annotation/urls.py b/cvat/apps/tf_annotation/urls.py deleted file mode 100644 index f84019be9693..000000000000 --- a/cvat/apps/tf_annotation/urls.py +++ /dev/null @@ -1,14 +0,0 @@ - -# Copyright (C) 2018 Intel Corporation -# -# SPDX-License-Identifier: MIT - -from django.urls import path -from . import views - -urlpatterns = [ - path('create/task/', views.create), - path('check/task/', views.check), - path('cancel/task/', views.cancel), - path('meta/get', views.get_meta_info), -] diff --git a/cvat/apps/tf_annotation/views.py b/cvat/apps/tf_annotation/views.py deleted file mode 100644 index e112860e5219..000000000000 --- a/cvat/apps/tf_annotation/views.py +++ /dev/null @@ -1,347 +0,0 @@ - -# Copyright (C) 2018-2020 Intel Corporation -# -# SPDX-License-Identifier: MIT - -from django.http import HttpResponse, JsonResponse, HttpResponseBadRequest -from rest_framework.decorators import api_view -from rules.contrib.views import permission_required, objectgetter -from cvat.apps.authentication.decorators import login_required -from cvat.apps.dataset_manager.task import put_task_data -from cvat.apps.engine.models import Task as TaskModel -from cvat.apps.engine.serializers import LabeledDataSerializer -from cvat.apps.engine.frame_provider import FrameProvider - -import django_rq -import os -import rq - -import tensorflow as tf -import numpy as np - -from PIL import Image -from cvat.apps.engine.log import slogger - - -def load_image_into_numpy(image): - (im_width, im_height) = image.size - return np.array(image.getdata()).reshape((im_height, im_width, 3)).astype(np.uint8) - - -def run_inference_engine_annotation(image_list, labels_mapping, treshold): - from cvat.apps.auto_annotation.inference_engine import make_plugin_or_core, make_network - - def _normalize_box(box, w, h, dw, dh): - xmin = min(int(box[0] * dw * w), w) - ymin = min(int(box[1] * dh * h), h) - xmax = min(int(box[2] * dw * w), w) - ymax = min(int(box[3] * dh * h), h) - return xmin, ymin, xmax, ymax - - result = {} - MODEL_PATH = os.environ.get('TF_ANNOTATION_MODEL_PATH') - if MODEL_PATH is None: - raise OSError('Model path env not found in the system.') - - core_or_plugin = make_plugin_or_core() - network = make_network('{}.xml'.format(MODEL_PATH), '{}.bin'.format(MODEL_PATH)) - input_blob_name = next(iter(network.inputs)) - output_blob_name = next(iter(network.outputs)) - if getattr(core_or_plugin, 'load_network', False): - executable_network = core_or_plugin.load_network(network, 'CPU') - else: - executable_network = core_or_plugin.load(network=network) - job = rq.get_current_job() - - del network - - try: - for image_num, im_name in enumerate(image_list): - - job.refresh() - if 'cancel' in job.meta: - del job.meta['cancel'] - job.save() - return None - job.meta['progress'] = image_num * 100 / len(image_list) - job.save_meta() - - image = Image.open(im_name) - width, height = image.size - image.thumbnail((600, 600), Image.ANTIALIAS) - dwidth, dheight = 600 / image.size[0], 600 / image.size[1] - image = image.crop((0, 0, 600, 600)) - image_np = load_image_into_numpy(image) - image_np = np.transpose(image_np, (2, 0, 1)) - prediction = executable_network.infer(inputs={input_blob_name: image_np[np.newaxis, ...]})[output_blob_name][0][0] - for obj in prediction: - obj_class = int(obj[1]) - obj_value = obj[2] - if obj_class and obj_class in labels_mapping and obj_value >= treshold: - label = labels_mapping[obj_class] - if label not in result: - result[label] = [] - xmin, ymin, xmax, ymax = _normalize_box(obj[3:7], width, height, dwidth, dheight) - result[label].append([image_num, xmin, ymin, xmax, ymax]) - finally: - del executable_network - del plugin - - return result - - -def run_tensorflow_annotation(frame_provider, labels_mapping, treshold): - def _normalize_box(box, w, h): - xmin = int(box[1] * w) - ymin = int(box[0] * h) - xmax = int(box[3] * w) - ymax = int(box[2] * h) - return xmin, ymin, xmax, ymax - - result = {} - model_path = os.environ.get('TF_ANNOTATION_MODEL_PATH') - if model_path is None: - raise OSError('Model path env not found in the system.') - job = rq.get_current_job() - - detection_graph = tf.Graph() - with detection_graph.as_default(): - od_graph_def = tf.GraphDef() - with tf.gfile.GFile(model_path + '.pb', 'rb') as fid: - serialized_graph = fid.read() - od_graph_def.ParseFromString(serialized_graph) - tf.import_graph_def(od_graph_def, name='') - - try: - config = tf.ConfigProto() - config.gpu_options.allow_growth=True - sess = tf.Session(graph=detection_graph, config=config) - frames = frame_provider.get_frames(frame_provider.Quality.ORIGINAL) - for image_num, (image, _) in enumerate(frames): - - job.refresh() - if 'cancel' in job.meta: - del job.meta['cancel'] - job.save() - return None - job.meta['progress'] = image_num * 100 / len(frame_provider) - job.save_meta() - - image = Image.open(image) - width, height = image.size - if width > 1920 or height > 1080: - image = image.resize((width // 2, height // 2), Image.ANTIALIAS) - image_np = load_image_into_numpy(image) - image_np_expanded = np.expand_dims(image_np, axis=0) - - image_tensor = detection_graph.get_tensor_by_name('image_tensor:0') - boxes = detection_graph.get_tensor_by_name('detection_boxes:0') - scores = detection_graph.get_tensor_by_name('detection_scores:0') - classes = detection_graph.get_tensor_by_name('detection_classes:0') - num_detections = detection_graph.get_tensor_by_name('num_detections:0') - (boxes, scores, classes, num_detections) = sess.run([boxes, scores, classes, num_detections], feed_dict={image_tensor: image_np_expanded}) - - for i in range(len(classes[0])): - if classes[0][i] in labels_mapping.keys(): - if scores[0][i] >= treshold: - xmin, ymin, xmax, ymax = _normalize_box(boxes[0][i], width, height) - label = labels_mapping[classes[0][i]] - if label not in result: - result[label] = [] - result[label].append([image_num, xmin, ymin, xmax, ymax]) - finally: - sess.close() - del sess - return result - -def convert_to_cvat_format(data): - result = { - "tracks": [], - "shapes": [], - "tags": [], - "version": 0, - } - - for label in data: - boxes = data[label] - for box in boxes: - result['shapes'].append({ - "type": "rectangle", - "label_id": label, - "frame": box[0], - "points": [box[1], box[2], box[3], box[4]], - "z_order": 0, - "group": None, - "occluded": False, - "attributes": [], - "source": "auto", - }) - - return result - -def create_thread(tid, labels_mapping, user): - try: - TRESHOLD = 0.5 - # Init rq job - job = rq.get_current_job() - job.meta['progress'] = 0 - job.save_meta() - # Get job indexes and segment length - db_task = TaskModel.objects.get(pk=tid) - # Get image list - image_list = FrameProvider(db_task.data) - - # Run auto annotation by tf - result = None - slogger.glob.info("tf annotation with tensorflow framework for task {}".format(tid)) - result = run_tensorflow_annotation(image_list, labels_mapping, TRESHOLD) - - if result is None: - slogger.glob.info('tf annotation for task {} canceled by user'.format(tid)) - return - - # Modify data format and save - result = convert_to_cvat_format(result) - serializer = LabeledDataSerializer(data = result) - if serializer.is_valid(raise_exception=True): - put_task_data(tid, result) - slogger.glob.info('tf annotation for task {} done'.format(tid)) - except Exception as ex: - try: - slogger.task[tid].exception('exception was occured during tf annotation of the task', exc_info=True) - except: - slogger.glob.exception('exception was occured during tf annotation of the task {}'.format(tid), exc_info=True) - raise ex - -@api_view(['POST']) -@login_required -def get_meta_info(request): - try: - queue = django_rq.get_queue('low') - tids = request.data - result = {} - for tid in tids: - job = queue.fetch_job('tf_annotation.create/{}'.format(tid)) - if job is not None: - result[tid] = { - "active": job.is_queued or job.is_started, - "success": not job.is_failed - } - - return JsonResponse(result) - except Exception as ex: - slogger.glob.exception('exception was occured during tf meta request', exc_info=True) - return HttpResponseBadRequest(str(ex)) - - -@login_required -@permission_required(perm=['engine.task.change'], - fn=objectgetter(TaskModel, 'tid'), raise_exception=True) -def create(request, tid): - slogger.glob.info('tf annotation create request for task {}'.format(tid)) - try: - db_task = TaskModel.objects.get(pk=tid) - queue = django_rq.get_queue('low') - job = queue.fetch_job('tf_annotation.create/{}'.format(tid)) - if job is not None and (job.is_started or job.is_queued): - raise Exception("The process is already running") - - db_labels = db_task.label_set.prefetch_related('attributespec_set').all() - db_labels = {db_label.id:db_label.name for db_label in db_labels} - - tf_annotation_labels = { - "person": 1, "bicycle": 2, "car": 3, "motorcycle": 4, "airplane": 5, - "bus": 6, "train": 7, "truck": 8, "boat": 9, "traffic_light": 10, - "fire_hydrant": 11, "stop_sign": 13, "parking_meter": 14, "bench": 15, - "bird": 16, "cat": 17, "dog": 18, "horse": 19, "sheep": 20, "cow": 21, - "elephant": 22, "bear": 23, "zebra": 24, "giraffe": 25, "backpack": 27, - "umbrella": 28, "handbag": 31, "tie": 32, "suitcase": 33, "frisbee": 34, - "skis": 35, "snowboard": 36, "sports_ball": 37, "kite": 38, "baseball_bat": 39, - "baseball_glove": 40, "skateboard": 41, "surfboard": 42, "tennis_racket": 43, - "bottle": 44, "wine_glass": 46, "cup": 47, "fork": 48, "knife": 49, "spoon": 50, - "bowl": 51, "banana": 52, "apple": 53, "sandwich": 54, "orange": 55, "broccoli": 56, - "carrot": 57, "hot_dog": 58, "pizza": 59, "donut": 60, "cake": 61, "chair": 62, - "couch": 63, "potted_plant": 64, "bed": 65, "dining_table": 67, "toilet": 70, - "tv": 72, "laptop": 73, "mouse": 74, "remote": 75, "keyboard": 76, "cell_phone": 77, - "microwave": 78, "oven": 79, "toaster": 80, "sink": 81, "refrigerator": 83, - "book": 84, "clock": 85, "vase": 86, "scissors": 87, "teddy_bear": 88, "hair_drier": 89, - "toothbrush": 90 - } - - labels_mapping = {} - for key, labels in db_labels.items(): - if labels in tf_annotation_labels.keys(): - labels_mapping[tf_annotation_labels[labels]] = key - - if not len(labels_mapping.values()): - raise Exception('No labels found for tf annotation') - - # Run tf annotation job - queue.enqueue_call(func=create_thread, - args=(tid, labels_mapping, request.user), - job_id='tf_annotation.create/{}'.format(tid), - timeout=604800) # 7 days - - slogger.task[tid].info('tensorflow annotation job enqueued with labels {}'.format(labels_mapping)) - - except Exception as ex: - try: - slogger.task[tid].exception("exception was occured during tensorflow annotation request", exc_info=True) - except: - pass - return HttpResponseBadRequest(str(ex)) - - return HttpResponse() - -@login_required -@permission_required(perm=['engine.task.access'], - fn=objectgetter(TaskModel, 'tid'), raise_exception=True) -def check(request, tid): - try: - queue = django_rq.get_queue('low') - job = queue.fetch_job('tf_annotation.create/{}'.format(tid)) - if job is not None and 'cancel' in job.meta: - return JsonResponse({'status': 'finished'}) - data = {} - if job is None: - data['status'] = 'unknown' - elif job.is_queued: - data['status'] = 'queued' - elif job.is_started: - data['status'] = 'started' - data['progress'] = job.meta['progress'] - elif job.is_finished: - data['status'] = 'finished' - job.delete() - else: - data['status'] = 'failed' - data['stderr'] = job.exc_info - job.delete() - - except Exception: - data['status'] = 'unknown' - - return JsonResponse(data) - - -@login_required -@permission_required(perm=['engine.task.change'], - fn=objectgetter(TaskModel, 'tid'), raise_exception=True) -def cancel(request, tid): - try: - queue = django_rq.get_queue('low') - job = queue.fetch_job('tf_annotation.create/{}'.format(tid)) - if job is None or job.is_finished or job.is_failed: - raise Exception('Task is not being annotated currently') - elif 'cancel' not in job.meta: - job.meta['cancel'] = True - job.save() - - except Exception as ex: - try: - slogger.task[tid].exception("cannot cancel tensorflow annotation for task #{}".format(tid), exc_info=True) - except: - pass - return HttpResponseBadRequest(str(ex)) - - return HttpResponse() diff --git a/cvat/requirements/base.txt b/cvat/requirements/base.txt index b2847cbdef61..aa6afd90befa 100644 --- a/cvat/requirements/base.txt +++ b/cvat/requirements/base.txt @@ -4,7 +4,7 @@ django-appconf==1.0.4 django-auth-ldap==2.2.0 django-cacheops==5.0.1 django-compressor==2.4 -django-rq==2.0.0 +django-rq==2.3.2 EasyProcess==0.3 Pillow==7.2.0 numpy==1.18.5 @@ -15,7 +15,7 @@ rcssmin==1.0.6 redis==3.5.3 rjsmin==1.1.0 requests==2.24.0 -rq==1.0.0 +rq==1.4.2 rq-scheduler==0.10.0 scipy==1.4.1 sqlparse==0.3.1 diff --git a/cvat/settings/base.py b/cvat/settings/base.py index 9155f2b38c68..15fe3264637a 100644 --- a/cvat/settings/base.py +++ b/cvat/settings/base.py @@ -99,6 +99,7 @@ def generate_ssh_keys(): 'cvat.apps.engine', 'cvat.apps.git', 'cvat.apps.restrictions', + 'cvat.apps.lambda_manager', 'django_rq', 'compressor', 'cacheops', @@ -160,25 +161,9 @@ def generate_ssh_keys(): 'REGISTER_SERIALIZER': 'cvat.apps.restrictions.serializers.RestrictedRegisterSerializer' } -if 'yes' == os.environ.get('TF_ANNOTATION', 'no'): - INSTALLED_APPS += ['cvat.apps.tf_annotation'] - -if 'yes' == os.environ.get('OPENVINO_TOOLKIT', 'no'): - INSTALLED_APPS += ['cvat.apps.auto_annotation'] - -if 'yes' == os.environ.get('OPENVINO_TOOLKIT', 'no') and os.environ.get('REID_MODEL_DIR', ''): - INSTALLED_APPS += ['cvat.apps.reid'] - -if 'yes' == os.environ.get('WITH_DEXTR', 'no'): - INSTALLED_APPS += ['cvat.apps.dextr_segmentation'] - if os.getenv('DJANGO_LOG_VIEWER_HOST'): INSTALLED_APPS += ['cvat.apps.log_viewer'] -# new feature by Mohammad -if 'yes' == os.environ.get('AUTO_SEGMENTATION', 'no'): - INSTALLED_APPS += ['cvat.apps.auto_segmentation'] - MIDDLEWARE = [ 'django.middleware.security.SecurityMiddleware', 'django.contrib.sessions.middleware.SessionMiddleware', @@ -254,6 +239,13 @@ def generate_ssh_keys(): } } +NUCLIO = { + 'SCHEME': 'http', + 'HOST': 'localhost', + 'PORT': 8070, + 'DEFAULT_TIMEOUT': 120 +} + RQ_SHOW_ADMIN_LINK = True RQ_EXCEPTION_HANDLERS = ['cvat.apps.engine.views.rq_handler'] diff --git a/cvat/settings/production.py b/cvat/settings/production.py index ae059bee06e8..d6cea8988adc 100644 --- a/cvat/settings/production.py +++ b/cvat/settings/production.py @@ -10,6 +10,8 @@ 'mod_wsgi.server', ] +NUCLIO['HOST'] = os.getenv('CVAT_NUCLIO_HOST', 'nuclio') + for key in RQ_QUEUES: RQ_QUEUES[key]['HOST'] = os.getenv('CVAT_REDIS_HOST', 'cvat_redis') diff --git a/cvat/urls.py b/cvat/urls.py index 6ae59f6b03aa..0d652e9b42fd 100644 --- a/cvat/urls.py +++ b/cvat/urls.py @@ -31,27 +31,14 @@ path('documentation/', include('cvat.apps.documentation.urls')), ] -if apps.is_installed('cvat.apps.tf_annotation'): - urlpatterns.append(path('tensorflow/annotation/', include('cvat.apps.tf_annotation.urls'))) - if apps.is_installed('cvat.apps.git'): urlpatterns.append(path('git/repository/', include('cvat.apps.git.urls'))) -if apps.is_installed('cvat.apps.reid'): - urlpatterns.append(path('reid/', include('cvat.apps.reid.urls'))) - -if apps.is_installed('cvat.apps.auto_annotation'): - urlpatterns.append(path('auto_annotation/', include('cvat.apps.auto_annotation.urls'))) - -if apps.is_installed('cvat.apps.dextr_segmentation'): - urlpatterns.append(path('dextr/', include('cvat.apps.dextr_segmentation.urls'))) - if apps.is_installed('cvat.apps.log_viewer'): urlpatterns.append(path('analytics/', include('cvat.apps.log_viewer.urls'))) +if apps.is_installed('cvat.apps.lambda_manager'): + urlpatterns.append(path('', include('cvat.apps.lambda_manager.urls'))) + if apps.is_installed('silk'): urlpatterns.append(path('profiler/', include('silk.urls'))) - -# new feature by Mohammad -if apps.is_installed('cvat.apps.auto_segmentation'): - urlpatterns.append(path('tensorflow/segmentation/', include('cvat.apps.auto_segmentation.urls'))) diff --git a/cvat_proxy/conf.d/cvat.conf.template b/cvat_proxy/conf.d/cvat.conf.template index 084ce189ea7a..287f72849d7e 100644 --- a/cvat_proxy/conf.d/cvat.conf.template +++ b/cvat_proxy/conf.d/cvat.conf.template @@ -12,7 +12,7 @@ server { proxy_set_header Host $http_host; proxy_pass_header Set-Cookie; - location ~* /api/.*|git/.*|tensorflow/.*|auto_annotation/.*|analytics/.*|static/.*|admin|admin/.*|documentation/.*|dextr/.*|reid/.* { + location ~* /api/.*|git/.*|analytics/.*|static/.*|admin|admin/.*|documentation/.* { proxy_pass http://cvat:8080; } diff --git a/datumaro/.gitignore b/datumaro/.gitignore new file mode 100644 index 000000000000..17c3ea8bcfbf --- /dev/null +++ b/datumaro/.gitignore @@ -0,0 +1 @@ +/datumaro.egg-info diff --git a/docker-compose.yml b/docker-compose.yml index 0b8aebfbd6b5..60a5965bdb54 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -32,7 +32,7 @@ services: cvat: container_name: cvat - image: cvat + image: cvat/server restart: always depends_on: - cvat_redis @@ -44,12 +44,9 @@ services: https_proxy: no_proxy: socks_proxy: - TF_ANNOTATION: "no" - AUTO_SEGMENTATION: "no" USER: "django" DJANGO_CONFIGURATION: "production" TZ: "Etc/UTC" - OPENVINO_TOOLKIT: "no" CLAM_AV: "no" environment: DJANGO_MODWSGI_EXTRA_ARGS: "" @@ -64,6 +61,7 @@ services: cvat_ui: container_name: cvat_ui + image: cvat/ui restart: always build: context: . @@ -97,6 +95,22 @@ services: - ./cvat_proxy/conf.d/cvat.conf.template:/etc/nginx/conf.d/cvat.conf.template:ro command: /bin/sh -c "envsubst '$$CVAT_HOST' < /etc/nginx/conf.d/cvat.conf.template > /etc/nginx/conf.d/default.conf && nginx -g 'daemon off;'" + serverless: + container_name: nuclio + image: quay.io/nuclio/dashboard:1.4.8-amd64 + restart: always + networks: + default: + aliases: + - nuclio + volumes: + - /tmp:/tmp + - /var/run/docker.sock:/var/run/docker.sock + environment: + NUCLIO_CHECK_FUNCTION_CONTAINERS_HEALTHINESS: "true" + ports: + - "8070:8070" + volumes: cvat_db: cvat_data: diff --git a/serverless/deploy.sh b/serverless/deploy.sh new file mode 100755 index 000000000000..556f1d3e9f9b --- /dev/null +++ b/serverless/deploy.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" + +nuctl create project cvat +nuctl deploy --project-name cvat \ + --path $SCRIPT_DIR/openvino/omz/public/faster_rcnn_inception_v2_coco/nuclio \ + --volume $SCRIPT_DIR/openvino/common:/opt/nuclio/common + +nuctl deploy --project-name cvat \ + --path $SCRIPT_DIR/openvino/omz/public/mask_rcnn_inception_resnet_v2_atrous_coco/nuclio \ + --volume $SCRIPT_DIR/openvino/common:/opt/nuclio/common + +nuctl deploy --project-name cvat \ + --path $SCRIPT_DIR/openvino/omz/public/yolo-v3-tf/nuclio \ + --volume $SCRIPT_DIR/openvino/common:/opt/nuclio/common + +nuctl deploy --project-name cvat \ + --path $SCRIPT_DIR/openvino/omz/intel/text-detection-0004/nuclio \ + --volume $SCRIPT_DIR/openvino/common:/opt/nuclio/common + +nuctl deploy --project-name cvat \ + --path $SCRIPT_DIR/openvino/omz/intel/semantic-segmentation-adas-0001/nuclio \ + --volume $SCRIPT_DIR/openvino/common:/opt/nuclio/common + +nuctl deploy --project-name cvat \ + --path $SCRIPT_DIR/openvino/omz/intel/person-reidentification-retail-300/nuclio \ + --volume $SCRIPT_DIR/openvino/common:/opt/nuclio/common + +nuctl deploy --project-name cvat \ + --path $SCRIPT_DIR/openvino/dextr/nuclio \ + --volume $SCRIPT_DIR/openvino/common:/opt/nuclio/common + +nuctl deploy --project-name cvat \ + --path $SCRIPT_DIR/tensorflow/matterport/mask_rcnn/nuclio + +nuctl deploy --project-name cvat \ + --path $SCRIPT_DIR/tensorflow/faster_rcnn_inception_v2_coco/nuclio + +nuctl get function diff --git a/serverless/openvino/common/model_loader.py b/serverless/openvino/common/model_loader.py new file mode 100644 index 000000000000..ca3354d1aee3 --- /dev/null +++ b/serverless/openvino/common/model_loader.py @@ -0,0 +1,72 @@ + +# Copyright (C) 2020 Intel Corporation +# +# SPDX-License-Identifier: MIT + +import cv2 +import numpy as np +from openvino.inference_engine import IECore + +class ModelLoader: + def __init__(self, model, weights): + ie_core = IECore() + network = ie_core.read_network(model, weights) + self._network = network + + # Check compatibility + supported_layers = ie_core.query_network(network, "CPU") + not_supported_layers = [l for l in network.layers.keys() if l not in supported_layers] + if len(not_supported_layers) != 0: + raise Exception( + "Following layers are not supported by the plugin for specified device {}:\n {}" + .format(ie_core.device, ", ".join(not_supported_layers))) + + # Initialize input blobs + self._input_info_name = None + for blob_name in network.inputs: + if len(network.inputs[blob_name].shape) == 4: + self._input_blob_name = blob_name + elif len(network.inputs[blob_name].shape) == 2: + self._input_info_name = blob_name + else: + raise RuntimeError( + "Unsupported {}D input layer '{}'. Only 2D and 4D input layers are supported" + .format(len(network.inputs[blob_name].shape), blob_name)) + + # Initialize output blob + self._output_blob_name = next(iter(network.outputs)) + + # Load network + self._net = ie_core.load_network(network, "CPU", num_requests=2) + input_type = network.inputs[self._input_blob_name] + self._input_layout = input_type if isinstance(input_type, list) else input_type.shape + + def infer(self, image, preprocessing=True): + image = np.array(image) + _, _, h, w = self._input_layout + if preprocessing: + image = image if image.shape[:-1] == (h, w) else cv2.resize(image, (w, h)) + if len(image.shape) < 3: # grayscale image + image = image[:, :, np.newaxis] + else: + if image.shape[2] == 4: # the image has alpha channel + image = image[:, :, :3] + + image = image.transpose((2, 0, 1)) # Change data layout from HWC to CHW + + inputs = {self._input_blob_name: image} + if self._input_info_name: + inputs[self._input_info_name] = [h, w, 1] + + results = self._net.infer(inputs) + if len(results) == 1: + return results[self._output_blob_name].copy() + else: + return results.copy() + + def input_size(self): + return self._input_layout[2:] + + @property + def layers(self): + return self._network.layers diff --git a/serverless/openvino/common/python3 b/serverless/openvino/common/python3 new file mode 100755 index 000000000000..fca7518d4099 --- /dev/null +++ b/serverless/openvino/common/python3 @@ -0,0 +1,7 @@ +#!/bin/bash + +args=$@ + +. /opt/intel/openvino/bin/setupvars.sh +PYTHONPATH=/opt/nuclio/common:$PYTHONPATH +/usr/bin/python3 $args diff --git a/serverless/openvino/dextr/nuclio/function.yaml b/serverless/openvino/dextr/nuclio/function.yaml new file mode 100644 index 000000000000..08fbf14f7d6d --- /dev/null +++ b/serverless/openvino/dextr/nuclio/function.yaml @@ -0,0 +1,54 @@ +metadata: + name: openvino.dextr + namespace: cvat + annotations: + name: DEXTR + type: interactor + spec: + framework: openvino + +spec: + description: Deep Extreme Cut + runtime: "python:3.6" + handler: main:handler + eventTimeout: 30s + env: + - name: NUCLIO_PYTHON_EXE_PATH + value: /opt/nuclio/common/python3 + + build: + image: cvat/openvino.dextr + baseImage: openvino/ubuntu18_runtime:2020.2 + + directives: + preCopy: + - kind: USER + value: root + - kind: WORKDIR + value: /opt/nuclio + - kind: RUN + value: ln -s /usr/bin/pip3 /usr/bin/pip + + postCopy: + - kind: RUN + value: curl -O https://download.01.org/openvinotoolkit/models_contrib/cvat/dextr_model_v1.zip + - kind: RUN + value: unzip dextr_model_v1.zip + - kind: RUN + value: pip3 install Pillow + - kind: USER + value: openvino + + triggers: + myHttpTrigger: + maxWorkers: 2 + kind: "http" + workerAvailabilityTimeoutMilliseconds: 10000 + attributes: + maxRequestBodySize: 33554432 # 32MB + + platform: + attributes: + restartPolicy: + name: always + maximumRetryCount: 3 diff --git a/serverless/openvino/dextr/nuclio/main.py b/serverless/openvino/dextr/nuclio/main.py new file mode 100644 index 000000000000..73617002c69d --- /dev/null +++ b/serverless/openvino/dextr/nuclio/main.py @@ -0,0 +1,26 @@ +import json +import base64 +from PIL import Image +import io +from model_handler import ModelHandler + +def init_context(context): + context.logger.info("Init context... 0%") + + model = ModelHandler() + setattr(context.user_data, 'model', model) + + context.logger.info("Init context...100%") + +def handler(context, event): + context.logger.info("call handler") + data = event.body + points = data["points"] + buf = io.BytesIO(base64.b64decode(data["image"].encode('utf-8'))) + image = Image.open(buf) + + polygon = context.user_data.model.handle(image, points) + return context.Response(body=json.dumps(polygon), + headers={}, + content_type='application/json', + status_code=200) diff --git a/serverless/openvino/dextr/nuclio/model_handler.py b/serverless/openvino/dextr/nuclio/model_handler.py new file mode 100644 index 000000000000..9dc74116805b --- /dev/null +++ b/serverless/openvino/dextr/nuclio/model_handler.py @@ -0,0 +1,84 @@ +# Copyright (C) 2018-2020 Intel Corporation +# +# SPDX-License-Identifier: MIT + +import cv2 +import numpy as np +import os +from model_loader import ModelLoader + +class ModelHandler: + def __init__(self): + base_dir = os.environ.get("MODEL_PATH", "/opt/nuclio") + model_xml = os.path.join(base_dir, "dextr.xml") + model_bin = os.path.join(base_dir, "dextr.bin") + self.model = ModelLoader(model_xml, model_bin) + + # Input: + # image: PIL image + # points: [[x1,y1], [x2,y2], [x3,y3], [x4,y4], ...] + # Output: + # polygon: [[x1,y1], [x2,y2], [x3,y3], [x4,y4], ...] + def handle(self, image, points): + DEXTR_PADDING = 50 + DEXTR_TRESHOLD = 0.9 + DEXTR_SIZE = 512 + + numpy_image = np.array(image) + points = np.asarray(points, dtype=int) + bounding_box = ( + max(min(points[:, 0]) - DEXTR_PADDING, 0), + max(min(points[:, 1]) - DEXTR_PADDING, 0), + min(max(points[:, 0]) + DEXTR_PADDING, numpy_image.shape[1] - 1), + min(max(points[:, 1]) + DEXTR_PADDING, numpy_image.shape[0] - 1) + ) + + # Prepare an image + numpy_cropped = np.array(image.crop(bounding_box)) + resized = cv2.resize(numpy_cropped, (DEXTR_SIZE, DEXTR_SIZE), + interpolation = cv2.INTER_CUBIC).astype(np.float32) + if len(resized.shape) == 2: # support grayscale images + resized = cv2.cvtColor(resized, cv2.COLOR_GRAY2RGB) + elif resized.shape[2] == 4: # remove alpha channel + resized = resized[:, :, :3] + + # Make a heatmap + points = points - [min(points[:, 0]), min(points[:, 1])] + [DEXTR_PADDING, DEXTR_PADDING] + points = (points * [DEXTR_SIZE / numpy_cropped.shape[1], DEXTR_SIZE / numpy_cropped.shape[0]]).astype(int) + heatmap = np.zeros(shape=resized.shape[:2], dtype=np.float64) + for point in points: + gaussian_x_axis = np.arange(0, DEXTR_SIZE, 1, float) - point[0] + gaussian_y_axis = np.arange(0, DEXTR_SIZE, 1, float)[:, np.newaxis] - point[1] + gaussian = np.exp(-4 * np.log(2) * ((gaussian_x_axis ** 2 + gaussian_y_axis ** 2) / 100)).astype(np.float64) + heatmap = np.maximum(heatmap, gaussian) + cv2.normalize(heatmap, heatmap, 0, 255, cv2.NORM_MINMAX) + + # Concat an image and a heatmap + input_dextr = np.concatenate((resized, heatmap[:, :, np.newaxis].astype(resized.dtype)), axis=2) + input_dextr = input_dextr.transpose((2,0,1)) + + pred = self.model.infer(input_dextr[np.newaxis, ...], False)[0, 0, :, :] + pred = cv2.resize(pred, tuple(reversed(numpy_cropped.shape[:2])), interpolation = cv2.INTER_CUBIC) + result = np.zeros(numpy_image.shape[:2]) + result[bounding_box[1]:bounding_box[1] + pred.shape[0], bounding_box[0]:bounding_box[0] + pred.shape[1]] = pred > DEXTR_TRESHOLD + + # Convert a mask to a polygon + result = np.array(result, dtype=np.uint8) + cv2.normalize(result,result,0,255,cv2.NORM_MINMAX) + contours = None + if int(cv2.__version__.split('.')[0]) > 3: + contours = cv2.findContours(result, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_TC89_KCOS)[0] + else: + contours = cv2.findContours(result, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_TC89_KCOS)[1] + + contours = max(contours, key=lambda arr: arr.size) + if contours.shape.count(1): + contours = np.squeeze(contours) + if contours.size < 3 * 2: + raise Exception('Less then three point have been detected. Can not build a polygon.') + + result = [] + for point in contours: + result.append([int(point[0]), int(point[1])]) + + return result diff --git a/serverless/openvino/omz/intel/person-reidentification-retail-300/nuclio/function.yaml b/serverless/openvino/omz/intel/person-reidentification-retail-300/nuclio/function.yaml new file mode 100644 index 000000000000..1057efb07786 --- /dev/null +++ b/serverless/openvino/omz/intel/person-reidentification-retail-300/nuclio/function.yaml @@ -0,0 +1,52 @@ +metadata: + name: openvino.omz.intel.person-reidentification-retail-0300 + namespace: cvat + annotations: + name: Person reidentification + type: reid + framework: openvino + spec: + +spec: + description: Person reidentification model for a general scenario + runtime: "python:3.6" + handler: main:handler + eventTimeout: 30s + env: + - name: NUCLIO_PYTHON_EXE_PATH + value: /opt/nuclio/common/python3 + + build: + image: cvat/openvino.omz.intel.person-reidentification-retail-0300 + baseImage: openvino/ubuntu18_dev:2020.2 + + directives: + preCopy: + - kind: USER + value: root + - kind: WORKDIR + value: /opt/nuclio + - kind: RUN + value: ln -s /usr/bin/pip3 /usr/bin/pip + - kind: RUN + value: /opt/intel/openvino/deployment_tools/open_model_zoo/tools/downloader/downloader.py --name person-reidentification-retail-0300 -o /opt/nuclio/open_model_zoo + - kind: RUN + value: /opt/intel/openvino/deployment_tools/open_model_zoo/tools/downloader/converter.py --name person-reidentification-retail-0300 --precisions FP32 -d /opt/nuclio/open_model_zoo -o /opt/nuclio/open_model_zoo + + postCopy: + - kind: USER + value: openvino + + triggers: + myHttpTrigger: + maxWorkers: 2 + kind: "http" + workerAvailabilityTimeoutMilliseconds: 10000 + attributes: + maxRequestBodySize: 33554432 # 32MB + + platform: + attributes: + restartPolicy: + name: always + maximumRetryCount: 3 diff --git a/serverless/openvino/omz/intel/person-reidentification-retail-300/nuclio/main.py b/serverless/openvino/omz/intel/person-reidentification-retail-300/nuclio/main.py new file mode 100644 index 000000000000..9197632ae830 --- /dev/null +++ b/serverless/openvino/omz/intel/person-reidentification-retail-300/nuclio/main.py @@ -0,0 +1,31 @@ +import json +import base64 +from PIL import Image +import io +from model_handler import ModelHandler + +def init_context(context): + context.logger.info("Init context... 0%") + + model = ModelHandler() + setattr(context.user_data, 'model', model) + + context.logger.info("Init context...100%") + +def handler(context, event): + context.logger.info("Run person-reidentification-retail-0300 model") + data = event.body + buf0 = io.BytesIO(base64.b64decode(data["image0"].encode('utf-8'))) + buf1 = io.BytesIO(base64.b64decode(data["image1"].encode('utf-8'))) + threshold = float(data.get("threshold", 0.5)) + max_distance = float(data.get("max_distance", 50)) + image0 = Image.open(buf0) + image1 = Image.open(buf1) + boxes0 = data["boxes0"] + boxes1 = data["boxes1"] + + results = context.user_data.model.infer(image0, boxes0, + image1, boxes1, threshold, max_distance) + + return context.Response(body=json.dumps(results), headers={}, + content_type='application/json', status_code=200) diff --git a/serverless/openvino/omz/intel/person-reidentification-retail-300/nuclio/model_handler.py b/serverless/openvino/omz/intel/person-reidentification-retail-300/nuclio/model_handler.py new file mode 100644 index 000000000000..481fa4c42cfd --- /dev/null +++ b/serverless/openvino/omz/intel/person-reidentification-retail-300/nuclio/model_handler.py @@ -0,0 +1,81 @@ +# Copyright (C) 2018 Intel Corporation +# +# SPDX-License-Identifier: MIT + +import math +import numpy +import os +from scipy.optimize import linear_sum_assignment +from scipy.spatial.distance import euclidean, cosine + +from model_loader import ModelLoader + +class ModelHandler: + def __init__(self): + base_dir = os.environ.get("MODEL_PATH", + "/opt/nuclio/open_model_zoo/intel/person-reidentification-retail-0300/FP32") + model_xml = os.path.join(base_dir, "person-reidentification-retail-0300.xml") + model_bin = os.path.join(base_dir, "person-reidentification-retail-0300.bin") + + self.model = ModelLoader(model_xml, model_bin) + + def infer(self, image0, boxes0, image1, boxes1, threshold, distance): + similarity_matrix = self._compute_similarity_matrix(image0, + boxes0, image1, boxes1, distance) + row_idx, col_idx = linear_sum_assignment(similarity_matrix) + results = [-1] * len(boxes0) + for idx0, idx1 in zip(row_idx, col_idx): + if similarity_matrix[idx0, idx1] <= threshold: + results[idx0] = int(idx1) + + return results + + def _match_boxes(self, box0, box1, distance): + cx0 = (box0["points"][0] + box0["points"][2]) / 2 + cy0 = (box0["points"][1] + box0["points"][3]) / 2 + cx1 = (box1["points"][0] + box1["points"][2]) / 2 + cy1 = (box1["points"][1] + box1["points"][3]) / 2 + is_good_distance = euclidean([cx0, cy0], [cx1, cy1]) <= distance + is_same_label = box0["label_id"] == box1["label_id"] + + return is_good_distance and is_same_label + + def _match_crops(self, crop0, crop1): + embedding0 = self.model.infer(crop0) + embedding1 = self.model.infer(crop1) + + embedding0 = embedding0.reshape(embedding0.size) + embedding1 = embedding1.reshape(embedding1.size) + + return cosine(embedding0, embedding1) + + def _compute_similarity_matrix(self, image0, boxes0, image1, boxes1, + distance): + def _int(number, upper): + return math.floor(numpy.clip(number, 0, upper - 1)) + + DISTANCE_INF = 1000.0 + + matrix = numpy.full([len(boxes0), len(boxes1)], DISTANCE_INF, dtype=float) + for row, box0 in enumerate(boxes0): + w0, h0 = image0.size + xtl0, xbr0, ytl0, ybr0 = ( + _int(box0["points"][0], w0), _int(box0["points"][2], w0), + _int(box0["points"][1], h0), _int(box0["points"][3], h0) + ) + + for col, box1 in enumerate(boxes1): + w1, h1 = image1.size + xtl1, xbr1, ytl1, ybr1 = ( + _int(box1["points"][0], w1), _int(box1["points"][2], w1), + _int(box1["points"][1], h1), _int(box1["points"][3], h1) + ) + + if not self._match_boxes(box0, box1, distance): + continue + + crop0 = image0.crop((xtl0, ytl0, xbr0, ybr0)) + crop1 = image1.crop((xtl1, ytl1, xbr1, ybr1)) + matrix[row][col] = self._match_crops(crop0, crop1) + + return matrix diff --git a/serverless/openvino/omz/intel/semantic-segmentation-adas-0001/nuclio/function.yaml b/serverless/openvino/omz/intel/semantic-segmentation-adas-0001/nuclio/function.yaml new file mode 100644 index 000000000000..0f3f4454fd72 --- /dev/null +++ b/serverless/openvino/omz/intel/semantic-segmentation-adas-0001/nuclio/function.yaml @@ -0,0 +1,78 @@ +metadata: + name: openvino.omz.semantic-segmentation-adas-0001 + namespace: cvat + annotations: + name: Semantic segmentation for ADAS + type: detector + framework: openvino + spec: | + [ + { "id": 0, "name": "road" }, + { "id": 1, "name": "sidewalk" }, + { "id": 2, "name": "building" }, + { "id": 3, "name": "wall" }, + { "id": 4, "name": "fence" }, + { "id": 5, "name": "pole" }, + { "id": 6, "name": "traffic light" }, + { "id": 7, "name": "traffic sign" }, + { "id": 8, "name": "vegetation" }, + { "id": 9, "name": "terrain" }, + { "id": 10, "name": "sky" }, + { "id": 11, "name": "person" }, + { "id": 12, "name": "rider" }, + { "id": 13, "name": "car" }, + { "id": 14, "name": "truck" }, + { "id": 15, "name": "bus" }, + { "id": 16, "name": "train" }, + { "id": 17, "name": "motorcycle" }, + { "id": 18, "name": "bicycle" }, + { "id": 19, "name": "ego-vehicle" }, + { "id": 20, "name": "background" } + ] + +spec: + description: Segmentation network to classify each pixel into typical 20 classes for ADAS + runtime: "python:3.6" + handler: main:handler + eventTimeout: 30s + env: + - name: NUCLIO_PYTHON_EXE_PATH + value: /opt/nuclio/common/python3 + + build: + image: cvat/openvino.omz.intel.semantic-segmentation-adas-0001 + baseImage: openvino/ubuntu18_dev:2020.2 + + directives: + preCopy: + - kind: USER + value: root + - kind: WORKDIR + value: /opt/nuclio + - kind: RUN + value: ln -s /usr/bin/pip3 /usr/bin/pip + - kind: RUN + value: /opt/intel/openvino/deployment_tools/open_model_zoo/tools/downloader/downloader.py --name semantic-segmentation-adas-0001 -o /opt/nuclio/open_model_zoo + + + postCopy: + - kind: RUN + value: apt update && DEBIAN_FRONTEND=noninteractive apt install --no-install-recommends -y python3-skimage + - kind: RUN + value: pip3 install "numpy<1.16.0" # workaround for skimage + - kind: USER + value: openvino + + triggers: + myHttpTrigger: + maxWorkers: 2 + kind: "http" + workerAvailabilityTimeoutMilliseconds: 10000 + attributes: + maxRequestBodySize: 33554432 # 32MB + + platform: + attributes: + restartPolicy: + name: always + maximumRetryCount: 3 diff --git a/serverless/openvino/omz/intel/semantic-segmentation-adas-0001/nuclio/main.py b/serverless/openvino/omz/intel/semantic-segmentation-adas-0001/nuclio/main.py new file mode 100644 index 000000000000..9942bc029779 --- /dev/null +++ b/serverless/openvino/omz/intel/semantic-segmentation-adas-0001/nuclio/main.py @@ -0,0 +1,32 @@ +import json +import base64 +from PIL import Image +import io +from model_handler import ModelHandler +import yaml + +def init_context(context): + context.logger.info("Init context... 0%") + + # Read labels + functionconfig = yaml.safe_load(open("/opt/nuclio/function.yaml")) + labels_spec = functionconfig['metadata']['annotations']['spec'] + labels = {item['id']: item['name'] for item in json.loads(labels_spec)} + + # Read the DL model + model = ModelHandler(labels) + setattr(context.user_data, 'model', model) + + context.logger.info("Init context...100%") + +def handler(context, event): + context.logger.info("Run semantic-segmentation-adas-0001 model") + data = event.body + buf = io.BytesIO(base64.b64decode(data["image"].encode('utf-8'))) + threshold = float(data.get("threshold", 0.5)) + image = Image.open(buf) + + results = context.user_data.model.infer(image, threshold) + + return context.Response(body=json.dumps(results), headers={}, + content_type='application/json', status_code=200) diff --git a/serverless/openvino/omz/intel/semantic-segmentation-adas-0001/nuclio/model_handler.py b/serverless/openvino/omz/intel/semantic-segmentation-adas-0001/nuclio/model_handler.py new file mode 100644 index 000000000000..5b4bb7221da5 --- /dev/null +++ b/serverless/openvino/omz/intel/semantic-segmentation-adas-0001/nuclio/model_handler.py @@ -0,0 +1,50 @@ +# Copyright (C) 2020 Intel Corporation +# +# SPDX-License-Identifier: MIT + +import os +import cv2 +import numpy as np +from skimage.measure import approximate_polygon, find_contours +from model_loader import ModelLoader + +class ModelHandler: + def __init__(self, labels): + base_dir = os.environ.get("MODEL_PATH", + "/opt/nuclio/open_model_zoo/intel/semantic-segmentation-adas-0001/FP32") + model_xml = os.path.join(base_dir, "semantic-segmentation-adas-0001.xml") + model_bin = os.path.join(base_dir, "semantic-segmentation-adas-0001.bin") + self.model = ModelLoader(model_xml, model_bin) + self.labels = labels + + def infer(self, image, threshold): + output_layer = self.model.infer(image) + + results = [] + mask = output_layer[0, 0, :, :] + width, height = mask.shape + + for i in range(len(self.labels)): + mask_by_label = np.zeros((width, height), dtype=np.uint8) + + mask_by_label = ((mask == float(i)) * 255).astype(np.float32) + mask_by_label = cv2.resize(mask_by_label, + dsize=(image.width, image.height), + interpolation=cv2.INTER_CUBIC) + + contours = find_contours(mask_by_label, 0.8) + + for contour in contours: + contour = np.flip(contour, axis=1) + contour = approximate_polygon(contour, tolerance=2.5) + if len(contour) < 3: + continue + + results.append({ + "confidence": None, + "label": self.labels.get(i, "unknown"), + "points": contour.ravel().tolist(), + "type": "polygon", + }) + + return results \ No newline at end of file diff --git a/serverless/openvino/omz/intel/text-detection-0004/nuclio/function.yaml b/serverless/openvino/omz/intel/text-detection-0004/nuclio/function.yaml new file mode 100644 index 000000000000..864ed2d1ff0c --- /dev/null +++ b/serverless/openvino/omz/intel/text-detection-0004/nuclio/function.yaml @@ -0,0 +1,53 @@ +metadata: + name: openvino.omz.intel.text-detection-0004 + namespace: cvat + annotations: + name: Text detection v4 + type: detector + framework: openvino + spec: | + [ + { "id": 1, "name": "text" } + ] + +spec: + description: Text detector based on PixelLink architecture with MobileNetV2-like as a backbone for indoor/outdoor scenes. + runtime: "python:3.6" + handler: main:handler + eventTimeout: 30s + env: + - name: NUCLIO_PYTHON_EXE_PATH + value: /opt/nuclio/common/python3 + + build: + image: cvat/openvino.omz.intel.text-detection-0004 + baseImage: openvino/ubuntu18_dev:2020.2 + + directives: + preCopy: + - kind: USER + value: root + - kind: WORKDIR + value: /opt/nuclio + - kind: RUN + value: ln -s /usr/bin/pip3 /usr/bin/pip + - kind: RUN + value: /opt/intel/openvino/deployment_tools/open_model_zoo/tools/downloader/downloader.py --name text-detection-0004 -o /opt/nuclio/open_model_zoo + + postCopy: + - kind: USER + value: openvino + + triggers: + myHttpTrigger: + maxWorkers: 2 + kind: "http" + workerAvailabilityTimeoutMilliseconds: 10000 + attributes: + maxRequestBodySize: 33554432 # 32MB + + platform: + attributes: + restartPolicy: + name: always + maximumRetryCount: 3 diff --git a/serverless/openvino/omz/intel/text-detection-0004/nuclio/main.py b/serverless/openvino/omz/intel/text-detection-0004/nuclio/main.py new file mode 100644 index 000000000000..ab54e76cb02d --- /dev/null +++ b/serverless/openvino/omz/intel/text-detection-0004/nuclio/main.py @@ -0,0 +1,34 @@ +import json +import base64 +from PIL import Image +import io +from model_handler import ModelHandler +import yaml + +def init_context(context): + context.logger.info("Init context... 0%") + + # Read labels + functionconfig = yaml.safe_load(open("/opt/nuclio/function.yaml")) + labels_spec = functionconfig['metadata']['annotations']['spec'] + labels = {item['id']: item['name'] for item in json.loads(labels_spec)} + + # Read the DL model + model = ModelHandler(labels) + setattr(context.user_data, 'model', model) + + context.logger.info("Init context...100%") + +def handler(context, event): + context.logger.info("Run text-detection-0004 model") + data = event.body + buf = io.BytesIO(base64.b64decode(data["image"].encode('utf-8'))) + pixel_threshold = float(data.get("pixel_threshold", 0.8)) + link_threshold = float(data.get("link_threshold", 0.8)) + image = Image.open(buf) + + results = context.user_data.model.infer(image, + pixel_threshold, link_threshold) + + return context.Response(body=json.dumps(results), headers={}, + content_type='application/json', status_code=200) diff --git a/utils/open_model_zoo/Retail/object_detection/text/pixel_link_mobilenet_v2/0004/pixel_link_mobilenet_v2.py b/serverless/openvino/omz/intel/text-detection-0004/nuclio/model_handler.py similarity index 83% rename from utils/open_model_zoo/Retail/object_detection/text/pixel_link_mobilenet_v2/0004/pixel_link_mobilenet_v2.py rename to serverless/openvino/omz/intel/text-detection-0004/nuclio/model_handler.py index b0f105ec5a12..6ea78c1e509f 100644 --- a/utils/open_model_zoo/Retail/object_detection/text/pixel_link_mobilenet_v2/0004/pixel_link_mobilenet_v2.py +++ b/serverless/openvino/omz/intel/text-detection-0004/nuclio/model_handler.py @@ -1,18 +1,21 @@ -# SPDX-License-Identifier: MIT` +# Copyright (C) 2020 Intel Corporation +# +# SPDX-License-Identifier: MIT +import os import cv2 import numpy as np - +from model_loader import ModelLoader class PixelLinkDecoder(): - def __init__(self): + def __init__(self, pixel_threshold, link_threshold): four_neighbours = False if four_neighbours: self._get_neighbours = self._get_neighbours_4 else: self._get_neighbours = self._get_neighbours_8 - self.pixel_conf_threshold = 0.8 - self.link_conf_threshold = 0.8 + self.pixel_conf_threshold = pixel_threshold + self.link_conf_threshold = link_threshold def decode(self, height, width, detections: dict): self.image_height = height @@ -186,12 +189,29 @@ def _decode(self): self._get_all() self._mask_to_bboxes() - -label = 1 -pcd = PixelLinkDecoder() -for detection in detections: - frame = detection['frame_id'] - pcd.decode(detection['frame_height'], detection['frame_width'], detection['detections']) - for box in pcd.bboxes: - box = [[int(b[0]), int(b[1])] for b in box] - results.add_polygon(box, label, frame) +class ModelHandler: + def __init__(self, labels): + base_dir = os.environ.get("MODEL_PATH", + "/opt/nuclio/open_model_zoo/intel/text-detection-0004/FP32") + model_xml = os.path.join(base_dir, "text-detection-0004.xml") + model_bin = os.path.join(base_dir, "text-detection-0004.bin") + self.model = ModelLoader(model_xml, model_bin) + self.labels = labels + + def infer(self, image, pixel_threshold, link_threshold): + output_layer = self.model.infer(image) + + results = [] + obj_class = 1 + pcd = PixelLinkDecoder(pixel_threshold, link_threshold) + + pcd.decode(image.height, image.width, output_layer) + for box in pcd.bboxes: + results.append({ + "confidence": None, + "label": self.labels.get(obj_class, "unknown"), + "points": box.ravel().tolist(), + "type": "polygon", + }) + + return results \ No newline at end of file diff --git a/serverless/openvino/omz/public/faster_rcnn_inception_v2_coco/nuclio/function.yaml b/serverless/openvino/omz/public/faster_rcnn_inception_v2_coco/nuclio/function.yaml new file mode 100644 index 000000000000..b8ea9f537fa5 --- /dev/null +++ b/serverless/openvino/omz/public/faster_rcnn_inception_v2_coco/nuclio/function.yaml @@ -0,0 +1,134 @@ +metadata: + name: openvino.omz.public.faster_rcnn_inception_v2_coco + namespace: cvat + annotations: + name: Faster RCNN + type: detector + framework: openvino + spec: | + [ + { "id": 1, "name": "person" }, + { "id": 2, "name": "bicycle" }, + { "id": 3, "name": "car" }, + { "id": 4, "name": "motorcycle" }, + { "id": 5, "name": "airplane" }, + { "id": 6, "name": "bus" }, + { "id": 7, "name": "train" }, + { "id": 8, "name": "truck" }, + { "id": 9, "name": "boat" }, + { "id":10, "name": "traffic_light" }, + { "id":11, "name": "fire_hydrant" }, + { "id":13, "name": "stop_sign" }, + { "id":14, "name": "parking_meter" }, + { "id":15, "name": "bench" }, + { "id":16, "name": "bird" }, + { "id":17, "name": "cat" }, + { "id":18, "name": "dog" }, + { "id":19, "name": "horse" }, + { "id":20, "name": "sheep" }, + { "id":21, "name": "cow" }, + { "id":22, "name": "elephant" }, + { "id":23, "name": "bear" }, + { "id":24, "name": "zebra" }, + { "id":25, "name": "giraffe" }, + { "id":27, "name": "backpack" }, + { "id":28, "name": "umbrella" }, + { "id":31, "name": "handbag" }, + { "id":32, "name": "tie" }, + { "id":33, "name": "suitcase" }, + { "id":34, "name": "frisbee" }, + { "id":35, "name": "skis" }, + { "id":36, "name": "snowboard" }, + { "id":37, "name": "sports_ball" }, + { "id":38, "name": "kite" }, + { "id":39, "name": "baseball_bat" }, + { "id":40, "name": "baseball_glove" }, + { "id":41, "name": "skateboard" }, + { "id":42, "name": "surfboard" }, + { "id":43, "name": "tennis_racket" }, + { "id":44, "name": "bottle" }, + { "id":46, "name": "wine_glass" }, + { "id":47, "name": "cup" }, + { "id":48, "name": "fork" }, + { "id":49, "name": "knife" }, + { "id":50, "name": "spoon" }, + { "id":51, "name": "bowl" }, + { "id":52, "name": "banana" }, + { "id":53, "name": "apple" }, + { "id":54, "name": "sandwich" }, + { "id":55, "name": "orange" }, + { "id":56, "name": "broccoli" }, + { "id":57, "name": "carrot" }, + { "id":58, "name": "hot_dog" }, + { "id":59, "name": "pizza" }, + { "id":60, "name": "donut" }, + { "id":61, "name": "cake" }, + { "id":62, "name": "chair" }, + { "id":63, "name": "couch" }, + { "id":64, "name": "potted_plant" }, + { "id":65, "name": "bed" }, + { "id":67, "name": "dining_table" }, + { "id":70, "name": "toilet" }, + { "id":72, "name": "tv" }, + { "id":73, "name": "laptop" }, + { "id":74, "name": "mouse" }, + { "id":75, "name": "remote" }, + { "id":76, "name": "keyboard" }, + { "id":77, "name": "cell_phone" }, + { "id":78, "name": "microwave" }, + { "id":79, "name": "oven" }, + { "id":80, "name": "toaster" }, + { "id":81, "name": "sink" }, + { "id":83, "name": "refrigerator" }, + { "id":84, "name": "book" }, + { "id":85, "name": "clock" }, + { "id":86, "name": "vase" }, + { "id":87, "name": "scissors" }, + { "id":88, "name": "teddy_bear" }, + { "id":89, "name": "hair_drier" }, + { "id":90, "name": "toothbrush" } + ] + +spec: + description: Faster RCNN inception v2 COCO via Intel OpenVINO toolkit + runtime: "python:3.6" + handler: main:handler + eventTimeout: 30s + env: + - name: NUCLIO_PYTHON_EXE_PATH + value: /opt/nuclio/common/python3 + + build: + image: cvat/openvino.omz.public.faster_rcnn_inception_v2_coco + baseImage: openvino/ubuntu18_dev:2020.2 + + directives: + preCopy: + - kind: USER + value: root + - kind: WORKDIR + value: /opt/nuclio + - kind: RUN + value: ln -s /usr/bin/pip3 /usr/bin/pip + - kind: RUN + value: /opt/intel/openvino/deployment_tools/open_model_zoo/tools/downloader/downloader.py --name faster_rcnn_inception_v2_coco -o /opt/nuclio/open_model_zoo + - kind: RUN + value: /opt/intel/openvino/deployment_tools/open_model_zoo/tools/downloader/converter.py --name faster_rcnn_inception_v2_coco --precisions FP32 -d /opt/nuclio/open_model_zoo -o /opt/nuclio/open_model_zoo + + postCopy: + - kind: USER + value: openvino + + triggers: + myHttpTrigger: + maxWorkers: 2 + kind: "http" + workerAvailabilityTimeoutMilliseconds: 10000 + attributes: + maxRequestBodySize: 33554432 # 32MB + + platform: + attributes: + restartPolicy: + name: always + maximumRetryCount: 3 diff --git a/serverless/openvino/omz/public/faster_rcnn_inception_v2_coco/nuclio/main.py b/serverless/openvino/omz/public/faster_rcnn_inception_v2_coco/nuclio/main.py new file mode 100644 index 000000000000..6ae5c801c0a1 --- /dev/null +++ b/serverless/openvino/omz/public/faster_rcnn_inception_v2_coco/nuclio/main.py @@ -0,0 +1,32 @@ +import json +import base64 +from PIL import Image +import io +from model_handler import ModelHandler +import yaml + +def init_context(context): + context.logger.info("Init context... 0%") + + # Read labels + functionconfig = yaml.safe_load(open("/opt/nuclio/function.yaml")) + labels_spec = functionconfig['metadata']['annotations']['spec'] + labels = {item['id']: item['name'] for item in json.loads(labels_spec)} + + # Read the DL model + model = ModelHandler(labels) + setattr(context.user_data, 'model', model) + + context.logger.info("Init context...100%") + +def handler(context, event): + context.logger.info("Run faster_rcnn_inception_v2_coco model") + data = event.body + buf = io.BytesIO(base64.b64decode(data["image"].encode('utf-8'))) + threshold = float(data.get("threshold", 0.5)) + image = Image.open(buf) + + results = context.user_data.model.infer(image, threshold) + + return context.Response(body=json.dumps(results), headers={}, + content_type='application/json', status_code=200) diff --git a/serverless/openvino/omz/public/faster_rcnn_inception_v2_coco/nuclio/model_handler.py b/serverless/openvino/omz/public/faster_rcnn_inception_v2_coco/nuclio/model_handler.py new file mode 100644 index 000000000000..608387796e89 --- /dev/null +++ b/serverless/openvino/omz/public/faster_rcnn_inception_v2_coco/nuclio/model_handler.py @@ -0,0 +1,39 @@ +# Copyright (C) 2020 Intel Corporation +# +# SPDX-License-Identifier: MIT + +import os +from model_loader import ModelLoader + +class ModelHandler: + def __init__(self, labels): + base_dir = os.environ.get("MODEL_PATH", + "/opt/nuclio/open_model_zoo/public/faster_rcnn_inception_v2_coco/FP32") + model_xml = os.path.join(base_dir, "faster_rcnn_inception_v2_coco.xml") + model_bin = os.path.join(base_dir, "faster_rcnn_inception_v2_coco.bin") + self.model = ModelLoader(model_xml, model_bin) + self.labels = labels + + def infer(self, image, threshold): + output_layer = self.model.infer(image) + + results = [] + prediction = output_layer[0][0] + for obj in prediction: + obj_class = int(obj[1]) + obj_value = obj[2] + obj_label = self.labels.get(obj_class, "unknown") + if obj_value >= threshold: + xtl = obj[3] * image.width + ytl = obj[4] * image.height + xbr = obj[5] * image.width + ybr = obj[6] * image.height + + results.append({ + "confidence": str(obj_value), + "label": obj_label, + "points": [xtl, ytl, xbr, ybr], + "type": "rectangle", + }) + + return results \ No newline at end of file diff --git a/serverless/openvino/omz/public/mask_rcnn_inception_resnet_v2_atrous_coco/nuclio/function.yaml b/serverless/openvino/omz/public/mask_rcnn_inception_resnet_v2_atrous_coco/nuclio/function.yaml new file mode 100644 index 000000000000..5b94d9545fae --- /dev/null +++ b/serverless/openvino/omz/public/mask_rcnn_inception_resnet_v2_atrous_coco/nuclio/function.yaml @@ -0,0 +1,141 @@ +# To build the function you need to adjust docker settings. Be sure that you +# have enough memory (more than 4GB). Look here how to do that +# https://stackoverflow.com/questions/44417159/docker-process-killed-with-cryptic-killed-message +metadata: + name: openvino.omz.public.mask_rcnn_inception_resnet_v2_atrous_coco + namespace: cvat + annotations: + name: Mask RCNN + type: detector + framework: openvino + spec: | + [ + { "id": 1, "name": "person" }, + { "id": 2, "name": "bicycle" }, + { "id": 3, "name": "car" }, + { "id": 4, "name": "motorcycle" }, + { "id": 5, "name": "airplane" }, + { "id": 6, "name": "bus" }, + { "id": 7, "name": "train" }, + { "id": 8, "name": "truck" }, + { "id": 9, "name": "boat" }, + { "id":10, "name": "traffic_light" }, + { "id":11, "name": "fire_hydrant" }, + { "id":13, "name": "stop_sign" }, + { "id":14, "name": "parking_meter" }, + { "id":15, "name": "bench" }, + { "id":16, "name": "bird" }, + { "id":17, "name": "cat" }, + { "id":18, "name": "dog" }, + { "id":19, "name": "horse" }, + { "id":20, "name": "sheep" }, + { "id":21, "name": "cow" }, + { "id":22, "name": "elephant" }, + { "id":23, "name": "bear" }, + { "id":24, "name": "zebra" }, + { "id":25, "name": "giraffe" }, + { "id":27, "name": "backpack" }, + { "id":28, "name": "umbrella" }, + { "id":31, "name": "handbag" }, + { "id":32, "name": "tie" }, + { "id":33, "name": "suitcase" }, + { "id":34, "name": "frisbee" }, + { "id":35, "name": "skis" }, + { "id":36, "name": "snowboard" }, + { "id":37, "name": "sports_ball" }, + { "id":38, "name": "kite" }, + { "id":39, "name": "baseball_bat" }, + { "id":40, "name": "baseball_glove" }, + { "id":41, "name": "skateboard" }, + { "id":42, "name": "surfboard" }, + { "id":43, "name": "tennis_racket" }, + { "id":44, "name": "bottle" }, + { "id":46, "name": "wine_glass" }, + { "id":47, "name": "cup" }, + { "id":48, "name": "fork" }, + { "id":49, "name": "knife" }, + { "id":50, "name": "spoon" }, + { "id":51, "name": "bowl" }, + { "id":52, "name": "banana" }, + { "id":53, "name": "apple" }, + { "id":54, "name": "sandwich" }, + { "id":55, "name": "orange" }, + { "id":56, "name": "broccoli" }, + { "id":57, "name": "carrot" }, + { "id":58, "name": "hot_dog" }, + { "id":59, "name": "pizza" }, + { "id":60, "name": "donut" }, + { "id":61, "name": "cake" }, + { "id":62, "name": "chair" }, + { "id":63, "name": "couch" }, + { "id":64, "name": "potted_plant" }, + { "id":65, "name": "bed" }, + { "id":67, "name": "dining_table" }, + { "id":70, "name": "toilet" }, + { "id":72, "name": "tv" }, + { "id":73, "name": "laptop" }, + { "id":74, "name": "mouse" }, + { "id":75, "name": "remote" }, + { "id":76, "name": "keyboard" }, + { "id":77, "name": "cell_phone" }, + { "id":78, "name": "microwave" }, + { "id":79, "name": "oven" }, + { "id":80, "name": "toaster" }, + { "id":81, "name": "sink" }, + { "id":83, "name": "refrigerator" }, + { "id":84, "name": "book" }, + { "id":85, "name": "clock" }, + { "id":86, "name": "vase" }, + { "id":87, "name": "scissors" }, + { "id":88, "name": "teddy_bear" }, + { "id":89, "name": "hair_drier" }, + { "id":90, "name": "toothbrush" } + ] + +spec: + description: Mask RCNN inception resnet v2 COCO via Intel OpenVINO + runtime: "python:3.6" + handler: main:handler + eventTimeout: 60s + env: + - name: NUCLIO_PYTHON_EXE_PATH + value: /opt/nuclio/common/python3 + + build: + image: cvat/openvino.omz.public.mask_rcnn_inception_resnet_v2_atrous_coco + baseImage: openvino/ubuntu18_dev:2020.2 + + directives: + preCopy: + - kind: USER + value: root + - kind: WORKDIR + value: /opt/nuclio + - kind: RUN + value: ln -s /usr/bin/pip3 /usr/bin/pip + - kind: RUN + value: /opt/intel/openvino/deployment_tools/open_model_zoo/tools/downloader/downloader.py --name mask_rcnn_inception_resnet_v2_atrous_coco -o /opt/nuclio/open_model_zoo + - kind: RUN + value: /opt/intel/openvino/deployment_tools/open_model_zoo/tools/downloader/converter.py --name mask_rcnn_inception_resnet_v2_atrous_coco --precisions FP32 -d /opt/nuclio/open_model_zoo -o /opt/nuclio/open_model_zoo + + postCopy: + - kind: RUN + value: apt update && DEBIAN_FRONTEND=noninteractive apt install --no-install-recommends -y python3-skimage + - kind: RUN + value: pip3 install "numpy<1.16.0" # workaround for skimage + - kind: USER + value: openvino + + triggers: + myHttpTrigger: + maxWorkers: 2 + kind: "http" + workerAvailabilityTimeoutMilliseconds: 10000 + attributes: + maxRequestBodySize: 33554432 # 32MB + + platform: + attributes: + restartPolicy: + name: always + maximumRetryCount: 3 diff --git a/serverless/openvino/omz/public/mask_rcnn_inception_resnet_v2_atrous_coco/nuclio/main.py b/serverless/openvino/omz/public/mask_rcnn_inception_resnet_v2_atrous_coco/nuclio/main.py new file mode 100644 index 000000000000..8fc7d285d2ff --- /dev/null +++ b/serverless/openvino/omz/public/mask_rcnn_inception_resnet_v2_atrous_coco/nuclio/main.py @@ -0,0 +1,32 @@ +import json +import base64 +from PIL import Image +import io +from model_handler import ModelHandler +import yaml + +def init_context(context): + context.logger.info("Init context... 0%") + + # Read labels + functionconfig = yaml.safe_load(open("/opt/nuclio/function.yaml")) + labels_spec = functionconfig['metadata']['annotations']['spec'] + labels = {item['id']: item['name'] for item in json.loads(labels_spec)} + + # Read the DL model + model = ModelHandler(labels) + setattr(context.user_data, 'model', model) + + context.logger.info("Init context...100%") + +def handler(context, event): + context.logger.info("Run mask_rcnn_inception_resnet_v2_atrous_coco model") + data = event.body + buf = io.BytesIO(base64.b64decode(data["image"].encode('utf-8'))) + threshold = float(data.get("threshold", 0.2)) + image = Image.open(buf) + + results = context.user_data.model.infer(image, threshold) + + return context.Response(body=json.dumps(results), headers={}, + content_type='application/json', status_code=200) diff --git a/serverless/openvino/omz/public/mask_rcnn_inception_resnet_v2_atrous_coco/nuclio/model_handler.py b/serverless/openvino/omz/public/mask_rcnn_inception_resnet_v2_atrous_coco/nuclio/model_handler.py new file mode 100644 index 000000000000..a508aaa4747d --- /dev/null +++ b/serverless/openvino/omz/public/mask_rcnn_inception_resnet_v2_atrous_coco/nuclio/model_handler.py @@ -0,0 +1,77 @@ +# Copyright (C) 2020 Intel Corporation +# +# SPDX-License-Identifier: MIT + +import os +import cv2 +import numpy as np +from model_loader import ModelLoader +from skimage.measure import approximate_polygon, find_contours + + +MASK_THRESHOLD = 0.5 + +# Ref: https://software.intel.com/en-us/forums/computer-vision/topic/804895 +def segm_postprocess(box: list, raw_cls_mask, im_h, im_w): + ymin, xmin, ymax, xmax = box + + width = int(abs(xmax - xmin)) + height = int(abs(ymax - ymin)) + + result = np.zeros((im_h, im_w), dtype=np.uint8) + resized_mask = cv2.resize(raw_cls_mask, dsize=(height, width), interpolation=cv2.INTER_CUBIC) + + # extract the ROI of the image + ymin = int(round(ymin)) + xmin = int(round(xmin)) + ymax = ymin + height + xmax = xmin + width + result[xmin:xmax, ymin:ymax] = (resized_mask>MASK_THRESHOLD).astype(np.uint8) * 255 + + return result + +class ModelHandler: + def __init__(self, labels): + base_dir = os.environ.get("MODEL_PATH", + "/opt/nuclio/open_model_zoo/public/mask_rcnn_inception_resnet_v2_atrous_coco/FP32") + model_xml = os.path.join(base_dir, "mask_rcnn_inception_resnet_v2_atrous_coco.xml") + model_bin = os.path.join(base_dir, "mask_rcnn_inception_resnet_v2_atrous_coco.bin") + self.model = ModelLoader(model_xml, model_bin) + self.labels = labels + + def infer(self, image, threshold): + output_layer = self.model.infer(image) + + results = [] + masks = output_layer['masks'] + boxes = output_layer['reshape_do_2d'] + + for index, box in enumerate(boxes): + obj_class = int(box[1]) + obj_value = box[2] + obj_label = self.labels.get(obj_class, "unknown") + if obj_value >= threshold: + xtl = box[3] * image.width + ytl = box[4] * image.height + xbr = box[5] * image.width + ybr = box[6] * image.height + mask = masks[index][obj_class - 1] + + mask = segm_postprocess((xtl, ytl, xbr, ybr), + mask, image.height, image.width) + + contours = find_contours(mask, MASK_THRESHOLD) + contour = contours[0] + contour = np.flip(contour, axis=1) + contour = approximate_polygon(contour, tolerance=2.5) + if len(contour) < 3: + continue + + results.append({ + "confidence": str(obj_value), + "label": obj_label, + "points": contour.ravel().tolist(), + "type": "polygon", + }) + + return results \ No newline at end of file diff --git a/serverless/openvino/omz/public/yolo-v3-tf/nuclio/function.yaml b/serverless/openvino/omz/public/yolo-v3-tf/nuclio/function.yaml new file mode 100644 index 000000000000..3ec733b7588d --- /dev/null +++ b/serverless/openvino/omz/public/yolo-v3-tf/nuclio/function.yaml @@ -0,0 +1,134 @@ +metadata: + name: openvino.omz.public.yolo-v3-tf + namespace: cvat + annotations: + name: YOLO v3 + type: detector + framework: openvino + spec: | + [ + { "id": 0, "name": "person" }, + { "id": 1, "name": "bicycle" }, + { "id": 2, "name": "car" }, + { "id": 3, "name": "motorbike" }, + { "id": 4, "name": "aeroplane" }, + { "id": 5, "name": "bus" }, + { "id": 6, "name": "train" }, + { "id": 7, "name": "truck" }, + { "id": 8, "name": "boat" }, + { "id": 9, "name": "traffic light" }, + { "id": 10, "name": "fire hydrant" }, + { "id": 11, "name": "stop sign" }, + { "id": 12, "name": "parking meter" }, + { "id": 13, "name": "bench" }, + { "id": 14, "name": "bird" }, + { "id": 15, "name": "cat" }, + { "id": 16, "name": "dog" }, + { "id": 17, "name": "horse" }, + { "id": 18, "name": "sheep" }, + { "id": 19, "name": "cow" }, + { "id": 20, "name": "elephant" }, + { "id": 21, "name": "bear" }, + { "id": 22, "name": "zebra" }, + { "id": 23, "name": "giraffe" }, + { "id": 24, "name": "backpack" }, + { "id": 25, "name": "umbrella" }, + { "id": 26, "name": "handbag" }, + { "id": 27, "name": "tie" }, + { "id": 28, "name": "suitcase" }, + { "id": 29, "name": "frisbee" }, + { "id": 30, "name": "skis" }, + { "id": 31, "name": "snowboard" }, + { "id": 32, "name": "sports ball" }, + { "id": 33, "name": "kite" }, + { "id": 34, "name": "baseball bat" }, + { "id": 35, "name": "baseball glove" }, + { "id": 36, "name": "skateboard" }, + { "id": 37, "name": "surfboard" }, + { "id": 38, "name": "tennis racket" }, + { "id": 39, "name": "bottle" }, + { "id": 40, "name": "wine glass" }, + { "id": 41, "name": "cup" }, + { "id": 42, "name": "fork" }, + { "id": 43, "name": "knife" }, + { "id": 44, "name": "spoon" }, + { "id": 45, "name": "bowl" }, + { "id": 46, "name": "banana" }, + { "id": 47, "name": "apple" }, + { "id": 48, "name": "sandwich" }, + { "id": 49, "name": "orange" }, + { "id": 50, "name": "broccoli" }, + { "id": 51, "name": "carrot" }, + { "id": 52, "name": "hot dog" }, + { "id": 53, "name": "pizza" }, + { "id": 54, "name": "donut" }, + { "id": 55, "name": "cake" }, + { "id": 56, "name": "chair" }, + { "id": 57, "name": "sofa" }, + { "id": 58, "name": "pottedplant" }, + { "id": 59, "name": "bed" }, + { "id": 60, "name": "diningtable" }, + { "id": 61, "name": "toilet" }, + { "id": 62, "name": "tvmonitor" }, + { "id": 63, "name": "laptop" }, + { "id": 64, "name": "mouse" }, + { "id": 65, "name": "remote" }, + { "id": 66, "name": "keyboard" }, + { "id": 67, "name": "cell phone" }, + { "id": 68, "name": "microwave" }, + { "id": 69, "name": "oven" }, + { "id": 70, "name": "toaster" }, + { "id": 71, "name": "sink" }, + { "id": 72, "name": "refrigerator" }, + { "id": 73, "name": "book" }, + { "id": 74, "name": "clock" }, + { "id": 75, "name": "vase" }, + { "id": 76, "name": "scissors" }, + { "id": 77, "name": "teddy bear" }, + { "id": 78, "name": "hair drier" }, + { "id": 79, "name": "toothbrush" } + ] + +spec: + description: YOLO v3 via Intel OpenVINO + runtime: "python:3.6" + handler: main:handler + eventTimeout: 30s + env: + - name: NUCLIO_PYTHON_EXE_PATH + value: /opt/nuclio/common/python3 + + build: + image: cvat/openvino.omz.public.yolo-v3-tf + baseImage: openvino/ubuntu18_dev:2020.2 + + directives: + preCopy: + - kind: USER + value: root + - kind: WORKDIR + value: /opt/nuclio + - kind: RUN + value: ln -s /usr/bin/pip3 /usr/bin/pip + - kind: RUN + value: /opt/intel/openvino/deployment_tools/open_model_zoo/tools/downloader/downloader.py --name yolo-v3-tf -o /opt/nuclio/open_model_zoo + - kind: RUN + value: /opt/intel/openvino/deployment_tools/open_model_zoo/tools/downloader/converter.py --name yolo-v3-tf --precisions FP32 -d /opt/nuclio/open_model_zoo -o /opt/nuclio/open_model_zoo + + postCopy: + - kind: USER + value: openvino + + triggers: + myHttpTrigger: + maxWorkers: 2 + kind: "http" + workerAvailabilityTimeoutMilliseconds: 10000 + attributes: + maxRequestBodySize: 33554432 # 32MB + + platform: + attributes: + restartPolicy: + name: always + maximumRetryCount: 3 diff --git a/serverless/openvino/omz/public/yolo-v3-tf/nuclio/main.py b/serverless/openvino/omz/public/yolo-v3-tf/nuclio/main.py new file mode 100644 index 000000000000..806ab654200d --- /dev/null +++ b/serverless/openvino/omz/public/yolo-v3-tf/nuclio/main.py @@ -0,0 +1,32 @@ +import json +import base64 +from PIL import Image +import io +from model_handler import ModelHandler +import yaml + +def init_context(context): + context.logger.info("Init context... 0%") + + # Read labels + functionconfig = yaml.safe_load(open("/opt/nuclio/function.yaml")) + labels_spec = functionconfig['metadata']['annotations']['spec'] + labels = {item['id']: item['name'] for item in json.loads(labels_spec)} + + # Read the DL model + model = ModelHandler(labels) + setattr(context.user_data, 'model', model) + + context.logger.info("Init context...100%") + +def handler(context, event): + context.logger.info("Run yolo-v3-tf model") + data = event.body + buf = io.BytesIO(base64.b64decode(data["image"].encode('utf-8'))) + threshold = float(data.get("threshold", 0.5)) + image = Image.open(buf) + + results = context.user_data.model.infer(image, threshold) + + return context.Response(body=json.dumps(results), headers={}, + content_type='application/json', status_code=200) diff --git a/serverless/openvino/omz/public/yolo-v3-tf/nuclio/model_handler.py b/serverless/openvino/omz/public/yolo-v3-tf/nuclio/model_handler.py new file mode 100644 index 000000000000..47fa58426be8 --- /dev/null +++ b/serverless/openvino/omz/public/yolo-v3-tf/nuclio/model_handler.py @@ -0,0 +1,160 @@ +# Copyright (C) 2020 Intel Corporation +# +# SPDX-License-Identifier: MIT + +import os +from math import exp +from model_loader import ModelLoader + +class YoloParams: + # ------------------------------------------- Extracting layer parameters ------------------------------------------ + # Magic numbers are copied from yolo samples + def __init__(self, param, side): + self.num = 3 if 'num' not in param else int(param['num']) + self.coords = 4 if 'coords' not in param else int(param['coords']) + self.classes = 80 if 'classes' not in param else int(param['classes']) + self.side = side + self.anchors = [10.0, 13.0, 16.0, 30.0, 33.0, 23.0, 30.0, 61.0, 62.0, 45.0, 59.0, 119.0, 116.0, 90.0, 156.0, + 198.0, + 373.0, 326.0] if 'anchors' not in param else [float(a) for a in param['anchors'].split(',')] + + self.isYoloV3 = False + + if param.get('mask'): + mask = [int(idx) for idx in param['mask'].split(',')] + self.num = len(mask) + + maskedAnchors = [] + for idx in mask: + maskedAnchors += [self.anchors[idx * 2], self.anchors[idx * 2 + 1]] + self.anchors = maskedAnchors + + self.isYoloV3 = True # Weak way to determine but the only one. + + +def entry_index(side, coord, classes, location, entry): + side_power_2 = side ** 2 + n = location // side_power_2 + loc = location % side_power_2 + return int(side_power_2 * (n * (coord + classes + 1) + entry) + loc) + + +def scale_bbox(x, y, h, w, class_id, confidence, h_scale, w_scale): + xmin = int((x - w / 2) * w_scale) + ymin = int((y - h / 2) * h_scale) + xmax = int(xmin + w * w_scale) + ymax = int(ymin + h * h_scale) + return dict(xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax, class_id=class_id, confidence=confidence) + + +def parse_yolo_region(blob, resized_image_shape, original_im_shape, params, threshold): + # ------------------------------------------ Validating output parameters ------------------------------------------ + _, _, out_blob_h, out_blob_w = blob.shape + assert out_blob_w == out_blob_h, "Invalid size of output blob. It sould be in NCHW layout and height should " \ + "be equal to width. Current height = {}, current width = {}" \ + "".format(out_blob_h, out_blob_w) + + # ------------------------------------------ Extracting layer parameters ------------------------------------------- + orig_im_h, orig_im_w = original_im_shape + resized_image_h, resized_image_w = resized_image_shape + objects = list() + predictions = blob.flatten() + side_square = params.side * params.side + + # ------------------------------------------- Parsing YOLO Region output ------------------------------------------- + for i in range(side_square): + row = i // params.side + col = i % params.side + for n in range(params.num): + obj_index = entry_index(params.side, params.coords, params.classes, n * side_square + i, params.coords) + scale = predictions[obj_index] + if scale < threshold: + continue + box_index = entry_index(params.side, params.coords, params.classes, n * side_square + i, 0) + # Network produces location predictions in absolute coordinates of feature maps. + # Scale it to relative coordinates. + x = (col + predictions[box_index + 0 * side_square]) / params.side + y = (row + predictions[box_index + 1 * side_square]) / params.side + # Value for exp is very big number in some cases so following construction is using here + try: + w_exp = exp(predictions[box_index + 2 * side_square]) + h_exp = exp(predictions[box_index + 3 * side_square]) + except OverflowError: + continue + # Depends on topology we need to normalize sizes by feature maps (up to YOLOv3) or by input shape (YOLOv3) + w = w_exp * params.anchors[2 * n] / (resized_image_w if params.isYoloV3 else params.side) + h = h_exp * params.anchors[2 * n + 1] / (resized_image_h if params.isYoloV3 else params.side) + for j in range(params.classes): + class_index = entry_index(params.side, params.coords, params.classes, n * side_square + i, + params.coords + 1 + j) + confidence = scale * predictions[class_index] + if confidence < threshold: + continue + objects.append(scale_bbox(x=x, y=y, h=h, w=w, class_id=j, confidence=confidence, + h_scale=orig_im_h, w_scale=orig_im_w)) + return objects + + +def intersection_over_union(box_1, box_2): + width_of_overlap_area = min(box_1['xmax'], box_2['xmax']) - max(box_1['xmin'], box_2['xmin']) + height_of_overlap_area = min(box_1['ymax'], box_2['ymax']) - max(box_1['ymin'], box_2['ymin']) + if width_of_overlap_area < 0 or height_of_overlap_area < 0: + area_of_overlap = 0 + else: + area_of_overlap = width_of_overlap_area * height_of_overlap_area + box_1_area = (box_1['ymax'] - box_1['ymin']) * (box_1['xmax'] - box_1['xmin']) + box_2_area = (box_2['ymax'] - box_2['ymin']) * (box_2['xmax'] - box_2['xmin']) + area_of_union = box_1_area + box_2_area - area_of_overlap + if area_of_union == 0: + return 0 + return area_of_overlap / area_of_union + + +class ModelHandler: + def __init__(self, labels): + base_dir = os.environ.get("MODEL_PATH", + "/opt/nuclio/open_model_zoo/public/yolo-v3-tf/FP32") + model_xml = os.path.join(base_dir, "yolo-v3-tf.xml") + model_bin = os.path.join(base_dir, "yolo-v3-tf.bin") + self.model = ModelLoader(model_xml, model_bin) + self.labels = labels + + def infer(self, image, threshold): + output_layer = self.model.infer(image) + + # Collecting object detection results + objects = [] + origin_im_size = (image.height, image.width) + for layer_name, out_blob in output_layer.items(): + out_blob = out_blob.reshape(self.model.layers[self.model.layers[layer_name].parents[0]].shape) + layer_params = YoloParams(self.model.layers[layer_name].params, out_blob.shape[2]) + objects += parse_yolo_region(out_blob, self.model.input_size(), + origin_im_size, layer_params, threshold) + + # Filtering overlapping boxes (non-maximum supression) + IOU_THRESHOLD = 0.4 + objects = sorted(objects, key=lambda obj : obj['confidence'], reverse=True) + for i, obj in enumerate(objects): + if obj['confidence'] == 0: + continue + for j in range(i + 1, len(objects)): + if intersection_over_union(obj, objects[j]) > IOU_THRESHOLD: + objects[j]['confidence'] = 0 + + results = [] + for obj in objects: + if obj['confidence'] >= threshold: + xtl = max(obj['xmin'], 0) + ytl = max(obj['ymin'], 0) + xbr = min(obj['xmax'], image.width) + ybr = min(obj['ymax'], image.height) + obj_class = int(obj['class_id']) + + results.append({ + "confidence": str(obj['confidence']), + "label": self.labels.get(obj_class, "unknown"), + "points": [xtl, ytl, xbr, ybr], + "type": "rectangle", + }) + + return results \ No newline at end of file diff --git a/serverless/pytorch/foolwood/siammask/nuclio/function.yaml b/serverless/pytorch/foolwood/siammask/nuclio/function.yaml new file mode 100644 index 000000000000..efdb0ca818a5 --- /dev/null +++ b/serverless/pytorch/foolwood/siammask/nuclio/function.yaml @@ -0,0 +1,57 @@ +metadata: + name: pth.foolwood.siammask + namespace: cvat + annotations: + name: SiamMask + type: tracker + spec: + framework: pytorch + +spec: + description: Fast Online Object Tracking and Segmentation + runtime: "python:3.6" + handler: main:handler + eventTimeout: 30s + env: + - name: PYTHONPATH + value: /opt/nuclio/SiamMask:/opt/nuclio/SiamMask/experiments/siammask_sharp + + build: + image: cvat/pth.foolwood.siammask + baseImage: continuumio/miniconda3 + + directives: + preCopy: + - kind: WORKDIR + value: /opt/nuclio + - kind: RUN + value: conda create -y -n siammask python=3.6 + - kind: RUN + value: source activate siammask + - kind: RUN + value: git clone https://github.com/foolwood/SiamMask.git + - kind: RUN + value: pip install -r SiamMask/requirements.txt + - kind: RUN + value: conda install -y gcc_linux-64 + - kind: RUN + value: cd SiamMask && bash make.sh && cd - + - kind: RUN + value: wget -P SiamMask/experiments/siammask_sharp http://www.robots.ox.ac.uk/~qwang/SiamMask_DAVIS.pth + + - kind: WORKDIR + value: /opt/nuclio/pysot + + triggers: + myHttpTrigger: + maxWorkers: 2 + kind: "http" + workerAvailabilityTimeoutMilliseconds: 10000 + attributes: + maxRequestBodySize: 33554432 # 32MB + + platform: + attributes: + restartPolicy: + name: always + maximumRetryCount: 3 diff --git a/serverless/pytorch/foolwood/siammask/nuclio/main.py b/serverless/pytorch/foolwood/siammask/nuclio/main.py new file mode 100644 index 000000000000..51c3669ac991 --- /dev/null +++ b/serverless/pytorch/foolwood/siammask/nuclio/main.py @@ -0,0 +1,27 @@ +import json +import base64 +from PIL import Image +import io +from model_handler import ModelHandler + +def init_context(context): + context.logger.info("Init context... 0%") + + # Read the DL model + model = ModelHandler() + setattr(context.user_data, 'model', model) + + context.logger.info("Init context...100%") + +def handler(context, event): + context.logger.info("Run SiamMask model") + data = event.body + buf = io.BytesIO(base64.b64decode(data["image"].encode('utf-8'))) + shape = data.get("shape") + state = data.get("state") + image = Image.open(buf) + + results = context.user_data.model.infer(image, shape, state) + + return context.Response(body=json.dumps(results), headers={}, + content_type='application/json', status_code=200) diff --git a/serverless/pytorch/foolwood/siammask/nuclio/model_handler.py b/serverless/pytorch/foolwood/siammask/nuclio/model_handler.py new file mode 100644 index 000000000000..de79c16d6df4 --- /dev/null +++ b/serverless/pytorch/foolwood/siammask/nuclio/model_handler.py @@ -0,0 +1,38 @@ +# Copyright (C) 2020 Intel Corporation +# +# SPDX-License-Identifier: MIT + +from tools.test import * +import os + +class ModelHandler: + def __init__(self): + # Setup device + self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') + torch.backends.cudnn.benchmark = True + + base_dir = "/opt/nuclio/SiamMask/experiments/siammask_sharp" + class configPath: + config = os.path.join(base_dir, "config_davis.json") + + self.config = load_config(configPath) + from custom import Custom + siammask = Custom(anchors=self.config['anchors']) + self.siammask = load_pretrain(siammask, os.path.join(base_dir, "SiamMask_DAVIS.pth")) + self.siammask.eval().to(self.device) + + + def infer(self, image, shape, state): + if state is None: # init tracking + x, y, w, h = shape + target_pos = np.array([x + w / 2, y + h / 2]) + target_sz = np.array([w, h]) + state = siamese_init(image, target_pos, target_sz, self.siammask, + self.config['hp'], device=self.device) + else: # track + state = siamese_track(state, image, mask_enable=True, refine_enable=True, + device=self.device) + shape = state['ploygon'].flatten() + + return {"shape": shape, "state": state} + diff --git a/serverless/tensorflow/faster_rcnn_inception_v2_coco/nuclio/function.yaml b/serverless/tensorflow/faster_rcnn_inception_v2_coco/nuclio/function.yaml new file mode 100644 index 000000000000..c58617cea040 --- /dev/null +++ b/serverless/tensorflow/faster_rcnn_inception_v2_coco/nuclio/function.yaml @@ -0,0 +1,132 @@ +metadata: + name: tf.faster_rcnn_inception_v2_coco + namespace: cvat + annotations: + name: Faster RCNN via Tensorflow + type: detector + framework: tensorflow + spec: | + [ + { "id": 1, "name": "person" }, + { "id": 2, "name": "bicycle" }, + { "id": 3, "name": "car" }, + { "id": 4, "name": "motorcycle" }, + { "id": 5, "name": "airplane" }, + { "id": 6, "name": "bus" }, + { "id": 7, "name": "train" }, + { "id": 8, "name": "truck" }, + { "id": 9, "name": "boat" }, + { "id":10, "name": "traffic_light" }, + { "id":11, "name": "fire_hydrant" }, + { "id":13, "name": "stop_sign" }, + { "id":14, "name": "parking_meter" }, + { "id":15, "name": "bench" }, + { "id":16, "name": "bird" }, + { "id":17, "name": "cat" }, + { "id":18, "name": "dog" }, + { "id":19, "name": "horse" }, + { "id":20, "name": "sheep" }, + { "id":21, "name": "cow" }, + { "id":22, "name": "elephant" }, + { "id":23, "name": "bear" }, + { "id":24, "name": "zebra" }, + { "id":25, "name": "giraffe" }, + { "id":27, "name": "backpack" }, + { "id":28, "name": "umbrella" }, + { "id":31, "name": "handbag" }, + { "id":32, "name": "tie" }, + { "id":33, "name": "suitcase" }, + { "id":34, "name": "frisbee" }, + { "id":35, "name": "skis" }, + { "id":36, "name": "snowboard" }, + { "id":37, "name": "sports_ball" }, + { "id":38, "name": "kite" }, + { "id":39, "name": "baseball_bat" }, + { "id":40, "name": "baseball_glove" }, + { "id":41, "name": "skateboard" }, + { "id":42, "name": "surfboard" }, + { "id":43, "name": "tennis_racket" }, + { "id":44, "name": "bottle" }, + { "id":46, "name": "wine_glass" }, + { "id":47, "name": "cup" }, + { "id":48, "name": "fork" }, + { "id":49, "name": "knife" }, + { "id":50, "name": "spoon" }, + { "id":51, "name": "bowl" }, + { "id":52, "name": "banana" }, + { "id":53, "name": "apple" }, + { "id":54, "name": "sandwich" }, + { "id":55, "name": "orange" }, + { "id":56, "name": "broccoli" }, + { "id":57, "name": "carrot" }, + { "id":58, "name": "hot_dog" }, + { "id":59, "name": "pizza" }, + { "id":60, "name": "donut" }, + { "id":61, "name": "cake" }, + { "id":62, "name": "chair" }, + { "id":63, "name": "couch" }, + { "id":64, "name": "potted_plant" }, + { "id":65, "name": "bed" }, + { "id":67, "name": "dining_table" }, + { "id":70, "name": "toilet" }, + { "id":72, "name": "tv" }, + { "id":73, "name": "laptop" }, + { "id":74, "name": "mouse" }, + { "id":75, "name": "remote" }, + { "id":76, "name": "keyboard" }, + { "id":77, "name": "cell_phone" }, + { "id":78, "name": "microwave" }, + { "id":79, "name": "oven" }, + { "id":80, "name": "toaster" }, + { "id":81, "name": "sink" }, + { "id":83, "name": "refrigerator" }, + { "id":84, "name": "book" }, + { "id":85, "name": "clock" }, + { "id":86, "name": "vase" }, + { "id":87, "name": "scissors" }, + { "id":88, "name": "teddy_bear" }, + { "id":89, "name": "hair_drier" }, + { "id":90, "name": "toothbrush" } + ] + +spec: + description: Faster RCNN from Tensorflow Object Detection API + runtime: "python:3.6" + handler: main:handler + eventTimeout: 30s + + build: + image: cvat/tf.faster_rcnn_inception_v2_coco + baseImage: tensorflow/tensorflow:2.1.1 + + directives: + preCopy: + - kind: RUN + value: apt install curl + - kind: WORKDIR + value: /opt/nuclio + + postCopy: + - kind: RUN + value: curl -O http://download.tensorflow.org/models/object_detection/faster_rcnn_inception_v2_coco_2018_01_28.tar.gz + - kind: RUN + value: tar -xzf faster_rcnn_inception_v2_coco_2018_01_28.tar.gz && rm faster_rcnn_inception_v2_coco_2018_01_28.tar.gz + - kind: RUN + value: ln -s faster_rcnn_inception_v2_coco_2018_01_28 faster_rcnn + - kind: RUN + value: pip install pillow pyyaml + + + triggers: + myHttpTrigger: + maxWorkers: 2 + kind: "http" + workerAvailabilityTimeoutMilliseconds: 10000 + attributes: + maxRequestBodySize: 33554432 # 32MB + + platform: + attributes: + restartPolicy: + name: always + maximumRetryCount: 3 diff --git a/serverless/tensorflow/faster_rcnn_inception_v2_coco/nuclio/main.py b/serverless/tensorflow/faster_rcnn_inception_v2_coco/nuclio/main.py new file mode 100644 index 000000000000..8bcad27cf1f0 --- /dev/null +++ b/serverless/tensorflow/faster_rcnn_inception_v2_coco/nuclio/main.py @@ -0,0 +1,48 @@ +import json +import base64 +import io +from PIL import Image +import yaml +from model_loader import ModelLoader + + +def init_context(context): + context.logger.info("Init context... 0%") + model_path = "/opt/nuclio/faster_rcnn/frozen_inference_graph.pb" + model_handler = ModelLoader(model_path) + setattr(context.user_data, 'model_handler', model_handler) + functionconfig = yaml.safe_load(open("/opt/nuclio/function.yaml")) + labels_spec = functionconfig['metadata']['annotations']['spec'] + labels = {item['id']: item['name'] for item in json.loads(labels_spec)} + setattr(context.user_data, "labels", labels) + context.logger.info("Init context...100%") + +def handler(context, event): + context.logger.info("Run faster_rcnn_inception_v2_coco model") + data = event.body + buf = io.BytesIO(base64.b64decode(data["image"].encode('utf-8'))) + threshold = float(data.get("threshold", 0.5)) + image = Image.open(buf) + + (boxes, scores, classes, num_detections) = context.user_data.model_handler.infer(image) + + results = [] + for i in range(int(num_detections[0])): + obj_class = int(classes[0][i]) + obj_score = scores[0][i] + obj_label = context.user_data.labels.get(obj_class, "unknown") + if obj_score >= threshold: + xtl = boxes[0][i][1] * image.width + ytl = boxes[0][i][0] * image.height + xbr = boxes[0][i][3] * image.width + ybr = boxes[0][i][2] * image.height + + results.append({ + "confidence": str(obj_score), + "label": obj_label, + "points": [xtl, ytl, xbr, ybr], + "type": "rectangle", + }) + + return context.Response(body=json.dumps(results), headers={}, + content_type='application/json', status_code=200) diff --git a/serverless/tensorflow/faster_rcnn_inception_v2_coco/nuclio/model_loader.py b/serverless/tensorflow/faster_rcnn_inception_v2_coco/nuclio/model_loader.py new file mode 100644 index 000000000000..8158eee31288 --- /dev/null +++ b/serverless/tensorflow/faster_rcnn_inception_v2_coco/nuclio/model_loader.py @@ -0,0 +1,43 @@ + +import numpy as np +from PIL import Image +import tensorflow.compat.v1 as tf +tf.disable_v2_behavior() + +class ModelLoader: + def __init__(self, model_path): + self.session = None + + detection_graph = tf.Graph() + with detection_graph.as_default(): + od_graph_def = tf.GraphDef() + with tf.gfile.GFile(model_path, 'rb') as fid: + serialized_graph = fid.read() + od_graph_def.ParseFromString(serialized_graph) + tf.import_graph_def(od_graph_def, name='') + + config = tf.ConfigProto() + config.gpu_options.allow_growth = True + self.session = tf.Session(graph=detection_graph, config=config) + + self.image_tensor = detection_graph.get_tensor_by_name('image_tensor:0') + self.boxes = detection_graph.get_tensor_by_name('detection_boxes:0') + self.scores = detection_graph.get_tensor_by_name('detection_scores:0') + self.classes = detection_graph.get_tensor_by_name('detection_classes:0') + self.num_detections = detection_graph.get_tensor_by_name('num_detections:0') + + def __del__(self): + if self.session: + self.session.close() + del self.session + + def infer(self, image): + width, height = image.size + if width > 1920 or height > 1080: + image = image.resize((width // 2, height // 2), Image.ANTIALIAS) + image_np = np.array(image.getdata()).reshape((image.height, image.width, 3)).astype(np.uint8) + image_np = np.expand_dims(image_np, axis=0) + + return self.session.run( + [self.boxes, self.scores, self.classes, self.num_detections], + feed_dict={self.image_tensor: image_np}) diff --git a/serverless/tensorflow/matterport/mask_rcnn/nuclio/function.yaml b/serverless/tensorflow/matterport/mask_rcnn/nuclio/function.yaml new file mode 100644 index 000000000000..b50868a6be95 --- /dev/null +++ b/serverless/tensorflow/matterport/mask_rcnn/nuclio/function.yaml @@ -0,0 +1,133 @@ +metadata: + name: tf.matterport.mask_rcnn + namespace: cvat + annotations: + name: Mask RCNN via Tensorflow + type: detector + framework: tensorflow + spec: | + [ + { "id": 0, "name": "BG" }, + { "id": 1, "name": "person" }, + { "id": 2, "name": "bicycle" }, + { "id": 3, "name": "car" }, + { "id": 4, "name": "motorcycle" }, + { "id": 5, "name": "airplane" }, + { "id": 6, "name": "bus" }, + { "id": 7, "name": "train" }, + { "id": 8, "name": "truck" }, + { "id": 9, "name": "boat" }, + { "id": 10, "name": "traffic_light" }, + { "id": 11, "name": "fire_hydrant" }, + { "id": 12, "name": "stop_sign" }, + { "id": 13, "name": "parking_meter" }, + { "id": 14, "name": "bench" }, + { "id": 15, "name": "bird" }, + { "id": 16, "name": "cat" }, + { "id": 17, "name": "dog" }, + { "id": 18, "name": "horse" }, + { "id": 19, "name": "sheep" }, + { "id": 20, "name": "cow" }, + { "id": 21, "name": "elephant" }, + { "id": 22, "name": "bear" }, + { "id": 23, "name": "zebra" }, + { "id": 24, "name": "giraffe" }, + { "id": 25, "name": "backpack" }, + { "id": 26, "name": "umbrella" }, + { "id": 27, "name": "handbag" }, + { "id": 28, "name": "tie" }, + { "id": 29, "name": "suitcase" }, + { "id": 30, "name": "frisbee" }, + { "id": 31, "name": "skis" }, + { "id": 32, "name": "snowboard" }, + { "id": 33, "name": "sports_ball" }, + { "id": 34, "name": "kite" }, + { "id": 35, "name": "baseball_bat" }, + { "id": 36, "name": "baseball_glove" }, + { "id": 37, "name": "skateboard" }, + { "id": 38, "name": "surfboard" }, + { "id": 39, "name": "tennis_racket" }, + { "id": 40, "name": "bottle" }, + { "id": 41, "name": "wine_glass" }, + { "id": 42, "name": "cup" }, + { "id": 43, "name": "fork" }, + { "id": 44, "name": "knife" }, + { "id": 45, "name": "spoon" }, + { "id": 46, "name": "bowl" }, + { "id": 47, "name": "banana" }, + { "id": 48, "name": "apple" }, + { "id": 49, "name": "sandwich" }, + { "id": 50, "name": "orange" }, + { "id": 51, "name": "broccoli" }, + { "id": 52, "name": "carrot" }, + { "id": 53, "name": "hot_dog" }, + { "id": 54, "name": "pizza" }, + { "id": 55, "name": "donut" }, + { "id": 56, "name": "cake" }, + { "id": 57, "name": "chair" }, + { "id": 58, "name": "couch" }, + { "id": 59, "name": "potted_plant" }, + { "id": 60, "name": "bed" }, + { "id": 61, "name": "dining_table" }, + { "id": 62, "name": "toilet" }, + { "id": 63, "name": "tv" }, + { "id": 64, "name": "laptop" }, + { "id": 65, "name": "mouse" }, + { "id": 66, "name": "remote" }, + { "id": 67, "name": "keyboard" }, + { "id": 68, "name": "cell_phone" }, + { "id": 69, "name": "microwave" }, + { "id": 70, "name": "oven" }, + { "id": 71, "name": "toaster" }, + { "id": 72, "name": "sink" }, + { "id": 73, "name": "refrigerator" }, + { "id": 74, "name": "book" }, + { "id": 75, "name": "clock" }, + { "id": 76, "name": "vase" }, + { "id": 77, "name": "scissors" }, + { "id": 78, "name": "teddy_bear" }, + { "id": 79, "name": "hair_drier" }, + { "id": 80, "name": "toothbrush" } + ] + +spec: + description: | + An implementation of Mask RCNN on Python 3, Keras, and TensorFlow. + + runtime: "python:3.6" + handler: main:handler + eventTimeout: 30s + env: + - name: MASK_RCNN_DIR + value: /opt/nuclio/Mask_RCNN + build: + image: cvat/tf.matterport.mask_rcnn + baseImage: tensorflow/tensorflow:2.1.0-py3 + directives: + postCopy: + - kind: WORKDIR + value: /opt/nuclio + - kind: RUN + value: apt update && apt install --no-install-recommends -y git curl libsm6 libxext6 libxrender-dev + - kind: RUN + value: git clone https://github.com/matterport/Mask_RCNN.git + - kind: RUN + value: curl -L https://github.com/matterport/Mask_RCNN/releases/download/v2.0/mask_rcnn_coco.h5 -o Mask_RCNN/mask_rcnn_coco.h5 + - kind: RUN + value: pip3 install -r Mask_RCNN/requirements.txt + - kind: RUN + value: pip3 install pycocotools tensorflow==1.13.1 keras==2.1.0 pillow pyyaml + + triggers: + myHttpTrigger: + maxWorkers: 2 + kind: "http" + workerAvailabilityTimeoutMilliseconds: 10000 + attributes: + maxRequestBodySize: 33554432 # 32MB + + platform: + attributes: + restartPolicy: + name: always + maximumRetryCount: 3 diff --git a/serverless/tensorflow/matterport/mask_rcnn/nuclio/main.py b/serverless/tensorflow/matterport/mask_rcnn/nuclio/main.py new file mode 100644 index 000000000000..95816dd45fee --- /dev/null +++ b/serverless/tensorflow/matterport/mask_rcnn/nuclio/main.py @@ -0,0 +1,32 @@ +import json +import base64 +from PIL import Image +import io +from model_loader import ModelLoader +import numpy as np +import yaml + + +def init_context(context): + context.logger.info("Init context... 0%") + + functionconfig = yaml.safe_load(open("/opt/nuclio/function.yaml")) + labels_spec = functionconfig['metadata']['annotations']['spec'] + labels = {item['id']: item['name'] for item in json.loads(labels_spec)} + + model_handler = ModelLoader(labels) + setattr(context.user_data, 'model_handler', model_handler) + + context.logger.info("Init context...100%") + +def handler(context, event): + context.logger.info("Run tf.matterport.mask_rcnn model") + data = event.body + buf = io.BytesIO(base64.b64decode(data["image"].encode('utf-8'))) + threshold = float(data.get("threshold", 0.2)) + image = Image.open(buf) + + results = context.user_data.model_handler.infer(np.array(image), threshold) + + return context.Response(body=json.dumps(results), headers={}, + content_type='application/json', status_code=200) \ No newline at end of file diff --git a/serverless/tensorflow/matterport/mask_rcnn/nuclio/model_loader.py b/serverless/tensorflow/matterport/mask_rcnn/nuclio/model_loader.py new file mode 100644 index 000000000000..b210338a642e --- /dev/null +++ b/serverless/tensorflow/matterport/mask_rcnn/nuclio/model_loader.py @@ -0,0 +1,79 @@ +# Copyright (C) 2018-2020 Intel Corporation +# +# SPDX-License-Identifier: MIT + +import os +import numpy as np +import sys +from skimage.measure import find_contours, approximate_polygon + +# workaround for tf.placeholder() is not compatible with eager execution +# https://github.com/tensorflow/tensorflow/issues/18165 +import tensorflow as tf +tf.compat.v1.disable_eager_execution() +#import tensorflow.compat.v1 as tf +# tf.disable_v2_behavior() + +# The directory should contain a clone of +# https://github.com/matterport/Mask_RCNN repository and +# downloaded mask_rcnn_coco.h5 model. +MASK_RCNN_DIR = os.environ.get('MASK_RCNN_DIR') +if MASK_RCNN_DIR: + sys.path.append(MASK_RCNN_DIR) # To find local version of the library + sys.path.append(os.path.join(MASK_RCNN_DIR, 'samples/coco')) + +from mrcnn import model as modellib +import coco + +class ModelLoader: + def __init__(self, labels): + COCO_MODEL_PATH = os.path.join(MASK_RCNN_DIR, "mask_rcnn_coco.h5") + if COCO_MODEL_PATH is None: + raise OSError('Model path env not found in the system.') + + class InferenceConfig(coco.CocoConfig): + # Set batch size to 1 since we'll be running inference on + # one image at a time. Batch size = GPU_COUNT * IMAGES_PER_GPU + GPU_COUNT = 1 + IMAGES_PER_GPU = 1 + + # Print config details + self.config = InferenceConfig() + self.config.display() + + self.model = modellib.MaskRCNN(mode="inference", + config=self.config, model_dir=MASK_RCNN_DIR) + self.model.load_weights(COCO_MODEL_PATH, by_name=True) + self.labels = labels + + def infer(self, image, threshold): + output = self.model.detect([image], verbose=1)[0] + + results = [] + MASK_THRESHOLD = 0.5 + for i in range(len(output["rois"])): + score = output["scores"][i] + class_id = output["class_ids"][i] + mask = output["masks"][:,:,i] + if score >= threshold: + mask = mask.astype(np.uint8) + contours = find_contours(mask, MASK_THRESHOLD) + # only one contour exist in our case + contour = contours[0] + contour = np.flip(contour, axis=1) + # Approximate the contour and reduce the number of points + contour = approximate_polygon(contour, tolerance=2.5) + if len(contour) < 3: + continue + label = self.labels[class_id] + + results.append({ + "confidence": str(score), + "label": label, + "points": contour.ravel().tolist(), + "type": "polygon", + }) + + return results + + diff --git a/utils/README.md b/utils/README.md index a768eedf7400..48b87c96e416 100644 --- a/utils/README.md +++ b/utils/README.md @@ -3,6 +3,6 @@ ## Description -This folder contains some useful utilities for Computer Vision Annotation Tool (CVAT). To read about a certain utility please choose a link: - [Auto Annotation Runner](auto_annotation/README.md) +This folder contains some useful utilities for Computer Vision Annotation Tool (CVAT). +To read about a certain utility please choose a link: - [Command line interface for working with CVAT tasks](cli/README.md) diff --git a/utils/auto_annotation/README.md b/utils/auto_annotation/README.md deleted file mode 100644 index 4bc646543053..000000000000 --- a/utils/auto_annotation/README.md +++ /dev/null @@ -1,95 +0,0 @@ -# Auto Annotation Runner - -A small command line program to test and run AutoAnnotation Scripts. - -## Instructions - -There are two modes to run this script in. If you already have a model uploaded into the server, and you're having -issues with running it in production, you can pass in the model name and a task id that you want to test against. - -```shell -# Note that this module can be found in cvat/utils/auto_annotation/run_model.py -$ python /path/to/run_model.py --model-name mymodel --task-id 4 -``` - -If you're running in docker, this can be useful way to debug your model. - -``` shell -$ docker exec -it cvat bash -ic 'python3 ~/cvat/apps/auto_annotation/run_model.py --model-name my-model --task-id 4 -``` - -If you are developing an auto annotation model or you can't get something uploaded into the server, -then you'll need to specify the individual inputs. - -```shell -# Note that this module can be found in cvat/utils/auto_annotation/run_model.py -$ python path/to/run_model.py --py /path/to/python/interp.py \ - --xml /path/to/xml/file.xml \ - --bin /path/to/bin/file.bin \ - --json /path/to/json/mapping/mapping.json -``` - -Some programs need to run unrestricted or as an administer. Use the `--unrestriced` flag to simulate. - -You can pass image files in to fully simulate your findings. Images are passed in as a list - -```shell -$ python /path/to/run_model.py --py /path/to/python/interp.py \ - --xml /path/to/xml/file.xml \ - --bin /path/to/bin/file.bin \ - --json /path/to/json/mapping/mapping.json \ - --image-files /path/to/img.jpg /path2/to/img2.png /path/to/img3.jpg -``` - -Additionally, it's sometimes useful to visualize your images. -Use the `--show-images` flag to have each image with the annotations pop up. - -```shell -$ python /path/to/run_model.py --py /path/to/python/interp.py \ - --xml /path/to/xml/file.xml \ - --bin /path/to/bin/file.bin \ - --json /path/to/json/mapping/mapping.json \ - --image-files /path/to/img.jpg /path2/to/img2.png /path/to/img3.jpg \ - --show-images -``` - -If you'd like to see the labels printed on the image, use the `--show-labels` flag - -```shell -$ python /path/to/run_model.py --py /path/to/python/interp.py \ - --xml /path/to/xml/file.xml \ - --bin /path/to/bin/file.bin \ - --json /path/to/json/mapping/mapping.json \ - --image-files /path/to/img.jpg /path2/to/img2.png /path/to/img3.jpg \ - --show-images \ - --show-labels -``` - -There's a command that let's you scan quickly by setting the length of time (in milliseconds) to display each image. -Use the `--show-image-delay` flag and set the appropriate time. -In this example, 2000 milliseconds is 2 seconds for each image. - -```shell -# Display each image in a window for 2 seconds -$ python /path/to/run_model.py --py /path/to/python/interp.py \ - --xml /path/to/xml/file.xml \ - --bin /path/to/bin/file.bin \ - --json /path/to/json/mapping/mapping.json \ - --image-files /path/to/img.jpg /path2/to/img2.png /path/to/img3.jpg \ - --show-images \ - --show-image-delay 2000 -``` - -Visualization isn't always enough. -The CVAT has a serialization step that can throw errors on model upload even after successful visualization. -You must install the necessary packages installed, but then you can add the `--serialize` command to ensure that your -results will serialize correctly. - -```shell -$ python /path/to/run_model.py --py /path/to/python/interp.py \ - --xml /path/to/xml/file.xml \ - --bin /path/to/bin/file.bin \ - --json /path/to/json/mapping/mapping.json \ - --image-files /path/to/img.jpg /path2/to/img2.png /path/to/img3.jpg \ - --serialize -``` diff --git a/utils/auto_annotation/run_model.py b/utils/auto_annotation/run_model.py deleted file mode 100644 index 9df63528ed4d..000000000000 --- a/utils/auto_annotation/run_model.py +++ /dev/null @@ -1,263 +0,0 @@ -import os -import sys -import json -import argparse -import random -import logging -import fnmatch -from operator import xor - -import numpy as np -import cv2 - -work_dir = os.path.dirname(os.path.abspath(__file__)) -cvat_dir = os.path.join(work_dir, '..', '..') - -sys.path.insert(0, cvat_dir) - -from cvat.apps.auto_annotation.inference import run_inference_engine_annotation - - -def _get_kwargs(): - parser = argparse.ArgumentParser() - parser.add_argument('--py', help='Path to the python interpt file') - parser.add_argument('--xml', help='Path to the xml file') - parser.add_argument('--bin', help='Path to the bin file') - parser.add_argument('--json', help='Path to the JSON mapping file') - - parser.add_argument('--model-name', help='Name of the model in the Model Manager') - parser.add_argument('--task-id', type=int, help='ID task used to test the model') - - parser.add_argument('--restricted', dest='restricted', action='store_true') - parser.add_argument('--unrestricted', dest='restricted', action='store_false') - parser.add_argument('--image-files', nargs='*', help='Paths to image files you want to test') - - parser.add_argument('--show-images', action='store_true', help='Show the results of the annotation in a window') - parser.add_argument('--show-image-delay', default=0, type=int, help='Displays the images for a set duration in milliseconds, default is until a key is pressed') - parser.add_argument('--serialize', default=False, action='store_true', help='Try to serialize the result') - parser.add_argument('--show-labels', action='store_true', help='Show the labels on the window') - - return vars(parser.parse_args()) - -def _init_django(settings): - import django - os.environ['DJANGO_SETTINGS_MODULE'] = settings - django.setup() - -def random_color(): - rgbl=[255,0,0] - random.shuffle(rgbl) - return tuple(rgbl) - - -def pairwise(iterable): - result = [] - for i in range(0, len(iterable) - 1, 2): - result.append((iterable[i], iterable[i+1])) - return np.array(result, dtype=np.int32) - -def find_min_y(array): - min_ = sys.maxsize - index = None - for i, pair in enumerate(array): - if pair[1] < min_: - min_ = pair[1] - index = i - - return array[index] - -def _get_docker_files(model_name: str, task_id: int): - _init_django('cvat.settings.development') - - from cvat.apps.auto_annotation.models import AnnotationModel - from cvat.apps.engine.models import Task as TaskModel - - task = TaskModel(pk=task_id) - model = AnnotationModel.objects.get(name=model_name) - - images_dir = task.data.get_data_dirname() - - py_file = model.interpretation_file.name - mapping_file = model.labelmap_file.name - xml_file = model.model_file.name - bin_file = model.weights_file.name - - image_files = [] - images_dir = os.path.abspath(images_dir) - for root, _, filenames in os.walk(images_dir): - for filename in fnmatch.filter(filenames, '*.jpg'): - image_files.append(os.path.join(root, filename)) - - return py_file, mapping_file, bin_file, xml_file, image_files - - -def main(): - kwargs = _get_kwargs() - - py_file = kwargs.get('py') - bin_file = kwargs.get('bin') - mapping_file = os.path.abspath(kwargs.get('json')) - xml_file = kwargs.get('xml') - - model_name = kwargs.get('model_name') - task_id = kwargs.get('task_id') - - is_docker = model_name and task_id - - # xor is `exclusive or`. English is: if one or the other but not both - if xor(bool(model_name), bool(task_id)): - logging.critical('Must provide both `--model-name` and `--task-id` together!') - return - - if is_docker: - files = _get_docker_files(model_name, task_id) - py_file = files[0] - mapping_file = files[1] - bin_file = files[2] - xml_file = files[3] - image_files = files[4] - else: - return_ = False - if not py_file: - logging.critical('Must provide --py file!') - return_ = True - if not bin_file: - logging.critical('Must provide --bin file!') - return_ = True - if not xml_file: - logging.critical('Must provide --xml file!') - return_ = True - if not mapping_file: - logging.critical('Must provide --json file!') - return_ = True - - if return_: - return - - if not os.path.isfile(py_file): - logging.critical('Py file not found! Check the path') - return - - if not os.path.isfile(bin_file): - logging.critical('Bin file is not found! Check path!') - return - - if not os.path.isfile(xml_file): - logging.critical('XML File not found! Check path!') - return - - if not os.path.isfile(mapping_file): - logging.critical('JSON file is not found! Check path!') - return - - with open(mapping_file) as json_file: - try: - mapping = json.load(json_file) - except json.decoder.JSONDecodeError: - logging.critical('JSON file not able to be parsed! Check file') - return - - try: - mapping = mapping['label_map'] - except KeyError: - logging.critical("JSON Mapping file must contain key `label_map`!") - logging.critical("Exiting") - return - - mapping = {int(k): v for k, v in mapping.items()} - - restricted = kwargs['restricted'] - - if not is_docker: - image_files = kwargs.get('image_files') - - if image_files: - image_data = [cv2.imread(f) for f in image_files] - else: - test_image = np.ones((1024, 1980, 3), np.uint8) * 255 - image_data = [test_image,] - attribute_spec = {} - - results = run_inference_engine_annotation(image_data, - xml_file, - bin_file, - mapping, - attribute_spec, - py_file, - restricted=restricted) - - - logging.warning('Inference didn\'t have any errors.') - show_images = kwargs.get('show_images', False) - - if show_images: - if image_files is None: - logging.critical("Warning, no images provided!") - logging.critical('Exiting without presenting results') - return - - if not results['shapes']: - logging.warning(str(results)) - logging.critical("No objects detected!") - return - - show_image_delay = kwargs['show_image_delay'] - show_labels = kwargs.get('show_labels') - - for index, data in enumerate(image_data): - for detection in results['shapes']: - if not detection['frame'] == index: - continue - points = detection['points'] - label_str = detection['label_id'] - - # Cv2 doesn't like floats for drawing - points = [int(p) for p in points] - color = random_color() - - if detection['type'] == 'rectangle': - cv2.rectangle(data, (points[0], points[1]), (points[2], points[3]), color, 3) - - if show_labels: - cv2.putText(data, label_str, (points[0], points[1] - 7), cv2.FONT_HERSHEY_COMPLEX, 0.6, color, 1) - - elif detection['type'] in ('polygon', 'polyline'): - # polylines is picky about datatypes - points = pairwise(points) - cv2.polylines(data, [points], 1, color) - - if show_labels: - min_point = find_min_y(points) - cv2.putText(data, label_str, (min_point[0], min_point[1] - 7), cv2.FONT_HERSHEY_COMPLEX, 0.6, color, 1) - - cv2.imshow(str(index), data) - cv2.waitKey(show_image_delay) - cv2.destroyWindow(str(index)) - - if kwargs['serialize']: - _init_django('cvat.settings.production') - - from cvat.apps.engine.serializers import LabeledDataSerializer - - # NOTE: We're actually using `run_inference_engine_annotation` - # incorrectly here. The `mapping` dict is supposed to be a mapping - # of integers -> integers and represents the transition from model - # integers to the labels in the database. We're using a mapping of - # integers -> strings. For testing purposes, this shortcut is fine. - # We just want to make sure everything works. Until, that is.... - # we want to test using the label serializer. Then we have to transition - # back to integers, otherwise the serializer complains about have a string - # where an integer is expected. We'll just brute force that. - - for shape in results['shapes']: - # Change the english label to an integer for serialization validation - shape['label_id'] = 1 - - serializer = LabeledDataSerializer(data=results) - - if not serializer.is_valid(): - logging.critical('Data unable to be serialized correctly!') - serializer.is_valid(raise_exception=True) - -if __name__ == '__main__': - main() diff --git a/utils/open_model_zoo/Retail/object_detection/text/pixel_link_mobilenet_v2/0001/mappings.json b/utils/open_model_zoo/Retail/object_detection/text/pixel_link_mobilenet_v2/0001/mappings.json deleted file mode 100644 index f6a0aa87964b..000000000000 --- a/utils/open_model_zoo/Retail/object_detection/text/pixel_link_mobilenet_v2/0001/mappings.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "label_map": { - "1": "text" - } -} diff --git a/utils/open_model_zoo/Retail/object_detection/text/pixel_link_mobilenet_v2/0001/pixel_link_mobilenet_v2.py b/utils/open_model_zoo/Retail/object_detection/text/pixel_link_mobilenet_v2/0001/pixel_link_mobilenet_v2.py deleted file mode 100644 index 77297cabdb18..000000000000 --- a/utils/open_model_zoo/Retail/object_detection/text/pixel_link_mobilenet_v2/0001/pixel_link_mobilenet_v2.py +++ /dev/null @@ -1,192 +0,0 @@ -import cv2 -import numpy as np - - -class PixelLinkDecoder(): - def __init__(self): - four_neighbours = False - if four_neighbours: - self._get_neighbours = self._get_neighbours_4 - else: - self._get_neighbours = self._get_neighbours_8 - self.pixel_conf_threshold = 0.8 - self.link_conf_threshold = 0.8 - - def decode(self, height, width, detections: dict): - self.image_height = height - self.image_width = width - self.pixel_scores = self._set_pixel_scores(detections['pixel_cls/add_2']) - self.link_scores = self._set_link_scores(detections['pixel_link/add_2']) - - self.pixel_mask = self.pixel_scores >= self.pixel_conf_threshold - self.link_mask = self.link_scores >= self.link_conf_threshold - self.points = list(zip(*np.where(self.pixel_mask))) - self.h, self.w = np.shape(self.pixel_mask) - self.group_mask = dict.fromkeys(self.points, -1) - self.bboxes = None - self.root_map = None - self.mask = None - - self._decode() - - def _softmax(self, x, axis=None): - return np.exp(x - self._logsumexp(x, axis=axis, keepdims=True)) - - def _logsumexp(self, a, axis=None, b=None, keepdims=False, return_sign=False): - if b is not None: - a, b = np.broadcast_arrays(a, b) - if np.any(b == 0): - a = a + 0. # promote to at least float - a[b == 0] = -np.inf - - a_max = np.amax(a, axis=axis, keepdims=True) - - if a_max.ndim > 0: - a_max[~np.isfinite(a_max)] = 0 - elif not np.isfinite(a_max): - a_max = 0 - - if b is not None: - b = np.asarray(b) - tmp = b * np.exp(a - a_max) - else: - tmp = np.exp(a - a_max) - - # suppress warnings about log of zero - with np.errstate(divide='ignore'): - s = np.sum(tmp, axis=axis, keepdims=keepdims) - if return_sign: - sgn = np.sign(s) - s *= sgn # /= makes more sense but we need zero -> zero - out = np.log(s) - - if not keepdims: - a_max = np.squeeze(a_max, axis=axis) - out += a_max - - if return_sign: - return out, sgn - else: - return out - - def _set_pixel_scores(self, pixel_scores): - "get softmaxed properly shaped pixel scores" - tmp = np.transpose(pixel_scores, (0, 2, 3, 1)) - return self._softmax(tmp, axis=-1)[0, :, :, 1] - - def _set_link_scores(self, link_scores): - "get softmaxed properly shaped links scores" - tmp = np.transpose(link_scores, (0, 2, 3, 1)) - tmp_reshaped = tmp.reshape(tmp.shape[:-1] + (8, 2)) - return self._softmax(tmp_reshaped, axis=-1)[0, :, :, :, 1] - - def _find_root(self, point): - root = point - update_parent = False - tmp = self.group_mask[root] - while tmp is not -1: - root = tmp - tmp = self.group_mask[root] - update_parent = True - if update_parent: - self.group_mask[point] = root - return root - - def _join(self, p1, p2): - root1 = self._find_root(p1) - root2 = self._find_root(p2) - if root1 != root2: - self.group_mask[root2] = root1 - - def _get_index(self, root): - if root not in self.root_map: - self.root_map[root] = len(self.root_map) + 1 - return self.root_map[root] - - def _get_all(self): - self.root_map = {} - self.mask = np.zeros_like(self.pixel_mask, dtype=np.int32) - - for point in self.points: - point_root = self._find_root(point) - bbox_idx = self._get_index(point_root) - self.mask[point] = bbox_idx - - def _get_neighbours_8(self, x, y): - w, h = self.w, self.h - tmp = [(0, x - 1, y - 1), (1, x, y - 1), - (2, x + 1, y - 1), (3, x - 1, y), - (4, x + 1, y), (5, x - 1, y + 1), - (6, x, y + 1), (7, x + 1, y + 1)] - - return [i for i in tmp if i[1] >= 0 and i[1] < w and i[2] >= 0 and i[2] < h] - - def _get_neighbours_4(self, x, y): - w, h = self.w, self.h - tmp = [(1, x, y - 1), - (3, x - 1, y), - (4, x + 1, y), - (6, x, y + 1)] - - return [i for i in tmp if i[1] >= 0 and i[1] < w and i[2] >= 0 and i[2] < h] - - def _mask_to_bboxes(self, min_area=300, min_height=10): - self.bboxes = [] - max_bbox_idx = self.mask.max() - mask_tmp = cv2.resize(self.mask, (self.image_width, self.image_height), interpolation=cv2.INTER_NEAREST) - - for bbox_idx in range(1, max_bbox_idx + 1): - bbox_mask = mask_tmp == bbox_idx - cnts, _ = cv2.findContours(bbox_mask.astype(np.uint8), cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) - if len(cnts) == 0: - continue - cnt = cnts[0] - rect, w, h = self._min_area_rect(cnt) - if min(w, h) < min_height: - continue - if w * h < min_area: - continue - self.bboxes.append(self._order_points(rect)) - - def _min_area_rect(self, cnt): - rect = cv2.minAreaRect(cnt) - w, h = rect[1] - box = cv2.boxPoints(rect) - box = np.int0(box) - return box, w, h - - def _order_points(self, rect): - """ (x, y) - Order: TL, TR, BR, BL - """ - tmp = np.zeros_like(rect) - sums = rect.sum(axis=1) - tmp[0] = rect[np.argmin(sums)] - tmp[2] = rect[np.argmax(sums)] - diff = np.diff(rect, axis=1) - tmp[1] = rect[np.argmin(diff)] - tmp[3] = rect[np.argmax(diff)] - return tmp - - def _decode(self): - for point in self.points: - y, x = point - neighbours = self._get_neighbours(x, y) - for n_idx, nx, ny in neighbours: - link_value = self.link_mask[y, x, n_idx] - pixel_cls = self.pixel_mask[ny, nx] - if link_value and pixel_cls: - self._join(point, (ny, nx)) - - self._get_all() - self._mask_to_bboxes() - - -label = 1 -pcd = PixelLinkDecoder() -for detection in detections: - frame = detection['frame_id'] - pcd.decode(detection['frame_height'], detection['frame_width'], detection['detections']) - for box in pcd.bboxes: - box = [[int(b[0]), int(b[1])] for b in box] - results.add_polygon(box, label, frame) diff --git a/utils/open_model_zoo/Retail/object_detection/text/pixel_link_mobilenet_v2/0004/mappings.json b/utils/open_model_zoo/Retail/object_detection/text/pixel_link_mobilenet_v2/0004/mappings.json deleted file mode 100644 index f6a0aa87964b..000000000000 --- a/utils/open_model_zoo/Retail/object_detection/text/pixel_link_mobilenet_v2/0004/mappings.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "label_map": { - "1": "text" - } -} diff --git a/utils/open_model_zoo/Retail/object_detection/text/pixel_link_mobilenet_v2/README.md b/utils/open_model_zoo/Retail/object_detection/text/pixel_link_mobilenet_v2/README.md deleted file mode 100644 index c7bac387865f..000000000000 --- a/utils/open_model_zoo/Retail/object_detection/text/pixel_link_mobilenet_v2/README.md +++ /dev/null @@ -1,5 +0,0 @@ -# Pixel Link - -* Model for the Detecting Scene Text vai Instance Segmentation -* Download using the `intel_model_zoo` using `$./downloader.py text-detection-0002` -* See [this Arxiv](https://arxiv.org/abs/1801.01315) link for the technical details diff --git a/utils/open_model_zoo/Transportation/semantic-segmentation-adas/interp.py b/utils/open_model_zoo/Transportation/semantic-segmentation-adas/interp.py deleted file mode 100644 index 58a87fa35d1b..000000000000 --- a/utils/open_model_zoo/Transportation/semantic-segmentation-adas/interp.py +++ /dev/null @@ -1,31 +0,0 @@ -import numpy as np -from skimage.measure import approximate_polygon, find_contours - -import cv2 - - -for frame_results in detections: - frame_height = frame_results['frame_height'] - frame_width = frame_results['frame_width'] - frame_number = frame_results['frame_id'] - detection = frame_results['detections'] - detection = detection[0, 0, :, :] - width, height = detection.shape - - for i in range(21): - zero = np.zeros((width,height),dtype=np.uint8) - - f = float(i) - zero = ((detection == f) * 255).astype(np.float32) - zero = cv2.resize(zero, dsize=(frame_width, frame_height), interpolation=cv2.INTER_CUBIC) - - contours = find_contours(zero, 0.8) - - for contour in contours: - contour = np.flip(contour, axis=1) - contour = approximate_polygon(contour, tolerance=2.5) - segmentation = contour.tolist() - if len(segmentation) < 3: - continue - - results.add_polygon(segmentation, i, frame_number) diff --git a/utils/open_model_zoo/Transportation/semantic-segmentation-adas/mapping.json b/utils/open_model_zoo/Transportation/semantic-segmentation-adas/mapping.json deleted file mode 100644 index cbda289d3e2f..000000000000 --- a/utils/open_model_zoo/Transportation/semantic-segmentation-adas/mapping.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "label_map": { - "0": "road", - "1": "sidewalk", - "2": "building", - "3": "wall", - "4": "fence", - "5": "pole", - "6": "traffic light", - "7": "traffic sign", - "8": "vegetation", - "9": "terrain", - "10": "sky", - "11": "person", - "12": "rider", - "13": "car", - "14": "truck", - "15": "bus", - "16": "train", - "17": "motorcycle", - "18": "bicycle", - "19": "ego-vehicle", - "20": "background" - } -} diff --git a/utils/open_model_zoo/faster_rcnn_inception_v2_coco/README.md b/utils/open_model_zoo/faster_rcnn_inception_v2_coco/README.md deleted file mode 100644 index a6ede4453a92..000000000000 --- a/utils/open_model_zoo/faster_rcnn_inception_v2_coco/README.md +++ /dev/null @@ -1,6 +0,0 @@ -# Faster R-CNN with Inception v2 (https://arxiv.org/pdf/1801.04381.pdf) pre-trained on the COCO dataset - -### What is it? -* This application allows you automatically to annotate many various objects on images. -* It uses [Faster RCNN Inception Resnet v2 Atrous Coco Model](http://download.tensorflow.org/models/object_detection/faster_rcnn_inception_resnet_v2_atrous_coco_2018_01_28.tar.gz) from [tensorflow detection model zoo](https://github.com/tensorflow/models/blob/master/research/object_detection/g3doc/detection_model_zoo.md) -* It can work on CPU (with Tensorflow or OpenVINO) or GPU (with Tensorflow GPU). diff --git a/utils/open_model_zoo/faster_rcnn_inception_v2_coco/interp.py b/utils/open_model_zoo/faster_rcnn_inception_v2_coco/interp.py deleted file mode 100644 index f8a8a60219f0..000000000000 --- a/utils/open_model_zoo/faster_rcnn_inception_v2_coco/interp.py +++ /dev/null @@ -1,19 +0,0 @@ -threshold = .5 - -for detection in detections: - frame_number = detection['frame_id'] - height = detection['frame_height'] - width = detection['frame_width'] - detection = detection['detections'] - - prediction = detection[0][0] - for obj in prediction: - obj_class = int(obj[1]) - obj_value = obj[2] - if obj_value >= threshold: - x = obj[3] * width - y = obj[4] * height - right = obj[5] * width - bottom = obj[6] * height - - results.add_box(x, y, right, bottom, obj_class, frame_number) diff --git a/utils/open_model_zoo/faster_rcnn_inception_v2_coco/mapping.json b/utils/open_model_zoo/faster_rcnn_inception_v2_coco/mapping.json deleted file mode 100644 index 3efdb307565f..000000000000 --- a/utils/open_model_zoo/faster_rcnn_inception_v2_coco/mapping.json +++ /dev/null @@ -1,84 +0,0 @@ -{ - "label_map": { - "1": "person", - "2": "bicycle", - "3": "car", - "4": "motorcycle", - "5": "airplane", - "6": "bus", - "7": "train", - "8": "truck", - "9": "boat", - "10": "traffic_light", - "11": "fire_hydrant", - "13": "stop_sign", - "14": "parking_meter", - "15": "bench", - "16": "bird", - "17": "cat", - "18": "dog", - "19": "horse", - "20": "sheep", - "21": "cow", - "22": "elephant", - "23": "bear", - "24": "zebra", - "25": "giraffe", - "27": "backpack", - "28": "umbrella", - "31": "handbag", - "32": "tie", - "33": "suitcase", - "34": "frisbee", - "35": "skis", - "36": "snowboard", - "37": "sports_ball", - "38": "kite", - "39": "baseball_bat", - "40": "baseball_glove", - "41": "skateboard", - "42": "surfboard", - "43": "tennis_racket", - "44": "bottle", - "46": "wine_glass", - "47": "cup", - "48": "fork", - "49": "knife", - "50": "spoon", - "51": "bowl", - "52": "banana", - "53": "apple", - "54": "sandwich", - "55": "orange", - "56": "broccoli", - "57": "carrot", - "58": "hot_dog", - "59": "pizza", - "60": "donut", - "61": "cake", - "62": "chair", - "63": "couch", - "64": "potted_plant", - "65": "bed", - "67": "dining_table", - "70": "toilet", - "72": "tv", - "73": "laptop", - "74": "mouse", - "75": "remote", - "76": "keyboard", - "77": "cell_phone", - "78": "microwave", - "79": "oven", - "80": "toaster", - "81": "sink", - "83": "refrigerator", - "84": "book", - "85": "clock", - "86": "vase", - "87": "scissors", - "88": "teddy_bear", - "89": "hair_drier", - "90": "toothbrush" - } -} diff --git a/utils/open_model_zoo/mask_rcnn_inception_resnet_v2_atrous_coco/README.md b/utils/open_model_zoo/mask_rcnn_inception_resnet_v2_atrous_coco/README.md deleted file mode 100644 index be7a82121bf0..000000000000 --- a/utils/open_model_zoo/mask_rcnn_inception_resnet_v2_atrous_coco/README.md +++ /dev/null @@ -1,32 +0,0 @@ -# mask_rcnn_inception_resnet_v2_atrous_coco - -## Use Case and High-Level Description - -Mask R-CNN Inception Resnet V2 Atrous is trained on COCO dataset and used for object instance segmentation. -For details, see a [paper](https://arxiv.org/pdf/1703.06870.pdf). - -## Specification - -| Metric | Value | -|---------------------------------|-------------------------------------------| -| Type | Instance segmentation | -| GFlops | 675.314 | -| MParams | 92.368 | -| Source framework | TensorFlow\* | - -## Legal Information - -[https://raw.githubusercontent.com/tensorflow/models/master/LICENSE]() - -## OpenVINO Conversion Notes - -In order to convert the code into the openvino format, please see the [following link](https://docs.openvinotoolkit.org/latest/_docs_MO_DG_prepare_model_convert_model_tf_specific_Convert_Object_Detection_API_Models.html#mask_r_cnn_topologies). - -The conversion command from the command line prompt will look something like the following. - -```shell -$ python /opt/intel/openvino/deployment_tools/model_optimizer/mo_tf.py \ - --input_model /path/to/frozen_inference_graph.pb \ - --tensorflow_use_custom_operations_config /opt/intel/openvino/deployment_tools/model_optimizer/extensions/front/tf/mask_rcnn_support.json \ - --tensorflow_object_detection_api_pipeline_config /path/to/pipeline.config -``` diff --git a/utils/open_model_zoo/mask_rcnn_inception_resnet_v2_atrous_coco/interp.py b/utils/open_model_zoo/mask_rcnn_inception_resnet_v2_atrous_coco/interp.py deleted file mode 100644 index 6625a8349330..000000000000 --- a/utils/open_model_zoo/mask_rcnn_inception_resnet_v2_atrous_coco/interp.py +++ /dev/null @@ -1,64 +0,0 @@ -import numpy as np -import cv2 -from skimage.measure import approximate_polygon, find_contours - - -MASK_THRESHOLD = .5 -PROBABILITY_THRESHOLD = 0.2 - - -# Ref: https://software.intel.com/en-us/forums/computer-vision/topic/804895 -def segm_postprocess(box: list, raw_cls_mask, im_h, im_w, threshold): - ymin, xmin, ymax, xmax = box - - width = int(abs(xmax - xmin)) - height = int(abs(ymax - ymin)) - - result = np.zeros((im_h, im_w), dtype=np.uint8) - resized_mask = cv2.resize(raw_cls_mask, dsize=(height, width), interpolation=cv2.INTER_CUBIC) - - # extract the ROI of the image - ymin = int(round(ymin)) - xmin = int(round(xmin)) - ymax = ymin + height - xmax = xmin + width - result[xmin:xmax, ymin:ymax] = (resized_mask>threshold).astype(np.uint8) * 255 - - return result - - -for detection in detections: - frame_number = detection['frame_id'] - height = detection['frame_height'] - width = detection['frame_width'] - detection = detection['detections'] - - masks = detection['masks'] - boxes = detection['reshape_do_2d'] - - for index, box in enumerate(boxes): - label = int(box[1]) - obj_value = box[2] - if obj_value >= PROBABILITY_THRESHOLD: - x = box[3] * width - y = box[4] * height - right = box[5] * width - bottom = box[6] * height - mask = masks[index][label - 1] - - mask = segm_postprocess((x, y, right, bottom), - mask, - height, - width, - MASK_THRESHOLD) - - contours = find_contours(mask, MASK_THRESHOLD) - contour = contours[0] - contour = np.flip(contour, axis=1) - contour = approximate_polygon(contour, tolerance=2.5) - segmentation = contour.tolist() - - - # NOTE: if you want to see the boxes, uncomment next line - # results.add_box(x, y, right, bottom, label, frame_number) - results.add_polygon(segmentation, label, frame_number) diff --git a/utils/open_model_zoo/mask_rcnn_inception_resnet_v2_atrous_coco/mapping.json b/utils/open_model_zoo/mask_rcnn_inception_resnet_v2_atrous_coco/mapping.json deleted file mode 100644 index 3efdb307565f..000000000000 --- a/utils/open_model_zoo/mask_rcnn_inception_resnet_v2_atrous_coco/mapping.json +++ /dev/null @@ -1,84 +0,0 @@ -{ - "label_map": { - "1": "person", - "2": "bicycle", - "3": "car", - "4": "motorcycle", - "5": "airplane", - "6": "bus", - "7": "train", - "8": "truck", - "9": "boat", - "10": "traffic_light", - "11": "fire_hydrant", - "13": "stop_sign", - "14": "parking_meter", - "15": "bench", - "16": "bird", - "17": "cat", - "18": "dog", - "19": "horse", - "20": "sheep", - "21": "cow", - "22": "elephant", - "23": "bear", - "24": "zebra", - "25": "giraffe", - "27": "backpack", - "28": "umbrella", - "31": "handbag", - "32": "tie", - "33": "suitcase", - "34": "frisbee", - "35": "skis", - "36": "snowboard", - "37": "sports_ball", - "38": "kite", - "39": "baseball_bat", - "40": "baseball_glove", - "41": "skateboard", - "42": "surfboard", - "43": "tennis_racket", - "44": "bottle", - "46": "wine_glass", - "47": "cup", - "48": "fork", - "49": "knife", - "50": "spoon", - "51": "bowl", - "52": "banana", - "53": "apple", - "54": "sandwich", - "55": "orange", - "56": "broccoli", - "57": "carrot", - "58": "hot_dog", - "59": "pizza", - "60": "donut", - "61": "cake", - "62": "chair", - "63": "couch", - "64": "potted_plant", - "65": "bed", - "67": "dining_table", - "70": "toilet", - "72": "tv", - "73": "laptop", - "74": "mouse", - "75": "remote", - "76": "keyboard", - "77": "cell_phone", - "78": "microwave", - "79": "oven", - "80": "toaster", - "81": "sink", - "83": "refrigerator", - "84": "book", - "85": "clock", - "86": "vase", - "87": "scissors", - "88": "teddy_bear", - "89": "hair_drier", - "90": "toothbrush" - } -} diff --git a/utils/open_model_zoo/yolov3/README.md b/utils/open_model_zoo/yolov3/README.md deleted file mode 100644 index 2e47953cb3f8..000000000000 --- a/utils/open_model_zoo/yolov3/README.md +++ /dev/null @@ -1,22 +0,0 @@ -# Object Detection YOLO V3 Python Demo, Async API Performance Showcase - -See [these instructions][1] for converting the yolo weights to the OpenVino format. - -As of OpenVINO 2019 R3, only tensorflow 1.13 and NetworkX 2.3. -These can be explicitly installed using the following command. - -```bash -python3 -m pip install tensorflow==1.13 networkx==2.3 -``` - - -Additionally, at the time of writing, the model optimizer required an input shape. - -``` bash -python3 mo_tf.py \ - --input_model /path/to/yolo_v3.pb \ - --tensorflow_use_custom_operations_config $MO_ROOT/extensions/front/tf/yolo_v3.json \ - --input_shape [1,416,416,3] -``` - -[1]: https://docs.openvinotoolkit.org/latest/_docs_MO_DG_prepare_model_convert_model_tf_specific_Convert_YOLO_From_Tensorflow.html diff --git a/utils/open_model_zoo/yolov3/interp.py b/utils/open_model_zoo/yolov3/interp.py deleted file mode 100644 index 4c76c85448d0..000000000000 --- a/utils/open_model_zoo/yolov3/interp.py +++ /dev/null @@ -1,160 +0,0 @@ -from math import exp - - -class Parser: - IOU_THRESHOLD = 0.4 - PROB_THRESHOLD = 0.5 - - def __init__(self): - self.objects = [] - - def scale_bbox(self, x, y, h, w, class_id, confidence, h_scale, w_scale): - xmin = int((x - w / 2) * w_scale) - ymin = int((y - h / 2) * h_scale) - xmax = int(xmin + w * w_scale) - ymax = int(ymin + h * h_scale) - - return dict(xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax, class_id=class_id, confidence=confidence) - - def entry_index(self, side, coord, classes, location, entry): - side_power_2 = side ** 2 - n = location // side_power_2 - loc = location % side_power_2 - return int(side_power_2 * (n * (coord + classes + 1) + entry) + loc) - - def intersection_over_union(self, box_1, box_2): - width_of_overlap_area = min(box_1['xmax'], box_2['xmax']) - max(box_1['xmin'], box_2['xmin']) - height_of_overlap_area = min(box_1['ymax'], box_2['ymax']) - max(box_1['ymin'], box_2['ymin']) - if width_of_overlap_area < 0 or height_of_overlap_area < 0: - area_of_overlap = 0 - else: - area_of_overlap = width_of_overlap_area * height_of_overlap_area - box_1_area = (box_1['ymax'] - box_1['ymin']) * (box_1['xmax'] - box_1['xmin']) - box_2_area = (box_2['ymax'] - box_2['ymin']) * (box_2['xmax'] - box_2['xmin']) - area_of_union = box_1_area + box_2_area - area_of_overlap - if area_of_union == 0: - return 0 - return area_of_overlap / area_of_union - - - def sort_objects(self): - self.objects = sorted(self.objects, key=lambda obj : obj['confidence'], reverse=True) - - for i in range(len(self.objects)): - if self.objects[i]['confidence'] == 0: - continue - for j in range(i + 1, len(self.objects)): - if self.intersection_over_union(self.objects[i], self.objects[j]) > self.IOU_THRESHOLD: - self.objects[j]['confidence'] = 0 - - def parse_yolo_region(self, blob: 'np.ndarray', original_shape: list, params: dict) -> list: - - # YOLO magic numbers - # See: https://github.com/opencv/open_model_zoo/blob/acf297c73db8cb3f68791ae1fad4a7cc4a6039e5/demos/python_demos/object_detection_demo_yolov3_async/object_detection_demo_yolov3_async.py#L61 - num = 3 - coords = 4 - classes = 80 - # ----------------- - - _, _, out_blob_h, out_blob_w = blob.shape - assert out_blob_w == out_blob_h, "Invalid size of output blob. It sould be in NCHW layout and height should " \ - "be equal to width. Current height = {}, current width = {}" \ - "".format(out_blob_h, out_blob_w) - - # ------ Extracting layer parameters -- - orig_im_h, orig_im_w = original_shape - predictions = blob.flatten() - side_square = params['side'] * params['side'] - - # ------ Parsing YOLO Region output -- - for i in range(side_square): - row = i // params['side'] - col = i % params['side'] - for n in range(num): - # -----entry index calcs------ - obj_index = self.entry_index(params['side'], coords, classes, n * side_square + i, coords) - scale = predictions[obj_index] - if scale < self.PROB_THRESHOLD: - continue - box_index = self.entry_index(params['side'], coords, classes, n * side_square + i, 0) - - # Network produces location predictions in absolute coordinates of feature maps. - # Scale it to relative coordinates. - x = (col + predictions[box_index + 0 * side_square]) / params['side'] * 416 - y = (row + predictions[box_index + 1 * side_square]) / params['side'] * 416 - # Value for exp is very big number in some cases so following construction is using here - try: - h_exp = exp(predictions[box_index + 3 * side_square]) - w_exp = exp(predictions[box_index + 2 * side_square]) - except OverflowError: - continue - - w = w_exp * params['anchors'][2 * n] - h = h_exp * params['anchors'][2 * n + 1] - - for j in range(classes): - class_index = self.entry_index(params['side'], coords, classes, n * side_square + i, - coords + 1 + j) - confidence = scale * predictions[class_index] - if confidence < self.PROB_THRESHOLD: - continue - - self.objects.append(self.scale_bbox(x=x, - y=y, - h=h, - w=w, - class_id=j, - confidence=confidence, - h_scale=(orig_im_h/416), - w_scale=(orig_im_w/416))) - - -for detection in detections: - frame_number = detection['frame_id'] - height = detection['frame_height'] - width = detection['frame_width'] - detection = detection['detections'] - - original_shape = (height, width) - - # https://github.com/opencv/open_model_zoo/blob/master/demos/python_demos/object_detection_demo_yolov3_async/object_detection_demo_yolov3_async.py#L72 - anchors = [10,13,16,30,33,23,30,61,62,45,59,119,116,90,156,198,373,326] - conv_6 = {'side': 13, 'mask': [6,7,8]} - conv_14 = {'side': 26, 'mask': [3,4,5]} - conv_22 = {'side': 52, 'mask': [0,1,2]} - - yolo_params = {'detector/yolo-v3/Conv_6/BiasAdd/YoloRegion': conv_6, - 'detector/yolo-v3/Conv_14/BiasAdd/YoloRegion': conv_14, - 'detector/yolo-v3/Conv_22/BiasAdd/YoloRegion': conv_22} - - for conv_net in yolo_params.values(): - mask = conv_net['mask'] - masked_anchors = [] - for idx in mask: - masked_anchors += [anchors[idx * 2], anchors[idx * 2 + 1]] - - conv_net['anchors'] = masked_anchors - - parser = Parser() - - for name, blob in detection.items(): - parser.parse_yolo_region(blob, original_shape, yolo_params[name]) - - parser.sort_objects() - - objects = [] - for obj in parser.objects: - if obj['confidence'] >= parser.PROB_THRESHOLD: - label = obj['class_id'] - xmin = obj['xmin'] - xmax = obj['xmax'] - ymin = obj['ymin'] - ymax = obj['ymax'] - - # Enforcing extra checks for bounding box coordinates - xmin = max(0,xmin) - ymin = max(0,ymin) - xmax = min(xmax,width) - ymax = min(ymax,height) - - results.add_box(xmin, ymin, xmax, ymax, label, frame_number) diff --git a/utils/open_model_zoo/yolov3/mapping.json b/utils/open_model_zoo/yolov3/mapping.json deleted file mode 100644 index bfb65a24cf0c..000000000000 --- a/utils/open_model_zoo/yolov3/mapping.json +++ /dev/null @@ -1,84 +0,0 @@ -{ - "label_map": { - "0": "person", - "1": "bicycle", - "2": "car", - "3": "motorbike", - "4": "aeroplane", - "5": "bus", - "6": "train", - "7": "truck", - "8": "boat", - "9": "traffic light", - "10": "fire hydrant", - "11": "stop sign", - "12": "parking meter", - "13": "bench", - "14": "bird", - "15": "cat", - "16": "dog", - "17": "horse", - "18": "sheep", - "19": "cow", - "20": "elephant", - "21": "bear", - "22": "zebra", - "23": "giraffe", - "24": "backpack", - "25": "umbrella", - "26": "handbag", - "27": "tie", - "28": "suitcase", - "29": "frisbee", - "30": "skis", - "31": "snowboard", - "32": "sports ball", - "33": "kite", - "34": "baseball bat", - "35": "baseball glove", - "36": "skateboard", - "37": "surfboard", - "38": "tennis racket", - "39": "bottle", - "40": "wine glass", - "41": "cup", - "42": "fork", - "43": "knife", - "44": "spoon", - "45": "bowl", - "46": "banana", - "47": "apple", - "48": "sandwich", - "49": "orange", - "50": "broccoli", - "51": "carrot", - "52": "hot dog", - "53": "pizza", - "54": "donut", - "55": "cake", - "56": "chair", - "57": "sofa", - "58": "pottedplant", - "59": "bed", - "60": "diningtable", - "61": "toilet", - "62": "tvmonitor", - "63": "laptop", - "64": "mouse", - "65": "remote", - "66": "keyboard", - "67": "cell phone", - "68": "microwave", - "69": "oven", - "70": "toaster", - "71": "sink", - "72": "refrigerator", - "73": "book", - "74": "clock", - "75": "vase", - "76": "scissors", - "77": "teddy bear", - "78": "hair drier", - "79": "toothbrush" - } -}