diff --git a/.travis.yml b/.travis.yml index 87631e76006..8cbae296ccc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,9 +9,9 @@ services: - docker before_script: - - docker-compose -f docker-compose.yml -f docker-compose.ci.yml up --build -d + - docker-compose -f docker-compose.yml -f docker-compose.ci.yml build script: - - docker exec -it cvat /bin/bash -c 'python3 manage.py test cvat/apps utils/cli' - - docker exec -it cvat /bin/bash -c 'python3 manage.py test datumaro/' - - docker exec -it cvat /bin/bash -c 'cd cvat-core && npm install && npm run test && npm run coveralls' + - docker-compose -f docker-compose.yml -f docker-compose.ci.yml run cvat_ci /bin/bash -c 'python3 manage.py test cvat/apps utils/cli' + - docker-compose -f docker-compose.yml -f docker-compose.ci.yml run cvat_ci /bin/bash -c 'python3 manage.py test datumaro/' + - docker-compose -f docker-compose.yml -f docker-compose.ci.yml run cvat_ci /bin/bash -c 'cd cvat-core && npm install && npm run test && npm run coveralls' diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 985f2fe9d6d..9072dd3b622 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -91,8 +91,8 @@ to changes in ``.env/bin/activate`` file are active. - Perform all steps in the automatic annotation section - Download ReID model and save it somewhere: ```sh - wget https://download.01.org/openvinotoolkit/2018_R5/open_model_zoo/person-reidentification-retail-0079/FP32/person-reidentification-retail-0079.xml -O reid.xml - wget https://download.01.org/openvinotoolkit/2018_R5/open_model_zoo/person-reidentification-retail-0079/FP32/person-reidentification-retail-0079.bin -O reid.bin + curl https://download.01.org/openvinotoolkit/2018_R5/open_model_zoo/person-reidentification-retail-0079/FP32/person-reidentification-retail-0079.xml -o reid.xml + curl https://download.01.org/openvinotoolkit/2018_R5/open_model_zoo/person-reidentification-retail-0079/FP32/person-reidentification-retail-0079.bin -o reid.bin ``` - Add next line to ``.env/bin/activate``: ```sh @@ -103,7 +103,7 @@ to changes in ``.env/bin/activate`` file are active. - Perform all steps in the automatic annotation section - Download Deep Extreme Cut model, unpack it, and save somewhere: ```sh -wget https://download.01.org/openvinotoolkit/models_contrib/cvat/dextr_model_v1.zip -O dextr.zip +curl https://download.01.org/openvinotoolkit/models_contrib/cvat/dextr_model_v1.zip -o dextr.zip unzip dextr.zip ``` - Add next lines to ``.env/bin/activate``: @@ -115,7 +115,7 @@ unzip dextr.zip ### Tensorflow RCNN - Download RCNN model, unpack it, and save it somewhere: ```sh -wget -O model.tar.gz http://download.tensorflow.org/models/object_detection/faster_rcnn_inception_resnet_v2_atrous_coco_2018_01_28.tar.gz && \ +curl http://download.tensorflow.org/models/object_detection/faster_rcnn_inception_resnet_v2_atrous_coco_2018_01_28.tar.gz -o model.tar.gz && \ tar -xzf model.tar.gz ``` - Add next lines to ``.env/bin/activate``: @@ -127,7 +127,7 @@ tar -xzf model.tar.gz ### Tensorflow Mask RCNN - Download Mask RCNN model, and save it somewhere: ```sh -wget https://github.com/matterport/Mask_RCNN/releases/download/v2.0/mask_rcnn_coco.h5 +curl https://github.com/matterport/Mask_RCNN/releases/download/v2.0/mask_rcnn_coco.h5 -o mask_rcnn_coco.h5 ``` - Add next lines to ``.env/bin/activate``: ```sh diff --git a/Dockerfile b/Dockerfile index 8c0bdab33ff..c2465f7fa90 100644 --- a/Dockerfile +++ b/Dockerfile @@ -22,9 +22,7 @@ ENV DJANGO_CONFIGURATION=${DJANGO_CONFIGURATION} # Install necessary apt packages RUN apt-get update && \ apt-get install -yq \ - python-software-properties \ - software-properties-common \ - wget && \ + software-properties-common && \ add-apt-repository ppa:mc3man/xerus-media -y && \ add-apt-repository ppa:mc3man/gstffmpeg-keep -y && \ apt-get update && \ @@ -40,14 +38,19 @@ RUN apt-get update && \ python3-dev \ python3-pip \ tzdata \ - unzip \ - unrar \ p7zip-full \ - git-core \ - libsm6 \ - libxext6 && \ - python3 -m pip install -U pip && \ - python3 -m pip install -U setuptools && \ + git \ + ssh \ + poppler-utils \ + curl && \ + curl https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | bash && \ + apt-get install -y git-lfs && git lfs install && \ + if [ -z ${socks_proxy} ]; then \ + echo export "GIT_SSH_COMMAND=\"ssh -o StrictHostKeyChecking=no -o ConnectTimeout=30\"" >> ${HOME}/.bashrc; \ + else \ + echo export "GIT_SSH_COMMAND=\"ssh -o StrictHostKeyChecking=no -o ConnectTimeout=30 -o ProxyCommand='nc -X 5 -x ${socks_proxy} %h %p'\"" >> ${HOME}/.bashrc; \ + fi && \ + python3 -m pip install --no-cache-dir -U pip setuptools && \ ln -fs /usr/share/zoneinfo/${TZ} /etc/localtime && \ dpkg-reconfigure -f noninteractive tzdata && \ add-apt-repository --remove ppa:mc3man/gstffmpeg-keep -y && \ @@ -69,8 +72,8 @@ ENV REID_MODEL_DIR=${HOME}/reid RUN if [ "$OPENVINO_TOOLKIT" = "yes" ]; then \ /tmp/components/openvino/install.sh && \ mkdir ${REID_MODEL_DIR} && \ - wget https://download.01.org/openvinotoolkit/2018_R5/open_model_zoo/person-reidentification-retail-0079/FP32/person-reidentification-retail-0079.xml -O reid/reid.xml && \ - wget https://download.01.org/openvinotoolkit/2018_R5/open_model_zoo/person-reidentification-retail-0079/FP32/person-reidentification-retail-0079.bin -O reid/reid.bin; \ + curl https://download.01.org/openvinotoolkit/2018_R5/open_model_zoo/person-reidentification-retail-0079/FP32/person-reidentification-retail-0079.xml -o reid/reid.xml && \ + curl https://download.01.org/openvinotoolkit/2018_R5/open_model_zoo/person-reidentification-retail-0079/FP32/person-reidentification-retail-0079.bin -o reid/reid.bin; \ fi # Tensorflow annotation support @@ -89,29 +92,6 @@ RUN if [ "$AUTO_SEGMENTATION" = "yes" ]; then \ bash -i /tmp/components/auto_segmentation/install.sh; \ fi -ARG WITH_TESTS -RUN if [ "$WITH_TESTS" = "yes" ]; then \ - wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - && \ - echo 'deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main' | tee /etc/apt/sources.list.d/google-chrome.list && \ - wget -qO- https://deb.nodesource.com/setup_9.x | bash - && \ - apt-get update && \ - DEBIAN_FRONTEND=noninteractive apt-get install -yq \ - google-chrome-stable \ - nodejs && \ - rm -rf /var/lib/apt/lists/*; \ - mkdir tests && cd tests && npm install \ - eslint \ - eslint-detailed-reporter \ - karma \ - karma-chrome-launcher \ - karma-coveralls \ - karma-coverage \ - karma-junit-reporter \ - karma-qunit \ - qunit; \ - echo "export PATH=~/tests/node_modules/.bin:${PATH}" >> ~/.bashrc; \ - fi - # Install and initialize CVAT, copy all necessary files COPY cvat/requirements/ /tmp/requirements/ COPY supervisord.conf mod_wsgi.conf wait-for-it.sh manage.py ${HOME}/ @@ -119,23 +99,6 @@ RUN python3 -m pip install --no-cache-dir -r /tmp/requirements/${DJANGO_CONFIGUR # pycocotools package is impossible to install with its dependencies by one pip install command RUN python3 -m pip install --no-cache-dir pycocotools==2.0.0 -# Install git application dependencies -RUN apt-get update && \ - apt-get install -y ssh netcat-openbsd git curl zip && \ - wget -qO /dev/stdout https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | bash && \ - apt-get install -y git-lfs && \ - git lfs install && \ - rm -rf /var/lib/apt/lists/* && \ - if [ -z ${socks_proxy} ]; then \ - echo export "GIT_SSH_COMMAND=\"ssh -o StrictHostKeyChecking=no -o ConnectTimeout=30\"" >> ${HOME}/.bashrc; \ - else \ - echo export "GIT_SSH_COMMAND=\"ssh -o StrictHostKeyChecking=no -o ConnectTimeout=30 -o ProxyCommand='nc -X 5 -x ${socks_proxy} %h %p'\"" >> ${HOME}/.bashrc; \ - fi - -# Install poppler for working with pdfs -RUN apt-get update && \ - apt install -y poppler-utils && \ - rm -rf /var/lib/apt/lists/* # CUDA support ARG CUDA_SUPPORT @@ -150,8 +113,8 @@ ENV WITH_DEXTR=${WITH_DEXTR} ENV DEXTR_MODEL_DIR=${HOME}/dextr RUN if [ "$WITH_DEXTR" = "yes" ]; then \ mkdir ${DEXTR_MODEL_DIR} -p && \ - wget https://download.01.org/openvinotoolkit/models_contrib/cvat/dextr_model_v1.zip -O ${DEXTR_MODEL_DIR}/dextr.zip && \ - unzip ${DEXTR_MODEL_DIR}/dextr.zip -d ${DEXTR_MODEL_DIR} && rm ${DEXTR_MODEL_DIR}/dextr.zip; \ + curl https://download.01.org/openvinotoolkit/models_contrib/cvat/dextr_model_v1.zip -o ${DEXTR_MODEL_DIR}/dextr.zip && \ + 7z e ${DEXTR_MODEL_DIR}/dextr.zip -o${DEXTR_MODEL_DIR} && rm ${DEXTR_MODEL_DIR}/dextr.zip; \ fi COPY ssh ${HOME}/.ssh diff --git a/Dockerfile.ci b/Dockerfile.ci new file mode 100644 index 00000000000..e3671196ee3 --- /dev/null +++ b/Dockerfile.ci @@ -0,0 +1,32 @@ +FROM cvat + +ENV DJANGO_CONFIGURATION=testing +USER root + +RUN curl https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - && \ + echo 'deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main' | tee /etc/apt/sources.list.d/google-chrome.list && \ + curl https://deb.nodesource.com/setup_9.x | bash - && \ + apt-get update && \ + DEBIAN_FRONTEND=noninteractive apt-get install -yq \ + google-chrome-stable \ + nodejs && \ + rm -rf /var/lib/apt/lists/*; + +RUN python3 -m pip install --no-cache-dir -r /tmp/requirements/${DJANGO_CONFIGURATION}.txt + +# RUN all commands below as 'django' user +USER ${USER} + +RUN mkdir -p tests && cd tests && npm install \ + eslint \ + eslint-detailed-reporter \ + karma \ + karma-chrome-launcher \ + karma-coveralls \ + karma-coverage \ + karma-junit-reporter \ + karma-qunit \ + qunit; \ + echo "export PATH=~/tests/node_modules/.bin:${PATH}" >> ~/.bashrc; + +ENTRYPOINT [] \ No newline at end of file diff --git a/components/auto_segmentation/install.sh b/components/auto_segmentation/install.sh index e336644fd13..d6881a68150 100755 --- a/components/auto_segmentation/install.sh +++ b/components/auto_segmentation/install.sh @@ -3,10 +3,11 @@ set -e +MASK_RCNN_URL=https://github.com/matterport/Mask_RCNN + cd ${HOME} && \ -git clone https://github.com/matterport/Mask_RCNN.git && \ -wget https://github.com/matterport/Mask_RCNN/releases/download/v2.0/mask_rcnn_coco.h5 && \ -mv mask_rcnn_coco.h5 Mask_RCNN/mask_rcnn_coco.h5 +git clone ${MASK_RCNN_URL}.git && \ +curl -L ${MASK_RCNN_URL}/releases/download/v2.0/mask_rcnn_coco.h5 -o Mask_RCNN/mask_rcnn_coco.h5 # TODO remove useless files # tensorflow and Keras are installed globally diff --git a/components/cuda/install.sh b/components/cuda/install.sh index 757120468a7..485ae989bde 100755 --- a/components/cuda/install.sh +++ b/components/cuda/install.sh @@ -30,8 +30,8 @@ apt-get update && apt-get install -y --no-install-recommends --allow-unauthentic libnccl2=$NCCL_VERSION-1+cuda9.0 \ libcudnn7=$CUDNN_VERSION-1+cuda9.0 && \ ln -s cuda-9.0 /usr/local/cuda && \ -rm -rf /var/lib/apt/lists/* \ + rm -rf /var/lib/apt/lists/* \ /etc/apt/sources.list.d/nvidia-ml.list /etc/apt/sources.list.d/cuda.list python3 -m pip uninstall -y tensorflow -python3 -m pip install --no-cache-dir tensorflow-gpu==1.15.0 +python3 -m pip install --no-cache-dir tensorflow-gpu==1.13.1 diff --git a/components/tf_annotation/install.sh b/components/tf_annotation/install.sh index fc5ed6b6309..8dc034832a2 100755 --- a/components/tf_annotation/install.sh +++ b/components/tf_annotation/install.sh @@ -7,7 +7,7 @@ set -e cd ${HOME} && \ -wget -O model.tar.gz http://download.tensorflow.org/models/object_detection/faster_rcnn_inception_resnet_v2_atrous_coco_2018_01_28.tar.gz && \ +curl http://download.tensorflow.org/models/object_detection/faster_rcnn_inception_resnet_v2_atrous_coco_2018_01_28.tar.gz -o model.tar.gz && \ tar -xzf model.tar.gz && rm model.tar.gz && \ mv faster_rcnn_inception_resnet_v2_atrous_coco_2018_01_28 ${HOME}/rcnn && cd ${HOME} && \ mv rcnn/frozen_inference_graph.pb rcnn/inference_graph.pb diff --git a/cvat/apps/documentation/installation.md b/cvat/apps/documentation/installation.md index b6d53014966..9931d743807 100644 --- a/cvat/apps/documentation/installation.md +++ b/cvat/apps/documentation/installation.md @@ -100,7 +100,7 @@ server. Proxy is an advanced topic and it is not covered by the guide. install it as well. Type commands below in a terminal window: ```sh - wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | sudo apt-key add - + curl https://dl-ssl.google.com/linux/linux_signing_key.pub | sudo apt-key add - sudo sh -c 'echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google-chrome.list' sudo apt-get update sudo apt-get install -y google-chrome-stable diff --git a/cvat/requirements/base.txt b/cvat/requirements/base.txt index 5eff2314d68..11d098ed992 100644 --- a/cvat/requirements/base.txt +++ b/cvat/requirements/base.txt @@ -39,10 +39,13 @@ django-rest-auth[with_social]==0.9.5 cython==0.29.13 matplotlib==3.0.3 scikit-image==0.15.0 -tensorflow==1.15.0 +tensorflow==1.13.1 keras==2.2.5 opencv-python==4.1.0.25 h5py==2.9.0 imgaug==0.2.9 django-cors-headers==3.2.0 furl==2.0.0 +# The package is used by pyunpack as a command line tool to support multiple +# archives. Don't use as a python module because it has GPL license. +patool==1.12 diff --git a/cvat/requirements/testing.txt b/cvat/requirements/testing.txt index b6468d78a44..f5f61eec145 100644 --- a/cvat/requirements/testing.txt +++ b/cvat/requirements/testing.txt @@ -1,2 +1,5 @@ -r development.txt -fakeredis==1.0.3 \ No newline at end of file +fakeredis==1.1.0 +# Fix dependencies for fakeredis 1.1.0 +# Pip will not reinstall six package if it is installed already +six==1.12.0 diff --git a/datumaro/datumaro/util/image.py b/datumaro/datumaro/util/image.py index ac0c1daf63d..46ef267c3d2 100644 --- a/datumaro/datumaro/util/image.py +++ b/datumaro/datumaro/util/image.py @@ -14,7 +14,7 @@ try: import cv2 _IMAGE_BACKEND = _IMAGE_BACKENDS.cv2 -except ModuleNotFoundError: +except ImportError: import PIL _IMAGE_BACKEND = _IMAGE_BACKENDS.PIL diff --git a/datumaro/requirements.txt b/datumaro/requirements.txt index a47f1149686..1fc9c3d52ee 100644 --- a/datumaro/requirements.txt +++ b/datumaro/requirements.txt @@ -2,10 +2,10 @@ Cython>=0.27.3 # include before pycocotools GitPython>=2.1.11 lxml>=4.4.1 matplotlib<3.1 # 3.1+ requires python3.6, but we have 3.5 in cvat -opencv-python>=4.1.0.25 +opencv-python-headless>=4.1.0.25 Pillow>=6.1.0 pycocotools>=2.0.0 PyYAML>=5.1.1 scikit-image>=0.15.0 tensorboardX>=1.8 -tensorflow>=1.12.0 +tensorflow==1.13.1 diff --git a/docker-compose.ci.yml b/docker-compose.ci.yml index 81bd47a6756..f67559124c1 100644 --- a/docker-compose.ci.yml +++ b/docker-compose.ci.yml @@ -1,12 +1,14 @@ version: "2.3" services: - cvat: + cvat_ci: + image: cvat_ci build: - args: - DJANGO_CONFIGURATION: "testing" - WITH_TESTS: "yes" network: host + context: . + dockerfile: Dockerfile.ci + depends_on: + - cvat environment: COVERALLS_REPO_TOKEN: TRAVIS: diff --git a/docker-compose.yml b/docker-compose.yml index 4a586475585..834f3560be2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -49,7 +49,6 @@ services: AUTO_SEGMENTATION: "no" USER: "django" DJANGO_CONFIGURATION: "production" - WITH_TESTS: "no" TZ: "Etc/UTC" OPENVINO_TOOLKIT: "no" environment: diff --git a/utils/tfrecords/requirements.txt b/utils/tfrecords/requirements.txt index 45e48c20153..616c04018e2 100644 --- a/utils/tfrecords/requirements.txt +++ b/utils/tfrecords/requirements.txt @@ -1,3 +1,3 @@ argparse==1.1 -tensorflow==1.15.0 +tensorflow==1.13.1 pathlib==1.0.1