Cleanup Dockerfiles for CVAT (#1060)

* Replaced wget by curl

* Moved CI stuff into Dockerfile.ci

* Use docker-compose to run commnands inside docker (need environment variables)

* Added patool again (to support different archive formats)

* Roll back tensorflow version: 1.15 -> 1.13.1

Fixed #982
Fixed #1017

* datumaro install tensorflow 2.x now. It breaks automatic annotation
using TF.

* Follow redirects in curl (auto_segmentation)

Author: nmanovic

.travis.yml

@@ -9,9 +9,9 @@ services:
   - docker
 
 before_script:
-  - docker-compose -f docker-compose.yml -f docker-compose.ci.yml up --build -d
+  - docker-compose -f docker-compose.yml -f docker-compose.ci.yml build
 
 script:
-  - docker exec -it cvat /bin/bash -c 'python3 manage.py test cvat/apps utils/cli'
-  - docker exec -it cvat /bin/bash -c 'python3 manage.py test datumaro/'
-  - docker exec -it cvat /bin/bash -c 'cd cvat-core && npm install && npm run test && npm run coveralls'
+  - docker-compose -f docker-compose.yml -f docker-compose.ci.yml run cvat_ci /bin/bash -c 'python3 manage.py test cvat/apps utils/cli'
+  - docker-compose -f docker-compose.yml -f docker-compose.ci.yml run cvat_ci /bin/bash -c 'python3 manage.py test datumaro/'
+  - docker-compose -f docker-compose.yml -f docker-compose.ci.yml run cvat_ci /bin/bash -c 'cd cvat-core && npm install && npm run test && npm run coveralls'

CONTRIBUTING.md

@@ -91,8 +91,8 @@ to changes in ``.env/bin/activate`` file are active.
 - Perform all steps in the automatic annotation section
 - Download ReID model and save it somewhere:
 ```sh
-    wget https://download.01.org/openvinotoolkit/2018_R5/open_model_zoo/person-reidentification-retail-0079/FP32/person-reidentification-retail-0079.xml -O reid.xml
-    wget https://download.01.org/openvinotoolkit/2018_R5/open_model_zoo/person-reidentification-retail-0079/FP32/person-reidentification-retail-0079.bin -O reid.bin
+    curl https://download.01.org/openvinotoolkit/2018_R5/open_model_zoo/person-reidentification-retail-0079/FP32/person-reidentification-retail-0079.xml -o reid.xml
+    curl https://download.01.org/openvinotoolkit/2018_R5/open_model_zoo/person-reidentification-retail-0079/FP32/person-reidentification-retail-0079.bin -o reid.bin
 ```
 - Add next line to ``.env/bin/activate``:
 ```sh
@@ -103,7 +103,7 @@ to changes in ``.env/bin/activate`` file are active.
 - Perform all steps in the automatic annotation section
 - Download Deep Extreme Cut model, unpack it, and save somewhere:
 ```sh
-wget https://download.01.org/openvinotoolkit/models_contrib/cvat/dextr_model_v1.zip -O dextr.zip
+curl https://download.01.org/openvinotoolkit/models_contrib/cvat/dextr_model_v1.zip -o dextr.zip
 unzip dextr.zip
 ```
 - Add next lines to ``.env/bin/activate``:
@@ -115,7 +115,7 @@ unzip dextr.zip
 ### Tensorflow RCNN
 - Download RCNN model, unpack it, and save it somewhere:
 ```sh
-wget -O model.tar.gz http://download.tensorflow.org/models/object_detection/faster_rcnn_inception_resnet_v2_atrous_coco_2018_01_28.tar.gz && \
+curl http://download.tensorflow.org/models/object_detection/faster_rcnn_inception_resnet_v2_atrous_coco_2018_01_28.tar.gz -o model.tar.gz && \
 tar -xzf model.tar.gz
 ```
 - Add next lines to ``.env/bin/activate``:
@@ -127,7 +127,7 @@ tar -xzf model.tar.gz
 ### Tensorflow Mask RCNN
 - Download Mask RCNN model, and save it somewhere:
 ```sh
-wget https://github.com/matterport/Mask_RCNN/releases/download/v2.0/mask_rcnn_coco.h5
+curl https://github.com/matterport/Mask_RCNN/releases/download/v2.0/mask_rcnn_coco.h5 -o mask_rcnn_coco.h5
 ```
 - Add next lines to ``.env/bin/activate``:
 ```sh

Dockerfile

@@ -22,9 +22,7 @@ ENV DJANGO_CONFIGURATION=${DJANGO_CONFIGURATION}
 # Install necessary apt packages
 RUN apt-get update && \
     apt-get install -yq \
-        python-software-properties \
-        software-properties-common \
-        wget && \
+        software-properties-common && \
     add-apt-repository ppa:mc3man/xerus-media -y && \
     add-apt-repository ppa:mc3man/gstffmpeg-keep -y && \
     apt-get update && \
@@ -40,14 +38,19 @@ RUN apt-get update && \
         python3-dev \
         python3-pip \
         tzdata \
-        unzip \
-        unrar \
         p7zip-full \
-        git-core \
-        libsm6 \
-        libxext6 && \
-    python3 -m pip install -U pip && \
-    python3 -m pip install -U setuptools && \
+        git \
+        ssh \
+        poppler-utils \
+        curl && \
+    curl https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | bash && \
+    apt-get install -y git-lfs && git lfs install && \
+    if [ -z ${socks_proxy} ]; then \
+        echo export "GIT_SSH_COMMAND=\"ssh -o StrictHostKeyChecking=no -o ConnectTimeout=30\"" >> ${HOME}/.bashrc; \
+    else \
+        echo export "GIT_SSH_COMMAND=\"ssh -o StrictHostKeyChecking=no -o ConnectTimeout=30 -o ProxyCommand='nc -X 5 -x ${socks_proxy} %h %p'\"" >> ${HOME}/.bashrc; \
+    fi && \
+    python3 -m pip install --no-cache-dir -U pip setuptools && \
     ln -fs /usr/share/zoneinfo/${TZ} /etc/localtime && \
     dpkg-reconfigure -f noninteractive tzdata && \
     add-apt-repository --remove ppa:mc3man/gstffmpeg-keep -y && \
@@ -69,8 +72,8 @@ ENV REID_MODEL_DIR=${HOME}/reid
 RUN if [ "$OPENVINO_TOOLKIT" = "yes" ]; then \
         /tmp/components/openvino/install.sh && \
         mkdir ${REID_MODEL_DIR} && \
-        wget https://download.01.org/openvinotoolkit/2018_R5/open_model_zoo/person-reidentification-retail-0079/FP32/person-reidentification-retail-0079.xml -O reid/reid.xml && \
-        wget https://download.01.org/openvinotoolkit/2018_R5/open_model_zoo/person-reidentification-retail-0079/FP32/person-reidentification-retail-0079.bin -O reid/reid.bin; \
+        curl https://download.01.org/openvinotoolkit/2018_R5/open_model_zoo/person-reidentification-retail-0079/FP32/person-reidentification-retail-0079.xml -o reid/reid.xml && \
+        curl https://download.01.org/openvinotoolkit/2018_R5/open_model_zoo/person-reidentification-retail-0079/FP32/person-reidentification-retail-0079.bin -o reid/reid.bin; \
     fi
 
 # Tensorflow annotation support
@@ -89,53 +92,13 @@ RUN if [ "$AUTO_SEGMENTATION" = "yes" ]; then \
     bash -i /tmp/components/auto_segmentation/install.sh; \
     fi
 
-ARG WITH_TESTS
-RUN if [ "$WITH_TESTS" = "yes" ]; then \
-        wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - && \
-        echo 'deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main' | tee /etc/apt/sources.list.d/google-chrome.list && \
-        wget -qO- https://deb.nodesource.com/setup_9.x | bash - && \
-        apt-get update && \
-        DEBIAN_FRONTEND=noninteractive apt-get install -yq \
-            google-chrome-stable \
-            nodejs && \
-        rm -rf /var/lib/apt/lists/*; \
-        mkdir tests && cd tests && npm install \
-            eslint \
-            eslint-detailed-reporter \
-            karma \
-            karma-chrome-launcher \
-            karma-coveralls \
-            karma-coverage \
-            karma-junit-reporter \
-            karma-qunit \
-            qunit; \
-        echo "export PATH=~/tests/node_modules/.bin:${PATH}" >> ~/.bashrc; \
-    fi
-
 # Install and initialize CVAT, copy all necessary files
 COPY cvat/requirements/ /tmp/requirements/
 COPY supervisord.conf mod_wsgi.conf wait-for-it.sh manage.py ${HOME}/
 RUN python3 -m pip install --no-cache-dir -r /tmp/requirements/${DJANGO_CONFIGURATION}.txt
 # pycocotools package is impossible to install with its dependencies by one pip install command
 RUN python3 -m pip install --no-cache-dir pycocotools==2.0.0
 
-# Install git application dependencies
-RUN apt-get update && \
-    apt-get install -y ssh netcat-openbsd git curl zip  && \
-    wget -qO /dev/stdout https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | bash && \
-    apt-get install -y git-lfs && \
-    git lfs install && \
-    rm -rf /var/lib/apt/lists/* && \
-    if [ -z ${socks_proxy} ]; then \
-        echo export "GIT_SSH_COMMAND=\"ssh -o StrictHostKeyChecking=no -o ConnectTimeout=30\"" >> ${HOME}/.bashrc; \
-    else \
-        echo export "GIT_SSH_COMMAND=\"ssh -o StrictHostKeyChecking=no -o ConnectTimeout=30 -o ProxyCommand='nc -X 5 -x ${socks_proxy} %h %p'\"" >> ${HOME}/.bashrc; \
-    fi
-
-# Install poppler for working with pdfs
-RUN apt-get update && \
-    apt install -y poppler-utils && \
-    rm -rf /var/lib/apt/lists/*
 
 # CUDA support
 ARG CUDA_SUPPORT
@@ -150,8 +113,8 @@ ENV WITH_DEXTR=${WITH_DEXTR}
 ENV DEXTR_MODEL_DIR=${HOME}/dextr
 RUN if [ "$WITH_DEXTR" = "yes" ]; then \
         mkdir ${DEXTR_MODEL_DIR} -p && \
-        wget https://download.01.org/openvinotoolkit/models_contrib/cvat/dextr_model_v1.zip -O ${DEXTR_MODEL_DIR}/dextr.zip && \
-        unzip ${DEXTR_MODEL_DIR}/dextr.zip -d ${DEXTR_MODEL_DIR} && rm ${DEXTR_MODEL_DIR}/dextr.zip; \
+        curl https://download.01.org/openvinotoolkit/models_contrib/cvat/dextr_model_v1.zip -o ${DEXTR_MODEL_DIR}/dextr.zip && \
+        7z e ${DEXTR_MODEL_DIR}/dextr.zip -o${DEXTR_MODEL_DIR} && rm ${DEXTR_MODEL_DIR}/dextr.zip; \
     fi
 
 COPY ssh ${HOME}/.ssh

Dockerfile.ci

@@ -0,0 +1,32 @@
+FROM cvat
+
+ENV DJANGO_CONFIGURATION=testing
+USER root
+
+RUN curl https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - && \
+    echo 'deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main' | tee /etc/apt/sources.list.d/google-chrome.list && \
+    curl https://deb.nodesource.com/setup_9.x | bash - && \
+    apt-get update && \
+    DEBIAN_FRONTEND=noninteractive apt-get install -yq \
+        google-chrome-stable \
+        nodejs && \
+    rm -rf /var/lib/apt/lists/*;
+
+RUN python3 -m pip install --no-cache-dir -r /tmp/requirements/${DJANGO_CONFIGURATION}.txt
+
+# RUN all commands below as 'django' user
+USER ${USER}
+
+RUN mkdir -p tests && cd tests && npm install \
+        eslint \
+        eslint-detailed-reporter \
+        karma \
+        karma-chrome-launcher \
+        karma-coveralls \
+        karma-coverage \
+        karma-junit-reporter \
+        karma-qunit \
+        qunit; \
+    echo "export PATH=~/tests/node_modules/.bin:${PATH}" >> ~/.bashrc;
+
+ENTRYPOINT []

components/auto_segmentation/install.sh

@@ -3,10 +3,11 @@
 
 set -e
 
+MASK_RCNN_URL=https://github.com/matterport/Mask_RCNN
+
 cd ${HOME} && \
-git clone https://github.com/matterport/Mask_RCNN.git && \
-wget https://github.com/matterport/Mask_RCNN/releases/download/v2.0/mask_rcnn_coco.h5 && \
-mv mask_rcnn_coco.h5 Mask_RCNN/mask_rcnn_coco.h5
+git clone ${MASK_RCNN_URL}.git && \
+curl -L ${MASK_RCNN_URL}/releases/download/v2.0/mask_rcnn_coco.h5 -o Mask_RCNN/mask_rcnn_coco.h5
 
 # TODO remove useless files
 # tensorflow and Keras are installed globally

components/cuda/install.sh

@@ -30,8 +30,8 @@ apt-get update && apt-get install -y --no-install-recommends --allow-unauthentic
     libnccl2=$NCCL_VERSION-1+cuda9.0 \
     libcudnn7=$CUDNN_VERSION-1+cuda9.0 && \
     ln -s cuda-9.0 /usr/local/cuda && \
-rm -rf /var/lib/apt/lists/* \
+    rm -rf /var/lib/apt/lists/* \
     /etc/apt/sources.list.d/nvidia-ml.list /etc/apt/sources.list.d/cuda.list
 
 python3 -m pip uninstall -y tensorflow
-python3 -m pip install --no-cache-dir tensorflow-gpu==1.15.0
+python3 -m pip install --no-cache-dir tensorflow-gpu==1.13.1

components/tf_annotation/install.sh

@@ -7,7 +7,7 @@
 set -e
 
 cd ${HOME} && \
-wget -O model.tar.gz http://download.tensorflow.org/models/object_detection/faster_rcnn_inception_resnet_v2_atrous_coco_2018_01_28.tar.gz && \
+curl http://download.tensorflow.org/models/object_detection/faster_rcnn_inception_resnet_v2_atrous_coco_2018_01_28.tar.gz -o model.tar.gz && \
 tar -xzf model.tar.gz && rm model.tar.gz && \
 mv faster_rcnn_inception_resnet_v2_atrous_coco_2018_01_28 ${HOME}/rcnn && cd ${HOME} && \
 mv rcnn/frozen_inference_graph.pb rcnn/inference_graph.pb

cvat/apps/documentation/installation.md

@@ -100,7 +100,7 @@ server. Proxy is an advanced topic and it is not covered by the guide.
     install it as well. Type commands below in a terminal window:
 
     ```sh
-    wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | sudo apt-key add -
+    curl https://dl-ssl.google.com/linux/linux_signing_key.pub | sudo apt-key add -
     sudo sh -c 'echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google-chrome.list'
     sudo apt-get update
     sudo apt-get install -y google-chrome-stable

cvat/requirements/base.txt

@@ -39,10 +39,13 @@ django-rest-auth[with_social]==0.9.5
 cython==0.29.13
 matplotlib==3.0.3
 scikit-image==0.15.0
-tensorflow==1.15.0
+tensorflow==1.13.1
 keras==2.2.5
 opencv-python==4.1.0.25
 h5py==2.9.0
 imgaug==0.2.9
 django-cors-headers==3.2.0
 furl==2.0.0
+# The package is used by pyunpack as a command line tool to support multiple
+# archives. Don't use as a python module because it has GPL license.
+patool==1.12

cvat/requirements/testing.txt

@@ -1,2 +1,5 @@
 -r development.txt
-fakeredis==1.0.3
+fakeredis==1.1.0
+# Fix dependencies for fakeredis 1.1.0
+# Pip will not reinstall six package if it is installed already
+six==1.12.0

datumaro/datumaro/util/image.py

@@ -14,7 +14,7 @@
 try:
     import cv2
     _IMAGE_BACKEND = _IMAGE_BACKENDS.cv2
-except ModuleNotFoundError:
+except ImportError:
     import PIL
     _IMAGE_BACKEND = _IMAGE_BACKENDS.PIL
 

datumaro/requirements.txt

@@ -2,10 +2,10 @@ Cython>=0.27.3 # include before pycocotools
 GitPython>=2.1.11
 lxml>=4.4.1
 matplotlib<3.1 # 3.1+ requires python3.6, but we have 3.5 in cvat
-opencv-python>=4.1.0.25
+opencv-python-headless>=4.1.0.25
 Pillow>=6.1.0
 pycocotools>=2.0.0
 PyYAML>=5.1.1
 scikit-image>=0.15.0
 tensorboardX>=1.8
-tensorflow>=1.12.0
+tensorflow==1.13.1

docker-compose.ci.yml

@@ -1,12 +1,14 @@
 version: "2.3"
 
 services:
-  cvat:
+  cvat_ci:
+    image: cvat_ci
     build:
-      args:
-        DJANGO_CONFIGURATION: "testing"
-        WITH_TESTS: "yes"
       network: host
+      context: .
+      dockerfile: Dockerfile.ci
+    depends_on:
+      - cvat
     environment:
       COVERALLS_REPO_TOKEN:
       TRAVIS:

docker-compose.yml

@@ -49,7 +49,6 @@ services:
         AUTO_SEGMENTATION: "no"
         USER: "django"
         DJANGO_CONFIGURATION: "production"
-        WITH_TESTS: "no"
         TZ: "Etc/UTC"
         OPENVINO_TOOLKIT: "no"
     environment:

utils/tfrecords/requirements.txt

@@ -1,3 +1,3 @@
 argparse==1.1
-tensorflow==1.15.0
+tensorflow==1.13.1
 pathlib==1.0.1