Cleanup Dockerfiles for CVAT (cvat-ai#1060)

* Replaced wget by curl * Moved CI stuff into Dockerfile.ci * Use docker-compose to run commnands inside docker (need environment variables) * Added patool again (to support different archive formats) * Roll back tensorflow version: 1.15 -> 1.13.1 Fixed cvat-ai#982 Fixed cvat-ai#1017 * datumaro install tensorflow 2.x now. It breaks automatic annotation using TF. * Follow redirects in curl (auto_segmentation)
cleemesser · Mar 5, 2020 · b51c8e8 · b51c8e8
1 parent fde944b
commit b51c8e8
Show file tree

Hide file tree

Showing 15 changed files with 84 additions and 81 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -9,9 +9,9 @@ services:
  - docker
 
 before_script:
- - docker-compose -f docker-compose.yml -f docker-compose.ci.yml up --build -d
+ - docker-compose -f docker-compose.yml -f docker-compose.ci.yml build
 
 script:
- - docker exec -it cvat /bin/bash -c 'python3 manage.py test cvat/apps utils/cli'
- - docker exec -it cvat /bin/bash -c 'python3 manage.py test datumaro/'
- - docker exec -it cvat /bin/bash -c 'cd cvat-core && npm install && npm run test && npm run coveralls'
+ - docker-compose -f docker-compose.yml -f docker-compose.ci.yml run cvat_ci /bin/bash -c 'python3 manage.py test cvat/apps utils/cli'
+ - docker-compose -f docker-compose.yml -f docker-compose.ci.yml run cvat_ci /bin/bash -c 'python3 manage.py test datumaro/'
+ - docker-compose -f docker-compose.yml -f docker-compose.ci.yml run cvat_ci /bin/bash -c 'cd cvat-core && npm install && npm run test && npm run coveralls'
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -91,8 +91,8 @@ to changes in ``.env/bin/activate`` file are active.
 - Perform all steps in the automatic annotation section
 - Download ReID model and save it somewhere:
 ```sh
- wget https://download.01.org/openvinotoolkit/2018_R5/open_model_zoo/person-reidentification-retail-0079/FP32/person-reidentification-retail-0079.xml -O reid.xml
- wget https://download.01.org/openvinotoolkit/2018_R5/open_model_zoo/person-reidentification-retail-0079/FP32/person-reidentification-retail-0079.bin -O reid.bin
+ curl https://download.01.org/openvinotoolkit/2018_R5/open_model_zoo/person-reidentification-retail-0079/FP32/person-reidentification-retail-0079.xml -o reid.xml
+ curl https://download.01.org/openvinotoolkit/2018_R5/open_model_zoo/person-reidentification-retail-0079/FP32/person-reidentification-retail-0079.bin -o reid.bin
 ```
 - Add next line to ``.env/bin/activate``:
 ```sh
@@ -103,7 +103,7 @@ to changes in ``.env/bin/activate`` file are active.
 - Perform all steps in the automatic annotation section
 - Download Deep Extreme Cut model, unpack it, and save somewhere:
 ```sh
-wget https://download.01.org/openvinotoolkit/models_contrib/cvat/dextr_model_v1.zip -O dextr.zip
+curl https://download.01.org/openvinotoolkit/models_contrib/cvat/dextr_model_v1.zip -o dextr.zip
 unzip dextr.zip
 ```
 - Add next lines to ``.env/bin/activate``:
@@ -115,7 +115,7 @@ unzip dextr.zip
 ### Tensorflow RCNN
 - Download RCNN model, unpack it, and save it somewhere:
 ```sh
-wget -O model.tar.gz http://download.tensorflow.org/models/object_detection/faster_rcnn_inception_resnet_v2_atrous_coco_2018_01_28.tar.gz && \
+curl http://download.tensorflow.org/models/object_detection/faster_rcnn_inception_resnet_v2_atrous_coco_2018_01_28.tar.gz -o model.tar.gz && \
 tar -xzf model.tar.gz
 ```
 - Add next lines to ``.env/bin/activate``:
@@ -127,7 +127,7 @@ tar -xzf model.tar.gz
 ### Tensorflow Mask RCNN
 - Download Mask RCNN model, and save it somewhere:
 ```sh
-wget https://github.com/matterport/Mask_RCNN/releases/download/v2.0/mask_rcnn_coco.h5
+curl https://github.com/matterport/Mask_RCNN/releases/download/v2.0/mask_rcnn_coco.h5 -o mask_rcnn_coco.h5
 ```
 - Add next lines to ``.env/bin/activate``:
 ```sh

diff --git a/Dockerfile b/Dockerfile
@@ -22,9 +22,7 @@ ENV DJANGO_CONFIGURATION=${DJANGO_CONFIGURATION}
 # Install necessary apt packages
 RUN apt-get update && \
  apt-get install -yq \
- python-software-properties \
- software-properties-common \
- wget && \
+ software-properties-common && \
  add-apt-repository ppa:mc3man/xerus-media -y && \
  add-apt-repository ppa:mc3man/gstffmpeg-keep -y && \
  apt-get update && \
@@ -40,14 +38,19 @@ RUN apt-get update && \
  python3-dev \
  python3-pip \
  tzdata \
- unzip \
- unrar \
  p7zip-full \
- git-core \
- libsm6 \
- libxext6 && \
- python3 -m pip install -U pip && \
- python3 -m pip install -U setuptools && \
+ git \
+ ssh \
+ poppler-utils \
+ curl && \
+ curl https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | bash && \
+ apt-get install -y git-lfs && git lfs install && \
+ if [ -z ${socks_proxy} ]; then \
+ echo export "GIT_SSH_COMMAND=\"ssh -o StrictHostKeyChecking=no -o ConnectTimeout=30\"" >> ${HOME}/.bashrc; \
+ else \
+ echo export "GIT_SSH_COMMAND=\"ssh -o StrictHostKeyChecking=no -o ConnectTimeout=30 -o ProxyCommand='nc -X 5 -x ${socks_proxy} %h %p'\"" >> ${HOME}/.bashrc; \
+ fi && \
+ python3 -m pip install --no-cache-dir -U pip setuptools && \
  ln -fs /usr/share/zoneinfo/${TZ} /etc/localtime && \
  dpkg-reconfigure -f noninteractive tzdata && \
  add-apt-repository --remove ppa:mc3man/gstffmpeg-keep -y && \
@@ -69,8 +72,8 @@ ENV REID_MODEL_DIR=${HOME}/reid
 RUN if [ "$OPENVINO_TOOLKIT" = "yes" ]; then \
  /tmp/components/openvino/install.sh && \
  mkdir ${REID_MODEL_DIR} && \
- wget https://download.01.org/openvinotoolkit/2018_R5/open_model_zoo/person-reidentification-retail-0079/FP32/person-reidentification-retail-0079.xml -O reid/reid.xml && \
- wget https://download.01.org/openvinotoolkit/2018_R5/open_model_zoo/person-reidentification-retail-0079/FP32/person-reidentification-retail-0079.bin -O reid/reid.bin; \
+ curl https://download.01.org/openvinotoolkit/2018_R5/open_model_zoo/person-reidentification-retail-0079/FP32/person-reidentification-retail-0079.xml -o reid/reid.xml && \
+ curl https://download.01.org/openvinotoolkit/2018_R5/open_model_zoo/person-reidentification-retail-0079/FP32/person-reidentification-retail-0079.bin -o reid/reid.bin; \
  fi
 
 # Tensorflow annotation support
@@ -89,53 +92,13 @@ RUN if [ "$AUTO_SEGMENTATION" = "yes" ]; then \
  bash -i /tmp/components/auto_segmentation/install.sh; \
  fi
 
-ARG WITH_TESTS
-RUN if [ "$WITH_TESTS" = "yes" ]; then \
- wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - && \
- echo 'deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main' | tee /etc/apt/sources.list.d/google-chrome.list && \
- wget -qO- https://deb.nodesource.com/setup_9.x | bash - && \
- apt-get update && \
- DEBIAN_FRONTEND=noninteractive apt-get install -yq \
- google-chrome-stable \
- nodejs && \
- rm -rf /var/lib/apt/lists/*; \
- mkdir tests && cd tests && npm install \
- eslint \
- eslint-detailed-reporter \
- karma \
- karma-chrome-launcher \
- karma-coveralls \
- karma-coverage \
- karma-junit-reporter \
- karma-qunit \
- qunit; \
- echo "export PATH=~/tests/node_modules/.bin:${PATH}" >> ~/.bashrc; \
- fi
-
 # Install and initialize CVAT, copy all necessary files
 COPY cvat/requirements/ /tmp/requirements/
 COPY supervisord.conf mod_wsgi.conf wait-for-it.sh manage.py ${HOME}/
 RUN python3 -m pip install --no-cache-dir -r /tmp/requirements/${DJANGO_CONFIGURATION}.txt
 # pycocotools package is impossible to install with its dependencies by one pip install command
 RUN python3 -m pip install --no-cache-dir pycocotools==2.0.0
 
-# Install git application dependencies
-RUN apt-get update && \
- apt-get install -y ssh netcat-openbsd git curl zip && \
- wget -qO /dev/stdout https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | bash && \
- apt-get install -y git-lfs && \
- git lfs install && \
- rm -rf /var/lib/apt/lists/* && \
- if [ -z ${socks_proxy} ]; then \
- echo export "GIT_SSH_COMMAND=\"ssh -o StrictHostKeyChecking=no -o ConnectTimeout=30\"" >> ${HOME}/.bashrc; \
- else \
- echo export "GIT_SSH_COMMAND=\"ssh -o StrictHostKeyChecking=no -o ConnectTimeout=30 -o ProxyCommand='nc -X 5 -x ${socks_proxy} %h %p'\"" >> ${HOME}/.bashrc; \
- fi
-
-# Install poppler for working with pdfs
-RUN apt-get update && \
- apt install -y poppler-utils && \
- rm -rf /var/lib/apt/lists/*
 
 # CUDA support
 ARG CUDA_SUPPORT
@@ -150,8 +113,8 @@ ENV WITH_DEXTR=${WITH_DEXTR}
 ENV DEXTR_MODEL_DIR=${HOME}/dextr
 RUN if [ "$WITH_DEXTR" = "yes" ]; then \
  mkdir ${DEXTR_MODEL_DIR} -p && \
- wget https://download.01.org/openvinotoolkit/models_contrib/cvat/dextr_model_v1.zip -O ${DEXTR_MODEL_DIR}/dextr.zip && \
- unzip ${DEXTR_MODEL_DIR}/dextr.zip -d ${DEXTR_MODEL_DIR} && rm ${DEXTR_MODEL_DIR}/dextr.zip; \
+ curl https://download.01.org/openvinotoolkit/models_contrib/cvat/dextr_model_v1.zip -o ${DEXTR_MODEL_DIR}/dextr.zip && \
+ 7z e ${DEXTR_MODEL_DIR}/dextr.zip -o${DEXTR_MODEL_DIR} && rm ${DEXTR_MODEL_DIR}/dextr.zip; \
  fi
 
 COPY ssh ${HOME}/.ssh

diff --git a/Dockerfile.ci b/Dockerfile.ci
@@ -0,0 +1,32 @@
+FROM cvat
+
+ENV DJANGO_CONFIGURATION=testing
+USER root
+
+RUN curl https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - && \
+ echo 'deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main' | tee /etc/apt/sources.list.d/google-chrome.list && \
+ curl https://deb.nodesource.com/setup_9.x | bash - && \
+ apt-get update && \
+ DEBIAN_FRONTEND=noninteractive apt-get install -yq \
+ google-chrome-stable \
+ nodejs && \
+ rm -rf /var/lib/apt/lists/*;
+
+RUN python3 -m pip install --no-cache-dir -r /tmp/requirements/${DJANGO_CONFIGURATION}.txt
+
+# RUN all commands below as 'django' user
+USER ${USER}
+
+RUN mkdir -p tests && cd tests && npm install \
+ eslint \
+ eslint-detailed-reporter \
+ karma \
+ karma-chrome-launcher \
+ karma-coveralls \
+ karma-coverage \
+ karma-junit-reporter \
+ karma-qunit \
+ qunit; \
+ echo "export PATH=~/tests/node_modules/.bin:${PATH}" >> ~/.bashrc;
+
+ENTRYPOINT []
diff --git a/components/auto_segmentation/install.sh b/components/auto_segmentation/install.sh
@@ -3,10 +3,11 @@
 
 set -e
 
+MASK_RCNN_URL=https://github.com/matterport/Mask_RCNN
+
 cd ${HOME} && \
-git clone https://github.com/matterport/Mask_RCNN.git && \
-wget https://github.com/matterport/Mask_RCNN/releases/download/v2.0/mask_rcnn_coco.h5 && \
-mv mask_rcnn_coco.h5 Mask_RCNN/mask_rcnn_coco.h5
+git clone ${MASK_RCNN_URL}.git && \
+curl -L ${MASK_RCNN_URL}/releases/download/v2.0/mask_rcnn_coco.h5 -o Mask_RCNN/mask_rcnn_coco.h5
 
 # TODO remove useless files
 # tensorflow and Keras are installed globally
diff --git a/components/cuda/install.sh b/components/cuda/install.sh
@@ -30,8 +30,8 @@ apt-get update && apt-get install -y --no-install-recommends --allow-unauthentic
  libnccl2=$NCCL_VERSION-1+cuda9.0 \
  libcudnn7=$CUDNN_VERSION-1+cuda9.0 && \
  ln -s cuda-9.0 /usr/local/cuda && \
-rm -rf /var/lib/apt/lists/* \
+ rm -rf /var/lib/apt/lists/* \
  /etc/apt/sources.list.d/nvidia-ml.list /etc/apt/sources.list.d/cuda.list
 
 python3 -m pip uninstall -y tensorflow
-python3 -m pip install --no-cache-dir tensorflow-gpu==1.15.0
+python3 -m pip install --no-cache-dir tensorflow-gpu==1.13.1
diff --git a/components/tf_annotation/install.sh b/components/tf_annotation/install.sh
@@ -7,7 +7,7 @@
 set -e
 
 cd ${HOME} && \
-wget -O model.tar.gz http://download.tensorflow.org/models/object_detection/faster_rcnn_inception_resnet_v2_atrous_coco_2018_01_28.tar.gz && \
+curl http://download.tensorflow.org/models/object_detection/faster_rcnn_inception_resnet_v2_atrous_coco_2018_01_28.tar.gz -o model.tar.gz && \
 tar -xzf model.tar.gz && rm model.tar.gz && \
 mv faster_rcnn_inception_resnet_v2_atrous_coco_2018_01_28 ${HOME}/rcnn && cd ${HOME} && \
 mv rcnn/frozen_inference_graph.pb rcnn/inference_graph.pb

diff --git a/cvat/apps/documentation/installation.md b/cvat/apps/documentation/installation.md
@@ -100,7 +100,7 @@ server. Proxy is an advanced topic and it is not covered by the guide.
  install it as well. Type commands below in a terminal window:
 
  ```sh
- wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | sudo apt-key add -
+ curl https://dl-ssl.google.com/linux/linux_signing_key.pub | sudo apt-key add -
  sudo sh -c 'echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google-chrome.list'
  sudo apt-get update
  sudo apt-get install -y google-chrome-stable

diff --git a/cvat/requirements/base.txt b/cvat/requirements/base.txt
@@ -39,10 +39,13 @@ django-rest-auth[with_social]==0.9.5
 cython==0.29.13
 matplotlib==3.0.3
 scikit-image==0.15.0
-tensorflow==1.15.0
+tensorflow==1.13.1
 keras==2.2.5
 opencv-python==4.1.0.25
 h5py==2.9.0
 imgaug==0.2.9
 django-cors-headers==3.2.0
 furl==2.0.0
+# The package is used by pyunpack as a command line tool to support multiple
+# archives. Don't use as a python module because it has GPL license.
+patool==1.12
diff --git a/cvat/requirements/testing.txt b/cvat/requirements/testing.txt
@@ -1,2 +1,5 @@
 -r development.txt
-fakeredis==1.0.3
+fakeredis==1.1.0
+# Fix dependencies for fakeredis 1.1.0
+# Pip will not reinstall six package if it is installed already
+six==1.12.0
diff --git a/datumaro/datumaro/util/image.py b/datumaro/datumaro/util/image.py
@@ -14,7 +14,7 @@
 try:
  import cv2
  _IMAGE_BACKEND = _IMAGE_BACKENDS.cv2
-except ModuleNotFoundError:
+except ImportError:
  import PIL
  _IMAGE_BACKEND = _IMAGE_BACKENDS.PIL
 

diff --git a/datumaro/requirements.txt b/datumaro/requirements.txt
@@ -2,10 +2,10 @@ Cython>=0.27.3 # include before pycocotools
 GitPython>=2.1.11
 lxml>=4.4.1
 matplotlib<3.1 # 3.1+ requires python3.6, but we have 3.5 in cvat
-opencv-python>=4.1.0.25
+opencv-python-headless>=4.1.0.25
 Pillow>=6.1.0
 pycocotools>=2.0.0
 PyYAML>=5.1.1
 scikit-image>=0.15.0
 tensorboardX>=1.8
-tensorflow>=1.12.0
+tensorflow==1.13.1
diff --git a/docker-compose.ci.yml b/docker-compose.ci.yml
@@ -1,12 +1,14 @@
 version: "2.3"
 
 services:
- cvat:
+ cvat_ci:
+ image: cvat_ci
  build:
- args:
- DJANGO_CONFIGURATION: "testing"
- WITH_TESTS: "yes"
  network: host
+ context: .
+ dockerfile: Dockerfile.ci
+ depends_on:
+ - cvat
  environment:
  COVERALLS_REPO_TOKEN:
  TRAVIS:

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -49,7 +49,6 @@ services:
  AUTO_SEGMENTATION: "no"
  USER: "django"
  DJANGO_CONFIGURATION: "production"
- WITH_TESTS: "no"
  TZ: "Etc/UTC"
  OPENVINO_TOOLKIT: "no"
  environment:

diff --git a/utils/tfrecords/requirements.txt b/utils/tfrecords/requirements.txt
@@ -1,3 +1,3 @@
 argparse==1.1
-tensorflow==1.15.0
+tensorflow==1.13.1
 pathlib==1.0.1