Merge pull request #483 from EpistasisLab/docker_updates

Docker updates

docker/dbmongo/Dockerfile

@@ -1,23 +1,25 @@
-FROM ubuntu:xenial
+FROM mongo:4.4.18
 WORKDIR /opt/
 
 ARG docker_filepath=docker/dbmongo/files
 
+RUN apt-get update && apt-get install -y dos2unix
+
 # RUN apt-get update
 # RUN apt-get install gpg wget
 # RUN wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | gpg --dearmor - | sudo tee /usr/share/keyrings/kitware-archive-keyring.gpg >/dev/null
 # RUN apt-get update
 
 
 #add repo for mongodb
-RUN echo "deb http://repo.mongodb.org/apt/ubuntu xenial/mongodb-org/3.2 multiverse" | tee /etc/apt/sources.list.d/mongodb-org-3.2.list
-RUN apt-get update --fix-missing && \
-  apt-get install -y --allow-unauthenticated \
-  vim mongodb-org npm openssh-client htop dos2unix \
-  net-tools iputils-ping \
-  --no-install-recommends && \
-  apt-get clean && \
-  rm -rf /var/lib/apt/lists/*
+# RUN echo "deb http://repo.mongodb.org/apt/ubuntu xenial/mongodb-org/3.2 multiverse" | tee /etc/apt/sources.list.d/mongodb-org-3.2.list
+# RUN apt-get update --fix-missing && \
+#   apt-get install -y --allow-unauthenticated \
+#   vim mongodb-org npm openssh-client htop dos2unix \
+#   net-tools iputils-ping \
+#   --no-install-recommends && \
+#   apt-get clean && \
+#   rm -rf /var/lib/apt/lists/*
 
 #check if we need to rebuild
 COPY ${docker_filepath}/mongod.conf /etc/
@@ -28,6 +30,7 @@ COPY ${docker_filepath}/sync.sh /root/
 RUN dos2unix /root/entrypoint.sh /root/sync.sh
 RUN dos2unix /root/users.json /root/projects.json
 RUN dos2unix /etc/mongod.conf
+RUN apt-get remove -y dos2unix && apt-get clean
 
 EXPOSE 27017
 

docker/lab/Dockerfile

@@ -1,23 +1,51 @@
-FROM python:3.7.11-stretch
+# FROM python:3.7.11-stretch
+FROM python:3.7.16-slim-buster
 
 #nodejs
-RUN wget --quiet https://nodejs.org/dist/v12.22.9/node-v12.22.9-linux-x64.tar.xz -O ~/node.tar.xz && \
-    tar -xvf ~/node.tar.xz -C /opt/ && \
-    rm ~/node.tar.xz
-ENV PATH /opt/node-v12.22.9-linux-x64/bin:$PATH
-
-ARG docker_filepath=docker/lab/files
-
-RUN apt-get update --fix-missing && apt-get install -y \
-    vim openssh-client openssh-server telnet apache2 \
-    net-tools iputils-ping xz-utils \
-    screen ngrep ca-cacert \
+# RUN wget --quiet https://nodejs.org/dist/v12.22.9/node-v12.22.9-linux-x64.tar.xz -O ~/node.tar.xz && \
+#     tar -xvf ~/node.tar.xz -C /opt/ && \
+#     rm ~/node.tar.xz
+# ENV PATH /opt/node-v12.22.9-linux-x64/bin:$PATH
+
+RUN apt-get update --fix-missing && \
+    apt-get install -y wget xz-utils git \
+    openssh-client openssh-server telnet apache2 \
+    net-tools iputils-ping \
+    screen ngrep \
     mercurial subversion \
-    build-essential cmake lsb-core cpio mesa-common-dev \
+    build-essential cmake cpio mesa-common-dev \
     dos2unix \
     --no-install-recommends && \
-    apt-get clean && \
-    rm -rf /var/lib/apt/lists/*
+    ARCH= && dpkgArch="$(dpkg --print-architecture)" \
+    && case "${dpkgArch##*-}" in \
+      amd64) ARCH='x64';; \
+      ppc64el) ARCH='ppc64le';; \
+      s390x) ARCH='s390x';; \
+      arm64) ARCH='arm64';; \
+      armhf) ARCH='armv7l';; \
+      i386) ARCH='x86';; \
+      *) echo "unsupported architecture"; exit 1 ;; \
+    esac \
+    && wget --quiet https://nodejs.org/dist/v16.18.1/node-v16.18.1-linux-$ARCH.tar.xz -O ~/node.tar.xz && \
+    tar -xvf ~/node.tar.xz -C /usr/local/ --strip-components=1 --no-same-owner && \
+    rm ~/node.tar.xz && \
+    ln -s /usr/local/bin/node /usr/local/bin/nodejs && \
+    rm -rf /var/lib/apt/lists/* && \
+    apt-get remove -y wget && \
+    apt-get clean
+
+ARG docker_filepath=docker/lab/files
+
+# RUN apt-get update --fix-missing && apt-get install -y \
+#     vim openssh-client openssh-server telnet apache2 \
+#     net-tools iputils-ping xz-utils \
+#     screen ngrep ca-cacert \
+#     mercurial subversion \
+#     build-essential cmake lsb-core cpio mesa-common-dev \
+#     dos2unix \
+#     --no-install-recommends && \
+#     apt-get clean && \
+#     rm -rf /var/lib/apt/lists/*
 
 # setup python environment
 COPY ${docker_filepath}/requirements.txt /root/

docker/lab/files/requirements.txt

@@ -1,11 +1,11 @@
-scikit-learn==0.22.1
-tqdm==4.32.2
-pymongo==3.8.0
-pandas==1.0.3
+scikit-learn==0.23.2
+tqdm==4.64.1
+pymongo==4.3.3
+pandas==1.0.5
 numpy==1.19.5
-cython==0.29.2
-xgboost==0.82
-simplejson==3.16.0
-requests==2.22.0
-joblib==0.16.0
--e git+https://github.com/lacava/surprise.git@1.1.1.1#egg=scikit-surprise
+cython==0.29.32
+xgboost==1.6.2
+simplejson==3.17.2
+requests==2.25.1
+joblib==1.2.0
+-e git+https://github.com/lacava/surprise.git@1.1.1.1#egg=scikit-surprise

docker/machine/Dockerfile

@@ -1,23 +1,39 @@
-FROM python:3.7.11-stretch
+# FROM python:3.7.11-stretch
+FROM python:3.7.16-slim-buster
 
 #nodejs
-RUN wget --quiet https://nodejs.org/dist/v12.22.9/node-v12.22.9-linux-x64.tar.xz -O ~/node.tar.xz && \
-    tar -xvf ~/node.tar.xz -C /opt/ && \
-    rm ~/node.tar.xz
-ENV PATH /opt/node-v12.22.9-linux-x64/bin:$PATH
+# RUN wget --quiet https://nodejs.org/dist/v12.22.9/node-v12.22.9-linux-x64.tar.xz -O ~/node.tar.xz && \
+#     tar -xvf ~/node.tar.xz -C /opt/ && \
+#     rm ~/node.tar.xz
+# ENV PATH /opt/node-v12.22.9-linux-x64/bin:$PATH
 
 ARG docker_filepath=docker/machine/files
 
 RUN apt-get update --fix-missing && apt-get install -y \
-    vim openssh-client openssh-server graphviz \
+    wget openssh-client openssh-server graphviz \
     net-tools iputils-ping xz-utils \
-    screen ngrep ca-cacert \
+    screen ngrep \
     mercurial subversion \
-    build-essential cmake lsb-core cpio mesa-common-dev \
+    build-essential cmake cpio mesa-common-dev \
     libglib2.0-0 libxext6 libsm6 libxrender1 dos2unix \
     --no-install-recommends && \
-    apt-get clean && \
-    rm -rf /var/lib/apt/lists/*
+    ARCH= && dpkgArch="$(dpkg --print-architecture)" \
+    && case "${dpkgArch##*-}" in \
+      amd64) ARCH='x64';; \
+      ppc64el) ARCH='ppc64le';; \
+      s390x) ARCH='s390x';; \
+      arm64) ARCH='arm64';; \
+      armhf) ARCH='armv7l';; \
+      i386) ARCH='x86';; \
+      *) echo "unsupported architecture"; exit 1 ;; \
+    esac \
+    && wget --quiet https://nodejs.org/dist/v16.18.1/node-v16.18.1-linux-$ARCH.tar.xz -O ~/node.tar.xz && \
+    tar -xvf ~/node.tar.xz -C /usr/local/ --strip-components=1 --no-same-owner && \
+    rm ~/node.tar.xz && \
+    ln -s /usr/local/bin/node /usr/local/bin/nodejs && \
+    rm -rf /var/lib/apt/lists/* && \
+    apt-get remove -y wget && \
+    apt-get clean
 
 # setup python environment
 COPY ${docker_filepath}/requirements.txt /root/

docker/machine/files/requirements.txt

@@ -1,11 +1,11 @@
-scikit-learn==0.22.1
-tqdm==4.32.2
-pymongo==3.8.0
-pandas==1.0.3
+scikit-learn==0.23.2
+tqdm==4.64.1
+pymongo==4.3.3
+pandas==1.0.5
 numpy==1.19.5
-matplotlib==3.1.0
+matplotlib==3.5.3
 mlxtend==0.16.0
-pydot==1.4.1
-requests==2.22.0
-xgboost==0.90
-joblib==0.16.0
+pydot==1.4.2
+requests==2.25.1
+xgboost==1.6.2
+joblib==1.2.0

docs/guides/developerGuide.md

@@ -111,6 +111,8 @@ The test results in html format can be found in the directory `.\target\test-rep
 
 Note: If the npm packages have been updated, the unit tests docker image need to be rebuild with `docker-compose -f .\docker-compose-unit-test.yml build`
 
+Note: It is best to prune docker volumes with `docker volume prune -f` or `docker system prune -a -f --volumes` before running the unit and integration tests, otherwise there may be errors about not being able to upload duplicate datasets.
+
 ### Integration
 - Type: Docker, runs [Jest](https://jestjs.io/)
 - Usage: `docker-compose -f .\docker-compose-int-test.yml up --abort-on-container-exit --force-recreate`