Docker build script robustness

docker/Dockerfile.ci_gpu

@@ -24,7 +24,7 @@ COPY install/ubuntu_install_sphinx.sh /install/ubuntu_install_sphinx.sh
 RUN bash /install/ubuntu_install_sphinx.sh
 
 # Fix recommonmark to latest version
-RUN git clone https://github.com/rtfd/recommonmark
+RUN git clone --depth=1 https://github.com/rtfd/recommonmark
 RUN cd recommonmark; python3 setup.py install
 
 # Enable doxygen for c++ doc build

docker/Dockerfile.demo_opencl

@@ -45,7 +45,7 @@ RUN echo "Cloning TVM source & submodules"
 ENV TVM_PAR_DIR="/usr"
 RUN mkdir -p TVM_PAR_DIR && \
 	cd ${TVM_PAR_DIR} && \
-	git clone https://github.com/dmlc/tvm --recursive
+	git clone --depth=1 https://github.com/dmlc/tvm --recursive
 #RUN git submodule update --init --recursive
 
 

docker/install/install_tvm_cpu.sh

@@ -1,5 +1,11 @@
+#!/bin/bash
+
+set -e
+set -u
+set -o pipefail
+
 cd /usr
-git clone https://github.com/dmlc/tvm --recursive
+git clone --depth=1 https://github.com/dmlc/tvm --recursive
 cd /usr/tvm
 echo set\(USE_LLVM llvm-config-6.0\) >> config.cmake
 echo set\(USE_RPC ON\) >> config.cmake

docker/install/install_tvm_gpu.sh

@@ -1,5 +1,11 @@
+#!/bin/bash
+
+set -e
+set -u
+set -o pipefail
+
 cd /usr
-git clone https://github.com/dmlc/tvm --recursive
+git clone --depth=1 https://github.com/dmlc/tvm --recursive
 cd /usr/tvm
 echo set\(USE_LLVM llvm-config-6.0\) >> config.cmake
 echo set\(USE_CUDA ON\) >> config.cmake

docker/install/ubuntu_install_androidsdk.sh

@@ -1,6 +1,9 @@
+#!/bin/bash
+
 . /etc/profile
 
 set -o errexit -o nounset
+set -o pipefail
 
 ANDROID_HOME=/opt/android-sdk-linux
 ASDKTOOLS_HOME=/opt/android-sdk-tools
@@ -58,7 +61,7 @@ EOF
 
 mkdir /root/.android 2>/dev/null || true
 touch /root/.android/repositories.cfg
-yes | sdkmanager --licenses --sdk_root="$ANDROID_HOME"
+(yes || true) | sdkmanager --licenses --sdk_root="$ANDROID_HOME"
 sdkmanager --verbose --package_file=/install/package-list-minimal.txt --sdk_root="$ANDROID_HOME"
 test -d "${ANDROID_HOME}/build-tools/27.0.3"
 test -d "${ANDROID_HOME}/ndk-bundle"

docker/install/ubuntu_install_antlr.sh

@@ -1,3 +1,9 @@
+#!/bin/bash
+
+set -e
+set -u
+set -o pipefail
+
 cd /usr/local/lib
 wget -q https://www.antlr.org/download/antlr-4.7.1-complete.jar
 cd -

docker/install/ubuntu_install_caffe2.sh

@@ -1,3 +1,9 @@
+#!/bin/bash
+
+set -e
+set -u
+set -o pipefail
+
 python3 -m caffe2.python.models.download -i -f squeezenet
 python3 -m caffe2.python.models.download -i -f resnet50
 python3 -m caffe2.python.models.download -i -f vgg19

docker/install/ubuntu_install_core.sh

@@ -1,5 +1,11 @@
+#!/bin/bash
+
+set -e
+set -u
+set -o pipefail
+
 # install libraries for building c++ core on ubuntu
-apt-get update && apt-get install -y --no-install-recommends --force-yes \
+apt-get update && apt-get install -y --no-install-recommends \
         git make libgtest-dev cmake wget unzip libtinfo-dev libz-dev\
         libcurl4-openssl-dev libopenblas-dev g++ sudo
 

docker/install/ubuntu_install_coreml.sh

@@ -1 +1,7 @@
+#!/bin/bash
+
+set -e
+set -u
+set -o pipefail
+
 pip3 install coremltools

docker/install/ubuntu_install_darknet.sh

@@ -1,3 +1,9 @@
+#!/bin/bash
+
+set -e
+set -u
+set -o pipefail
+
 #install the necessary dependancies, cffi, opencv
 wget -q 'https://github.com/siju-samuel/darknet/blob/master/lib/libdarknet.so?raw=true' -O libdarknet.so
 pip2 install opencv-python cffi

docker/install/ubuntu_install_emscripten.sh

@@ -1,3 +1,9 @@
+#!/bin/bash
+
+set -e
+set -u
+set -o pipefail
+
 alias make="make -j4"
 
 # Get latest cmake

docker/install/ubuntu_install_gluoncv.sh

@@ -1 +1,7 @@
+#!/bin/bash
+
+set -e
+set -u
+set -o pipefail
+
 pip3 install gluoncv

docker/install/ubuntu_install_golang.sh

@@ -1,4 +1,11 @@
+#!/bin/bash
+
+set -e
+set -u
+set -o pipefail
+
 #install the necessary dependancies for golang build
-apt-get update && apt-get install -y golang-1.10-go
-apt-get update && apt-get install -y golang-1.10-doc
-apt-get update && apt-get install -y golint
+apt-get update
+apt-get install -y golang-1.10-go
+apt-get install -y golang-1.10-doc
+apt-get install -y golint

docker/install/ubuntu_install_gradle.sh

@@ -1,6 +1,9 @@
+#!/bin/bash
+
 . /etc/profile
 
 set -o errexit -o nounset
+set -o pipefail
 
 GRADLE_HOME=/opt/gradle
 GRADLE_VERSION=4.10-rc-2

docker/install/ubuntu_install_iverilog.sh

@@ -1,4 +1,10 @@
-apt-get install -y --no-install-recommends --force-yes make bison flex
+#!/bin/bash
+
+set -e
+set -u
+set -o pipefail
+
+apt-get install -y --no-install-recommends make bison flex
 wget -q ftp://icarus.com/pub/eda/verilog/v10/verilog-10.1.tar.gz
 tar xf verilog-10.1.tar.gz
 cd verilog-10.1

docker/install/ubuntu_install_java.sh

@@ -1,4 +1,8 @@
+#!/bin/bash
+
 set -o errexit -o nounset
+set -o pipefail
+
 apt-get update && apt-get install -y openjdk-8-jdk maven
 test -d "/usr/lib/jvm/java-8-openjdk-amd64/jre"
 echo "export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/jre" >> /etc/profile

docker/install/ubuntu_install_keras.sh

@@ -1,2 +1,8 @@
+#!/bin/bash
+
+set -e
+set -u
+set -o pipefail
+
 pip2 install keras tensorflow h5py
 pip3 install keras tensorflow h5py

docker/install/ubuntu_install_llvm.sh

@@ -1,3 +1,9 @@
+#!/bin/bash
+
+set -e
+set -u
+set -o pipefail
+
 echo deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-4.0 main\
      >> /etc/apt/sources.list.d/llvm.list
 echo deb-src http://apt.llvm.org/xenial/ llvm-toolchain-xenial-4.0 main\
@@ -19,4 +25,4 @@ echo deb-src http://apt.llvm.org/xenial/ llvm-toolchain-xenial main\
      >> /etc/apt/sources.list.d/llvm.list
 
 wget -q -O - http://apt.llvm.org/llvm-snapshot.gpg.key|sudo apt-key add -
-apt-get update && apt-get install -y --force-yes llvm-4.0 llvm-5.0 llvm-6.0 clang-6.0
+apt-get update && apt-get install -y llvm-4.0 llvm-5.0 llvm-6.0 clang-6.0

docker/install/ubuntu_install_mxnet.sh

@@ -1 +1,7 @@
+#!/bin/bash
+
+set -e
+set -u
+set -o pipefail
+
 pip3 install mxnet

docker/install/ubuntu_install_nnpack.sh

@@ -1,11 +1,13 @@
-apt-get update && apt-get install -y --no-install-recommends --force-yes git cmake
+#!/bin/bash
 
+set -e
+set -u
+set -o pipefail
+
+apt-get update && apt-get install -y --no-install-recommends git cmake
 
-git clone https://github.com/Maratyszcza/NNPACK NNPACK
-cd NNPACK
 # TODO: specific tag?
-git checkout 1e005b0c2
-cd -
+git clone --branch=1e005b0c2 --depth=1 https://github.com/Maratyszcza/NNPACK NNPACK
 
 mkdir -p NNPACK/build
 cd NNPACK/build

docker/install/ubuntu_install_nodejs.sh

@@ -1,4 +1,16 @@
-apt-get update && apt-get install -y curl
+#!/bin/bash
+
+set -e
+set -u
+set -o pipefail
+
+apt-get update
+apt-get install -y curl
+
+# The node install script fetched and executed here will update the
+# apt source list, hence the second apt-get update is necessary.
 curl -s -S -L https://deb.nodesource.com/setup_6.x | bash -
-apt-get update && apt-get install -y nodejs
+apt-get update
+apt-get install -y nodejs
+
 npm install eslint jsdoc ws

docker/install/ubuntu_install_onnx.sh

@@ -1,3 +1,9 @@
+#!/bin/bash
+
+set -e
+set -u
+set -o pipefail
+
 # fix to certain version for now
 pip2 install onnx>=1.1.0
 pip3 install onnx>=1.1.0

docker/install/ubuntu_install_opencl.sh

@@ -1,5 +1,11 @@
+#!/bin/bash
+
+set -e
+set -u
+set -o pipefail
+
 # Install OpenCL runtime in nvidia docker.
-apt-get update && apt-get install -y --no-install-recommends --force-yes \
+apt-get update && apt-get install -y --no-install-recommends \
         ocl-icd-opencl-dev \
         clinfo && \
     rm -rf /var/lib/apt/lists/*

docker/install/ubuntu_install_opengl.sh

@@ -1,4 +1,10 @@
+#!/bin/bash
+
+set -e
+set -u
+set -o pipefail
+
 apt-get update --fix-missing
 
-apt-get install -y --no-install-recommends --force-yes \
-        libgl1-mesa-dev libglfw3-dev
+apt-get install -y --no-install-recommends \
+        libgl1-mesa-dev libglfw3-dev

docker/install/ubuntu_install_python.sh

@@ -1,10 +1,19 @@
+#!/bin/bash
+
+set -e
+set -u
+set -o pipefail
+
 # install python and pip, don't modify this, modify install_python_package.sh
-apt-get update && apt-get install -y python-dev
+apt-get update
+apt-get install -y python-dev
 
 # python 3.6
-apt-get update && apt-get install -y software-properties-common
-add-apt-repository ppa:jonathonf/python-3.6 &&\
-    apt-get update && apt-get install -y python-pip python-dev python3.6 python3.6-dev
+apt-get install -y software-properties-common
+
+add-apt-repository ppa:jonathonf/python-3.6
+apt-get update
+apt-get install -y python-pip python-dev python3.6 python3.6-dev
 
 rm -f /usr/bin/python3 && ln -s /usr/bin/python3.6 /usr/bin/python3
 

docker/install/ubuntu_install_python_package.sh

@@ -1,3 +1,9 @@
+#!/bin/bash
+
+set -e
+set -u
+set -o pipefail
+
 # install libraries for python package on ubuntu
 pip2 install nose pylint==2.2.2 six numpy nose-timer cython decorator scipy tornado typing antlr4-python2-runtime attrs
 pip3 install nose pylint==2.2.2 six numpy nose-timer cython decorator scipy tornado typed_ast pytest mypy orderedset antlr4-python3-runtime attrs

docker/install/ubuntu_install_redis.sh

@@ -1,3 +1,9 @@
+#!/bin/bash
+
+set -e
+set -u
+set -o pipefail
+
 apt-get update && apt-get install -y redis-server
 pip2 install xgboost psutil
 pip3 install xgboost psutil

docker/install/ubuntu_install_rocm.sh

@@ -1,4 +1,10 @@
+#!/bin/bash
+
+set -e
+set -u
+set -o pipefail
+
 # Install ROCm cross compilation toolchain.
 wget -qO - http://repo.radeon.com/rocm/apt/debian/rocm.gpg.key | sudo apt-key add -
 echo deb [arch=amd64] http://repo.radeon.com/rocm/apt/debian/ xenial main > /etc/apt/sources.list.d/rocm.list
-apt-get update && apt-get install -y --force-yes rocm-dev
+apt-get update && apt-get install -y rocm-dev

docker/install/ubuntu_install_rust.sh

@@ -1,9 +1,15 @@
-apt-get update && apt-get install -y --no-install-recommends --force-yes curl
+#!/bin/bash
+
+set -e
+set -u
+set -o pipefail
+
+apt-get update && apt-get install -y --no-install-recommends curl
 
 export RUSTUP_HOME=/opt/rust
 export CARGO_HOME=/opt/rust
 # this rustc is one supported by the installed version of rust-sgx-sdk
-curl -s -S https://sh.rustup.rs -sSf | sh -s -- -y --no-modify-path --default-toolchain nightly-2019-01-28
+curl -s -S -L https://sh.rustup.rs -sSf | sh -s -- -y --no-modify-path --default-toolchain nightly-2019-01-28
 . $CARGO_HOME/env
 rustup toolchain add nightly
 rustup component add rust-src

docker/install/ubuntu_install_sgx.sh

@@ -1,21 +1,26 @@
-apt-get update && apt-get install -y --no-install-recommends --force-yes \
+#!/bin/bash
+
+set -e
+set -u
+set -o pipefail
+
+apt-get update && apt-get install -y --no-install-recommends \
     build-essential git cmake \
     wget python pkg-config software-properties-common \
     autoconf automake libtool ocaml \
     protobuf-compiler libprotobuf-dev \
     libssl-dev libcurl4-openssl-dev curl
 
-git clone https://github.com/intel/linux-sgx.git
+git clone --branch=sgx_2.2 --depth=1 https://github.com/intel/linux-sgx.git
 cd linux-sgx
-git checkout sgx_2.2
-curl -s -S 'https://gist.githubusercontent.com/nhynes/c770b0e91610f8c020a8d1a803a1e7cb/raw/8f5372d9cb88929b3cc49a384943bb363bc06827/intel-sgx.patch' | git apply
+curl -s -S -L 'https://gist.githubusercontent.com/nhynes/c770b0e91610f8c020a8d1a803a1e7cb/raw/8f5372d9cb88929b3cc49a384943bb363bc06827/intel-sgx.patch' | git apply
 ./download_prebuilt.sh
 make -j4 sdk && make -j4 sdk_install_pkg
 ./linux/installer/bin/sgx_linux_x64_sdk*.bin --prefix /opt
 cd -
 
-git clone https://github.com/baidu/rust-sgx-sdk.git /opt/rust-sgx-sdk
+tag=6098af # v1.0.5
+git clone --branch=$tag --depth=1 https://github.com/baidu/rust-sgx-sdk.git /opt/rust-sgx-sdk
 cd /opt/rust-sgx-sdk
-git checkout 6098af # v1.0.5
-curl -s -S 'https://gist.githubusercontent.com/nhynes/37164039c5d3f33aa4f123e4ba720036/raw/b0de575fe937231799930764e76c664b92975163/rust-sgx-sdk.diff' | git apply
+curl -s -S -L 'https://gist.githubusercontent.com/nhynes/37164039c5d3f33aa4f123e4ba720036/raw/b0de575fe937231799930764e76c664b92975163/rust-sgx-sdk.diff' | git apply
 cd -