diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 70cc849..30c0c28 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -5,7 +5,7 @@ on: branches: [ main ] paths-ignore: - "*.md" - + pull_request: branches: [ main ] paths-ignore: @@ -216,8 +216,8 @@ jobs: - uses: actions/checkout@v2 - run: | source ./tool.sh - build_image tf1-cu100 latest docker_core/Dockerfile --build-arg "BASE_IMG=cuda_10.0" --build-arg "ARG_PROFILE_PYTHON=tf1" - alias_image tf1-cu100 latest tf1 latest + build_image tf1-cuda100 latest docker_core/Dockerfile --build-arg "BASE_IMG=cuda_10.0" --build-arg "ARG_PROFILE_PYTHON=tf1" + alias_image tf1-cuda100 latest tf1 latest push_image qpod_tf2: @@ -228,11 +228,11 @@ jobs: - uses: actions/checkout@v2 - run: | source ./tool.sh - build_image tf2-cu112 latest docker_core/Dockerfile --build-arg "BASE_IMG=cuda_11.2" --build-arg "ARG_PROFILE_PYTHON=tf2" - alias_image tf2-cu112 latest torch latest + build_image tf2-cuda112 latest docker_core/Dockerfile --build-arg "BASE_IMG=cuda_11.2" --build-arg "ARG_PROFILE_PYTHON=tf2" + alias_image tf2-cuda112 latest torch latest push_image - qpod_torch_cu116: + qpod_torch_cuda116: name: qpod/torch-cuda116 needs: qpod_cuda_11_6 runs-on: ubuntu-latest @@ -240,22 +240,22 @@ jobs: - uses: actions/checkout@v2 - run: | source ./tool.sh - build_image torch-cu116 latest docker_core/Dockerfile --build-arg "BASE_IMG=cuda_11.6" --build-arg "ARG_PROFILE_PYTHON=torch" - alias_image torch-cu116 latest torch latest + build_image torch-cuda116 latest docker_core/Dockerfile --build-arg "BASE_IMG=cuda_11.6" --build-arg "ARG_PROFILE_PYTHON=torch" + alias_image torch-cuda116 latest torch latest push_image - qpod_torch_cu102: - name: qpod/torch-cu102 + qpod_torch_cuda102: + name: qpod/torch-cuda102 needs: qpod_cuda_10_2 runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - run: | source ./tool.sh - build_image torch-cu102 latest docker_core/Dockerfile --build-arg "BASE_IMG=cuda_10.2" --build-arg "ARG_PROFILE_PYTHON=torch" + build_image torch-cuda102 latest docker_core/Dockerfile --build-arg "BASE_IMG=cuda_10.2" --build-arg "ARG_PROFILE_PYTHON=torch" push_image - qpod_paddle_cu116: + qpod_paddle_cuda116: name: qpod/paddle-cuda116 needs: qpod_cuda_11_6 runs-on: ubuntu-latest @@ -263,8 +263,19 @@ jobs: - uses: actions/checkout@v2 - run: | source ./tool.sh - build_image paddle-cu116 latest docker_core/Dockerfile --build-arg "BASE_IMG=cuda_11.6" --build-arg "ARG_PROFILE_PYTHON=paddle" - alias_image paddle-cu116 latest paddle latest + build_image paddle-cuda116 latest docker_core/Dockerfile --build-arg "BASE_IMG=cuda_11.6" --build-arg "ARG_PROFILE_PYTHON=paddle" + alias_image paddle-cuda116 latest paddle latest + push_image + + qpod_paddle_cuda102: + name: qpod/paddle-cuda102 + needs: qpod_cuda_10_2 + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - run: | + source ./tool.sh + build_image paddle-cuda102 latest docker_core/Dockerfile --build-arg "BASE_IMG=cuda_10.2" --build-arg "ARG_PROFILE_PYTHON=paddle" push_image diff --git a/docker_atom/work/script-setup.sh b/docker_atom/work/script-setup.sh index 3d084b1..77f1327 100644 --- a/docker_atom/work/script-setup.sh +++ b/docker_atom/work/script-setup.sh @@ -24,17 +24,17 @@ setup_conda_postprocess() { && conda update --all --quiet --yes # These conda pkgs shouldn't be removed (otherwise will cause RemoveError) since they are directly reqiuired by conda: pip setuptools pycosat pyopenssl requests ruamel_yaml - CONDA_PY_PKGS=`conda list | grep "py3" | cut -d " " -f 1 | sed "/#/d;/conda/d;/pip/d;/setuptools/d;/pycosat/d;/pyopenssl/d;/requests/d;/ruamel_yaml/d;"` \ - && conda remove --force -yq $CONDA_PY_PKGS \ - && pip install -UIq pip setuptools $CONDA_PY_PKGS + CONDA_PY_PKGS=$(conda list | grep "py3" | cut -d " " -f 1 | sed "/#/d;/conda/d;/pip/d;/setuptools/d;/pycosat/d;/pyopenssl/d;/requests/d;/ruamel_yaml/d;") \ + && conda remove --force -yq "${CONDA_PY_PKGS}" \ + && pip install -UIq pip setuptools "${CONDA_PY_PKGS}" # Print Conda and Python packages information in the docker build log echo "@ Version of Conda & Python:" && conda info && conda list | grep -v "" } setup_conda_with_mamba() { - VERSION_PTYHON=$1; shift 1; - mamba install -y --root-prefix="${CONDA_PREFIX}" --prefix="${CONDA_PREFIX}" -c "conda-forge" conda pip python="${VERSION_PTYHON:-3.10}" + VERSION_PYTHON=$1; shift 1; + mamba install -y --root-prefix="${CONDA_PREFIX}" --prefix="${CONDA_PREFIX}" -c "conda-forge" conda pip python="${VERSION_PYTHON:-3.10}" rm -rf /opt/conda/pkgs/* setup_conda_postprocess } @@ -59,17 +59,17 @@ setup_nvtop() { # Install Utilities `nvtop` sudo apt-get -qq update --fix-missing && sudo apt-get -qq install -y --no-install-recommends libncurses5-dev - DIRECTORY=`pwd` + DIRECTORY=$(pwd) cd /tmp \ && git clone https://github.com/Syllo/nvtop.git \ && mkdir -pv nvtop/build && cd nvtop/build \ - && LIB_PATH=`find / -name "libnvidia-ml*" 2>/dev/null` \ - && cmake .. -DCMAKE_LIBRARY_PATH="`dirname $LIB_PATH`" .. \ + && LIB_PATH=$(find / -name "libnvidia-ml*" 2>/dev/null) \ + && cmake .. -DCMAKE_LIBRARY_PATH="$(dirname ${LIB_PATH})" .. \ && make && sudo make install \ && nvtop --version - cd $DIRECTORY && rm -rf /tmp/nvtop + cd "${DIRECTORY}" && rm -rf /tmp/nvtop sudo apt-get -qq remove -y libncurses5-dev } @@ -77,9 +77,9 @@ setup_nvtop() { setup_java_base() { VERSION_JDK=11 \ - && URL_OPENJDK=`curl -sL https://jdk.java.net/archive/ | grep 'linux-x64_bin.tar' | grep -v sha256 | sed -n 's/.*href="\([^"]*\).*/\1/p' | grep "jdk${VERSION_JDK}" | head -n 1` \ + && URL_OPENJDK=$(curl -sL https://jdk.java.net/archive/ | grep 'linux-x64_bin.tar' | grep -v sha256 | sed -n 's/.*href="\([^"]*\).*/\1/p' | grep "jdk${VERSION_JDK}" | head -n 1) \ && echo "Installing JDK from: ${URL_OPENJDK}" \ - && install_tar_gz ${URL_OPENJDK} && mv /opt/jdk-* /opt/jdk \ + && install_tar_gz "${URL_OPENJDK}" && mv /opt/jdk-* /opt/jdk \ && ln -sf /opt/jdk/bin/* /usr/bin/ \ && echo "@ Version of Java (java/javac):" && java -version && javac -version } @@ -104,8 +104,8 @@ setup_node() { && export PATH=/opt/node/bin:$PATH \ && npm install -g npm yarn \ && ln -sf /opt/node/bin/* /usr/bin/ \ - && echo "@ Version of Node, npm, and yarn:" `node -v` `npm -v` \ - && echo "@ Version of Yarn:" `yarn -v` + && echo "@ Version of Node, npm, and yarn: $(node -v) $(npm -v)" \ + && echo "@ Version of Yarn: $(yarn -v)" } @@ -116,7 +116,7 @@ setup_R_base() { && echo "options(repos=structure(c(CRAN=\"https://cloud.r-project.org\")))" >> /etc/R/Rprofile.site \ && R -e "install.packages(c('devtools'),clean=T,quiet=T);" \ && ( type java && type R && R CMD javareconf || true ) \ - && echo "@ Version of R:" && R --version + && echo "@ Version of R: $(R --version)" } @@ -147,7 +147,7 @@ setup_R_rstudio() { # Remove RStudio's pandoc and pandoc-proc to reduce size if they are already installed in the jpy-latex step. ( which pandoc && rm /opt/rstudio-server/bin/pandoc/pandoc || true ) \ && ( which pandoc-citeproc && rm /opt/rstudio-server/bin/pandoc/pandoc-citeproc || true ) \ - && echo "@ Version of rstudio-server:" && rstudio-server version + && echo "@ Version of rstudio-server: $(rstudio-server version)" } @@ -168,7 +168,7 @@ setup_R_rshiny() { # hack shiny-server to allow run in root user: https://github.com/rstudio/shiny-server/pull/391 sed -i "s/throw new Error/logger.warn/g" /opt/shiny-server/lib/worker/app-worker.js \ - && echo "@ Version of shiny-server:" && shiny-server --version + && echo "@ Version of shiny-server: $(shiny-server --version)" } @@ -184,9 +184,9 @@ setup_R_datascience() { setup_GO() { GO_VERSION=$(curl -sL https://github.com/golang/go/releases.atom | grep 'releases/tag' | head -1 | grep -Po '\d[\d.]+') \ && GO_URL="https://dl.google.com/go/go$GO_VERSION.linux-$(dpkg --print-architecture).tar.gz" \ - && install_tar_gz $GO_URL go \ + && install_tar_gz "${GO_URL}" go \ && ln -sf /opt/go/bin/go /usr/bin/ \ - && echo "@ Version of golang:" && go version + && echo "@ Version of golang: $(go version)" } @@ -198,7 +198,7 @@ setup_julia() { && mkdir -p /opt/julia/pkg \ && echo "import Libdl; push!(Libdl.DL_LOAD_PATH, \"/opt/conda/lib\")" >> /opt/julia/etc/julia/startup.jl \ && echo "DEPOT_PATH[1]=\"/opt/julia/pkg\"" >> /opt/julia/etc/julia/startup.jl \ - && echo "@ Version of Julia" && julia --version + && echo "@ Version of Julia: $(julia --version)" } @@ -215,15 +215,14 @@ setup_octave() { install_apt /opt/utils/install_list_octave.apt \ && install_octave /opt/utils/install_list_octave.pkg \ - && echo "@ Version of Octave and installed packages:" \ - && /opt/octave/bin/octave --version + && echo "@ Version of Octave and installed packages: $(/opt/octave/bin/octave --version)" } setup_traefik() { TRAEFIK_VERSION=$(curl -sL https://github.com/traefik/traefik/releases.atom | grep 'releases/tag' | head -1 | grep -Po '\d[\d.]+') \ && TRAEFIK_URL="https://github.com/traefik/traefik/releases/download/v${TRAEFIK_VERSION}/traefik_v${TRAEFIK_VERSION}_linux_$(dpkg --print-architecture).tar.gz" \ - && install_tar_gz $TRAEFIK_URL traefik \ + && install_tar_gz "${TRAEFIK_URL}" traefik \ && ln -sf /opt/traefik /usr/bin/ \ - && echo "@ Version of traefik:" && traefik version + && echo "@ Version of traefik: $(traefik version)" } diff --git a/docker_atom/work/script-utils.sh b/docker_atom/work/script-utils.sh index 7f9304d..6bed357 100644 --- a/docker_atom/work/script-utils.sh +++ b/docker_atom/work/script-utils.sh @@ -11,7 +11,7 @@ install_conda() { cat $1 | cut -d "%" -f 1 | sed '/^$/d' | xargs -r -n1 conda install_mamba() { cat $1 | cut -d "%" -f 1 | sed '/^$/d' | xargs -r -n1 mamba install -yq --root-prefix="${CONDA_PREFIX}" --prefix="${CONDA_PREFIX}" ; } # function to install python packages with pip from a text file which lists package names (add comments with % char) -install_pip() { cat $1 | cut -d "%" -f 1 | sed '/^$/d' | xargs -r -n1 pip install -U --pre ; } +install_pip() { cat $1 | cut -d "%" -f 1 | sed '/^$/d' | xargs -r -n1 pip install -U --root-user-action=ignore --pre ; } # function to install R packages from a text file which lists package names (add comments with % char, use quiet=T to be less verbose) install_R() { R -e "options(Ncpus=4);lapply(scan('$1','c',comment.char='%'),function(x){cat(x,system.time(install.packages(x,clean=T,quiet=T)),'\n')})"; } diff --git a/docker_core/work/install_list_PY_datascience.pip b/docker_core/work/install_list_PY_datascience.pip index fd82c85..7a8773c 100644 --- a/docker_core/work/install_list_PY_datascience.pip +++ b/docker_core/work/install_list_PY_datascience.pip @@ -7,5 +7,5 @@ sortedcontainers sortedcollections % Enhanced Data Structur lxml beautifulsoup4 pyquery numpydoc sphinx % Parse/Build documents pandas pyarrow xlrd openpyxl beautifulsoup4 % Data preparation patsy statsmodels sympy pywavelets numexpr pycrypto % Statistics, Maths, and related Utilities -scikit-learn scipy mlflow % Statistical ML, Data Mining +scikit-learn scipy % Statistical ML, Data Mining bokeh seaborn altair vincent % Data visualization diff --git a/docker_dev/work/script-extend.sh b/docker_dev/work/script-extend.sh index 413707f..6397409 100644 --- a/docker_dev/work/script-extend.sh +++ b/docker_dev/work/script-extend.sh @@ -5,8 +5,8 @@ setup_jupyter_base() { pip install -Uq jupyterhub jupyterlab notebook ipywidgets qpod_hub "mistune>=0.8.1,<2" \ && jupyter nbextension enable --py widgetsnbextension \ && jupyter labextension install @jupyter-widgets/jupyterlab-manager \ - && echo "@ Version of Jupyter Notebook/JupyterLab:" \ - `jupyter notebook --version` `jupyter lab --version` + && echo "@ Version of Jupyter Notebook/JupyterLab: $(jupyter notebook --version)" \ + && echo "@ Version of Jupyter Notebook/JupyterLab: $(jupyter lab --version)" } @@ -74,7 +74,7 @@ setup_vscode_base() { && install_tar_gz "https://github.com/cdr/code-server/releases/download/v${VERSION_CODER}/code-server-${VERSION_CODER}-linux-amd64.tar.gz" \ && mv /opt/code-server* /opt/code-server \ && ln -s /opt/code-server/bin/code-server /usr/bin/ \ - && printf "#!/bin/bash\n/opt/code-server/bin/code-server --port=8888 --auth=none --disable-telemetry $HOME\n" > /usr/local/bin/start-code-server.sh \ + && printf "#!/bin/bash\n/opt/code-server/bin/code-server --port=8888 --auth=none --disable-telemetry ${HOME}\n" > /usr/local/bin/start-code-server.sh \ && chmod u+x /usr/local/bin/start-code-server.sh \ - && echo "@ coder-server Version:" && code-server -v + && echo "@ coder-server Version: $(code-server -v)" }