diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 9603404..51c8f5a 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -16,27 +16,22 @@ env: DOCKER_REGISTRY_PASSWORD: ${{ secrets.DOCKER_REGISTRY_PASSWORD }} jobs: - build-base: + qpod_base: + name: qpod/base runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - run: echo '{"experimental":true}' | sudo tee /etc/docker/daemon.json && sudo service docker restart && docker info - - - name: docker - atom/base - run: | + - run: | source ./tool.sh - build_image atom latest docker_atom/Dockerfile && push_image - build_image base latest docker_base/Dockerfile - alias_image base latest py-mini latest - push_image + build_image atom latest docker_atom/Dockerfile && push_image + build_image base latest docker_base/Dockerfile && alias_image base latest python latest && push_image - build-cuda-base: + qpod_cuda_11: + name: qpod/cuda_11.x runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - run: echo '{"experimental":true}' | sudo tee /etc/docker/daemon.json && sudo service docker restart - - name: docker - cuda - run: | + - run: | source ./tool.sh build_image cuda_11.4 latest docker_atom/Dockerfile --build-arg "BASE_IMG=nvidia/cuda:11.4.1-cudnn8-runtime-ubuntu20.04" @@ -48,19 +43,36 @@ jobs: build_image cuda_11.2 latest docker_base/Dockerfile --build-arg "BASE_IMG=cuda_11.2" build_image cuda_11.2 latest docker_cuda/Dockerfile --build-arg "BASE_IMG=cuda_11.2" && push_image - # For legacy tensorflow 1.15 support + qpod_cuda_10: + name: qpod/cuda_10.x + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - run: | + source ./tool.sh + + # cuda_10.0 - for legacy tensorflow 1.15 support build_image cuda_10.0 latest docker_atom/Dockerfile --build-arg "BASE_IMG=nvidia/cuda:10.0-cudnn7-runtime-ubuntu18.04" build_image cuda_10.0 latest docker_base/Dockerfile --build-arg "BASE_IMG=cuda_10.0" build_image cuda_10.0 latest docker_cuda/Dockerfile --build-arg "BASE_IMG=cuda_10.0" && push_image - build-core: - needs: build-base + # cuda_10.1 + build_image cuda_10.1 latest docker_atom/Dockerfile --build-arg "BASE_IMG=nvidia/cuda:10.1-cudnn7-runtime-ubuntu18.04" + build_image cuda_10.1 latest docker_base/Dockerfile --build-arg "BASE_IMG=cuda_10.1" + build_image cuda_10.1 latest docker_cuda/Dockerfile --build-arg "BASE_IMG=cuda_10.1" && push_image + + # cuda_10.2 + build_image cuda_10.2 latest docker_atom/Dockerfile --build-arg "BASE_IMG=nvidia/cuda:10.2-cudnn7-runtime-ubuntu18.04" + build_image cuda_10.2 latest docker_base/Dockerfile --build-arg "BASE_IMG=cuda_10.2" + build_image cuda_10.2 latest docker_cuda/Dockerfile --build-arg "BASE_IMG=cuda_10.2" && push_image + + qpod_core: + name: qpod/core + needs: qpod_base runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - run: echo '{"experimental":true}' | sudo tee /etc/docker/daemon.json && sudo service docker restart - - name: docker-core - run: | + - run: | source ./tool.sh build_image core latest docker_core/Dockerfile \ --build-arg "ARG_PROFILE_PYTHON=base,datascience,database,nlp,cv,bio,chem,tf2,torch" \ @@ -73,77 +85,145 @@ jobs: --build-arg "ARG_PROFILE_LATEX=base,cjk" push_image - build-py: - needs: build-base + qpod_py-data: + name: qpod/py-data + needs: qpod_base runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - run: echo '{"experimental":true}' | sudo tee /etc/docker/daemon.json && sudo service docker restart - - name: docker-py - run: | - source ./tool.sh - build_image py-data latest docker_core/Dockerfile --build-arg "ARG_PROFILE_PYTHON=datascience,database" && push_image - build_image py-nlp latest docker_core/Dockerfile --build-arg "ARG_PROFILE_PYTHON=datascience,nlp" && push_image - build_image py-cv latest docker_core/Dockerfile --build-arg "ARG_PROFILE_PYTHON=datascience,cv" && push_image - build_image py-bio latest docker_core/Dockerfile --build-arg "ARG_PROFILE_PYTHON=datascience,bio" && push_image - build_image py-bio latest docker_core/Dockerfile --build-arg "ARG_PROFILE_PYTHON=datascience,chem" && push_image - build_image py-std latest docker_core/Dockerfile \ - --build-arg "ARG_PROFILE_PYTHON=datascience,database,nlp,cv,bio,chem" \ - && push_image - build_image py-jdk latest docker_core/Dockerfile \ - --build-arg "ARG_PROFILE_PYTHON=datascience,database,nlp,cv,bio,chem" \ - --build-arg "ARG_PROFILE_JAVA=base" \ - && push_image + - run: source ./tool.sh && build_image py-data latest docker_core/Dockerfile --build-arg "ARG_PROFILE_PYTHON=datascience,database" && push_image + + qpod_py-nlp: + name: qpod/py-nlp + needs: qpod_base + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - run: source ./tool.sh && build_image py-nlp latest docker_core/Dockerfile --build-arg "ARG_PROFILE_PYTHON=datascience,nlp" && push_image + + qpod_py-cv: + name: qpod/py-cv + needs: qpod_base + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - run: source ./tool.sh && build_image py-cv latest docker_core/Dockerfile --build-arg "ARG_PROFILE_PYTHON=datascience,cv" && push_image + + qpod_py-bio: + name: qpod/py-bio + needs: qpod_base + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - run: source ./tool.sh && build_image py-bio latest docker_core/Dockerfile --build-arg "ARG_PROFILE_PYTHON=datascience,bio" && push_image + + qpod_py-chem: + name: qpod/py-chem + needs: qpod_base + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - run: source ./tool.sh && build_image py-chem latest docker_core/Dockerfile --build-arg "ARG_PROFILE_PYTHON=datascience,chem" && push_image + + qpod_py-std: + name: qpod/py-std + needs: qpod_base + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - run: source ./tool.sh && build_image py-std latest docker_core/Dockerfile --build-arg "ARG_PROFILE_PYTHON=datascience,database,nlp,cv,bio,chem" && push_image + + qpod_py-jdk: + name: qpod/py-jdk + needs: qpod_base + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - run: source ./tool.sh && build_image py-jdk latest docker_core/Dockerfile --build-arg "ARG_PROFILE_PYTHON=datascience,database,nlp,cv,bio,chem" --build-arg "ARG_PROFILE_JAVA=base" && push_image + + qpod_r-base: + name: qpod/r-base + needs: qpod_base + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - run: source ./tool.sh build_image r-base latest docker_core/Dockerfile --build-arg "ARG_PROFILE_R=base" && push_image - build-r: - needs: build-base + qpod_r-std: + name: qpod/r-std + needs: qpod_base runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - run: echo '{"experimental":true}' | sudo tee /etc/docker/daemon.json && sudo service docker restart - - name: docker-R - run: | + - run: source ./tool.sh && build_image r-std latest docker_core/Dockerfile --build-arg "ARG_PROFILE_R=base,datascience" --build-arg "ARG_PROFILE_JAVA=base" && push_image + + qpod_r-latex: + name: qpod/r-latex + needs: qpod_base + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - run: | source ./tool.sh - build_image r-mini latest docker_core/Dockerfile \ - --build-arg "ARG_PROFILE_R=base" \ - && push_image - build_image r-std latest docker_core/Dockerfile \ - --build-arg "ARG_PROFILE_R=base,datascience" \ - --build-arg "ARG_PROFILE_JAVA=base" \ - && push_image build_image r-latex latest docker_core/Dockerfile \ --build-arg "ARG_PROFILE_R=base,datascience" \ --build-arg "ARG_PROFILE_JAVA=base" \ - --build-arg "ARG_PROFILE_LATEX=base,cjk" \ - && push_image + --build-arg "ARG_PROFILE_LATEX=base,cjk" + push_image + + qpod_r-studio: + name: qpod/r-studio + needs: qpod_base + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - run: | + source ./tool.sh build_image r-studio latest docker_core/Dockerfile \ --build-arg "ARG_PROFILE_R=base,datascience,rstudio,rshiny" \ --build-arg "ARG_PROFILE_JAVA=base" \ - --build-arg "ARG_PROFILE_LATEX=base,cjk" \ - && push_image + --build-arg "ARG_PROFILE_LATEX=base,cjk" + push_image - build-other: - needs: build-base + qpod_node: + name: qpod/node + needs: qpod_base runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - run: echo '{"experimental":true}' | sudo tee /etc/docker/daemon.json && sudo service docker restart - - name: docker-other - run: | - source ./tool.sh - build_image node latest docker_core/Dockerfile --build-arg "ARG_PROFILE_NODEJS=base" && push_image - build_image jdk latest docker_core/Dockerfile --build-arg "ARG_PROFILE_JAVA=base" && push_image - build_image julia latest docker_core/Dockerfile --build-arg "ARG_PROFILE_JULIA=base" && push_image - build_image go latest docker_core/Dockerfile --build-arg "ARG_PROFILE_GO=base" && push_image + - run: source ./tool.sh && build_image node latest docker_core/Dockerfile --build-arg "ARG_PROFILE_NODEJS=base" && push_image + + qpod_jdk: + name: qpod/jdk + needs: qpod_base + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - run: source ./tool.sh && build_image jdk latest docker_core/Dockerfile --build-arg "ARG_PROFILE_JAVA=base" && push_image + + qpod_julia: + name: qpod/julia + needs: qpod_base + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - run: source ./tool.sh && build_image julia latest docker_core/Dockerfile --build-arg "ARG_PROFILE_JULIA=base" && push_image - build-cuda-core: - needs: build-cuda-base + qpod_go: + name: qpod/go + needs: qpod_base runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - run: echo '{"experimental":true}' | sudo tee /etc/docker/daemon.json && sudo service docker restart - - name: docker-cuda-core - run: | + - run: source ./tool.sh && build_image go latest docker_core/Dockerfile --build-arg "ARG_PROFILE_GO=base" && push_image + + qpod_tf1: + name: qpod/tf1 + needs: qpod_cuda_10 + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - run: | source ./tool.sh build_image py-cuda-10.0 latest docker_core/Dockerfile \ @@ -151,6 +231,15 @@ jobs: --build-arg "BASE_IMG=cuda_10.0" alias_image py-cuda-10.0 latest tf1 latest && push_image + qpod_core-cuda: + name: qpod/core-cuda + needs: qpod_cuda_11 + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - run: | + source ./tool.sh + build_image full-cuda-11.2 latest docker_core/Dockerfile \ --build-arg "BASE_IMG=cuda_11.2" \ --build-arg "ARG_PROFILE_PYTHON=base,datascience,database,nlp,cv,bio,chem,tf2,torch" \ @@ -162,28 +251,26 @@ jobs: --build-arg "ARG_PROFILE_LATEX=base,cjk" alias_image full-cuda-11.2 latest core-cuda latest && push_image - build-dev-core: - needs: build-core + qpod_core-dev: + name: qpod/core-dev + needs: qpod_core runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - run: echo '{"experimental":true}' | sudo tee /etc/docker/daemon.json && sudo service docker restart - - name: docker-other - run: | + - run: | source ./tool.sh build_image core-dev latest docker_dev/Dockerfile \ --build-arg "ARG_PROFILE_JUPYTER=base,kernels,extensions" \ --build-arg "ARG_PROFILE_VSCODE=base" alias_image core-dev latest full latest && push_image - build-dev-cuda: - needs: build-cuda-core + qpod_cuda-dev: + name: qpod/cuda-dev + needs: qpod_core-cuda runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - run: echo '{"experimental":true}' | sudo tee /etc/docker/daemon.json && sudo service docker restart - - name: docker-dev-cuda - run: | + - run: | source ./tool.sh build_image cuda-dev latest docker_dev/Dockerfile \ --build-arg "BASE_IMG=core-cuda" \ diff --git a/README.md b/README.md index be3c3a4..dcce823 100644 --- a/README.md +++ b/README.md @@ -70,7 +70,7 @@ Typically, you can choose `full` / `full-cuda` if you have enough disk space and | py-std | https://hub.docker.com/r/qpod/py-std | qpod/base | Python environment including all the packages mentioned above installed. | | py-jdk | https://hub.docker.com/r/qpod/py-jdk | qpod/base | `py-std` plus OpenJDK. (no LaTex) | | r-mini | https://hub.docker.com/r/qpod/r-mini | qpod/base | Minimal R environment -- no JDK, no R data science packages, no LaTex. | -| r-std | https://hub.docker.com/r/qpod/r-std | qpod/base | Standard R environment for data science -- including popular R data science packages. (OpenJDK included since many R packages need Java, no LaTex.) | +| r-std | https://hub.docker.com/r/qpod/r-std | qpod/base | Standard R environment for data science -- including popular R data science packages. (OpenJDK included since many R packages need Java, no LaTex.) | | r-latex | https://hub.docker.com/r/qpod/r-latex | qpod/base | `r-std` + LaTex -- this is the full R environment if you do not need RStudio. | | r-studio | https://hub.docker.com/r/qpod/r-studio | qpod/base | Full R environment if you want to use RStudio. `r-latex` + RStudio + RShiny. | | node | https://hub.docker.com/r/qpod/node | qpod/base | Minimal NodeJS environment (including npm and yarn). | @@ -173,8 +173,8 @@ Minor modifications are expected to port to `arm64`, `ppc64le` platform. Although `conda` is installed, we do not recommend to use conda to install a lib/package, because: -- `conda` does not reuse the existing system library yet if a system lib is already installed -- `conda` installs it again. -- `conda` does not provide a stable Linux system library repository yet, for example, some lib works well on `debian:jessie` but fail on `debian:stretch`. +- `conda` repo mirrors are generally not avaliable in restricted enterprise LAN, especially in fincial/medical related companies. +- `conda` does not reuse the existing system library yet if a system lib is already installed -- `conda` installs it again; and `conda` does not provide a stable Linux system library repository yet, for example, some lib works well on `debian:jessie` but fail on `debian:stretch`. ### Customization diff --git a/docker_atom/work/script-setup.sh b/docker_atom/work/script-setup.sh index de6f690..b255107 100644 --- a/docker_atom/work/script-setup.sh +++ b/docker_atom/work/script-setup.sh @@ -41,7 +41,9 @@ setup_nvtop() { setup_java_base() { - URL_OPENJDK=`curl -sL https://jdk.java.net/archive/ | grep 'linux-x64_bin.tar' | sed -n 's/.*href="\([^"]*\).*/\1/p' | head -n 1` \ + VERSION_JDK=11 \ + && URL_OPENJDK=`curl -sL https://jdk.java.net/archive/ | grep 'linux-x64_bin.tar' | grep -v sha256 | sed -n 's/.*href="\([^"]*\).*/\1/p' | grep "jdk${VERSION_JDK}" | head -n 1` \ + && echo "Installing JDK from: ${URL_OPENJDK}" \ && install_tar_gz ${URL_OPENJDK} && mv /opt/jdk-* /opt/jdk \ && ln -s /opt/jdk/bin/* /usr/bin/ \ && echo "@ Version of Java (java/javac):" && java -version && javac -version diff --git a/docker_dev/work/install_list_JPY_extend.pip b/docker_dev/work/install_list_JPY_extend.pip index 33ff6fd..22a8601 100644 --- a/docker_dev/work/install_list_JPY_extend.pip +++ b/docker_dev/work/install_list_JPY_extend.pip @@ -3,8 +3,8 @@ ipyparallel ipynb % IPython extensions nbresuse % Notebook extension -jupyter_nbextensions_configurator % Jupyter Notebook extension configurator -jupyter_contrib_nbextensions % Jupyter Notebook third-party extensions +# jupyter_nbextensions_configurator % Jupyter Notebook extension configurator +# jupyter_contrib_nbextensions % Jupyter Notebook third-party extensions jupyterlab_server % Jupyter Lab Server jupyterlab-git % Jupyter Lab Git extension diff --git a/docker_dev/work/jupyter_notebook_config.json b/docker_dev/work/jupyter_notebook_config.json index 249d955..3196589 100644 --- a/docker_dev/work/jupyter_notebook_config.json +++ b/docker_dev/work/jupyter_notebook_config.json @@ -1,9 +1,11 @@ { - "NotebookApp": { + "ServerApp": { "ip": "0.0.0.0", "port": 8888, + "root_dir": "/root", + "default_url": "/lab", "allow_root": true, - "notebook_dir": "/root", - "default_url": "/lab" + "allow_origin": "*", + "open_browser": false } } diff --git a/docker_dev/work/script-extend.sh b/docker_dev/work/script-extend.sh index 4a8fe70..aedb1e5 100644 --- a/docker_dev/work/script-extend.sh +++ b/docker_dev/work/script-extend.sh @@ -18,33 +18,33 @@ setup_jupyter_kernels() { # Install Bash Kernel pip install -Uq bash_kernel && python -m bash_kernel.install --sys-prefix - which npm \ + which npm \ && npm install -g --unsafe-perm ijavascript \ && /opt/node/bin/ijsinstall --install=global --spec-path=full \ && mv /usr/local/share/jupyter/kernels/javascript /opt/conda/share/jupyter/kernels/ - which R \ + which R \ && R -e "install.packages('IRkernel')" \ && R -e "IRkernel::installspec(user=FALSE)" \ && mv /usr/local/share/jupyter/kernels/ir /opt/conda/share/jupyter/kernels/ - which java \ - && pip install -Uq beakerx pandas py4j \ - && beakerx install \ - && jupyter labextension install beakerx-jupyterlab + which java \ + && pip install -Uq pandas py4j + #&& pip install beakerx && beakerx install \ + #&& jupyter labextension install beakerx-jupyterlab - which julia \ + which julia \ && julia -e "using Pkg; Pkg.add(\"IJulia\"); Pkg.precompile();" \ && mv ~/.local/share/jupyter/kernels/julia* /opt/conda/share/jupyter/kernels/ - which go \ + which go \ && export GOPATH=/opt/go/path \ && go get -u github.com/gopherdata/gophernotes \ && mkdir -p /opt/conda/share/jupyter/kernels/gophernotes \ && cp $GOPATH/src/github.com/gopherdata/gophernotes/kernel/* /opt/conda/share/jupyter/kernels/gophernotes \ && ln -s $GOPATH/bin/gophernotes /usr/bin/ - which octave \ + which octave \ && export PATH=/opt/octave/bin:$PATH \ && pip install -Uq octave_kernel @@ -53,18 +53,12 @@ setup_jupyter_kernels() { setup_jupyter_extensions() { - install_apt /opt/utils/install_list_JPY_extend.apt \ + install_apt /opt/utils/install_list_JPY_extend.apt \ && install_pip /opt/utils/install_list_JPY_extend.pip - jupyter nbextension install --sys-prefix --py ipyparallel - jupyter nbextension enable --sys-prefix --py ipyparallel - jupyter serverextension enable --sys-prefix --py ipyparallel - ipcluster nbextension enable - - jupyter nbextensions_configurator enable --sys-prefix - jupyter contrib nbextension install --sys-prefix - - jupyter serverextension enable --sys-prefix --py jupyterlab_git + # jupyter nbextensions_configurator enable --sys-prefix + # jupyter contrib nbextension install --sys-prefix + # jupyter serverextension enable --sys-prefix --py jupyterlab_git jupyter labextension install --no-build \ @jupyterlab/toc @jupyterlab/shortcutui @jupyterlab/git @jupyterlab/latex \ @@ -81,7 +75,7 @@ setup_jupyter_extensions() { setup_vscode_base() { VERSION_CODER=$(curl -sL https://github.com/cdr/code-server/releases.atom | grep "releases/tag" | head -1 | grep -Po '(\d[\d|.]+)') \ - && install_tar_gz "https://github.com/cdr/code-server/releases/download/v${VERSION_CODER}/code-server-${VERSION_CODER}-linux-x86_64.tar.gz" \ + && install_tar_gz "https://github.com/cdr/code-server/releases/download/v${VERSION_CODER}/code-server-${VERSION_CODER}-linux-amd64.tar.gz" \ && mv /opt/code-server* /opt/code-server \ && ln -s /opt/code-server/bin/code-server /usr/bin/ \ && printf "#!/bin/bash\n/opt/code-server/bin/code-server --port=8888 --auth=none --disable-telemetry $HOME\n" > /usr/local/bin/start-code-server.sh \ diff --git a/tool.sh b/tool.sh index 36bf87f..0d4b666 100644 --- a/tool.sh +++ b/tool.sh @@ -14,6 +14,8 @@ fi export NAMESPACE=$(echo "${REGISTRY_URL:-"docker.io"}/${CI_PROJECT_NAMESPACE}" | awk '{print tolower($0)}') echo '---->' $GITHUB_REPOSITORY $NAMESPACE +echo '{"experimental":true}' | sudo tee /etc/docker/daemon.json && sudo service docker restart + build_image() { echo $@ ; IMG=$1; TAG=$2; FILE=$3; shift 3; VER=`date +%Y.%m%d`;