Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

CI optimize #58

Merged
merged 7 commits into from
Oct 3, 2021
+194 −107
Merged

CI optimize #58

Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
7190b4f
update ci
haobibo Aug 30, 2021
701471d
typo
haobibo Aug 30, 2021
3a85b48
typo in job name
haobibo Aug 30, 2021
ff35cb1
bug fix
haobibo Aug 30, 2021
4e9a85f
update ci
haobibo Aug 30, 2021
4f00eca
bugfix: code-server install
haobibo Sep 28, 2021
ca5eed5
remove leagcy pkgs
Sep 30, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
243 changes: 165 additions & 78 deletions .github/workflows/docker.yml
View file
Original file line number Diff line number Diff line change
@@ -16,27 +16,22 @@ env:
DOCKER_REGISTRY_PASSWORD: ${{ secrets.DOCKER_REGISTRY_PASSWORD }}

jobs:
build-base:
qpod_base:
name: qpod/base
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- run: echo '{"experimental":true}' | sudo tee /etc/docker/daemon.json && sudo service docker restart && docker info

- name: docker - atom/base
run: |
- run: |
source ./tool.sh
build_image atom latest docker_atom/Dockerfile && push_image
build_image base latest docker_base/Dockerfile
alias_image base latest py-mini latest
push_image
build_image atom latest docker_atom/Dockerfile && push_image
build_image base latest docker_base/Dockerfile && alias_image base latest python latest && push_image
build-cuda-base:
qpod_cuda_11:
name: qpod/cuda_11.x
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- run: echo '{"experimental":true}' | sudo tee /etc/docker/daemon.json && sudo service docker restart
- name: docker - cuda
run: |
- run: |
source ./tool.sh
build_image cuda_11.4 latest docker_atom/Dockerfile --build-arg "BASE_IMG=nvidia/cuda:11.4.1-cudnn8-runtime-ubuntu20.04"
@@ -48,19 +43,36 @@ jobs:
build_image cuda_11.2 latest docker_base/Dockerfile --build-arg "BASE_IMG=cuda_11.2"
build_image cuda_11.2 latest docker_cuda/Dockerfile --build-arg "BASE_IMG=cuda_11.2" && push_image
# For legacy tensorflow 1.15 support
qpod_cuda_10:
name: qpod/cuda_10.x
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- run: |
source ./tool.sh
# cuda_10.0 - for legacy tensorflow 1.15 support
build_image cuda_10.0 latest docker_atom/Dockerfile --build-arg "BASE_IMG=nvidia/cuda:10.0-cudnn7-runtime-ubuntu18.04"
build_image cuda_10.0 latest docker_base/Dockerfile --build-arg "BASE_IMG=cuda_10.0"
build_image cuda_10.0 latest docker_cuda/Dockerfile --build-arg "BASE_IMG=cuda_10.0" && push_image
build-core:
needs: build-base
# cuda_10.1
build_image cuda_10.1 latest docker_atom/Dockerfile --build-arg "BASE_IMG=nvidia/cuda:10.1-cudnn7-runtime-ubuntu18.04"
build_image cuda_10.1 latest docker_base/Dockerfile --build-arg "BASE_IMG=cuda_10.1"
build_image cuda_10.1 latest docker_cuda/Dockerfile --build-arg "BASE_IMG=cuda_10.1" && push_image
# cuda_10.2
build_image cuda_10.2 latest docker_atom/Dockerfile --build-arg "BASE_IMG=nvidia/cuda:10.2-cudnn7-runtime-ubuntu18.04"
build_image cuda_10.2 latest docker_base/Dockerfile --build-arg "BASE_IMG=cuda_10.2"
build_image cuda_10.2 latest docker_cuda/Dockerfile --build-arg "BASE_IMG=cuda_10.2" && push_image
qpod_core:
name: qpod/core
needs: qpod_base
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- run: echo '{"experimental":true}' | sudo tee /etc/docker/daemon.json && sudo service docker restart
- name: docker-core
run: |
- run: |
source ./tool.sh
build_image core latest docker_core/Dockerfile \
--build-arg "ARG_PROFILE_PYTHON=base,datascience,database,nlp,cv,bio,chem,tf2,torch" \
@@ -73,84 +85,161 @@ jobs:
--build-arg "ARG_PROFILE_LATEX=base,cjk"
push_image
build-py:
needs: build-base
qpod_py-data:
name: qpod/py-data
needs: qpod_base
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- run: echo '{"experimental":true}' | sudo tee /etc/docker/daemon.json && sudo service docker restart
- name: docker-py
run: |
source ./tool.sh
build_image py-data latest docker_core/Dockerfile --build-arg "ARG_PROFILE_PYTHON=datascience,database" && push_image
build_image py-nlp latest docker_core/Dockerfile --build-arg "ARG_PROFILE_PYTHON=datascience,nlp" && push_image
build_image py-cv latest docker_core/Dockerfile --build-arg "ARG_PROFILE_PYTHON=datascience,cv" && push_image
build_image py-bio latest docker_core/Dockerfile --build-arg "ARG_PROFILE_PYTHON=datascience,bio" && push_image
build_image py-bio latest docker_core/Dockerfile --build-arg "ARG_PROFILE_PYTHON=datascience,chem" && push_image
build_image py-std latest docker_core/Dockerfile \
--build-arg "ARG_PROFILE_PYTHON=datascience,database,nlp,cv,bio,chem" \
&& push_image
build_image py-jdk latest docker_core/Dockerfile \
--build-arg "ARG_PROFILE_PYTHON=datascience,database,nlp,cv,bio,chem" \
--build-arg "ARG_PROFILE_JAVA=base" \
&& push_image
- run: source ./tool.sh && build_image py-data latest docker_core/Dockerfile --build-arg "ARG_PROFILE_PYTHON=datascience,database" && push_image

qpod_py-nlp:
name: qpod/py-nlp
needs: qpod_base
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- run: source ./tool.sh && build_image py-nlp latest docker_core/Dockerfile --build-arg "ARG_PROFILE_PYTHON=datascience,nlp" && push_image

qpod_py-cv:
name: qpod/py-cv
needs: qpod_base
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- run: source ./tool.sh && build_image py-cv latest docker_core/Dockerfile --build-arg "ARG_PROFILE_PYTHON=datascience,cv" && push_image

qpod_py-bio:
name: qpod/py-bio
needs: qpod_base
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- run: source ./tool.sh && build_image py-bio latest docker_core/Dockerfile --build-arg "ARG_PROFILE_PYTHON=datascience,bio" && push_image

qpod_py-chem:
name: qpod/py-chem
needs: qpod_base
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- run: source ./tool.sh && build_image py-chem latest docker_core/Dockerfile --build-arg "ARG_PROFILE_PYTHON=datascience,chem" && push_image

qpod_py-std:
name: qpod/py-std
needs: qpod_base
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- run: source ./tool.sh && build_image py-std latest docker_core/Dockerfile --build-arg "ARG_PROFILE_PYTHON=datascience,database,nlp,cv,bio,chem" && push_image

qpod_py-jdk:
name: qpod/py-jdk
needs: qpod_base
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- run: source ./tool.sh && build_image py-jdk latest docker_core/Dockerfile --build-arg "ARG_PROFILE_PYTHON=datascience,database,nlp,cv,bio,chem" --build-arg "ARG_PROFILE_JAVA=base" && push_image

qpod_r-base:
name: qpod/r-base
needs: qpod_base
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- run: source ./tool.sh build_image r-base latest docker_core/Dockerfile --build-arg "ARG_PROFILE_R=base" && push_image

build-r:
needs: build-base
qpod_r-std:
name: qpod/r-std
needs: qpod_base
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- run: echo '{"experimental":true}' | sudo tee /etc/docker/daemon.json && sudo service docker restart
- name: docker-R
run: |
- run: source ./tool.sh && build_image r-std latest docker_core/Dockerfile --build-arg "ARG_PROFILE_R=base,datascience" --build-arg "ARG_PROFILE_JAVA=base" && push_image

qpod_r-latex:
name: qpod/r-latex
needs: qpod_base
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- run: |
source ./tool.sh
build_image r-mini latest docker_core/Dockerfile \
--build-arg "ARG_PROFILE_R=base" \
&& push_image
build_image r-std latest docker_core/Dockerfile \
--build-arg "ARG_PROFILE_R=base,datascience" \
--build-arg "ARG_PROFILE_JAVA=base" \
&& push_image
build_image r-latex latest docker_core/Dockerfile \
--build-arg "ARG_PROFILE_R=base,datascience" \
--build-arg "ARG_PROFILE_JAVA=base" \
--build-arg "ARG_PROFILE_LATEX=base,cjk" \
&& push_image
--build-arg "ARG_PROFILE_LATEX=base,cjk"
push_image
qpod_r-studio:
name: qpod/r-studio
needs: qpod_base
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- run: |
source ./tool.sh
build_image r-studio latest docker_core/Dockerfile \
--build-arg "ARG_PROFILE_R=base,datascience,rstudio,rshiny" \
--build-arg "ARG_PROFILE_JAVA=base" \
--build-arg "ARG_PROFILE_LATEX=base,cjk" \
&& push_image
--build-arg "ARG_PROFILE_LATEX=base,cjk"
push_image
build-other:
needs: build-base
qpod_node:
name: qpod/node
needs: qpod_base
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- run: echo '{"experimental":true}' | sudo tee /etc/docker/daemon.json && sudo service docker restart
- name: docker-other
run: |
source ./tool.sh
build_image node latest docker_core/Dockerfile --build-arg "ARG_PROFILE_NODEJS=base" && push_image
build_image jdk latest docker_core/Dockerfile --build-arg "ARG_PROFILE_JAVA=base" && push_image
build_image julia latest docker_core/Dockerfile --build-arg "ARG_PROFILE_JULIA=base" && push_image
build_image go latest docker_core/Dockerfile --build-arg "ARG_PROFILE_GO=base" && push_image
- run: source ./tool.sh && build_image node latest docker_core/Dockerfile --build-arg "ARG_PROFILE_NODEJS=base" && push_image

qpod_jdk:
name: qpod/jdk
needs: qpod_base
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- run: source ./tool.sh && build_image jdk latest docker_core/Dockerfile --build-arg "ARG_PROFILE_JAVA=base" && push_image

qpod_julia:
name: qpod/julia
needs: qpod_base
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- run: source ./tool.sh && build_image julia latest docker_core/Dockerfile --build-arg "ARG_PROFILE_JULIA=base" && push_image

build-cuda-core:
needs: build-cuda-base
qpod_go:
name: qpod/go
needs: qpod_base
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- run: echo '{"experimental":true}' | sudo tee /etc/docker/daemon.json && sudo service docker restart
- name: docker-cuda-core
run: |
- run: source ./tool.sh && build_image go latest docker_core/Dockerfile --build-arg "ARG_PROFILE_GO=base" && push_image

qpod_tf1:
name: qpod/tf1
needs: qpod_cuda_10
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- run: |
source ./tool.sh
build_image py-cuda-10.0 latest docker_core/Dockerfile \
--build-arg "ARG_PROFILE_PYTHON=tf1,datascience,nlp,cv" \
--build-arg "BASE_IMG=cuda_10.0"
alias_image py-cuda-10.0 latest tf1 latest && push_image
qpod_core-cuda:
name: qpod/core-cuda
needs: qpod_cuda_11
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- run: |
source ./tool.sh
build_image full-cuda-11.2 latest docker_core/Dockerfile \
--build-arg "BASE_IMG=cuda_11.2" \
--build-arg "ARG_PROFILE_PYTHON=base,datascience,database,nlp,cv,bio,chem,tf2,torch" \
@@ -162,28 +251,26 @@ jobs:
--build-arg "ARG_PROFILE_LATEX=base,cjk"
alias_image full-cuda-11.2 latest core-cuda latest && push_image
build-dev-core:
needs: build-core
qpod_core-dev:
name: qpod/core-dev
needs: qpod_core
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- run: echo '{"experimental":true}' | sudo tee /etc/docker/daemon.json && sudo service docker restart
- name: docker-other
run: |
- run: |
source ./tool.sh
build_image core-dev latest docker_dev/Dockerfile \
--build-arg "ARG_PROFILE_JUPYTER=base,kernels,extensions" \
--build-arg "ARG_PROFILE_VSCODE=base"
alias_image core-dev latest full latest && push_image
build-dev-cuda:
needs: build-cuda-core
qpod_cuda-dev:
name: qpod/cuda-dev
needs: qpod_core-cuda
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- run: echo '{"experimental":true}' | sudo tee /etc/docker/daemon.json && sudo service docker restart
- name: docker-dev-cuda
run: |
- run: |
source ./tool.sh
build_image cuda-dev latest docker_dev/Dockerfile \
--build-arg "BASE_IMG=core-cuda" \
6 changes: 3 additions & 3 deletions README.md
View file
Original file line number Diff line number Diff line change
@@ -70,7 +70,7 @@ Typically, you can choose `full` / `full-cuda` if you have enough disk space and
| py-std | https://hub.docker.com/r/qpod/py-std | qpod/base | Python environment including all the packages mentioned above installed. |
| py-jdk | https://hub.docker.com/r/qpod/py-jdk | qpod/base | `py-std` plus OpenJDK. (no LaTex) |
| r-mini | https://hub.docker.com/r/qpod/r-mini | qpod/base | Minimal R environment -- no JDK, no R data science packages, no LaTex. |
| r-std | https://hub.docker.com/r/qpod/r-std | qpod/base | Standard R environment for data science -- including popular R data science packages. (OpenJDK included since many R packages need Java, no LaTex.) |
| r-std | https://hub.docker.com/r/qpod/r-std | qpod/base | Standard R environment for data science -- including popular R data science packages. (OpenJDK included since many R packages need Java, no LaTex.) |
| r-latex | https://hub.docker.com/r/qpod/r-latex | qpod/base | `r-std` + LaTex -- this is the full R environment if you do not need RStudio. |
| r-studio | https://hub.docker.com/r/qpod/r-studio | qpod/base | Full R environment if you want to use RStudio. `r-latex` + RStudio + RShiny. |
| node | https://hub.docker.com/r/qpod/node | qpod/base | Minimal NodeJS environment (including npm and yarn). |
@@ -173,8 +173,8 @@ Minor modifications are expected to port to `arm64`, `ppc64le` platform.

Although `conda` is installed, we do not recommend to use conda to install a lib/package, because:

- `conda` does not reuse the existing system library yet if a system lib is already installed -- `conda` installs it again.
- `conda` does not provide a stable Linux system library repository yet, for example, some lib works well on `debian:jessie` but fail on `debian:stretch`.
- `conda` repo mirrors are generally not avaliable in restricted enterprise LAN, especially in fincial/medical related companies.
- `conda` does not reuse the existing system library yet if a system lib is already installed -- `conda` installs it again; and `conda` does not provide a stable Linux system library repository yet, for example, some lib works well on `debian:jessie` but fail on `debian:stretch`.

### Customization

4 changes: 3 additions & 1 deletion docker_atom/work/script-setup.sh
View file
Original file line number Diff line number Diff line change
@@ -41,7 +41,9 @@ setup_nvtop() {


setup_java_base() {
URL_OPENJDK=`curl -sL https://jdk.java.net/archive/ | grep 'linux-x64_bin.tar' | sed -n 's/.*href="\([^"]*\).*/\1/p' | head -n 1` \
VERSION_JDK=11 \
&& URL_OPENJDK=`curl -sL https://jdk.java.net/archive/ | grep 'linux-x64_bin.tar' | grep -v sha256 | sed -n 's/.*href="\([^"]*\).*/\1/p' | grep "jdk${VERSION_JDK}" | head -n 1` \
&& echo "Installing JDK from: ${URL_OPENJDK}" \
&& install_tar_gz ${URL_OPENJDK} && mv /opt/jdk-* /opt/jdk \
&& ln -s /opt/jdk/bin/* /usr/bin/ \
&& echo "@ Version of Java (java/javac):" && java -version && javac -version
4 changes: 2 additions & 2 deletions docker_dev/work/install_list_JPY_extend.pip
View file
Original file line number Diff line number Diff line change
@@ -3,8 +3,8 @@

ipyparallel ipynb % IPython extensions
nbresuse % Notebook extension
jupyter_nbextensions_configurator % Jupyter Notebook extension configurator
jupyter_contrib_nbextensions % Jupyter Notebook third-party extensions
# jupyter_nbextensions_configurator % Jupyter Notebook extension configurator
# jupyter_contrib_nbextensions % Jupyter Notebook third-party extensions

jupyterlab_server % Jupyter Lab Server
jupyterlab-git % Jupyter Lab Git extension
8 changes: 5 additions & 3 deletions docker_dev/work/jupyter_notebook_config.json
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,11 @@
{
"NotebookApp": {
"ServerApp": {
"ip": "0.0.0.0",
"port": 8888,
"root_dir": "/root",
"default_url": "/lab",
"allow_root": true,
"notebook_dir": "/root",
"default_url": "/lab"
"allow_origin": "*",
"open_browser": false
}
}
34 changes: 14 additions & 20 deletions docker_dev/work/script-extend.sh
View file
Original file line number Diff line number Diff line change
@@ -18,33 +18,33 @@ setup_jupyter_kernels() {
# Install Bash Kernel
pip install -Uq bash_kernel && python -m bash_kernel.install --sys-prefix

which npm \
which npm \
&& npm install -g --unsafe-perm ijavascript \
&& /opt/node/bin/ijsinstall --install=global --spec-path=full \
&& mv /usr/local/share/jupyter/kernels/javascript /opt/conda/share/jupyter/kernels/

which R \
which R \
&& R -e "install.packages('IRkernel')" \
&& R -e "IRkernel::installspec(user=FALSE)" \
&& mv /usr/local/share/jupyter/kernels/ir /opt/conda/share/jupyter/kernels/

which java \
&& pip install -Uq beakerx pandas py4j \
&& beakerx install \
&& jupyter labextension install beakerx-jupyterlab
which java \
&& pip install -Uq pandas py4j
#&& pip install beakerx && beakerx install \
#&& jupyter labextension install beakerx-jupyterlab

which julia \
which julia \
&& julia -e "using Pkg; Pkg.add(\"IJulia\"); Pkg.precompile();" \
&& mv ~/.local/share/jupyter/kernels/julia* /opt/conda/share/jupyter/kernels/

which go \
which go \
&& export GOPATH=/opt/go/path \
&& go get -u github.com/gopherdata/gophernotes \
&& mkdir -p /opt/conda/share/jupyter/kernels/gophernotes \
&& cp $GOPATH/src/github.com/gopherdata/gophernotes/kernel/* /opt/conda/share/jupyter/kernels/gophernotes \
&& ln -s $GOPATH/bin/gophernotes /usr/bin/

which octave \
which octave \
&& export PATH=/opt/octave/bin:$PATH \
&& pip install -Uq octave_kernel

@@ -53,18 +53,12 @@ setup_jupyter_kernels() {


setup_jupyter_extensions() {
install_apt /opt/utils/install_list_JPY_extend.apt \
install_apt /opt/utils/install_list_JPY_extend.apt \
&& install_pip /opt/utils/install_list_JPY_extend.pip

jupyter nbextension install --sys-prefix --py ipyparallel
jupyter nbextension enable --sys-prefix --py ipyparallel
jupyter serverextension enable --sys-prefix --py ipyparallel
ipcluster nbextension enable

jupyter nbextensions_configurator enable --sys-prefix
jupyter contrib nbextension install --sys-prefix

jupyter serverextension enable --sys-prefix --py jupyterlab_git
# jupyter nbextensions_configurator enable --sys-prefix
# jupyter contrib nbextension install --sys-prefix
# jupyter serverextension enable --sys-prefix --py jupyterlab_git

jupyter labextension install --no-build \
@jupyterlab/toc @jupyterlab/shortcutui @jupyterlab/git @jupyterlab/latex \
@@ -81,7 +75,7 @@ setup_jupyter_extensions() {

setup_vscode_base() {
VERSION_CODER=$(curl -sL https://github.com/cdr/code-server/releases.atom | grep "releases/tag" | head -1 | grep -Po '(\d[\d|.]+)') \
&& install_tar_gz "https://github.com/cdr/code-server/releases/download/v${VERSION_CODER}/code-server-${VERSION_CODER}-linux-x86_64.tar.gz" \
&& install_tar_gz "https://github.com/cdr/code-server/releases/download/v${VERSION_CODER}/code-server-${VERSION_CODER}-linux-amd64.tar.gz" \
&& mv /opt/code-server* /opt/code-server \
&& ln -s /opt/code-server/bin/code-server /usr/bin/ \
&& printf "#!/bin/bash\n/opt/code-server/bin/code-server --port=8888 --auth=none --disable-telemetry $HOME\n" > /usr/local/bin/start-code-server.sh \
2 changes: 2 additions & 0 deletions tool.sh
View file
Original file line number Diff line number Diff line change
@@ -14,6 +14,8 @@ fi
export NAMESPACE=$(echo "${REGISTRY_URL:-"docker.io"}/${CI_PROJECT_NAMESPACE}" | awk '{print tolower($0)}')
echo '---->' $GITHUB_REPOSITORY $NAMESPACE

echo '{"experimental":true}' | sudo tee /etc/docker/daemon.json && sudo service docker restart

build_image() {
echo $@ ;
IMG=$1; TAG=$2; FILE=$3; shift 3; VER=`date +%Y.%m%d`;