CI optimize (#58)

haobibo · web-flow · commit 61e21710e201 · 2021-10-03T20:37:12.000+08:00
diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
@@ -16,27 +16,22 @@ env:
   DOCKER_REGISTRY_PASSWORD: ${{ secrets.DOCKER_REGISTRY_PASSWORD }}
 
 jobs:
-  build-base:
+  qpod_base:
+    name: qpod/base
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - run: echo '{"experimental":true}' | sudo tee /etc/docker/daemon.json && sudo service docker restart && docker info
-
-      - name: docker - atom/base
-        run: |
+      - run: |
           source ./tool.sh
-          build_image atom latest  docker_atom/Dockerfile && push_image
-          build_image base latest  docker_base/Dockerfile
-          alias_image base latest  py-mini latest
-          push_image
+          build_image atom latest docker_atom/Dockerfile && push_image
+          build_image base latest docker_base/Dockerfile && alias_image base latest  python latest && push_image
 
-  build-cuda-base:
+  qpod_cuda_11:
+    name: qpod/cuda_11.x
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - run: echo '{"experimental":true}' | sudo tee /etc/docker/daemon.json && sudo service docker restart
-      - name: docker - cuda
-        run: |
+      - run: |
           source ./tool.sh
 
           build_image cuda_11.4 latest docker_atom/Dockerfile --build-arg "BASE_IMG=nvidia/cuda:11.4.1-cudnn8-runtime-ubuntu20.04"
@@ -48,19 +43,36 @@ jobs:
           build_image cuda_11.2 latest docker_base/Dockerfile --build-arg "BASE_IMG=cuda_11.2"
           build_image cuda_11.2 latest docker_cuda/Dockerfile --build-arg "BASE_IMG=cuda_11.2" && push_image
 
-          # For legacy tensorflow 1.15 support
+  qpod_cuda_10:
+    name: qpod/cuda_10.x
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - run: |
+          source ./tool.sh
+
+          # cuda_10.0 - for legacy tensorflow 1.15 support
           build_image cuda_10.0 latest docker_atom/Dockerfile --build-arg "BASE_IMG=nvidia/cuda:10.0-cudnn7-runtime-ubuntu18.04"
           build_image cuda_10.0 latest docker_base/Dockerfile --build-arg "BASE_IMG=cuda_10.0"
           build_image cuda_10.0 latest docker_cuda/Dockerfile --build-arg "BASE_IMG=cuda_10.0" && push_image
 
-  build-core:
-    needs: build-base
+          # cuda_10.1
+          build_image cuda_10.1 latest docker_atom/Dockerfile --build-arg "BASE_IMG=nvidia/cuda:10.1-cudnn7-runtime-ubuntu18.04"
+          build_image cuda_10.1 latest docker_base/Dockerfile --build-arg "BASE_IMG=cuda_10.1"
+          build_image cuda_10.1 latest docker_cuda/Dockerfile --build-arg "BASE_IMG=cuda_10.1" && push_image
+
+          # cuda_10.2
+          build_image cuda_10.2 latest docker_atom/Dockerfile --build-arg "BASE_IMG=nvidia/cuda:10.2-cudnn7-runtime-ubuntu18.04"
+          build_image cuda_10.2 latest docker_base/Dockerfile --build-arg "BASE_IMG=cuda_10.2"
+          build_image cuda_10.2 latest docker_cuda/Dockerfile --build-arg "BASE_IMG=cuda_10.2" && push_image
+
+  qpod_core:
+    name: qpod/core
+    needs: qpod_base
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - run: echo '{"experimental":true}' | sudo tee /etc/docker/daemon.json && sudo service docker restart
-      - name: docker-core
-        run: |
+      - run: |
           source ./tool.sh
           build_image core latest  docker_core/Dockerfile \
             --build-arg "ARG_PROFILE_PYTHON=base,datascience,database,nlp,cv,bio,chem,tf2,torch" \
@@ -73,84 +85,161 @@ jobs:
             --build-arg "ARG_PROFILE_LATEX=base,cjk"
           push_image
 
-  build-py:
-    needs: build-base
+  qpod_py-data:
+    name: qpod/py-data
+    needs: qpod_base
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - run: echo '{"experimental":true}' | sudo tee /etc/docker/daemon.json && sudo service docker restart
-      - name: docker-py
-        run: |
-          source ./tool.sh
-          build_image py-data latest docker_core/Dockerfile --build-arg "ARG_PROFILE_PYTHON=datascience,database"   && push_image
-          build_image py-nlp  latest docker_core/Dockerfile --build-arg "ARG_PROFILE_PYTHON=datascience,nlp"        && push_image
-          build_image py-cv   latest docker_core/Dockerfile --build-arg "ARG_PROFILE_PYTHON=datascience,cv"         && push_image
-          build_image py-bio  latest docker_core/Dockerfile --build-arg "ARG_PROFILE_PYTHON=datascience,bio"        && push_image
-          build_image py-bio  latest docker_core/Dockerfile --build-arg "ARG_PROFILE_PYTHON=datascience,chem"       && push_image
-          build_image py-std  latest docker_core/Dockerfile \
-            --build-arg "ARG_PROFILE_PYTHON=datascience,database,nlp,cv,bio,chem" \
-            && push_image
-          build_image py-jdk  latest docker_core/Dockerfile \
-            --build-arg "ARG_PROFILE_PYTHON=datascience,database,nlp,cv,bio,chem" \
-            --build-arg "ARG_PROFILE_JAVA=base" \
-            && push_image
+      - run: source ./tool.sh && build_image py-data latest docker_core/Dockerfile --build-arg "ARG_PROFILE_PYTHON=datascience,database" && push_image
+
+  qpod_py-nlp:
+    name: qpod/py-nlp
+    needs: qpod_base
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - run: source ./tool.sh && build_image py-nlp  latest docker_core/Dockerfile --build-arg "ARG_PROFILE_PYTHON=datascience,nlp" && push_image
+
+  qpod_py-cv:
+    name: qpod/py-cv
+    needs: qpod_base
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - run: source ./tool.sh && build_image py-cv   latest docker_core/Dockerfile --build-arg "ARG_PROFILE_PYTHON=datascience,cv" && push_image
+
+  qpod_py-bio:
+    name: qpod/py-bio
+    needs: qpod_base
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - run: source ./tool.sh && build_image py-bio  latest docker_core/Dockerfile --build-arg "ARG_PROFILE_PYTHON=datascience,bio" && push_image
+
+  qpod_py-chem:
+    name: qpod/py-chem
+    needs: qpod_base
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - run: source ./tool.sh && build_image py-chem latest docker_core/Dockerfile --build-arg "ARG_PROFILE_PYTHON=datascience,chem" && push_image
+
+  qpod_py-std:
+    name: qpod/py-std
+    needs: qpod_base
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - run: source ./tool.sh && build_image py-std  latest docker_core/Dockerfile --build-arg "ARG_PROFILE_PYTHON=datascience,database,nlp,cv,bio,chem" && push_image
+
+  qpod_py-jdk:
+    name: qpod/py-jdk
+    needs: qpod_base
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - run: source ./tool.sh && build_image py-jdk  latest docker_core/Dockerfile --build-arg "ARG_PROFILE_PYTHON=datascience,database,nlp,cv,bio,chem" --build-arg "ARG_PROFILE_JAVA=base" && push_image
+
+  qpod_r-base:
+    name: qpod/r-base
+    needs: qpod_base
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - run: source ./tool.sh build_image r-base latest docker_core/Dockerfile --build-arg "ARG_PROFILE_R=base" && push_image
 
-  build-r:
-    needs: build-base
+  qpod_r-std:
+    name: qpod/r-std
+    needs: qpod_base
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - run: echo '{"experimental":true}' | sudo tee /etc/docker/daemon.json && sudo service docker restart
-      - name: docker-R
-        run: |
+      - run: source ./tool.sh && build_image r-std latest docker_core/Dockerfile --build-arg "ARG_PROFILE_R=base,datascience" --build-arg "ARG_PROFILE_JAVA=base" && push_image
+
+  qpod_r-latex:
+    name: qpod/r-latex
+    needs: qpod_base
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - run: |
           source ./tool.sh
-          build_image r-mini    latest docker_core/Dockerfile \
-            --build-arg "ARG_PROFILE_R=base" \
-            && push_image
-          build_image r-std     latest docker_core/Dockerfile \
-            --build-arg "ARG_PROFILE_R=base,datascience" \
-            --build-arg "ARG_PROFILE_JAVA=base" \
-            && push_image
           build_image r-latex   latest docker_core/Dockerfile \
             --build-arg "ARG_PROFILE_R=base,datascience" \
             --build-arg "ARG_PROFILE_JAVA=base" \
-            --build-arg "ARG_PROFILE_LATEX=base,cjk" \
-            && push_image
+            --build-arg "ARG_PROFILE_LATEX=base,cjk"
+          push_image
+
+  qpod_r-studio:
+    name: qpod/r-studio
+    needs: qpod_base
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - run: |
+          source ./tool.sh
           build_image r-studio  latest docker_core/Dockerfile \
             --build-arg "ARG_PROFILE_R=base,datascience,rstudio,rshiny" \
             --build-arg "ARG_PROFILE_JAVA=base" \
-            --build-arg "ARG_PROFILE_LATEX=base,cjk" \
-            && push_image
+            --build-arg "ARG_PROFILE_LATEX=base,cjk"
+          push_image
 
-  build-other:
-    needs: build-base
+  qpod_node:
+    name: qpod/node
+    needs: qpod_base
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - run: echo '{"experimental":true}' | sudo tee /etc/docker/daemon.json && sudo service docker restart
-      - name: docker-other
-        run: |
-          source ./tool.sh
-          build_image node      latest docker_core/Dockerfile --build-arg "ARG_PROFILE_NODEJS=base" && push_image
-          build_image jdk       latest docker_core/Dockerfile --build-arg "ARG_PROFILE_JAVA=base"   && push_image
-          build_image julia     latest docker_core/Dockerfile --build-arg "ARG_PROFILE_JULIA=base"  && push_image
-          build_image go        latest docker_core/Dockerfile --build-arg "ARG_PROFILE_GO=base"     && push_image
+      - run: source ./tool.sh && build_image node latest docker_core/Dockerfile --build-arg "ARG_PROFILE_NODEJS=base" && push_image
+
+  qpod_jdk:
+    name: qpod/jdk
+    needs: qpod_base
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - run: source ./tool.sh && build_image jdk latest docker_core/Dockerfile --build-arg "ARG_PROFILE_JAVA=base"   && push_image
+
+  qpod_julia:
+    name: qpod/julia
+    needs: qpod_base
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - run: source ./tool.sh && build_image julia latest docker_core/Dockerfile --build-arg "ARG_PROFILE_JULIA=base"  && push_image
 
-  build-cuda-core:
-    needs: build-cuda-base
+  qpod_go:
+    name: qpod/go
+    needs: qpod_base
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - run: echo '{"experimental":true}' | sudo tee /etc/docker/daemon.json && sudo service docker restart
-      - name: docker-cuda-core
-        run: |
+      - run: source ./tool.sh && build_image go latest docker_core/Dockerfile --build-arg "ARG_PROFILE_GO=base"     && push_image
+
+  qpod_tf1:
+    name: qpod/tf1
+    needs: qpod_cuda_10
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - run: |
           source ./tool.sh
 
           build_image py-cuda-10.0 latest docker_core/Dockerfile \
             --build-arg "ARG_PROFILE_PYTHON=tf1,datascience,nlp,cv" \
             --build-arg "BASE_IMG=cuda_10.0"
           alias_image py-cuda-10.0 latest tf1 latest && push_image
 
+  qpod_core-cuda:
+    name: qpod/core-cuda
+    needs: qpod_cuda_11
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - run: |
+          source ./tool.sh
+
           build_image full-cuda-11.2 latest docker_core/Dockerfile \
             --build-arg "BASE_IMG=cuda_11.2" \
             --build-arg "ARG_PROFILE_PYTHON=base,datascience,database,nlp,cv,bio,chem,tf2,torch" \
@@ -162,28 +251,26 @@ jobs:
             --build-arg "ARG_PROFILE_LATEX=base,cjk"
           alias_image full-cuda-11.2 latest core-cuda latest && push_image
 
-  build-dev-core:
-    needs: build-core
+  qpod_core-dev:
+    name: qpod/core-dev
+    needs: qpod_core
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - run: echo '{"experimental":true}' | sudo tee /etc/docker/daemon.json && sudo service docker restart
-      - name: docker-other
-        run: |
+      - run: |
           source ./tool.sh
           build_image core-dev latest docker_dev/Dockerfile \
             --build-arg "ARG_PROFILE_JUPYTER=base,kernels,extensions" \
             --build-arg "ARG_PROFILE_VSCODE=base"
           alias_image core-dev latest full latest && push_image
 
-  build-dev-cuda:
-    needs: build-cuda-core
+  qpod_cuda-dev:
+    name: qpod/cuda-dev
+    needs: qpod_core-cuda
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - run: echo '{"experimental":true}' | sudo tee /etc/docker/daemon.json && sudo service docker restart
-      - name: docker-dev-cuda
-        run: |
+      - run: |
           source ./tool.sh
           build_image cuda-dev latest docker_dev/Dockerfile \
             --build-arg "BASE_IMG=core-cuda" \
diff --git a/README.md b/README.md
@@ -70,7 +70,7 @@ Typically, you can choose `full` / `full-cuda` if you have enough disk space and
 |             py-std            | https://hub.docker.com/r/qpod/py-std   | qpod/base | Python environment including all the packages mentioned above installed.                                                                                                                                                                                                                                 |
 |             py-jdk            | https://hub.docker.com/r/qpod/py-jdk   | qpod/base | `py-std` plus OpenJDK. (no LaTex)                                                                                                                                                                                                                                                                        |
 |             r-mini            | https://hub.docker.com/r/qpod/r-mini   | qpod/base | Minimal R environment -- no JDK, no R data science packages, no LaTex.                                                                                                                                                                                                                                   |
-|             r-std             | https://hub.docker.com/r/qpod/r-std    | qpod/base | Standard R environment for data science -- including popular R data science packages. (OpenJDK included since many R packages need Java, no LaTex.)                                                                                                                                                      |
+|             r-std             | https://hub.docker.com/r/qpod/r-std   | qpod/base | Standard R environment for data science -- including popular R data science packages. (OpenJDK included since many R packages need Java, no LaTex.)                                                                                                                                                      |
 |            r-latex            | https://hub.docker.com/r/qpod/r-latex  | qpod/base | `r-std` + LaTex -- this is the full R environment if you do not need RStudio.                                                                                                                                                                                                                            |
 |            r-studio           | https://hub.docker.com/r/qpod/r-studio | qpod/base | Full R environment if you want to use RStudio. `r-latex` + RStudio + RShiny.                                                                                                                                                                                                                             |
 |              node             | https://hub.docker.com/r/qpod/node     | qpod/base | Minimal NodeJS environment (including npm and yarn).                                                                                                                                                                                                                                                     |
@@ -173,8 +173,8 @@ Minor modifications are expected to port to `arm64`, `ppc64le` platform.
 
 Although `conda` is installed, we do not recommend to use conda to install a lib/package, because:
 
-- `conda` does not reuse the existing system library yet if a system lib is already installed -- `conda` installs it again.
-- `conda` does not provide a stable Linux system library repository yet, for example, some lib works well on `debian:jessie` but fail on `debian:stretch`.
+- `conda` repo mirrors are generally not avaliable in restricted enterprise LAN, especially in fincial/medical related companies.
+- `conda` does not reuse the existing system library yet if a system lib is already installed -- `conda` installs it again; and `conda` does not provide a stable Linux system library repository yet, for example, some lib works well on `debian:jessie` but fail on `debian:stretch`.
 
 ### Customization
 
diff --git a/docker_atom/work/script-setup.sh b/docker_atom/work/script-setup.sh
@@ -41,7 +41,9 @@ setup_nvtop() {
 
 
 setup_java_base() {
-      URL_OPENJDK=`curl -sL https://jdk.java.net/archive/ | grep 'linux-x64_bin.tar' | sed -n 's/.*href="\([^"]*\).*/\1/p' | head -n 1` \
+      VERSION_JDK=11 \
+   && URL_OPENJDK=`curl -sL https://jdk.java.net/archive/ | grep 'linux-x64_bin.tar' | grep -v sha256 | sed -n 's/.*href="\([^"]*\).*/\1/p' | grep "jdk${VERSION_JDK}" | head -n 1`  \
+   && echo "Installing JDK from: ${URL_OPENJDK}" \
    && install_tar_gz ${URL_OPENJDK} && mv /opt/jdk-* /opt/jdk \
    && ln -s /opt/jdk/bin/* /usr/bin/ \
    && echo "@ Version of Java (java/javac):" && java -version && javac -version
diff --git a/docker_dev/work/install_list_JPY_extend.pip b/docker_dev/work/install_list_JPY_extend.pip
@@ -3,8 +3,8 @@
 
 ipyparallel ipynb                       % IPython extensions
 nbresuse                                % Notebook extension
-jupyter_nbextensions_configurator       % Jupyter Notebook extension configurator
-jupyter_contrib_nbextensions            % Jupyter Notebook third-party extensions
+# jupyter_nbextensions_configurator       % Jupyter Notebook extension configurator
+# jupyter_contrib_nbextensions            % Jupyter Notebook third-party extensions
 
 jupyterlab_server                       % Jupyter Lab Server
 jupyterlab-git                          % Jupyter Lab Git extension
diff --git a/docker_dev/work/jupyter_notebook_config.json b/docker_dev/work/jupyter_notebook_config.json
@@ -1,9 +1,11 @@
 {
-  "NotebookApp": {
+  "ServerApp": {
     "ip": "0.0.0.0",
     "port": 8888,
+    "root_dir": "/root",
+    "default_url": "/lab",
     "allow_root": true,
-    "notebook_dir": "/root",
-    "default_url": "/lab"
+    "allow_origin": "*",
+    "open_browser": false
   }
 }
diff --git a/docker_dev/work/script-extend.sh b/docker_dev/work/script-extend.sh
diff --git a/tool.sh b/tool.sh

Original file line number	Diff line number	Diff line change
`@@ -1,9 +1,11 @@`
`1`	`1`	`{`
`2`		`- "NotebookApp": {`
	`2`	`+ "ServerApp": {`
`3`	`3`	`"ip": "0.0.0.0",`
`4`	`4`	`"port": 8888,`
	`5`	`+ "root_dir": "/root",`
	`6`	`+ "default_url": "/lab",`
`5`	`7`	`"allow_root": true,`
`6`		`- "notebook_dir": "/root",`
`7`		`- "default_url": "/lab"`
	`8`	`+ "allow_origin": "*",`
	`9`	`+ "open_browser": false`
`8`	`10`	`}`
`9`	`11`	`}`