From 819e881a1686cd0a342dc4da56451561706df7be Mon Sep 17 00:00:00 2001
From: Bibo Hao <haobibo@users.noreply.github.com>
Date: Fri, 1 Apr 2022 15:15:23 +0800
Subject: [PATCH 1/2] fix torch install
---
.github/workflows/docker.yml | 28 +++++++++++++++++++++++-
docker_atom/work/script-setup.sh | 3 ++-
docker_core/Dockerfile | 7 +++---
docker_core/work/install_list_PY_nlp.pip | 5 ++---
4 files changed, 35 insertions(+), 8 deletions(-)
diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
index 8b36f8f..d945b23 100644
--- a/.github/workflows/docker.yml
+++ b/.github/workflows/docker.yml
@@ -234,10 +234,36 @@ jobs:
- run: |
source ./tool.sh
build_image py-cuda-10.0 latest docker_core/Dockerfile \
- --build-arg "ARG_PROFILE_PYTHON=tf1,datascience,nlp,cv" \
+ --build-arg "ARG_PROFILE_PYTHON=tf1" \
--build-arg "BASE_IMG=cuda_10.0"
alias_image py-cuda-10.0 latest tf1 latest && push_image
+ qpod_tf2:
+ name: qpod/tf2
+ needs: qpod_cuda_11_2
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v2
+ - run: |
+ source ./tool.sh
+ build_image py-cuda-11.2 latest docker_core/Dockerfile \
+ --build-arg "ARG_PROFILE_PYTHON=tf2" \
+ --build-arg "BASE_IMG=cuda_11.2"
+ alias_image py-cuda-11.2 latest tf2 latest && push_image
+
+ qpod_torch:
+ name: qpod/torch
+ needs: qpod_cuda_11_3
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v2
+ - run: |
+ source ./tool.sh
+ build_image py-cuda-11.3 latest docker_core/Dockerfile \
+ --build-arg "ARG_PROFILE_PYTHON=torch" \
+ --build-arg "BASE_IMG=cuda_11.3"
+ alias_image py-cuda-11.3 latest torch latest && push_image
+
qpod_core-cuda:
name: qpod/core-cuda
needs: qpod_cuda_11_3
diff --git a/docker_atom/work/script-setup.sh b/docker_atom/work/script-setup.sh
index 46e24eb..09f886b 100644
--- a/docker_atom/work/script-setup.sh
+++ b/docker_atom/work/script-setup.sh
@@ -33,7 +33,8 @@ setup_conda_postprocess() {
}
setup_conda_with_mamba() {
- mamba install -y --root-prefix="${CONDA_PREFIX}" --prefix="${CONDA_PREFIX}" conda pip -c "conda-forge/linux-64"
+ VERSION_PTYHON=$1; shift 1;
+ mamba install -y --root-prefix="${CONDA_PREFIX}" --prefix="${CONDA_PREFIX}" -c "conda-forge" conda pip python="${VERSION_PTYHON:-3.9}"
rm -rf /opt/conda/pkgs/*
setup_conda_postprocess
}
diff --git a/docker_core/Dockerfile b/docker_core/Dockerfile
index 6f6cd28..ab1c27b 100644
--- a/docker_core/Dockerfile
+++ b/docker_core/Dockerfile
@@ -54,15 +54,16 @@ RUN source /opt/utils/script-utils.sh \
&& ( which java && echo "py4j % Install py4j if Java exists" >> /opt/utils/install_list_PY_datascience.pip || true ) \
|| true ) \
&& for profile in $(echo $ARG_PROFILE_PYTHON | tr "," "\n") ; do ( install_pip "/opt/utils/install_list_PY_${profile}.pip" || true ) ; done \
- # Handle tensorflow installation 1.x/2.x, cpu/gpu
+ # Handle tensorflow installation 1.x/2.x, cpu/gpu: https://www.tensorflow.org/install/source#gpu
&& ( [[ ${ARG_PROFILE_PYTHON} == *"tf"* ]] \
&& TF=$( [ -x "$(command -v nvcc)" ] && echo "tensorflow-gpu" || echo "tensorflow") \
&& V=$([[ ${ARG_PROFILE_PYTHON} == *"tf1"* ]] && echo "1" || echo "2" ) \
&& pip install --pre -U "${TF}==${V}.*" \
|| true ) \
- # Handle pytorch installation 1.x only, cpu/gpu
+ # Handle pytorch installation 1.x only, cpu/gpu: https://pytorch.org/get-started/locally/
&& ( [[ ${ARG_PROFILE_PYTHON} == *"torch"* ]] \
- && IDX=$( [ -x "$(command -v nvcc)" ] && echo "cu${CUDA_VER//./}" || echo "cpu" ) \
+ && CUDA_VER=$(echo "${CUDA_VERSION:0:4}" | sed 's/\.//' ) \
+ && IDX=$( [ -x "$(command -v nvcc)" ] && echo "cu${CUDA_VER:-113}" || echo "cpu" ) \
&& pip install --pre -U torch -f "https://download.pytorch.org/whl/${IDX}/torch_stable.html" \
|| true )
diff --git a/docker_core/work/install_list_PY_nlp.pip b/docker_core/work/install_list_PY_nlp.pip
index 9101bdc..a1c20f0 100644
--- a/docker_core/work/install_list_PY_nlp.pip
+++ b/docker_core/work/install_list_PY_nlp.pip
@@ -1,6 +1,5 @@
% This file contains python packages to be installed with pip line by line.
% Use percent char as line comment separator.
-pyparsing astroid % Parsing, AST
-nltk spacy PyStemmer % NLP
-transformers % transformers
+pyparsing astroid nltk % Parsing, AST
+transformers datasets % transformers
From 17a4ed6d750848d5daa2fc0a7d9bffceb7da5dbe Mon Sep 17 00:00:00 2001
From: Bibo Hao <haobibo@users.noreply.github.com>
Date: Fri, 1 Apr 2022 17:38:50 +0800
Subject: [PATCH 2/2] remove NodeJS from default imgs
---
docker_core/Dockerfile | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/docker_core/Dockerfile b/docker_core/Dockerfile
index ab1c27b..ce647db 100644
--- a/docker_core/Dockerfile
+++ b/docker_core/Dockerfile
@@ -7,7 +7,7 @@ FROM ${BASE_NAMESPACE:+$BASE_NAMESPACE/}${BASE_IMG}
LABEL maintainer="haobibo@gmail.com"
-ARG ARG_PROFILE_NODEJS=base
+ARG ARG_PROFILE_NODEJS
ARG ARG_PROFILE_JAVA