Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Update PaddlePaddle cuda version #68

Merged
merged 1 commit into from
Aug 29, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 25 additions & 14 deletions .github/workflows/docker.yml
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ on:
branches: [ main ]
paths-ignore:
- "*.md"

pull_request:
branches: [ main ]
paths-ignore:
Expand Down Expand Up @@ -216,8 +216,8 @@ jobs:
- uses: actions/checkout@v2
- run: |
source ./tool.sh
build_image tf1-cu100 latest docker_core/Dockerfile --build-arg "BASE_IMG=cuda_10.0" --build-arg "ARG_PROFILE_PYTHON=tf1"
alias_image tf1-cu100 latest tf1 latest
build_image tf1-cuda100 latest docker_core/Dockerfile --build-arg "BASE_IMG=cuda_10.0" --build-arg "ARG_PROFILE_PYTHON=tf1"
alias_image tf1-cuda100 latest tf1 latest
push_image

qpod_tf2:
Expand All @@ -228,43 +228,54 @@ jobs:
- uses: actions/checkout@v2
- run: |
source ./tool.sh
build_image tf2-cu112 latest docker_core/Dockerfile --build-arg "BASE_IMG=cuda_11.2" --build-arg "ARG_PROFILE_PYTHON=tf2"
alias_image tf2-cu112 latest torch latest
build_image tf2-cuda112 latest docker_core/Dockerfile --build-arg "BASE_IMG=cuda_11.2" --build-arg "ARG_PROFILE_PYTHON=tf2"
alias_image tf2-cuda112 latest torch latest
push_image

qpod_torch_cu116:
qpod_torch_cuda116:
name: qpod/torch-cuda116
needs: qpod_cuda_11_6
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- run: |
source ./tool.sh
build_image torch-cu116 latest docker_core/Dockerfile --build-arg "BASE_IMG=cuda_11.6" --build-arg "ARG_PROFILE_PYTHON=torch"
alias_image torch-cu116 latest torch latest
build_image torch-cuda116 latest docker_core/Dockerfile --build-arg "BASE_IMG=cuda_11.6" --build-arg "ARG_PROFILE_PYTHON=torch"
alias_image torch-cuda116 latest torch latest
push_image

qpod_torch_cu102:
name: qpod/torch-cu102
qpod_torch_cuda102:
name: qpod/torch-cuda102
needs: qpod_cuda_10_2
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- run: |
source ./tool.sh
build_image torch-cu102 latest docker_core/Dockerfile --build-arg "BASE_IMG=cuda_10.2" --build-arg "ARG_PROFILE_PYTHON=torch"
build_image torch-cuda102 latest docker_core/Dockerfile --build-arg "BASE_IMG=cuda_10.2" --build-arg "ARG_PROFILE_PYTHON=torch"
push_image

qpod_paddle_cu116:
qpod_paddle_cuda116:
name: qpod/paddle-cuda116
needs: qpod_cuda_11_6
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- run: |
source ./tool.sh
build_image paddle-cu116 latest docker_core/Dockerfile --build-arg "BASE_IMG=cuda_11.6" --build-arg "ARG_PROFILE_PYTHON=paddle"
alias_image paddle-cu116 latest paddle latest
build_image paddle-cuda116 latest docker_core/Dockerfile --build-arg "BASE_IMG=cuda_11.6" --build-arg "ARG_PROFILE_PYTHON=paddle"
alias_image paddle-cuda116 latest paddle latest
push_image

qpod_paddle_cuda102:
name: qpod/paddle-cuda102
needs: qpod_cuda_10_2
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- run: |
source ./tool.sh
build_image paddle-cuda102 latest docker_core/Dockerfile --build-arg "BASE_IMG=cuda_10.2" --build-arg "ARG_PROFILE_PYTHON=paddle"
push_image


Expand Down
45 changes: 22 additions & 23 deletions docker_atom/work/script-setup.sh
Original file line number Diff line number Diff line change
Expand Up @@ -24,17 +24,17 @@ setup_conda_postprocess() {
&& conda update --all --quiet --yes

# These conda pkgs shouldn't be removed (otherwise will cause RemoveError) since they are directly reqiuired by conda: pip setuptools pycosat pyopenssl requests ruamel_yaml
CONDA_PY_PKGS=`conda list | grep "py3" | cut -d " " -f 1 | sed "/#/d;/conda/d;/pip/d;/setuptools/d;/pycosat/d;/pyopenssl/d;/requests/d;/ruamel_yaml/d;"` \
&& conda remove --force -yq $CONDA_PY_PKGS \
&& pip install -UIq pip setuptools $CONDA_PY_PKGS
CONDA_PY_PKGS=$(conda list | grep "py3" | cut -d " " -f 1 | sed "/#/d;/conda/d;/pip/d;/setuptools/d;/pycosat/d;/pyopenssl/d;/requests/d;/ruamel_yaml/d;") \
&& conda remove --force -yq "${CONDA_PY_PKGS}" \
&& pip install -UIq pip setuptools "${CONDA_PY_PKGS}"

# Print Conda and Python packages information in the docker build log
echo "@ Version of Conda & Python:" && conda info && conda list | grep -v "<pip>"
}

setup_conda_with_mamba() {
VERSION_PTYHON=$1; shift 1;
mamba install -y --root-prefix="${CONDA_PREFIX}" --prefix="${CONDA_PREFIX}" -c "conda-forge" conda pip python="${VERSION_PTYHON:-3.10}"
VERSION_PYTHON=$1; shift 1;
mamba install -y --root-prefix="${CONDA_PREFIX}" --prefix="${CONDA_PREFIX}" -c "conda-forge" conda pip python="${VERSION_PYTHON:-3.10}"
rm -rf /opt/conda/pkgs/*
setup_conda_postprocess
}
Expand All @@ -59,27 +59,27 @@ setup_nvtop() {
# Install Utilities `nvtop`
sudo apt-get -qq update --fix-missing && sudo apt-get -qq install -y --no-install-recommends libncurses5-dev

DIRECTORY=`pwd`
DIRECTORY=$(pwd)

cd /tmp \
&& git clone https://github.com/Syllo/nvtop.git \
&& mkdir -pv nvtop/build && cd nvtop/build \
&& LIB_PATH=`find / -name "libnvidia-ml*" 2>/dev/null` \
&& cmake .. -DCMAKE_LIBRARY_PATH="`dirname $LIB_PATH`" .. \
&& LIB_PATH=$(find / -name "libnvidia-ml*" 2>/dev/null) \
&& cmake .. -DCMAKE_LIBRARY_PATH="$(dirname ${LIB_PATH})" .. \
&& make && sudo make install \
&& nvtop --version

cd $DIRECTORY && rm -rf /tmp/nvtop
cd "${DIRECTORY}" && rm -rf /tmp/nvtop

sudo apt-get -qq remove -y libncurses5-dev
}


setup_java_base() {
VERSION_JDK=11 \
&& URL_OPENJDK=`curl -sL https://jdk.java.net/archive/ | grep 'linux-x64_bin.tar' | grep -v sha256 | sed -n 's/.*href="\([^"]*\).*/\1/p' | grep "jdk${VERSION_JDK}" | head -n 1` \
&& URL_OPENJDK=$(curl -sL https://jdk.java.net/archive/ | grep 'linux-x64_bin.tar' | grep -v sha256 | sed -n 's/.*href="\([^"]*\).*/\1/p' | grep "jdk${VERSION_JDK}" | head -n 1) \
&& echo "Installing JDK from: ${URL_OPENJDK}" \
&& install_tar_gz ${URL_OPENJDK} && mv /opt/jdk-* /opt/jdk \
&& install_tar_gz "${URL_OPENJDK}" && mv /opt/jdk-* /opt/jdk \
&& ln -sf /opt/jdk/bin/* /usr/bin/ \
&& echo "@ Version of Java (java/javac):" && java -version && javac -version
}
Expand All @@ -104,8 +104,8 @@ setup_node() {
&& export PATH=/opt/node/bin:$PATH \
&& npm install -g npm yarn \
&& ln -sf /opt/node/bin/* /usr/bin/ \
&& echo "@ Version of Node, npm, and yarn:" `node -v` `npm -v` \
&& echo "@ Version of Yarn:" `yarn -v`
&& echo "@ Version of Node, npm, and yarn: $(node -v) $(npm -v)" \
&& echo "@ Version of Yarn: $(yarn -v)"
}


Expand All @@ -116,7 +116,7 @@ setup_R_base() {
&& echo "options(repos=structure(c(CRAN=\"https://cloud.r-project.org\")))" >> /etc/R/Rprofile.site \
&& R -e "install.packages(c('devtools'),clean=T,quiet=T);" \
&& ( type java && type R && R CMD javareconf || true ) \
&& echo "@ Version of R:" && R --version
&& echo "@ Version of R: $(R --version)"
}


Expand Down Expand Up @@ -147,7 +147,7 @@ setup_R_rstudio() {
# Remove RStudio's pandoc and pandoc-proc to reduce size if they are already installed in the jpy-latex step.
( which pandoc && rm /opt/rstudio-server/bin/pandoc/pandoc || true ) \
&& ( which pandoc-citeproc && rm /opt/rstudio-server/bin/pandoc/pandoc-citeproc || true ) \
&& echo "@ Version of rstudio-server:" && rstudio-server version
&& echo "@ Version of rstudio-server: $(rstudio-server version)"
}


Expand All @@ -168,7 +168,7 @@ setup_R_rshiny() {

# hack shiny-server to allow run in root user: https://github.com/rstudio/shiny-server/pull/391
sed -i "s/throw new Error/logger.warn/g" /opt/shiny-server/lib/worker/app-worker.js \
&& echo "@ Version of shiny-server:" && shiny-server --version
&& echo "@ Version of shiny-server: $(shiny-server --version)"
}


Expand All @@ -184,9 +184,9 @@ setup_R_datascience() {
setup_GO() {
GO_VERSION=$(curl -sL https://github.com/golang/go/releases.atom | grep 'releases/tag' | head -1 | grep -Po '\d[\d.]+') \
&& GO_URL="https://dl.google.com/go/go$GO_VERSION.linux-$(dpkg --print-architecture).tar.gz" \
&& install_tar_gz $GO_URL go \
&& install_tar_gz "${GO_URL}" go \
&& ln -sf /opt/go/bin/go /usr/bin/ \
&& echo "@ Version of golang:" && go version
&& echo "@ Version of golang: $(go version)"
}


Expand All @@ -198,7 +198,7 @@ setup_julia() {
&& mkdir -p /opt/julia/pkg \
&& echo "import Libdl; push!(Libdl.DL_LOAD_PATH, \"/opt/conda/lib\")" >> /opt/julia/etc/julia/startup.jl \
&& echo "DEPOT_PATH[1]=\"/opt/julia/pkg\"" >> /opt/julia/etc/julia/startup.jl \
&& echo "@ Version of Julia" && julia --version
&& echo "@ Version of Julia: $(julia --version)"
}


Expand All @@ -215,15 +215,14 @@ setup_octave() {

install_apt /opt/utils/install_list_octave.apt \
&& install_octave /opt/utils/install_list_octave.pkg \
&& echo "@ Version of Octave and installed packages:" \
&& /opt/octave/bin/octave --version
&& echo "@ Version of Octave and installed packages: $(/opt/octave/bin/octave --version)"
}


setup_traefik() {
TRAEFIK_VERSION=$(curl -sL https://github.com/traefik/traefik/releases.atom | grep 'releases/tag' | head -1 | grep -Po '\d[\d.]+') \
&& TRAEFIK_URL="https://github.com/traefik/traefik/releases/download/v${TRAEFIK_VERSION}/traefik_v${TRAEFIK_VERSION}_linux_$(dpkg --print-architecture).tar.gz" \
&& install_tar_gz $TRAEFIK_URL traefik \
&& install_tar_gz "${TRAEFIK_URL}" traefik \
&& ln -sf /opt/traefik /usr/bin/ \
&& echo "@ Version of traefik:" && traefik version
&& echo "@ Version of traefik: $(traefik version)"
}
2 changes: 1 addition & 1 deletion docker_atom/work/script-utils.sh
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ install_conda() { cat $1 | cut -d "%" -f 1 | sed '/^$/d' | xargs -r -n1 conda
install_mamba() { cat $1 | cut -d "%" -f 1 | sed '/^$/d' | xargs -r -n1 mamba install -yq --root-prefix="${CONDA_PREFIX}" --prefix="${CONDA_PREFIX}" ; }

# function to install python packages with pip from a text file which lists package names (add comments with % char)
install_pip() { cat $1 | cut -d "%" -f 1 | sed '/^$/d' | xargs -r -n1 pip install -U --pre ; }
install_pip() { cat $1 | cut -d "%" -f 1 | sed '/^$/d' | xargs -r -n1 pip install -U --root-user-action=ignore --pre ; }

# function to install R packages from a text file which lists package names (add comments with % char, use quiet=T to be less verbose)
install_R() { R -e "options(Ncpus=4);lapply(scan('$1','c',comment.char='%'),function(x){cat(x,system.time(install.packages(x,clean=T,quiet=T)),'\n')})"; }
Expand Down
2 changes: 1 addition & 1 deletion docker_core/work/install_list_PY_datascience.pip
Original file line number Diff line number Diff line change
Expand Up @@ -7,5 +7,5 @@ sortedcontainers sortedcollections % Enhanced Data Structur
lxml beautifulsoup4 pyquery numpydoc sphinx % Parse/Build documents
pandas pyarrow xlrd openpyxl beautifulsoup4 % Data preparation
patsy statsmodels sympy pywavelets numexpr pycrypto % Statistics, Maths, and related Utilities
scikit-learn scipy mlflow % Statistical ML, Data Mining
scikit-learn scipy % Statistical ML, Data Mining
bokeh seaborn altair vincent % Data visualization
8 changes: 4 additions & 4 deletions docker_dev/work/script-extend.sh
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,8 @@ setup_jupyter_base() {
pip install -Uq jupyterhub jupyterlab notebook ipywidgets qpod_hub "mistune>=0.8.1,<2" \
&& jupyter nbextension enable --py widgetsnbextension \
&& jupyter labextension install @jupyter-widgets/jupyterlab-manager \
&& echo "@ Version of Jupyter Notebook/JupyterLab:" \
`jupyter notebook --version` `jupyter lab --version`
&& echo "@ Version of Jupyter Notebook/JupyterLab: $(jupyter notebook --version)" \
&& echo "@ Version of Jupyter Notebook/JupyterLab: $(jupyter lab --version)"
}


Expand Down Expand Up @@ -74,7 +74,7 @@ setup_vscode_base() {
&& install_tar_gz "https://github.com/cdr/code-server/releases/download/v${VERSION_CODER}/code-server-${VERSION_CODER}-linux-amd64.tar.gz" \
&& mv /opt/code-server* /opt/code-server \
&& ln -s /opt/code-server/bin/code-server /usr/bin/ \
&& printf "#!/bin/bash\n/opt/code-server/bin/code-server --port=8888 --auth=none --disable-telemetry $HOME\n" > /usr/local/bin/start-code-server.sh \
&& printf "#!/bin/bash\n/opt/code-server/bin/code-server --port=8888 --auth=none --disable-telemetry ${HOME}\n" > /usr/local/bin/start-code-server.sh \
&& chmod u+x /usr/local/bin/start-code-server.sh \
&& echo "@ coder-server Version:" && code-server -v
&& echo "@ coder-server Version: $(code-server -v)"
}