From e272ef61d3f1374dceeaf6e0cf876c6743ff1950 Mon Sep 17 00:00:00 2001 From: Bibo Hao Date: Mon, 21 Nov 2022 17:09:03 +0800 Subject: [PATCH 1/3] add db-client scripts --- docker_atom/work/script-setup-db-clients.sh | 42 +++++++++++++++++++++ docker_atom/work/script-setup.sh | 7 ++-- docker_atom/work/script-utils.sh | 16 ++++---- docker_cuda/Dockerfile | 3 ++ 4 files changed, 57 insertions(+), 11 deletions(-) create mode 100644 docker_atom/work/script-setup-db-clients.sh diff --git a/docker_atom/work/script-setup-db-clients.sh b/docker_atom/work/script-setup-db-clients.sh new file mode 100644 index 00000000..73d4dd3a --- /dev/null +++ b/docker_atom/work/script-setup-db-clients.sh @@ -0,0 +1,42 @@ +source /opt/utils/script-utils.sh + + +setup_postgresql_client() { + local VER_PG=${VERSION_PG:-"14"} + # from: https://www.postgresql.org/download/linux/ubuntu/ + echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list + curl https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add - + sudo apt-get update + # will download ~9MB files and use ~55MB disk after installation + sudo apt-get -y install "postgresql-client-${VER_PG}" + echo "@ Version of psql client: $(psql --version)" +} + + +setup_mysql_client() { + sudo apt-get update + # will download ~5MB files and use ~76MB disk after installation + sudo apt-get -y install mysql-client + echo "@ Version of mysql client: $(mysql --version)" +} + + +setup_mongosh_client() { + # from: https://www.mongodb.com/docs/mongodb-shell/install/ + echo "deb [ arch=amd64,arm64 ] https://repo.mongodb.org/apt/ubuntu $(lsb_release -cs)/mongodb-org/6.0 multiverse" > /etc/apt/sources.list.d/mongodb-org-6.0.list + curl -sL https://www.mongodb.org/static/pgp/server-6.0.asc | sudo apt-key add - + sudo apt-get update + # will download ~38MB files and use ~218MB disk after installation + sudo apt-get -y install mongodb-mongosh + echo "@ Version of mongosh client: $(mongosh --version)" +} + + +setup_redis_client() { + # from https://redis.io/docs/getting-started/installation/install-redis-on-linux/ + curl -fsSL https://packages.redis.io/gpg | sudo gpg --dearmor -o /usr/share/keyrings/redis-archive-keyring.gpg + echo "deb [signed-by=/usr/share/keyrings/redis-archive-keyring.gpg] https://packages.redis.io/deb $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/redis.list + sudo apt-get update + sudo apt-get -y install redis-tools + echo "@ Version of redis-cli: $(redis-cli --version)" +} diff --git a/docker_atom/work/script-setup.sh b/docker_atom/work/script-setup.sh index a3ec3270..5fa199e4 100644 --- a/docker_atom/work/script-setup.sh +++ b/docker_atom/work/script-setup.sh @@ -29,7 +29,7 @@ trusted-host=pypi.python.org pypi.org files.pythonhosted.org EOF fi - echo "export PATH=$PATH:${CONDA_PREFIX}/bin" >> /etc/profile + echo 'export PATH=${PATH}:${CONDA_PREFIX:-"/opt/conda"}/bin' >> /etc/profile ln -sf "${CONDA_PREFIX}/bin/conda" /usr/bin/ conda config --system --prepend channels conda-forge \ @@ -53,9 +53,10 @@ EOF } setup_conda_with_mamba() { - mkdir -pv "${CONDA_PREFIX}" + local PREFIX="${CONDA_PREFIX:-/opt/conda}" + mkdir -pv "${PREFIX}" VERSION_PYTHON=${1:-"3.10"}; shift 1; - mamba install -y --root-prefix="${CONDA_PREFIX}" --prefix="${CONDA_PREFIX}" -c "conda-forge" conda pip python="${VERSION_PYTHON}" + mamba install -y --root-prefix="${PREFIX}" --prefix="${PREFIX}" -c "conda-forge" conda pip python="${VERSION_PYTHON}" setup_conda_postprocess } diff --git a/docker_atom/work/script-utils.sh b/docker_atom/work/script-utils.sh index 6bed357f..a4bfe925 100644 --- a/docker_atom/work/script-utils.sh +++ b/docker_atom/work/script-utils.sh @@ -4,7 +4,7 @@ set -x install_echo() { cat $1 | cut -d "%" -f 1 | sed '/^$/d' | xargs -r -n1 printf '%s\n' ; } # function to install apt-get packages from a text file which lists package names (add comments with % char) -install_apt() { apt-get -qq update -yq --fix-missing && apt-get -qq install -yq --no-install-recommends `cat $1 | cut -d '%' -f 1` ; } +install_apt() { apt-get -qq update -yq --fix-missing && apt-get -qq install -yq --no-install-recommends $(cat "$1" | cut -d '%' -f 1) ; } # function to install conda packages from a text file which lists package names (add comments with % char) install_conda() { cat $1 | cut -d "%" -f 1 | sed '/^$/d' | xargs -r -n1 conda install -yq ; } @@ -44,8 +44,8 @@ install_mvn() { cat $1 | cut -d "%" -f 1 | xargs -r -n1 -I {} mvn dependency:cop install__clean(){ which apt-get && apt-get autoremove -y && apt-get clean && rm -rf /var/lib/apt/lists/* which mamba && mamba clean -ya && rm -rf ~/micromamba - which conda && conda clean -ya && ( rm -rf /opt/conda/pkgs/* || true ) - find /opt/conda/lib | grep -E "(__pycache__|\.pyc|\.pyo$)" | xargs rm -rf + which conda && conda clean -ya && ( rm -rf "${CONDA_PREFIX:-/opt/conda}"/pkgs/* || true ) + find "${CONDA_PREFIX:-/opt/conda}"/lib | grep -E "(__pycache__|\.pyc|\.pyo$)" | xargs rm -rf which npm && npm cache clean --force rm -rf /opt/conda/share/jupyter/lab/staging ( rm -rf /tmp/.* /tmp/* /var/log/* /var/cache/* /root/.cache /root/.* || true ) @@ -72,21 +72,21 @@ list_installed_packages() { } fix_permission() { - DIRECTORY=$1; GROUP_ID=$2; shift 2; - for d in "$DIRECTORY"; do + GROUP_ID=${1:-0}; shift 1; + for d in "$@"; do find "${d}" \ ! \( \ -group "${GROUP_ID}" \ -a -perm -g+rwX \ \) \ - -exec chgrp "${GROUP_ID}" {} \; \ - -exec chmod g+rwX {} \; + -exec chgrp "${GROUP_ID}" -- {} \+ \ + -exec chmod g+rwX -- {} \+ # setuid, setgid *on directories only* find "${d}" \ \( \ -type d \ -a ! -perm -6000 \ \) \ - -exec chmod +6000 {} \; + -exec chmod +6000 -- {} \+; done } diff --git a/docker_cuda/Dockerfile b/docker_cuda/Dockerfile index 366f4f09..279ae2e7 100644 --- a/docker_cuda/Dockerfile +++ b/docker_cuda/Dockerfile @@ -6,6 +6,9 @@ FROM ${BASE_NAMESPACE:+$BASE_NAMESPACE/}${BASE_IMG} LABEL maintainer="haobibo@gmail.com" +# Let NVIDIA docker ignore cuda requirement check +ENV NVIDIA_DISABLE_REUIRE=1 + # For cuda version 10.0, the image is solely serverd for legacy tensorflow 1.15, which requires python 3.7 # For tensorflow 2.x or torch, python>=3.9 is supported. RUN echo ${CUDA_VERSION} && nvcc --version \ From 015715e6a85b98ea88561841458792bfc5d73a09 Mon Sep 17 00:00:00 2001 From: Bibo Hao Date: Tue, 22 Nov 2022 23:24:08 +0800 Subject: [PATCH 2/3] Update Dockerfile --- docker_base/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/docker_base/Dockerfile b/docker_base/Dockerfile index b83167d1..e665b3ab 100644 --- a/docker_base/Dockerfile +++ b/docker_base/Dockerfile @@ -15,6 +15,7 @@ RUN source /opt/utils/script-setup.sh \ && echo "Install tini:" && setup_tini \ && echo "Install Mamba:" && setup_mamba \ && echo "Install Python ${PYTHON_VERSION} and conda:" && setup_conda_with_mamba ${PYTHON_VERSION} \ + && fix-permissions 0 ${CONDA_PREFIX} \ && echo "Replace system Python3 with conda Python - note that /bin and /sbin are symlinks of /usr/bin in docker image ubuntu" \ && PYTHON_VERSION_DEFAULT=$(py3versions -v -i) \ && sed -i "s/${PYTHON_VERSION_DEFAULT}/${PYTHON_VERSION}/g" /usr/share/python3/debian_defaults \ From eeea085767e0dd628141ae63d996973fc0df8918 Mon Sep 17 00:00:00 2001 From: Bibo Hao Date: Tue, 22 Nov 2022 23:30:49 +0800 Subject: [PATCH 3/3] Update Dockerfile --- docker_base/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker_base/Dockerfile b/docker_base/Dockerfile index e665b3ab..3f9f4cfe 100644 --- a/docker_base/Dockerfile +++ b/docker_base/Dockerfile @@ -15,7 +15,7 @@ RUN source /opt/utils/script-setup.sh \ && echo "Install tini:" && setup_tini \ && echo "Install Mamba:" && setup_mamba \ && echo "Install Python ${PYTHON_VERSION} and conda:" && setup_conda_with_mamba ${PYTHON_VERSION} \ - && fix-permissions 0 ${CONDA_PREFIX} \ + && fix_permission 0 ${CONDA_PREFIX} \ && echo "Replace system Python3 with conda Python - note that /bin and /sbin are symlinks of /usr/bin in docker image ubuntu" \ && PYTHON_VERSION_DEFAULT=$(py3versions -v -i) \ && sed -i "s/${PYTHON_VERSION_DEFAULT}/${PYTHON_VERSION}/g" /usr/share/python3/debian_defaults \