diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 95bcb33a..504b552e 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -15,8 +15,12 @@ on: workflow_dispatch: env: + # DOCKER_REGISTRY_USER and DOCKER_REGISTRY_PASSWORD is required for docker image push, they should be set in CI secrets. DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }} DOCKER_REGISTRY_PASSWORD: ${{ secrets.DOCKER_REGISTRY_PASSWORD }} + REGISTRY_URL: "docker.io" # docker.io or other registry URL, DOCKER_REGISTRY_USER/DOCKER_REGISTRY_PASSWORD to be set in CI env. + BUILDKIT_PROGRESS: "plain" # Full logs for CI build. + jobs: qpod_base: diff --git a/docker_atom/work/script-setup-db-clients.sh b/docker_atom/work/script-setup-db-clients.sh index 73d4dd3a..889cdf36 100644 --- a/docker_atom/work/script-setup-db-clients.sh +++ b/docker_atom/work/script-setup-db-clients.sh @@ -4,8 +4,8 @@ source /opt/utils/script-utils.sh setup_postgresql_client() { local VER_PG=${VERSION_PG:-"14"} # from: https://www.postgresql.org/download/linux/ubuntu/ - echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list - curl https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add - + echo "deb https://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" | sudo tee /etc/apt/sources.list.d/pgdg.list + curl "https://www.postgresql.org/media/keys/ACCC4CF8.asc" | sudo apt-key add - sudo apt-get update # will download ~9MB files and use ~55MB disk after installation sudo apt-get -y install "postgresql-client-${VER_PG}" diff --git a/docker_atom/work/script-setup.sh b/docker_atom/work/script-setup.sh index 6ad78b4c..cf38d96b 100644 --- a/docker_atom/work/script-setup.sh +++ b/docker_atom/work/script-setup.sh @@ -121,12 +121,13 @@ setup_java_base() { && echo "@ Version of Java (java/javac):" && java -version && javac -version } + setup_java_maven() { - VERSION_MAVEN=$1; shift 1; VERSION_MAVEN=${VERSION_MAVEN:-"3.8.6"} \ + VERSION_MAVEN=$(curl -sL https://maven.apache.org/download.cgi | grep 'latest' | head -1 | grep -Po '\d[\d.]+') \ && install_zip "http://archive.apache.org/dist/maven/maven-3/${VERSION_MAVEN}/binaries/apache-maven-${VERSION_MAVEN}-bin.zip" \ && mv "/opt/apache-maven-${VERSION_MAVEN}" /opt/maven \ && ln -sf /opt/maven/bin/mvn* /usr/bin/ \ - && echo "@ Version of Maven:" && mvn --version + && echo "@ Version of Maven: $(mvn --version)" } @@ -220,7 +221,7 @@ setup_R_datascience() { setup_GO() { GO_VERSION=$(curl -sL https://github.com/golang/go/releases.atom | grep 'releases/tag' | head -1 | grep -Po '\d[\d.]+') \ - && GO_URL="https://dl.google.com/go/go$GO_VERSION.linux-$(dpkg --print-architecture).tar.gz" \ + && GO_URL="https://dl.google.com/go/go${GO_VERSION}.linux-$(dpkg --print-architecture).tar.gz" \ && install_tar_gz "${GO_URL}" go \ && ln -sf /opt/go/bin/go /usr/bin/ \ && echo "@ Version of golang: $(go version)" @@ -259,3 +260,21 @@ setup_rust() { && echo "@ Version of rustup: $(rustup --version)" \ && echo "@ Version of rustc: $(rustc --version)" } + + +setup_bazel() { + BAZEL_VERSION=$(curl -sL https://github.com/bazelbuild/bazel/releases.atom | grep 'releases/tag' | head -1 | grep -Po '\d[\d.]+' ) \ + && BAZEL_URL="https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/bazel-${BAZEL_VERSION}-installer-linux-x86_64.sh" \ + && curl -o /tmp/bazel.sh -sL "${BAZEL_URL}" && chmod +x /tmp/bazel.sh \ + && /tmp/bazel.sh && rm /tmp/bazel.sh \ + && echo "@ Version of bazel: $(bazel --version)" +} + + +setup_gradle() { + GRADLE_VERSION=$(curl -sL https://github.com/gradle/gradle/releases.atom | grep 'releases/tag' | grep -v 'M' | head -1 | grep -Po '\d[\d.]+' ) \ + && install_zip "https://downloads.gradle.org/distributions/gradle-${GRADLE_VERSION}-bin.zip" \ + && mv /opt/gradle* /opt/gradle \ + && ln -sf /opt/gradle/bin/gradle /usr/bin \ + && echo "@ Version of gradle: $(gradle --version)" +} diff --git a/docker_dev/work/script-extend.sh b/docker_dev/work/script-extend.sh index 63974096..5edc9df2 100644 --- a/docker_dev/work/script-extend.sh +++ b/docker_dev/work/script-extend.sh @@ -1,8 +1,7 @@ source /opt/utils/script-utils.sh setup_jupyter_base() { - # TEMP fix: nbconvert requires mistune>=0.8.1,<2 for now - pip install -Uq jupyterhub jupyterlab notebook ipywidgets qpod_hub "mistune>=0.8.1,<2" \ + pip install -Uq --pre jupyterhub jupyterlab notebook ipywidgets qpod_hub \ && jupyter nbextension enable --py widgetsnbextension \ && jupyter labextension install @jupyter-widgets/jupyterlab-manager \ && echo "@ Version of Jupyter Notebook/JupyterLab: $(jupyter notebook --version)" \ diff --git a/tool.sh b/tool.sh index b221cf18..52fd9b90 100755 --- a/tool.sh +++ b/tool.sh @@ -1,10 +1,6 @@ #!/bin/bash set -xu -export REGISTRY_URL="docker.io" # docker.io or other registry URL, DOCKER_REGISTRY_USER/DOCKER_REGISTRY_PASSWORD to be set in CI env. -export BUILDKIT_PROGRESS="plain" # Full logs for CI build. -# DOCKER_REGISTRY_USER and DOCKER_REGISTRY_PASSWORD is required for docker image push, they should be set in CI secrets. - CI_PROJECT_NAME=${GITHUB_REPOSITORY:-"QPod/docker-images"} CI_PROJECT_BRANCH=${GITHUB_HEAD_REF:-"main"} CI_PROJECT_SPACE=$(echo "${CI_PROJECT_BRANCH}" | cut -f1 -d'/') @@ -18,23 +14,29 @@ else fi export NAMESPACE=$(echo "${REGISTRY_URL:-"docker.io"}/${CI_PROJECT_NAMESPACE}" | awk '{print tolower($0)}') + echo "--------> CI_PROJECT_NAMESPACE=${CI_PROJECT_NAMESPACE}" -echo "--------> Docker Repo=${NAMESPACE}" +echo "--------> DOCKER_REGISTRY_NAMESPACE=${NAMESPACE}" + +if [ -f /etc/docker/daemon.json ]; then + jq '.experimental=true' /etc/docker/daemon.json > /tmp/daemon.json && sudo mv /tmp/daemon.json /etc/docker/ \ + && ( sudo service docker restart || true ) +else + docker info +fi -jq '.experimental=true' /etc/docker/daemon.json > /tmp/daemon.json && sudo mv /tmp/daemon.json /etc/docker/ -sudo service docker restart build_image() { echo "$@" ; IMG=$1; TAG=$2; FILE=$3; shift 3; VER=$(date +%Y.%m%d.%H%M); WORKDIR="$(dirname $FILE)"; - docker build --squash --compress --force-rm=true -t "${NAMESPACE}/${IMG}:${TAG}" -f "$FILE" --build-arg "BASE_NAMESPACE=${NAMESPACE}" "$@" "${WORKDIR}" ; + docker build --compress --force-rm=true -t "${NAMESPACE}/${IMG}:${TAG}" -f "$FILE" --build-arg "BASE_NAMESPACE=${NAMESPACE}" "$@" "${WORKDIR}" ; docker tag "${NAMESPACE}/${IMG}:${TAG}" "${NAMESPACE}/${IMG}:${VER}" ; } build_image_no_tag() { echo "$@" ; IMG=$1; TAG=$2; FILE=$3; shift 3; WORKDIR="$(dirname $FILE)"; - docker build --squash --compress --force-rm=true -t "${NAMESPACE}/${IMG}:${TAG}" -f "$FILE" --build-arg "BASE_NAMESPACE=${NAMESPACE}" "$@" "${WORKDIR}" ; + docker build --compress --force-rm=true -t "${NAMESPACE}/${IMG}:${TAG}" -f "$FILE" --build-arg "BASE_NAMESPACE=${NAMESPACE}" "$@" "${WORKDIR}" ; } build_image_common() {