Split libvcx image build in 2 phases in CI (#174)

Signed-off-by: Patrik Stas <patrik.stas@absa.africa>
hyperledger · Nov 3, 2020 · 5407e99 · 5407e99
1 parent bca8470
commit 5407e99
Show file tree

Hide file tree

Showing 4 changed files with 111 additions and 52 deletions.
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -17,11 +17,13 @@ jobs:
  CACHE_KEY_ANDROID: ${{ steps.mainstep.outputs.CACHE_KEY_ANDROID }}
  CACHE_KEY_LIBVCX: ${{ steps.mainstep.outputs.CACHE_KEY_LIBVCX }}
  CACHE_KEY_CODECOV: ${{ steps.mainstep.outputs.CACHE_KEY_CODECOV }}
+ CACHE_KEY_ALPINE_CORE: ${{ steps.mainstep.outputs.CACHE_KEY_ALPINE_CORE }}
  DOCKER_IMG_NAME_POOL: ${{ steps.mainstep.outputs.DOCKER_IMG_NAME_POOL }}
  DOCKER_IMG_NAME_ANDROID: ${{ steps.mainstep.outputs.DOCKER_IMG_NAME_ANDROID }}
  DOCKER_IMG_NAME_AGENCY: ${{ steps.mainstep.outputs.DOCKER_IMG_NAME_AGENCY }}
  DOCKER_IMG_NAME_LIBVCX: ${{ steps.mainstep.outputs.DOCKER_IMG_NAME_LIBVCX }}
  DOCKER_IMG_NAME_CODECOV: ${{ steps.mainstep.outputs.DOCKER_IMG_NAME_CODECOV }}
+ DOCKER_IMG_NAME_ALPINE_CORE: ${{ steps.mainstep.outputs.DOCKER_IMG_NAME_ALPINE_CORE }}
  PUBLISH_VERSION: ${{ steps.mainstep.outputs.PUBLISH_VERSION }}
  RELEASE: ${{ steps.mainstep.outputs.RELEASE }}
  steps:
@@ -95,6 +97,7 @@ jobs:
  LIBVCX_NODE_AGENT_HASH=${{ hashFiles('agents/node') }}
  JAVA_WRAPPERS_HASH=${{ hashFiles('wrappers/java') }}
  POOL_DOCKERFILE_HASH=${{ hashFiles('ci/indy-pool.dockerfile')}}
+ ALPINE_CORE_HASH=${{ hashFiles('ci/alpine_core.dockerfile')}}
 
  LIBVCX_HASH=${LIBVCX_SOURCE_HASH:0:15}-${LIBVCX_NODE_WRAPPERS_HASH:0:15}-${LIBVCX_NODE_AGENT_HASH:0:15}-${LIBVCX_DOCKERFILE_HASH:0:15}
  CODECOV_HASH=${LIBVCX_SOURCE_HASH:0:23}-${CODECOV_DOCKERFILE_HASH:0:23}
@@ -107,12 +110,7 @@ jobs:
  echo ::set-env name=CACHE_KEY_CODECOV::$CODECOV_HASH
  echo ::set-env name=CACHE_KEY_ANDROID::$ANDROID_HASH
  echo ::set-env name=CACHE_KEY_POOL::$POOL_HASH
-
- DOCKER_IMG_NAME_AGENCY="docker.pkg.github.com/absaoss/vcxagencynode/vcxagency-node:0.1.2"
- echo ::set-env name=DOCKER_IMG_NAME_AGENCY::$(echo $DOCKER_IMG_NAME_AGENCY)
-
- DOCKER_IMG_NAME_ANDROID="android-test"
- echo ::set-env name=DOCKER_IMG_NAME_ANDROID::$(echo $DOCKER_IMG_NAME_ANDROID)
+ echo ::set-env name=CACHE_KEY_ALPINE_CORE::$ALPINE_CORE_HASH
 
  - name: Set and print outputs
  id: mainstep
@@ -126,9 +124,11 @@ jobs:
  echo "::set-output name=CACHE_KEY_CODECOV::$CACHE_KEY_CODECOV"
  echo "::set-output name=CACHE_KEY_ANDROID::$CACHE_KEY_ANDROID"
  echo "::set-output name=CACHE_KEY_POOL::$CACHE_KEY_POOL"
+ echo "::set-output name=CACHE_KEY_ALPINE_CORE::$CACHE_KEY_ALPINE_CORE"
 
- echo "::set-output name=DOCKER_IMG_NAME_AGENCY::$DOCKER_IMG_NAME_AGENCY"
- echo "::set-output name=DOCKER_IMG_NAME_ANDROID::$CACHE_KEY_ANDROID"
+ echo "::set-output name=DOCKER_IMG_NAME_ALPINE_CORE::alpine-core"
+ echo "::set-output name=DOCKER_IMG_NAME_AGENCY::docker.pkg.github.com/absaoss/vcxagencynode/vcxagency-node:0.1.2"
+ echo "::set-output name=DOCKER_IMG_NAME_ANDROID::android-test"
  echo "::set-output name=DOCKER_IMG_NAME_LIBVCX::libvcx:$CACHE_KEY_LIBVCX"
  echo "::set-output name=DOCKER_IMG_NAME_CODECOV::codecov:$CACHE_KEY_CODECOV"
  echo "::set-output name=DOCKER_IMG_NAME_POOL::indypool:$CACHE_KEY_POOL"
@@ -167,18 +167,69 @@ jobs:
  docker images
  docker image ls --format "{{.Repository}}:{{.Tag}}" | grep "$DOCKER_IMG_NAME_POOL" || { echo "Image $DOCKER_IMG_NAME_POOL was not found!" ; exit 1; }
 
- build-image-libvcx:
+ build-image-alpine-core:
  needs: workflow-setup
  runs-on: ubuntu-latest
+ env:
+ DOCKER_BUILDKIT: 1
+ steps:
+ - name: Load up custom variables
+ run: |
+ echo ::set-env name=CACHE_KEY_ALPINE_CORE::$(echo ${{needs.workflow-setup.outputs.CACHE_KEY_ALPINE_CORE}})
+ echo ::set-env name=DOCKER_IMG_NAME_ALPINE_CORE::$(echo ${{needs.workflow-setup.outputs.DOCKER_IMG_NAME_ALPINE_CORE}})
+ - name: Git checkout
+ uses: actions/checkout@v2
+ - name: Try load from cache.
+ id: cache-image-alpinecore
+ uses: actions/cache@v2
+ with:
+ path: /tmp/imgcache_alpine_core
+ key: ${{ env.CACHE_KEY_ALPINE_CORE }}
+ - name: If NOT found in cache, build and cache image.
+ if: steps.cache-image-alpinecore.outputs.cache-hit != 'true'
+ run: |
+ set -x
+ docker build --build-arg "USER_ID=$UID" \
+ -f ci/alpine_core.dockerfile \
+ -t "$DOCKER_IMG_NAME_ALPINE_CORE" \
+ .
+ mkdir -p /tmp/imgcache_alpine_core
+ docker save "$DOCKER_IMG_NAME_ALPINE_CORE" > /tmp/imgcache_alpine_core/img_alpine_core.rar
+
+ - name: Load alpine_core image from cache
+ run: |
+ docker load < /tmp/imgcache_alpine_core/img_alpine_core.rar
+ - name: Verify alpine_core image was loaded
+ run: |
+ docker image ls --format "{{.Repository}}:{{.Tag}}" | grep "$DOCKER_IMG_NAME_ALPINE_CORE" || { echo "Image $DOCKER_IMG_NAME_ALPINE_CORE was not found!" ; exit 1; }
+
+ build-image-libvcx:
+ needs: [workflow-setup, build-image-alpine-core]
+ runs-on: ubuntu-latest
  env:
  DOCKER_BUILDKIT: 1
  steps:
  - name: Load up custom variables
  run: |
  echo ::set-env name=CACHE_KEY_LIBVCX::$(echo ${{needs.workflow-setup.outputs.CACHE_KEY_LIBVCX}})
  echo ::set-env name=DOCKER_IMG_NAME_LIBVCX::$(echo ${{needs.workflow-setup.outputs.DOCKER_IMG_NAME_LIBVCX}})
+ echo ::set-env name=CACHE_KEY_ALPINE_CORE::$(echo ${{needs.workflow-setup.outputs.CACHE_KEY_ALPINE_CORE}})
+ echo ::set-env name=DOCKER_IMG_NAME_ALPINE_CORE::$(echo ${{needs.workflow-setup.outputs.DOCKER_IMG_NAME_ALPINE_CORE}})
  - name: Git checkout
  uses: actions/checkout@v2
+ - name: Try load from cache.
+ id: cache-image-alpinecore
+ uses: actions/cache@v2
+ with:
+ path: /tmp/imgcache_alpine_core
+ key: ${{ env.CACHE_KEY_ALPINE_CORE }}
+ - name: If alpine_core NOT found in cache, error
+ if: steps.cache-image-alpinecore.outputs.cache-hit != 'true'
+ run: exit -1
+ - name: Load alpine_core image from cache
+ run: |
+ docker load < /tmp/imgcache_alpine_core/img_alpine_core.rar
+ rm /tmp/imgcache_alpine_core/img_alpine_core.rar
  - name: Try load from cache.
  id: cache-image-libvcx
  uses: actions/cache@v2
@@ -189,8 +240,7 @@ jobs:
  if: steps.cache-image-libvcx.outputs.cache-hit != 'true'
  run: |
  set -x
- docker build --build-arg "NPMJS_TOKEN=$NPMJS_TOKEN" \
- --build-arg "USER_ID=$UID" \
+ docker build --build-arg "ALPINE_CORE_IMAGE=$DOCKER_IMG_NAME_ALPINE_CORE" \
  -f ci/libvcx.dockerfile \
  -t "$DOCKER_IMG_NAME_LIBVCX" \
  .
@@ -962,7 +1012,6 @@ jobs:
  - name: Verify libvcx image were loaded
  run: |
  docker image ls --format "{{.Repository}}:{{.Tag}}" | grep "$DOCKER_IMG_NAME_LIBVCX" || { echo "Image $DOCKER_IMG_NAME_LIBVCX was not found!" ; exit 1; }
-
  - name: Docker Login
  uses: azure/docker-login@v1
  with:
@@ -1093,4 +1142,4 @@ jobs:
  upload_url: ${{ needs.make-release.outputs.RELEASE_UPLOAD_URL }}
  asset_path: ./libvcx-ios-${{ needs.workflow-setup.outputs.PUBLISH_VERSION }}-universal.zip
  asset_name: libvcx-ios-${{ needs.workflow-setup.outputs.PUBLISH_VERSION }}-universal.zip
- asset_content_type: application/zip
+ asset_content_type: application/zip
diff --git a/ci/alpine_core.dockerfile b/ci/alpine_core.dockerfile
@@ -0,0 +1,44 @@
+FROM alpine:3.12 AS builder
+
+ARG UID=1000
+ARG GID=1000
+
+
+ARG INDYSDK_PATH=/home/indy/indy-sdk
+ARG INDYSDK_REPO=https://github.com/hyperledger/indy-sdk.git
+ARG INDYSDK_REVISION=v1.15.0
+
+ENV RUST_LOG=warning
+
+RUN addgroup -g $GID indy && adduser -u $UID -D -G indy indy
+
+RUN apk update && apk upgrade && \
+ apk add --no-cache \
+ build-base \
+ cargo \
+ git \
+ libsodium-dev \
+ libzmq \
+ openssl-dev \
+ zeromq-dev
+
+ARG RUST_VER="1.45.2"
+RUN curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain $RUST_VER
+
+USER indy
+WORKDIR /home/indy
+
+RUN git clone $INDYSDK_REPO && cd indy-sdk && git checkout $INDYSDK_REVISION
+
+RUN cargo build --release --manifest-path=$INDYSDK_PATH/libindy/Cargo.toml
+
+USER root
+RUN mv $INDYSDK_PATH/libindy/target/release/libindy.so /usr/lib
+
+USER indy
+RUN cargo build --release --manifest-path=$INDYSDK_PATH/libnullpay/Cargo.toml
+RUN cargo build --release --manifest-path=$INDYSDK_PATH/experimental/plugins/postgres_storage/Cargo.toml
+
+USER root
+RUN mv $INDYSDK_PATH/libnullpay/target/release/libnullpay.so .
+RUN mv $INDYSDK_PATH/experimental/plugins/postgres_storage/target/release/libindystrgpostgres.so .
diff --git a/ci/libvcx.dockerfile b/ci/libvcx.dockerfile
@@ -1,51 +1,15 @@
-FROM alpine:3.12 AS builder
-
-ARG UID=1000
-ARG GID=1000
-
-ARG INDYSDK_PATH=/home/indy/indy-sdk
-ARG INDYSDK_REPO=https://github.com/hyperledger/indy-sdk.git
-ARG INDYSDK_REVISION=v1.15.0
-
-ENV RUST_LOG=warning
-ARG RUST_VER="1.45.2"
-
-RUN addgroup -g $GID indy && adduser -u $UID -D -G indy indy
-
-RUN apk update && apk upgrade && \
- apk add --no-cache \
- build-base \
- cargo \
- git \
- libsodium-dev \
- libzmq \
- openssl-dev \
- zeromq-dev
-
-RUN curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain $RUST_VER
-
+ARG ALPINE_CORE_IMAGE
+FROM ${ALPINE_CORE_IMAGE} as builder
 USER indy
 WORKDIR /home/indy
 
 COPY --chown=indy ./ ./
 
-RUN git clone $INDYSDK_REPO && \
- cd indy-sdk && git checkout $INDYSDK_REVISION
-
-RUN cargo build --release --manifest-path=$INDYSDK_PATH/libindy/Cargo.toml
-
-USER root
-RUN mv $INDYSDK_PATH/libindy/target/release/libindy.so /usr/lib
-
 USER indy
 RUN cargo build --release --manifest-path=/home/indy/libvcx/Cargo.toml
-RUN cargo build --release --manifest-path=$INDYSDK_PATH/libnullpay/Cargo.toml
-RUN cargo build --release --manifest-path=$INDYSDK_PATH/experimental/plugins/postgres_storage/Cargo.toml
 
 USER root
 RUN mv /home/indy/libvcx/target/release/libvcx.so .
-RUN mv $INDYSDK_PATH/libnullpay/target/release/libnullpay.so .
-RUN mv $INDYSDK_PATH/experimental/plugins/postgres_storage/target/release/libindystrgpostgres.so .
 
 FROM alpine:3.12
 
@@ -76,8 +40,10 @@ RUN apk add --no-cache \
  npm \
  make \
  openssl-dev \
- rust \
  python2 \
  zeromq-dev
 
+ARG RUST_VER="1.45.2"
+RUN curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain $RUST_VER
+
 USER node
diff --git a/libvcx/tails.txt/GedWbCuEYRcXK1ndGUrygAwSw6GDHnEWpoeVuLrZEhan b/libvcx/tails.txt/GedWbCuEYRcXK1ndGUrygAwSw6GDHnEWpoeVuLrZEhan