Merge remote-tracking branch 'origin/revert-2308-ci/do-not-run-test-o…

…n-push' into revert-2308-ci/do-not-run-test-on-push
dashpay · Nov 19, 2024 · a526ceb · a526ceb
2 parents 79bb789 + 89e91cd
commit a526ceb
Show file tree

Hide file tree

Showing 50 changed files with 905 additions and 529 deletions.
diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile
@@ -44,16 +44,30 @@ ENV PATH=$CARGO_HOME/bin:$PATH
 COPY rust-toolchain.toml .
 RUN TOOLCHAIN_VERSION="$(grep channel rust-toolchain.toml | awk '{print $3}' | tr -d '"')" && \
     curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- \
-    --profile minimal \
     -y \
     --default-toolchain "${TOOLCHAIN_VERSION}" \
     --target wasm32-unknown-unknown
 
-# Install wasm-bindgen-cli in the same profile as other components, to sacrifice some performance & disk space to gain
-# better build caching
-RUN if [[ -z "${SCCACHE_MEMCACHED}" ]] ; then unset SCCACHE_MEMCACHED ; fi ; \
-    RUSTFLAGS="-C target-feature=-crt-static" \
-    # Meanwhile if you want to update wasm-bindgen you also need to update version in:
-    #  - packages/wasm-dpp/Cargo.toml
-    #  - packages/wasm-dpp/scripts/build-wasm.sh
-    cargo install wasm-bindgen-cli@0.2.86 --locked
+# Download and install cargo-binstall
+ENV BINSTALL_VERSION=1.10.11
+RUN set -ex; \
+    if [ "$TARGETARCH" = "amd64" ]; then \
+        CARGO_BINSTALL_ARCH="x86_64-unknown-linux-musl"; \
+    elif [ "$TARGETARCH" = "arm64" ] || [ "$TARGETARCH" = "aarch64" ]; then \
+        CARGO_BINSTALL_ARCH="aarch64-unknown-linux-musl"; \
+    else \
+        echo "Unsupported architecture: $TARGETARCH"; exit 1; \
+    fi; \
+    DOWNLOAD_URL="https://github.com/cargo-bins/cargo-binstall/releases/download/v${BINSTALL_VERSION}/cargo-binstall-${CARGO_BINSTALL_ARCH}.tgz"; \
+    curl -L --fail --show-error "$DOWNLOAD_URL" -o /tmp/cargo-binstall.tgz; \
+    tar -xzf /tmp/cargo-binstall.tgz -C /tmp cargo-binstall; \
+    chmod +x /tmp/cargo-binstall; \
+    /tmp/cargo-binstall -y --force cargo-binstall; \
+    rm /tmp/cargo-binstall; \
+    cargo binstall -V
+
+RUN cargo binstall wasm-bindgen-cli@0.2.86 --locked \
+                                           --no-discover-github-token \
+                                           --disable-telemetry \
+                                           --no-track \
+                                           --no-confirm
diff --git a/.github/actions/docker/action.yaml b/.github/actions/docker/action.yaml
@@ -22,12 +22,6 @@ inputs:
     description: User name to use when pushing images to Docker Hub
   dockerhub_token:
     description: Docker Hub token to use
-  cache_mounts:
-    description: Load cache mounts cache
-    default: |
-      cargo_registry_index
-      cargo_registry_cache
-      cargo_git
   cargo_profile:
     description: Cargo build profile, i.e release or dev
     default: dev
@@ -43,6 +37,9 @@ inputs:
   aws_secret_access_key:
     description: AWS secret access key
     required: true
+  cache_to_name:
+    description: 'Save cache to name manifest (should be used only on default branch)'
+    default: 'false'
 outputs:
   digest:
     value: ${{ steps.docker_build.outputs.digest }}
@@ -65,17 +62,10 @@ runs:
         install: true
         driver-opts: env.BUILDKIT_STEP_LOG_MAX_SIZE=10485760
         cleanup: false
-        config-inline: |
+        buildkitd-config-inline: |
           [worker.oci]
             gc = false
 
-    - name: Load Docker mount cache
-      uses: dcginfra/buildkit-cache-dance/inject@s5cmd
-      if: ${{ inputs.cache_mounts != '' }}
-      with:
-        bucket: ${{ inputs.bucket }}
-        mounts: ${{ inputs.cache_mounts }}
-
     - name: Set Docker tags and labels from image tag
       id: docker_meta
       uses: docker/metadata-action@v5
@@ -90,16 +80,83 @@ runs:
       id: layer_cache_settings
       with:
         name: ${{ inputs.image_name }}
+        region: ${{ inputs.region }}
         bucket: ${{ inputs.bucket }}
+        cache_to_name: ${{ inputs.cache_to_name }}
+
+    - name: Set HOME variable to github context
+      shell: bash
+      run: echo "HOME=$HOME" >> $GITHUB_ENV
+
+    - name: Cargo cache for Docker
+      uses: actions/cache@v4
+      id: cargo-cache
+      with:
+        path: |
+          ${{ env.HOME }}/cargo-cache-registry-index
+          ${{ env.HOME }}/cargo-cache-registry-cache
+          ${{ env.HOME }}/cargo-cache-git-db
+        key: ${{ runner.os }}-cargo-${{ hashFiles('Cargo.lock') }}
+        restore-keys: |
+          ${{ runner.os }}-cargo-
+
+    - name: Inject cargo cache into docker
+      uses: reproducible-containers/buildkit-cache-dance@v3.1.2
+      with:
+        cache-map: |
+          {
+            "${{ env.HOME }}/cargo-cache-registry-index": {
+              "target": "/root/.cargo/registry/index",
+              "id": "cargo_registry_index"
+            },
+            "${{ env.HOME }}/cargo-cache-registry-cache": {
+              "target": "/root/.cargo/registry/cache",
+              "id": "cargo_registry_cache"
+            },
+            "${{ env.HOME }}/cargo-cache-git-db": {
+              "target": "/root/.cargo/git/db",
+              "id": "cargo_git"
+            }
+          }
+        skip-extraction: ${{ steps.cargo-cache.outputs.cache-hit }}
+
+    - name: Yarn unplugged cache for Docker
+      uses: actions/cache@v4
+      id: yarn-cache
+      with:
+        path: ${{ env.HOME }}/yarn-unplugged-cache
+        key: ${{ inputs.platform }}-yarn-unplugged-${{ hashFiles('yarn.lock') }}
+        restore-keys: |
+          ${{ inputs.platform }}-yarn-unplugged-
+
+    - name: Set arch
+      id: arch
+      uses: actions/github-script@v6
+      with:
+        result-encoding: 'string'
+        script: return '${{ inputs.platform }}'.replace('linux/', '');
+
+    - name: Inject cargo cache into docker
+      uses: reproducible-containers/buildkit-cache-dance@v3.1.2
+      with:
+        cache-map: |
+          {
+            "${{ env.HOME }}/yarn-unplugged-cache": {
+              "target": "/tmp/unplugged",
+              "id": "unplugged_${{ steps.arch.outputs.result }}"
+            }
+          }
+        skip-extraction: ${{ steps.yarn-cache.outputs.cache-hit }}
 
     - name: Build and push Docker image ${{ inputs.image }}
       id: docker_build
-      uses: docker/build-push-action@v5
+      uses: docker/build-push-action@v6
       with:
         context: .
         builder: ${{ steps.buildx.outputs.name }}
         target: ${{ inputs.target }}
         labels: ${{ steps.docker_meta.outputs.labels }}
+        push: ${{ inputs.push_tags }}
         tags: ${{ inputs.push_tags == 'true' && steps.docker_meta.outputs.tags || '' }}
         platforms: ${{ inputs.platform }}
         build-args: |
@@ -113,10 +170,3 @@ runs:
         cache-from: ${{ steps.layer_cache_settings.outputs.cache_from }}
         cache-to: ${{ steps.layer_cache_settings.outputs.cache_to }}
         outputs: type=image,name=${{ inputs.image_org }}/${{ inputs.image_name }},push-by-digest=${{ inputs.push_tags != 'true' }},name-canonical=true,push=true
-
-    - name: Save Docker mount cache
-      uses: dcginfra/buildkit-cache-dance/extract@s5cmd
-      if: ${{ inputs.cache_mounts != '' }}
-      with:
-        bucket: ${{ inputs.bucket }}
-        mounts: ${{ inputs.cache_mounts }}
diff --git a/.github/actions/librocksdb/action.yaml b/.github/actions/librocksdb/action.yaml
@@ -21,14 +21,12 @@ inputs:
 runs:
   using: composite
   steps:
-    # Cache librocksdb using s3 bucket
-    - name: Restore cached librocksdb from S3
+    - name: Cache librocksdb
+      uses: actions/cache@v4
       id: librocksdb-cache
-      uses: strophy/actions-cache@opendal-update
       with:
-        bucket: ${{ inputs.bucket }}
-        path: /opt/rocksdb
         key: librocksdb/${{ inputs.version }}/${{ runner.os }}/${{ runner.arch }}
+        path: /opt/rocksdb
 
     - if: ${{ steps.librocksdb-cache.outputs.cache-hit != 'true' || inputs.force == 'true' }}
       shell: bash

diff --git a/.github/actions/local-network/action.yaml b/.github/actions/local-network/action.yaml
@@ -11,32 +11,27 @@ runs:
     - name: Setup Node.JS
       uses: ./.github/actions/nodejs
 
-    - name: Restore JS build artifacts
-      uses: strophy/actions-cache@opendal-update
+    - name: Download JS build artifacts
+      uses: actions/download-artifact@v4
       with:
-        bucket: multi-runner-cache-x1xibo9c
-        root: actions-cache
-        path: build-js-artifacts-${{ github.sha }}.tar
-        key: build-js-artifacts/${{ github.sha }}
-
-    - name: Unpack JS build artifacts archive
-      shell: bash
-      run: tar -xf build-js-artifacts-${{ github.sha }}.tar
+        name: js-build-${{ github.sha }}
+        path: packages
 
     - name: Get dashmate fingerprint
       id: dashmate-fingerprint
       shell: bash
       run: echo "sha=$(git log -1 --format="%h" -- packages/dashmate)" >> $GITHUB_OUTPUT
 
-    # TODO: Use upload artifacts action instead
+    - name: Set HOME variable to github context
+      shell: bash
+      run: echo "HOME=$HOME" >> $GITHUB_ENV
+
     - name: Restore local network data
       id: local-network-data
-      uses: strophy/actions-cache/restore@opendal-update
+      uses: actions/cache/restore@v4
       with:
-        bucket: multi-runner-cache-x1xibo9c
-        root: local-network-data
         path: |
-          /home/ubuntu/.dashmate
+          ${{ env.HOME }}/.dashmate
           **/.env
           dashmate_volumes_dump
         key: local-network-volumes/${{ steps.dashmate-fingerprint.outputs.sha }}
@@ -68,12 +63,10 @@ runs:
       if: steps.local-network-data.outputs.cache-hit != 'true'
 
     - name: Save local network data
-      uses: strophy/actions-cache/save@opendal-update
+      uses: actions/cache/save@v4
       with:
-        bucket: multi-runner-cache-x1xibo9c
-        root: local-network-data
         path: |
-          /home/ubuntu/.dashmate
+          ${{ env.HOME }}/.dashmate
           **/.env
           dashmate_volumes_dump
         key: local-network-volumes/${{ steps.dashmate-fingerprint.outputs.sha }}
@@ -90,9 +83,9 @@ runs:
         docker tag ${{ inputs.image_org }}/dashmate-helper:$SHA_TAG dashpay/dashmate-helper:$VERSION
 
         # Replace DAPI and Drive images with new org and tag in dashmate config
-        sed -i -E "s/dashpay\/(drive|dapi):[^\"]+/${{ inputs.image_org }}\/\1:${SHA_TAG}/g" /home/ubuntu/.dashmate/config.json
+        sed -i -E "s/dashpay\/(drive|dapi):[^\"]+/${{ inputs.image_org }}\/\1:${SHA_TAG}/g" ${{ env.HOME }}/.dashmate/config.json
 
-        cat /home/ubuntu/.dashmate/config.json
+        cat ${{ env.HOME }}/.dashmate/config.json
 
     - name: Start local network
       shell: bash

diff --git a/.github/actions/nodejs/action.yaml b/.github/actions/nodejs/action.yaml
@@ -17,11 +17,9 @@ runs:
       shell: bash
       run: npm config set audit false
 
-    - name: Cache NPM build artifacts (S3 bucket cache)
-      uses: strophy/actions-cache@opendal-update
+    - name: Cache NPM build artifacts
+      uses: actions/cache@v4
       with:
-        bucket: multi-runner-cache-x1xibo9c
-        root: actions-cache
         path: |
           .yarn/unplugged
         key: ${{ runner.os }}/yarn/unplugged/${{ runner.arch }}/${{ hashFiles('yarn.lock') }}

diff --git a/.github/actions/rust/action.yaml b/.github/actions/rust/action.yaml
@@ -17,6 +17,7 @@ inputs:
     required: false
     default: "true"
 
+# TODO: Cache deps here to save 1 minute
 runs:
   using: composite
   steps:
@@ -32,10 +33,8 @@ runs:
         echo "TOOLCHAIN_VERSION=$TOOLCHAIN_VERSION" >> $GITHUB_ENV
         echo "::set-output name=version::$TOOLCHAIN_VERSION"
 
-    # TODO: Move to AMI and build every day
     - uses: dtolnay/rust-toolchain@master
       name: Install Rust toolchain
-      id: install_rust
       with:
         toolchain: ${{ steps.rust_toolchain.outputs.version }}
         target: ${{ inputs.target }}
@@ -60,7 +59,6 @@ runs:
             ;;
         esac
 
-    # TODO: Move to AMI and build every day
     - name: Check if protoc is installed
       id: check-protoc
       shell: bash
@@ -84,38 +82,35 @@ runs:
         echo "PROTOC=${HOME}/.local/bin/protoc" >> $GITHUB_ENV
         export PATH="${PATH}:${HOME}/.local/bin"
 
-    - name: Run sccache-cache
-      uses: mozilla-actions/sccache-action@v0.0.3
+    - name: Install sccache-cache
+      uses: mozilla-actions/sccache-action@v0.0.6
       with:
-        version: "v0.7.1" # Must be the same as in Dockerfile
+        version: "v0.8.2" # Must be the same as in Dockerfile
       if: inputs.cache == 'true'
 
-    - name: Hash ref_name
-      id: hashed-ref-name
+    - name: Set HOME variable to github context
       shell: bash
-      run: echo "key=$(echo '${{ github.ref_name }}' | sha256sum | cut -d ' ' -f1)" >> $GITHUB_OUTPUT
+      run: echo "HOME=$HOME" >> $GITHUB_ENV
 
-    - name: Cache cargo registry (S3 bucket cache)
-      uses: strophy/actions-cache@opendal-update
+    - name: Cache cargo registry
+      uses: actions/cache@v4
       if: inputs.cache == 'true'
       with:
-        bucket: multi-runner-cache-x1xibo9c
-        root: actions-cache
         path: |
-          /home/ubuntu/.cargo/registry/index
-          /home/ubuntu/.cargo/registry/cache
-          /home/ubuntu/.cargo/git
+          ${{ env.HOME }}/.cargo/registry/index
+          ${{ env.HOME }}/.cargo/registry/cache
+          ${{ env.HOME }}/.cargo/git
         key: ${{ runner.os }}/cargo/registry/${{ hashFiles('**/Cargo.lock') }}
         restore-keys: |
           ${{ runner.os }}/cargo/registry/${{ hashFiles('**/Cargo.lock') }}
           ${{ runner.os }}/cargo/registry/
 
-    # TODO: Move to AMI and build every day
     - name: Install clang
       id: deps-clang
       shell: bash
       if: runner.os == 'Linux'
       run: |
         sudo apt update -qq
-        sudo apt install -qq --yes clang llvm
+        # snappy is required by rust rocksdb
+        sudo apt install -qq --yes clang llvm libsnappy-dev
         sudo update-alternatives --set cc /usr/bin/clang