Merge pull request #326 from clearmatics/ci-cache-test

Integration tests on CI
clearmatics · Feb 9, 2021 · 42ffe31 · 42ffe31
2 parents d1db76e + 18bda78
commit 42ffe31
Show file tree

Hide file tree

Showing 10 changed files with 468 additions and 69 deletions.
diff --git a/.github/workflows/code-scanning.yml b/.github/workflows/code-scanning.yml
@@ -46,6 +46,11 @@ jobs:
         uses: actions/setup-python@v2
         with:
           python-version: '3.x'
+      - name: Cache pip
+        uses: actions/cache@v2
+        with:
+          path: ~/.cache/pip
+          key: codeql-py-pip-${{ hashFiles('**/setup.py') }}-${{ runner.os }}
       - name: Custom dependencies installation
         run: |
           cd client
@@ -63,4 +68,3 @@ jobs:
           setup-python-dependencies: false
       - name: Perform CodeQL Analysis
         uses: github/codeql-action/analyze@v1
-
diff --git a/.github/workflows/onpullrequest-build-ubuntu.yml b/.github/workflows/onpullrequest-build-ubuntu.yml
@@ -1,19 +1,130 @@
 # Actions for pull requests only
-name: zeth-ci-pull-request
+name: zeth-ci-onpullrequest-build-ubuntu
 
 on:
   pull_request:
 
 jobs:
 
-  build-linux-full:
+  # Job to build the grpc libraries. The grpc build directory is populated and
+  # cached so that all other jobs can mark this job as a prerequisite and just
+  # run `make install`.
+  #
+  # NOTE: the version number here (in `key` and in script arguments) must be
+  # kept in sync with the key used by the jobs.
+  onpr-build-grpc:
     runs-on: ubuntu-20.04
+    steps:
+    - uses: actions/checkout@v2
+    - name: Cache grpc
+      uses: actions/cache@v2
+      with:
+        key: grpc-1.31.x-${{ runner.os }}
+        path: depends/grpc
+    - name: Build grpc
+      run: if ! [ -d depends/grpc ] ; then scripts/install_grpc /usr v1.31.x depends/grpc ; fi
+
+  # Extract the commits of submodules for use by cache steps
+  onpr-submodules:
+    runs-on: ubuntu-20.04
+    outputs:
+      commits: ${{ steps.get-commits.outputs.commits }}
+    steps:
+    - uses: actions/checkout@v2
+    - name: Get Submodule Commits
+      id: get-commits
+      run: |
+        git submodule sync
+        echo "::set-output name=commits::"`git submodule status depends/libsodium | grep -oe '[0-9a-fA-F]\+' | head -c 8`-`git submodule status depends/libsnark | grep -oe '[0-9a-fA-F]\+' | head -c 8`
+
+  # Run the prover testing python scripts
+  prover-tests-linux:
+    runs-on: ubuntu-20.04
+    needs: [onpr-build-grpc, onpr-submodules]
+    strategy:
+      matrix:
+        curve: [ BLS12_377, ALT_BN128 ]
+    steps:
+    - uses: actions/checkout@v1
+      with:
+        submodules: recursive
+    - uses: actions/setup-node@v1
+      with:
+        node-version: 10
+    - name: Cache grpc
+      uses: actions/cache@v2
+      with:
+        key: grpc-1.31.x-${{ runner.os }}
+        path: depends/grpc
+    - name: Cache ccache
+      uses: actions/cache@v2
+      with:
+        key: prover-tests-ccache-${{ needs.onpr-submodules.outputs.commits }}-${{ matrix.curve }}-${{ runner.os }}
+        path: ~/.ccache
+    - name: Cache pip
+      uses: actions/cache@v2
+      with:
+        path: |
+          ~/.cache/pip
+          ~/Library/Caches/pip
+        key: prover-tests-pip-${{ hashFiles('**/setup.py') }}-${{ runner.os }}
+    - name: Cache npm
+      uses: actions/cache@v2
+      with:
+        path: |
+          ~/.npm
+          depends/ganache-cli/node_modules
+        key: prover-tests-npm-${{ hashFiles('**/package-lock.json') }}-${{ runner.os }}
+    - name: Install dependencies
+      run: |
+        scripts/install_grpc /usr v1.31.x depends/grpc
+        sudo apt install -y ccache
+    - name: Execute
+      run: CI_CONFIG=Release CI_CURVE=${{ matrix.curve }} CI_PROVER_TESTS=1 scripts/ci build
+
+  # Run all unit tests and integration tests
+  integration-tests-linux:
+    runs-on: ubuntu-20.04
+    needs: [onpr-build-grpc, onpr-submodules]
     strategy:
       matrix:
         curve: [ BLS12_377, ALT_BN128 ]
     steps:
     - uses: actions/checkout@v1
       with:
         submodules: recursive
+    - uses: actions/setup-node@v1
+      with:
+        node-version: 10
+    - name: Cache grpc
+      uses: actions/cache@v2
+      with:
+        key: grpc-1.31.x-${{ runner.os }}
+        path: depends/grpc
+    # ccache in this job seems to make the tests crash occasionally. Disabling
+    # until the cause is understood.
+    # - name: Cache ccache
+    #   uses: actions/cache@v2
+    #   with:
+    #     key: integration-tests-ccache-${{ needs.onpr-submodules.outputs.commits }}-${{ matrix.curve }}-${{ runner.os }}
+    #     path: ~/.ccache
+    - name: Cache pip
+      uses: actions/cache@v2
+      with:
+        path: |
+          ~/.cache/pip
+          ~/Library/Caches/pip
+        key: integration-tests-pip-${{ hashFiles('**/setup.py') }}-${{ runner.os }}
+    - name: Cache npm
+      uses: actions/cache@v2
+      with:
+        path: |
+          ~/.npm
+          depends/ganache-cli/node_modules
+        key: integration-tests-npm-${{ hashFiles('**/package-lock.json') }}-${{ runner.os }}
+    - name: Install dependencies
+      run: |
+        scripts/install_grpc /usr v1.31.x depends/grpc
+        sudo apt install -y ccache
     - name: Execute
-      run: CI_EVENT_NAME=pull_request CI_USE_DOCKER=1 CI_CONFIG=Release CI_CURVE=${{ matrix.curve }} scripts/ci build
+      run: CI_CONFIG=Release CI_CURVE=${{ matrix.curve }} CI_FULL_TESTS=1 CI_INTEGRATION_TESTS=1 scripts/ci build
diff --git a/.github/workflows/onpush-build-macos.yml b/.github/workflows/onpush-build-macos.yml
@@ -5,21 +5,45 @@ on:
   push:
 
 env:
-  MACOS_BREW_PACKAGES: "pkg-config libomp"
+  MACOS_BREW_PACKAGES: "pkg-config libomp ccache"
 
 jobs:
 
+  # Extract the commits of submodules for use by cache steps
+  submodules:
+    runs-on: ubuntu-20.04
+    outputs:
+      commits: ${{ steps.get-commits.outputs.commits }}
+    steps:
+    - uses: actions/checkout@v2
+    - name: Get Submodule Commits
+      id: get-commits
+      run: |
+        git submodule sync
+        echo "::set-output name=commits::"`git submodule status depends/libsodium | grep -oe '[0-9a-fA-F]\+' | head -c 8`-`git submodule status depends/libsnark | grep -oe '[0-9a-fA-F]\+' | head -c 8`
+
+  # Main build
   build-macos:
     runs-on: macos-10.15
+    needs: submodules
     strategy:
       matrix:
         config: [ Debug, Release ]
     steps:
-    - uses: actions/checkout@v1
+    - uses: actions/checkout@v2
       with:
         submodules: recursive
+    - name: Cache ccache
+      uses: actions/cache@v2
+      with:
+        key: submodules-${{ needs.submodules.outputs.commits }}-${{ runner.os }}-${{ matrix.config }}
+        path: ~/Library/Caches/ccache
+    - name: Cache pip (for mpc tests)
+      uses: actions/cache@v2
+      with:
+        path: ~/Library/Caches/pip
+        key: build-macos-pip-${{ hashFiles('**/setup.py') }}-${{ runner.os }}
     - name: Install Dependencies
       run: brew install ${MACOS_BREW_PACKAGES}
     - name: Execute
-      run: CI_CONFIG=${{ matrix.config }} scripts/ci build
-
+      run: CI_MPC_TESTS=1 CI_CONFIG=${{ matrix.config }} scripts/ci build
diff --git a/.github/workflows/onpush-build-ubuntu.yml b/.github/workflows/onpush-build-ubuntu.yml
@@ -6,33 +6,85 @@ on:
 
 jobs:
 
+  # Job to build and cache the grpc libraries. The grpc build directory is
+  # populated and cached so that all other jobs can mark this job as a
+  # prerequisite and just run `make install`.
+  #
+  # NOTE: the version number here (in `key` and in script arguments) must be
+  # kept in sync with the key used by the jobs.
+  build-grpc:
+    runs-on: ubuntu-20.04
+    steps:
+    - uses: actions/checkout@v2
+    - name: Cache grpc
+      uses: actions/cache@v2
+      with:
+        key: grpc-1.31.x-${{ runner.os }}
+        path: depends/grpc
+    - name: Build grpc
+      run: if ! [ -d depends/grpc ] ; then scripts/install_grpc /usr v1.31.x depends/grpc ; fi
+
   build-linux:
     runs-on: ubuntu-20.04
+    needs: build-grpc
     strategy:
       matrix:
         config: [ Debug, Release ]
     steps:
-    - uses: actions/checkout@v1
+    - uses: actions/checkout@v2
       with:
         submodules: recursive
+    - name: Cache grpc
+      uses: actions/cache@v2
+      with:
+        key: grpc-1.31.x-${{ runner.os }}
+        path: depends/grpc
+    - name: Cache pip (for mpc tests)
+      uses: actions/cache@v2
+      with:
+        path: ~/.cache/pip
+        key: build-linux-pip-${{ hashFiles('**/setup.py') }}-${{ runner.os }}
+    - name: Install dependencies
+      run: |
+        scripts/install_grpc /usr v1.31.x depends/grpc
+        sudo apt install -y ccache
     - name: Execute
-      run: CI_CHECK_FORMAT=1 CI_USE_DOCKER=1 CI_CONFIG=${{ matrix.config }} scripts/ci build
+      run: CI_CHECK_FORMAT=1 CI_MPC_TESTS=1 CI_CONFIG=${{ matrix.config }} scripts/ci build
 
   build-linux-pghr13:
     runs-on: ubuntu-20.04
+    needs: build-grpc
     steps:
-    - uses: actions/checkout@v1
+    - uses: actions/checkout@v2
       with:
         submodules: recursive
+    - name: Cache grpc
+      uses: actions/cache@v2
+      with:
+        key: grpc-1.31.x-${{ runner.os }}
+        path: depends/grpc
+    - name: Install dependencies
+      run: |
+        scripts/install_grpc /usr v1.31.x depends/grpc
+        sudo apt install -y ccache
     - name: Execute
-      run: CI_USE_DOCKER=1 CI_CONFIG=Release CI_ZKSNARK=PGHR13 scripts/ci build
+      run: CI_CONFIG=Release CI_ZKSNARK=PGHR13 scripts/ci build
 
   build-linux-bls12-377:
     runs-on: ubuntu-20.04
+    needs: build-grpc
     steps:
     - uses: actions/checkout@v1
       with:
         submodules: recursive
+    - name: Cache grpc
+      uses: actions/cache@v2
+      with:
+        key: grpc-1.31.x-${{ runner.os }}
+        path: depends/grpc
+    - name: Install dependencies
+      run: |
+        scripts/install_grpc /usr v1.31.x depends/grpc
+        sudo apt install -y ccache
     - name: Execute
-      run: CI_USE_DOCKER=1 CI_CONFIG=Release CI_CURVE=BLS12_377 scripts/ci build
-
+      run: CI_CONFIG=Release CI_CURVE=BLS12_377 scripts/ci build
diff --git a/.github/workflows/onpush-checks.yml b/.github/workflows/onpush-checks.yml
@@ -1,5 +1,5 @@
 # Check actions (linters, analysis tools etc.)
-name: zeth-ci-push-checks
+name: zeth-ci-onpush-checks
 
 on:
   push:
@@ -9,28 +9,26 @@ jobs:
   check-contracts:
     runs-on: ubuntu-20.04
     steps:
-    - uses: actions/checkout@v1
+    - uses: actions/checkout@v2
       with:
         submodules: recursive
-    - uses: actions/setup-node@v1
-      with:
-        node-version: 10
-    - name: Cache npm
-      uses: actions/cache@v1
+    - uses: actions/setup-node@v2
       with:
-        path: ~/.npm
-        key: check-contracts-npm-${{ runner.os }}-${{ hashFiles('**/package-lock.json') }}
-    # Cache ganache-cli/node_modules to avoid recompiling native code
-    - name: Cache depends/ganache-cli/node_modules
-      uses: actions/cache@v1
+        node-version: '10'
+    - name: Cache npm (incl ganache-cli/node_modules)
+      uses: actions/cache@v2
       with:
-        path: depends/ganache-cli/node_modules
-        key: check-contracts-ganache-node-modules-${{ runner.os }}-${{ hashFiles('depends/ganache-cli/package-lock.json') }}
+        path: |
+          ~/.npm
+          depends/ganache-cli/node_modules
+        key: check-contracts-npm-${{ hashFiles('**/package-lock.json') }}-${{ runner.os }}
     - name: Cache pip
       uses: actions/cache@v2
       with:
-        path: ~/.cache/pip
-        key: check-contracts-pip-${{ runner.os }}-${{ hashFiles('client/setup.py') }}
+        path: |
+          ~/.cache/pip
+          ~/.solcx
+        key: check-contracts-pip-solcx-${{ hashFiles('**/setup.py') }}-${{ runner.os }}
     - name: Check Contracts
       run: scripts/ci check_contracts
 
@@ -44,7 +42,7 @@ jobs:
       uses: actions/cache@v2
       with:
         path: ~/.cache/pip
-        key: check-client-pip-${{ runner.os }}-${{ hashFiles('client/setup.py') }}
+        key: check-client-pip-${{ hashFiles('**/setup.py') }}-${{ runner.os }}
     - name: Check Client
       run: scripts/ci check_client
 

diff --git a/depends/CMakeLists.txt b/depends/CMakeLists.txt
@@ -23,11 +23,12 @@ endif()
 
 # libsodium
 set(LIBSODIUM_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/libsodium)
+set(LIBSODIUM_CONFIGURE_CMD "CC=\"${CMAKE_C_COMPILER_LAUNCHER} ${CMAKE_C_COMPILER}\" CXX=\"${CMAKE_C_COMPILER_LAUNCHER} ${CMAKE_CXX_COMPILER}\" ./configure --prefix=<INSTALL_DIR> --enable-minimal --disable-shared --enable-static")
 
 include(ExternalProject)
 ExternalProject_Add(
   libsodium
   SOURCE_DIR ${LIBSODIUM_SOURCE_DIR}
-  CONFIGURE_COMMAND cd ${LIBSODIUM_SOURCE_DIR} && DO_NOT_UPDATE_CONFIG_SCRIPTS=1 ./autogen.sh && ./configure --prefix=<INSTALL_DIR> --enable-minimal --disable-shared --enable-static
+  CONFIGURE_COMMAND cd ${LIBSODIUM_SOURCE_DIR} && DO_NOT_UPDATE_CONFIG_SCRIPTS=1 ./autogen.sh && eval ${LIBSODIUM_CONFIGURE_CMD}
   BUILD_COMMAND cd ${LIBSODIUM_SOURCE_DIR} && make ${MAKE_BUILD_FLAGS}
   INSTALL_COMMAND cd ${LIBSODIUM_SOURCE_DIR} && make install)