Parallelize test-lib and RPC tests

GaloisInc · May 6, 2021 · 952d072 · 952d072
1 parent f06a741
commit 952d072
Show file tree

Hide file tree

Showing 3 changed files with 114 additions and 48 deletions.
diff --git a/.github/ci.sh b/.github/ci.sh
@@ -37,10 +37,6 @@ retry() {
   done
 }
 
-setup_external_tools() {
-  cabal v2-install --install-method=copy --installdir="$BIN" test-lib
-}
-
 setup_dist_bins() {
   extract_exe "cryptol" "dist/bin"
   extract_exe "cryptol-html" "dist/bin"
@@ -125,12 +121,6 @@ install_system_deps() {
   is_exe "$BIN" z3 && is_exe "$BIN" cvc4 && is_exe "$BIN" yices
 }
 
-test_dist() {
-  setup_external_tools
-  echo "test-runner version: $($BIN/test-runner --version)"
-  $BIN/test-runner --ext=.icry -F -b --exe=dist/bin/cryptol tests
-}
-
 check_docs() {
   ./cry build exe:check-exercises
   find ./docs/ProgrammingCryptol -name '*.tex' -print0 | xargs -0 -n1 cabal v2-exec check-exercises

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -9,7 +9,7 @@ on:
   workflow_dispatch:
 
 env:
-  Z3_VERSION: "4.8.8"
+  Z3_VERSION: "4.8.10"
   CVC4_VERSION: "4.1.8"
   YICES_VERSION: "2.6.2"
 
@@ -54,11 +54,13 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, macos-latest, windows-latest]
-        ghc: ["8.6.5", "8.8.4", "8.10.2"]
+        ghc-version: ["8.6.5", "8.8.4", "8.10.2"]
         exclude:
           # https://gitlab.haskell.org/ghc/ghc/-/issues/18550
           - os: windows-latest
-            ghc: 8.10.2
+            ghc-version: 8.10.2
+    outputs:
+      test-lib-json: ${{ steps.test-lib.outputs.targets-json }}
     env:
       VERSION: ${{ needs.config.outputs.version }}
       RELEASE: ${{ needs.config.outputs.release }}
@@ -71,20 +73,20 @@ jobs:
         with:
           python-version: '3.x'
 
-      - uses: actions/setup-haskell@v1
+      - uses: haskell/actions/setup@v1
         id: setup-haskell
         with:
-          ghc-version: ${{ matrix.ghc }}
+          ghc-version: ${{ matrix.ghc-version }}
 
       - uses: actions/cache@v2
         name: Cache cabal store
         with:
           path: |
             ${{ steps.setup-haskell.outputs.cabal-store }}
             dist-newstyle
-          key: cabal-${{ runner.os }}-${{ matrix.ghc }}-${{ hashFiles(format('cabal.GHC-{0}.config', matrix.ghc)) }}-${{ github.sha }}
+          key: cabal-${{ runner.os }}-${{ matrix.ghc-version }}-${{ hashFiles(format('cabal.GHC-{0}.config', matrix.ghc-version)) }}-${{ github.sha }}
           restore-keys: |
-            cabal-${{ runner.os }}-${{ matrix.ghc }}-${{ hashFiles(format('cabal.GHC-{0}.config', matrix.ghc)) }}-
+            cabal-${{ runner.os }}-${{ matrix.ghc-version }}-${{ hashFiles(format('cabal.GHC-{0}.config', matrix.ghc-version)) }}-
 
       - shell: bash
         run: .github/ci.sh install_system_deps
@@ -113,6 +115,20 @@ jobs:
             make
             "
 
+      - shell: bash
+        name: Partition test-lib tests
+        id: test-lib
+        run: |
+          set -x
+          cabal v2-install --install-method=copy --installdir="./bin" test-lib
+          cmd="cat \$1.stdout"
+          if ${{ runner.os == 'Windows' }}; then
+              cmd="cat \$1.stdout.mingw32 2>/dev/null || $cmd"
+          fi
+          ./bin/test-runner --ext=.icry -r ./output --exe=$(which bash) -F -c -F "$cmd" -F -- ./tests
+          TARGETS_JSON=$(echo -n "$(ls -1 ./output/tests)" | jq -Rsc 'split("\n")')
+          echo "::set-output name=targets-json::$TARGETS_JSON"
+
       - shell: bash
         run: .github/ci.sh bundle_files
 
@@ -146,6 +162,12 @@ jobs:
           if-no-files-found: error
           retention-days: ${{ needs.config.outputs.retention-days }}
 
+      - if: matrix.ghc-version == '8.6.5'
+        uses: actions/upload-artifact@v2
+        with:
+          path: dist/bin
+          name: ${{ runner.os }}-bins
+
       - uses: actions/upload-artifact@v2
         if: runner.os == 'Windows'
         with:
@@ -154,13 +176,69 @@ jobs:
           if-no-files-found: error
           retention-days: ${{ needs.config.outputs.retention-days }}
 
-      - shell: bash
-        run: .github/ci.sh test_dist
+  test:
+    runs-on: ${{ matrix.os }}
+    needs: [build]
+    strategy:
+      fail-fast: false
+      matrix:
+        suite: [test-lib]
+        target: ${{ fromJson(needs.build.outputs.test-lib-json) }}
+        os: [ubuntu-latest, macos-latest, windows-latest]
+        continue-on-error: [false]
+        include:
+          - suite: rpc
+            target: ''
+            os: ubuntu-latest
+            continue-on-error: false
+          - suite: rpc
+            target: ''
+            os: macos-latest
+            continue-on-error: false
+          - suite: rpc
+            target: ''
+            os: windows
+            continue-on-error: true  # TODO: get Python client to work on Windows
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          submodules: true
+
+      - uses: haskell/actions/setup@v1
+        with:
+          ghc-version: '8.10.2'
+
+      - uses: actions/download-artifact@v2
+        with:
+          name: "${{ runner.os }}-bins"
+          path: dist/bin
 
-      # TODO: get Python client to work on Windows
       - shell: bash
-        run: .github/ci.sh test_rpc
-        if: runner.os != 'Windows'
+        run: |
+          set -x
+          chmod +x dist/bin/cryptol
+          .github/ci.sh install_system_deps
+          ghc_ver="$(ghc --numeric-version)"
+          cp cabal.GHC-"$ghc_ver".config cabal.project.freeze
+          cabal v2-update
+
+      - if: matrix.suite == 'test-lib'
+        shell: bash
+        continue-on-error: ${{ matrix.continue-on-error }}
+        name: test-lib ${{ matrix.target }}
+        run: |
+          set -x
+          cabal v2-install -j --install-method=copy --installdir="./bin" test-lib
+          export PATH=$PWD/bin:$PWD/dist/bin:$PATH
+          ./bin/test-runner --ext=.icry -F -b --exe=dist/bin/cryptol ./tests/${{ matrix.target }}
+
+      - if: matrix.suite == 'rpc'
+        shell: bash
+        continue-on-error: ${{ matrix.continue-on-error }}
+        run: |
+          set -x
+          export PATH=$PWD/bin:$PWD/dist/bin:$PATH
+          cryptol-remote-api/run_rpc_tests.sh
 
   build-push-image:
     runs-on: ubuntu-latest

diff --git a/cryptol-remote-api/run_rpc_tests.sh b/cryptol-remote-api/run_rpc_tests.sh
@@ -2,11 +2,6 @@
 
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 
-pushd $DIR
-
-cabal v2-build exe:cryptol-remote-api
-cabal v2-build exe:cryptol-eval-server
-
 pushd $DIR/python
 
 NUM_FAILS=0
@@ -23,32 +18,35 @@ if [ $? -ne 0 ]; then
   NUM_FAILS=$(($NUM_FAILS+1))
 fi
 
-export CRYPTOL_SERVER=$(cabal v2-exec which cryptol-remote-api)
-if [[ -x "$CRYPTOL_SERVER" ]]; then
-  echo "Running cryptol-remote-api tests..."
-  echo "Using server $CRYPTOL_SERVER"
-  python3 -m unittest discover tests/cryptol
-  if [ $? -ne 0 ]; then
-      NUM_FAILS=$(($NUM_FAILS+1))
+export CRYPTOL_SERVER=$(which cryptol-remote-api)
+if [[ ! -x "$CRYPTOL_SERVER" ]]; then
+  export CRYPTOL_SERVER=$(cabal v2-exec which cryptol-remote-api)
+  if [[ ! -x "$CRYPTOL_SERVER" ]]; then
+    echo "could not locate cryptol-remote-api executable"
+    exit 1
   fi
-else
-  echo "could not find the cryptol-remote-api via `cabal v2-exec which`"
-  NUM_FAILS=$(($NUM_FAILS+1))
+fi
+echo "Running cryptol-remote-api tests..."
+echo "Using server $CRYPTOL_SERVER"
+python3 -m unittest discover tests/cryptol
+if [ $? -ne 0 ]; then
+    NUM_FAILS=$(($NUM_FAILS+1))
 fi
 
-export CRYPTOL_SERVER=$(cabal v2-exec which cryptol-eval-server)
-if [[ -x "$CRYPTOL_SERVER" ]]; then
-  echo "Running cryptol-eval-server tests..."
-  echo "Using server $CRYPTOL_SERVER"
-  python3 -m unittest discover tests/cryptol_eval
-  if [ $? -ne 0 ]; then
-      NUM_FAILS=$(($NUM_FAILS+1))
+export CRYPTOL_SERVER=$(which cryptol-eval-server)
+if [[ ! -x "$CRYPTOL_SERVER" ]]; then
+  export CRYPTOL_SERVER=$(cabal v2-exec which cryptol-eval-server)
+  if [[ ! -x "$CRYPTOL_SERVER" ]]; then
+    echo "could not locate cryptol-eval-server executable"
+    exit 1
   fi
-else
-  echo "could not find the cryptol-eval-server via `cabal v2-exec which`"
-  NUM_FAILS=$(($NUM_FAILS+1))
 fi
-popd
+echo "Running cryptol-eval-server tests..."
+echo "Using server $CRYPTOL_SERVER"
+python3 -m unittest discover tests/cryptol_eval
+if [ $? -ne 0 ]; then
+    NUM_FAILS=$(($NUM_FAILS+1))
+fi
 
 popd