Parallelize test-lib and RPC tests

GaloisInc · Apr 28, 2021 · 4a46412 · 4a46412
1 parent f06a741
commit 4a46412
Show file tree

Hide file tree

Showing 3 changed files with 87 additions and 41 deletions.
diff --git a/.github/ci.sh b/.github/ci.sh
@@ -37,10 +37,6 @@ retry() {
   done
 }
 
-setup_external_tools() {
-  cabal v2-install --install-method=copy --installdir="$BIN" test-lib
-}
-
 setup_dist_bins() {
   extract_exe "cryptol" "dist/bin"
   extract_exe "cryptol-html" "dist/bin"
@@ -125,12 +121,6 @@ install_system_deps() {
   is_exe "$BIN" z3 && is_exe "$BIN" cvc4 && is_exe "$BIN" yices
 }
 
-test_dist() {
-  setup_external_tools
-  echo "test-runner version: $($BIN/test-runner --version)"
-  $BIN/test-runner --ext=.icry -F -b --exe=dist/bin/cryptol tests
-}
-
 check_docs() {
   ./cry build exe:check-exercises
   find ./docs/ProgrammingCryptol -name '*.tex' -print0 | xargs -0 -n1 cabal v2-exec check-exercises

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -59,6 +59,8 @@ jobs:
           # https://gitlab.haskell.org/ghc/ghc/-/issues/18550
           - os: windows-latest
             ghc: 8.10.2
+    outputs:
+      test-lib-json: ${{ steps.test-lib.outputs.targets-json }}
     env:
       VERSION: ${{ needs.config.outputs.version }}
       RELEASE: ${{ needs.config.outputs.release }}
@@ -71,7 +73,7 @@ jobs:
         with:
           python-version: '3.x'
 
-      - uses: actions/setup-haskell@v1
+      - uses: haskell/actions/setup@v1
         id: setup-haskell
         with:
           ghc-version: ${{ matrix.ghc }}
@@ -113,6 +115,28 @@ jobs:
             make
             "
 
+      - shell: bash
+        name: Partition test-lib tests
+        id: test-lib
+        run: |
+          set -x
+          cabal v2-install --install-method=copy --installdir="./bin" test-lib
+          stdout=$(mktemp)
+          cat > $stdout << EndOfScript
+          if ${{ runner.os == 'Windows' }}; then
+              file="\$(cygpath -u "\$1").stdout"
+          else
+              file="\$1.stdout"
+          fi
+          echo "\$file" >> $PWD/lol
+          cat "\$file"
+          EndOfScript
+          ${{ runner.os == 'Windows' }} || chmod +x $stdout
+          ./bin/test-runner --ext=.icry -r ./output --exe=bash -F $stdout ./tests || true
+          cat ./lol
+          TARGETS_JSON=$(echo -n "$(ls -1 ./output/tests)" | jq -Rsc 'split("\n")')
+          echo "::set-output name=targets-json::$TARGETS_JSON"
+
       - shell: bash
         run: .github/ci.sh bundle_files
 
@@ -146,6 +170,12 @@ jobs:
           if-no-files-found: error
           retention-days: ${{ needs.config.outputs.retention-days }}
 
+      - if: matrix.ghc == '8.6.5'
+        uses: actions/upload-artifact@v2
+        with:
+          path: dist/bin
+          name: ${{ runner.os }}-bins
+
       - uses: actions/upload-artifact@v2
         if: runner.os == 'Windows'
         with:
@@ -154,13 +184,41 @@ jobs:
           if-no-files-found: error
           retention-days: ${{ needs.config.outputs.retention-days }}
 
+  test-lib:
+    runs-on: ${{ matrix.os }}
+    needs: [build]
+    strategy:
+      fail-fast: false
+      matrix:
+        target: ${{ fromJson(needs.build.outputs.test-lib-json) }}
+        os: [ubuntu-latest, macos-latest, windows-latest]
+    steps:
+      - uses: actions/download-artifact@v2
+        with:
+          name: "${{ runner.os }}-bins"
+          path: dist/bin
       - shell: bash
-        run: .github/ci.sh test_dist
+        run: |
+          cabal v2-install --install-method=copy --installdir="./bin" test-lib
+          echo "test-runner version: $($BIN/test-runner --version)"
+          ./bin/test-runner --ext=.icry -F -b --exe=dist/bin/cryptol ./tests/${{ matrix.target }}
 
-      # TODO: get Python client to work on Windows
+  test-rpc:
+    runs-on: ${{ matrix.os }}
+    needs: [build]
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macos-latest, windows-latest]
+        exclude:
+          - os: windows-latest  # TODO: get Python client to work on Windows
+    steps:
+      - uses: actions/download-artifact@v2
+        with:
+          name: "${{ runner.os }}-bins"
+          path: dist/bin
       - shell: bash
-        run: .github/ci.sh test_rpc
-        if: runner.os != 'Windows'
+        run: PATH=$PWD/dist/bin:$PATH cryptol-remote-api/run_rpc_tests.sh
 
   build-push-image:
     runs-on: ubuntu-latest

diff --git a/cryptol-remote-api/run_rpc_tests.sh b/cryptol-remote-api/run_rpc_tests.sh
@@ -2,11 +2,6 @@
 
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 
-pushd $DIR
-
-cabal v2-build exe:cryptol-remote-api
-cabal v2-build exe:cryptol-eval-server
-
 pushd $DIR/python
 
 NUM_FAILS=0
@@ -23,32 +18,35 @@ if [ $? -ne 0 ]; then
   NUM_FAILS=$(($NUM_FAILS+1))
 fi
 
-export CRYPTOL_SERVER=$(cabal v2-exec which cryptol-remote-api)
-if [[ -x "$CRYPTOL_SERVER" ]]; then
-  echo "Running cryptol-remote-api tests..."
-  echo "Using server $CRYPTOL_SERVER"
-  python3 -m unittest discover tests/cryptol
-  if [ $? -ne 0 ]; then
-      NUM_FAILS=$(($NUM_FAILS+1))
+export CRYPTOL_SERVER=$(which cryptol-remote-api)
+if [[ ! -x "$CRYPTOL_SERVER" ]]; then
+  export CRYPTOL_SERVER=$(cabal v2-exec which cryptol-remote-api)
+  if [[ ! -x "$CRYPTOL_SERVER" ]]; then
+    echo "could not locate cryptol-remote-api executable"
+    exit 1
   fi
-else
-  echo "could not find the cryptol-remote-api via `cabal v2-exec which`"
-  NUM_FAILS=$(($NUM_FAILS+1))
+fi
+echo "Running cryptol-remote-api tests..."
+echo "Using server $CRYPTOL_SERVER"
+python3 -m unittest discover tests/cryptol
+if [ $? -ne 0 ]; then
+    NUM_FAILS=$(($NUM_FAILS+1))
 fi
 
-export CRYPTOL_SERVER=$(cabal v2-exec which cryptol-eval-server)
-if [[ -x "$CRYPTOL_SERVER" ]]; then
-  echo "Running cryptol-eval-server tests..."
-  echo "Using server $CRYPTOL_SERVER"
-  python3 -m unittest discover tests/cryptol_eval
-  if [ $? -ne 0 ]; then
-      NUM_FAILS=$(($NUM_FAILS+1))
+export CRYPTOL_SERVER=$(which cryptol-eval-server)
+if [[ ! -x "$CRYPTOL_SERVER" ]]; then
+  export CRYPTOL_SERVER=$(cabal v2-exec which cryptol-eval-server)
+  if [[ ! -x "$CRYPTOL_SERVER" ]]; then
+    echo "could not locate cryptol-eval-server executable"
+    exit 1
   fi
-else
-  echo "could not find the cryptol-eval-server via `cabal v2-exec which`"
-  NUM_FAILS=$(($NUM_FAILS+1))
 fi
-popd
+echo "Running cryptol-eval-server tests..."
+echo "Using server $CRYPTOL_SERVER"
+python3 -m unittest discover tests/cryptol_eval
+if [ $? -ne 0 ]; then
+    NUM_FAILS=$(($NUM_FAILS+1))
+fi
 
 popd