Skip to content

Commit

Permalink
Parallelize test-lib and RPC tests
Browse files Browse the repository at this point in the history
  • Loading branch information
Lisanna Dettwyler committed Apr 28, 2021
1 parent f06a741 commit 4a46412
Show file tree
Hide file tree
Showing 3 changed files with 87 additions and 41 deletions.
10 changes: 0 additions & 10 deletions .github/ci.sh
Original file line number Diff line number Diff line change
Expand Up @@ -37,10 +37,6 @@ retry() {
done
}

setup_external_tools() {
cabal v2-install --install-method=copy --installdir="$BIN" test-lib
}

setup_dist_bins() {
extract_exe "cryptol" "dist/bin"
extract_exe "cryptol-html" "dist/bin"
Expand Down Expand Up @@ -125,12 +121,6 @@ install_system_deps() {
is_exe "$BIN" z3 && is_exe "$BIN" cvc4 && is_exe "$BIN" yices
}

test_dist() {
setup_external_tools
echo "test-runner version: $($BIN/test-runner --version)"
$BIN/test-runner --ext=.icry -F -b --exe=dist/bin/cryptol tests
}

check_docs() {
./cry build exe:check-exercises
find ./docs/ProgrammingCryptol -name '*.tex' -print0 | xargs -0 -n1 cabal v2-exec check-exercises
Expand Down
68 changes: 63 additions & 5 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,8 @@ jobs:
# https://gitlab.haskell.org/ghc/ghc/-/issues/18550
- os: windows-latest
ghc: 8.10.2
outputs:
test-lib-json: ${{ steps.test-lib.outputs.targets-json }}
env:
VERSION: ${{ needs.config.outputs.version }}
RELEASE: ${{ needs.config.outputs.release }}
Expand All @@ -71,7 +73,7 @@ jobs:
with:
python-version: '3.x'

- uses: actions/setup-haskell@v1
- uses: haskell/actions/setup@v1
id: setup-haskell
with:
ghc-version: ${{ matrix.ghc }}
Expand Down Expand Up @@ -113,6 +115,28 @@ jobs:
make
"
- shell: bash
name: Partition test-lib tests
id: test-lib
run: |
set -x
cabal v2-install --install-method=copy --installdir="./bin" test-lib
stdout=$(mktemp)
cat > $stdout << EndOfScript
if ${{ runner.os == 'Windows' }}; then
file="\$(cygpath -u "\$1").stdout"
else
file="\$1.stdout"
fi
echo "\$file" >> $PWD/lol
cat "\$file"
EndOfScript
${{ runner.os == 'Windows' }} || chmod +x $stdout
./bin/test-runner --ext=.icry -r ./output --exe=bash -F $stdout ./tests || true
cat ./lol
TARGETS_JSON=$(echo -n "$(ls -1 ./output/tests)" | jq -Rsc 'split("\n")')
echo "::set-output name=targets-json::$TARGETS_JSON"
- shell: bash
run: .github/ci.sh bundle_files

Expand Down Expand Up @@ -146,6 +170,12 @@ jobs:
if-no-files-found: error
retention-days: ${{ needs.config.outputs.retention-days }}

- if: matrix.ghc == '8.6.5'
uses: actions/upload-artifact@v2
with:
path: dist/bin
name: ${{ runner.os }}-bins

- uses: actions/upload-artifact@v2
if: runner.os == 'Windows'
with:
Expand All @@ -154,13 +184,41 @@ jobs:
if-no-files-found: error
retention-days: ${{ needs.config.outputs.retention-days }}

test-lib:
runs-on: ${{ matrix.os }}
needs: [build]
strategy:
fail-fast: false
matrix:
target: ${{ fromJson(needs.build.outputs.test-lib-json) }}
os: [ubuntu-latest, macos-latest, windows-latest]
steps:
- uses: actions/download-artifact@v2
with:
name: "${{ runner.os }}-bins"
path: dist/bin
- shell: bash
run: .github/ci.sh test_dist
run: |
cabal v2-install --install-method=copy --installdir="./bin" test-lib
echo "test-runner version: $($BIN/test-runner --version)"
./bin/test-runner --ext=.icry -F -b --exe=dist/bin/cryptol ./tests/${{ matrix.target }}
# TODO: get Python client to work on Windows
test-rpc:
runs-on: ${{ matrix.os }}
needs: [build]
strategy:
fail-fast: false
matrix:
os: [ubuntu-latest, macos-latest, windows-latest]
exclude:
- os: windows-latest # TODO: get Python client to work on Windows
steps:
- uses: actions/download-artifact@v2
with:
name: "${{ runner.os }}-bins"
path: dist/bin
- shell: bash
run: .github/ci.sh test_rpc
if: runner.os != 'Windows'
run: PATH=$PWD/dist/bin:$PATH cryptol-remote-api/run_rpc_tests.sh

build-push-image:
runs-on: ubuntu-latest
Expand Down
50 changes: 24 additions & 26 deletions cryptol-remote-api/run_rpc_tests.sh
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,6 @@

DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"

pushd $DIR

cabal v2-build exe:cryptol-remote-api
cabal v2-build exe:cryptol-eval-server

pushd $DIR/python

NUM_FAILS=0
Expand All @@ -23,32 +18,35 @@ if [ $? -ne 0 ]; then
NUM_FAILS=$(($NUM_FAILS+1))
fi

export CRYPTOL_SERVER=$(cabal v2-exec which cryptol-remote-api)
if [[ -x "$CRYPTOL_SERVER" ]]; then
echo "Running cryptol-remote-api tests..."
echo "Using server $CRYPTOL_SERVER"
python3 -m unittest discover tests/cryptol
if [ $? -ne 0 ]; then
NUM_FAILS=$(($NUM_FAILS+1))
export CRYPTOL_SERVER=$(which cryptol-remote-api)
if [[ ! -x "$CRYPTOL_SERVER" ]]; then
export CRYPTOL_SERVER=$(cabal v2-exec which cryptol-remote-api)
if [[ ! -x "$CRYPTOL_SERVER" ]]; then
echo "could not locate cryptol-remote-api executable"
exit 1
fi
else
echo "could not find the cryptol-remote-api via `cabal v2-exec which`"
NUM_FAILS=$(($NUM_FAILS+1))
fi
echo "Running cryptol-remote-api tests..."
echo "Using server $CRYPTOL_SERVER"
python3 -m unittest discover tests/cryptol
if [ $? -ne 0 ]; then
NUM_FAILS=$(($NUM_FAILS+1))
fi

export CRYPTOL_SERVER=$(cabal v2-exec which cryptol-eval-server)
if [[ -x "$CRYPTOL_SERVER" ]]; then
echo "Running cryptol-eval-server tests..."
echo "Using server $CRYPTOL_SERVER"
python3 -m unittest discover tests/cryptol_eval
if [ $? -ne 0 ]; then
NUM_FAILS=$(($NUM_FAILS+1))
export CRYPTOL_SERVER=$(which cryptol-eval-server)
if [[ ! -x "$CRYPTOL_SERVER" ]]; then
export CRYPTOL_SERVER=$(cabal v2-exec which cryptol-eval-server)
if [[ ! -x "$CRYPTOL_SERVER" ]]; then
echo "could not locate cryptol-eval-server executable"
exit 1
fi
else
echo "could not find the cryptol-eval-server via `cabal v2-exec which`"
NUM_FAILS=$(($NUM_FAILS+1))
fi
popd
echo "Running cryptol-eval-server tests..."
echo "Using server $CRYPTOL_SERVER"
python3 -m unittest discover tests/cryptol_eval
if [ $? -ne 0 ]; then
NUM_FAILS=$(($NUM_FAILS+1))
fi

popd

Expand Down

0 comments on commit 4a46412

Please sign in to comment.