diff --git a/.github/actions/benchmark/action.yml b/.github/actions/benchmark/action.yml index 75a64ed7df..b2a923f81f 100644 --- a/.github/actions/benchmark/action.yml +++ b/.github/actions/benchmark/action.yml @@ -7,34 +7,34 @@ inputs: system: description: 'Name of system (e.g. GPU name)' required: true - options: + futhark-options: description: 'Options to pass to futhark bench' required: false default: '' + slurm-options: + description: 'Options to pass to srun' + required: false + default: '' runs: using: "composite" steps: - - uses: actions/download-artifact@v3 - with: - name: futhark-nightly-linux-x86_64.tar.xz - - name: Setup compiler - shell: bash - run: | - tar xvf futhark-nightly-linux-x86_64.tar.xz - echo "$PWD/futhark-nightly-linux-x86_64/bin" >> $GITHUB_PATH - - name: Get data + - name: Download Benchmarks. shell: bash run: | - cd futhark-benchmarks && pwd && ./get-data.sh external-data.txt - - name: Debug - shell: bash - run: | - env - - name: Run benchmarks - shell: bash - run: | - futhark bench futhark-benchmarks --ignore-files /lib/ --backend=${{inputs.backend}} --json futhark-${{inputs.backend}}-${{inputs.system}}-$GITHUB_SHA.json --exclude no_${{inputs.system}} ${{inputs.options}} + cd futhark-benchmarks + ./get-data.sh external-data.txt + + - uses: ./.github/actions/futhark-slurm + with: + script: | + futhark bench futhark-benchmarks \ + --backend ${{inputs.backend}} \ + --exclude no_${{inputs.system}} \ + --json futhark-${{inputs.backend}}-${{inputs.system}}-$GITHUB_SHA.json \ + --ignore-files /lib/ ${{inputs.futhark-options}} + slurm-options: ${{inputs.slurm-options}} + - uses: actions/upload-artifact@v1 with: name: futhark-${{inputs.backend}}-${{inputs.system}}-${{ github.sha }}.json - path: futhark-${{inputs.backend}}-${{inputs.system}}-${{ github.sha }}.json + path: futhark-${{inputs.backend}}-${{inputs.system}}-${{ github.sha }}.json \ No newline at end of file diff --git a/.github/actions/futhark-slurm/action.yml b/.github/actions/futhark-slurm/action.yml new file mode 100644 index 0000000000..a5f7e6c531 --- /dev/null +++ b/.github/actions/futhark-slurm/action.yml @@ -0,0 +1,40 @@ +name: 'Futhark Script' +description: 'Run script on slurm if available where Futhark will be available.' +inputs: + script: + description: 'Script to run' + required: true + slurm-options: + description: 'Options to pass to srun' + required: false + default: '' +runs: + using: "composite" + steps: + - uses: actions/download-artifact@v3 + with: + name: futhark-nightly-linux-x86_64.tar.xz + + - name: Install from nightly tarball + shell: bash + run: | + tar xvf futhark-nightly-linux-x86_64.tar.xz + make -C futhark-nightly-linux-x86_64/ install PREFIX=$HOME/.local + echo "$HOME/.local/bin" >> $GITHUB_PATH + + - uses: ./.github/actions/is-slurm + id: slurm + + - if: steps.slurm.outputs.is-slurm == 'false' + shell: bash + run: | + ${{inputs.script}} + + - if: steps.slurm.outputs.is-slurm == 'true' + shell: bash + run: | + printf '#!/bin/bash + ${{inputs.script}}' > temp.sh + chmod +x temp.sh + srun --time=0-08:00:00 ${{inputs.slurm-options}} temp.sh + rm temp.sh diff --git a/.github/actions/is-slurm/action.yml b/.github/actions/is-slurm/action.yml new file mode 100644 index 0000000000..1d2f8e63da --- /dev/null +++ b/.github/actions/is-slurm/action.yml @@ -0,0 +1,21 @@ +name: 'Is Slurm Installed' +description: 'Checks if slurm is installed by checking if srun can be used.' + +outputs: + is-slurm: + description: "If slurm is used." + value: ${{ steps.slurm.outputs.is-slurm }} + +runs: + using: "composite" + steps: + - name: Check if slurm can be run + id: slurm + shell: bash + run: | + printf '#!/bin/bash + if ! srun --version &> /dev/null; then + echo "is-slurm=false" + else + echo "is-slurm=true" + fi' | bash >> $GITHUB_OUTPUT diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 9bd41141e4..b1d54a0814 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -185,78 +185,55 @@ jobs: # nix-shell --pure --run "make docs" test-interpreter: - runs-on: ubuntu-22.04 + runs-on: hendrix needs: [build-linux-nix] - + if: github.repository == 'diku-dk/futhark' steps: - uses: actions/checkout@v3 - - uses: actions/download-artifact@v3 + - uses: ./.github/actions/futhark-slurm with: - name: futhark-nightly-linux-x86_64.tar.xz - - - name: Install from nightly tarball - run: | - tar xvf futhark-nightly-linux-x86_64.tar.xz - make -C futhark-nightly-linux-x86_64/ install PREFIX=$HOME/.local - echo "$HOME/.local/bin" >> $GITHUB_PATH - - - run: | - futhark test -i tests + script: | + futhark test -i tests + slurm-options: --cpus-per-task=2 test-c: - runs-on: ubuntu-22.04 + runs-on: hendrix needs: [build-linux-nix] - + if: github.repository == 'diku-dk/futhark' steps: - uses: actions/checkout@v3 - - name: Install dependencies - run: | - sudo apt-get update - sudo apt-get install -y python3-jsonschema - - - uses: actions/download-artifact@v3 - with: - name: futhark-nightly-linux-x86_64.tar.xz - - - name: Install from nightly tarball - run: | - tar xvf futhark-nightly-linux-x86_64.tar.xz - make -C futhark-nightly-linux-x86_64/ install PREFIX=$HOME/.local - echo "$HOME/.local/bin" >> $GITHUB_PATH - - run: | + module load python/3.9.9 + pip install jsonschema export CFLAGS="-fsanitize=undefined -fsanitize=address -fno-sanitize-recover -O" - futhark test -c --backend=c tests --no-tuning - make -C tests_lib/c -j + + - uses: ./.github/actions/futhark-slurm + with: + script: | + futhark test tests -c --backend=c --no-tuning + make -C tests_lib/c -j + slurm-options: --cpus-per-task=2 test-multicore: - runs-on: ubuntu-22.04 + runs-on: hendrix needs: [build-linux-nix] - + if: github.repository == 'diku-dk/futhark' steps: - uses: actions/checkout@v3 - - name: Install dependencies - run: | - sudo apt-get update - sudo apt-get install -y python3-jsonschema - - - uses: actions/download-artifact@v3 - with: - name: futhark-nightly-linux-x86_64.tar.xz - - - name: Install from nightly tarball - run: | - tar xvf futhark-nightly-linux-x86_64.tar.xz - make -C futhark-nightly-linux-x86_64/ install PREFIX=$HOME/.local - echo "$HOME/.local/bin" >> $GITHUB_PATH - - run: | + module load python/3.9.9 + pip install jsonschema export CFLAGS="-fsanitize=undefined -fsanitize=address -fno-sanitize-recover -O" - futhark test -c --backend=multicore tests --no-tuning - make -C tests_lib/c FUTHARK_BACKEND=multicore + + - uses: ./.github/actions/futhark-slurm + with: + script: | + futhark test tests -c --backend=multicore --no-tuning + make -C tests_lib/c FUTHARK_BACKEND=multicore + slurm-options: --cpus-per-task=2 test-ispc: runs-on: ubuntu-22.04 @@ -289,30 +266,27 @@ jobs: nix-shell --run 'make -C tests_lib/c FUTHARK_BACKEND=ispc' test-python: - runs-on: ubuntu-22.04 + runs-on: hendrix needs: [build-linux-nix] - + if: github.repository == 'diku-dk/futhark' steps: - uses: actions/checkout@v3 - - name: Install dependencies - run: | - sudo apt-get update - sudo apt-get install -y python3-numpy + - run: | + module load python/3.9.9 + pip install numpy - - uses: actions/download-artifact@v3 + - uses: ./.github/actions/futhark-slurm with: - name: futhark-nightly-linux-x86_64.tar.xz - - - name: Install from nightly tarball - run: | - tar xvf futhark-nightly-linux-x86_64.tar.xz - make -C futhark-nightly-linux-x86_64/ install PREFIX=$HOME/.local - echo "$HOME/.local/bin" >> $GITHUB_PATH - - - run: | - futhark test -c --no-terminal --no-tuning --backend=python --exclude=no_python --exclude=compiled tests - make -C tests_lib/python -j + script: | + futhark test tests \ + -c \ + --no-tuning \ + --backend=python \ + --exclude=no_python \ + --exclude=compiled + make -C tests_lib/python -j + slurm-options: --cpus-per-task=2 test-oclgrind: runs-on: ubuntu-22.04 @@ -335,74 +309,68 @@ jobs: futhark test tests -c --no-terminal --backend=opencl --exclude=compiled --cache-extension=cache --pass-option=--build-option=-O0 --runner=tools/oclgrindrunner.sh test-opencl: - runs-on: opencl + runs-on: hendrix needs: [build-linux-nix] if: github.repository == 'diku-dk/futhark' steps: - uses: actions/checkout@v3 - - uses: actions/download-artifact@v3 - with: - name: futhark-nightly-linux-x86_64.tar.xz - - name: Setup compiler - run: | - tar xvf futhark-nightly-linux-x86_64.tar.xz - echo "$PWD/futhark-nightly-linux-x86_64/bin" >> $GITHUB_PATH + - run: | - futhark test tests --no-terminal --backend=opencl --cache-extension=cache --concurrency=16 - - name: Setup python - run: | - virtualenv --no-site-packages virtualenv + module load python/3.9.9 + virtualenv virtualenv source virtualenv/bin/activate pip install jsonschema - - run: | - source virtualenv/bin/activate - FUTHARK_BACKEND=opencl make -C tests_lib/c -j + + - uses: ./.github/actions/futhark-slurm + with: + script: | + futhark test tests \ + --backend=opencl \ + --cache-extension=cache + source virtualenv/bin/activate + FUTHARK_BACKEND=opencl make -C tests_lib/c -j + slurm-options: -p gpu --gres=gpu:1 test-pyopencl: - runs-on: opencl + runs-on: hendrix needs: [build-linux-nix] if: github.repository == 'diku-dk/futhark' steps: - uses: actions/checkout@v3 - - uses: actions/download-artifact@v3 - with: - name: futhark-nightly-linux-x86_64.tar.xz - - name: Setup compiler - run: | - tar xvf futhark-nightly-linux-x86_64.tar.xz - echo "$PWD/futhark-nightly-linux-x86_64/bin" >> $GITHUB_PATH - - name: Setup python - run: | - virtualenv --no-site-packages virtualenv - source virtualenv/bin/activate - pip install numpy pyopencl jsonschema + - run: | + module load python/3.9.9 + virtualenv virtualenv source virtualenv/bin/activate - futhark test tests --no-terminal --backend=pyopencl --concurrency=16 + pip install numpy pyopencl jsonschema + + - uses: ./.github/actions/futhark-slurm + with: + script: | + source virtualenv/bin/activate + futhark test tests --no-terminal --backend=pyopencl + slurm-options: -p gpu --gres=gpu:1 test-cuda: - runs-on: cuda + runs-on: hendrix needs: [build-linux-nix] if: github.repository == 'diku-dk/futhark' steps: - uses: actions/checkout@v3 - - uses: actions/download-artifact@v3 - with: - name: futhark-nightly-linux-x86_64.tar.xz - - name: Setup compiler - run: | - tar xvf futhark-nightly-linux-x86_64.tar.xz - echo "$PWD/futhark-nightly-linux-x86_64/bin" >> $GITHUB_PATH + - run: | - futhark test tests --no-terminal --backend=cuda --concurrency=16 - - name: Setup python - run: | - virtualenv --no-site-packages virtualenv + module load python/3.9.9 + virtualenv virtualenv source virtualenv/bin/activate pip install jsonschema - - run: | - source virtualenv/bin/activate - FUTHARK_BACKEND=cuda make -C tests_lib/c -j + + - uses: ./.github/actions/futhark-slurm + with: + script: | + futhark test tests --no-terminal --backend=cuda + source virtualenv/bin/activate + FUTHARK_BACKEND=cuda make -C tests_lib/c -j + slurm-options: -p gpu --gres=gpu:1 test-wasm: runs-on: ubuntu-22.04 @@ -434,7 +402,12 @@ jobs: run: | node --version export EMCFLAGS="-sINITIAL_MEMORY=2147418112 -O1" # 2gb - 64kb... largest value of memory - futhark test -c --backend=wasm --runner=./tools/node-simd.sh --no-tuning --exclude=no_wasm tests + futhark test \ + -c \ + --backend=wasm \ + --runner=./tools/node-simd.sh \ + --no-tuning \ + --exclude=no_wasm tests test-wasm-multicore: runs-on: ubuntu-22.04 @@ -612,9 +585,71 @@ jobs: backend: opencl system: MI100 + benchmark-titanx-cuda: + runs-on: hendrix + needs: [build-linux-nix] + if: github.repository == 'diku-dk/futhark' && !github.event.pull_request.draft + steps: + - uses: actions/checkout@v3 + with: + submodules: recursive + fetch-depth: 0 + - uses: ./.github/actions/benchmark + with: + backend: cuda + system: titanx + slurm-options: -p gpu --gres=gpu:titanx:1 + futhark-options: --exclude=mem_16gb + + benchmark-titanx-opencl: + runs-on: hendrix + needs: [build-linux-nix] + if: github.repository == 'diku-dk/futhark' && !github.event.pull_request.draft + steps: + - uses: actions/checkout@v3 + with: + submodules: recursive + fetch-depth: 0 + - uses: ./.github/actions/benchmark + with: + backend: opencl + system: titanx + slurm-options: -p gpu --gres=gpu:titanx:1 + futhark-options: --exclude=mem_16gb + + benchmark-titanrtx-cuda: + runs-on: hendrix + needs: [build-linux-nix] + if: github.repository == 'diku-dk/futhark' && !github.event.pull_request.draft + steps: + - uses: actions/checkout@v3 + with: + submodules: recursive + fetch-depth: 0 + - uses: ./.github/actions/benchmark + with: + backend: cuda + system: titanrtx + slurm-options: -p gpu --gres=gpu:titanrtx:1 + + benchmark-titanrtx-opencl: + runs-on: hendrix + needs: [build-linux-nix] + if: github.repository == 'diku-dk/futhark' && !github.event.pull_request.draft + steps: + - uses: actions/checkout@v3 + with: + submodules: recursive + fetch-depth: 0 + - uses: ./.github/actions/benchmark + with: + backend: opencl + system: titanrtx + slurm-options: -p gpu --gres=gpu:titanrtx:1 + benchmark-results: runs-on: ubuntu-22.04 - needs: [benchmark-A100-cuda, benchmark-A100-opencl, benchmark-MI100-opencl] + needs: [benchmark-A100-cuda, benchmark-A100-opencl, benchmark-MI100-opencl, benchmark-titanx-cuda, benchmark-titanx-opencl, benchmark-titanrtx-cuda, benchmark-titanrtx-opencl] if: github.ref == 'refs/heads/master' steps: - name: Install SSH key