CI #3782
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| on: | |
| push: | |
| branches: [main] | |
| pull_request: | |
| schedule: | |
| - cron: "35 03 * * *" # Daily at 8:35 PM PDT, 7:35 PM PST. | |
| workflow_dispatch: | |
| inputs: | |
| should_bench: | |
| description: "Should Benchmark? (`true`)" | |
| required: true | |
| default: "false" | |
| jobs: | |
| pre_job: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| should_skip: ${{ steps.skip_check.outputs.should_skip }} | |
| steps: | |
| - id: skip_check | |
| uses: fkirc/skip-duplicate-actions@v5.3.0 | |
| with: | |
| cancel_others: "true" | |
| check: | |
| name: Check | |
| if: ${{ needs.pre_job.outputs.should_skip != 'true' || github.event_name != 'pull_request' }} | |
| timeout-minutes: 10 | |
| needs: pre_job | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| rust_release: [pinned-nightly, latest-nightly] | |
| exclude: | |
| # For non-pull requests, event_name != 'pull_request' will be true, | |
| # and nothing is truthy, so the entire && operator will resolve to | |
| # 'nothing'. Then the || operator will resolve to 'nothing' so we | |
| # will exclude 'nothing'. https://stackoverflow.com/a/73822998 | |
| - rust_release: ${{ (needs.pre_job.outputs.should_skip != 'true' && 'nothing') || 'pinned-nightly' }} | |
| - rust_release: ${{ (github.event_name != 'pull_request' && 'nothing') || 'latest-nightly' }} | |
| steps: | |
| - name: Checkout sources | |
| uses: actions/checkout@v3 | |
| - name: Install nightly toolchain | |
| uses: actions-rs/toolchain@v1 | |
| with: | |
| profile: minimal | |
| toolchain: nightly | |
| override: ${{ matrix.rust_release == 'latest-nightly' }} | |
| - name: Run cargo check | |
| uses: actions-rs/cargo@v1 | |
| with: | |
| command: check | |
| args: --all-targets --features python | |
| check-wasm: | |
| name: Check WebAssembly | |
| if: ${{ needs.pre_job.outputs.should_skip != 'true' || github.event_name != 'pull_request' }} | |
| timeout-minutes: 10 | |
| needs: pre_job | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| rust_release: [pinned-nightly, latest-nightly] | |
| exclude: | |
| # For non-pull requests, event_name != 'pull_request' will be true, | |
| # and nothing is truthy, so the entire && operator will resolve to | |
| # 'nothing'. Then the || operator will resolve to 'nothing' so we | |
| # will exclude 'nothing'. https://stackoverflow.com/a/73822998 | |
| - rust_release: ${{ (needs.pre_job.outputs.should_skip != 'true' && 'nothing') || 'pinned-nightly' }} | |
| - rust_release: ${{ (github.event_name != 'pull_request' && 'nothing') || 'latest-nightly' }} | |
| steps: | |
| - name: Checkout sources | |
| uses: actions/checkout@v3 | |
| - name: Install nightly toolchain | |
| uses: actions-rs/toolchain@v1 | |
| with: | |
| profile: minimal | |
| toolchain: nightly | |
| target: wasm32-unknown-unknown | |
| override: ${{ matrix.rust_release == 'latest-nightly' }} | |
| - name: Check hydroflow_lang | |
| uses: actions-rs/cargo@v1 | |
| with: | |
| command: check | |
| args: -p hydroflow_lang --target wasm32-unknown-unknown | |
| test: | |
| name: Test Suite | |
| if: ${{ needs.pre_job.outputs.should_skip != 'true' || github.event_name != 'pull_request' }} | |
| timeout-minutes: 25 | |
| needs: pre_job | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| rust_release: [pinned-nightly, latest-nightly] | |
| exclude: | |
| # For non-pull requests, event_name != 'pull_request' will be true, | |
| # and nothing is truthy, so the entire && operator will resolve to | |
| # 'nothing'. Then the || operator will resolve to 'nothing' so we | |
| # will exclude 'nothing'. https://stackoverflow.com/a/73822998 | |
| - rust_release: ${{ (needs.pre_job.outputs.should_skip != 'true' && 'nothing') || 'pinned-nightly' }} | |
| - rust_release: ${{ (github.event_name != 'pull_request' && 'nothing') || 'latest-nightly' }} | |
| steps: | |
| - name: Checkout sources | |
| uses: actions/checkout@v3 | |
| - name: Install nightly toolchain | |
| uses: actions-rs/toolchain@v1 | |
| with: | |
| profile: minimal | |
| toolchain: nightly | |
| override: ${{ matrix.rust_release == 'latest-nightly' }} | |
| - name: Run cargo test all targets (which does not include doctests) | |
| uses: actions-rs/cargo@v1 | |
| with: | |
| command: test | |
| args: --no-fail-fast --all-targets --features python | |
| - name: Run cargo doc tests | |
| uses: actions-rs/cargo@v1 | |
| with: | |
| command: test | |
| args: --no-fail-fast --doc --features python | |
| test-wasm: | |
| name: Test Suite (WebAssembly) | |
| if: ${{ needs.pre_job.outputs.should_skip != 'true' || github.event_name != 'pull_request' }} | |
| timeout-minutes: 15 | |
| needs: pre_job | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| rust_release: [pinned-nightly, latest-nightly] | |
| exclude: | |
| # For non-pull requests, event_name != 'pull_request' will be true, | |
| # and nothing is truthy, so the entire && operator will resolve to | |
| # 'nothing'. Then the || operator will resolve to 'nothing' so we | |
| # will exclude 'nothing'. https://stackoverflow.com/a/73822998 | |
| - rust_release: ${{ (needs.pre_job.outputs.should_skip != 'true' && 'nothing') || 'pinned-nightly' }} | |
| - rust_release: ${{ (github.event_name != 'pull_request' && 'nothing') || 'latest-nightly' }} | |
| steps: | |
| - name: Checkout sources | |
| uses: actions/checkout@v3 | |
| - name: Install nightly toolchain | |
| uses: actions-rs/toolchain@v1 | |
| with: | |
| profile: minimal | |
| toolchain: nightly | |
| target: wasm32-unknown-unknown | |
| override: ${{ matrix.rust_release == 'latest-nightly' }} | |
| - name: Get wasm-bindgen version | |
| id: wasm-bindgen-version | |
| run: echo "VERSION=$(cargo pkgid wasm-bindgen-shared | cut -d '@' -f2)" >> "$GITHUB_OUTPUT" | |
| - name: Install WebAssembly test runner | |
| uses: actions-rs/cargo@v1 | |
| with: | |
| command: install | |
| args: wasm-bindgen-cli@${{ steps.wasm-bindgen-version.outputs.VERSION }} | |
| - name: Run cargo test | |
| uses: actions-rs/cargo@v1 | |
| env: | |
| CARGO_TARGET_WASM32_UNKNOWN_UNKNOWN_RUNNER: wasm-bindgen-test-runner | |
| with: | |
| command: test | |
| args: -p hydroflow --target wasm32-unknown-unknown --tests --no-fail-fast | |
| test-cli: | |
| name: Test CLI | |
| if: ${{ needs.pre_job.outputs.should_skip != 'true' || github.event_name != 'pull_request' }} | |
| timeout-minutes: 25 | |
| needs: pre_job | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout sources | |
| uses: actions/checkout@v3 | |
| - name: Install nightly toolchain | |
| uses: actions-rs/toolchain@v1 | |
| with: | |
| profile: minimal | |
| toolchain: nightly | |
| - name: Install Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.11' | |
| - name: Build CLI | |
| run: | | |
| cd hydro_cli | |
| python -m venv .venv | |
| source .venv/bin/activate | |
| pip install maturin | |
| maturin develop | |
| - name: Configure core dumps | |
| run: | | |
| mkdir cli-core-dumps | |
| echo "$PWD/cli-core-dumps/corefile-%e-%p-%t" | sudo tee /proc/sys/kernel/core_pattern | |
| - name: Run Python tests | |
| run: | | |
| ulimit -c unlimited | |
| cd hydro_cli | |
| source .venv/bin/activate | |
| cd python_tests | |
| pip install -r requirements.txt | |
| RUST_BACKTRACE=1 pytest *.py | |
| - name: Print backtraces | |
| if: ${{ failure() }} | |
| run: | | |
| cd cli-core-dumps | |
| sudo apt-get install gdb | |
| for file in $(ls); do | |
| echo "Backtrace for $file" | |
| rust-gdb -ex "thread apply all bt" -batch $PWD/../.venv/bin/python $file | |
| done | |
| cd .. | |
| - name: Upload core dumps | |
| if: ${{ failure() }} | |
| uses: actions/upload-artifact@master | |
| with: | |
| name: cli-core-dumps | |
| path: ./cli-core-dumps/* | |
| lints: | |
| name: Lints | |
| if: ${{ needs.pre_job.outputs.should_skip != 'true' || github.event_name != 'pull_request' }} | |
| timeout-minutes: 10 | |
| needs: pre_job | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| rust_release: [pinned-nightly, latest-nightly] | |
| exclude: | |
| # For non-pull requests, event_name != 'pull_request' will be true, | |
| # and nothing is truthy, so the entire && operator will resolve to | |
| # 'nothing'. Then the || operator will resolve to 'nothing' so we | |
| # will exclude 'nothing'. https://stackoverflow.com/a/73822998 | |
| - rust_release: ${{ (needs.pre_job.outputs.should_skip != 'true' && 'nothing') || 'pinned-nightly' }} | |
| - rust_release: ${{ (github.event_name != 'pull_request' && 'nothing') || 'latest-nightly' }} | |
| steps: | |
| - name: Checkout sources | |
| uses: actions/checkout@v3 | |
| - name: Install nightly toolchain | |
| uses: actions-rs/toolchain@v1 | |
| with: | |
| profile: minimal | |
| toolchain: nightly | |
| override: ${{ matrix.rust_release == 'latest-nightly' }} | |
| components: rustfmt, clippy | |
| - name: Run cargo fmt | |
| uses: actions-rs/cargo@v1 | |
| with: | |
| command: fmt | |
| args: --all -- --check | |
| - name: Run cargo clippy | |
| uses: actions-rs/cargo@v1 | |
| with: | |
| command: clippy | |
| args: --all-targets --features python -- -D warnings | |
| build-website: | |
| name: Build Website | |
| if: ${{ needs.pre_job.outputs.should_skip != 'true' || github.event_name != 'pull_request' }} | |
| timeout-minutes: 25 | |
| needs: pre_job | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout sources | |
| uses: actions/checkout@v3 | |
| - name: Build Website | |
| run: bash build_docs.bash x86_64-linux-gnu-ubuntu-20.04 | |
| docs: | |
| name: Docs (rustdoc) | |
| timeout-minutes: 10 | |
| needs: pre_job | |
| if: ${{ needs.pre_job.outputs.should_skip != 'true' }} | |
| runs-on: ubuntu-latest | |
| env: | |
| WWW_DIR: target | |
| steps: | |
| - name: Checkout sources | |
| uses: actions/checkout@v3 | |
| - name: Install nightly toolchain | |
| uses: actions-rs/toolchain@v1 | |
| with: | |
| profile: minimal | |
| toolchain: nightly | |
| - name: Run cargo doc | |
| uses: actions-rs/cargo@v1 | |
| with: | |
| command: doc | |
| args: --no-deps | |
| env: | |
| RUSTDOCFLAGS: -Dwarnings | |
| - name: Move design docs to output | |
| shell: bash | |
| run: | | |
| mv design_docs "$WWW_DIR/" | |
| - name: Push to gh-pages | |
| # Do not deploy PRs. | |
| if: ${{ github.event_name != 'pull_request' }} | |
| shell: bash | |
| run: | | |
| git -C "$WWW_DIR" init -q | |
| git -C "$WWW_DIR" remote add origin "$(git remote get-url origin)" | |
| git -C "$WWW_DIR" config credential.helper "$(git config credential.helper)" | |
| git -C "$WWW_DIR" config 'http.https://github.com/.extraheader' "$(git config 'http.https://github.com/.extraheader')" | |
| git -C "$WWW_DIR" config core.autocrlf input | |
| git -C "$WWW_DIR" config core.safecrlf false | |
| git -C "$WWW_DIR" fetch origin gh-pages:gh-pages || true | |
| git -C "$WWW_DIR" symbolic-ref HEAD refs/heads/gh-pages | |
| git -C "$WWW_DIR" reset | |
| git -C "$WWW_DIR" add doc design_docs | |
| if git -C "$WWW_DIR" -c 'user.name=github-actions[bot]' -c 'user.email=41898282+github-actions[bot]@users.noreply.github.com' \ | |
| commit -m "Update rustdoc $(date -I) $(git rev-parse HEAD)"; | |
| then | |
| git -C "$WWW_DIR" push -u origin gh-pages --quiet | |
| else | |
| echo 'No changes to commit' | |
| fi | |
| benches: | |
| name: Benchmarks | |
| timeout-minutes: 30 | |
| needs: pre_job | |
| if: | | |
| needs.pre_job.outputs.should_skip != 'true' && | |
| ( | |
| github.event_name == 'schedule' | |
| || (github.event_name == 'workflow_dispatch' && github.event.inputs.should_bench == 'true') | |
| || (github.event_name == 'push' && contains(github.event.head_commit.message, '[ci-bench]')) | |
| || ( | |
| github.event_name == 'pull_request' | |
| && ( | |
| contains(github.event.pull_request.title, '[ci-bench]') | |
| || contains(github.event.pull_request.body, '[ci-bench]') | |
| ) | |
| ) | |
| ) | |
| runs-on: ubuntu-latest | |
| env: | |
| WWW_DIR: target | |
| steps: | |
| - name: Checkout sources | |
| uses: actions/checkout@v3 | |
| - name: Install nightly toolchain | |
| uses: actions-rs/toolchain@v1 | |
| with: | |
| profile: minimal | |
| toolchain: nightly | |
| - name: Checkout gh-pages | |
| shell: bash | |
| run: | | |
| mkdir -p "$WWW_DIR" | |
| git -C "$WWW_DIR" init -q | |
| git -C "$WWW_DIR" remote add origin "$(git remote get-url origin)" | |
| git -C "$WWW_DIR" config credential.helper "$(git config credential.helper)" | |
| git -C "$WWW_DIR" config 'http.https://github.com/.extraheader' "$(git config 'http.https://github.com/.extraheader')" | |
| git -C "$WWW_DIR" config core.autocrlf input | |
| git -C "$WWW_DIR" config core.safecrlf false | |
| git -C "$WWW_DIR" checkout -b gh-pages | |
| git -C "$WWW_DIR" fetch origin gh-pages | |
| git -C "$WWW_DIR" reset --soft origin/gh-pages | |
| git -C "$WWW_DIR" reset | |
| git -C "$WWW_DIR" checkout -- bench criterion | |
| mkdir -p "$WWW_DIR/bench" | |
| - name: Run benchmark | |
| run: | | |
| time cargo bench -p benches -- hydroflow --output-format bencher | tee output.txt | |
| time cargo bench -p benches -- micro/ops/ --output-format bencher | tee -a output.txt | |
| - name: Generate benchmark page | |
| uses: benchmark-action/github-action-benchmark@v1 | |
| with: | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| tool: cargo | |
| output-file-path: output.txt | |
| external-data-json-path: target/bench/data.json | |
| - name: Write benchmark JSON | |
| run: echo 'window.BENCHMARK_DATA = ' | cat - "$WWW_DIR/bench/data.json" > "$WWW_DIR/bench/data.js" | |
| - name: Write benchmark HTML | |
| shell: bash | |
| run: node --input-type=module -e "$(curl -sSL https://raw.githubusercontent.com/benchmark-action/github-action-benchmark/master/src/default_index_html.ts) console.log(DEFAULT_INDEX_HTML)" > "$WWW_DIR/bench/index.html" | |
| - name: Push to gh-pages | |
| # Do not deploy PRs, only benchmark main branch. | |
| if: ${{ github.event_name != 'pull_request' && github.ref == 'refs/heads/main' }} | |
| shell: bash | |
| run: | | |
| cp -r .github/gh-pages/* .github/gh-pages/.gitignore "$WWW_DIR/" | |
| git -C "$WWW_DIR" fetch origin gh-pages | |
| git -C "$WWW_DIR" reset --soft origin/gh-pages | |
| git -C "$WWW_DIR" reset | |
| git -C "$WWW_DIR" add bench criterion $(ls .github/gh-pages) | |
| if git -C "$WWW_DIR" -c 'user.name=github-actions[bot]' -c 'user.email=41898282+github-actions[bot]@users.noreply.github.com' \ | |
| commit -m "Update Benchmarks $(date -I) $(git rev-parse HEAD)"; | |
| then | |
| git -C "$WWW_DIR" push -u origin gh-pages --quiet | |
| else | |
| echo 'No changes to commit' | |
| fi |