refactor(paxos): make leader election generic over logs #6961
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| on: | |
| push: | |
| branches: | |
| - main | |
| - feature/** | |
| pull_request: | |
| schedule: | |
| - cron: "35 03 * * *" # Daily at 8:35 PM PDT, 7:35 PM PST. | |
| workflow_dispatch: | |
| inputs: | |
| should_bench: | |
| description: "Should Benchmark? (`true`)" | |
| required: true | |
| default: "false" | |
| jobs: | |
| pre_job: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| should_skip: ${{ steps.skip_check.outputs.should_skip }} | |
| steps: | |
| - id: skip_check | |
| uses: fkirc/skip-duplicate-actions@v5.3.0 | |
| with: | |
| cancel_others: "true" | |
| check: | |
| name: Lint and Check | |
| if: ${{ needs.pre_job.outputs.should_skip != 'true' || github.event_name != 'pull_request' }} | |
| timeout-minutes: 20 | |
| needs: pre_job | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [ubuntu-latest, windows-latest] | |
| rust_release: [pinned-nightly, latest-nightly] | |
| exclude: | |
| # For non-pull requests, event_name != 'pull_request' will be true, and 'nothing' is | |
| # truthy, so the entire && operator will resolve to 'nothing'. Then the || operator will | |
| # resolve to 'nothing' so we will exclude 'nothing'. https://stackoverflow.com/a/73822998 | |
| - rust_release: ${{ (needs.pre_job.outputs.should_skip != 'true' && 'nothing') || 'pinned-nightly' }} | |
| - rust_release: ${{ (github.event_name != 'pull_request' && 'nothing') || 'latest-nightly' }} | |
| env: | |
| CARGO_TERM_COLOR: always | |
| CARGO_INCREMENTAL: 0 | |
| CARGO_PROFILE_DEV_STRIP: "debuginfo" | |
| CARGO_PROFILE_TEST_STRIP: "debuginfo" | |
| CARGO_PROFILE_RELEASE_STRIP: "debuginfo" | |
| steps: | |
| - name: Checkout sources | |
| uses: actions/checkout@v3 | |
| - name: Install nightly toolchain | |
| uses: actions-rs/toolchain@v1 | |
| with: | |
| profile: minimal | |
| toolchain: nightly | |
| override: ${{ matrix.rust_release == 'latest-nightly' }} | |
| components: rustfmt, clippy | |
| - name: Run sccache-cache | |
| if: matrix.rust_release == 'pinned-nightly' | |
| uses: mozilla-actions/sccache-action@v0.0.4 | |
| - name: Set Rust caching env vars | |
| if: matrix.rust_release == 'pinned-nightly' | |
| uses: actions/github-script@v6 | |
| with: | |
| script: | | |
| core.exportVariable('SCCACHE_GHA_ENABLED', 'true'); | |
| core.exportVariable('RUSTC_WRAPPER', 'sccache'); | |
| - name: Run cargo fmt | |
| if: ${{ matrix.os == 'ubuntu-latest' }} | |
| run: cargo fmt --all -- --check | |
| - name: Run cargo clippy | |
| run: cargo clippy --all-targets --features python -- -D warnings | |
| - name: Run cargo check | |
| run: cargo check --all-targets --features python | |
| - name: Run cargo check (no default features) | |
| run: cargo check --all-targets --no-default-features | |
| check-wasm: | |
| name: Check WebAssembly | |
| if: ${{ needs.pre_job.outputs.should_skip != 'true' || github.event_name != 'pull_request' }} | |
| timeout-minutes: 10 | |
| needs: pre_job | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| rust_release: [pinned-nightly, latest-nightly] | |
| exclude: | |
| # For non-pull requests, event_name != 'pull_request' will be true, and 'nothing' is | |
| # truthy, so the entire && operator will resolve to 'nothing'. Then the || operator will | |
| # resolve to 'nothing' so we will exclude 'nothing'. https://stackoverflow.com/a/73822998 | |
| - rust_release: ${{ (needs.pre_job.outputs.should_skip != 'true' && 'nothing') || 'pinned-nightly' }} | |
| - rust_release: ${{ (github.event_name != 'pull_request' && 'nothing') || 'latest-nightly' }} | |
| steps: | |
| - name: Checkout sources | |
| uses: actions/checkout@v3 | |
| - name: Install nightly toolchain | |
| uses: actions-rs/toolchain@v1 | |
| with: | |
| profile: minimal | |
| toolchain: nightly | |
| target: wasm32-unknown-unknown | |
| override: ${{ matrix.rust_release == 'latest-nightly' }} | |
| - name: Check hydroflow_lang | |
| uses: actions-rs/cargo@v1 | |
| with: | |
| command: check | |
| args: -p hydroflow_lang --target wasm32-unknown-unknown | |
| test: | |
| name: Test Suite | |
| if: ${{ needs.pre_job.outputs.should_skip != 'true' || github.event_name != 'pull_request' }} | |
| timeout-minutes: 35 | |
| needs: pre_job | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [ubuntu-latest, windows-latest] | |
| rust_release: [pinned-nightly, latest-nightly] | |
| exclude: | |
| # For non-pull requests, event_name != 'pull_request' will be true, and 'nothing' is | |
| # truthy, so the entire && operator will resolve to 'nothing'. Then the || operator will | |
| # resolve to 'nothing' so we will exclude 'nothing'. https://stackoverflow.com/a/73822998 | |
| - rust_release: ${{ (needs.pre_job.outputs.should_skip != 'true' && 'nothing') || 'pinned-nightly' }} | |
| - rust_release: ${{ (github.event_name != 'pull_request' && 'nothing') || 'latest-nightly' }} | |
| - os: ${{ (github.event_name != 'pull_request' && 'nothing') || 'windows-latest' }} | |
| env: | |
| CARGO_TERM_COLOR: always | |
| CARGO_INCREMENTAL: 0 | |
| CARGO_PROFILE_DEV_STRIP: "debuginfo" | |
| CARGO_PROFILE_TEST_STRIP: "debuginfo" | |
| CARGO_PROFILE_RELEASE_STRIP: "debuginfo" | |
| steps: | |
| - name: Checkout sources | |
| uses: actions/checkout@v3 | |
| - name: Install nightly toolchain | |
| uses: actions-rs/toolchain@v1 | |
| with: | |
| profile: minimal | |
| toolchain: nightly | |
| override: ${{ matrix.rust_release == 'latest-nightly' }} | |
| - name: Run sccache-cache | |
| if: matrix.rust_release == 'pinned-nightly' | |
| uses: mozilla-actions/sccache-action@v0.0.4 | |
| - name: Set Rust caching env vars | |
| if: matrix.rust_release == 'pinned-nightly' | |
| uses: actions/github-script@v6 | |
| with: | |
| script: | | |
| core.exportVariable('SCCACHE_GHA_ENABLED', 'true'); | |
| core.exportVariable('RUSTC_WRAPPER', 'sccache'); | |
| - name: Install cargo-nextest (linux) | |
| if: ${{ matrix.os == 'ubuntu-latest' }} | |
| run: curl -LsSf https://get.nexte.st/latest/linux | tar zxf - -C ${CARGO_HOME:-~/.cargo}/bin | |
| - name: Install cargo-nextest (windows) | |
| if: ${{ matrix.os == 'windows-latest' }} | |
| shell: bash | |
| run: curl -LsSf https://get.nexte.st/latest/windows-tar | tar zxf - -C ${CARGO_HOME:-~/.cargo}/bin | |
| - name: Run cargo nextest on all targets | |
| run: cargo nextest run --no-fail-fast --features python --features deploy --all-targets | |
| - name: Run doctests | |
| run: cargo test --no-fail-fast --features python --features deploy --doc | |
| - name: Install Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.11' | |
| - name: Build CLI (Linux) | |
| if: ${{ matrix.os == 'ubuntu-latest' }} | |
| run: | | |
| cd hydro_deploy/hydro_cli | |
| python -m venv .venv | |
| source .venv/bin/activate | |
| pip install maturin | |
| maturin develop | |
| - name: Build CLI (Windows) | |
| if: ${{ matrix.os == 'windows-latest' }} | |
| run: | | |
| cd hydro_deploy/hydro_cli | |
| python -m venv .venv | |
| .venv/Scripts/activate | |
| pip install maturin | |
| maturin develop | |
| - name: Run Python tests (Linux) | |
| if: ${{ matrix.os == 'ubuntu-latest' }} | |
| run: | | |
| cd hydro_deploy/hydro_cli | |
| source .venv/bin/activate | |
| cd python_tests | |
| pip install -r requirements.txt | |
| RUST_BACKTRACE=1 pytest | |
| - name: Run Python tests (Windows) | |
| if: ${{ matrix.os == 'windows-latest' }} | |
| run: | | |
| cd hydro_deploy/hydro_cli | |
| .venv/Scripts/activate | |
| cd python_tests | |
| pip install -r requirements.txt | |
| pytest | |
| test-wasm: | |
| name: Test Suite (WebAssembly) | |
| if: ${{ needs.pre_job.outputs.should_skip != 'true' || github.event_name != 'pull_request' }} | |
| timeout-minutes: 15 | |
| needs: pre_job | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| rust_release: [pinned-nightly, latest-nightly] | |
| exclude: | |
| # For non-pull requests, event_name != 'pull_request' will be true, and 'nothing' is | |
| # truthy, so the entire && operator will resolve to 'nothing'. Then the || operator will | |
| # resolve to 'nothing' so we will exclude 'nothing'. https://stackoverflow.com/a/73822998 | |
| - rust_release: ${{ (needs.pre_job.outputs.should_skip != 'true' && 'nothing') || 'pinned-nightly' }} | |
| - rust_release: ${{ (github.event_name != 'pull_request' && 'nothing') || 'latest-nightly' }} | |
| steps: | |
| - name: Checkout sources | |
| uses: actions/checkout@v3 | |
| - name: Install nightly toolchain | |
| uses: actions-rs/toolchain@v1 | |
| with: | |
| profile: minimal | |
| toolchain: nightly | |
| target: wasm32-unknown-unknown | |
| override: ${{ matrix.rust_release == 'latest-nightly' }} | |
| - name: Get wasm-bindgen version | |
| id: wasm-bindgen-version | |
| run: echo "VERSION=$(cargo pkgid wasm-bindgen-shared | cut -d '@' -f2)" >> "$GITHUB_OUTPUT" | |
| - name: Install WebAssembly test runner | |
| uses: actions-rs/cargo@v1 | |
| with: | |
| command: install | |
| args: wasm-bindgen-cli@${{ steps.wasm-bindgen-version.outputs.VERSION }} | |
| - name: Run cargo test | |
| uses: actions-rs/cargo@v1 | |
| env: | |
| CARGO_TARGET_WASM32_UNKNOWN_UNKNOWN_RUNNER: wasm-bindgen-test-runner | |
| with: | |
| command: test | |
| args: -p hydroflow --target wasm32-unknown-unknown --tests --no-fail-fast | |
| build-website: | |
| name: Build Website | |
| if: ${{ needs.pre_job.outputs.should_skip != 'true' || github.event_name != 'pull_request' }} | |
| timeout-minutes: 25 | |
| needs: pre_job | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout sources | |
| uses: actions/checkout@v3 | |
| - name: Build Website | |
| run: bash build_docs.bash x86_64-linux-gnu-ubuntu-20.04 | |
| docs: | |
| name: Docs (rustdoc) | |
| timeout-minutes: 10 | |
| needs: pre_job | |
| if: ${{ needs.pre_job.outputs.should_skip != 'true' }} | |
| runs-on: ubuntu-latest | |
| env: | |
| WWW_DIR: target | |
| steps: | |
| - name: Checkout sources | |
| uses: actions/checkout@v3 | |
| - name: Install nightly toolchain | |
| uses: actions-rs/toolchain@v1 | |
| with: | |
| profile: minimal | |
| toolchain: nightly | |
| - name: Run cargo doc | |
| uses: actions-rs/cargo@v1 | |
| with: | |
| command: doc | |
| args: --no-deps | |
| env: | |
| RUSTDOCFLAGS: -Dwarnings | |
| - name: Move design docs to output | |
| shell: bash | |
| run: | | |
| mv design_docs "$WWW_DIR/" | |
| - name: Push to gh-pages | |
| # Do not deploy PRs. | |
| if: ${{ github.event_name != 'pull_request' }} | |
| shell: bash | |
| run: | | |
| git -C "$WWW_DIR" init -q | |
| git -C "$WWW_DIR" remote add origin "$(git remote get-url origin)" | |
| git -C "$WWW_DIR" config credential.helper "$(git config credential.helper)" | |
| git -C "$WWW_DIR" config 'http.https://github.com/.extraheader' "$(git config 'http.https://github.com/.extraheader')" | |
| git -C "$WWW_DIR" config core.autocrlf input | |
| git -C "$WWW_DIR" config core.safecrlf false | |
| git -C "$WWW_DIR" fetch origin gh-pages:gh-pages || true | |
| git -C "$WWW_DIR" symbolic-ref HEAD refs/heads/gh-pages | |
| git -C "$WWW_DIR" reset | |
| git -C "$WWW_DIR" add doc design_docs | |
| if git -C "$WWW_DIR" -c 'user.name=github-actions[bot]' -c 'user.email=41898282+github-actions[bot]@users.noreply.github.com' \ | |
| commit -m "Update rustdoc $(date -I) $(git rev-parse HEAD)"; | |
| then | |
| git -C "$WWW_DIR" push -u origin gh-pages --quiet | |
| else | |
| echo 'No changes to commit' | |
| fi | |
| benches: | |
| name: Benchmarks | |
| timeout-minutes: 30 | |
| needs: pre_job | |
| if: | | |
| needs.pre_job.outputs.should_skip != 'true' && | |
| ( | |
| github.event_name == 'schedule' | |
| || (github.event_name == 'workflow_dispatch' && github.event.inputs.should_bench == 'true') | |
| || (github.event_name == 'push' && contains(github.event.head_commit.message, '[ci-bench]')) | |
| || ( | |
| github.event_name == 'pull_request' | |
| && ( | |
| contains(github.event.pull_request.title, '[ci-bench]') | |
| || contains(github.event.pull_request.body, '[ci-bench]') | |
| ) | |
| ) | |
| ) | |
| runs-on: ubuntu-latest | |
| env: | |
| WWW_DIR: target | |
| steps: | |
| - name: Checkout sources | |
| uses: actions/checkout@v3 | |
| - name: Install nightly toolchain | |
| uses: actions-rs/toolchain@v1 | |
| with: | |
| profile: minimal | |
| toolchain: nightly | |
| - name: Checkout gh-pages | |
| shell: bash | |
| run: | | |
| mkdir -p "$WWW_DIR" | |
| git -C "$WWW_DIR" init -q | |
| git -C "$WWW_DIR" remote add origin "$(git remote get-url origin)" | |
| git -C "$WWW_DIR" config credential.helper "$(git config credential.helper)" | |
| git -C "$WWW_DIR" config 'http.https://github.com/.extraheader' "$(git config 'http.https://github.com/.extraheader')" | |
| git -C "$WWW_DIR" config core.autocrlf input | |
| git -C "$WWW_DIR" config core.safecrlf false | |
| git -C "$WWW_DIR" checkout -b gh-pages | |
| git -C "$WWW_DIR" fetch origin gh-pages | |
| git -C "$WWW_DIR" reset --soft origin/gh-pages | |
| git -C "$WWW_DIR" reset | |
| git -C "$WWW_DIR" checkout -- bench criterion | |
| mkdir -p "$WWW_DIR/bench" | |
| - name: Run benchmark | |
| run: | | |
| time cargo bench -p benches -- hydroflow --output-format bencher | tee output.txt | |
| time cargo bench -p benches -- micro/ops/ --output-format bencher | tee -a output.txt | |
| - name: Generate benchmark page | |
| uses: benchmark-action/github-action-benchmark@v1 | |
| with: | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| tool: cargo | |
| output-file-path: output.txt | |
| external-data-json-path: target/bench/data.json | |
| - name: Write benchmark JSON | |
| run: echo 'window.BENCHMARK_DATA = ' | cat - "$WWW_DIR/bench/data.json" > "$WWW_DIR/bench/data.js" | |
| - name: Write benchmark HTML | |
| shell: bash | |
| run: node --input-type=module -e "$(curl -sSL https://raw.githubusercontent.com/benchmark-action/github-action-benchmark/master/src/default_index_html.ts) console.log(DEFAULT_INDEX_HTML)" > "$WWW_DIR/bench/index.html" | |
| - name: Push to gh-pages | |
| # Do not deploy PRs, only benchmark main branch. | |
| if: ${{ github.event_name != 'pull_request' && github.ref == 'refs/heads/main' }} | |
| shell: bash | |
| run: | | |
| cp -r .github/gh-pages/* .github/gh-pages/.gitignore "$WWW_DIR/" | |
| git -C "$WWW_DIR" fetch origin gh-pages | |
| git -C "$WWW_DIR" reset --soft origin/gh-pages | |
| git -C "$WWW_DIR" reset | |
| git -C "$WWW_DIR" add bench criterion $(ls .github/gh-pages) | |
| if git -C "$WWW_DIR" -c 'user.name=github-actions[bot]' -c 'user.email=41898282+github-actions[bot]@users.noreply.github.com' \ | |
| commit -m "Update Benchmarks $(date -I) $(git rev-parse HEAD)"; | |
| then | |
| git -C "$WWW_DIR" push -u origin gh-pages --quiet | |
| else | |
| echo 'No changes to commit' | |
| fi |