feat: make obkv wal opening more parallelly (#1129) #72
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Copyright 2022 CeresDB Project Authors. Licensed under Apache-2.0. | |
| name: CI | |
| on: | |
| merge_group: | |
| workflow_dispatch: | |
| push: | |
| branches: | |
| - main | |
| paths-ignore: | |
| - 'docs/**' | |
| - 'etc/**' | |
| - '**.md' | |
| - '.dockerignore' | |
| - 'docker/**' | |
| pull_request: | |
| paths-ignore: | |
| - 'docs/**' | |
| - 'etc/**' | |
| - '**.md' | |
| - '.dockerignore' | |
| - 'docker/**' | |
| # Common environment variables | |
| env: | |
| RUSTFLAGS: "-C debuginfo=1" | |
| CARGO_TERM_COLOR: always | |
| RUST_BACKTRACE: "1" | |
| CERESMETA_IMAGE_NAME: ceresdb/ceresmeta-server:latest | |
| CERESDB_IMAGE_NAME: ceresdb/ceresdb-server:latest | |
| LOCK_FILE: Cargo.lock | |
| RUST_VERSION: nightly-2023-02-02 | |
| jobs: | |
| style-check: | |
| name: style-check | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 60 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| submodules: true | |
| - run: | | |
| rustup set auto-self-update disable | |
| rustup toolchain install ${RUST_VERSION} --profile minimal | |
| - name: Release Disk Quota | |
| run: | | |
| sudo rm -rf /usr/local/lib/android # release about 10 GB | |
| sudo rm -rf /usr/share/dotnet # release about 20GB | |
| - name: Cache Rust Dependencies | |
| uses: actions/cache@v3 | |
| with: | |
| path: | | |
| ~/.cargo | |
| ./target | |
| key: debug-${{ runner.os }}-${{ hashFiles('rust-toolchain') }}-${{ hashFiles('Cargo.lock') }} | |
| restore-keys: | | |
| debug-${{ runner.os }}-${{ hashFiles('rust-toolchain') }}- | |
| debug-${{ runner.os }}- | |
| debug- | |
| - name: Ensure Disk Quota | |
| run: | | |
| make ensure-disk-quota | |
| - name: Setup Build Environment | |
| run: | | |
| sudo apt update | |
| sudo apt install --yes protobuf-compiler | |
| - name: Install clippy rustfmt | |
| run: | | |
| rustup component add clippy | |
| rustup component add rustfmt | |
| cargo install cargo-sort | |
| - name: Run Style Check | |
| run: | | |
| make check-license | |
| make clippy | |
| make fmt | |
| make check-cargo-toml | |
| - name: Report Disk Usage | |
| if: always() | |
| run: | | |
| du -sh ./target | |
| df -h | |
| unit-test: | |
| name: unit-test | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 60 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| submodules: true | |
| - run: | | |
| rustup set auto-self-update disable | |
| rustup toolchain install ${RUST_VERSION} --profile minimal | |
| - name: Release Disk Quota | |
| run: | | |
| sudo rm -rf /usr/local/lib/android # release about 10 GB | |
| sudo rm -rf /usr/share/dotnet # release about 20GB | |
| - name: Cache Rust Dependencies | |
| uses: actions/cache@v3 | |
| with: | |
| path: | | |
| ~/.cargo | |
| ./target | |
| key: debug-${{ runner.os }}-${{ hashFiles('rust-toolchain') }}-${{ hashFiles('Cargo.lock') }} | |
| restore-keys: | | |
| debug-${{ runner.os }}-${{ hashFiles('rust-toolchain') }}- | |
| debug-${{ runner.os }}- | |
| debug- | |
| - name: Setup Build Environment | |
| run: | | |
| sudo apt update | |
| sudo apt install --yes protobuf-compiler | |
| - name: Backup Lock File | |
| run: | | |
| cp ${LOCK_FILE} ${LOCK_FILE}.bak | |
| - name: Run Unit Tests | |
| run: | | |
| make test-ut | |
| echo "Checking if ${LOCK_FILE} has changed..." | |
| - name: Check Lock File | |
| run: | | |
| diff ${LOCK_FILE} ${LOCK_FILE}.bak | |
| integration-test: | |
| name: integration-test | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 60 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| submodules: true | |
| - name: Cache Rust Dependencies | |
| uses: actions/cache@v3 | |
| with: | |
| path: | | |
| ~/.cargo | |
| ./target | |
| key: debug-${{ runner.os }}-${{ hashFiles('rust-toolchain') }}-${{ hashFiles('Cargo.lock') }} | |
| restore-keys: | | |
| debug-${{ runner.os }}-${{ hashFiles('rust-toolchain') }}- | |
| debug-${{ runner.os }}- | |
| debug- | |
| - run: | | |
| rustup set auto-self-update disable | |
| rustup toolchain install ${RUST_VERSION} --profile minimal | |
| - name: Release Disk Quota | |
| run: | | |
| sudo rm -rf /usr/local/lib/android # release about 10 GB | |
| sudo rm -rf /usr/share/dotnet # release about 20GB | |
| - name: Setup Build Environment | |
| run: | | |
| sudo apt update | |
| sudo apt install --yes protobuf-compiler | |
| - name: Run integration tests | |
| run: | | |
| make integration-test | |
| env: | |
| RUST_BACKTRACE: "1" | |
| - name: Upload Logs | |
| if: always() | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: CI-${{ github.head_ref }}.${{ github.sha }} | |
| path: | | |
| /tmp/ceresdb-stdout.log | |
| /tmp/ceresmeta.log | |
| /tmp/ceresdb-stdout-0.log | |
| /tmp/ceresdb-stdout-1.log | |
| sdk-test: | |
| name: sdk-test | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 60 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| submodules: true | |
| - name: Cache Rust Dependencies | |
| uses: actions/cache@v3 | |
| with: | |
| path: | | |
| ~/.cargo | |
| ./target | |
| key: debug-${{ runner.os }}-${{ hashFiles('rust-toolchain') }}-${{ hashFiles('Cargo.lock') }} | |
| restore-keys: | | |
| debug-${{ runner.os }}-${{ hashFiles('rust-toolchain') }}- | |
| debug-${{ runner.os }}- | |
| debug- | |
| - run: | | |
| rustup set auto-self-update disable | |
| rustup toolchain install ${RUST_VERSION} --profile minimal | |
| - name: Release Disk Quota | |
| run: | | |
| sudo rm -rf /usr/local/lib/android # release about 10 GB | |
| sudo rm -rf /usr/share/dotnet # release about 20GB | |
| - name: Setup Build Environment | |
| run: | | |
| sudo apt update | |
| sudo apt install --yes protobuf-compiler | |
| - name: Build and Run CeresDB | |
| run: | | |
| make build-debug | |
| nohup ./target/debug/ceresdb-server -c docs/minimal.toml > /tmp/ceresdb-stdout.log & | |
| sleep 10 | |
| - name: Run Go SDK tests | |
| working-directory: integration_tests | |
| run: | | |
| make run-go | |
| - name: Run Java SDK tests | |
| working-directory: integration_tests | |
| run: | | |
| make run-java | |
| - name: Run Rust SDK tests | |
| working-directory: integration_tests | |
| run: | | |
| make run-rust | |
| - name: Run MySQL client tests | |
| working-directory: integration_tests | |
| run: | | |
| make run-mysql | |
| - name: Run Prometheus query tests | |
| working-directory: integration_tests | |
| run: | | |
| make run-prom | |
| - name: Run OpenTSDB tests | |
| working-directory: integration_tests | |
| run: | | |
| make run-opentsdb | |
| - name: Upload Logs | |
| if: always() | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: sdk-test-${{ github.head_ref }}.${{ github.sha }} | |
| path: | | |
| /tmp/ceresdb-stdout.log | |
| recovery-test: | |
| name: recovery-test | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 60 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| submodules: true | |
| - name: Cache Rust Dependencies | |
| uses: actions/cache@v3 | |
| with: | |
| path: | | |
| ~/.cargo | |
| ./target | |
| key: debug-${{ runner.os }}-${{ hashFiles('rust-toolchain') }}-${{ hashFiles('Cargo.lock') }} | |
| restore-keys: | | |
| debug-${{ runner.os }}-${{ hashFiles('rust-toolchain') }}- | |
| debug-${{ runner.os }}- | |
| debug- | |
| - run: | | |
| rustup set auto-self-update disable | |
| rustup toolchain install ${RUST_VERSION} --profile minimal | |
| - name: Release Disk Quota | |
| run: | | |
| sudo rm -rf /usr/local/lib/android # release about 10 GB | |
| sudo rm -rf /usr/share/dotnet # release about 20GB | |
| - name: Setup Build Environment | |
| run: | | |
| sudo apt update | |
| sudo apt install --yes protobuf-compiler | |
| - name: Run recovery tests | |
| working-directory: integration_tests | |
| run: | | |
| make run-recovery | |
| - name: Upload Logs | |
| if: always() | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: recovery-test-${{ github.head_ref }}.${{ github.sha }} | |
| path: | | |
| /tmp/ceresdb-stdout.log |