feat: support re-acquire shard lock in a fast way (#1251) #106
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Copyright 2023 The CeresDB Authors | |
| # | |
| # Licensed under the Apache License, Version 2.0 (the "License"); | |
| # you may not use this file except in compliance with the License. | |
| # You may obtain a copy of the License at | |
| # | |
| # http://www.apache.org/licenses/LICENSE-2.0 | |
| # | |
| # Unless required by applicable law or agreed to in writing, software | |
| # distributed under the License is distributed on an "AS IS" BASIS, | |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
| # See the License for the specific language governing permissions and | |
| # limitations under the License. | |
| name: CI | |
| on: | |
| merge_group: | |
| workflow_dispatch: | |
| push: | |
| branches: | |
| - main | |
| paths-ignore: | |
| - 'docs/**' | |
| - 'etc/**' | |
| - '**.md' | |
| - '.dockerignore' | |
| - 'docker/**' | |
| pull_request: | |
| paths-ignore: | |
| - 'docs/**' | |
| - 'etc/**' | |
| - '**.md' | |
| - '.dockerignore' | |
| - 'docker/**' | |
| # Common environment variables | |
| env: | |
| RUSTFLAGS: "-C debuginfo=1" | |
| CARGO_TERM_COLOR: always | |
| RUST_BACKTRACE: "1" | |
| CERESMETA_IMAGE_NAME: ceresdb/ceresmeta-server:latest | |
| CERESDB_IMAGE_NAME: ceresdb/ceresdb-server:latest | |
| LOCK_FILE: Cargo.lock | |
| RUST_VERSION: nightly-2023-08-28 | |
| jobs: | |
| style-check: | |
| name: style-check | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 60 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| submodules: true | |
| - run: | | |
| rustup set auto-self-update disable | |
| rustup toolchain install ${RUST_VERSION} --profile minimal | |
| - name: Release Disk Quota | |
| run: | | |
| sudo rm -rf /usr/local/lib/android # release about 10 GB | |
| sudo rm -rf /usr/share/dotnet # release about 20GB | |
| - name: Check License Header | |
| uses: korandoru/hawkeye@v3 | |
| - name: Cache Rust Dependencies | |
| uses: actions/cache@v3 | |
| with: | |
| path: | | |
| ~/.cargo | |
| ./target | |
| key: debug-${{ runner.os }}-${{ hashFiles('rust-toolchain') }}-${{ hashFiles('Cargo.lock') }} | |
| restore-keys: | | |
| debug-${{ runner.os }}-${{ hashFiles('rust-toolchain') }}- | |
| debug-${{ runner.os }}- | |
| debug- | |
| - name: Ensure Disk Quota | |
| run: | | |
| make ensure-disk-quota | |
| - name: Setup Build Environment | |
| run: | | |
| sudo apt update | |
| sudo apt install --yes protobuf-compiler | |
| - name: Install check binaries | |
| run: | | |
| rustup component add clippy | |
| rustup component add rustfmt | |
| cargo install cargo-sort | |
| - name: Run Style Check | |
| run: | | |
| make clippy | |
| make fmt | |
| make check-cargo-toml | |
| - name: Report Disk Usage | |
| if: always() | |
| run: | | |
| du -sh ./target | |
| df -h | |
| unit-test: | |
| name: unit-test | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 60 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| submodules: true | |
| - run: | | |
| rustup set auto-self-update disable | |
| rustup toolchain install ${RUST_VERSION} --profile minimal | |
| - name: Release Disk Quota | |
| run: | | |
| sudo rm -rf /usr/local/lib/android # release about 10 GB | |
| sudo rm -rf /usr/share/dotnet # release about 20GB | |
| - name: Cache Rust Dependencies | |
| uses: actions/cache@v3 | |
| with: | |
| path: | | |
| ~/.cargo | |
| ./target | |
| key: debug-${{ runner.os }}-${{ hashFiles('rust-toolchain') }}-${{ hashFiles('Cargo.lock') }} | |
| restore-keys: | | |
| debug-${{ runner.os }}-${{ hashFiles('rust-toolchain') }}- | |
| debug-${{ runner.os }}- | |
| debug- | |
| - name: Setup Build Environment | |
| run: | | |
| sudo apt update | |
| sudo apt install --yes protobuf-compiler | |
| - name: Backup Lock File | |
| run: | | |
| cp ${LOCK_FILE} ${LOCK_FILE}.bak | |
| - name: Run Unit Tests | |
| run: | | |
| make test-ut | |
| echo "Checking if ${LOCK_FILE} has changed..." | |
| - name: Check Lock File | |
| run: | | |
| diff ${LOCK_FILE} ${LOCK_FILE}.bak | |
| integration-test: | |
| name: integration-test | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 60 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| submodules: true | |
| - name: Cache Rust Dependencies | |
| uses: actions/cache@v3 | |
| with: | |
| path: | | |
| ~/.cargo | |
| ./target | |
| key: debug-${{ runner.os }}-${{ hashFiles('rust-toolchain') }}-${{ hashFiles('Cargo.lock') }} | |
| restore-keys: | | |
| debug-${{ runner.os }}-${{ hashFiles('rust-toolchain') }}- | |
| debug-${{ runner.os }}- | |
| debug- | |
| - run: | | |
| rustup set auto-self-update disable | |
| rustup toolchain install ${RUST_VERSION} --profile minimal | |
| - name: Release Disk Quota | |
| run: | | |
| sudo rm -rf /usr/local/lib/android # release about 10 GB | |
| sudo rm -rf /usr/share/dotnet # release about 20GB | |
| - name: Setup Build Environment | |
| run: | | |
| sudo apt update | |
| sudo apt install --yes protobuf-compiler | |
| - name: Run integration tests | |
| run: | | |
| make integration-test | |
| env: | |
| RUST_BACKTRACE: "1" | |
| - name: Upload Logs | |
| if: always() | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: CI-${{ github.sha }} | |
| path: | | |
| /tmp/ceresdb-stdout.log | |
| /tmp/ceresmeta.log | |
| /tmp/ceresdb-stdout-0.log | |
| /tmp/ceresdb-stdout-1.log | |
| sdk-test: | |
| name: sdk-test | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 60 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| submodules: true | |
| - name: Cache Rust Dependencies | |
| uses: actions/cache@v3 | |
| with: | |
| path: | | |
| ~/.cargo | |
| ./target | |
| key: debug-${{ runner.os }}-${{ hashFiles('rust-toolchain') }}-${{ hashFiles('Cargo.lock') }} | |
| restore-keys: | | |
| debug-${{ runner.os }}-${{ hashFiles('rust-toolchain') }}- | |
| debug-${{ runner.os }}- | |
| debug- | |
| - run: | | |
| rustup set auto-self-update disable | |
| rustup toolchain install ${RUST_VERSION} --profile minimal | |
| - name: Release Disk Quota | |
| run: | | |
| sudo rm -rf /usr/local/lib/android # release about 10 GB | |
| sudo rm -rf /usr/share/dotnet # release about 20GB | |
| - name: Setup Build Environment | |
| run: | | |
| sudo apt update | |
| sudo apt install --yes protobuf-compiler | |
| - name: Build and Run CeresDB | |
| run: | | |
| make build-debug | |
| nohup ./target/debug/ceresdb-server -c docs/minimal.toml > /tmp/ceresdb-stdout.log & | |
| sleep 10 | |
| - name: Run Go SDK tests | |
| working-directory: integration_tests | |
| run: | | |
| make run-go | |
| - name: Run Java SDK tests | |
| working-directory: integration_tests | |
| run: | | |
| make run-java | |
| - name: Run Rust SDK tests | |
| working-directory: integration_tests | |
| run: | | |
| make run-rust | |
| - name: Run MySQL client tests | |
| working-directory: integration_tests | |
| run: | | |
| make run-mysql | |
| - name: Run PostgreSQL client tests | |
| working-directory: integration_tests | |
| run: | | |
| make run-postgresql | |
| - name: Run Prometheus query tests | |
| working-directory: integration_tests | |
| run: | | |
| make run-prom | |
| - name: Run OpenTSDB tests | |
| working-directory: integration_tests | |
| run: | | |
| make run-opentsdb | |
| - name: Upload Logs | |
| if: always() | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: sdk-test-${{ github.sha }} | |
| path: | | |
| /tmp/ceresdb-stdout.log | |
| recovery-test: | |
| name: recovery-test | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 60 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| submodules: true | |
| - name: Cache Rust Dependencies | |
| uses: actions/cache@v3 | |
| with: | |
| path: | | |
| ~/.cargo | |
| ./target | |
| key: debug-${{ runner.os }}-${{ hashFiles('rust-toolchain') }}-${{ hashFiles('Cargo.lock') }} | |
| restore-keys: | | |
| debug-${{ runner.os }}-${{ hashFiles('rust-toolchain') }}- | |
| debug-${{ runner.os }}- | |
| debug- | |
| - run: | | |
| rustup set auto-self-update disable | |
| rustup toolchain install ${RUST_VERSION} --profile minimal | |
| - name: Release Disk Quota | |
| run: | | |
| sudo rm -rf /usr/local/lib/android # release about 10 GB | |
| sudo rm -rf /usr/share/dotnet # release about 20GB | |
| - name: Setup Build Environment | |
| run: | | |
| sudo apt update | |
| sudo apt install --yes protobuf-compiler | |
| - name: Run recovery tests | |
| working-directory: integration_tests | |
| run: | | |
| make run-recovery | |
| - name: Upload Logs | |
| if: always() | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: recovery-test-${{ github.sha }} | |
| path: | | |
| /tmp/ceresdb-stdout.log | |
| dist-query-test: | |
| name: dist-query-test | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 60 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| submodules: true | |
| - name: Cache Rust Dependencies | |
| uses: actions/cache@v3 | |
| with: | |
| path: | | |
| ~/.cargo | |
| ./target | |
| key: debug-${{ runner.os }}-${{ hashFiles('rust-toolchain') }}-${{ hashFiles('Cargo.lock') }} | |
| restore-keys: | | |
| debug-${{ runner.os }}-${{ hashFiles('rust-toolchain') }}- | |
| debug-${{ runner.os }}- | |
| debug- | |
| - run: | | |
| rustup set auto-self-update disable | |
| rustup toolchain install ${RUST_VERSION} --profile minimal | |
| - name: Release Disk Quota | |
| run: | | |
| sudo rm -rf /usr/local/lib/android # release about 10 GB | |
| sudo rm -rf /usr/share/dotnet # release about 20GB | |
| - name: Setup Build Environment | |
| run: | | |
| sudo apt update | |
| sudo apt install --yes protobuf-compiler | |
| - name: Run dist query tests | |
| working-directory: integration_tests | |
| run: | | |
| make run-dist-query | |
| - name: Upload Logs | |
| if: always() | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: dist-query-test-${{ github.sha }} | |
| path: | | |
| /tmp/ceresdb-stdout.log |