diff --git a/.github/workflows/check.yaml b/.github/workflows/check.yaml index cfc113cfbd44..cdd575065e11 100644 --- a/.github/workflows/check.yaml +++ b/.github/workflows/check.yaml @@ -6,7 +6,7 @@ jobs: steps: - uses: actions/setup-go@v2 with: - go-version: 1.16 + go-version: 1.17 - name: Checkout code uses: actions/checkout@v2 - name: Restore cache diff --git a/.github/workflows/pd-docker-image.yaml b/.github/workflows/pd-docker-image.yaml new file mode 100644 index 000000000000..355fa122ec67 --- /dev/null +++ b/.github/workflows/pd-docker-image.yaml @@ -0,0 +1,21 @@ +name: PD Docker Image +on: + push: + branches: + - master + pull_request: + branches: + - master +jobs: + chunks: + runs-on: ubuntu-latest + strategy: + fail-fast: true + steps: + - uses: actions/setup-go@v2 + with: + go-version: 1.17 + - name: Checkout code + uses: actions/checkout@v2 + - name: Make + run: make docker-image diff --git a/.github/workflows/pd-tests.yaml b/.github/workflows/pd-tests.yaml index c29a68ec11a2..0307ce2aa12f 100644 --- a/.github/workflows/pd-tests.yaml +++ b/.github/workflows/pd-tests.yaml @@ -1,28 +1,24 @@ -name: PD Tests +name: PD Test on: push: branches: - master - - release-4.0 - - release-5.* pull_request: branches: - master - - release-4.0 - - release-5.* jobs: chunks: runs-on: ubuntu-latest strategy: fail-fast: true matrix: - worker_id: [1, 2, 3, 4] + worker_id: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] outputs: - job-total: ${{ strategy.job-total }} + job-total: 10 steps: - uses: actions/setup-go@v2 with: - go-version: 1.16 + go-version: 1.17 - name: Checkout code uses: actions/checkout@v2 - name: Restore cache @@ -34,31 +30,12 @@ jobs: **/.tools **/.dashboard_download_cache key: ${{ runner.os }}-go-${{ matrix.worker_id }}-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-go-${{ matrix.worker_id }} - ${{ runner.os }}-go- - - name: Dispatch Packages - id: packages-units - env: - WORKER_ID: ${{ matrix.worker_id }} - # github.com/tikv/pd/tests/server/tso is the integration test of TSO, which will take a long time, - # will be run independently in the TSO Function Test. - run: | - go list ./... | grep -v -E "github.com/tikv/pd/server/api|github.com/tikv/pd/tests/client|github.com/tikv/pd/tests/server/tso" > packages.list - total=$(expr ${{ strategy.job-total }} - 1) - echo "Dispatched ${total} normal chunks" - split packages.list -n r/${total} packages_unit_ -a 1 --numeric-suffixes=1 - echo "Dispatched 2 special task to the last chunk (the special tests take a long time)" - echo github.com/tikv/pd/server/api > packages_unit_${{ strategy.job-total }} - echo github.com/tikv/pd/tests/client >> packages_unit_${{ strategy.job-total }} - packages="{$(cat packages_unit_${WORKER_ID} |tr "\n" ",")}" - echo "This worker will test the chunk - ${packages}" - echo "::set-output name=list::${packages}" - name: Make Test env: WORKER_ID: ${{ matrix.worker_id }} + WORKER_COUNT: 10 run: | - make test-with-cover TEST_PKGS="${{ steps.packages-units.outputs.list }}" + make ci-test-job JOB_COUNT=$WORKER_COUNT JOB_INDEX=$WORKER_ID mv covprofile covprofile_$WORKER_ID sed -i "/failpoint_binding/d" covprofile_$WORKER_ID - name: Upload coverage result ${{ matrix.worker_id }} @@ -79,10 +56,7 @@ jobs: - name: Merge env: TOTAL_JOBS: ${{needs.chunks.outputs.job-total}} - run: | - echo ${TOTAL_JOBS} - for i in $(seq 1 $TOTAL_JOBS); do cat covprofile_$i >> covprofile; done - cat covprofile + run: for i in $(seq 1 $TOTAL_JOBS); do cat covprofile_$i >> covprofile; done - name: Send coverage uses: codecov/codecov-action@v1 with: diff --git a/.github/workflows/tso-consistency-test.yaml b/.github/workflows/tso-consistency-test.yaml index a3e1de811df3..a489485d21cc 100644 --- a/.github/workflows/tso-consistency-test.yaml +++ b/.github/workflows/tso-consistency-test.yaml @@ -10,7 +10,7 @@ jobs: steps: - uses: actions/setup-go@v2 with: - go-version: 1.16 + go-version: 1.17 - name: Checkout code uses: actions/checkout@v2 - name: Make TSO Consistency Test diff --git a/.github/workflows/tso-function-test.yaml b/.github/workflows/tso-function-test.yaml index 0ce9cbddf732..cd493f332725 100644 --- a/.github/workflows/tso-function-test.yaml +++ b/.github/workflows/tso-function-test.yaml @@ -12,7 +12,7 @@ jobs: steps: - uses: actions/setup-go@v2 with: - go-version: 1.16 + go-version: 1.17 - name: Checkout code uses: actions/checkout@v2 - name: Make TSO Function Test diff --git a/Makefile b/Makefile index 3f356f18c92f..ccd9432611ff 100644 --- a/Makefile +++ b/Makefile @@ -98,6 +98,17 @@ stores-dump: .PHONY: pd-ctl pd-tso-bench pd-recover pd-analysis pd-heartbeat-bench simulator regions-dump stores-dump +#### Docker image #### + +docker-image: + $(eval DOCKER_PS_EXIT_CODE=$(shell docker ps > /dev/null 2>&1 ; echo $$?)) + @if [ $(DOCKER_PS_EXIT_CODE) -ne 0 ]; then \ + echo "Encountered problem while invoking docker cli. Is the docker daemon running?"; \ + fi + docker build --no-cache -t tikv/pd . + +.PHONY: docker-image + #### Build utils ### swagger-spec: install-tools @@ -133,7 +144,7 @@ install-tools: #### Static checks #### -check: install-tools static tidy check-plugin errdoc check-testing-t docker-build-test +check: install-tools static tidy check-plugin errdoc check-testing-t static: install-tools @ # Not running vet and fmt through metalinter becauase it ends up looking at vendor @@ -157,13 +168,6 @@ errdoc: install-tools @echo "generator errors.toml" ./scripts/check-errdoc.sh -docker-build-test: - $(eval DOCKER_PS_EXIT_CODE=$(shell docker ps > /dev/null 2>&1 ; echo $$?)) - @if [ $(DOCKER_PS_EXIT_CODE) -ne 0 ]; then \ - echo "Encountered problem while invoking docker cli. Is the docker daemon running?"; \ - fi - docker build --no-cache -t tikv/pd . - check-testing-t: ./scripts/check-testing-t.sh @@ -226,15 +230,12 @@ basic-test: install-tools go test $(BASIC_TEST_PKGS) || { $(FAILPOINT_DISABLE); exit 1; } @$(FAILPOINT_DISABLE) -test-with-cover: install-tools dashboard-ui - # testing all pkgs (expect TSO consistency test) with converage... +ci-test-job: install-tools dashboard-ui + @$(DEADLOCK_ENABLE) @$(FAILPOINT_ENABLE) - for PKG in $(TEST_PKGS); do\ - set -euo pipefail;\ - CGO_ENABLED=1 go test -race -covermode=atomic -coverprofile=coverage.tmp -coverpkg=./... $$PKG 2>&1 | grep -v "no packages being tested" && tail -n +2 coverage.tmp >> covprofile || { $(FAILPOINT_DISABLE); rm coverage.tmp && exit 1;}; \ - rm coverage.tmp;\ - done + CGO_ENABLED=1 go test -race -covermode=atomic -coverprofile=covprofile -coverpkg=./... $(shell ./scripts/ci-subtask.sh $(JOB_COUNT) $(JOB_INDEX)) || { $(FAILPOINT_DISABLE) && $(DEADLOCK_DISABLE) && exit 1;} @$(FAILPOINT_DISABLE) + @$(DEADLOCK_DISABLE) TSO_INTEGRATION_TEST_PKGS := $(PD_PKG)/tests/server/tso diff --git a/scripts/ci-subtask.sh b/scripts/ci-subtask.sh new file mode 100755 index 000000000000..6cd1e659a0bf --- /dev/null +++ b/scripts/ci-subtask.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +# ./ci-subtask.sh + +packages=(`go list ./...`) +dirs=(`find . -iname "*_test.go" -exec dirname {} \; | sort -u | sed -e "s/^\./github.com\/tikv\/pd/"`) +tasks=($(comm -12 <(printf "%s\n" "${packages[@]}") <(printf "%s\n" "${dirs[@]}"))) + +weight () { + [[ $1 == "github.com/tikv/pd/server/api" ]] && return 30 + [[ $1 == "github.com/tikv/pd/server/schedule" ]] && return 30 + [[ $1 == "github.com/tikv/pd/tests/client" ]] && return 30 + [[ $1 =~ "pd/tests" ]] && return 5 + return 1 +} + +scores=(`seq "$1" | xargs -I{} echo 0`) + +res=() +for t in ${tasks[@]}; do + min_i=0 + for i in ${!scores[@]}; do + [[ ${scores[i]} -lt ${scores[$min_i]} ]] && min_i=$i + done + weight $t + scores[$min_i]=$((${scores[$min_i]} + $?)) + [[ $(($min_i+1)) -eq $2 ]] && res+=($t) +done + +printf "%s " "${res[@]}" diff --git a/server/schedule/region_scatterer_test.go b/server/schedule/region_scatterer_test.go index b708ade6f817..b77e226f9a7d 100644 --- a/server/schedule/region_scatterer_test.go +++ b/server/schedule/region_scatterer_test.go @@ -63,28 +63,13 @@ var _ = Suite(&testScatterRegionSuite{}) type testScatterRegionSuite struct{} -func (s *testScatterRegionSuite) TestSixStores(c *C) { - s.scatter(c, 6, 100, false) - s.scatter(c, 6, 100, true) - s.scatter(c, 6, 1000, false) - s.scatter(c, 6, 1000, true) -} - -func (s *testScatterRegionSuite) TestFiveStores(c *C) { - s.scatter(c, 5, 100, false) - s.scatter(c, 5, 100, true) - s.scatter(c, 5, 1000, false) - s.scatter(c, 5, 1000, true) -} - -func (s *testScatterRegionSuite) TestSixSpecialStores(c *C) { - s.scatterSpecial(c, 3, 6, 100) - s.scatterSpecial(c, 3, 6, 1000) -} - -func (s *testScatterRegionSuite) TestFiveSpecialStores(c *C) { - s.scatterSpecial(c, 5, 5, 100) - s.scatterSpecial(c, 5, 5, 1000) +func (s *testScatterRegionSuite) TestScatterRegions(c *C) { + s.scatter(c, 5, 50, true) + s.scatter(c, 5, 500, true) + s.scatter(c, 6, 50, true) + s.scatter(c, 5, 50, false) + s.scatterSpecial(c, 3, 6, 50) + s.scatterSpecial(c, 5, 5, 50) } func (s *testScatterRegionSuite) checkOperator(op *operator.Operator, c *C) {