diff --git a/.github/workflows/pd-tests.yaml b/.github/workflows/pd-tests.yaml index c29a68ec11a2..6f8d9b379704 100644 --- a/.github/workflows/pd-tests.yaml +++ b/.github/workflows/pd-tests.yaml @@ -16,7 +16,7 @@ jobs: strategy: fail-fast: true matrix: - worker_id: [1, 2, 3, 4] + worker_id: [1, 2, 3, 4, 5, 6, 7, 8] outputs: job-total: ${{ strategy.job-total }} steps: @@ -37,28 +37,11 @@ jobs: restore-keys: | ${{ runner.os }}-go-${{ matrix.worker_id }} ${{ runner.os }}-go- - - name: Dispatch Packages - id: packages-units - env: - WORKER_ID: ${{ matrix.worker_id }} - # github.com/tikv/pd/tests/server/tso is the integration test of TSO, which will take a long time, - # will be run independently in the TSO Function Test. - run: | - go list ./... | grep -v -E "github.com/tikv/pd/server/api|github.com/tikv/pd/tests/client|github.com/tikv/pd/tests/server/tso" > packages.list - total=$(expr ${{ strategy.job-total }} - 1) - echo "Dispatched ${total} normal chunks" - split packages.list -n r/${total} packages_unit_ -a 1 --numeric-suffixes=1 - echo "Dispatched 2 special task to the last chunk (the special tests take a long time)" - echo github.com/tikv/pd/server/api > packages_unit_${{ strategy.job-total }} - echo github.com/tikv/pd/tests/client >> packages_unit_${{ strategy.job-total }} - packages="{$(cat packages_unit_${WORKER_ID} |tr "\n" ",")}" - echo "This worker will test the chunk - ${packages}" - echo "::set-output name=list::${packages}" - name: Make Test env: WORKER_ID: ${{ matrix.worker_id }} run: | - make test-with-cover TEST_PKGS="${{ steps.packages-units.outputs.list }}" + make ci-test-job JOB_COUNT=8 JOB_INDEX=${{ matrix.worker_id }} mv covprofile covprofile_$WORKER_ID sed -i "/failpoint_binding/d" covprofile_$WORKER_ID - name: Upload coverage result ${{ matrix.worker_id }} diff --git a/Makefile b/Makefile index 98477c597529..ba660850bee3 100644 --- a/Makefile +++ b/Makefile @@ -226,14 +226,9 @@ basic-test: install-tools go test $(BASIC_TEST_PKGS) || { $(FAILPOINT_DISABLE); exit 1; } @$(FAILPOINT_DISABLE) -test-with-cover: install-tools dashboard-ui - # testing all pkgs (expect TSO consistency test) with converage... +ci-test-job: install-tools @$(FAILPOINT_ENABLE) - for PKG in $(TEST_PKGS); do\ - set -euo pipefail;\ - CGO_ENABLED=1 go test -race -covermode=atomic -coverprofile=coverage.tmp -coverpkg=./... $$PKG 2>&1 | grep -v "no packages being tested" && tail -n +2 coverage.tmp >> covprofile || { $(FAILPOINT_DISABLE); rm coverage.tmp && exit 1;}; \ - rm coverage.tmp;\ - done + CGO_ENABLED=1 go test -race -covermode=atomic -coverprofile=covprofile -coverpkg=./... $(shell ./scripts/ci-subtask.sh $(JOB_COUNT) $(JOB_INDEX)) || { $(FAILPOINT_DISABLE) && exit 1;} @$(FAILPOINT_DISABLE) TSO_INTEGRATION_TEST_PKGS := $(PD_PKG)/tests/server/tso diff --git a/scripts/ci-subtask.sh b/scripts/ci-subtask.sh new file mode 100755 index 000000000000..99b736ea8914 --- /dev/null +++ b/scripts/ci-subtask.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +# ./ci-subtask.sh + +packages=(`go list ./...`) +dirs=(`find . -iname "*_test.go" -exec dirname {} \; | sort -u | sed -e "s/^\./github.com\/tikv\/pd/"`) +tasks=($(comm -12 <(printf "%s\n" "${packages[@]}") <(printf "%s\n" "${dirs[@]}"))) + +weight () { + if [[ $1 == "github.com/tikv/pd/server/api" ]]; then return 30; fi + if [[ $1 == "github.com/tikv/pd/server/tso" ]]; then return 30; fi + if [[ $1 == "github.com/tikv/pd/server/schedule" ]]; then return 30; fi + if [[ $1 == "github.com/tikv/pd/tests/client" ]]; then return 30; fi + if [[ $1 =~ "pd/tests" ]]; then return 5; fi + return 1 +} + +scores=(`seq "$1" | xargs -I{} echo 0`) + +res=() +for t in ${tasks[@]}; do + min_i=0 + for i in ${!scores[@]}; do + if [[ ${scores[i]} -lt ${scores[$min_i]} ]]; then min_i=$i; fi + done + weight $t + scores[$min_i]=$((${scores[$min_i]} + $?)) + if [[ $(($min_i+1)) -eq $2 ]]; then res+=($t); fi +done + +printf "%s " "${res[@]}"