test: update github action

ref tikv#4399

Signed-off-by: disksing <i@disksing.com>

.github/workflows/pd-tests.yaml

@@ -16,7 +16,7 @@ jobs:
  strategy:
  fail-fast: true
  matrix:
- worker_id: [1, 2, 3, 4]
+ worker_id: [1, 2, 3, 4, 5, 6, 7, 8]
  outputs:
  job-total: ${{ strategy.job-total }}
  steps:
@@ -37,28 +37,11 @@ jobs:
  restore-keys: |
  ${{ runner.os }}-go-${{ matrix.worker_id }}
  ${{ runner.os }}-go-
- - name: Dispatch Packages
- id: packages-units
- env:
- WORKER_ID: ${{ matrix.worker_id }}
- # github.com/tikv/pd/tests/server/tso is the integration test of TSO, which will take a long time,
- # will be run independently in the TSO Function Test.
- run: |
- go list ./... | grep -v -E "github.com/tikv/pd/server/api|github.com/tikv/pd/tests/client|github.com/tikv/pd/tests/server/tso" > packages.list
- total=$(expr ${{ strategy.job-total }} - 1)
- echo "Dispatched ${total} normal chunks"
- split packages.list -n r/${total} packages_unit_ -a 1 --numeric-suffixes=1
- echo "Dispatched 2 special task to the last chunk (the special tests take a long time)"
- echo github.com/tikv/pd/server/api > packages_unit_${{ strategy.job-total }}
- echo github.com/tikv/pd/tests/client >> packages_unit_${{ strategy.job-total }}
- packages="{$(cat packages_unit_${WORKER_ID} |tr "\n" ",")}"
- echo "This worker will test the chunk - ${packages}"
- echo "::set-output name=list::${packages}"
  - name: Make Test
  env:
  WORKER_ID: ${{ matrix.worker_id }}
  run: |
- make test-with-cover TEST_PKGS="${{ steps.packages-units.outputs.list }}"
+ make ci-test-job JOB_COUNT=8 JOB_INDEX=${{ matrix.worker_id }}
  mv covprofile covprofile_$WORKER_ID
  sed -i "/failpoint_binding/d" covprofile_$WORKER_ID
  - name: Upload coverage result ${{ matrix.worker_id }}

Makefile

@@ -226,14 +226,9 @@ basic-test: install-tools
  go test $(BASIC_TEST_PKGS) || { $(FAILPOINT_DISABLE); exit 1; }
  @$(FAILPOINT_DISABLE)
 
-test-with-cover: install-tools dashboard-ui
- # testing all pkgs (expect TSO consistency test) with converage...
+ci-test-job: install-tools
  @$(FAILPOINT_ENABLE)
- for PKG in $(TEST_PKGS); do\
- set -euo pipefail;\
- CGO_ENABLED=1 go test -race -covermode=atomic -coverprofile=coverage.tmp -coverpkg=./... $$PKG 2>&1 | grep -v "no packages being tested" && tail -n +2 coverage.tmp >> covprofile || { $(FAILPOINT_DISABLE); rm coverage.tmp && exit 1;}; \
- rm coverage.tmp;\
- done
+ CGO_ENABLED=1 go test -race -covermode=atomic -coverprofile=covprofile -coverpkg=./... $(shell ./scripts/ci-subtask.sh $(JOB_COUNT) $(JOB_INDEX)) || { $(FAILPOINT_DISABLE) && exit 1;}
  @$(FAILPOINT_DISABLE)
 
 TSO_INTEGRATION_TEST_PKGS := $(PD_PKG)/tests/server/tso

scripts/ci-subtask.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+
+# ./ci-subtask.sh <TOTAL_TASK_N> <TASK_INDEX>
+
+packages=(`go list ./...`)
+dirs=(`find . -iname "*_test.go" -exec dirname {} \; | sort -u | sed -e "s/^\./github.com\/tikv\/pd/"`)
+tasks=($(comm -12 <(printf "%s\n" "${packages[@]}") <(printf "%s\n" "${dirs[@]}")))
+
+weight () {
+ if [[ $1 == "github.com/tikv/pd/server/api" ]]; then return 30; fi
+ if [[ $1 == "github.com/tikv/pd/server/tso" ]]; then return 30; fi
+ if [[ $1 == "github.com/tikv/pd/server/schedule" ]]; then return 30; fi
+ if [[ $1 == "github.com/tikv/pd/tests/client" ]]; then return 30; fi
+ if [[ $1 =~ "pd/tests" ]]; then return 5; fi
+ return 1
+}
+
+scores=(`seq "$1" | xargs -I{} echo 0`)
+
+res=()
+for t in ${tasks[@]}; do
+ min_i=0
+ for i in ${!scores[@]}; do
+ if [[ ${scores[i]} -lt ${scores[$min_i]} ]]; then min_i=$i; fi
+ done
+ weight $t
+ scores[$min_i]=$((${scores[$min_i]} + $?))
+ if [[ $(($min_i+1)) -eq $2 ]]; then res+=($t); fi
+done
+
+printf "%s " "${res[@]}"