Skip to content

Commit

Permalink
build: update GitHub Actions
Browse files Browse the repository at this point in the history
ref tikv#4399

Signed-off-by: disksing <i@disksing.com>
  • Loading branch information
disksing committed Jan 14, 2022
1 parent 32832a3 commit 9205dd3
Show file tree
Hide file tree
Showing 7 changed files with 112 additions and 69 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/check.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ jobs:
steps:
- uses: actions/setup-go@v2
with:
go-version: 1.16
go-version: 1.17
- name: Checkout code
uses: actions/checkout@v2
- name: Restore cache
Expand Down
66 changes: 66 additions & 0 deletions .github/workflows/pd-cover.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
name: PD Coverage
on:
push:
branches:
- master
pull_request:
branches:
- master
jobs:
chunks:
runs-on: ubuntu-latest
strategy:
fail-fast: true
matrix:
worker_id: [1, 2, 3, 4]
outputs:
job-total: 4
steps:
- uses: actions/setup-go@v2
with:
go-version: 1.17
- name: Checkout code
uses: actions/checkout@v2
- name: Restore cache
uses: actions/cache@v2
with:
path: |
~/go/pkg/mod
~/.cache/go-build
**/.tools
**/.dashboard_download_cache
key: ${{ runner.os }}-go-${{ matrix.worker_id }}-${{ hashFiles('**/go.sum') }}
- name: Make Test
env:
WORKER_ID: ${{ matrix.worker_id }}
WORKER_COUNT: 4
run: |
make ci-cover-job JOB_COUNT=$WORKER_COUNT JOB_INDEX=$WORKER_ID
mv covprofile covprofile_$WORKER_ID
sed -i "/failpoint_binding/d" covprofile_$WORKER_ID
- name: Upload coverage result ${{ matrix.worker_id }}
uses: actions/upload-artifact@v2
with:
name: cover-reports
path: covprofile_${{ matrix.worker_id }}
report-coverage:
needs: chunks
runs-on: ubuntu-latest
steps:
- name: Checkout code
uses: actions/checkout@v2
- name: Download chunk report
uses: actions/download-artifact@v2
with:
name: cover-reports
- name: Merge
env:
TOTAL_JOBS: ${{needs.chunks.outputs.job-total}}
run: for i in $(seq 1 $TOTAL_JOBS); do cat covprofile_$i >> covprofile; done
- name: Send coverage
uses: codecov/codecov-action@v1
with:
token: ${{ secrets.CODECOV }}
file: ./covprofile
flags: unittests
name: codecov-umbrella
62 changes: 3 additions & 59 deletions .github/workflows/pd-tests.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -16,13 +16,11 @@ jobs:
strategy:
fail-fast: true
matrix:
worker_id: [1, 2, 3, 4]
outputs:
job-total: ${{ strategy.job-total }}
worker_id: [1, 2, 3, 4, 5, 6, 7, 8]
steps:
- uses: actions/setup-go@v2
with:
go-version: 1.16
go-version: 1.17
- name: Checkout code
uses: actions/checkout@v2
- name: Restore cache
Expand All @@ -34,59 +32,5 @@ jobs:
**/.tools
**/.dashboard_download_cache
key: ${{ runner.os }}-go-${{ matrix.worker_id }}-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-go-${{ matrix.worker_id }}
${{ runner.os }}-go-
- name: Dispatch Packages
id: packages-units
env:
WORKER_ID: ${{ matrix.worker_id }}
# github.com/tikv/pd/tests/server/tso is the integration test of TSO, which will take a long time,
# will be run independently in the TSO Function Test.
run: |
go list ./... | grep -v -E "github.com/tikv/pd/server/api|github.com/tikv/pd/tests/client|github.com/tikv/pd/tests/server/tso" > packages.list
total=$(expr ${{ strategy.job-total }} - 1)
echo "Dispatched ${total} normal chunks"
split packages.list -n r/${total} packages_unit_ -a 1 --numeric-suffixes=1
echo "Dispatched 2 special task to the last chunk (the special tests take a long time)"
echo github.com/tikv/pd/server/api > packages_unit_${{ strategy.job-total }}
echo github.com/tikv/pd/tests/client >> packages_unit_${{ strategy.job-total }}
packages="{$(cat packages_unit_${WORKER_ID} |tr "\n" ",")}"
echo "This worker will test the chunk - ${packages}"
echo "::set-output name=list::${packages}"
- name: Make Test
env:
WORKER_ID: ${{ matrix.worker_id }}
run: |
make test-with-cover TEST_PKGS="${{ steps.packages-units.outputs.list }}"
mv covprofile covprofile_$WORKER_ID
sed -i "/failpoint_binding/d" covprofile_$WORKER_ID
- name: Upload coverage result ${{ matrix.worker_id }}
uses: actions/upload-artifact@v2
with:
name: cover-reports
path: covprofile_${{ matrix.worker_id }}
report-coverage:
needs: chunks
runs-on: ubuntu-latest
steps:
- name: Checkout code
uses: actions/checkout@v2
- name: Download chunk report
uses: actions/download-artifact@v2
with:
name: cover-reports
- name: Merge
env:
TOTAL_JOBS: ${{needs.chunks.outputs.job-total}}
run: |
echo ${TOTAL_JOBS}
for i in $(seq 1 $TOTAL_JOBS); do cat covprofile_$i >> covprofile; done
cat covprofile
- name: Send coverage
uses: codecov/codecov-action@v1
with:
token: ${{ secrets.CODECOV }}
file: ./covprofile
flags: unittests
name: codecov-umbrella
run: make ci-test-job JOB_COUNT=8 JOB_INDEX=${{ matrix.worker_id }}
2 changes: 1 addition & 1 deletion .github/workflows/tso-consistency-test.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ jobs:
steps:
- uses: actions/setup-go@v2
with:
go-version: 1.16
go-version: 1.17
- name: Checkout code
uses: actions/checkout@v2
- name: Make TSO Consistency Test
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/tso-function-test.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ jobs:
steps:
- uses: actions/setup-go@v2
with:
go-version: 1.16
go-version: 1.17
- name: Checkout code
uses: actions/checkout@v2
- name: Make TSO Function Test
Expand Down
16 changes: 9 additions & 7 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -226,14 +226,16 @@ basic-test: install-tools
go test $(BASIC_TEST_PKGS) || { $(FAILPOINT_DISABLE); exit 1; }
@$(FAILPOINT_DISABLE)

test-with-cover: install-tools dashboard-ui
# testing all pkgs (expect TSO consistency test) with converage...
ci-test-job: install-tools dashboard-ui
@$(DEADLOCK_ENABLE)
@$(FAILPOINT_ENABLE)
CGO_ENABLED=1 go test -race $(shell ./scripts/ci-subtask.sh $(JOB_COUNT) $(JOB_INDEX)) || { $(FAILPOINT_DISABLE) && $(DEADLOCK_DISABLE) && exit 1;}
@$(FAILPOINT_DISABLE)
@$(DEADLOCK_DISABLE)

ci-cover-job: install-tools dashboard-ui
@$(FAILPOINT_ENABLE)
for PKG in $(TEST_PKGS); do\
set -euo pipefail;\
CGO_ENABLED=1 go test -race -covermode=atomic -coverprofile=coverage.tmp -coverpkg=./... $$PKG 2>&1 | grep -v "no packages being tested" && tail -n +2 coverage.tmp >> covprofile || { $(FAILPOINT_DISABLE); rm coverage.tmp && exit 1;}; \
rm coverage.tmp;\
done
go test -covermode=atomic -coverprofile=covprofile -coverpkg=./... $(shell ./scripts/ci-subtask.sh $(JOB_COUNT) $(JOB_INDEX)) || { $(FAILPOINT_DISABLE) && exit 1;}
@$(FAILPOINT_DISABLE)

TSO_INTEGRATION_TEST_PKGS := $(PD_PKG)/tests/server/tso
Expand Down
31 changes: 31 additions & 0 deletions scripts/ci-subtask.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
#!/bin/bash

# ./ci-subtask.sh <TOTAL_TASK_N> <TASK_INDEX>

packages=(`go list ./...`)
dirs=(`find . -iname "*_test.go" -exec dirname {} \; | sort -u | sed -e "s/^\./github.com\/tikv\/pd/"`)
tasks=($(comm -12 <(printf "%s\n" "${packages[@]}") <(printf "%s\n" "${dirs[@]}")))

weight () {
if [[ $1 == "github.com/tikv/pd/server/api" ]]; then return 30; fi
if [[ $1 == "github.com/tikv/pd/server/tso" ]]; then return 30; fi
if [[ $1 == "github.com/tikv/pd/server/schedule" ]]; then return 30; fi
if [[ $1 == "github.com/tikv/pd/tests/client" ]]; then return 30; fi
if [[ $1 =~ "pd/tests" ]]; then return 5; fi
return 1
}

scores=(`seq "$1" | xargs -I{} echo 0`)

res=()
for t in ${tasks[@]}; do
min_i=0
for i in ${!scores[@]}; do
if [[ ${scores[i]} -lt ${scores[$min_i]} ]]; then min_i=$i; fi
done
weight $t
scores[$min_i]=$((${scores[$min_i]} + $?))
if [[ $(($min_i+1)) -eq $2 ]]; then res+=($t); fi
done

printf "%s " "${res[@]}"

0 comments on commit 9205dd3

Please sign in to comment.