Skip to content

Commit

Permalink
Merge branch 'master' into feature/bucket
Browse files Browse the repository at this point in the history
  • Loading branch information
bufferflies committed Feb 25, 2022
2 parents f0456e3 + 4b6f00c commit c1f10d6
Show file tree
Hide file tree
Showing 79 changed files with 3,015 additions and 892 deletions.
7 changes: 6 additions & 1 deletion .github/workflows/pd-tests.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,13 @@ on:
push:
branches:
- master
- release-4.0
- release-5.*
pull_request:
branches:
- master
- release-4.0
- release-5.*
jobs:
chunks:
runs-on: ubuntu-latest
Expand Down Expand Up @@ -35,7 +39,8 @@ jobs:
WORKER_ID: ${{ matrix.worker_id }}
WORKER_COUNT: 10
run: |
make ci-test-job JOB_COUNT=$WORKER_COUNT JOB_INDEX=$WORKER_ID
[ $WORKER_ID -ne $WORKER_COUNT ] && make ci-test-job JOB_COUNT=$(($WORKER_COUNT-1)) JOB_INDEX=$WORKER_ID
[ $WORKER_ID -eq $WORKER_COUNT ] && make ci-test-job-submod
mv covprofile covprofile_$WORKER_ID
sed -i "/failpoint_binding/d" covprofile_$WORKER_ID
- name: Upload coverage result ${{ matrix.worker_id }}
Expand Down
2 changes: 2 additions & 0 deletions .github/workflows/tso-function-test.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,11 @@ on:
push:
branches:
- master
- release-5.*
pull_request:
branches:
- master
- release-5.*
jobs:
tso-function-test:
runs-on: ubuntu-latest
Expand Down
23 changes: 16 additions & 7 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@ pd-server-basic:
pd-ctl:
CGO_ENABLED=0 go build -gcflags '$(GCFLAGS)' -ldflags '$(LDFLAGS)' -o $(BUILD_BIN_PATH)/pd-ctl tools/pd-ctl/main.go
pd-tso-bench:
CGO_ENABLED=0 go build -o $(BUILD_BIN_PATH)/pd-tso-bench tools/pd-tso-bench/main.go
cd tools/pd-tso-bench && CGO_ENABLED=0 go build -o $(BUILD_BIN_PATH)/pd-tso-bench main.go
pd-recover:
CGO_ENABLED=0 go build -gcflags '$(GCFLAGS)' -ldflags '$(LDFLAGS)' -o $(BUILD_BIN_PATH)/pd-recover tools/pd-recover/main.go
pd-analysis:
Expand Down Expand Up @@ -147,17 +147,20 @@ install-tools:
check: install-tools static tidy check-plugin errdoc check-testing-t

static: install-tools
@ # Not running vet and fmt through metalinter becauase it ends up looking at vendor
@ echo "gofmt ..."
@ gofmt -s -l -d $(PACKAGE_DIRECTORIES) 2>&1 | awk '{ print } END { if (NR > 0) { exit 1 } }'
@ echo "golangci-lint ..."
@ golangci-lint run $(PACKAGE_DIRECTORIES)
@ echo "revive ..."
@revive -formatter friendly -config revive.toml $(PACKAGES)
@ revive -formatter friendly -config revive.toml $(PACKAGES)

@ for mod in $(SUBMODULES); do cd $$mod && $(MAKE) static && cd - > /dev/null; done

tidy:
go mod tidy
@ go mod tidy
git diff --quiet go.mod go.sum

@ for mod in $(SUBMODULES); do cd $$mod && $(MAKE) tidy && cd - > /dev/null; done

check-plugin:
@echo "checking plugin"
Expand Down Expand Up @@ -215,6 +218,9 @@ TEST_PKGS := $(filter $(shell find . -iname "*_test.go" -exec dirname {} \; | \
sort -u | sed -e "s/^\./github.com\/tikv\/pd/"),$(PACKAGES))
BASIC_TEST_PKGS := $(filter-out $(PD_PKG)/tests%,$(TEST_PKGS))

SUBMODULES := $(filter $(shell find . -iname "go.mod" -exec dirname {} \;),\
$(filter-out .,$(shell find . -iname "Makefile" -exec dirname {} \;)))

test: install-tools
# testing all pkgs...
@$(DEADLOCK_ENABLE)
Expand All @@ -232,9 +238,12 @@ basic-test: install-tools
ci-test-job: install-tools dashboard-ui
@$(DEADLOCK_ENABLE)
@$(FAILPOINT_ENABLE)
CGO_ENABLED=1 go test -race -covermode=atomic -coverprofile=covprofile -coverpkg=./... $(shell ./scripts/ci-subtask.sh $(JOB_COUNT) $(JOB_INDEX)) || { $(FAILPOINT_DISABLE) && $(DEADLOCK_DISABLE) && exit 1;}
@$(FAILPOINT_DISABLE)
@$(DEADLOCK_DISABLE)
CGO_ENABLED=1 go test -race -covermode=atomic -coverprofile=covprofile -coverpkg=./... $(shell ./scripts/ci-subtask.sh $(JOB_COUNT) $(JOB_INDEX))

ci-test-job-submod: install-tools dashboard-ui
@$(DEADLOCK_ENABLE)
@$(FAILPOINT_ENABLE)
@ for mod in $(SUBMODULES); do cd $$mod && $(MAKE) ci-test-job && cd - > /dev/null && cat $$mod/covprofile >> covprofile; done

TSO_INTEGRATION_TEST_PKGS := $(PD_PKG)/tests/server/tso

Expand Down
41 changes: 41 additions & 0 deletions client/Makefile
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
# Copyright 2022 TiKV Project Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

GO_TOOLS_BIN_PATH := $(shell pwd)/../.tools/bin
PATH := $(GO_TOOLS_BIN_PATH):$(PATH)
SHELL := env PATH='$(PATH)' GOBIN='$(GO_TOOLS_BIN_PATH)' $(shell which bash)

default: static tidy test

test:
CGO_ENABLE=1 go test -race -cover

basic-test:
CGO_ENABLE=1 go test

ci-test-job:
CGO_ENABLED=1 go test -race -covermode=atomic -coverprofile=covprofile -coverpkg=../... github.com/tikv/pd/client

install-tools:
cd .. && $(MAKE) install-tools

static: install-tools
@ gofmt -s -l -d . 2>&1 | awk '{ print } END { if (NR > 0) { exit 1 } }'
@ golangci-lint run -c ../.golangci.yml ./...
@ revive -formatter friendly -config ../revive.toml .

tidy:
@ go mod tidy
git diff --quiet go.mod go.sum

7 changes: 4 additions & 3 deletions client/base_client.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,8 +27,9 @@ import (
"github.com/pingcap/failpoint"
"github.com/pingcap/kvproto/pkg/pdpb"
"github.com/pingcap/log"
"github.com/tikv/pd/pkg/errs"
"github.com/tikv/pd/pkg/grpcutil"
"github.com/tikv/pd/client/errs"
"github.com/tikv/pd/client/grpcutil"
"github.com/tikv/pd/client/tlsutil"
"go.uber.org/zap"
"google.golang.org/grpc"
)
Expand Down Expand Up @@ -399,7 +400,7 @@ func (c *baseClient) getOrCreateGRPCConn(addr string) (*grpc.ClientConn, error)
if ok {
return conn.(*grpc.ClientConn), nil
}
tlsCfg, err := grpcutil.TLSConfig{
tlsCfg, err := tlsutil.TLSConfig{
CAPath: c.security.CAPath,
CertPath: c.security.CertPath,
KeyPath: c.security.KeyPath,
Expand Down
4 changes: 2 additions & 2 deletions client/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -29,8 +29,8 @@ import (
"github.com/pingcap/kvproto/pkg/metapb"
"github.com/pingcap/kvproto/pkg/pdpb"
"github.com/pingcap/log"
"github.com/tikv/pd/pkg/errs"
"github.com/tikv/pd/pkg/grpcutil"
"github.com/tikv/pd/client/errs"
"github.com/tikv/pd/client/grpcutil"
"go.uber.org/zap"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
Expand Down
2 changes: 1 addition & 1 deletion client/client_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ import (
. "github.com/pingcap/check"
"github.com/pingcap/errors"
"github.com/pingcap/kvproto/pkg/pdpb"
"github.com/tikv/pd/pkg/testutil"
"github.com/tikv/pd/client/testutil"
"go.uber.org/goleak"
"google.golang.org/grpc"
)
Expand Down
61 changes: 61 additions & 0 deletions client/errs/errno.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
// Copyright 2022 TiKV Project Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package errs

import "github.com/pingcap/errors"

const (
// NotLeaderErr indicates the the non-leader member received the requests which should be received by leader.
NotLeaderErr = "is not leader"
// MismatchLeaderErr indicates the the non-leader member received the requests which should be received by leader.
MismatchLeaderErr = "mismatch leader id"
)

// client errors
var (
ErrClientCreateTSOStream = errors.Normalize("create TSO stream failed, %s", errors.RFCCodeText("PD:client:ErrClientCreateTSOStream"))
ErrClientGetTSOTimeout = errors.Normalize("get TSO timeout", errors.RFCCodeText("PD:client:ErrClientGetTSOTimeout"))
ErrClientGetTSO = errors.Normalize("get TSO failed, %v", errors.RFCCodeText("PD:client:ErrClientGetTSO"))
ErrClientGetLeader = errors.Normalize("get leader from %v error", errors.RFCCodeText("PD:client:ErrClientGetLeader"))
ErrClientGetMember = errors.Normalize("get member failed", errors.RFCCodeText("PD:client:ErrClientGetMember"))
)

// grpcutil errors
var (
ErrSecurityConfig = errors.Normalize("security config error: %s", errors.RFCCodeText("PD:grpcutil:ErrSecurityConfig"))
)

// The third-party project error.
// url errors
var (
ErrURLParse = errors.Normalize("parse url error", errors.RFCCodeText("PD:url:ErrURLParse"))
)

// grpc errors
var (
ErrGRPCDial = errors.Normalize("dial error", errors.RFCCodeText("PD:grpc:ErrGRPCDial"))
ErrCloseGRPCConn = errors.Normalize("close gRPC connection failed", errors.RFCCodeText("PD:grpc:ErrCloseGRPCConn"))
)

// etcd errors
var (
ErrEtcdTLSConfig = errors.Normalize("etcd TLS config error", errors.RFCCodeText("PD:etcd:ErrEtcdTLSConfig"))
)

// crypto
var (
ErrCryptoX509KeyPair = errors.Normalize("x509 keypair error", errors.RFCCodeText("PD:crypto:ErrCryptoX509KeyPair"))
ErrCryptoAppendCertsFromPEM = errors.Normalize("cert pool append certs error", errors.RFCCodeText("PD:crypto:ErrCryptoAppendCertsFromPEM"))
)
36 changes: 36 additions & 0 deletions client/errs/errs.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
// Copyright 2022 TiKV Project Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package errs

import (
"github.com/pingcap/errors"
"go.uber.org/zap"
"go.uber.org/zap/zapcore"
)

// ZapError is used to make the log output easier.
func ZapError(err error, causeError ...error) zap.Field {
if err == nil {
return zap.Skip()
}
if e, ok := err.(*errors.Error); ok {
if len(causeError) >= 1 {
err = e.Wrap(causeError[0]).FastGenWithCause()
} else {
err = e.FastGenByArgs()
}
}
return zap.Field{Key: "error", Type: zapcore.ErrorType, Interface: err}
}
18 changes: 18 additions & 0 deletions client/go.mod
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
module github.com/tikv/pd/client

go 1.16

require (
github.com/opentracing/opentracing-go v1.2.0
github.com/pingcap/check v0.0.0-20211026125417-57bd13f7b5f0
github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c
github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00
github.com/pingcap/kvproto v0.0.0-20220106070556-3fa8fa04f898
github.com/pingcap/log v0.0.0-20211215031037-e024ba4eb0ee
github.com/prometheus/client_golang v1.11.0
go.uber.org/goleak v1.1.11
go.uber.org/zap v1.20.0
google.golang.org/grpc v1.43.0
)

replace github.com/pingcap/kvproto v0.0.0-20220215045702-d229fcc888c8 => github.com/bufferflies/kvproto v0.0.0-20220224121437-27ddbf52d12f
Loading

0 comments on commit c1f10d6

Please sign in to comment.