Skip to content

Commit

Permalink
add failpoint raftBeforeAdvance
Browse files Browse the repository at this point in the history
Signed-off-by: Chao Chen <chaochn@amazon.com>
  • Loading branch information
chaochn47 committed May 10, 2023
1 parent 5d1130f commit 7ec6f42
Show file tree
Hide file tree
Showing 8 changed files with 156 additions and 34 deletions.
9 changes: 9 additions & 0 deletions bill-of-materials.json
Original file line number Diff line number Diff line change
Expand Up @@ -521,6 +521,15 @@
}
]
},
{
"project": "go.etcd.io/gofail/runtime",
"licenses": [
{
"type": "Apache License 2.0",
"confidence": 1
}
]
},
{
"project": "go.etcd.io/raft/v3",
"licenses": [
Expand Down
6 changes: 4 additions & 2 deletions server/etcdserver/raft.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,12 +23,13 @@ import (

"go.uber.org/zap"

"go.etcd.io/raft/v3"
"go.etcd.io/raft/v3/raftpb"

"go.etcd.io/etcd/client/pkg/v3/logutil"
"go.etcd.io/etcd/pkg/v3/contention"
"go.etcd.io/etcd/server/v3/etcdserver/api/rafthttp"
serverstorage "go.etcd.io/etcd/server/v3/storage"
"go.etcd.io/raft/v3"
"go.etcd.io/raft/v3/raftpb"
)

const (
Expand Down Expand Up @@ -307,6 +308,7 @@ func (r *raftNode) start(rh *raftReadyHandler) {
notifyc <- struct{}{}
}

// gofail: var raftBeforeAdvance struct{}
r.Advance()
case <-r.stopped:
return
Expand Down
25 changes: 25 additions & 0 deletions tests/failpoint/cluster_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
// Copyright 2023 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package failpoint

import (
"testing"
)

// TestMemberPromoteMemberNotLearnerWithFailpoint ensures that promoting a voting member fails with injected failpoint .
// make gofail-enable && pushd tests/failpoint && go test -v -run TestMemberPromoteMemberNotLearnerWithFailpoint && popd
func TestMemberPromoteMemberNotLearnerWithFailpoint(t *testing.T) {
MemberPromoteMemberNotLearnerTest(t, NewFailpoint("raftBeforeAdvance", `sleep(100)`))
}
64 changes: 64 additions & 0 deletions tests/failpoint/cluster_test_common.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
// Copyright 2023 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package failpoint

import (
"context"
"strings"
"testing"

"github.com/stretchr/testify/require"

integration2 "go.etcd.io/etcd/tests/v3/framework/integration"
)

func MemberPromoteMemberNotLearnerTest(t *testing.T, f Failpoint) {
integration2.BeforeTest(t)

require.NoError(t, f.Enable())
defer func() {
require.NoError(t, f.Disable())
}()

clus := integration2.NewCluster(t, &integration2.ClusterConfig{Size: 3})
defer clus.Terminate(t)
// member promote request can be sent to any server in cluster,
// the request will be auto-forwarded to leader on server-side.
// This test explicitly includes the server-side forwarding by
// sending the request to follower.
leaderIdx := clus.WaitLeader(t)
followerIdx := (leaderIdx + 1) % 3
cli := clus.Client(followerIdx)

resp, err := cli.MemberList(context.Background())
if err != nil {
t.Fatalf("failed to list member %v", err)
}
if len(resp.Members) != 3 {
t.Fatalf("number of members = %d, want %d", len(resp.Members), 3)
}

// promoting any of the voting members in cluster should fail
expectedErrKeywords := "can only promote a learner member"
for _, m := range resp.Members {
_, err = cli.MemberPromote(context.Background(), m.ID)
if err == nil {
t.Fatalf("expect promoting voting member to fail, got no error")
}
if !strings.Contains(err.Error(), expectedErrKeywords) {
t.Fatalf("expect error to contain %s, got %s", expectedErrKeywords, err.Error())
}
}
}
49 changes: 49 additions & 0 deletions tests/failpoint/failpoint.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
// Copyright 2023 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package failpoint

import (
gofail "go.etcd.io/gofail/runtime"
)

var EmptyFailPoint = &failpointFunc{}

type Failpoint interface {
Enable() error
Disable() error
}

type failpointFunc struct {
name string
payload string
}

func (f *failpointFunc) Enable() error {
if f == nil || len(f.name) == 0 {
return nil
}
return gofail.Enable(f.name, f.payload)
}

func (f *failpointFunc) Disable() error {
if f == nil || len(f.name) == 0 {
return nil
}
return gofail.Disable(f.name)
}

func NewFailpoint(name, payload string) *failpointFunc {
return &failpointFunc{name, payload}
}
1 change: 1 addition & 0 deletions tests/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ require (
go.etcd.io/etcd/etcdutl/v3 v3.6.0-alpha.0
go.etcd.io/etcd/pkg/v3 v3.6.0-alpha.0
go.etcd.io/etcd/server/v3 v3.6.0-alpha.0
go.etcd.io/gofail v0.1.0
go.etcd.io/raft/v3 v3.0.0-20221201111702-eaa6808e1f7a
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.37.0
go.opentelemetry.io/otel v1.14.0
Expand Down
2 changes: 2 additions & 0 deletions tests/go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -264,6 +264,8 @@ github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ=
go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw=
go.etcd.io/gofail v0.1.0 h1:XItAMIhOojXFQMgrxjnd2EIIHun/d5qL0Pf7FzVTkFg=
go.etcd.io/gofail v0.1.0/go.mod h1:VZBCXYGZhHAinaBiiqYvuDynvahNsAyLFwB3kEHKz1M=
go.etcd.io/raft/v3 v3.0.0-20221201111702-eaa6808e1f7a h1:Znv2XJyAf/fsJsFNt9toO8uyXwwHQ44wxqsvdSxipj4=
go.etcd.io/raft/v3 v3.0.0-20221201111702-eaa6808e1f7a/go.mod h1:eMshmuwXLWZrjHXN8ZgYrOMQRSbHqi5M84DEZWhG+o4=
go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
Expand Down
34 changes: 2 additions & 32 deletions tests/integration/clientv3/cluster_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ import (
"time"

"go.etcd.io/etcd/client/pkg/v3/types"
"go.etcd.io/etcd/tests/v3/failpoint"
integration2 "go.etcd.io/etcd/tests/v3/framework/integration"
)

Expand Down Expand Up @@ -294,38 +295,7 @@ func TestMemberPromote(t *testing.T) {

// TestMemberPromoteMemberNotLearner ensures that promoting a voting member fails.
func TestMemberPromoteMemberNotLearner(t *testing.T) {
integration2.BeforeTest(t)

clus := integration2.NewCluster(t, &integration2.ClusterConfig{Size: 3})
defer clus.Terminate(t)

// member promote request can be sent to any server in cluster,
// the request will be auto-forwarded to leader on server-side.
// This test explicitly includes the server-side forwarding by
// sending the request to follower.
leaderIdx := clus.WaitLeader(t)
followerIdx := (leaderIdx + 1) % 3
cli := clus.Client(followerIdx)

resp, err := cli.MemberList(context.Background())
if err != nil {
t.Fatalf("failed to list member %v", err)
}
if len(resp.Members) != 3 {
t.Fatalf("number of members = %d, want %d", len(resp.Members), 3)
}

// promoting any of the voting members in cluster should fail
expectedErrKeywords := "can only promote a learner member"
for _, m := range resp.Members {
_, err = cli.MemberPromote(context.Background(), m.ID)
if err == nil {
t.Fatalf("expect promoting voting member to fail, got no error")
}
if !strings.Contains(err.Error(), expectedErrKeywords) {
t.Fatalf("expect error to contain %s, got %s", expectedErrKeywords, err.Error())
}
}
failpoint.MemberPromoteMemberNotLearnerTest(t, failpoint.EmptyFailPoint)
}

// TestMemberPromoteMemberNotExist ensures that promoting a member that does not exist in cluster fails.
Expand Down

0 comments on commit 7ec6f42

Please sign in to comment.