diff --git a/bill-of-materials.json b/bill-of-materials.json index db45756dc769..b5cf5d624f07 100644 --- a/bill-of-materials.json +++ b/bill-of-materials.json @@ -521,6 +521,15 @@ } ] }, + { + "project": "go.etcd.io/gofail/runtime", + "licenses": [ + { + "type": "Apache License 2.0", + "confidence": 1 + } + ] + }, { "project": "go.etcd.io/raft/v3", "licenses": [ diff --git a/server/etcdserver/raft.go b/server/etcdserver/raft.go index 6e50b417f6a2..e2f26117d985 100644 --- a/server/etcdserver/raft.go +++ b/server/etcdserver/raft.go @@ -23,12 +23,13 @@ import ( "go.uber.org/zap" + "go.etcd.io/raft/v3" + "go.etcd.io/raft/v3/raftpb" + "go.etcd.io/etcd/client/pkg/v3/logutil" "go.etcd.io/etcd/pkg/v3/contention" "go.etcd.io/etcd/server/v3/etcdserver/api/rafthttp" serverstorage "go.etcd.io/etcd/server/v3/storage" - "go.etcd.io/raft/v3" - "go.etcd.io/raft/v3/raftpb" ) const ( @@ -307,6 +308,7 @@ func (r *raftNode) start(rh *raftReadyHandler) { notifyc <- struct{}{} } + // gofail: var raftBeforeAdvance struct{} r.Advance() case <-r.stopped: return diff --git a/tests/failpoint/cluster_test.go b/tests/failpoint/cluster_test.go new file mode 100644 index 000000000000..3947d50c3164 --- /dev/null +++ b/tests/failpoint/cluster_test.go @@ -0,0 +1,25 @@ +// Copyright 2023 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package failpoint + +import ( + "testing" +) + +// TestMemberPromoteMemberNotLearnerWithFailpoint ensures that promoting a voting member fails with injected failpoint . +// make gofail-enable && pushd tests/failpoint && go test -v -run TestMemberPromoteMemberNotLearnerWithFailpoint && popd +func TestMemberPromoteMemberNotLearnerWithFailpoint(t *testing.T) { + MemberPromoteMemberNotLearnerTest(t, NewFailpoint("raftBeforeAdvance", `sleep(100)`)) +} diff --git a/tests/failpoint/cluster_test_common.go b/tests/failpoint/cluster_test_common.go new file mode 100644 index 000000000000..0fa00975be6d --- /dev/null +++ b/tests/failpoint/cluster_test_common.go @@ -0,0 +1,65 @@ +// Copyright 2023 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package failpoint + +import ( + "context" + "strings" + "testing" + + "github.com/stretchr/testify/require" + + integration2 "go.etcd.io/etcd/tests/v3/framework/integration" +) + +func MemberPromoteMemberNotLearnerTest(t *testing.T, f Failpoint) { + integration2.BeforeTest(t) + + require.NoError(t, f.Enable()) + defer func() { + require.NoError(t, f.Disable()) + }() + + clus := integration2.NewCluster(t, &integration2.ClusterConfig{Size: 3}) + defer clus.Terminate(t) + + // member promote request can be sent to any server in cluster, + // the request will be auto-forwarded to leader on server-side. + // This test explicitly includes the server-side forwarding by + // sending the request to follower. + leaderIdx := clus.WaitLeader(t) + followerIdx := (leaderIdx + 1) % 3 + cli := clus.Client(followerIdx) + + resp, err := cli.MemberList(context.Background()) + if err != nil { + t.Fatalf("failed to list member %v", err) + } + if len(resp.Members) != 3 { + t.Fatalf("number of members = %d, want %d", len(resp.Members), 3) + } + + // promoting any of the voting members in cluster should fail + expectedErrKeywords := "can only promote a learner member" + for _, m := range resp.Members { + _, err = cli.MemberPromote(context.Background(), m.ID) + if err == nil { + t.Fatalf("expect promoting voting member to fail, got no error") + } + if !strings.Contains(err.Error(), expectedErrKeywords) { + t.Fatalf("expect error to contain %s, got %s", expectedErrKeywords, err.Error()) + } + } +} diff --git a/tests/failpoint/failpoint.go b/tests/failpoint/failpoint.go new file mode 100644 index 000000000000..cf77e7714825 --- /dev/null +++ b/tests/failpoint/failpoint.go @@ -0,0 +1,49 @@ +// Copyright 2023 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package failpoint + +import ( + gofail "go.etcd.io/gofail/runtime" +) + +var EmptyFailPoint = &failpointFunc{} + +type Failpoint interface { + Enable() error + Disable() error +} + +type failpointFunc struct { + name string + payload string +} + +func (f *failpointFunc) Enable() error { + if f == nil || len(f.name) == 0 { + return nil + } + return gofail.Enable(f.name, f.payload) +} + +func (f *failpointFunc) Disable() error { + if f == nil || len(f.name) == 0 { + return nil + } + return gofail.Disable(f.name) +} + +func NewFailpoint(name, payload string) *failpointFunc { + return &failpointFunc{name, payload} +} diff --git a/tests/go.mod b/tests/go.mod index 5966b5f0f415..7c090896103f 100644 --- a/tests/go.mod +++ b/tests/go.mod @@ -32,6 +32,7 @@ require ( go.etcd.io/etcd/etcdutl/v3 v3.6.0-alpha.0 go.etcd.io/etcd/pkg/v3 v3.6.0-alpha.0 go.etcd.io/etcd/server/v3 v3.6.0-alpha.0 + go.etcd.io/gofail v0.1.0 go.etcd.io/raft/v3 v3.0.0-20221201111702-eaa6808e1f7a go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.37.0 go.opentelemetry.io/otel v1.14.0 diff --git a/tests/go.sum b/tests/go.sum index 003ad77836ff..bc7936860f01 100644 --- a/tests/go.sum +++ b/tests/go.sum @@ -264,6 +264,8 @@ github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ= go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= +go.etcd.io/gofail v0.1.0 h1:XItAMIhOojXFQMgrxjnd2EIIHun/d5qL0Pf7FzVTkFg= +go.etcd.io/gofail v0.1.0/go.mod h1:VZBCXYGZhHAinaBiiqYvuDynvahNsAyLFwB3kEHKz1M= go.etcd.io/raft/v3 v3.0.0-20221201111702-eaa6808e1f7a h1:Znv2XJyAf/fsJsFNt9toO8uyXwwHQ44wxqsvdSxipj4= go.etcd.io/raft/v3 v3.0.0-20221201111702-eaa6808e1f7a/go.mod h1:eMshmuwXLWZrjHXN8ZgYrOMQRSbHqi5M84DEZWhG+o4= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= diff --git a/tests/integration/clientv3/cluster_test.go b/tests/integration/clientv3/cluster_test.go index 822855f1365c..6795b7554a93 100644 --- a/tests/integration/clientv3/cluster_test.go +++ b/tests/integration/clientv3/cluster_test.go @@ -24,6 +24,7 @@ import ( "time" "go.etcd.io/etcd/client/pkg/v3/types" + "go.etcd.io/etcd/tests/v3/failpoint" integration2 "go.etcd.io/etcd/tests/v3/framework/integration" ) @@ -294,38 +295,7 @@ func TestMemberPromote(t *testing.T) { // TestMemberPromoteMemberNotLearner ensures that promoting a voting member fails. func TestMemberPromoteMemberNotLearner(t *testing.T) { - integration2.BeforeTest(t) - - clus := integration2.NewCluster(t, &integration2.ClusterConfig{Size: 3}) - defer clus.Terminate(t) - - // member promote request can be sent to any server in cluster, - // the request will be auto-forwarded to leader on server-side. - // This test explicitly includes the server-side forwarding by - // sending the request to follower. - leaderIdx := clus.WaitLeader(t) - followerIdx := (leaderIdx + 1) % 3 - cli := clus.Client(followerIdx) - - resp, err := cli.MemberList(context.Background()) - if err != nil { - t.Fatalf("failed to list member %v", err) - } - if len(resp.Members) != 3 { - t.Fatalf("number of members = %d, want %d", len(resp.Members), 3) - } - - // promoting any of the voting members in cluster should fail - expectedErrKeywords := "can only promote a learner member" - for _, m := range resp.Members { - _, err = cli.MemberPromote(context.Background(), m.ID) - if err == nil { - t.Fatalf("expect promoting voting member to fail, got no error") - } - if !strings.Contains(err.Error(), expectedErrKeywords) { - t.Fatalf("expect error to contain %s, got %s", expectedErrKeywords, err.Error()) - } - } + failpoint.MemberPromoteMemberNotLearnerTest(t, failpoint.EmptyFailPoint) } // TestMemberPromoteMemberNotExist ensures that promoting a member that does not exist in cluster fails.