Skip to content

Commit

Permalink
Merge pull request #17173 from ZhouJianMS/member-replace-3.4
Browse files Browse the repository at this point in the history
[3.4] member replace e2e test
  • Loading branch information
ahrtr authored Jan 12, 2024
2 parents 3ee1503 + 2f8d7d0 commit 148ba41
Show file tree
Hide file tree
Showing 3 changed files with 151 additions and 0 deletions.
106 changes: 106 additions & 0 deletions tests/e2e/ctl_v3_member_no_proxy_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,106 @@
// Copyright 2023 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

//go:build !cluster_proxy

package e2e

import (
"context"
"math/rand"
"os"
"testing"
"time"

"github.com/stretchr/testify/require"

"go.etcd.io/etcd/etcdserver"
"go.etcd.io/etcd/pkg/testutil"
)

func TestMemberReplace(t *testing.T) {
defer testutil.AfterTest(t)
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
defer cancel()

epc, err := newEtcdProcessCluster(t, &etcdProcessClusterConfig{
clusterSize: 3,
keepDataDir: true,
corruptCheckTime: time.Second,
})
require.NoError(t, err)
defer epc.Close()

memberIdx := rand.Int() % len(epc.procs)
member := epc.procs[memberIdx]
memberName := member.Config().name
var endpoints []string
for i := 1; i < len(epc.procs); i++ {
endpoints = append(endpoints, epc.procs[(memberIdx+i)%len(epc.procs)].EndpointsGRPC()...)
}
cc := NewEtcdctl(endpoints, clientNonTLS, false, false)

memberID, found, err := getMemberIdByName(ctx, cc, memberName)
require.NoError(t, err)
require.Equal(t, found, true, "Member not found")

// Need to wait health interval for cluster to accept member changes
time.Sleep(etcdserver.HealthInterval)

t.Logf("Removing member %s", memberName)
_, err = cc.MemberRemove(memberID)
require.NoError(t, err)
_, found, err = getMemberIdByName(ctx, cc, memberName)
require.NoError(t, err)
require.Equal(t, found, false, "Expected member to be removed")
for member.IsRunning() {
member.Close()
time.Sleep(10 * time.Millisecond)
}

t.Logf("Removing member %s data", memberName)
err = os.RemoveAll(member.Config().dataDirPath)
require.NoError(t, err)

t.Logf("Adding member %s back", memberName)
removedMemberPeerUrl := member.Config().purl.String()
// 3.4 has probability that the leader election is still in progress.
// So we need to retry until the member is added back.
for {
_, err = cc.MemberAdd(memberName, []string{removedMemberPeerUrl})
if err == nil {
break
}
time.Sleep(100 * time.Millisecond)
}
require.NoError(t, err)
member.Config().args = patchArgs(member.Config().args, "initial-cluster-state", "existing")
require.NoError(t, err)

// Sleep 100ms to bypass the known issue https://github.com/etcd-io/etcd/issues/16687.
time.Sleep(100 * time.Millisecond)
t.Logf("Starting member %s", memberName)
err = member.Start()
require.NoError(t, err)
executeUntil(ctx, t, func() {
for {
_, found, err := getMemberIdByName(ctx, cc, memberName)
if err != nil || !found {
time.Sleep(10 * time.Millisecond)
continue
}
break
}
})
}
18 changes: 18 additions & 0 deletions tests/e2e/etcdctl.go
Original file line number Diff line number Diff line change
Expand Up @@ -115,6 +115,24 @@ func (ctl *Etcdctl) MemberList() (*clientv3.MemberListResponse, error) {
return &resp, err
}

func (ctl *Etcdctl) MemberAdd(name string, peerURLs []string) (*clientv3.MemberAddResponse, error) {
if ctl.v2 {
panic("Unsupported method for v2")
}
var resp clientv3.MemberAddResponse
err := ctl.spawnJsonCmd(&resp, "", "member", "add", name, "--peer-urls", strings.Join(peerURLs, ","))
return &resp, err
}

func (ctl *Etcdctl) MemberRemove(id uint64) (*clientv3.MemberRemoveResponse, error) {
if ctl.v2 {
panic("Unsupported method for v2")
}
var resp clientv3.MemberRemoveResponse
err := ctl.spawnJsonCmd(&resp, "", "member", "remove", fmt.Sprintf("%x", id))
return &resp, err
}

func (ctl *Etcdctl) Compact(rev int64) (*clientv3.CompactResponse, error) {
if ctl.v2 {
panic("Unsupported method for v2")
Expand Down
27 changes: 27 additions & 0 deletions tests/e2e/utils.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ package e2e
import (
"context"
"fmt"
"strings"
"testing"
"time"

Expand Down Expand Up @@ -129,3 +130,29 @@ func fillEtcdWithData(ctx context.Context, c *clientv3.Client, dbSize int) error
}
return g.Wait()
}

func getMemberIdByName(ctx context.Context, c *Etcdctl, name string) (id uint64, found bool, err error) {
resp, err := c.MemberList()
if err != nil {
return 0, false, err
}
for _, member := range resp.Members {
if name == member.Name {
return member.ID, true, nil
}
}
return 0, false, nil
}

// Different implementations here since 3.4 e2e test framework does not have "initial-cluster-state" as a default argument
// Append new flag if not exist, otherwise replace the value
func patchArgs(args []string, flag, newValue string) []string {
for i, arg := range args {
if strings.Contains(arg, flag) {
args[i] = fmt.Sprintf("--%s=%s", flag, newValue)
return args
}
}
args = append(args, fmt.Sprintf("--%s=%s", flag, newValue))
return args
}

0 comments on commit 148ba41

Please sign in to comment.