add e2e test

server/auth/range_perm_cache.go

@@ -131,6 +131,8 @@ func (as *authStore) refreshRangePermCache(tx AuthReadTx) {
 	as.rangePermCacheMu.Lock()
 	defer as.rangePermCacheMu.Unlock()
 
+	as.lg.Debug("Refreshing rangePermCache")
+
 	as.rangePermCache = make(map[string]*unifiedRangePermissions)
 
 	users := tx.UnsafeGetAllUsers()

tests/e2e/ctl_v3_auth_cluster_test.go

@@ -0,0 +1,138 @@
+package e2e
+
+import (
+	"context"
+	"fmt"
+	clientv3 "go.etcd.io/etcd/client/v3"
+	"go.etcd.io/etcd/tests/v3/framework/config"
+	"go.etcd.io/etcd/tests/v3/framework/e2e"
+	"sync"
+	"testing"
+)
+
+const (
+	numKeys      = 10
+	writeThreads = 10
+
+	rootUser     = "root"
+	rootPassword = "root"
+	rootRole     = "root"
+	testUser     = "test"
+	testPassword = "testPassword"
+	testRole     = "testRole"
+)
+
+func TestAuthCluster(t *testing.T) {
+	e2e.BeforeTest(t)
+	cfg := e2e.NewConfigNoTLS()
+	cfg.ClusterSize = 1
+	cfg.SnapshotCount = 2
+	cfg.UserName = rootUser
+	cfg.Password = rootPassword
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	epc, err := e2e.NewEtcdProcessCluster(ctx, t, cfg)
+	if err != nil {
+		t.Fatalf("could not start etcd process cluster (%v)", err)
+	}
+	defer func() {
+		if err := epc.Close(); err != nil {
+			t.Fatalf("could not close test cluster (%v)", err)
+		}
+	}()
+
+	epcClient := epc.Client()
+	createUsers(ctx, t, epcClient)
+
+	if _, err := epcClient.AuthEnable(ctx); err != nil {
+		t.Fatalf("could not enable Auth: (%v)", err)
+	}
+
+	writeKeys(ctx, epc.Client().WithAuth(testUser, testPassword), "test")
+
+	if err := epc.StartNewProc(ctx, t); err != nil {
+		t.Fatalf("could not start second etcd process (%v)", err)
+	}
+
+	writeKeys(ctx, epc.Client().WithAuth(testUser, testPassword), "test_two_nodes")
+
+	hashKvs, err := epc.Client().HashKV(ctx, numKeys*2)
+	if err != nil {
+		t.Fatalf("could not HashKV (%v)", err)
+	}
+
+	revisionSet := false
+	var revision int64
+	var hash uint32
+	for _, v := range hashKvs {
+		if !revisionSet {
+			revision = v.Header.GetRevision()
+			hash = v.Hash
+			revisionSet = true
+			continue
+		}
+		if revision != v.Header.GetRevision() || hash != v.Hash {
+			t.Errorf("Inconsistent revisions found:")
+			for _, v := range hashKvs {
+				t.Errorf("%+v", v)
+			}
+			t.Fail()
+		}
+	}
+}
+
+func createUsers(ctx context.Context, t *testing.T, client *e2e.EtcdctlV3) {
+	if _, err := client.UserAdd(ctx, rootUser, rootPassword, config.UserAddOptions{}); err != nil {
+		t.Fatalf("could not add root user (%v)", err)
+	}
+	if _, err := client.UserGrantRole(ctx, rootUser, rootRole); err != nil {
+		t.Fatalf("could not grant root role to root user (%v)", err)
+	}
+
+	if _, err := client.RoleAdd(ctx, testRole); err != nil {
+		t.Fatalf("could not create 'test' role (%v)", err)
+	}
+	if _, err := client.RoleGrantPermission(ctx, testRole, "/test/", "/test0", clientv3.PermissionType(clientv3.PermReadWrite)); err != nil {
+		t.Fatalf("could not RoleGrantPermission (%v)", err)
+	}
+	if _, err := client.UserAdd(ctx, testUser, testPassword, config.UserAddOptions{}); err != nil {
+		t.Fatalf("could not add user test (%v)", err)
+	}
+	if _, err := client.UserGrantRole(ctx, testUser, testRole); err != nil {
+		t.Fatalf("could not grant test role user (%v)", err)
+	}
+}
+
+func writeKeys(ctx context.Context, client *e2e.EtcdctlV3, value string) {
+	type kv struct {
+		key   string
+		value string
+	}
+	q := make(chan kv)
+
+	wg := sync.WaitGroup{}
+	for i := 0; i <= writeThreads; i++ {
+		wg.Add(1)
+		go func() {
+			for {
+				item, ok := <-q
+				if !ok {
+					break
+				}
+				if err := client.Put(ctx, item.key, item.value, config.PutOptions{}); err != nil {
+					panic(err)
+				}
+			}
+			wg.Done()
+		}()
+	}
+
+	for i := 0; i <= numKeys; i++ {
+		q <- kv{key: fmt.Sprintf("/test/%d", i), value: value}
+	}
+	close(q)
+
+	wg.Wait()
+}

tests/framework/e2e/cluster.go

@@ -182,6 +182,10 @@ type EtcdProcessClusterConfig struct {
 	CorruptCheckTime        time.Duration
 	CompactHashCheckEnabled bool
 	CompactHashCheckTime    time.Duration
+
+	// user/password pair to be used for client connections
+	UserName string
+	Password string
 }
 
 // NewEtcdProcessCluster launches a new cluster from etcd processes, returning
@@ -216,7 +220,7 @@ func InitEtcdProcessCluster(t testing.TB, cfg *EtcdProcessClusterConfig) (*EtcdP
 	etcdCfgs := cfg.EtcdAllServerProcessConfigs(t)
 	epc := &EtcdProcessCluster{
 		Cfg:     cfg,
-		lg:      zaptest.NewLogger(t),
+		lg:      cfg.Logger,
 		Procs:   make([]EtcdProcess, cfg.ClusterSize),
 		nextSeq: cfg.ClusterSize,
 	}
@@ -602,7 +606,11 @@ func (epc *EtcdProcessCluster) Stop() (err error) {
 }
 
 func (epc *EtcdProcessCluster) Client() *EtcdctlV3 {
-	return NewEtcdctl(epc.Cfg, epc.EndpointsV3())
+	client := NewEtcdctl(epc.Cfg, epc.EndpointsV3())
+	if epc.Cfg.UserName != "" {
+		client = client.WithAuth(epc.Cfg.UserName, epc.Cfg.Password)
+	}
+	return client
 }
 
 func (epc *EtcdProcessCluster) Close() error {

tests/framework/e2e/etcdctl.go

@@ -337,8 +337,8 @@ func (ctl *EtcdctlV3) HashKV(ctx context.Context, rev int64) ([]*clientv3.HashKV
 		return nil, err
 	}
 	resp := make([]*clientv3.HashKVResponse, len(epHashKVs))
-	for _, e := range epHashKVs {
-		resp = append(resp, e.HashKV)
+	for i, e := range epHashKVs {
+		resp[i] = e.HashKV
 	}
 	return resp, err
 }
@@ -466,15 +466,41 @@ func (ctl *EtcdctlV3) AlarmDisarm(ctx context.Context, _ *clientv3.AlarmMember)
 }
 
 func (ctl *EtcdctlV3) AuthEnable(ctx context.Context) (*clientv3.AuthEnableResponse, error) {
-	var resp clientv3.AuthEnableResponse
-	err := ctl.spawnJsonCmd(ctx, &resp, "auth", "enable")
-	return &resp, err
+	args := []string{"auth", "enable", "-w", "json"}
+	cmd, err := SpawnCmd(append(ctl.cmdArgs(), args...), nil)
+	if err != nil {
+		return nil, err
+	}
+	err = cmd.Send(strings.Join(args, " "))
+	if err != nil {
+		return nil, err
+	}
+	defer cmd.Close()
+
+	_, err = cmd.ExpectWithContext(ctx, "Authentication Enabled")
+	if err != nil {
+		return nil, err
+	}
+	return &clientv3.AuthEnableResponse{}, nil
 }
 
 func (ctl *EtcdctlV3) AuthDisable(ctx context.Context) (*clientv3.AuthDisableResponse, error) {
-	var resp clientv3.AuthDisableResponse
-	err := ctl.spawnJsonCmd(ctx, &resp, "auth", "disable")
-	return &resp, err
+	args := []string{"auth", "disable", "-w", "json"}
+	cmd, err := SpawnCmd(append(ctl.cmdArgs(), args...), nil)
+	if err != nil {
+		return nil, err
+	}
+	err = cmd.Send(strings.Join(args, " "))
+	if err != nil {
+		return nil, err
+	}
+	defer cmd.Close()
+
+	_, err = cmd.ExpectWithContext(ctx, "Authentication Disabled")
+	if err != nil {
+		return nil, err
+	}
+	return &clientv3.AuthDisableResponse{}, nil
 }
 
 func (ctl *EtcdctlV3) AuthStatus(ctx context.Context) (*clientv3.AuthStatusResponse, error) {