tests: Support multiple keys in linearizability tests

tests/linearizability/model.go

@@ -46,16 +46,19 @@ type EtcdResponse struct {
 	Err          error
 }
 
+type PossibleStates []EtcdState
+
 type EtcdState struct {
-	Revision int64
-	Key      string
-	Value    string
+	Revision  int64
+	KeyValues map[string]string
 }
 
 var etcdModel = porcupine.Model{
-	Init: func() interface{} { return "[]" },
+	Init: func() interface{} {
+		return "[]" // empty PossibleStates
+	},
 	Step: func(st interface{}, in interface{}, out interface{}) (bool, interface{}) {
-		var states []EtcdState
+		var states PossibleStates
 		err := json.Unmarshal([]byte(st.(string)), &states)
 		if err != nil {
 			panic(err)
@@ -101,87 +104,90 @@ var etcdModel = porcupine.Model{
 	},
 }
 
-func step(states []EtcdState, request EtcdRequest, response EtcdResponse) (bool, []EtcdState) {
+func step(states PossibleStates, request EtcdRequest, response EtcdResponse) (bool, PossibleStates) {
 	if len(states) == 0 {
-		return true, initStates(request, response)
+		// states were not initialized
+		if response.Err != nil {
+			return true, nil
+		}
+		return true, PossibleStates{initState(request, response)}
 	}
 	if response.Err != nil {
-		// Add addition states for failed request in case of failed request was persisted.
-		states = append(states, applyRequest(states, request)...)
+		states = applyFailedRequest(states, request)
 	} else {
-		// Remove states that didn't lead to response we got.
-		states = filterStateMatchesResponse(states, request, response)
+		states = applyRequest(states, request, response)
 	}
 	return len(states) > 0, states
 }
 
-func applyRequest(states []EtcdState, request EtcdRequest) []EtcdState {
-	newStates := make([]EtcdState, 0, len(states))
-	for _, s := range states {
-		newState, _ := stepState(s, request)
-		newStates = append(newStates, newState)
-	}
-	return newStates
-}
-
-func filterStateMatchesResponse(states []EtcdState, request EtcdRequest, response EtcdResponse) []EtcdState {
-	newStates := make([]EtcdState, 0, len(states))
-	for _, s := range states {
-		newState, expectResponse := stepState(s, request)
-		if expectResponse == response {
-			newStates = append(newStates, newState)
-		}
-	}
-	return newStates
-}
-
-func initStates(request EtcdRequest, response EtcdResponse) []EtcdState {
-	if response.Err != nil {
-		return []EtcdState{}
-	}
+// initState tries to create etcd state based on the first request.
+func initState(request EtcdRequest, response EtcdResponse) EtcdState {
 	state := EtcdState{
-		Key:      request.Key,
-		Revision: response.Revision,
+		Revision:  response.Revision,
+		KeyValues: map[string]string{},
 	}
 	switch request.Op {
 	case Get:
 		if response.GetData != "" {
-			state.Value = response.GetData
+			state.KeyValues[request.Key] = response.GetData
 		}
 	case Put:
-		state.Value = request.PutData
+		state.KeyValues[request.Key] = request.PutData
 	case Delete:
 	case Txn:
 		if response.TxnSucceeded {
-			state.Value = request.TxnNewData
+			state.KeyValues[request.Key] = request.TxnNewData
 		}
-		return []EtcdState{}
 	default:
 		panic("Unknown operation")
 	}
-	return []EtcdState{state}
+	return state
+}
+
+// applyFailedRequest handles a failed requests, one that it's not known if it was persisted or not.
+func applyFailedRequest(states PossibleStates, request EtcdRequest) PossibleStates {
+	for _, s := range states {
+		newState, _ := applyRequestToSingleState(s, request)
+		states = append(states, newState)
+	}
+	return states
+}
+
+// applyRequest handles a successful request by applying it to possible states and checking if they match the response.
+func applyRequest(states PossibleStates, request EtcdRequest, response EtcdResponse) PossibleStates {
+	newStates := make(PossibleStates, 0, len(states))
+	for _, s := range states {
+		newState, expectResponse := applyRequestToSingleState(s, request)
+		if expectResponse == response {
+			newStates = append(newStates, newState)
+		}
+	}
+	return newStates
 }
 
-func stepState(s EtcdState, request EtcdRequest) (EtcdState, EtcdResponse) {
-	if s.Key != request.Key {
-		panic("multiple keys not supported")
+// applyRequestToSingleState handles a successful request, returning updated state and response it would generate.
+func applyRequestToSingleState(s EtcdState, request EtcdRequest) (EtcdState, EtcdResponse) {
+	newKVs := map[string]string{}
+	for k, v := range s.KeyValues {
+		newKVs[k] = v
 	}
+	s.KeyValues = newKVs
 	resp := EtcdResponse{}
 	switch request.Op {
 	case Get:
-		resp.GetData = s.Value
+		resp.GetData = s.KeyValues[request.Key]
 	case Put:
-		s.Value = request.PutData
+		s.KeyValues[request.Key] = request.PutData
 		s.Revision += 1
 	case Delete:
-		if s.Value != "" {
-			s.Value = ""
+		if _, ok := s.KeyValues[request.Key]; ok {
+			delete(s.KeyValues, request.Key)
 			s.Revision += 1
 			resp.Deleted = 1
 		}
 	case Txn:
-		if s.Value == request.TxnExpectData {
-			s.Value = request.TxnNewData
+		if val := s.KeyValues[request.Key]; val == request.TxnExpectData {
+			s.KeyValues[request.Key] = request.TxnNewData
 			s.Revision += 1
 			resp.TxnSucceeded = true
 		}

tests/linearizability/model_test.go

@@ -51,19 +51,16 @@ func TestModel(t *testing.T) {
 		{
 			name: "Get response data should match put",
 			operations: []testOperation{
-				{req: EtcdRequest{Op: Put, Key: "key", PutData: "1"}, resp: EtcdResponse{Revision: 1}},
-				{req: EtcdRequest{Op: Get, Key: "key"}, resp: EtcdResponse{GetData: "2", Revision: 1}, failure: true},
-				{req: EtcdRequest{Op: Get, Key: "key"}, resp: EtcdResponse{GetData: "2", Revision: 2}, failure: true},
-				{req: EtcdRequest{Op: Get, Key: "key"}, resp: EtcdResponse{GetData: "1", Revision: 1}},
-			},
-		},
-		{
-			name: "Get revision should be equal to put",
-			operations: []testOperation{
-				{req: EtcdRequest{Op: Put, Key: "key"}, resp: EtcdResponse{Revision: 2}},
-				{req: EtcdRequest{Op: Get, Key: "key"}, resp: EtcdResponse{Revision: 1}, failure: true},
-				{req: EtcdRequest{Op: Get, Key: "key"}, resp: EtcdResponse{Revision: 3}, failure: true},
-				{req: EtcdRequest{Op: Get, Key: "key"}, resp: EtcdResponse{Revision: 2}},
+				{req: EtcdRequest{Op: Put, Key: "key1", PutData: "11"}, resp: EtcdResponse{Revision: 1}},
+				{req: EtcdRequest{Op: Put, Key: "key2", PutData: "12"}, resp: EtcdResponse{Revision: 2}},
+				{req: EtcdRequest{Op: Get, Key: "key1"}, resp: EtcdResponse{GetData: "11", Revision: 1}, failure: true},
+				{req: EtcdRequest{Op: Get, Key: "key1"}, resp: EtcdResponse{GetData: "12", Revision: 1}, failure: true},
+				{req: EtcdRequest{Op: Get, Key: "key1"}, resp: EtcdResponse{GetData: "12", Revision: 2}, failure: true},
+				{req: EtcdRequest{Op: Get, Key: "key1"}, resp: EtcdResponse{GetData: "11", Revision: 2}},
+				{req: EtcdRequest{Op: Get, Key: "key2"}, resp: EtcdResponse{GetData: "11", Revision: 2}, failure: true},
+				{req: EtcdRequest{Op: Get, Key: "key2"}, resp: EtcdResponse{GetData: "12", Revision: 1}, failure: true},
+				{req: EtcdRequest{Op: Get, Key: "key2"}, resp: EtcdResponse{GetData: "11", Revision: 1}, failure: true},
+				{req: EtcdRequest{Op: Get, Key: "key2"}, resp: EtcdResponse{GetData: "12", Revision: 2}},
 			},
 		},
 		{
@@ -97,7 +94,7 @@ func TestModel(t *testing.T) {
 		{
 			name: "Put can fail and be lost before delete",
 			operations: []testOperation{
-				{req: EtcdRequest{Op: Get, Key: "key"}, resp: EtcdResponse{Revision: 1}},
+				{req: EtcdRequest{Op: Delete, Key: "key"}, resp: EtcdResponse{Revision: 1}},
 				{req: EtcdRequest{Op: Put, Key: "key", PutData: "2"}, resp: EtcdResponse{Err: errors.New("failed")}},
 				{req: EtcdRequest{Op: Delete, Key: "key"}, resp: EtcdResponse{Revision: 1}},
 			},
@@ -138,7 +135,7 @@ func TestModel(t *testing.T) {
 			name: "Put can fail but be persisted and increase revision before delete",
 			operations: []testOperation{
 				// One failed request, one persisted.
-				{req: EtcdRequest{Op: Get, Key: "key"}, resp: EtcdResponse{Revision: 1}},
+				{req: EtcdRequest{Op: Delete, Key: "key"}, resp: EtcdResponse{Revision: 1}},
 				{req: EtcdRequest{Op: Put, Key: "key", PutData: "2"}, resp: EtcdResponse{Err: errors.New("failed")}},
 				{req: EtcdRequest{Op: Delete, Key: "key"}, resp: EtcdResponse{Deleted: 1, Revision: 1}, failure: true},
 				{req: EtcdRequest{Op: Delete, Key: "key"}, resp: EtcdResponse{Deleted: 1, Revision: 2}, failure: true},
@@ -173,11 +170,20 @@ func TestModel(t *testing.T) {
 		{
 			name: "Delete only increases revision on success",
 			operations: []testOperation{
-				{req: EtcdRequest{Op: Put, Key: "key", PutData: "1"}, resp: EtcdResponse{Revision: 1}},
-				{req: EtcdRequest{Op: Delete, Key: "key"}, resp: EtcdResponse{Deleted: 1, Revision: 1}, failure: true},
-				{req: EtcdRequest{Op: Delete, Key: "key"}, resp: EtcdResponse{Deleted: 1, Revision: 2}},
-				{req: EtcdRequest{Op: Delete, Key: "key"}, resp: EtcdResponse{Deleted: 0, Revision: 3}, failure: true},
-				{req: EtcdRequest{Op: Delete, Key: "key"}, resp: EtcdResponse{Deleted: 0, Revision: 2}},
+				{req: EtcdRequest{Op: Put, Key: "key1", PutData: "11"}, resp: EtcdResponse{Revision: 1}},
+				{req: EtcdRequest{Op: Put, Key: "key2", PutData: "12"}, resp: EtcdResponse{Revision: 2}},
+				{req: EtcdRequest{Op: Delete, Key: "key1"}, resp: EtcdResponse{Deleted: 1, Revision: 2}, failure: true},
+				{req: EtcdRequest{Op: Delete, Key: "key1"}, resp: EtcdResponse{Deleted: 1, Revision: 3}},
+				{req: EtcdRequest{Op: Delete, Key: "key1"}, resp: EtcdResponse{Deleted: 0, Revision: 4}, failure: true},
+				{req: EtcdRequest{Op: Delete, Key: "key1"}, resp: EtcdResponse{Deleted: 0, Revision: 3}},
+			},
+		},
+		{
+			name: "Delete not existing key",
+			operations: []testOperation{
+				{req: EtcdRequest{Op: Get, Key: "key"}, resp: EtcdResponse{Revision: 1}},
+				{req: EtcdRequest{Op: Delete, Key: "key"}, resp: EtcdResponse{Deleted: 1, Revision: 2}, failure: true},
+				{req: EtcdRequest{Op: Delete, Key: "key"}, resp: EtcdResponse{Deleted: 0, Revision: 1}},
 			},
 		},
 		{
@@ -287,8 +293,10 @@ func TestModel(t *testing.T) {
 		{
 			name: "Txn can expect on empty key",
 			operations: []testOperation{
-				{req: EtcdRequest{Op: Get, Key: "key"}, resp: EtcdResponse{Revision: 1}},
-				{req: EtcdRequest{Op: Txn, Key: "key", TxnExpectData: "", TxnNewData: "2"}, resp: EtcdResponse{Revision: 2, TxnSucceeded: true}},
+				{req: EtcdRequest{Op: Get, Key: "key1"}, resp: EtcdResponse{Revision: 1}},
+				{req: EtcdRequest{Op: Txn, Key: "key1", TxnExpectData: "", TxnNewData: "2"}, resp: EtcdResponse{Revision: 2, TxnSucceeded: true}},
+				{req: EtcdRequest{Op: Txn, Key: "key2", TxnExpectData: "", TxnNewData: "3"}, resp: EtcdResponse{Revision: 3, TxnSucceeded: true}},
+				{req: EtcdRequest{Op: Txn, Key: "key3", TxnExpectData: "4", TxnNewData: "4"}, resp: EtcdResponse{Revision: 4}, failure: true},
 			},
 		},
 		{

tests/linearizability/traffic.go

@@ -25,16 +25,16 @@ import (
 )
 
 var (
-	DefaultTraffic Traffic = readWriteSingleKey{key: "key", writes: []opChance{{operation: Put, chance: 90}, {operation: Delete, chance: 5}, {operation: Txn, chance: 5}}}
+	DefaultTraffic Traffic = readWriteSingleKey{keyCount: 4, writes: []opChance{{operation: Put, chance: 60}, {operation: Delete, chance: 20}, {operation: Txn, chance: 20}}}
 )
 
 type Traffic interface {
 	Run(ctx context.Context, c *recordingClient, limiter *rate.Limiter, ids idProvider)
 }
 
 type readWriteSingleKey struct {
-	key    string
-	writes []opChance
+	keyCount int
+	writes   []opChance
 }
 
 type opChance struct {
@@ -50,41 +50,42 @@ func (t readWriteSingleKey) Run(ctx context.Context, c *recordingClient, limiter
 			return
 		default:
 		}
+		key := fmt.Sprintf("%d", rand.Int()%t.keyCount)
 		// Execute one read per one write to avoid operation history include too many failed writes when etcd is down.
-		resp, err := t.Read(ctx, c, limiter)
+		resp, err := t.Read(ctx, c, limiter, key)
 		if err != nil {
 			continue
 		}
 		// Provide each write with unique id to make it easier to validate operation history.
-		t.Write(ctx, c, limiter, ids.RequestId(), resp)
+		t.Write(ctx, c, limiter, key, fmt.Sprintf("%d", ids.RequestId()), resp)
 	}
 }
 
-func (t readWriteSingleKey) Read(ctx context.Context, c *recordingClient, limiter *rate.Limiter) ([]*mvccpb.KeyValue, error) {
+func (t readWriteSingleKey) Read(ctx context.Context, c *recordingClient, limiter *rate.Limiter, key string) ([]*mvccpb.KeyValue, error) {
 	getCtx, cancel := context.WithTimeout(ctx, 20*time.Millisecond)
-	resp, err := c.Get(getCtx, t.key)
+	resp, err := c.Get(getCtx, key)
 	cancel()
 	if err == nil {
 		limiter.Wait(ctx)
 	}
 	return resp, err
 }
 
-func (t readWriteSingleKey) Write(ctx context.Context, c *recordingClient, limiter *rate.Limiter, id int, kvs []*mvccpb.KeyValue) error {
+func (t readWriteSingleKey) Write(ctx context.Context, c *recordingClient, limiter *rate.Limiter, key string, newValue string, lastValues []*mvccpb.KeyValue) error {
 	putCtx, cancel := context.WithTimeout(ctx, 20*time.Millisecond)
 
 	var err error
 	switch t.pickWriteOperation() {
 	case Put:
-		err = c.Put(putCtx, t.key, fmt.Sprintf("%d", id))
+		err = c.Put(putCtx, key, newValue)
 	case Delete:
-		err = c.Delete(putCtx, t.key)
+		err = c.Delete(putCtx, key)
 	case Txn:
-		var value string
-		if len(kvs) != 0 {
-			value = string(kvs[0].Value)
+		var expectValue string
+		if len(lastValues) != 0 {
+			expectValue = string(lastValues[0].Value)
 		}
-		err = c.Txn(putCtx, t.key, value, fmt.Sprintf("%d", id))
+		err = c.Txn(putCtx, key, expectValue, newValue)
 	default:
 		panic("invalid operation")
 	}