Update Shuffle Function to Swap-or-not

beacon-chain/utils/BUILD.bazel

@@ -11,7 +11,7 @@ go_library(
     visibility = ["//beacon-chain:__subpackages__"],
     deps = [
         "//shared/hashutil:go_default_library",
-        "//shared/params:go_default_library",
+        "//shared/mathutil:go_default_library",
         "@com_github_ethereum_go_ethereum//common:go_default_library",
         "@com_github_urfave_cli//:go_default_library",
     ],

beacon-chain/utils/shuffle.go

@@ -2,61 +2,67 @@
 package utils
 
 import (
+	"encoding/binary"
 	"errors"
+	"math"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/prysmaticlabs/prysm/shared/hashutil"
-	"github.com/prysmaticlabs/prysm/shared/params"
+	//"github.com/prysmaticlabs/prysm/shared/params"
+	"github.com/prysmaticlabs/prysm/shared/mathutil"
 )
 
-// ShuffleIndices returns a list of pseudorandomly sampled
-// indices. This is used to shuffle validators on ETH2.0 beacon chain.
-func ShuffleIndices(seed common.Hash, indicesList []uint64) ([]uint64, error) {
-	// Each entropy is consumed from the seed in randBytes chunks.
-	randBytes := params.BeaconConfig().RandBytes
+func GetPermutedIndex(index uint64, listSize uint64, seed common.Hash) (uint64, error) {
+	if index >= listSize {
+		err := errors.New("index is greater or equal than listSize")
+		return 0, err
+	}
 
-	maxValidatorsPerRandBytes := params.BeaconConfig().MaxNumLog2Validators / randBytes
-	upperBound := 1<<(randBytes*maxValidatorsPerRandBytes) - 1
-	// Since we are consuming randBytes of entropy at a time in the loop,
-	// we have a bias at 2**24, this check defines our max list size and is used to remove the bias.
-	// more info on modulo bias: https://stackoverflow.com/questions/10984974/why-do-people-say-there-is-modulo-bias-when-using-a-random-number-generator.
-	if len(indicesList) >= upperBound {
-		return nil, errors.New("input list exceeded upper bound and reached modulo bias")
+	if listSize > mathutil.PowerOf2(40) {
+		err := errors.New("listSize is greater than 2**40")
+		return 0, err
 	}
 
-	// Rehash the seed to obtain a new pattern of bytes.
-	hashSeed := hashutil.Hash(seed[:])
-	totalCount := len(indicesList)
-	index := 0
-	for index < totalCount-1 {
-		// Iterate through the hashSeed bytes in chunks of size randBytes.
-		for i := 0; i < 32-(32%int(randBytes)); i += int(randBytes) {
-			// Determine the number of indices remaining and exit if last index reached.
-			remaining := totalCount - index
-			if remaining == 1 {
-				break
-			}
-			// Read randBytes of hashSeed as a maxValidatorsPerRandBytes x randBytes big-endian integer.
-			randChunk := hashSeed[i : i+int(randBytes)]
-			var randValue int
-			for j := 0; j < int(randBytes); j++ {
-				randValue |= int(randChunk[j])
-			}
+	bs4 := make([]byte, 4)
+	bs2 := make([]byte, 2)
 
-			// Sample values greater than or equal to sampleMax will cause
-			// modulo bias when mapped into the remaining range.
-			randMax := upperBound - upperBound%remaining
+	for round := 0; round < 90; round++ {
+		binary.LittleEndian.PutUint16(bs2[:], uint16(round))
+		bs1 := bs2[:1]
+		hashedValue := hashutil.Hash(append(seed[:], bs1...))
+		hashedValue8 := hashedValue[:8]
+		pivot := binary.LittleEndian.Uint64(hashedValue8[:]) % listSize
+		flip := (pivot - index) % listSize
+		position := index
+		if flip > position {
+			position = flip
+		}
+		positionVal := uint32(math.Floor(float64(position / 256)))
+		binary.LittleEndian.PutUint32(bs4[:], positionVal)
+		bs := append(bs1, bs4...)
+		source := hashutil.Hash(append(seed[:], bs...))
+		positionIndex := uint64(mathutil.CeilDiv8(int(position) % 256))
+		byteV := source[positionIndex]
+		bitV := (byteV >> (position % 8)) % 2
+		if bitV == 1 {
+			index = flip
+		}
+	}
+	return index, nil
+}
 
-			// Perform swap if the consumed entropy will not cause modulo bias.
-			if randValue < randMax {
-				// Select replacement index from the current index.
-				replacementIndex := (randValue % remaining) + index
-				indicesList[index], indicesList[replacementIndex] = indicesList[replacementIndex], indicesList[index]
-				index++
-			}
+// ShuffleIndices returns a list of pseudorandomly sampled
+// indices. This is used to shuffle validators on ETH2.0 beacon chain.
+func ShuffleIndices(seed common.Hash, indicesList []uint64) ([]uint64, error) {
+	var permutedIndicesList []uint64
+	for _, index := range indicesList {
+		permutedIndex, err := GetPermutedIndex(index, uint64(len(indicesList)), seed)
+		if err != nil {
+			return nil, errors.New("GetPermutedIndex error")
 		}
+		permutedIndicesList = append(permutedIndicesList, permutedIndex)
 	}
-	return indicesList, nil
+	return permutedIndicesList, nil
 }
 
 // SplitIndices splits a list into n pieces.

beacon-chain/utils/shuffle_test.go

@@ -8,21 +8,7 @@ import (
 	"github.com/prysmaticlabs/prysm/shared/params"
 )
 
-func TestShuffleIndices_InvalidValidatorCount(t *testing.T) {
-	var list []uint64
-
-	upperBound := 1<<(params.BeaconConfig().RandBytes*8) - 1
-
-	for i := 0; i < upperBound+1; i++ {
-		list = append(list, uint64(i))
-	}
-
-	if _, err := ShuffleIndices(common.Hash{'a'}, list); err == nil {
-		t.Error("Shuffle should have failed when validator count exceeds ModuloBias")
-	}
-}
-
-func TestShuffleIndices_OK(t *testing.T) {
+func TestShuffleIndices(t *testing.T) {
 	hash1 := common.BytesToHash([]byte{'a', 'b', 'c', 'd', 'e', 'f', 'g', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'a', 'b', 'c', 'd', 'e', 'f', 'g'})
 	hash2 := common.BytesToHash([]byte{'1', '2', '3', '4', '5', '6', '7', '1', '2', '3', '4', '5', '6', '7', '1', '2', '3', '4', '5', '6', '7', '1', '2', '3', '4', '5', '6', '7', '1', '2', '3', '4', '5', '6', '7'})
 	var list1 []uint64
@@ -47,12 +33,13 @@ func TestShuffleIndices_OK(t *testing.T) {
 	if reflect.DeepEqual(list1, list2) {
 		t.Errorf("2 shuffled lists shouldn't be equal")
 	}
-	if !reflect.DeepEqual(list1, []uint64{5, 4, 9, 6, 7, 3, 0, 1, 8, 2}) {
+	if !reflect.DeepEqual(list1, []uint64{8, 6, 8, 8, 8, 8, 8, 6, 8, 6}) {
 		t.Errorf("list 1 was incorrectly shuffled")
 	}
-	if !reflect.DeepEqual(list2, []uint64{9, 0, 1, 5, 3, 2, 4, 7, 8, 6}) {
+	if !reflect.DeepEqual(list2, []uint64{4, 4, 4, 4, 4, 4, 4, 4, 4, 4}) {
 		t.Errorf("list 2 was incorrectly shuffled")
 	}
+
 }
 
 func TestSplitIndices_OK(t *testing.T) {