From 9b65960de085e6aa8006badfb8b011d5aa6e4ed2 Mon Sep 17 00:00:00 2001 From: zemyblue Date: Thu, 18 Jun 2020 19:45:29 +0900 Subject: [PATCH 1/4] Fix the floating-point problem of sampling --- libs/rand/sampling.go | 30 +++++++++++++----------------- libs/rand/sampling_test.go | 21 ++++++++++++++++----- 2 files changed, 29 insertions(+), 22 deletions(-) diff --git a/libs/rand/sampling.go b/libs/rand/sampling.go index 2f4000259..9deece547 100644 --- a/libs/rand/sampling.go +++ b/libs/rand/sampling.go @@ -2,7 +2,6 @@ package rand import ( "fmt" - "math" "math/big" s "sort" ) @@ -74,6 +73,12 @@ func moveWinnerToLast(candidates []Candidate, winner int) { const uint64Mask = uint64(0x7FFFFFFFFFFFFFFF) +// precisionForSelection is a value to be corrected to increase precision when calculating voting power as an integer. +const precisionForSelection = uint64(1000) + +// precisionCorrectionForSelection is a value corrected for accuracy of voting power +const precisionCorrectionForSelection = uint64(1000) + var divider *big.Int func init() { @@ -92,9 +97,6 @@ func randomThreshold(seed *uint64, total uint64) uint64 { return a.Uint64() } -// `RandomSamplingWithoutReplacement` elects winners among candidates without replacement -// so it updates rewards of winners. This function continues to elect winners until the both of two -// conditions(minSamplingCount, minPriorityPercent) are met. func RandomSamplingWithoutReplacement( seed uint64, candidates []Candidate, minSamplingCount int) (winners []Candidate) { @@ -135,22 +137,16 @@ func RandomSamplingWithoutReplacement( winnerNum, minSamplingCount, winnersPriority, totalPriority, threshold)) } } - compensationProportions := make([]float64, winnerNum) - for i := winnerNum - 2; i >= 0; i-- { // last winner doesn't get compensation reward - compensationProportions[i] = compensationProportions[i+1] + 1/float64(losersPriorities[i]) + correction := totalPriority * precisionForSelection + compensationProportions := make([]uint64, winnerNum) + for i := winnerNum - 2; i >= 0; i-- { + compensationProportions[i] = compensationProportions[i+1] + correction/losersPriorities[i] } winners = candidates[len(candidates)-winnerNum:] - winPoints := make([]float64, len(winners)) - totalWinPoint := float64(0) + winPoints := make([]uint64, len(winners)) for i, winner := range winners { - winPoints[i] = 1 + float64(winner.Priority())*compensationProportions[i] - totalWinPoint += winPoints[i] - } - for i, winner := range winners { - if winPoints[i] > math.MaxInt64 || winPoints[i] < 0 { - panic(fmt.Sprintf("winPoint is invalid: %f", winPoints[i])) - } - winner.SetWinPoint(int64(float64(totalPriority) * winPoints[i] / totalWinPoint)) + winPoints[i] = correction + winner.Priority()*compensationProportions[i] + winner.SetWinPoint(int64(winPoints[i] / (correction / precisionCorrectionForSelection))) } return winners } diff --git a/libs/rand/sampling_test.go b/libs/rand/sampling_test.go index 1783b5801..1f2cbd238 100644 --- a/libs/rand/sampling_test.go +++ b/libs/rand/sampling_test.go @@ -104,7 +104,7 @@ func TestRandomSamplingWithoutReplacement1Candidate(t *testing.T) { winners := RandomSamplingWithoutReplacement(0, candidates, 1) assert.True(t, len(winners) == 1) assert.True(t, candidates[0] == winners[0]) - assert.True(t, winners[0].(*Element).winPoint == 1000) + assert.True(t, uint64(winners[0].(*Element).winPoint) == precisionForSelection) resetWinPoint(candidates) winners2 := RandomSamplingWithoutReplacement(0, candidates, 0) @@ -176,11 +176,14 @@ func TestRandomSamplingWithoutReplacementIncludingZeroStakingPower(t *testing.T) assert.True(t, len(winners2) == 90) } -func accumulateAndResetReward(candidate []Candidate, acc []uint64) { +func accumulateAndResetReward(candidate []Candidate, acc []uint64) uint64 { + totalWinPoint := uint64(0) for i, c := range candidate { acc[i] += uint64(c.(*Element).winPoint) + totalWinPoint += uint64(c.(*Element).winPoint) c.(*Element).winPoint = 0 } + return totalWinPoint } func TestDivider(t *testing.T) { @@ -277,14 +280,22 @@ func TestRandomSamplingWithoutReplacementEquity(t *testing.T) { // good condition candidates := newCandidates(100, func(i int) uint64 { return 1000000 + rand.Uint64()&0xFFFFF }) + totalStaking := uint64(0) + for _, c := range candidates { + totalStaking += c.Priority() + } + accumulatedRewards := make([]uint64, 100) + totalAccumulateRewards := uint64(0) for i := 0; i < loopCount; i++ { RandomSamplingWithoutReplacement(uint64(i), candidates, 99) - accumulateAndResetReward(candidates, accumulatedRewards) + totalAccumulateRewards += accumulateAndResetReward(candidates, accumulatedRewards) } for i := 0; i < 99; i++ { - rewardPerStakingDiff := - math.Abs(float64(accumulatedRewards[i])/float64(candidates[i].Priority())/float64(loopCount) - 1) + rewardRate := float64(accumulatedRewards[i]) / float64(totalAccumulateRewards) + stakingRate := float64(candidates[i].Priority()) / float64(totalStaking) + rate := rewardRate / stakingRate + rewardPerStakingDiff := math.Abs(1 - rate) assert.True(t, rewardPerStakingDiff < 0.01) } From 6e4206a3f509fdd2421b197feabcb62eeb167ecc Mon Sep 17 00:00:00 2001 From: zemyblue Date: Thu, 25 Jun 2020 15:38:05 +0900 Subject: [PATCH 2/4] fix integer overflow problem of `winPoint`. --- libs/rand/sampling.go | 28 +++++++++++++++++----------- libs/rand/sampling_test.go | 16 ++++++++++++++++ 2 files changed, 33 insertions(+), 11 deletions(-) diff --git a/libs/rand/sampling.go b/libs/rand/sampling.go index 9deece547..4d7f85a0b 100644 --- a/libs/rand/sampling.go +++ b/libs/rand/sampling.go @@ -87,11 +87,8 @@ func init() { } func randomThreshold(seed *uint64, total uint64) uint64 { - if int64(total) < 0 { - panic(fmt.Sprintf("total priority is overflow: %d", total)) - } - totalBig := big.NewInt(int64(total)) - a := big.NewInt(int64(nextRandom(seed) & uint64Mask)) + totalBig := new(big.Int).SetUint64(total) + a := new(big.Int).SetUint64(nextRandom(seed) & uint64Mask) a.Mul(a, totalBig) a.Div(a, divider) return a.Uint64() @@ -137,20 +134,29 @@ func RandomSamplingWithoutReplacement( winnerNum, minSamplingCount, winnersPriority, totalPriority, threshold)) } } - correction := totalPriority * precisionForSelection - compensationProportions := make([]uint64, winnerNum) + correction := new(big.Int).SetUint64(totalPriority) + correction = correction.Mul(correction, new(big.Int).SetUint64(precisionForSelection)) + compensationProportions := make([]big.Int, winnerNum) for i := winnerNum - 2; i >= 0; i-- { - compensationProportions[i] = compensationProportions[i+1] + correction/losersPriorities[i] + additionalCompensation := new(big.Int).Div(correction, new(big.Int).SetUint64(losersPriorities[i])) + compensationProportions[i].Add(&compensationProportions[i+1], additionalCompensation) } winners = candidates[len(candidates)-winnerNum:] - winPoints := make([]uint64, len(winners)) + recalibration := new(big.Int).Div(correction, new(big.Int).SetUint64(precisionCorrectionForSelection)) for i, winner := range winners { - winPoints[i] = correction + winner.Priority()*compensationProportions[i] - winner.SetWinPoint(int64(winPoints[i] / (correction / precisionCorrectionForSelection))) + // winPoint = correction + winner.Priority() * compensationProportions[i] + winPoint := new(big.Int).SetUint64(winner.Priority()) + winPoint.Mul(winPoint, &compensationProportions[i]) + winPoint.Add(winPoint, correction) + + winner.SetWinPoint(winPoint.Div(winPoint, recalibration).Int64()) } return winners } +// sumTotalPriority calculate the sum of all candidate's priority(weight) +// and the sum should be less then or equal to MaxUint64 +// TODO We need to check the total weight doesn't over MaxUint64 in somewhere not here. func sumTotalPriority(candidates []Candidate) (sum uint64) { for _, candi := range candidates { sum += candi.Priority() diff --git a/libs/rand/sampling_test.go b/libs/rand/sampling_test.go index 1f2cbd238..084e86466 100644 --- a/libs/rand/sampling_test.go +++ b/libs/rand/sampling_test.go @@ -2,6 +2,7 @@ package rand import ( "fmt" + "github.com/tendermint/tendermint/types/time" "math" "math/rand" s "sort" @@ -176,6 +177,21 @@ func TestRandomSamplingWithoutReplacementIncludingZeroStakingPower(t *testing.T) assert.True(t, len(winners2) == 90) } +func TestRandomSamplingWithoutReplacementOverflow(t *testing.T) { + rand.Seed(time.Now().Unix()) + number := 100 + candidates := newCandidates(number, func(i int) uint64 { return math.MaxUint64 / uint64(number) }) + winners := RandomSamplingWithoutReplacement(rand.Uint64(), candidates, 64) + lastWinPoint := int64(math.MaxInt64) + for _, w := range winners { + element := w.(*Element) + assert.True(t, element.winPoint > 0) + assert.True(t, element.winPoint <= lastWinPoint) + lastWinPoint = element.winPoint + } + assert.Equal(t, lastWinPoint, int64(precisionForSelection)) +} + func accumulateAndResetReward(candidate []Candidate, acc []uint64) uint64 { totalWinPoint := uint64(0) for i, c := range candidate { From e1851021e22ad9793c76fc861556b643fd1efe0b Mon Sep 17 00:00:00 2001 From: zemyblue Date: Thu, 25 Jun 2020 16:15:06 +0900 Subject: [PATCH 3/4] fix lint warning --- libs/rand/sampling_test.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/libs/rand/sampling_test.go b/libs/rand/sampling_test.go index 084e86466..4ae34dfa9 100644 --- a/libs/rand/sampling_test.go +++ b/libs/rand/sampling_test.go @@ -2,13 +2,11 @@ package rand import ( "fmt" - "github.com/tendermint/tendermint/types/time" + "github.com/stretchr/testify/assert" "math" "math/rand" s "sort" "testing" - - "github.com/stretchr/testify/assert" ) type Element struct { @@ -178,7 +176,6 @@ func TestRandomSamplingWithoutReplacementIncludingZeroStakingPower(t *testing.T) } func TestRandomSamplingWithoutReplacementOverflow(t *testing.T) { - rand.Seed(time.Now().Unix()) number := 100 candidates := newCandidates(number, func(i int) uint64 { return math.MaxUint64 / uint64(number) }) winners := RandomSamplingWithoutReplacement(rand.Uint64(), candidates, 64) From ff9862ed19e5bc1910e0f422e3683854db2f8d26 Mon Sep 17 00:00:00 2001 From: zemyblue Date: Thu, 25 Jun 2020 16:22:41 +0900 Subject: [PATCH 4/4] fix lint warning --- libs/rand/sampling_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/libs/rand/sampling_test.go b/libs/rand/sampling_test.go index 4ae34dfa9..e7b261907 100644 --- a/libs/rand/sampling_test.go +++ b/libs/rand/sampling_test.go @@ -2,11 +2,12 @@ package rand import ( "fmt" - "github.com/stretchr/testify/assert" "math" "math/rand" s "sort" "testing" + + "github.com/stretchr/testify/assert" ) type Element struct {