refactor: Update median tracking for historacle pricing

proto/umee/oracle/v1/genesis.proto

@@ -23,8 +23,8 @@ message GenesisState {
   repeated AggregateExchangeRateVote aggregate_exchange_rate_votes = 6
       [(gogoproto.nullable) = false];
   repeated HistoricPrice               historic_prices  = 8 [(gogoproto.nullable) = false];
-  repeated ExchangeRateTuple medians          = 7 [(gogoproto.nullable) = false];
-  repeated ExchangeRateTuple medianDeviations = 9 [(gogoproto.nullable) = false];
+  repeated HistoricPrice medians          = 7 [(gogoproto.nullable) = false];
+  repeated HistoricPrice medianDeviations = 9 [(gogoproto.nullable) = false];
 }
 
 // FeederDelegation is the address for where oracle feeder authority are

proto/umee/oracle/v1/oracle.proto

@@ -41,23 +41,19 @@ message Params {
     (gogoproto.customtype) = "github.com/cosmos/cosmos-sdk/types.Dec",
     (gogoproto.nullable)   = false
   ];
-  // Stamp Period represents the amount of blocks the historacle module
-  // waits before recording a set of prices from the oracle.
-  uint64 stamp_period = 9;
-  // Prune Period represents the maximum amount of blocks which we want
-  // to keep a record of our set of exchange rates.
-  uint64 prune_period = 10;
-  // Median Period represents the amount blocks we will wait between
-  // calculating the median and standard deviation of the median of
-  // historic prices in the last Prune Period.
-  uint64 median_period = 11;
-    // Historic Asset List is a list of assets which will use the historic
-  // price stamping protection methodology (mainly manipulatable assets).
-  // Any assets not on this list will not be stamped.
-  repeated Denom historic_accept_list = 12 [
-    (gogoproto.castrepeated) = "DenomList",
-    (gogoproto.nullable)     = false
-  ];
+  // Historic Stamp Period represents the amount of blocks the oracle
+  // module waits before recording a new historic price.
+  uint64 historic_stamp_period = 9;
+  // Median Stamp Period represents the amount blocks the oracle module
+  // waits between calculating and stamping a new median and standard
+  // deviation of that median.
+  uint64 median_stamp_period = 10;
+  // Maximum Price Stamps represents the maximum amount of historic prices
+  // the oracle module will store before pruning via FIFO.
+  uint64 maximum_price_stamps = 11;
+  // Maximum Median Stamps represents the maximum amount of medians the
+  // oracle module will store before pruning via FIFO.
+  uint64 maximum_median_stamps = 12;
 }
 
 // Denom - the object to hold configurations of each denom

util/bytes.go

@@ -1,5 +1,9 @@
 package util
 
+import (
+	"encoding/binary"
+)
+
 // ConcatBytes creates a new slice by merging list of bytes and leaving empty amount of margin
 // bytes at the end
 func ConcatBytes(margin int, bzs ...[]byte) []byte {
@@ -15,3 +19,9 @@ func ConcatBytes(margin int, bzs ...[]byte) []byte {
 	}
 	return out
 }
+
+func UintWithNullPrefix(n uint64) []byte {
+	bz := make([]byte, 9)
+	binary.LittleEndian.PutUint64(bz[1:], n)
+	return bz
+}

util/bytes_test.go

@@ -1,7 +1,12 @@
 package util
 
-import "testing"
-import "github.com/stretchr/testify/require"
+import (
+	"encoding/binary"
+	"math"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
 
 func TestMergeBytes(t *testing.T) {
 	require := require.New(t)
@@ -22,3 +27,13 @@ func TestMergeBytes(t *testing.T) {
 		require.Equal(tc.out, ConcatBytes(tc.inMargin, tc.in...), i)
 	}
 }
+
+func TestUintWithNullPrefix(t *testing.T) {
+	expected := []byte{0}
+	num := make([]byte, 8)
+	binary.LittleEndian.PutUint64(num, math.MaxUint64)
+	expected = append(expected, num...)
+
+	out := UintWithNullPrefix(math.MaxUint64)
+	require.Equal(t, expected, out)
+}

util/pricestats.go

@@ -0,0 +1,78 @@
+package util
+
+import (
+	"sort"
+
+	sdk "github.com/cosmos/cosmos-sdk/types"
+)
+
+// Median returns the median of a list of prices.
+func Median(prices []sdk.Dec) sdk.Dec {
+	lenPrices := len(prices)
+	if lenPrices == 0 {
+		return sdk.ZeroDec()
+	}
+
+	sort.Slice(prices, func(i, j int) bool {
+		return prices[i].BigInt().
+			Cmp(prices[j].BigInt()) < 0
+	})
+
+	if lenPrices%2 == 0 {
+		return prices[lenPrices/2-1].
+			Add(prices[lenPrices/2]).
+			QuoInt64(2)
+	}
+	return prices[lenPrices/2]
+}
+
+// MedianDeviation returns the standard deviation around the
+// median of a list of prices.
+// MedianDeviation = ∑((price - median)^2 / len(prices))
+func MedianDeviation(median sdk.Dec, prices []sdk.Dec) sdk.Dec {
+	lenPrices := len(prices)
+	medianDeviation := sdk.ZeroDec()
+
+	for _, price := range prices {
+		medianDeviation = medianDeviation.Add(price.
+			Sub(median).Abs().Power(2).
+			QuoInt64(int64(lenPrices)))
+	}
+
+	return medianDeviation
+}
+
+// Average returns the average value of a list of prices
+func Average(prices []sdk.Dec) sdk.Dec {
+	lenPrices := len(prices)
+	sumPrices := sdk.ZeroDec()
+	if lenPrices == 0 {
+		return sdk.ZeroDec()
+	}
+
+	for _, price := range prices {
+		sumPrices = sumPrices.Add(price)
+	}
+
+	return sumPrices.QuoInt64(int64(lenPrices))
+}
+
+// Max returns the max value of a list of prices
+func Max(prices []sdk.Dec) sdk.Dec {
+	sort.Slice(prices, func(i, j int) bool {
+		return prices[i].BigInt().
+			Cmp(prices[j].BigInt()) < 0
+	})
+
+	return prices[len(prices)-1]
+}
+
+// Min returns the min value of a list of prices
+func Min(prices []sdk.Dec) sdk.Dec {
+	sort.Slice(prices, func(i, j int) bool {
+		return prices[i].BigInt().
+			Cmp(prices[j].BigInt()) < 0
+	})
+
+	return prices[0]
+}

util/pricestats_test.go

@@ -0,0 +1,74 @@
+package util
+
+import (
+	"testing"
+
+	sdk "github.com/cosmos/cosmos-sdk/types"
+	"github.com/stretchr/testify/require"
+)
+
+func TestMedian(t *testing.T) {
+	require := require.New(t)
+	prices := []sdk.Dec{
+		sdk.MustNewDecFromStr("1.1"),
+		sdk.MustNewDecFromStr("1.05"),
+		sdk.MustNewDecFromStr("1.15"),
+		sdk.MustNewDecFromStr("1.2"),
+	}
+
+	median := Median(prices)
+	require.Equal(sdk.MustNewDecFromStr("1.125"), median)
+}
+
+func TestMedianDeviation(t *testing.T) {
+	require := require.New(t)
+	prices := []sdk.Dec{
+		sdk.MustNewDecFromStr("1.1"),
+		sdk.MustNewDecFromStr("1.05"),
+		sdk.MustNewDecFromStr("1.15"),
+		sdk.MustNewDecFromStr("1.2"),
+	}
+	median := sdk.MustNewDecFromStr("1.125")
+
+	medianDeviation := MedianDeviation(median, prices)
+	require.Equal(sdk.MustNewDecFromStr("0.003125"), medianDeviation)
+}
+
+func TestAverage(t *testing.T) {
+	require := require.New(t)
+	prices := []sdk.Dec{
+		sdk.MustNewDecFromStr("1.1"),
+		sdk.MustNewDecFromStr("1.05"),
+		sdk.MustNewDecFromStr("1.15"),
+		sdk.MustNewDecFromStr("1.2"),
+	}
+
+	average := Average(prices)
+	require.Equal(sdk.MustNewDecFromStr("1.125"), average)
+}
+
+func TestMin(t *testing.T) {
+	require := require.New(t)
+	prices := []sdk.Dec{
+		sdk.MustNewDecFromStr("1.1"),
+		sdk.MustNewDecFromStr("1.05"),
+		sdk.MustNewDecFromStr("1.15"),
+		sdk.MustNewDecFromStr("1.2"),
+	}
+
+	min := Min(prices)
+	require.Equal(sdk.MustNewDecFromStr("1.05"), min)
+}
+
+func TestMax(t *testing.T) {
+	require := require.New(t)
+	prices := []sdk.Dec{
+		sdk.MustNewDecFromStr("1.1"),
+		sdk.MustNewDecFromStr("1.05"),
+		sdk.MustNewDecFromStr("1.15"),
+		sdk.MustNewDecFromStr("1.2"),
+	}
+
+	max := Max(prices)
+	require.Equal(sdk.MustNewDecFromStr("1.2"), max)
+}

x/oracle/abci.go

@@ -1,6 +1,7 @@
 package oracle
 
 import (
+	"strings"
 	"time"
 
 	"github.com/cosmos/cosmos-sdk/telemetry"
@@ -41,11 +42,6 @@ func EndBlocker(ctx sdk.Context, k keeper.Keeper, experimental bool) error {
 
 		k.ClearExchangeRates(ctx)
 
-		if isPeriodLastBlock(ctx, params.MedianPeriod) && experimental {
-			k.ClearMedians(ctx)
-			k.ClearMedianDeviations(ctx)
-		}
-
 		// NOTE: it filters out inactive or jailed validators
 		ballotDenomSlice := k.OrganizeBallotByDenom(ctx, validatorClaimMap)
 
@@ -63,14 +59,14 @@ func EndBlocker(ctx sdk.Context, k keeper.Keeper, experimental bool) error {
 			}
 
 			if experimental {
-				// Stamp rate every stamp period if asset is set to have historic stats tracked
-				if isPeriodLastBlock(ctx, params.StampPeriod) && params.HistoricAcceptList.Contains(ballotDenom.Denom) {
-					k.AddHistoricPrice(ctx, ballotDenom.Denom, exchangeRate)
+				// Stamp historic price if historic stamp period has passed
+				if isPeriodLastBlock(ctx, params.HistoricStampPeriod) {
+					k.AddHistoricPrice(ctx, strings.ToUpper(ballotDenom.Denom), exchangeRate)
 				}
 
-				// Set median price every median period if asset is set to have historic stats tracked
-				if isPeriodLastBlock(ctx, params.MedianPeriod) && params.HistoricAcceptList.Contains(ballotDenom.Denom) {
-					k.CalcAndSetMedian(ctx, ballotDenom.Denom)
+				// Calculate and stamp median/median deviation if median stamp period has passed
+				if isPeriodLastBlock(ctx, params.MedianStampPeriod) {
+					k.CalcAndSetMedian(ctx, strings.ToUpper(ballotDenom.Denom))
 				}
 			}
 		}
@@ -108,11 +104,15 @@ func EndBlocker(ctx sdk.Context, k keeper.Keeper, experimental bool) error {
 		k.SlashAndResetMissCounters(ctx)
 	}
 
-	// Prune historic prices every prune period
-	if isPeriodLastBlock(ctx, params.PrunePeriod) && experimental {
-		pruneBlock := uint64(ctx.BlockHeight()) - params.PrunePeriod
-		for _, v := range params.HistoricAcceptList {
-			k.DeleteHistoricPrice(ctx, v.String(), pruneBlock)
+	// Prune historic prices and medians outside pruning period determined by
+	// the stamp period multiplied by the max stamps.
+	if experimental {
+		pruneHistoricBlock := uint64(ctx.BlockHeight()) - (params.HistoricStampPeriod * params.MaximumPriceStamps)
+		pruneMedianBlock := uint64(ctx.BlockHeight()) - (params.MedianStampPeriod * params.MaximumMedianStamps)
+		for _, v := range params.AcceptList {
+			k.DeleteHistoricPrice(ctx, strings.ToUpper(v.SymbolDenom), pruneHistoricBlock)
+			k.DeleteMedian(ctx, strings.ToUpper(v.SymbolDenom), pruneMedianBlock)
+			k.DeleteMedianDeviation(ctx, strings.ToUpper(v.SymbolDenom), pruneMedianBlock)
 		}
 	}
 

x/oracle/abci_test.go

@@ -42,7 +42,6 @@ func (s *IntegrationTestSuite) SetupTest() {
 	app := umeeapp.Setup(s.T(), isCheckTx, 1)
 	ctx := app.BaseApp.NewContext(isCheckTx, tmproto.Header{
 		ChainID: fmt.Sprintf("test-chain-%s", tmrand.Str(4)),
-		Height:  int64(types.DefaultMedianPeriod) - 1,
 	})
 
 	oracle.InitGenesis(ctx, app.OracleKeeper, *types.DefaultGenesisState())
@@ -90,18 +89,32 @@ func (s *IntegrationTestSuite) TestEndblockerExperimentalFlag() {
 	// add historic price and calcSet median stats
 	app.OracleKeeper.AddHistoricPrice(s.ctx, displayDenom, sdk.MustNewDecFromStr("1.0"))
 	app.OracleKeeper.CalcAndSetMedian(s.ctx, displayDenom)
+	medianPruneBlock := ctx.BlockHeight() + int64(types.DefaultMaximumMedianStamps*types.DefaultMedianStampPeriod)
+	ctx = ctx.WithBlockHeight(medianPruneBlock)
 
-	// with experimental flag off median stats don't get cleared
+	// with experimental flag off median doesn't get deleted
 	oracle.EndBlocker(ctx, app.OracleKeeper, false)
-	median, err := app.OracleKeeper.GetMedian(s.ctx, displayDenom)
-	s.Require().NoError(err)
-	s.Require().Equal(sdk.MustNewDecFromStr("1.0"), median)
-
-	// with experimental flag on median stats get cleared
+	medians := []types.HistoricPrice{}
+	app.OracleKeeper.IterateAllMedianPrices(
+		ctx,
+		func(median types.HistoricPrice) bool {
+			medians = append(medians, median)
+			return false
+		},
+	)
+	s.Require().Equal(1, len(medians))
+
+	// with experimental flag on median gets deleted
 	oracle.EndBlocker(ctx, app.OracleKeeper, true)
-	median, err = app.OracleKeeper.GetMedian(s.ctx, displayDenom)
-	s.Require().Error(err)
-	s.Require().Equal(sdk.ZeroDec(), median)
+	experimentalMedians := []types.HistoricPrice{}
+	app.OracleKeeper.IterateAllMedianPrices(
+		ctx,
+		func(median types.HistoricPrice) bool {
+			medians = append(experimentalMedians, median)
+			return false
+		},
+	)
+	s.Require().Equal(0, len(experimentalMedians))
 }
 
 func TestOracleTestSuite(t *testing.T) {