refactor: Update median tracking for historacle pricing

proto/umee/oracle/v1/genesis.proto

@@ -23,8 +23,8 @@ message GenesisState {
   repeated AggregateExchangeRateVote aggregate_exchange_rate_votes = 6
       [(gogoproto.nullable) = false];
   repeated HistoricPrice               historic_prices  = 8 [(gogoproto.nullable) = false];
-  repeated ExchangeRateTuple medians          = 7 [(gogoproto.nullable) = false];
-  repeated ExchangeRateTuple medianDeviations = 9 [(gogoproto.nullable) = false];
+  repeated HistoricPrice medians          = 7 [(gogoproto.nullable) = false];
+  repeated HistoricPrice medianDeviations = 9 [(gogoproto.nullable) = false];
 }
 
 // FeederDelegation is the address for where oracle feeder authority are

proto/umee/oracle/v1/oracle.proto

@@ -41,23 +41,19 @@ message Params {
     (gogoproto.customtype) = "github.com/cosmos/cosmos-sdk/types.Dec",
     (gogoproto.nullable)   = false
   ];
-  // Stamp Period represents the amount of blocks the historacle module
-  // waits before recording a set of prices from the oracle.
-  uint64 stamp_period = 9;
-  // Prune Period represents the maximum amount of blocks which we want
-  // to keep a record of our set of exchange rates.
-  uint64 prune_period = 10;
-  // Median Period represents the amount blocks we will wait between
-  // calculating the median and standard deviation of the median of
-  // historic prices in the last Prune Period.
-  uint64 median_period = 11;
-    // Historic Asset List is a list of assets which will use the historic
-  // price stamping protection methodology (mainly manipulatable assets).
-  // Any assets not on this list will not be stamped.
-  repeated Denom historic_accept_list = 12 [
-    (gogoproto.castrepeated) = "DenomList",
-    (gogoproto.nullable)     = false
-  ];
+  // Historic Stamp Period represents the amount of blocks the oracle
+  // module waits before recording a new historic price.
+  uint64 historic_stamp_period = 9;
+  // Median Stamp Period represents the amount blocks the oracle module
+  // waits between calculating and stamping a new median and standard
+  // deviation of that median.
+  uint64 median_stamp_period = 10;
+  // Maximum Price Stamps represents the maximum amount of historic prices
+  // the oracle module will store before pruning via FIFO.
+  uint64 maximum_price_stamps = 11;
+  // Maximum Median Stamps represents the maximum amount of medians the
+  // oracle module will store before pruning via FIFO.
+  uint64 maximum_median_stamps = 12;
 }
 
 // Denom - the object to hold configurations of each denom

util/bytes.go

@@ -1,5 +1,9 @@
 package util
 
+import (
+	"encoding/binary"
+)
+
 // ConcatBytes creates a new slice by merging list of bytes and leaving empty amount of margin
 // bytes at the end
 func ConcatBytes(margin int, bzs ...[]byte) []byte {
@@ -15,3 +19,9 @@ func ConcatBytes(margin int, bzs ...[]byte) []byte {
 	}
 	return out
 }
+
+func UintWithNullPrefix(n uint64) []byte {
+	bz := make([]byte, 9)
+	binary.LittleEndian.PutUint64(bz[1:], n)
+	return bz
+}

util/bytes_test.go

@@ -1,7 +1,12 @@
 package util
 
-import "testing"
-import "github.com/stretchr/testify/require"
+import (
+	"encoding/binary"
+	"math"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
 
 func TestMergeBytes(t *testing.T) {
 	require := require.New(t)
@@ -22,3 +27,13 @@ func TestMergeBytes(t *testing.T) {
 		require.Equal(tc.out, ConcatBytes(tc.inMargin, tc.in...), i)
 	}
 }
+
+func TestUintWithNullPrefix(t *testing.T) {
+	expected := []byte{0}
+	num := make([]byte, 8)
+	binary.LittleEndian.PutUint64(num, math.MaxUint64)
+	expected = append(expected, num...)
+
+	out := UintWithNullPrefix(math.MaxUint64)
+	require.Equal(t, expected, out)
+}

util/pricestats.go

@@ -0,0 +1,100 @@
+package util
+
+import (
+	"fmt"
+	"sort"
+
+	sdk "github.com/cosmos/cosmos-sdk/types"
+)
+
+var (
+	ErrEmptyList = fmt.Errorf("empty price list passed in")
+)
+
+// Median returns the median of a list of prices. Returns error
+// if prices is empty list.
+func Median(prices []sdk.Dec) (sdk.Dec, error) {
+	lenPrices := len(prices)
+	if lenPrices == 0 {
+		return sdk.ZeroDec(), ErrEmptyList
+	}
+
+	sort.Slice(prices, func(i, j int) bool {
+		return prices[i].BigInt().
+			Cmp(prices[j].BigInt()) < 0
+	})
+
+	if lenPrices%2 == 0 {
+		return prices[lenPrices/2-1].
+			Add(prices[lenPrices/2]).
+			QuoInt64(2), nil
+	}
+	return prices[lenPrices/2], nil
+}
+
+// MedianDeviation returns the standard deviation around the
+// median of a list of prices. Returns error if prices is empty list.
+// MedianDeviation = ∑((price - median)^2 / len(prices))
+func MedianDeviation(median sdk.Dec, prices []sdk.Dec) (sdk.Dec, error) {
+	medianDeviation := sdk.ZeroDec()
+	lenPrices := len(prices)
+	if lenPrices == 0 {
+		return medianDeviation, ErrEmptyList
+	}
+
+	for _, price := range prices {
+		medianDeviation = medianDeviation.Add(price.
+			Sub(median).Abs().Power(2).
+			QuoInt64(int64(lenPrices)))
+	}
+
+	return medianDeviation, nil
+}
+
+// Average returns the average value of a list of prices. Returns error
+// if prices is empty list.
+func Average(prices []sdk.Dec) (sdk.Dec, error) {
+	lenPrices := len(prices)
+	if lenPrices == 0 {
+		return sdk.ZeroDec(), ErrEmptyList
+	}
+
+	sumPrices := sdk.ZeroDec()
+	for _, price := range prices {
+		sumPrices = sumPrices.Add(price)
+	}
+
+	return sumPrices.QuoInt64(int64(lenPrices)), nil
+}
+
+// Max returns the max value of a list of prices. Returns error
+// if prices is empty list.
+func Max(prices []sdk.Dec) (sdk.Dec, error) {
+	lenPrices := len(prices)
+	if lenPrices == 0 {
+		return sdk.ZeroDec(), ErrEmptyList
+	}
+
+	sort.Slice(prices, func(i, j int) bool {
+		return prices[i].BigInt().
+			Cmp(prices[j].BigInt()) < 0
+	})
+
+	return prices[len(prices)-1], nil
+}
+
+// Min returns the min value of a list of prices. Returns error
+// if prices is empty list.
+func Min(prices []sdk.Dec) (sdk.Dec, error) {
+	lenPrices := len(prices)
+	if lenPrices == 0 {
+		return sdk.ZeroDec(), ErrEmptyList
+	}
+
+	sort.Slice(prices, func(i, j int) bool {
+		return prices[i].BigInt().
+			Cmp(prices[j].BigInt()) < 0
+	})
+
+	return prices[0], nil
+}

util/pricestats_test.go

@@ -0,0 +1,99 @@
+package util
+
+import (
+	"testing"
+
+	sdk "github.com/cosmos/cosmos-sdk/types"
+	"github.com/stretchr/testify/require"
+)
+
+func TestMedian(t *testing.T) {
+	require := require.New(t)
+	prices := []sdk.Dec{
+		sdk.MustNewDecFromStr("1.1"),
+		sdk.MustNewDecFromStr("1.05"),
+		sdk.MustNewDecFromStr("1.15"),
+		sdk.MustNewDecFromStr("1.2"),
+	}
+
+	median, err := Median(prices)
+	require.NoError(err)
+	require.Equal(sdk.MustNewDecFromStr("1.125"), median)
+
+	// test empty prices list
+	median, err = Median([]sdk.Dec{})
+	require.ErrorIs(err, ErrEmptyList)
+}
+
+func TestMedianDeviation(t *testing.T) {
+	require := require.New(t)
+	prices := []sdk.Dec{
+		sdk.MustNewDecFromStr("1.1"),
+		sdk.MustNewDecFromStr("1.05"),
+		sdk.MustNewDecFromStr("1.15"),
+		sdk.MustNewDecFromStr("1.2"),
+	}
+	median := sdk.MustNewDecFromStr("1.125")
+
+	medianDeviation, err := MedianDeviation(median, prices)
+	require.NoError(err)
+	require.Equal(sdk.MustNewDecFromStr("0.003125"), medianDeviation)
+
+	// test empty prices list
+	medianDeviation, err = MedianDeviation(median, []sdk.Dec{})
+	require.ErrorIs(err, ErrEmptyList)
+}
+
+func TestAverage(t *testing.T) {
+	require := require.New(t)
+	prices := []sdk.Dec{
+		sdk.MustNewDecFromStr("1.1"),
+		sdk.MustNewDecFromStr("1.05"),
+		sdk.MustNewDecFromStr("1.15"),
+		sdk.MustNewDecFromStr("1.2"),
+	}
+
+	average, err := Average(prices)
+	require.NoError(err)
+	require.Equal(sdk.MustNewDecFromStr("1.125"), average)
+
+	// test empty prices list
+	average, err = Average([]sdk.Dec{})
+	require.ErrorIs(err, ErrEmptyList)
+}
+
+func TestMin(t *testing.T) {
+	require := require.New(t)
+	prices := []sdk.Dec{
+		sdk.MustNewDecFromStr("1.1"),
+		sdk.MustNewDecFromStr("1.05"),
+		sdk.MustNewDecFromStr("1.15"),
+		sdk.MustNewDecFromStr("1.2"),
+	}
+
+	min, err := Min(prices)
+	require.NoError(err)
+	require.Equal(sdk.MustNewDecFromStr("1.05"), min)
+
+	// test empty prices list
+	min, err = Min([]sdk.Dec{})
+	require.ErrorIs(err, ErrEmptyList)
+}
+
+func TestMax(t *testing.T) {
+	require := require.New(t)
+	prices := []sdk.Dec{
+		sdk.MustNewDecFromStr("1.1"),
+		sdk.MustNewDecFromStr("1.05"),
+		sdk.MustNewDecFromStr("1.15"),
+		sdk.MustNewDecFromStr("1.2"),
+	}
+
+	max, err := Max(prices)
+	require.NoError(err)
+	require.Equal(sdk.MustNewDecFromStr("1.2"), max)
+
+	// test empty prices list
+	max, err = Max([]sdk.Dec{})
+	require.ErrorIs(err, ErrEmptyList)
+}

x/oracle/abci.go

@@ -1,6 +1,7 @@
 package oracle
 
 import (
+	"strings"
 	"time"
 
 	"github.com/cosmos/cosmos-sdk/telemetry"
@@ -41,11 +42,6 @@ func EndBlocker(ctx sdk.Context, k keeper.Keeper, experimental bool) error {
 
 		k.ClearExchangeRates(ctx)
 
-		if isPeriodLastBlock(ctx, params.MedianPeriod) && experimental {
-			k.ClearMedians(ctx)
-			k.ClearMedianDeviations(ctx)
-		}
-
 		// NOTE: it filters out inactive or jailed validators
 		ballotDenomSlice := k.OrganizeBallotByDenom(ctx, validatorClaimMap)
 
@@ -63,14 +59,16 @@ func EndBlocker(ctx sdk.Context, k keeper.Keeper, experimental bool) error {
 			}
 
 			if experimental {
-				// Stamp rate every stamp period if asset is set to have historic stats tracked
-				if isPeriodLastBlock(ctx, params.StampPeriod) && params.HistoricAcceptList.Contains(ballotDenom.Denom) {
-					k.AddHistoricPrice(ctx, ballotDenom.Denom, exchangeRate)
+				// Stamp historic price if historic stamp period has passed
+				if isPeriodLastBlock(ctx, params.HistoricStampPeriod) {
+					k.AddHistoricPrice(ctx, strings.ToUpper(ballotDenom.Denom), exchangeRate)
 				}
 
-				// Set median price every median period if asset is set to have historic stats tracked
-				if isPeriodLastBlock(ctx, params.MedianPeriod) && params.HistoricAcceptList.Contains(ballotDenom.Denom) {
-					k.CalcAndSetMedian(ctx, ballotDenom.Denom)
+				// Calculate and stamp median/median deviation if median stamp period has passed
+				if isPeriodLastBlock(ctx, params.MedianStampPeriod) {
+					if err = k.CalcAndSetMedian(ctx, strings.ToUpper(ballotDenom.Denom)); err != nil {
+						return err
+					}
 				}
 			}
 		}
@@ -108,11 +106,15 @@ func EndBlocker(ctx sdk.Context, k keeper.Keeper, experimental bool) error {
 		k.SlashAndResetMissCounters(ctx)
 	}
 
-	// Prune historic prices every prune period
-	if isPeriodLastBlock(ctx, params.PrunePeriod) && experimental {
-		pruneBlock := uint64(ctx.BlockHeight()) - params.PrunePeriod
-		for _, v := range params.HistoricAcceptList {
-			k.DeleteHistoricPrice(ctx, v.String(), pruneBlock)
+	// Prune historic prices and medians outside pruning period determined by
+	// the stamp period multiplied by the max stamps.
+	if experimental {
+		pruneHistoricBlock := uint64(ctx.BlockHeight()) - (params.HistoricStampPeriod * params.MaximumPriceStamps)
+		pruneMedianBlock := uint64(ctx.BlockHeight()) - (params.MedianStampPeriod * params.MaximumMedianStamps)
+		for _, v := range params.AcceptList {
+			k.DeleteHistoricPrice(ctx, strings.ToUpper(v.SymbolDenom), pruneHistoricBlock)
+			k.DeleteMedian(ctx, strings.ToUpper(v.SymbolDenom), pruneMedianBlock)
+			k.DeleteMedianDeviation(ctx, strings.ToUpper(v.SymbolDenom), pruneMedianBlock)
 		}
 	}