diff --git a/price-feeder/CHANGELOG.md b/price-feeder/CHANGELOG.md
index b480613e86..7071de5b1e 100644
--- a/price-feeder/CHANGELOG.md
+++ b/price-feeder/CHANGELOG.md
@@ -56,6 +56,7 @@ Ref: https://keepachangelog.com/en/1.0.0/
- [#580](https://github.com/umee-network/umee/pull/580) Update Kraken provider to use WebSocket.
- [#592](https://github.com/umee-network/umee/pull/592) Add subscribe ticker function to the following providers: Binance, Huobi, Kraken, and Okx.
- [#601](https://github.com/umee-network/umee/pull/601) Use TVWAP formula for determining prices when available.
+- [#609](https://github.com/umee-network/umee/pull/609) TVWAP faulty provider detection.
### Bug Fixes
diff --git a/price-feeder/oracle/oracle.go b/price-feeder/oracle/oracle.go
index bcb9560479..0cb6618ce0 100644
--- a/price-feeder/oracle/oracle.go
+++ b/price-feeder/oracle/oracle.go
@@ -34,6 +34,10 @@ const (
tickerTimeout = 1000 * time.Millisecond
)
+// deviationThreshold defines how many 𝜎 a provider can be away from the mean
+// without being considered faulty.
+var deviationThreshold = sdk.MustNewDecFromStr("2")
+
// PreviousPrevote defines a structure for defining the previous prevote
// submitted on-chain.
type PreviousPrevote struct {
@@ -152,12 +156,12 @@ func (o *Oracle) GetPrices() map[string]sdk.Dec {
// determined in the config. If candles are available, uses TVWAP in order
// to determine prices. If candles are not available, uses the most recent prices
// with VWAP. Warns the the user of any missing prices, and filters out any faulty
-// providers which do not report prices within 2𝜎 of the others.
+// providers which do not report prices or candles within 2𝜎 of the others.
func (o *Oracle) SetPrices(ctx context.Context, acceptList oracletypes.DenomList) error {
g := new(errgroup.Group)
mtx := new(sync.Mutex)
- providerPrices := make(map[string]map[string]provider.TickerPrice)
- providerCandles := make(map[string]map[string][]provider.CandlePrice)
+ providerPrices := make(provider.AggregatedProviderPrices)
+ providerCandles := make(provider.AggregatedProviderCandles)
requiredRates := make(map[string]struct{})
for providerName, currencyPairs := range o.providerPairs {
@@ -249,8 +253,13 @@ func (o *Oracle) SetPrices(ctx context.Context, acceptList oracletypes.DenomList
}
}
+ filteredCandles, err := o.filterCandleDeviations(providerCandles)
+ if err != nil {
+ return err
+ }
+
// attempt to use candles for tvwap calculations
- tvwapPrices, err := ComputeTVWAP(providerCandles)
+ tvwapPrices, err := ComputeTVWAP(filteredCandles)
if err != nil {
return err
}
@@ -258,7 +267,7 @@ func (o *Oracle) SetPrices(ctx context.Context, acceptList oracletypes.DenomList
// If TVWAP candles are not available or were filtered out due to staleness,
// use most recent prices & VWAP instead.
if len(tvwapPrices) == 0 {
- filteredProviderPrices, err := o.filterDeviations(providerPrices)
+ filteredProviderPrices, err := o.filterTickerDeviations(providerPrices)
if err != nil {
return err
}
@@ -352,40 +361,44 @@ func (o *Oracle) getOrSetProvider(ctx context.Context, providerName string) (pro
return priceProvider, nil
}
-// filterDeviations find the standard deviations of the prices of
-// all assets, and filter out any providers that are not within 2𝜎 of the mean.
-func (o *Oracle) filterDeviations(
- prices map[string]map[string]provider.TickerPrice) (
- map[string]map[string]provider.TickerPrice, error,
-) {
+// filterTickerDeviations finds the standard deviations of the prices of
+// all assets, and filters out any providers that are not within 2𝜎 of the mean.
+func (o *Oracle) filterTickerDeviations(
+ prices provider.AggregatedProviderPrices,
+) (provider.AggregatedProviderPrices, error) {
var (
- filteredPrices = make(map[string]map[string]provider.TickerPrice)
- threshold = sdk.MustNewDecFromStr("2")
+ filteredPrices = make(provider.AggregatedProviderPrices)
+ priceMap = make(map[string]map[string]sdk.Dec)
)
- deviations, means, err := StandardDeviation(prices)
+ for providerName, providerPrices := range prices {
+ if _, ok := priceMap[providerName]; !ok {
+ priceMap[providerName] = make(map[string]sdk.Dec)
+ }
+ for base, price := range providerPrices {
+ priceMap[providerName][base] = price.Price
+ }
+ }
+
+ deviations, means, err := StandardDeviation(priceMap)
if err != nil {
- return make(map[string]map[string]provider.TickerPrice), nil
+ return nil, err
}
- // Accept any prices that are within 2𝜎, or for which we couldn't get 𝜎
- for providerName, priceMap := range prices {
- for base, price := range priceMap {
+ // accept any prices that are within 2𝜎, or for which we couldn't get 𝜎
+ for providerName, priceTickers := range prices {
+ for base, ticker := range priceTickers {
if _, ok := deviations[base]; !ok ||
- (price.Price.GTE(means[base].Sub(deviations[base].Mul(threshold))) &&
- price.Price.LTE(means[base].Add(deviations[base].Mul(threshold)))) {
+ (ticker.Price.GTE(means[base].Sub(deviations[base].Mul(deviationThreshold))) &&
+ ticker.Price.LTE(means[base].Add(deviations[base].Mul(deviationThreshold)))) {
if _, ok := filteredPrices[providerName]; !ok {
filteredPrices[providerName] = make(map[string]provider.TickerPrice)
}
- filteredPrices[providerName][base] = provider.TickerPrice{
- Price: price.Price,
- Volume: price.Volume,
- }
+ filteredPrices[providerName][base] = ticker
} else {
telemetry.IncrCounter(1, "failure", "provider")
- o.logger.Warn().Str("base", base).Str("provider", providerName).Msg(
- "provider deviating from other prices",
- )
+ o.logger.Warn().Str("base", base).Str("provider", providerName).Str(
+ "price", ticker.Price.String()).Msg("provider deviating from other prices")
}
}
}
@@ -393,6 +406,65 @@ func (o *Oracle) filterDeviations(
return filteredPrices, nil
}
+// filterCandleDeviations finds the standard deviations of the tvwaps of
+// all assets, and filters out any providers that are not within 2𝜎 of the mean.
+func (o *Oracle) filterCandleDeviations(
+ candles provider.AggregatedProviderCandles,
+) (provider.AggregatedProviderCandles, error) {
+ var (
+ filteredCandles = make(provider.AggregatedProviderCandles)
+ tvwaps = make(map[string]map[string]sdk.Dec)
+ )
+
+ for providerName, c := range candles {
+ candlePrices := make(provider.AggregatedProviderCandles)
+
+ for assetName, asset := range c {
+ if _, ok := candlePrices[providerName]; !ok {
+ candlePrices[providerName] = make(map[string][]provider.CandlePrice)
+ }
+ candlePrices[providerName][assetName] = asset
+ }
+
+ tvwap, err := ComputeTVWAP(candlePrices)
+ if err != nil {
+ return nil, err
+ }
+
+ for assetName, asset := range tvwap {
+ if _, ok := tvwaps[providerName]; !ok {
+ tvwaps[providerName] = make(map[string]sdk.Dec)
+ }
+ tvwaps[providerName][assetName] = asset
+ }
+ }
+
+ deviations, means, err := StandardDeviation(tvwaps)
+ if err != nil {
+ return nil, err
+ }
+
+ // accept any tvwaps that are within 2𝜎, or for which we couldn't get 𝜎
+ for providerName, priceMap := range tvwaps {
+ for base, price := range priceMap {
+ if _, ok := deviations[base]; !ok ||
+ (price.GTE(means[base].Sub(deviations[base].Mul(deviationThreshold))) &&
+ price.LTE(means[base].Add(deviations[base].Mul(deviationThreshold)))) {
+ if _, ok := filteredCandles[providerName]; !ok {
+ filteredCandles[providerName] = make(map[string][]provider.CandlePrice)
+ }
+ filteredCandles[providerName][base] = candles[providerName][base]
+ } else {
+ telemetry.IncrCounter(1, "failure", "provider")
+ o.logger.Warn().Str("base", base).Str("provider", providerName).Str(
+ "price", price.String()).Msg("provider deviating from other candles")
+ }
+ }
+ }
+
+ return filteredCandles, nil
+}
+
func (o *Oracle) checkAcceptList(params oracletypes.Params) {
for _, denom := range params.AcceptList {
symbol := strings.ToUpper(denom.SymbolDenom)
diff --git a/price-feeder/oracle/provider/provider.go b/price-feeder/oracle/provider/provider.go
index 6892d54124..d3c14394ab 100644
--- a/price-feeder/oracle/provider/provider.go
+++ b/price-feeder/oracle/provider/provider.go
@@ -33,6 +33,10 @@ type TickerPrice struct {
Volume sdk.Dec // 24h volume
}
+// AggregatedProviderPrices defines a type alias for a map
+// of provider -> asset -> TickerPrice
+type AggregatedProviderPrices map[string]map[string]TickerPrice
+
// CandlePrice defines price, volume, and time information for an
// exchange rate.
type CandlePrice struct {
@@ -41,6 +45,10 @@ type CandlePrice struct {
TimeStamp int64 // timestamp
}
+// AggregatedProviderCandles defines a type alias for a map
+// of provider -> asset -> []CandlePrice
+type AggregatedProviderCandles map[string]map[string][]CandlePrice
+
// preventRedirect avoid any redirect in the http.Client the request call
// will not return an error, but a valid response with redirect response code.
func preventRedirect(_ *http.Request, _ []*http.Request) error {
diff --git a/price-feeder/oracle/util.go b/price-feeder/oracle/util.go
index e1bdb9082e..24b22d7b0e 100644
--- a/price-feeder/oracle/util.go
+++ b/price-feeder/oracle/util.go
@@ -36,7 +36,7 @@ func vwap(weightedPrices map[string]sdk.Dec, volumeSum map[string]sdk.Dec) (map[
// of provider => { => , ...}.
//
// Ref: https://en.wikipedia.org/wiki/Volume-weighted_average_price
-func ComputeVWAP(prices map[string]map[string]provider.TickerPrice) (map[string]sdk.Dec, error) {
+func ComputeVWAP(prices provider.AggregatedProviderPrices) (map[string]sdk.Dec, error) {
var (
weightedPrices = make(map[string]sdk.Dec)
volumeSum = make(map[string]sdk.Dec)
@@ -68,7 +68,7 @@ func ComputeVWAP(prices map[string]map[string]provider.TickerPrice) (map[string]
// provider => { => , ...}.
//
// Ref : https://en.wikipedia.org/wiki/Time-weighted_average_price
-func ComputeTVWAP(prices map[string]map[string][]provider.CandlePrice) (map[string]sdk.Dec, error) {
+func ComputeTVWAP(prices provider.AggregatedProviderCandles) (map[string]sdk.Dec, error) {
var (
weightedPrices = make(map[string]sdk.Dec)
volumeSum = make(map[string]sdk.Dec)
@@ -121,7 +121,7 @@ func ComputeTVWAP(prices map[string]map[string][]provider.CandlePrice) (map[stri
// StandardDeviation returns maps of the standard deviations and means of assets.
// Will skip calculating for an asset if there are less than 3 prices.
func StandardDeviation(
- prices map[string]map[string]provider.TickerPrice) (
+ prices map[string]map[string]sdk.Dec) (
map[string]sdk.Dec, map[string]sdk.Dec, error,
) {
var (
@@ -140,8 +140,8 @@ func StandardDeviation(
priceSlice[base] = []sdk.Dec{}
}
- priceSums[base] = priceSums[base].Add(tp.Price)
- priceSlice[base] = append(priceSlice[base], tp.Price)
+ priceSums[base] = priceSums[base].Add(tp)
+ priceSlice[base] = append(priceSlice[base], tp)
}
}
diff --git a/price-feeder/oracle/util_test.go b/price-feeder/oracle/util_test.go
index fd4617aa88..9043ac8c43 100644
--- a/price-feeder/oracle/util_test.go
+++ b/price-feeder/oracle/util_test.go
@@ -85,11 +85,11 @@ func TestStandardDeviation(t *testing.T) {
deviation sdk.Dec
}
testCases := map[string]struct {
- prices map[string]map[string]provider.TickerPrice
+ prices map[string]map[string]sdk.Dec
expected map[string]deviation
}{
"empty prices": {
- prices: make(map[string]map[string]provider.TickerPrice),
+ prices: make(map[string]map[string]sdk.Dec),
expected: map[string]deviation{},
},
"nil prices": {
@@ -97,63 +97,35 @@ func TestStandardDeviation(t *testing.T) {
expected: map[string]deviation{},
},
"not enough prices": {
- prices: map[string]map[string]provider.TickerPrice{
+ prices: map[string]map[string]sdk.Dec{
config.ProviderBinance: {
- "ATOM": provider.TickerPrice{
- Price: sdk.MustNewDecFromStr("28.21000000"),
- },
- "UMEE": provider.TickerPrice{
- Price: sdk.MustNewDecFromStr("1.13000000"),
- },
- "LUNA": provider.TickerPrice{
- Price: sdk.MustNewDecFromStr("64.87000000"),
- },
+ "ATOM": sdk.MustNewDecFromStr("28.21000000"),
+ "UMEE": sdk.MustNewDecFromStr("1.13000000"),
+ "LUNA": sdk.MustNewDecFromStr("64.87000000"),
},
config.ProviderKraken: {
- "ATOM": provider.TickerPrice{
- Price: sdk.MustNewDecFromStr("28.23000000"),
- },
- "UMEE": provider.TickerPrice{
- Price: sdk.MustNewDecFromStr("1.13050000"),
- },
- "LUNA": provider.TickerPrice{
- Price: sdk.MustNewDecFromStr("64.85000000"),
- },
+ "ATOM": sdk.MustNewDecFromStr("28.23000000"),
+ "UMEE": sdk.MustNewDecFromStr("1.13050000"),
+ "LUNA": sdk.MustNewDecFromStr("64.85000000"),
},
},
expected: map[string]deviation{},
},
"some prices": {
- prices: map[string]map[string]provider.TickerPrice{
+ prices: map[string]map[string]sdk.Dec{
config.ProviderBinance: {
- "ATOM": provider.TickerPrice{
- Price: sdk.MustNewDecFromStr("28.21000000"),
- },
- "UMEE": provider.TickerPrice{
- Price: sdk.MustNewDecFromStr("1.13000000"),
- },
- "LUNA": provider.TickerPrice{
- Price: sdk.MustNewDecFromStr("64.87000000"),
- },
+ "ATOM": sdk.MustNewDecFromStr("28.21000000"),
+ "UMEE": sdk.MustNewDecFromStr("1.13000000"),
+ "LUNA": sdk.MustNewDecFromStr("64.87000000"),
},
config.ProviderKraken: {
- "ATOM": provider.TickerPrice{
- Price: sdk.MustNewDecFromStr("28.23000000"),
- },
- "UMEE": provider.TickerPrice{
- Price: sdk.MustNewDecFromStr("1.13050000"),
- },
+ "ATOM": sdk.MustNewDecFromStr("28.23000000"),
+ "UMEE": sdk.MustNewDecFromStr("1.13050000"),
},
config.ProviderOsmosis: {
- "ATOM": provider.TickerPrice{
- Price: sdk.MustNewDecFromStr("28.40000000"),
- },
- "UMEE": provider.TickerPrice{
- Price: sdk.MustNewDecFromStr("1.14000000"),
- },
- "LUNA": provider.TickerPrice{
- Price: sdk.MustNewDecFromStr("64.10000000"),
- },
+ "ATOM": sdk.MustNewDecFromStr("28.40000000"),
+ "UMEE": sdk.MustNewDecFromStr("1.14000000"),
+ "LUNA": sdk.MustNewDecFromStr("64.10000000"),
},
},
expected: map[string]deviation{
@@ -169,39 +141,22 @@ func TestStandardDeviation(t *testing.T) {
},
"non empty prices": {
- prices: map[string]map[string]provider.TickerPrice{
+ prices: map[string]map[string]sdk.Dec{
config.ProviderBinance: {
- "ATOM": provider.TickerPrice{
- Price: sdk.MustNewDecFromStr("28.21000000"),
- },
- "UMEE": provider.TickerPrice{
- Price: sdk.MustNewDecFromStr("1.13000000"),
- },
- "LUNA": provider.TickerPrice{
- Price: sdk.MustNewDecFromStr("64.87000000"),
- },
+ "ATOM": sdk.MustNewDecFromStr("28.21000000"),
+
+ "UMEE": sdk.MustNewDecFromStr("1.13000000"),
+ "LUNA": sdk.MustNewDecFromStr("64.87000000"),
},
config.ProviderKraken: {
- "ATOM": provider.TickerPrice{
- Price: sdk.MustNewDecFromStr("28.23000000"),
- },
- "UMEE": provider.TickerPrice{
- Price: sdk.MustNewDecFromStr("1.13050000"),
- },
- "LUNA": provider.TickerPrice{
- Price: sdk.MustNewDecFromStr("64.85000000"),
- },
+ "ATOM": sdk.MustNewDecFromStr("28.23000000"),
+ "UMEE": sdk.MustNewDecFromStr("1.13050000"),
+ "LUNA": sdk.MustNewDecFromStr("64.85000000"),
},
config.ProviderOsmosis: {
- "ATOM": provider.TickerPrice{
- Price: sdk.MustNewDecFromStr("28.40000000"),
- },
- "UMEE": provider.TickerPrice{
- Price: sdk.MustNewDecFromStr("1.14000000"),
- },
- "LUNA": provider.TickerPrice{
- Price: sdk.MustNewDecFromStr("64.10000000"),
- },
+ "ATOM": sdk.MustNewDecFromStr("28.40000000"),
+ "UMEE": sdk.MustNewDecFromStr("1.14000000"),
+ "LUNA": sdk.MustNewDecFromStr("64.10000000"),
},
},
expected: map[string]deviation{