From c1afade6f44ef8cd29662c6b744309e7969fca2a Mon Sep 17 00:00:00 2001 From: rbajollari Date: Fri, 3 Feb 2023 16:04:48 -0500 Subject: [PATCH 1/5] fix currency tracker --- .../config/currency_provider_tracker.go | 88 ++++++++++++------- price-feeder/price-feeder.example.toml | 25 +----- 2 files changed, 59 insertions(+), 54 deletions(-) diff --git a/price-feeder/config/currency_provider_tracker.go b/price-feeder/config/currency_provider_tracker.go index 5497b48497..da1489c658 100644 --- a/price-feeder/config/currency_provider_tracker.go +++ b/price-feeder/config/currency_provider_tracker.go @@ -17,6 +17,7 @@ const ( coinGeckoTickersEndpoint = "tickers" osmosisV2RestURL = "https://api.osmo-api.network.umee.cc" osmosisV2AssetPairsEndpoint = "assetpairs" + requestTimeout = time.Second * 2 trackingPeriod = time.Hour * 24 ) @@ -32,6 +33,7 @@ type ( logger zerolog.Logger pairs []CurrencyPair coinIDSymbolMap map[string]string // ex: map["ATOM"] = "cosmos" + osmosisAPIPairs map[string]string // ex: map["BASE"] = "QUOTE" CurrencyProviders map[string][]string // map of price feeder currencies and what exchanges support them CurrencyProviderMin map[string]int // map of price feeder currencies and min required providers for them } @@ -80,6 +82,10 @@ func NewCurrencyProviderTracker( return nil, err } + if err := currencyProviderTracker.setOsmosisAPIPairs(); err != nil { + return nil, err + } + if err := currencyProviderTracker.setCurrencyProviders(); err != nil { return nil, err } @@ -111,51 +117,69 @@ func (t *CurrencyProviderTracker) setCoinIDSymbolMap() error { } for _, coin := range listResponse { - t.coinIDSymbolMap[coin.Symbol] = coin.ID + if _, ok := t.coinIDSymbolMap[coin.Symbol]; !ok { + t.coinIDSymbolMap[coin.Symbol] = coin.ID + } } return nil } -// setCurrencyProviders queries CoinGecko's tickers endpoint and the osmosis-api assetpairs -// endpoint to get all the exchanges that support each price feeder currency pair and store -// it in the CurrencyProviders map. -func (t *CurrencyProviderTracker) setCurrencyProviders() error { - for _, pair := range t.pairs { - // check if CoinGecko API supports pair - pairBaseID := t.coinIDSymbolMap[strings.ToLower(pair.Base)] - coinGeckoResp, err := http.Get(fmt.Sprintf("%s/%s/%s", coinGeckoRestURL, pairBaseID, coinGeckoTickersEndpoint)) - if err != nil { - return err - } - defer coinGeckoResp.Body.Close() +// setOsmosisV2CurrencyProviders queries the osmosis-api assetpairs endpoint to get +func (t *CurrencyProviderTracker) setOsmosisAPIPairs() error { + client := &http.Client{ + Timeout: requestTimeout, + } + t.osmosisAPIPairs = make(map[string]string) - var tickerResponse coinTickerResponse - if err = json.NewDecoder(coinGeckoResp.Body).Decode(&tickerResponse); err != nil { + osmosisResp, err := client.Get(fmt.Sprintf("%s/%s", osmosisV2RestURL, osmosisV2AssetPairsEndpoint)) + if err != nil { + t.logger.Error().Err(err).Msg("Failed to query osmosis api assetpairs endpoint") + } else { + defer osmosisResp.Body.Close() + var assetPairsResponse []assetPair + if err = json.NewDecoder(osmosisResp.Body).Decode(&assetPairsResponse); err != nil { return err } - for _, ticker := range tickerResponse.Tickers { - if ticker.Target == pair.Quote { - t.CurrencyProviders[pair.Base] = append(t.CurrencyProviders[pair.Base], ticker.Market.Name) - } + for _, assetPair := range assetPairsResponse { + t.osmosisAPIPairs[assetPair.Base] = assetPair.Quote } + } - // check if osmosis-api supports pair - osmosisResp, err := http.Get(fmt.Sprintf("%s/%s", osmosisV2RestURL, osmosisV2AssetPairsEndpoint)) - if err != nil { - return err - } - defer osmosisResp.Body.Close() + return nil +} - var assetPairsResponse []assetPair - if err = json.NewDecoder(osmosisResp.Body).Decode(&assetPairsResponse); err != nil { - return err +// setCurrencyProviders queries CoinGecko's tickers endpoint to get all the exchanges +// that support each price feeder currency pair and store it in the CurrencyProviders map. +func (t *CurrencyProviderTracker) setCurrencyProviders() error { + client := &http.Client{ + Timeout: requestTimeout, + } + for _, pair := range t.pairs { + t.CurrencyProviders[pair.Base] = []string{} + + // check if its a pair supported by the osmosis api + if t.osmosisAPIPairs[strings.ToUpper(pair.Base)] == strings.ToUpper(pair.Quote) { + t.CurrencyProviders[pair.Base] = append(t.CurrencyProviders[pair.Base], "osmosisv2") } - for _, assetPair := range assetPairsResponse { - if pair.Base == assetPair.Base && pair.Quote == assetPair.Quote { - t.CurrencyProviders[pair.Base] = append(t.CurrencyProviders[pair.Base], "osmosisv2") + // check if CoinGecko API supports pair + pairBaseID := t.coinIDSymbolMap[strings.ToLower(pair.Base)] + coinGeckoResp, err := client.Get(fmt.Sprintf("%s/%s/%s", coinGeckoRestURL, pairBaseID, coinGeckoTickersEndpoint)) + if err != nil { + t.logger.Error().Err(err).Msg(fmt.Sprintf("Failed to query coin gecko api tickers endpoint for %s", pair.Base)) + } else { + defer coinGeckoResp.Body.Close() + var tickerResponse coinTickerResponse + if err = json.NewDecoder(coinGeckoResp.Body).Decode(&tickerResponse); err != nil { + return err + } + + for _, ticker := range tickerResponse.Tickers { + if ticker.Target == pair.Quote { + t.CurrencyProviders[pair.Base] = append(t.CurrencyProviders[pair.Base], ticker.Market.Name) + } } } } @@ -168,7 +192,7 @@ func (t *CurrencyProviderTracker) setCurrencyProviders() error { // set to 3 providers. func (t *CurrencyProviderTracker) setCurrencyProviderMin() { for base, exchanges := range t.CurrencyProviders { - if len(exchanges) < 3 { + if len(exchanges) < 3 && len(exchanges) > 0 { // can be 0 if request errored t.CurrencyProviderMin[base] = len(exchanges) } else { t.CurrencyProviderMin[base] = 3 diff --git a/price-feeder/price-feeder.example.toml b/price-feeder/price-feeder.example.toml index 3d78a4367c..a290e9a4ea 100644 --- a/price-feeder/price-feeder.example.toml +++ b/price-feeder/price-feeder.example.toml @@ -50,10 +50,6 @@ threshold = "2" base = "OSMO" threshold = "2" -[[deviation_thresholds]] -base = "BTC" -threshold = "1.5" - [[deviation_thresholds]] base = "stATOM" threshold = "2" @@ -208,21 +204,6 @@ providers = [ ] quote = "USD" -[[currency_pairs]] -base = "BTC" -providers = [ - "coinbase", -] -quote = "USD" - -[[currency_pairs]] -base = "BTC" -providers = [ - "huobi", - "kraken", -] -quote = "USDT" - [[currency_pairs]] base = "stATOM" providers = [ @@ -245,13 +226,13 @@ providers = [ quote = "OSMO" [account] -address = "umee15nejfgcaanqpw25ru4arvfd0fwy6j8clccvwx4" +address = "umee1zypqa76je7pxsdwkfah6mu9a583sju6xjavygg" chain_id = "umee-local-testnet" -validator = "umeevaloper12tysz6mzrawenca2t3t7ltym4hfjj8a5upsn2k" +validator = "umeevaloper1zypqa76je7pxsdwkfah6mu9a583sju6xjettez" [keyring] backend = "test" -dir = "/Users/username/.umee" +dir = "/Users/ryanbajollari/.umee" [rpc] grpc_endpoint = "localhost:9090" From 9f27ff8222cef8dab2260f8457b837651e932268 Mon Sep 17 00:00:00 2001 From: rbajollari Date: Fri, 3 Feb 2023 16:11:03 -0500 Subject: [PATCH 2/5] price-feeder e2e timeout --- tests/e2e/e2e_setup_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/e2e/e2e_setup_test.go b/tests/e2e/e2e_setup_test.go index 4eee368f96..89bb53e16a 100644 --- a/tests/e2e/e2e_setup_test.go +++ b/tests/e2e/e2e_setup_test.go @@ -1029,7 +1029,7 @@ func (s *IntegrationTestSuite) runPriceFeeder() { return len(prices) > 0 }, - time.Second*90, // 1:30 min + time.Minute, time.Second, "price-feeder not healthy", ) From 353e4afc5a70f8c504000307fcc45b3018d1b409 Mon Sep 17 00:00:00 2001 From: rbajollari Date: Fri, 3 Feb 2023 16:15:52 -0500 Subject: [PATCH 3/5] example config --- price-feeder/price-feeder.example.toml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/price-feeder/price-feeder.example.toml b/price-feeder/price-feeder.example.toml index a290e9a4ea..e391d449ed 100644 --- a/price-feeder/price-feeder.example.toml +++ b/price-feeder/price-feeder.example.toml @@ -226,13 +226,13 @@ providers = [ quote = "OSMO" [account] -address = "umee1zypqa76je7pxsdwkfah6mu9a583sju6xjavygg" +address = "umee15nejfgcaanqpw25ru4arvfd0fwy6j8clccvwx4" chain_id = "umee-local-testnet" -validator = "umeevaloper1zypqa76je7pxsdwkfah6mu9a583sju6xjettez" +validator = "umeevaloper12tysz6mzrawenca2t3t7ltym4hfjj8a5upsn2k" [keyring] backend = "test" -dir = "/Users/ryanbajollari/.umee" +dir = "/Users/username/.umee" [rpc] grpc_endpoint = "localhost:9090" From 69822466f8d86be575759332056adab3aa7832a6 Mon Sep 17 00:00:00 2001 From: rbajollari Date: Mon, 6 Feb 2023 15:00:37 -0500 Subject: [PATCH 4/5] PR comments --- .../config/currency_provider_tracker.go | 50 +++++++++++-------- 1 file changed, 28 insertions(+), 22 deletions(-) diff --git a/price-feeder/config/currency_provider_tracker.go b/price-feeder/config/currency_provider_tracker.go index da1489c658..9c45632f18 100644 --- a/price-feeder/config/currency_provider_tracker.go +++ b/price-feeder/config/currency_provider_tracker.go @@ -33,7 +33,6 @@ type ( logger zerolog.Logger pairs []CurrencyPair coinIDSymbolMap map[string]string // ex: map["ATOM"] = "cosmos" - osmosisAPIPairs map[string]string // ex: map["BASE"] = "QUOTE" CurrencyProviders map[string][]string // map of price feeder currencies and what exchanges support them CurrencyProviderMin map[string]int // map of price feeder currencies and min required providers for them } @@ -82,11 +81,12 @@ func NewCurrencyProviderTracker( return nil, err } - if err := currencyProviderTracker.setOsmosisAPIPairs(); err != nil { + osmosisAPIPairs, err := currencyProviderTracker.getOsmosisAPIPairs() + if err != nil { return nil, err } - if err := currencyProviderTracker.setCurrencyProviders(); err != nil { + if err := currencyProviderTracker.setCurrencyProviders(osmosisAPIPairs); err != nil { return nil, err } @@ -125,42 +125,40 @@ func (t *CurrencyProviderTracker) setCoinIDSymbolMap() error { return nil } -// setOsmosisV2CurrencyProviders queries the osmosis-api assetpairs endpoint to get -func (t *CurrencyProviderTracker) setOsmosisAPIPairs() error { +// getOsmosisAPIPairs queries the osmosis-api assetpairs endpoint to get the asset pairs +// supported by it. +func (t *CurrencyProviderTracker) getOsmosisAPIPairs() (map[string]string, error) { client := &http.Client{ Timeout: requestTimeout, } - t.osmosisAPIPairs = make(map[string]string) + osmosisAPIPairs := make(map[string]string) osmosisResp, err := client.Get(fmt.Sprintf("%s/%s", osmosisV2RestURL, osmosisV2AssetPairsEndpoint)) if err != nil { - t.logger.Error().Err(err).Msg("Failed to query osmosis api assetpairs endpoint") - } else { - defer osmosisResp.Body.Close() - var assetPairsResponse []assetPair - if err = json.NewDecoder(osmosisResp.Body).Decode(&assetPairsResponse); err != nil { - return err - } + return nil, err + } + defer osmosisResp.Body.Close() + var assetPairsResponse []assetPair + if err = json.NewDecoder(osmosisResp.Body).Decode(&assetPairsResponse); err != nil { + return nil, err + } - for _, assetPair := range assetPairsResponse { - t.osmosisAPIPairs[assetPair.Base] = assetPair.Quote - } + for _, assetPair := range assetPairsResponse { + osmosisAPIPairs[assetPair.Base] = assetPair.Quote } - return nil + return osmosisAPIPairs, nil } // setCurrencyProviders queries CoinGecko's tickers endpoint to get all the exchanges // that support each price feeder currency pair and store it in the CurrencyProviders map. -func (t *CurrencyProviderTracker) setCurrencyProviders() error { +func (t *CurrencyProviderTracker) setCurrencyProviders(osmosisAPIPairs map[string]string) error { client := &http.Client{ Timeout: requestTimeout, } for _, pair := range t.pairs { - t.CurrencyProviders[pair.Base] = []string{} - // check if its a pair supported by the osmosis api - if t.osmosisAPIPairs[strings.ToUpper(pair.Base)] == strings.ToUpper(pair.Quote) { + if osmosisAPIPairs[strings.ToUpper(pair.Base)] == strings.ToUpper(pair.Quote) { t.CurrencyProviders[pair.Base] = append(t.CurrencyProviders[pair.Base], "osmosisv2") } @@ -181,6 +179,10 @@ func (t *CurrencyProviderTracker) setCurrencyProviders() error { t.CurrencyProviders[pair.Base] = append(t.CurrencyProviders[pair.Base], ticker.Market.Name) } } + + if len(t.CurrencyProviders[pair.Base]) == 0 { + t.logger.Warn().Msg(fmt.Sprintf("Coin gecko found 0 providers for %s", pair.Base)) + } } } @@ -210,7 +212,11 @@ func (t *CurrencyProviderTracker) trackCurrencyProviders(ctx context.Context) { case <-ctx.Done(): return case <-time.After(trackingPeriod): - if err := t.setCurrencyProviders(); err != nil { + osmosisAPIPairs, err := t.getOsmosisAPIPairs() + if err != nil { + t.logger.Error().Err(err).Msg("failed to query osmosis-api for available asset pairs") + } + if err := t.setCurrencyProviders(osmosisAPIPairs); err != nil { t.logger.Error().Err(err).Msg("failed to set available providers for currencies") } From 81dde7be7832e7b541c21911fb0c97face377158 Mon Sep 17 00:00:00 2001 From: rbajollari Date: Tue, 7 Feb 2023 15:12:29 -0500 Subject: [PATCH 5/5] comment and changelog --- price-feeder/CHANGELOG.md | 2 ++ price-feeder/config/currency_provider_tracker.go | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/price-feeder/CHANGELOG.md b/price-feeder/CHANGELOG.md index 8afdefd3bc..a770373483 100644 --- a/price-feeder/CHANGELOG.md +++ b/price-feeder/CHANGELOG.md @@ -46,6 +46,8 @@ Ref: https://keepachangelog.com/en/1.0.0/ ## Unreleased +- [1781](https://github.com/umee-network/umee/pull/1781) BTC was removed from example config since WBTC is used by the `Oracle` module instead. + ## [v2.0.2](https://github.com/umee-network/umee/releases/tag/price-feeder/v2.0.2) 2022-12-14 ### Bugs diff --git a/price-feeder/config/currency_provider_tracker.go b/price-feeder/config/currency_provider_tracker.go index 9c45632f18..6a8696a551 100644 --- a/price-feeder/config/currency_provider_tracker.go +++ b/price-feeder/config/currency_provider_tracker.go @@ -194,7 +194,9 @@ func (t *CurrencyProviderTracker) setCurrencyProviders(osmosisAPIPairs map[strin // set to 3 providers. func (t *CurrencyProviderTracker) setCurrencyProviderMin() { for base, exchanges := range t.CurrencyProviders { - if len(exchanges) < 3 && len(exchanges) > 0 { // can be 0 if request errored + // If CoinGecko returns 0 or 3 and above providers for a given pair, the + // minimum providers enforced will default to 3. + if len(exchanges) < 3 && len(exchanges) > 0 { t.CurrencyProviderMin[base] = len(exchanges) } else { t.CurrencyProviderMin[base] = 3