Merge branch 'main' into tyler/blo-1459-simapp-doesnt-work-with-xupgrade

cmd/constants/markets.go → cmd/constants/marketmaps/markets.go

@@ -1,4 +1,4 @@
-package constants
+package marketmaps
 
 import (
 	"encoding/json"
@@ -17,6 +17,125 @@ var (
 	CoinMarketCapMarketMapJSON = `
 {
     "markets": {
+	  "W/USD": {
+        "ticker": {
+          "currency_pair": {
+            "Base": "W",
+            "Quote": "USD"
+          },
+          "decimals": 8,
+          "min_provider_count": 1,
+          "enabled": true
+        },
+        "provider_configs": [
+          {
+            "name": "coinmarketcap_api",
+            "off_chain_ticker": "29587"
+          }
+        ]
+      },
+	  "TON/USD": {
+        "ticker": {
+          "currency_pair": {
+            "Base": "TON",
+            "Quote": "USD"
+          },
+          "decimals": 8,
+          "min_provider_count": 1,
+          "enabled": true
+        },
+        "provider_configs": [
+          {
+            "name": "coinmarketcap_api",
+            "off_chain_ticker": "11419"
+          }
+        ]
+      },
+	  "ZRO/USD": {
+        "ticker": {
+          "currency_pair": {
+            "Base": "ZRO",
+            "Quote": "USD"
+          },
+          "decimals": 8,
+          "min_provider_count": 1,
+          "enabled": true
+        },
+        "provider_configs": [
+          {
+            "name": "coinmarketcap_api",
+            "off_chain_ticker": "26997"
+          }
+        ]
+      },
+	  "CHZ/USD": {
+        "ticker": {
+          "currency_pair": {
+            "Base": "CHZ",
+            "Quote": "USD"
+          },
+          "decimals": 8,
+          "min_provider_count": 1,
+          "enabled": true
+        },
+        "provider_configs": [
+          {
+            "name": "coinmarketcap_api",
+            "off_chain_ticker": "4066"
+          }
+        ]
+      },
+	  "ZK/USD": {
+        "ticker": {
+          "currency_pair": {
+            "Base": "ZK",
+            "Quote": "USD"
+          },
+          "decimals": 8,
+          "min_provider_count": 1,
+          "enabled": true
+        },
+        "provider_configs": [
+          {
+            "name": "coinmarketcap_api",
+            "off_chain_ticker": "24091"
+          }
+        ]
+      },
+	  "BODEN/USD": {
+        "ticker": {
+          "currency_pair": {
+            "Base": "BODEN",
+            "Quote": "USD"
+          },
+          "decimals": 8,
+          "min_provider_count": 1,
+          "enabled": true
+        },
+        "provider_configs": [
+          {
+            "name": "coinmarketcap_api",
+            "off_chain_ticker": "29687"
+          }
+        ]
+      },
+	  "ETHFI/USD": {
+        "ticker": {
+          "currency_pair": {
+            "Base": "ETHFI",
+            "Quote": "USD"
+          },
+          "decimals": 8,
+          "min_provider_count": 1,
+          "enabled": true
+        },
+        "provider_configs": [
+          {
+            "name": "coinmarketcap_api",
+            "off_chain_ticker": "29814"
+          }
+        ]
+      },
       "KHAI/USD": {
         "ticker": {
           "currency_pair": {

cmd/constants/markets_test.go → cmd/constants/marketmaps/markets_test.go

@@ -1,38 +1,38 @@
-package constants_test
+package marketmaps_test
 
 import (
 	"encoding/json"
 	"testing"
 
 	"github.com/stretchr/testify/require"
 
-	"github.com/skip-mev/slinky/cmd/constants"
+	"github.com/skip-mev/slinky/cmd/constants/marketmaps"
 	mmtypes "github.com/skip-mev/slinky/x/marketmap/types"
 )
 
 func TestMarkets(t *testing.T) {
 	// Unmarshal the RaydiumMarketMapJSON into RaydiumMarketMap.
 	var mm mmtypes.MarketMap
-	require.NoError(t, json.Unmarshal([]byte(constants.RaydiumMarketMapJSON), &mm))
+	require.NoError(t, json.Unmarshal([]byte(marketmaps.RaydiumMarketMapJSON), &mm))
 	require.NoError(t, mm.ValidateBasic())
 
 	// Unmarshal the CoreMarketMapJSON into CoreMarketMap.
 	var mm2 mmtypes.MarketMap
-	require.NoError(t, json.Unmarshal([]byte(constants.CoreMarketMapJSON), &mm2))
+	require.NoError(t, json.Unmarshal([]byte(marketmaps.CoreMarketMapJSON), &mm2))
 	require.NoError(t, mm2.ValidateBasic())
 
 	// Unmarshal the UniswapV3BaseMarketMapJSON into UniswapV3BaseMarketMap.
 	var mm3 mmtypes.MarketMap
-	require.NoError(t, json.Unmarshal([]byte(constants.UniswapV3BaseMarketMapJSON), &mm3))
-	require.NoError(t, constants.UniswapV3BaseMarketMap.ValidateBasic())
+	require.NoError(t, json.Unmarshal([]byte(marketmaps.UniswapV3BaseMarketMapJSON), &mm3))
+	require.NoError(t, marketmaps.UniswapV3BaseMarketMap.ValidateBasic())
 
 	// Unmarshal the CoinGeckoMarketMapJSON into CoinGeckoMarketMap.
 	var mm4 mmtypes.MarketMap
-	require.NoError(t, json.Unmarshal([]byte(constants.CoinGeckoMarketMapJSON), &mm4))
-	require.NoError(t, constants.CoinGeckoMarketMap.ValidateBasic())
+	require.NoError(t, json.Unmarshal([]byte(marketmaps.CoinGeckoMarketMapJSON), &mm4))
+	require.NoError(t, marketmaps.CoinGeckoMarketMap.ValidateBasic())
 
 	// Unmarshal the CoinMarketCapMarketMapJSON into CoinMarketCapMarketMap.
 	var mm5 mmtypes.MarketMap
-	require.NoError(t, json.Unmarshal([]byte(constants.CoinMarketCapMarketMapJSON), &mm5))
-	require.NoError(t, constants.CoinMarketCapMarketMap.ValidateBasic())
+	require.NoError(t, json.Unmarshal([]byte(marketmaps.CoinMarketCapMarketMapJSON), &mm5))
+	require.NoError(t, marketmaps.CoinMarketCapMarketMap.ValidateBasic())
 }

cmd/constants/providers.go

@@ -162,11 +162,17 @@ var (
 			API:  dydx.DefaultResearchAPIConfig,
 			Type: mmtypes.ConfigType,
 		},
+		{
+			Name: dydx.ResearchCMCAPIHandlerName,
+			API:  dydx.DefaultResearchCMCAPIConfig,
+			Type: mmtypes.ConfigType,
+		},
 	}
 
 	MarketMapProviderNames = map[string]struct{}{
-		dydx.Name:                   {},
-		dydx.ResearchAPIHandlerName: {},
-		marketmap.Name:              {},
+		dydx.Name:                      {},
+		dydx.ResearchAPIHandlerName:    {},
+		dydx.ResearchCMCAPIHandlerName: {},
+		marketmap.Name:                 {},
 	}
 )

metrics.md

@@ -52,9 +52,9 @@ Slinky exposes metrics on the `/metrics` endpoint on port `8002` by default. The
 
 There are three primary health metrics that are exposed by Slinky:
 
-* [`side_car_health_check_system_updates_total`](#side_car_health_check_system_updates_total): This metric is a counter that increments every time the side-car updates its internal state. This is a good indicator of the side-car's overall health.
-* [`side_car_health_check_ticker_updates_total`](#side_car_health_check_ticker_updates_total): This metric is a counter that increments every time the side-car updates the price of a given market. This is a good indicator of the overall health of a given market.
-* [`side_car_health_check_provider_updates_total`](#side_car_health_check_provider_updates_total): This metric is a counter that increments every time the side-car utilizes a given providers market data. This is a good indicator of the health of a given provider. Note that providers may not be responsible for every market. However, the side-car correctly tracks the number of expected updates for each provider.
+* (RECOMMENDED) [`side_car_health_check_system_updates_total`](#side_car_health_check_system_updates_total): This metric is a counter that increments every time the side-car updates its internal state. This is a good indicator of the side-car's overall health.
+* (RECOMMENDED) [`side_car_health_check_ticker_updates_total`](#side_car_health_check_ticker_updates_total): This metric is a counter that increments every time the side-car updates the price of a given market. This is a good indicator of the overall health of a given market.
+* (OPTIONAL) [`side_car_health_check_provider_updates_total`](#side_car_health_check_provider_updates_total): This metric is a counter that increments every time the side-car utilizes a given providers market data. This is a good indicator of the health of a given provider. Note that providers may not be responsible for every market. However, the side-car correctly tracks the number of expected updates for each provider.
 
 Given the additional context above, we should expect the health metrics to be increasing at a rate of 2.0 updates/sec (every 500 milliseconds) for each market and provider. If the rate of updates is lower than expected, this may indicate an issue with the side-car.
 

providers/apis/dydx/multi_market_map_fetcher.go

@@ -7,11 +7,14 @@ import (
 
 	"go.uber.org/zap"
 
+	"github.com/skip-mev/slinky/cmd/constants/marketmaps"
 	"github.com/skip-mev/slinky/oracle/config"
+	"github.com/skip-mev/slinky/providers/apis/coinmarketcap"
 	apihandlers "github.com/skip-mev/slinky/providers/base/api/handlers"
 	"github.com/skip-mev/slinky/providers/base/api/metrics"
 	providertypes "github.com/skip-mev/slinky/providers/types"
 	mmclient "github.com/skip-mev/slinky/service/clients/marketmap/types"
+	mmtypes "github.com/skip-mev/slinky/x/marketmap/types"
 )
 
 var (
@@ -86,7 +89,12 @@ func DefaultDYDXResearchMarketMapFetcher(
 		return nil, err
 	}
 
-	return NewDYDXResearchMarketMapFetcher(mainnetFetcher, researchFetcher, logger), nil
+	return NewDYDXResearchMarketMapFetcher(
+		mainnetFetcher,
+		researchFetcher,
+		logger,
+		api.Name == ResearchCMCAPIHandlerName,
+	), nil
 }
 
 // MultiMarketMapRestAPIFetcher is an implementation of a RestAPIFetcher that wraps
@@ -101,14 +109,22 @@ type MultiMarketMapRestAPIFetcher struct {
 
 	// logger is the logger for the fetcher
 	logger *zap.Logger
+
+	// isCMCOnly is a flag that indicates whether the fetcher should only return CoinMarketCap markets.
+	isCMCOnly bool
 }
 
 // NewDYDXResearchMarketMapFetcher returns an aggregated market-map among the dydx mainnet and the dydx research json.
-func NewDYDXResearchMarketMapFetcher(mainnetFetcher, researchFetcher mmclient.MarketMapFetcher, logger *zap.Logger) *MultiMarketMapRestAPIFetcher {
+func NewDYDXResearchMarketMapFetcher(
+	mainnetFetcher, researchFetcher mmclient.MarketMapFetcher,
+	logger *zap.Logger,
+	isCMCOnly bool,
+) *MultiMarketMapRestAPIFetcher {
 	return &MultiMarketMapRestAPIFetcher{
 		dydxMainnetFetcher:  mainnetFetcher,
 		dydxResearchFetcher: researchFetcher,
 		logger:              logger.With(zap.String("module", "dydx-research-market-map-fetcher")),
+		isCMCOnly:           isCMCOnly,
 	}
 }
 
@@ -142,29 +158,71 @@ func (f *MultiMarketMapRestAPIFetcher) Fetch(ctx context.Context, chains []mmcli
 	wg.Wait()
 
 	dydxMainnetMarketMapResponse := <-dydxMainnetResponseChan
-
 	dydxResearchMarketMapResponse := <-dydxResearchResponseChan
 
-	// combine the two market maps
 	// if the dydx mainnet market-map response failed, return the dydx mainnet failed response
 	if _, ok := dydxMainnetMarketMapResponse.UnResolved[DYDXChain]; ok {
 		f.logger.Error("dydx mainnet market-map fetch failed", zap.Any("response", dydxMainnetMarketMapResponse))
 		return dydxMainnetMarketMapResponse
 	}
 
+	// if the dydx research market-map response failed, return the dydx research failed response
+	if _, ok := dydxResearchMarketMapResponse.UnResolved[DYDXChain]; ok {
+		f.logger.Error("dydx research market-map fetch failed", zap.Any("response", dydxResearchMarketMapResponse))
+		return dydxResearchMarketMapResponse
+	}
+
 	// otherwise, add all markets from dydx research
 	dydxMainnetMarketMap := dydxMainnetMarketMapResponse.Resolved[DYDXChain].Value.MarketMap
 
-	if resolved, ok := dydxResearchMarketMapResponse.Resolved[DYDXChain]; ok {
+	resolved, ok := dydxResearchMarketMapResponse.Resolved[DYDXChain]
+	if ok {
 		for ticker, market := range resolved.Value.MarketMap.Markets {
 			// if the market is not already in the dydx mainnet market-map, add it
 			if _, ok := dydxMainnetMarketMap.Markets[ticker]; !ok {
 				f.logger.Debug("adding market from dydx research", zap.String("ticker", ticker))
 				dydxMainnetMarketMap.Markets[ticker] = market
 			}
 		}
-	} else {
-		return dydxResearchMarketMapResponse
+	}
+
+	// if the fetcher is only for CoinMarketCap markets, filter out all non-CMC markets
+	if f.isCMCOnly {
+		for ticker, market := range dydxMainnetMarketMap.Markets {
+			market.Ticker.MinProviderCount = 1
+			dydxMainnetMarketMap.Markets[ticker] = market
+
+			var (
+				seenCMC     = false
+				cmcProvider mmtypes.ProviderConfig
+			)
+
+			for _, provider := range market.ProviderConfigs {
+				if provider.Name == coinmarketcap.Name {
+					seenCMC = true
+					cmcProvider = provider
+				}
+			}
+
+			// if we saw a CMC provider, add it to the market
+			if seenCMC {
+				market.ProviderConfigs = []mmtypes.ProviderConfig{cmcProvider}
+				dydxMainnetMarketMap.Markets[ticker] = market
+				continue
+			}
+
+			// If we did not see a CMC provider, we can attempt to add it using the CMC marketmap
+			cmcMarket, ok := marketmaps.CoinMarketCapMarketMap.Markets[ticker]
+			if !ok {
+				f.logger.Info("did not find CMC market for ticker", zap.String("ticker", ticker))
+				delete(dydxMainnetMarketMap.Markets, ticker)
+				continue
+			}
+
+			// add the CMC provider to the market
+			market.ProviderConfigs = cmcMarket.ProviderConfigs
+			dydxMainnetMarketMap.Markets[ticker] = market
+		}
 	}
 
 	// validate the combined market-map

providers/apis/dydx/multi_market_map_fetcher_test.go

@@ -21,7 +21,7 @@ func TestDYDXMultiMarketMapFetcher(t *testing.T) {
 	dydxMainnetMMFetcher := apihandlermocks.NewAPIFetcher[mmclient.Chain, *mmtypes.MarketMapResponse](t)
 	dydxResearchMMFetcher := apihandlermocks.NewAPIFetcher[mmclient.Chain, *mmtypes.MarketMapResponse](t)
 
-	fetcher := dydx.NewDYDXResearchMarketMapFetcher(dydxMainnetMMFetcher, dydxResearchMMFetcher, zap.NewExample())
+	fetcher := dydx.NewDYDXResearchMarketMapFetcher(dydxMainnetMMFetcher, dydxResearchMMFetcher, zap.NewExample(), false)
 
 	t.Run("test that if the mainnet api-price fetcher response is unresolved, we return it", func(t *testing.T) {
 		ctx := context.Background()

providers/apis/dydx/parse.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/skip-mev/slinky/oracle/constants"
 	slinkytypes "github.com/skip-mev/slinky/pkg/types"
+	"github.com/skip-mev/slinky/providers/apis/coinmarketcap"
 	"github.com/skip-mev/slinky/providers/apis/defi/raydium"
 	"github.com/skip-mev/slinky/providers/apis/defi/uniswapv3"
 	dydxtypes "github.com/skip-mev/slinky/providers/apis/dydx/types"
@@ -49,6 +50,7 @@ var ProviderMapping = map[string]string{
 	"Raydium":              raydium.Name,
 	"UniswapV3-Ethereum":   uniswapv3.ProviderNames[constants.ETHEREUM],
 	"UniswapV3-Base":       uniswapv3.ProviderNames[constants.BASE],
+	coinmarketcap.Name:     coinmarketcap.Name,
 }
 
 // ConvertMarketParamsToMarketMap converts a dYdX market params response to a slinky market map response.