diff --git a/cmd/constants/marketmaps/markets.go b/cmd/constants/marketmaps/markets.go index 55a25f073..77f8d7666 100644 --- a/cmd/constants/marketmaps/markets.go +++ b/cmd/constants/marketmaps/markets.go @@ -5224,7 +5224,7 @@ var ( } }, { - "name": "bitstamp_ws", + "name": "bitstamp_api", "off_chain_ticker": "hbarusd" }, { diff --git a/cmd/constants/providers.go b/cmd/constants/providers.go index 963dce8cc..a8c8d1d16 100644 --- a/cmd/constants/providers.go +++ b/cmd/constants/providers.go @@ -6,6 +6,7 @@ import ( "github.com/skip-mev/slinky/oracle/constants" "github.com/skip-mev/slinky/oracle/types" binanceapi "github.com/skip-mev/slinky/providers/apis/binance" + bitstampapi "github.com/skip-mev/slinky/providers/apis/bitstamp" coinbaseapi "github.com/skip-mev/slinky/providers/apis/coinbase" "github.com/skip-mev/slinky/providers/apis/coingecko" "github.com/skip-mev/slinky/providers/apis/coinmarketcap" @@ -55,6 +56,11 @@ var ( API: binanceapi.DefaultNonUSAPIConfig, Type: types.ConfigType, }, + { + Name: bitstampapi.Name, + API: bitstampapi.DefaultAPIConfig, + Type: types.ConfigType, + }, { Name: coinbaseapi.Name, API: coinbaseapi.DefaultAPIConfig, diff --git a/docs/validator/validator-connect-config.mdx b/docs/validator/validator-connect-config.mdx index 90cf0839a..b6a39f6fb 100644 --- a/docs/validator/validator-connect-config.mdx +++ b/docs/validator/validator-connect-config.mdx @@ -46,6 +46,7 @@ required to fetch relevant data. Currently supported API Providers: - binance_api +- bitstamp_api - coinbase_api - coingecko_api - gecko_terminal_api diff --git a/oracle/config/README.md b/oracle/config/README.md index ee4737521..e077ec750 100644 --- a/oracle/config/README.md +++ b/oracle/config/README.md @@ -429,7 +429,7 @@ Sample configuration: "type": "price_provider" }, { - "name": "bitstamp_ws", + "name": "bitstamp_api", "api": { "enabled": false, "timeout": 0, @@ -445,7 +445,7 @@ Sample configuration: "maxBufferSize": 1024, "reconnectionTimeout": 10000000000, "wss": "wss://ws.bitstamp.net", - "name": "bitstamp_ws", + "name": "bitstamp_api", "readBufferSize": 0, "writeBufferSize": 0, "handshakeTimeout": 45000000000, diff --git a/providers/apis/README.md b/providers/apis/README.md index 05a03c5d6..6696e3b7b 100644 --- a/providers/apis/README.md +++ b/providers/apis/README.md @@ -16,6 +16,11 @@ The current set of supported providers are: * `curl https://api.binance.com/api/v3/exchangeInfo | jq` * Check if a given market is supported: * `curl https://api.binance.com/api/v3/ticker/price?symbol=BTCUSDT | jq` +* [Bitstamp](./bitstamp/README.md) - Bitstamp is a cryptocurrency exchange that provides a free API for fetching cryptocurrency data. Bitstamp is a **primary data source** for the oracle. + * Check all supported markets: + * `curl https://www.bitstamp.net/api/v2/trading-pairs-info/ | jq` + * Check if a given market is supported: + * `curl https://www.bitstamp.net/api/v2/ticker/{btcusd} | jq` * [Coinbase](./coinbase/README.md) - Coinbase is a cryptocurrency exchange that provides a free API for fetching cryptocurrency data. Coinbase is a **primary data source** for the oracle. * Check all supported markets: * `curl https://api.exchange.coinbase.com/currencies | jq` diff --git a/providers/apis/bitstamp/README.md b/providers/apis/bitstamp/README.md new file mode 100644 index 000000000..e09d93096 --- /dev/null +++ b/providers/apis/bitstamp/README.md @@ -0,0 +1,5 @@ +# Bitstamp Provider + +## Overview + +The Bitstamp provider is used to fetch the spot price for cryptocurrencies from the [Bitstamp API](https://www.bitstamp.net/api/). As standard, all clients can make 400 requests per second. There is a default limit threshold of 10,000 requests per 10 minutes in place. diff --git a/providers/apis/bitstamp/api_handler.go b/providers/apis/bitstamp/api_handler.go new file mode 100644 index 000000000..c6cb4c8a4 --- /dev/null +++ b/providers/apis/bitstamp/api_handler.go @@ -0,0 +1,120 @@ +package bitstamp + +import ( + "encoding/json" + "fmt" + "net/http" + "time" + + providertypes "github.com/skip-mev/slinky/providers/types" + + "github.com/skip-mev/slinky/pkg/math" + + "github.com/skip-mev/slinky/oracle/config" + "github.com/skip-mev/slinky/oracle/types" +) + +var _ types.PriceAPIDataHandler = (*APIHandler)(nil) + +// APIHandler implements the PriceAPIDataHandler interface for Bitstamp. +// for more information about the Bitstamp API, refer to the following link: +// https://www.bitstamp.net/api/#section/What-is-API +type APIHandler struct { + // api is the config for the Bitstamp API. + api config.APIConfig + // cache maintains the latest set of tickers seen by the handler. + cache types.ProviderTickers +} + +// NewAPIHandler returns a new Bitstamp PriceAPIDataHandler. +func NewAPIHandler( + api config.APIConfig, +) (types.PriceAPIDataHandler, error) { + if api.Name != Name { + return nil, fmt.Errorf("expected api config name %s, got %s", Name, api.Name) + } + + if !api.Enabled { + return nil, fmt.Errorf("api config for %s is not enabled", Name) + } + + if err := api.ValidateBasic(); err != nil { + return nil, fmt.Errorf("invalid api config for %s: %w", Name, err) + } + + return &APIHandler{ + api: api, + cache: types.NewProviderTickers(), + }, nil +} + +// CreateURL returns the URL that is used to fetch data from the Bitstamp API for the +// given tickers. +func (h *APIHandler) CreateURL( + tickers []types.ProviderTicker, +) (string, error) { + if len(tickers) == 0 { + return "", fmt.Errorf("no tickers provided") + } + + for _, ticker := range tickers { + h.cache.Add(ticker) + } + + return h.api.Endpoints[0].URL, nil +} + +// ParseResponse parses the response from the Bitstamp API and returns a GetResponse. Each +// of the tickers supplied will get a response or an error. Note that the bitstamp API +// returns all prices, some of which are not needed. The response is filtered to only +// include the prices that are needed (i.e. the pairs that are in the cache). +func (h *APIHandler) ParseResponse( + tickers []types.ProviderTicker, + resp *http.Response, +) types.PriceResponse { + var result MarketTickerResponse + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return types.NewPriceResponseWithErr( + tickers, + providertypes.NewErrorWithCode(err, providertypes.ErrorFailedToDecode), + ) + } + + var ( + resolved = make(types.ResolvedPrices) + unresolved = make(types.UnResolvedPrices) + ) + + for _, feed := range result { + // Filter out the responses that are not expected. + ticker, ok := h.cache.FromOffChainTicker(feed.Pair) + if !ok { + continue + } + + price, err := math.Float64StringToBigFloat(feed.Last) + if err != nil { + wErr := fmt.Errorf("failed to convert price %s to big.Float: %w", feed.Last, err) + unresolved[ticker] = providertypes.UnresolvedResult{ + ErrorWithCode: providertypes.NewErrorWithCode(wErr, providertypes.ErrorFailedToParsePrice), + } + continue + } + + resolved[ticker] = types.NewPriceResult(price, time.Now().UTC()) + } + + // Add currency pairs that received no response to the unresolved map. + for _, ticker := range tickers { + _, resolvedOk := resolved[ticker] + _, unresolvedOk := unresolved[ticker] + + if !resolvedOk && !unresolvedOk { + unresolved[ticker] = providertypes.UnresolvedResult{ + ErrorWithCode: providertypes.NewErrorWithCode(fmt.Errorf("no response"), providertypes.ErrorNoResponse), + } + } + } + + return types.NewPriceResponse(resolved, unresolved) +} diff --git a/providers/apis/bitstamp/api_handler_test.go b/providers/apis/bitstamp/api_handler_test.go new file mode 100644 index 000000000..d915eedcb --- /dev/null +++ b/providers/apis/bitstamp/api_handler_test.go @@ -0,0 +1,273 @@ +package bitstamp_test + +import ( + "fmt" + "math/big" + "net/http" + "testing" + "time" + + "github.com/skip-mev/slinky/oracle/types" + "github.com/skip-mev/slinky/providers/apis/bitstamp" + "github.com/skip-mev/slinky/providers/base/testutils" + providertypes "github.com/skip-mev/slinky/providers/types" + "github.com/stretchr/testify/require" +) + +var ( + btcusd = types.DefaultProviderTicker{ + OffChainTicker: "BTC/USD", + } + ethusd = types.DefaultProviderTicker{ + OffChainTicker: "ETH/USD", + } +) + +func TestCreateURL(t *testing.T) { + testCases := []struct { + name string + cps []types.ProviderTicker + url string + expectedErr bool + }{ + { + name: "empty", + cps: []types.ProviderTicker{}, + url: "", + expectedErr: true, + }, + { + name: "valid single", + cps: []types.ProviderTicker{ + btcusd, + }, + url: bitstamp.DefaultAPIConfig.Endpoints[0].URL, + expectedErr: false, + }, + { + name: "valid multiple", + cps: []types.ProviderTicker{ + btcusd, + ethusd, + }, + url: bitstamp.DefaultAPIConfig.Endpoints[0].URL, + expectedErr: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + h, err := bitstamp.NewAPIHandler(bitstamp.DefaultAPIConfig) + require.NoError(t, err) + + url, err := h.CreateURL(tc.cps) + if tc.expectedErr { + require.Error(t, err) + } else { + require.NoError(t, err) + require.Equal(t, tc.url, url) + } + }) + } +} + +func TestParseResponse(t *testing.T) { + testCases := []struct { + name string + cps []types.ProviderTicker + response *http.Response + expected types.PriceResponse + }{ + { + name: "valid single", + cps: []types.ProviderTicker{ + btcusd, + }, + response: testutils.CreateResponseFromJSON( + ` +[ + { + "ask": "2211.00", + "bid": "2188.97", + "high": "2811.00", + "last": "2211.00", + "low": "2188.97", + "open": "2211.00", + "open_24": "2211.00", + "pair": "BTC/USD", + "percent_change_24": "13.57", + "side": "0", + "timestamp": "1643640186", + "volume": "213.26801100", + "vwap": "2189.80" + } +] + `, + ), + expected: types.NewPriceResponse( + types.ResolvedPrices{ + btcusd: { + Value: big.NewFloat(2211.00), + }, + }, + types.UnResolvedPrices{}, + ), + }, + { + name: "valid multiple", + cps: []types.ProviderTicker{ + btcusd, + ethusd, + }, + response: testutils.CreateResponseFromJSON( + ` +[ + { + "ask": "2211.00", + "bid": "2188.97", + "high": "2811.00", + "last": "2211.00", + "low": "2188.97", + "open": "2211.00", + "open_24": "2211.00", + "pair": "BTC/USD", + "percent_change_24": "13.57", + "side": "0", + "timestamp": "1643640186", + "volume": "213.26801100", + "vwap": "2189.80" + }, + { + "ask": "2211.00", + "bid": "2188.97", + "high": "2811.00", + "last": "420.69", + "low": "2188.97", + "open": "2211.00", + "open_24": "2211.00", + "pair": "ETH/USD", + "percent_change_24": "13.57", + "side": "0", + "timestamp": "1643640186", + "volume": "213.26801100", + "vwap": "2189.80" + } +] + `, + ), + expected: types.NewPriceResponse( + types.ResolvedPrices{ + btcusd: { + Value: big.NewFloat(2211.00), + }, + ethusd: { + Value: big.NewFloat(420.69), + }, + }, + types.UnResolvedPrices{}, + ), + }, + { + name: "bad response", + cps: []types.ProviderTicker{ + btcusd, + }, + response: testutils.CreateResponseFromJSON( + `shout out my label that's me`, + ), + expected: types.NewPriceResponse( + types.ResolvedPrices{}, + types.UnResolvedPrices{ + btcusd: providertypes.UnresolvedResult{ + ErrorWithCode: providertypes.NewErrorWithCode(fmt.Errorf("no response"), providertypes.ErrorAPIGeneral), + }, + }, + ), + }, + { + name: "bad price response", + cps: []types.ProviderTicker{ + btcusd, + }, + response: testutils.CreateResponseFromJSON( + ` +[ + { + "ask": "2211.00", + "bid": "2188.97", + "high": "2811.00", + "last": "$2211.00", + "low": "2188.97", + "open": "2211.00", + "open_24": "2211.00", + "pair": "BTC/USD", + "percent_change_24": "13.57", + "side": "0", + "timestamp": "1643640186", + "volume": "213.26801100", + "vwap": "2189.80" + }, +] + `, + ), + expected: types.NewPriceResponse( + types.ResolvedPrices{}, + types.UnResolvedPrices{ + btcusd: providertypes.UnresolvedResult{ + ErrorWithCode: providertypes.NewErrorWithCode(fmt.Errorf("invalid syntax"), providertypes.ErrorAPIGeneral), + }, + }, + ), + }, + { + name: "no response", + cps: []types.ProviderTicker{ + btcusd, + ethusd, + }, + response: testutils.CreateResponseFromJSON( + `[]`, + ), + expected: types.NewPriceResponse( + types.ResolvedPrices{}, + types.UnResolvedPrices{ + btcusd: providertypes.UnresolvedResult{ + ErrorWithCode: providertypes.NewErrorWithCode(fmt.Errorf("no response"), providertypes.ErrorAPIGeneral), + }, + ethusd: providertypes.UnresolvedResult{ + ErrorWithCode: providertypes.NewErrorWithCode(fmt.Errorf("no response"), providertypes.ErrorAPIGeneral), + }, + }, + ), + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + h, err := bitstamp.NewAPIHandler(bitstamp.DefaultAPIConfig) + require.NoError(t, err) + + // Update the cache since it is assumed that createURL is executed before ParseResponse. + _, err = h.CreateURL(tc.cps) + require.NoError(t, err) + + now := time.Now() + resp := h.ParseResponse(tc.cps, tc.response) + + require.Len(t, resp.Resolved, len(tc.expected.Resolved)) + require.Len(t, resp.UnResolved, len(tc.expected.UnResolved)) + + for cp, result := range tc.expected.Resolved { + require.Contains(t, resp.Resolved, cp) + r := resp.Resolved[cp] + require.Equal(t, result.Value.SetPrec(18), r.Value.SetPrec(18)) + require.True(t, r.Timestamp.After(now)) + } + + for cp := range tc.expected.UnResolved { + require.Contains(t, resp.UnResolved, cp) + require.Error(t, resp.UnResolved[cp]) + } + }) + } +} diff --git a/providers/apis/bitstamp/utils.go b/providers/apis/bitstamp/utils.go new file mode 100644 index 000000000..d493ad3e5 --- /dev/null +++ b/providers/apis/bitstamp/utils.go @@ -0,0 +1,65 @@ +package bitstamp + +import ( + "time" + + "github.com/skip-mev/slinky/oracle/config" +) + +// NOTE: All documentation for this file can be located on the Bitstamp GitHub +// API documentation: https://www.bitstamp.net/api/v2/ticker/. This +// API does not require a subscription to use (i.e. No API key is required). + +const ( + // Name is the name of the Bitstamp provider. + Name = "bitstamp_api" + + // URL is the base URL of the Bitstamp API. The URL returns all prices, some + // of which are not needed. + URL = "https://www.bitstamp.net/api/v2/ticker/" +) + +// DefaultAPIConfig is the default configuration for the Bitstamp API. +var DefaultAPIConfig = config.APIConfig{ + Name: Name, + Atomic: true, + Enabled: true, + Timeout: 3000 * time.Millisecond, + Interval: 3000 * time.Millisecond, + ReconnectTimeout: 2000 * time.Millisecond, + MaxQueries: 1, + Endpoints: []config.Endpoint{{URL: URL}}, +} + +// MarketTickerResponse is the expected response returned by the Bitstamp API. +// +// ex. +// +// [ +// +// { +// "ask": "2211.00", +// "bid": "2188.97", +// "high": "2811.00", +// "last": "2211.00", +// "low": "2188.97", +// "open": "2211.00", +// "open_24": "2211.00", +// "pair": "BTC/USD", +// "percent_change_24": "13.57", +// "side": "0", +// "timestamp": "1643640186", +// "volume": "213.26801100", +// "vwap": "2189.80" +// } +// +// ] +// +// ref: https://www.bitstamp.net/api/v2/ticker/ +type MarketTickerResponse []MarketTickerData + +// MarketTickerData is the data returned by the Bitstamp API. +type MarketTickerData struct { + Last string `json:"last"` + Pair string `json:"pair"` +} diff --git a/providers/apis/dydx/parse.go b/providers/apis/dydx/parse.go index 6f10b8813..b153652b0 100644 --- a/providers/apis/dydx/parse.go +++ b/providers/apis/dydx/parse.go @@ -9,6 +9,7 @@ import ( "github.com/skip-mev/slinky/oracle/constants" slinkytypes "github.com/skip-mev/slinky/pkg/types" + "github.com/skip-mev/slinky/providers/apis/bitstamp" "github.com/skip-mev/slinky/providers/apis/coinmarketcap" "github.com/skip-mev/slinky/providers/apis/defi/raydium" "github.com/skip-mev/slinky/providers/apis/defi/uniswapv3" @@ -17,7 +18,6 @@ import ( "github.com/skip-mev/slinky/providers/volatile" "github.com/skip-mev/slinky/providers/websockets/binance" "github.com/skip-mev/slinky/providers/websockets/bitfinex" - "github.com/skip-mev/slinky/providers/websockets/bitstamp" "github.com/skip-mev/slinky/providers/websockets/bybit" "github.com/skip-mev/slinky/providers/websockets/coinbase" "github.com/skip-mev/slinky/providers/websockets/cryptodotcom" @@ -231,10 +231,6 @@ func ConvertDenomByProvider(denom string, exchange string) (string, error) { } return denom, nil - case exchange == bitstamp.Name: - if strings.Contains(denom, "/") { - return strings.ToLower(strings.ReplaceAll(denom, "/", "")), nil - } case exchange == raydium.Name: // split the ticker by /, and expect there to at least be two values fields := strings.Split(denom, RaydiumTickerSeparator) @@ -246,5 +242,4 @@ func ConvertDenomByProvider(denom string, exchange string) (string, error) { default: return denom, nil } - return "", nil } diff --git a/providers/factories/oracle/api.go b/providers/factories/oracle/api.go index 8f3dab0c6..953ed9607 100644 --- a/providers/factories/oracle/api.go +++ b/providers/factories/oracle/api.go @@ -6,6 +6,7 @@ import ( "net/http" "strings" + "github.com/skip-mev/slinky/providers/apis/bitstamp" "github.com/skip-mev/slinky/providers/apis/defi/raydium" "go.uber.org/zap" @@ -70,6 +71,8 @@ func APIQueryHandlerFactory( switch providerName := cfg.Name; { case providerName == binance.Name: apiDataHandler, err = binance.NewAPIHandler(cfg.API) + case providerName == bitstamp.Name: + apiDataHandler, err = bitstamp.NewAPIHandler(cfg.API) case providerName == coinbaseapi.Name: apiDataHandler, err = coinbaseapi.NewAPIHandler(cfg.API) case providerName == coingecko.Name: