some improvements on univ4 scraper suite.

Author: jppade

cmd/assetCollectionService/go.mod

@@ -3,10 +3,10 @@ module github.com/diadata-org/diadata/assetCollectionService
 go 1.22
 
 require (
-	github.com/diadata-org/diadata v1.4.573
+	github.com/diadata-org/diadata v1.4.575
 	github.com/sirupsen/logrus v1.9.3
 )
-replace github.com/diadata-org/diadata => /home/philipp/go/src/github.com/diadata-org/diadata
+
 require (
 	filippo.io/edwards25519 v1.0.0 // indirect
 	github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 // indirect

cmd/exchange-scrapers/collector/collector.go

@@ -48,6 +48,7 @@ var (
 		dia.BifrostExchange,
 		dia.HydrationExchange,
 		dia.UniswapExchangeV3Celo,
+		dia.UniswapExchangeV4,
 	}
 
 	exchange = flag.String("exchange", "", "which exchange")

cmd/exchange-scrapers/collector/go.mod

@@ -3,11 +3,11 @@ module github.com/diadata-org/diadata/exchange-scrapers/collector
 go 1.22
 
 require (
-	github.com/diadata-org/diadata v1.4.573
+	github.com/diadata-org/diadata v1.4.575
 	github.com/segmentio/kafka-go v0.4.35
 	github.com/sirupsen/logrus v1.9.3
 )
-replace github.com/diadata-org/diadata => /home/philipp/go/src/github.com/diadata-org/diadata
+
 require (
 	filippo.io/edwards25519 v1.0.0 // indirect
 	github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 // indirect

cmd/liquidityScraper/go.mod

@@ -3,10 +3,10 @@ module github.com/diadata-org/diadata/liquidityScraper
 go 1.22
 
 require (
-	github.com/diadata-org/diadata v1.4.573
+	github.com/diadata-org/diadata v1.4.575
 	github.com/sirupsen/logrus v1.9.3
 )
-replace github.com/diadata-org/diadata => /home/philipp/go/src/github.com/diadata-org/diadata
+
 require (
 	filippo.io/edwards25519 v1.0.0 // indirect
 	github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 // indirect

pkg/dia/scraper/exchange-scrapers/UniswapV4Scraper.go

@@ -52,53 +52,30 @@ type UniswapV4Scraper struct {
 	exchangeName           string
 	startBlock             uint64
 	waitTime               int
-	listenByAddress        bool
 	chanTrades             chan *dia.Trade
 	factoryContractAddress common.Address
+	thresholdSlippage      float64
 }
 
-var (
-	fullPoolsUniswapV4 *[]string
-)
-
 // NewUniswapV4Scraper returns a new UniswapV4Scraper
 func NewUniswapV4Scraper(exchange dia.Exchange, scrape bool, relDB *models.RelDB) *UniswapV4Scraper {
-	log.Info("NewUniswapScraper ", exchange.Name)
-	log.Info("NewUniswapScraper Address ", exchange.Contract)
+	log.Info("NewUniswapV4Scraper ", exchange.Name)
+	log.Info("NewUniswapV4Scraper Address ", exchange.Contract)
 
 	var (
-		s               *UniswapV4Scraper
-		listenByAddress bool
-		err             error
+		s   *UniswapV4Scraper
+		err error
 	)
 
-	listenByAddress, err = strconv.ParseBool(utils.Getenv("LISTEN_BY_ADDRESS", "false"))
-	if err != nil {
-		log.Fatal("parse LISTEN_BY_ADDRESS: ", err)
-	}
-
 	switch exchange.Name {
 	case dia.UniswapExchangeV4:
-		s = makeUniswapV4Scraper(exchange, listenByAddress, "", "", "200", uint64(12369621))
+		s = makeUniswapV4Scraper(exchange, "", "", "200", uint64(12369621))
 
 	}
 
 	s.relDB = relDB
 	s.poolMap = make(map[[32]byte]dia.Pool)
 
-	// Only include pools with (minimum) liquidity bigger than given env var.
-	liquidityThreshold, err := strconv.ParseFloat(utils.Getenv("LIQUIDITY_THRESHOLD", "0"), 64)
-	if err != nil {
-		liquidityThreshold = float64(0)
-		log.Warnf("parse liquidity threshold:  %v. Set to default %v", err, liquidityThreshold)
-	}
-	// Only include pools with (minimum) liquidity USD value bigger than given env var.
-	liquidityThresholdUSD, err := strconv.ParseFloat(utils.Getenv("LIQUIDITY_THRESHOLD_USD", "0"), 64)
-	if err != nil {
-		liquidityThresholdUSD = float64(0)
-		log.Warnf("parse liquidity threshold:  %v. Set to default %v", err, liquidityThresholdUSD)
-	}
-
 	pingNodeInterval, err := strconv.ParseInt(utils.Getenv("PING_SERVER", "0"), 10, 64)
 	if err != nil {
 		log.Error("parse PING_SERVER: ", err)
@@ -114,7 +91,7 @@ func NewUniswapV4Scraper(exchange dia.Exchange, scrape bool, relDB *models.RelDB
 }
 
 // makeUniswapV4Scraper returns a uniswap scraper as used in NewUniswapV4Scraper.
-func makeUniswapV4Scraper(exchange dia.Exchange, listenByAddress bool, restDial string, wsDial string, waitMilliseconds string, startBlock uint64) *UniswapV4Scraper {
+func makeUniswapV4Scraper(exchange dia.Exchange, restDial string, wsDial string, waitMilliseconds string, startBlock uint64) *UniswapV4Scraper {
 	var restClient, wsClient *ethclient.Client
 	var err error
 	var s *UniswapV4Scraper
@@ -148,33 +125,22 @@ func makeUniswapV4Scraper(exchange dia.Exchange, listenByAddress bool, restDial
 		error:                  nil,
 		chanTrades:             make(chan *dia.Trade),
 		waitTime:               waitTime,
-		listenByAddress:        listenByAddress,
 		startBlock:             startBlock,
 		factoryContractAddress: common.HexToAddress(exchange.Contract),
 	}
+
+	s.thresholdSlippage, err = strconv.ParseFloat(utils.Getenv(strings.ToUpper(s.exchangeName)+"_THRESHOLD_SLIPPAGE", "0.005"), 64)
+	if err != nil {
+		log.Error("Parse THRESHOLD_SLIPPAGE: ", err)
+		s.thresholdSlippage = 0.001
+	}
+
 	return s
 }
 
 // runs in a goroutine until s is closed
 func (s *UniswapV4Scraper) mainLoop() {
 
-	var err error
-	reverseBasetokens, err = getReverseTokensFromConfig("uniswapv4/reverse_tokens/" + s.exchangeName + "Basetoken")
-	if err != nil {
-		log.Error("error getting basetokens for which pairs should be reversed: ", err)
-	}
-	log.Infof("reverse the following basetokens on %s: %v", s.exchangeName, reverseBasetokens)
-	reverseQuotetokens, err = getReverseTokensFromConfig("uniswapv4/reverse_tokens/" + s.exchangeName + "Quotetoken")
-	if err != nil {
-		log.Error("error getting quotetokens for which pairs should be reversed: ", err)
-	}
-	log.Infof("reverse the following quotetokens on %s: %v", s.exchangeName, reverseQuotetokens)
-	fullPoolsUniswapV4, err = getReverseTokensFromConfig("uniswapv4/fullPools/" + s.exchangeName + "FullPools")
-	if err != nil {
-		log.Error("error getting fullPools for which pairs should be reversed: ", err)
-	}
-	log.Infof("Take into account both directions of a trade on the following pools: %v", fullPoolsUniswapV4)
-
 	time.Sleep(4 * time.Second)
 	s.run = true
 
@@ -193,12 +159,19 @@ func (s *UniswapV4Scraper) mainLoop() {
 			rawSwap, ok := <-sink
 			if ok {
 
-				// TO DO: Can we use liquidity field in order to assess admissibility of a trade?
+				slippage := computeSlippage(rawSwap.SqrtPriceX96, rawSwap.Amount0, rawSwap.Amount1, rawSwap.Liquidity)
+				log.Infof("slippage: %v", slippage)
+
 				swap, err := s.normalizeRawSwap(rawSwap)
 				if err != nil {
 					log.Error("normalizeRawSwap: ", err)
 					continue
 				}
+				if slippage > s.thresholdSlippage {
+					log.Warn("slippage above threshold: ", slippage)
+					continue
+				}
+
 				s.sendTrade(swap, hex.EncodeToString(rawSwap.Id[:]))
 
 			}
@@ -238,30 +211,6 @@ func (s *UniswapV4Scraper) sendTrade(swap UniswapV4Swap, poolID string) {
 		VerifiedPair:   true,
 	}
 
-	// switch {
-	// case utils.Contains(reverseBasetokens, pool.Token1.Address.Hex()):
-	// 	// If we need quotation of a base token, reverse pair
-	// 	tSwapped, err := dia.SwapTrade(*t)
-	// 	if err == nil {
-	// 		t = &tSwapped
-	// 	}
-	// case utils.Contains(reverseQuotetokens, pool.Token0.Address.Hex()):
-	// 	// If we need quotation of a base token, reverse pair
-	// 	tSwapped, err := dia.SwapTrade(*t)
-	// 	if err == nil {
-	// 		t = &tSwapped
-	// 	}
-	// }
-
-	// if utils.Contains(fullPoolsUniswapV4, pool.Address.Hex()) {
-	// 	tSwapped, err := dia.SwapTrade(*t)
-	// 	if err == nil {
-	// 		if tSwapped.Price > 0 {
-	// 			s.chanTrades <- &tSwapped
-	// 		}
-	// 	}
-	// }
-
 	if price > 0 {
 		log.Infof("Got trade on pair %s: %v", t.Pair, t)
 		log.Info("------")
@@ -281,44 +230,25 @@ func (s *UniswapV4Scraper) normalizeRawSwap(rawSwap *uniswapcontractv4.Poolmanag
 	pool, ok := s.poolMap[rawSwap.Id]
 	if !ok {
 		pool, err = s.relDB.GetPoolByAddress(dia.ETHEREUM, hex.EncodeToString(rawSwap.Id[:]))
-		if err != nil || len(pool.Assetvolumes) != 2 {
+		if err != nil {
 			return
 		}
-		if len(pool.Assetvolumes) == 2 {
-			s.poolMap[rawSwap.Id] = pool
+		if len(pool.Assetvolumes) != 2 {
+			err = errors.New("not enough assets in pool")
+			return
 		}
+		s.poolMap[rawSwap.Id] = pool
 	}
 
 	asset0 := pool.Assetvolumes[pool.Assetvolumes[0].Index].Asset
 	asset1 := pool.Assetvolumes[pool.Assetvolumes[1].Index].Asset
-	log.Infof("%s-%s", asset0.Symbol, asset1.Symbol)
 	decimals0 := int(asset0.Decimals)
 	decimals1 := int(asset1.Decimals)
 	amount0Big := new(big.Float).Quo(big.NewFloat(0).SetInt(rawSwap.Amount0), new(big.Float).SetFloat64(math.Pow10(decimals0)))
 	amount1Big := new(big.Float).Quo(big.NewFloat(0).SetInt(rawSwap.Amount1), new(big.Float).SetFloat64(math.Pow10(decimals1)))
 	amount0, _ := amount0Big.Float64()
 	amount1, _ := amount1Big.Float64()
 
-	// slippage0, _ := new(big.Float).Quo(amount0Big, big.NewFloat(0).SetInt(rawSwap.Liquidity)).Float64()
-	// slippage1, _ := new(big.Float).Quo(amount1Big, big.NewFloat(0).SetInt(rawSwap.Liquidity)).Float64()
-	// normalizedLiquidity, _ := new(big.Float).Quo(big.NewFloat(0).SetInt(rawSwap.Liquidity), new(big.Float).SetFloat64(math.Pow10(18))).Float64()
-	// log.Infof("slippage0 -- slippage1 -- liquidity: %v -- %v -- %v", slippage0, slippage1, normalizedLiquidity)
-
-	var pairTicker string
-	if amount0 < 0 {
-		pairTicker = asset1.Symbol + "-" + asset0.Symbol
-	} else {
-		pairTicker = asset0.Symbol + "-" + asset1.Symbol
-	}
-	slippage := computeSlippage(rawSwap.SqrtPriceX96, rawSwap.Amount0, rawSwap.Amount1, rawSwap.Liquidity)
-	if slippage != nil {
-		log.Infof(
-			"%s -- slippage: %s",
-			pairTicker,
-			slippage.String(),
-		)
-	}
-
 	normalizedSwap = UniswapV4Swap{
 		ID:        rawSwap.Raw.TxHash.Hex(),
 		Timestamp: time.Now().Unix(),
@@ -424,22 +354,24 @@ func (ps *UniswapPairV4Scraper) Pair() dia.ExchangePair {
 	return ps.pair
 }
 
-func computeSlippage(sqrtPriceX96 *big.Int, amount0 *big.Int, amount1 *big.Int, liquidity *big.Int) (slippage *big.Float) {
+func computeSlippage(sqrtPriceX96 *big.Int, amount0 *big.Int, amount1 *big.Int, liquidity *big.Int) (slippage float64) {
 
 	price := new(big.Float).Quo(big.NewFloat(0).SetInt(sqrtPriceX96), new(big.Float).SetFloat64(math.Pow(2, 96)))
 
 	if amount0.Sign() < 0 {
 		// token0 -> token1
 		amount0Abs := big.NewInt(0).Abs(amount0)
 		numerator := big.NewFloat(0).Mul(big.NewFloat(0).SetInt(amount0Abs), price)
-		return new(big.Float).Quo(numerator, big.NewFloat(0).SetInt(liquidity))
+		slippage, _ = new(big.Float).Quo(numerator, big.NewFloat(0).SetInt(liquidity)).Float64()
+		return
 	} else if amount1.Sign() < 0 {
 		// token1 -> token0
 		numerator := big.NewFloat(0).SetInt(big.NewInt(0).Abs(amount1))
 		denominator := big.NewFloat(0).Mul(big.NewFloat(0).SetInt(liquidity), price)
-		return new(big.Float).Quo(numerator, denominator)
+		slippage, _ = new(big.Float).Quo(numerator, denominator).Float64()
+		return
 	}
 	log.Infof("sqrtPrice -- amount0 -- amount1 -- liquidity: %s -- %s -- %s -- %s", sqrtPriceX96.String(), amount0.String(), amount1.String(), liquidity.String())
-	return nil
+	return 0
 
 }