Upgrade libp2p (enables go 1.21 support)

This change is primarily intended to support go 1.21, but as a
side-effect requires updating libp2p, which in turn triggers an update
of golang.org/x/exp which creates quite a bit of (simple) churn in the
slice sorting.

This change introduces a new `cmp.Compare` function which can be used to
return an integer satisfying the compare interface for slice sorting.

In order to continue to support mplex for libp2p, the change references
github.com/libp2p/go-libp2p-mplex instead.  Please see the PR at
libp2p/go-libp2p#2498 for the official usptream
comment that indicates official support for mplex being moved to this
location.

.github/workflows/ci.yml

@@ -32,7 +32,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-go@v4
         with:
-          go-version: '1.19'
+          go-version: '1.20'
       - name: Install dependencies on Linux
         if: runner.os == 'Linux'
         run: sudo apt update && sudo apt install build-essential
@@ -84,7 +84,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-go@v4
         with:
-          go-version: '1.19'
+          go-version: '1.20'
 
       - uses: actions/cache@v3
         with:

cmd/sentinel/sentinel/config.go

@@ -21,8 +21,8 @@ import (
 	"github.com/ledgerwatch/erigon/cl/clparams"
 	"github.com/ledgerwatch/log/v3"
 	"github.com/libp2p/go-libp2p"
+	mplex "github.com/libp2p/go-libp2p-mplex"
 	"github.com/libp2p/go-libp2p/core/crypto"
-	"github.com/libp2p/go-libp2p/p2p/muxer/mplex"
 	"github.com/libp2p/go-libp2p/p2p/security/noise"
 	"github.com/libp2p/go-libp2p/p2p/transport/tcp"
 	"github.com/multiformats/go-multiaddr"

cmd/silkworm_api/snapshot_idx.go

@@ -2,6 +2,10 @@ package main
 
 import (
 	"fmt"
+	"os"
+	"path/filepath"
+	"time"
+
 	"github.com/ledgerwatch/erigon-lib/common/background"
 	"github.com/ledgerwatch/erigon-lib/common/datadir"
 	"github.com/ledgerwatch/erigon-lib/downloader/snaptype"
@@ -12,9 +16,6 @@ import (
 	"github.com/ledgerwatch/log/v3"
 	"github.com/urfave/cli/v2"
 	"golang.org/x/sync/errgroup"
-	"os"
-	"path/filepath"
-	"time"
 )
 
 // Build snapshot indexes for given snapshot files.
@@ -55,7 +56,7 @@ func FindIf(segments []snaptype.FileInfo, predicate func(snaptype.FileInfo) bool
 }
 
 func buildIndex(cliCtx *cli.Context, dataDir string, snapshotPaths []string) error {
-	logger, err := debug.Setup(cliCtx, true /* rootLogger */)
+	logger, _, err := debug.Setup(cliCtx, true /* rootLogger */)
 	if err != nil {
 		return err
 	}

core/blockchain.go

@@ -27,6 +27,7 @@ import (
 
 	"github.com/ledgerwatch/erigon-lib/chain"
 	libcommon "github.com/ledgerwatch/erigon-lib/common"
+	"github.com/ledgerwatch/erigon-lib/common/cmp"
 	"github.com/ledgerwatch/erigon-lib/common/fixedgas"
 
 	"github.com/ledgerwatch/erigon/common/math"
@@ -184,7 +185,7 @@ func ExecuteBlockEphemerally(
 
 		stateSyncReceipt := &types.Receipt{}
 		if chainConfig.Consensus == chain.BorConsensus && len(blockLogs) > 0 {
-			slices.SortStableFunc(blockLogs, func(i, j *types.Log) bool { return i.Index < j.Index })
+			slices.SortStableFunc(blockLogs, func(i, j *types.Log) int { return cmp.Compare(i.Index, j.Index) })
 
 			if len(blockLogs) > len(logs) {
 				stateSyncReceipt.Logs = blockLogs[len(logs):] // get state-sync logs from `state.Logs()`

erigon-lib/common/cmp/cmp.go

@@ -44,3 +44,14 @@ func Max[T constraints.Ordered](a, b T) T {
 	}
 	return b
 }
+
+func Compare[T constraints.Ordered](a, b T) int {
+	switch {
+	case a < b:
+		return -1
+	case a == b:
+		return 0
+	default:
+		return 1
+	}
+}

erigon-lib/compress/compress.go

@@ -33,6 +33,7 @@ import (
 
 	"github.com/c2h5oh/datasize"
 	"github.com/ledgerwatch/erigon-lib/common"
+	"github.com/ledgerwatch/erigon-lib/common/cmp"
 	dir2 "github.com/ledgerwatch/erigon-lib/common/dir"
 	"github.com/ledgerwatch/erigon-lib/etl"
 	"github.com/ledgerwatch/log/v3"
@@ -300,17 +301,17 @@ func (db *DictionaryBuilder) Less(i, j int) bool {
 	return db.items[i].score < db.items[j].score
 }
 
-func dictionaryBuilderLess(i, j *Pattern) bool {
+func dictionaryBuilderCmp(i, j *Pattern) int {
 	if i.score == j.score {
-		return bytes.Compare(i.word, j.word) < 0
+		return bytes.Compare(i.word, j.word)
 	}
-	return i.score < j.score
+	return cmp.Compare(i.score, j.score)
 }
 
 func (db *DictionaryBuilder) Swap(i, j int) {
 	db.items[i], db.items[j] = db.items[j], db.items[i]
 }
-func (db *DictionaryBuilder) Sort() { slices.SortFunc(db.items, dictionaryBuilderLess) }
+func (db *DictionaryBuilder) Sort() { slices.SortFunc(db.items, dictionaryBuilderCmp) }
 
 func (db *DictionaryBuilder) Push(x interface{}) {
 	db.items = append(db.items, x.(*Pattern))
@@ -383,11 +384,11 @@ type Pattern struct {
 type PatternList []*Pattern
 
 func (pl PatternList) Len() int { return len(pl) }
-func patternListLess(i, j *Pattern) bool {
+func patternListCmp(i, j *Pattern) int {
 	if i.uses == j.uses {
-		return bits.Reverse64(i.code) < bits.Reverse64(j.code)
+		return cmp.Compare(bits.Reverse64(i.code), bits.Reverse64(j.code))
 	}
-	return i.uses < j.uses
+	return cmp.Compare(i.uses, j.uses)
 }
 
 // PatternHuff is an intermediate node in a huffman tree of patterns
@@ -555,11 +556,11 @@ type PositionList []*Position
 
 func (pl PositionList) Len() int { return len(pl) }
 
-func positionListLess(i, j *Position) bool {
+func positionListCmp(i, j *Position) int {
 	if i.uses == j.uses {
-		return bits.Reverse64(i.code) < bits.Reverse64(j.code)
+		return cmp.Compare(bits.Reverse64(i.code), bits.Reverse64(j.code))
 	}
-	return i.uses < j.uses
+	return cmp.Compare(i.uses, j.uses)
 }
 
 type PositionHeap []*PositionHuff
@@ -569,10 +570,14 @@ func (ph PositionHeap) Len() int {
 }
 
 func (ph PositionHeap) Less(i, j int) bool {
+	return ph.Compare(i, j) < 0
+}
+
+func (ph PositionHeap) Compare(i, j int) int {
 	if ph[i].uses == ph[j].uses {
-		return ph[i].tieBreaker < ph[j].tieBreaker
+		return cmp.Compare(ph[i].tieBreaker, ph[j].tieBreaker)
 	}
-	return ph[i].uses < ph[j].uses
+	return cmp.Compare(ph[i].uses, ph[j].uses)
 }
 
 func (ph *PositionHeap) Swap(i, j int) {

erigon-lib/compress/parallel_compress.go

@@ -469,7 +469,7 @@ func reducedict(ctx context.Context, trace bool, logPrefix, segmentFilePath stri
 			distribution[len(p.word)]++
 		}
 	}
-	slices.SortFunc(patternList, patternListLess)
+	slices.SortFunc(patternList, patternListCmp)
 	logCtx := make([]interface{}, 0, 8)
 	logCtx = append(logCtx, "patternList.Len", patternList.Len())
 
@@ -551,7 +551,7 @@ func reducedict(ctx context.Context, trace bool, logPrefix, segmentFilePath stri
 	}
 	//fmt.Printf("patternsSize = %d\n", patternsSize)
 	// Write all the pattens
-	slices.SortFunc(patternList, patternListLess)
+	slices.SortFunc(patternList, patternListCmp)
 	for _, p := range patternList {
 		ns := binary.PutUvarint(numBuf[:], uint64(p.depth))
 		if _, err = cw.Write(numBuf[:ns]); err != nil {
@@ -574,7 +574,7 @@ func reducedict(ctx context.Context, trace bool, logPrefix, segmentFilePath stri
 		positionList = append(positionList, p)
 		pos2code[pos] = p
 	}
-	slices.SortFunc(positionList, positionListLess)
+	slices.SortFunc(positionList, positionListCmp)
 	i = 0
 	// Build Huffman tree for codes
 	var posHeap PositionHeap
@@ -632,7 +632,7 @@ func reducedict(ctx context.Context, trace bool, logPrefix, segmentFilePath stri
 	}
 	//fmt.Printf("posSize = %d\n", posSize)
 	// Write all the positions
-	slices.SortFunc(positionList, positionListLess)
+	slices.SortFunc(positionList, positionListCmp)
 	for _, p := range positionList {
 		ns := binary.PutUvarint(numBuf[:], uint64(p.depth))
 		if _, err = cw.Write(numBuf[:ns]); err != nil {

erigon-lib/downloader/snaptype/files.go

@@ -20,12 +20,14 @@ import (
 	"encoding/hex"
 	"errors"
 	"fmt"
-	"github.com/anacrolix/torrent/metainfo"
 	"os"
 	"path/filepath"
 	"strconv"
 	"strings"
 
+	"github.com/anacrolix/torrent/metainfo"
+
+	"github.com/ledgerwatch/erigon-lib/common/cmp"
 	"github.com/ledgerwatch/erigon-lib/common/dir"
 	"golang.org/x/exp/slices"
 )
@@ -209,20 +211,20 @@ func ParseDir(dir string) (res []FileInfo, err error) {
 		}
 		res = append(res, meta)
 	}
-	slices.SortFunc(res, func(i, j FileInfo) bool {
+	slices.SortFunc(res, func(i, j FileInfo) int {
 		if i.Version != j.Version {
-			return i.Version < j.Version
+			return cmp.Compare(i.Version, j.Version)
 		}
 		if i.From != j.From {
-			return i.From < j.From
+			return cmp.Compare(i.From, j.From)
 		}
 		if i.To != j.To {
-			return i.To < j.To
+			return cmp.Compare(i.To, j.To)
 		}
 		if i.T != j.T {
-			return i.T < j.T
+			return cmp.Compare(i.T, j.T)
 		}
-		return i.Ext < j.Ext
+		return cmp.Compare(i.Ext, j.Ext)
 	})
 
 	return res, nil

erigon-lib/go.mod

@@ -33,7 +33,7 @@ require (
 	github.com/stretchr/testify v1.8.4
 	github.com/tidwall/btree v1.6.0
 	golang.org/x/crypto v0.13.0
-	golang.org/x/exp v0.0.0-20230711023510-fffb14384f22
+	golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63
 	golang.org/x/sync v0.3.0
 	golang.org/x/sys v0.12.0
 	golang.org/x/time v0.3.0
@@ -104,10 +104,10 @@ require (
 	go.etcd.io/bbolt v1.3.6 // indirect
 	go.opentelemetry.io/otel v1.8.0 // indirect
 	go.opentelemetry.io/otel/trace v1.8.0 // indirect
-	golang.org/x/mod v0.11.0 // indirect
-	golang.org/x/net v0.12.0 // indirect
+	golang.org/x/mod v0.12.0 // indirect
+	golang.org/x/net v0.14.0 // indirect
 	golang.org/x/text v0.13.0 // indirect
-	golang.org/x/tools v0.7.0 // indirect
+	golang.org/x/tools v0.12.1-0.20230815132531-74c255bcf846 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	modernc.org/libc v1.22.3 // indirect

erigon-lib/go.sum

@@ -425,8 +425,8 @@ golang.org/x/crypto v0.5.0/go.mod h1:NK/OQwhpMQP3MwtdjgLlYHnH9ebylxKWv3e0fK+mkQU
 golang.org/x/crypto v0.13.0 h1:mvySKfSWJ+UKUii46M40LOvyWfN0s2U+46/jDd0e6Ck=
 golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
-golang.org/x/exp v0.0.0-20230711023510-fffb14384f22 h1:FqrVOBQxQ8r/UwwXibI0KMolVhvFiGobSfdE33deHJM=
-golang.org/x/exp v0.0.0-20230711023510-fffb14384f22/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc=
+golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63 h1:m64FZMko/V45gv0bNmrNYoDEq8U5YUhetc9cBWKS1TQ=
+golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63/go.mod h1:0v4NqG35kSWCMzLaMeX+IQrlSnVE/bqGSyC2cz/9Le8=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
 golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
@@ -435,8 +435,8 @@ golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHl
 golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
-golang.org/x/mod v0.11.0 h1:bUO06HqtnRcc/7l71XBe4WcqTZ+3AH1J59zWDDwLKgU=
-golang.org/x/mod v0.11.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc=
+golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -465,8 +465,8 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug
 golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
 golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws=
 golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
-golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50=
-golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA=
+golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14=
+golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -545,8 +545,8 @@ golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roY
 golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
-golang.org/x/tools v0.7.0 h1:W4OVu8VVOaIO0yzWMNdepAulS7YfoS3Zabrm8DOXXU4=
-golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s=
+golang.org/x/tools v0.12.1-0.20230815132531-74c255bcf846 h1:Vve/L0v7CXXuxUmaMGIEK/dEeq7uiqb5qBgQrZzIE7E=
+golang.org/x/tools v0.12.1-0.20230815132531-74c255bcf846/go.mod h1:Sc0INKfu04TlqNoRA1hgpFZbhYXHPr4V5DzpSBTPqQM=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

erigon-lib/gointerfaces/remote/sort.go

@@ -6,9 +6,9 @@ import (
 	"github.com/ledgerwatch/erigon-lib/gointerfaces/types"
 )
 
-func NodeInfoReplyLess(i, j *types.NodeInfoReply) bool {
+func NodeInfoReplyCmp(i, j *types.NodeInfoReply) int {
 	if cmp := strings.Compare(i.Name, j.Name); cmp != 0 {
-		return cmp == -1
+		return cmp
 	}
-	return strings.Compare(i.Enode, j.Enode) == -1
+	return strings.Compare(i.Enode, j.Enode)
 }

erigon-lib/gointerfaces/remote/sort_test.go

@@ -49,7 +49,7 @@ func TestSort(t *testing.T) {
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 
-			slices.SortFunc(tt.got.NodesInfo, remote.NodeInfoReplyLess)
+			slices.SortFunc(tt.got.NodesInfo, remote.NodeInfoReplyCmp)
 			assert.Equal(t, tt.want, tt.got)
 		})
 	}

erigon-lib/patricia/patricia.go

@@ -21,6 +21,7 @@ import (
 	"math/bits"
 	"strings"
 
+	"github.com/ledgerwatch/erigon-lib/common/cmp"
 	"github.com/ledgerwatch/erigon-lib/sais"
 	"golang.org/x/exp/slices"
 )
@@ -699,7 +700,7 @@ func (mf2 *MatchFinder2) FindLongestMatches(data []byte) []Match {
 		return mf2.matches
 	}
 	//sort.Sort(&mf2.matches)
-	slices.SortFunc(mf2.matches, func(i, j Match) bool { return i.Start < j.Start })
+	slices.SortFunc(mf2.matches, func(i, j Match) int { return cmp.Compare(i.Start, j.Start) })
 
 	lastEnd := mf2.matches[0].End
 	j := 1

eth/backend.go

@@ -1092,7 +1092,7 @@ func (s *Ethereum) NodesInfo(limit int) (*remote.NodesInfoReply, error) {
 	}
 
 	nodesInfo := &remote.NodesInfoReply{NodesInfo: nodes}
-	slices.SortFunc(nodesInfo.NodesInfo, remote.NodeInfoReplyLess)
+	slices.SortFunc(nodesInfo.NodesInfo, remote.NodeInfoReplyCmp)
 
 	return nodesInfo, nil
 }

eth/stagedsync/stage_interhashes.go

@@ -347,7 +347,7 @@ func (p *HashPromoter) Promote(logPrefix string, from, to uint64, storage bool,
 	}
 
 	if !storage { // delete Intermediate hashes of deleted accounts
-		slices.SortFunc(deletedAccounts, func(a, b []byte) bool { return bytes.Compare(a, b) < 0 })
+		slices.SortFunc(deletedAccounts, bytes.Compare)
 		for _, k := range deletedAccounts {
 			if err := p.tx.ForPrefix(kv.TrieOfStorage, k, func(k, v []byte) error {
 				if err := p.tx.Delete(kv.TrieOfStorage, k); err != nil {
@@ -445,7 +445,7 @@ func (p *HashPromoter) UnwindOnHistoryV3(logPrefix string, unwindFrom, unwindTo
 	}
 
 	// delete Intermediate hashes of deleted accounts
-	slices.SortFunc(deletedAccounts, func(a, b []byte) bool { return bytes.Compare(a, b) < 0 })
+	slices.SortFunc(deletedAccounts, bytes.Compare)
 	for _, k := range deletedAccounts {
 		if err := p.tx.ForPrefix(kv.TrieOfStorage, k, func(k, v []byte) error {
 			if err := p.tx.Delete(kv.TrieOfStorage, k); err != nil {
@@ -533,7 +533,7 @@ func (p *HashPromoter) Unwind(logPrefix string, s *StageState, u *UnwindState, s
 	}
 
 	if !storage { // delete Intermediate hashes of deleted accounts
-		slices.SortFunc(deletedAccounts, func(a, b []byte) bool { return bytes.Compare(a, b) < 0 })
+		slices.SortFunc(deletedAccounts, bytes.Compare)
 		for _, k := range deletedAccounts {
 			if err := p.tx.ForPrefix(kv.TrieOfStorage, k, func(k, v []byte) error {
 				if err := p.tx.Delete(kv.TrieOfStorage, k); err != nil {