chore: sharding tests update

.github/docker-compose/nwaku.yml

@@ -0,0 +1,6 @@
+services:
+  nwaku:
+    image: "harbor.status.im/wakuorg/nwaku:latest"
+    command: ["--relay", "--store", "--nodekey=1122334455667788990011223344556677889900112233445566778899001122"]
+    ports:
+      - "60000"

.github/workflows/ci.yml

@@ -52,13 +52,6 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v3
-
-    - uses: xom9ikk/dotenv@v2
-      with:
-        path: ".github/"
-    - run: |
-        echo "go_version=${{ env.GO_VERSION }}" >> $GITHUB_OUTPUT
-
     - run: |
         VERSION=$(cat ./VERSION)
         echo "waku_version=$VERSION" >> $GITHUB_OUTPUT
@@ -73,9 +66,9 @@ jobs:
         uses: actions/checkout@v3
 
       - name: Install Go
-        uses: actions/setup-go@v4
+        uses: actions/setup-go@v5
         with:
-          go-version: ${{ needs.env.outputs.go_version }}
+          go-version-file: 'go.mod'
           cache: false
 
       - name: Execute golangci-lint
@@ -112,9 +105,9 @@ jobs:
           key: ${{ runner.os }}-vendor-modules-${{ steps.submodules.outputs.hash }}
 
       - name: Install Go
-        uses: actions/setup-go@v4
+        uses: actions/setup-go@v5
         with:
-          go-version: ${{ needs.env.outputs.go_version }}
+          go-version-file: 'go.mod'
           cache: false
 
       - name: Build binary
@@ -154,9 +147,9 @@ jobs:
           key: ${{ runner.os }}-vendor-modules-${{ steps.submodules.outputs.hash }}
 
       - name: Install Go
-        uses: actions/setup-go@v4
+        uses: actions/setup-go@v5
         with:
-          go-version: ${{ needs.env.outputs.go_version }} 
+          go-version-file: 'go.mod'
           cache: false
 
       - name: "Run tests"
@@ -166,3 +159,11 @@ jobs:
         run: |
           docker compose -f .github/docker-compose/ganache.yml up -d
           make test-onchain${{ matrix.tests == 'test-with-race' && '-with-race' || '' }}
+
+      - name: "Run storev3 tests"
+        run: |
+          docker compose -f .github/docker-compose/nwaku.yml up -d
+          NWAKU_HOST=$(docker-compose -f .github/docker-compose/nwaku.yml port nwaku 60000)
+          NWAKU_PORT=$(echo $NWAKU_HOST | cut -d ":" -f 2)
+          sleep 5
+          make test-storev3 TEST_STOREV3_NODE="/ip4/127.0.0.1/tcp/${NWAKU_PORT}/p2p/16Uiu2HAmMGhfSTUzKbsjMWxc6T1X4wiTWSF1bEWSLjAukCm7KiHV"

Makefile

@@ -219,3 +219,7 @@ test-postgres:
 
 test-postgres-with-race:
 	${GOCMD} test -race -p 1 -v -count 1 -tags="${PG_BUILD_TAGS}" github.com/waku-org/go-waku/waku/persistence/...
+
+TEST_STOREV3_NODE ?=
+test-storev3:
+	TEST_STOREV3_NODE=${TEST_STOREV3_NODE} ${GOCMD} test -p 1 -v -count 1 -tags="${BUILD_TAGS} include_storev3_tests" github.com/waku-org/go-waku/waku/v2/protocol/store/...

cmd/waku/node.go

@@ -46,10 +46,10 @@ import (
 	"github.com/waku-org/go-waku/waku/v2/node"
 	wprotocol "github.com/waku-org/go-waku/waku/v2/protocol"
 	"github.com/waku-org/go-waku/waku/v2/protocol/filter"
+	"github.com/waku-org/go-waku/waku/v2/protocol/legacy_store"
 	"github.com/waku-org/go-waku/waku/v2/protocol/lightpush"
 	"github.com/waku-org/go-waku/waku/v2/protocol/peer_exchange"
 	"github.com/waku-org/go-waku/waku/v2/protocol/relay"
-	"github.com/waku-org/go-waku/waku/v2/protocol/store"
 	"github.com/waku-org/go-waku/waku/v2/utils"
 
 	humanize "github.com/dustin/go-humanize"
@@ -336,7 +336,7 @@ func Execute(options NodeOptions) error {
 
 	//For now assuming that static peers added support/listen on all topics specified via commandLine.
 	staticPeers := map[protocol.ID][]multiaddr.Multiaddr{
-		store.StoreID_v20beta4:            options.Store.Nodes,
+		legacy_store.StoreID_v20beta4:     options.Store.Nodes,
 		lightpush.LightPushID_v20beta1:    options.LightPush.Nodes,
 		rendezvous.RendezvousID:           options.Rendezvous.Nodes,
 		filter.FilterSubscribeID_v20beta1: options.Filter.Nodes,

cmd/waku/server/rest/store.go

@@ -12,8 +12,8 @@ import (
 	"github.com/go-chi/chi/v5"
 	"github.com/multiformats/go-multiaddr"
 	"github.com/waku-org/go-waku/waku/v2/node"
-	"github.com/waku-org/go-waku/waku/v2/protocol/store"
-	"github.com/waku-org/go-waku/waku/v2/protocol/store/pb"
+	"github.com/waku-org/go-waku/waku/v2/protocol/legacy_store"
+	"github.com/waku-org/go-waku/waku/v2/protocol/legacy_store/pb"
 )
 
 type StoreService struct {
@@ -55,9 +55,9 @@ func NewStoreService(node *node.WakuNode, m *chi.Mux) *StoreService {
 	return s
 }
 
-func getStoreParams(r *http.Request) (*store.Query, []store.HistoryRequestOption, error) {
-	query := &store.Query{}
-	var options []store.HistoryRequestOption
+func getStoreParams(r *http.Request) (*legacy_store.Query, []legacy_store.HistoryRequestOption, error) {
+	query := &legacy_store.Query{}
+	var options []legacy_store.HistoryRequestOption
 	var err error
 	peerAddrStr := r.URL.Query().Get("peerAddr")
 	var m multiaddr.Multiaddr
@@ -66,12 +66,12 @@ func getStoreParams(r *http.Request) (*store.Query, []store.HistoryRequestOption
 		if err != nil {
 			return nil, nil, err
 		}
-		options = append(options, store.WithPeerAddr(m))
+		options = append(options, legacy_store.WithPeerAddr(m))
 	} else {
 		// The user didn't specify a peer address and self-node is configured as a store node.
 		// In this case we assume that the user is willing to retrieve the messages stored by
 		// the local/self store node.
-		options = append(options, store.WithLocalQuery())
+		options = append(options, legacy_store.WithLocalQuery())
 	}
 
 	query.PubsubTopic = r.URL.Query().Get("pubsubTopic")
@@ -131,14 +131,14 @@ func getStoreParams(r *http.Request) (*store.Query, []store.HistoryRequestOption
 
 		cursor.PubsubTopic = query.PubsubTopic
 
-		options = append(options, store.WithCursor(cursor))
+		options = append(options, legacy_store.WithCursor(cursor))
 	}
 
 	pageSizeStr := r.URL.Query().Get("pageSize")
 	ascendingStr := r.URL.Query().Get("ascending")
 	if ascendingStr != "" || pageSizeStr != "" {
 		ascending := true
-		pageSize := uint64(store.DefaultPageSize)
+		pageSize := uint64(legacy_store.DefaultPageSize)
 		if ascendingStr != "" {
 			ascending, err = strconv.ParseBool(ascendingStr)
 			if err != nil {
@@ -151,12 +151,12 @@ func getStoreParams(r *http.Request) (*store.Query, []store.HistoryRequestOption
 			if err != nil {
 				return nil, nil, err
 			}
-			if pageSize > store.MaxPageSize {
-				pageSize = store.MaxPageSize
+			if pageSize > legacy_store.MaxPageSize {
+				pageSize = legacy_store.MaxPageSize
 			}
 		}
 
-		options = append(options, store.WithPaging(ascending, pageSize))
+		options = append(options, legacy_store.WithPaging(ascending, pageSize))
 	}
 
 	return query, options, nil
@@ -166,7 +166,7 @@ func writeStoreError(w http.ResponseWriter, code int, err error) {
 	writeResponse(w, StoreResponse{ErrorMessage: err.Error()}, code)
 }
 
-func toStoreResponse(result *store.Result) StoreResponse {
+func toStoreResponse(result *legacy_store.Result) StoreResponse {
 	response := StoreResponse{}
 
 	cursor := result.Cursor()
@@ -202,7 +202,7 @@ func (d *StoreService) getV1Messages(w http.ResponseWriter, r *http.Request) {
 	ctx, cancel := context.WithTimeout(r.Context(), 5*time.Second)
 	defer cancel()
 
-	result, err := d.node.Store().Query(ctx, *query, options...)
+	result, err := d.node.LegacyStore().Query(ctx, *query, options...)
 	if err != nil {
 		writeStoreError(w, http.StatusInternalServerError, err)
 		return

cmd/waku/server/utils.go

@@ -6,6 +6,7 @@ import (
 
 	"github.com/libp2p/go-libp2p/core/protocol"
 	"github.com/waku-org/go-waku/waku/v2/protocol/filter"
+	"github.com/waku-org/go-waku/waku/v2/protocol/legacy_store"
 	"github.com/waku-org/go-waku/waku/v2/protocol/lightpush"
 	"github.com/waku-org/go-waku/waku/v2/protocol/relay"
 	"github.com/waku-org/go-waku/waku/v2/protocol/store"
@@ -16,7 +17,8 @@ func IsWakuProtocol(protocol protocol.ID) bool {
 		protocol == filter.FilterSubscribeID_v20beta1 ||
 		protocol == relay.WakuRelayID_v200 ||
 		protocol == lightpush.LightPushID_v20beta1 ||
-		protocol == store.StoreID_v20beta4
+		protocol == legacy_store.StoreID_v20beta4 ||
+		protocol == store.StoreQueryID_v300
 }
 
 type Base64URLByte []byte

examples/basic-relay/main.go

@@ -191,7 +191,7 @@ func write(ctx context.Context, wakuNode *node.WakuNode, contentTopic string, ms
 	if err != nil {
 		log.Error("Error sending a message", zap.Error(err))
 	}
-	log.Info("Published msg,", zap.String("data", string(msg.Payload)), logging.HexBytes("hash", hash))
+	log.Info("Published msg,", zap.String("data", string(msg.Payload)), logging.HexBytes("hash", hash.Bytes()))
 }
 
 func writeLoop(ctx context.Context, wakuNode *node.WakuNode, contentTopic string) {

examples/chat2/chat.go

@@ -17,11 +17,11 @@ import (
 	"github.com/waku-org/go-waku/waku/v2/payload"
 	"github.com/waku-org/go-waku/waku/v2/protocol"
 	"github.com/waku-org/go-waku/waku/v2/protocol/filter"
+	"github.com/waku-org/go-waku/waku/v2/protocol/legacy_store"
 	"github.com/waku-org/go-waku/waku/v2/protocol/lightpush"
 	wpb "github.com/waku-org/go-waku/waku/v2/protocol/pb"
 	"github.com/waku-org/go-waku/waku/v2/protocol/relay"
 	wrln "github.com/waku-org/go-waku/waku/v2/protocol/rln"
-	"github.com/waku-org/go-waku/waku/v2/protocol/store"
 	"github.com/waku-org/go-waku/waku/v2/utils"
 	"google.golang.org/protobuf/proto"
 )
@@ -368,10 +368,10 @@ func (c *Chat) retrieveHistory(connectionWg *sync.WaitGroup) {
 		return
 	}
 
-	var storeOpt store.HistoryRequestOption
+	var storeOpt legacy_store.HistoryRequestOption
 	if c.options.Store.Node == nil {
 		c.ui.InfoMessage("No store node configured. Choosing one at random...")
-		storeOpt = store.WithAutomaticPeerSelection()
+		storeOpt = legacy_store.WithAutomaticPeerSelection()
 	} else {
 		peerID, err := (*c.options.Store.Node).ValueForProtocol(multiaddr.P_P2P)
 		if err != nil {
@@ -383,22 +383,22 @@ func (c *Chat) retrieveHistory(connectionWg *sync.WaitGroup) {
 			c.ui.ErrorMessage(err)
 			return
 		}
-		storeOpt = store.WithPeer(pID)
+		storeOpt = legacy_store.WithPeer(pID)
 		c.ui.InfoMessage(fmt.Sprintf("Querying historic messages from %s", peerID))
 
 	}
 
 	tCtx, cancel := context.WithTimeout(c.ctx, 10*time.Second)
 	defer cancel()
 
-	q := store.Query{
+	q := legacy_store.Query{
 		ContentTopics: []string{options.ContentTopic},
 	}
 
-	response, err := c.node.Store().Query(tCtx, q,
-		store.WithAutomaticRequestID(),
+	response, err := c.node.LegacyStore().Query(tCtx, q,
+		legacy_store.WithAutomaticRequestID(),
 		storeOpt,
-		store.WithPaging(false, 100))
+		legacy_store.WithPaging(false, 100))
 
 	if err != nil {
 		c.ui.ErrorMessage(fmt.Errorf("could not query storenode: %w", err))

examples/chat2/exec.go

@@ -12,9 +12,9 @@ import (
 	"github.com/waku-org/go-waku/waku/v2/node"
 	"github.com/waku-org/go-waku/waku/v2/peerstore"
 	"github.com/waku-org/go-waku/waku/v2/protocol/filter"
+	"github.com/waku-org/go-waku/waku/v2/protocol/legacy_store"
 	"github.com/waku-org/go-waku/waku/v2/protocol/lightpush"
 	"github.com/waku-org/go-waku/waku/v2/protocol/pb"
-	"github.com/waku-org/go-waku/waku/v2/protocol/store"
 )
 
 func execute(options Options) {
@@ -77,7 +77,7 @@ func execute(options Options) {
 		return
 	}
 
-	err = addPeer(wakuNode, options.Store.Node, options.Relay.Topics.Value(), store.StoreID_v20beta4)
+	err = addPeer(wakuNode, options.Store.Node, options.Relay.Topics.Value(), legacy_store.StoreID_v20beta4)
 	if err != nil {
 		fmt.Println(err.Error())
 		return

library/lightpush.go

@@ -6,7 +6,6 @@ import (
 
 	"github.com/waku-org/go-waku/waku/v2/protocol/pb"
 
-	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/libp2p/go-libp2p/core/peer"
 	"github.com/waku-org/go-waku/waku/v2/protocol/lightpush"
 )
@@ -42,7 +41,8 @@ func lightpushPublish(instance *WakuInstance, msg *pb.WakuMessage, pubsubTopic s
 	}
 
 	hash, err := instance.node.Lightpush().Publish(ctx, msg, lpOptions...)
-	return hexutil.Encode(hash), err
+
+	return hash.String(), err
 }
 
 // LightpushPublish is used to publish a WakuMessage in a pubsub topic using Lightpush protocol

library/node.go

@@ -16,7 +16,6 @@ import (
 	"go.uber.org/zap"
 	"go.uber.org/zap/zapcore"
 
-	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/ethereum/go-ethereum/crypto"
 	pubsub "github.com/libp2p/go-libp2p-pubsub"
 	"github.com/libp2p/go-libp2p/core/peer"
@@ -444,7 +443,7 @@ type subscriptionMsg struct {
 
 func toSubscriptionMessage(msg *protocol.Envelope) *subscriptionMsg {
 	return &subscriptionMsg{
-		MessageID:   hexutil.Encode(msg.Hash()),
+		MessageID:   msg.Hash().String(),
 		PubsubTopic: msg.PubsubTopic(),
 		Message:     msg.Message(),
 	}

library/relay.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"time"
 
-	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/waku-org/go-waku/waku/v2/protocol"
 	"github.com/waku-org/go-waku/waku/v2/protocol/pb"
 	"github.com/waku-org/go-waku/waku/v2/protocol/relay"
@@ -40,7 +39,8 @@ func relayPublish(instance *WakuInstance, msg *pb.WakuMessage, pubsubTopic strin
 	}
 
 	hash, err := instance.node.Relay().Publish(ctx, msg, relay.WithPubSubTopic(pubsubTopic))
-	return hexutil.Encode(hash), err
+
+	return hash.String(), err
 }
 
 // RelayPublish publishes a message using waku relay and returns the message ID