From eca448296c6f69fe9669017d2ba88a8358cfb28e Mon Sep 17 00:00:00 2001 From: Richard Ramos Date: Fri, 26 Apr 2024 09:18:38 -0400 Subject: [PATCH] chore: run nwaku as github action --- .github/.env | 1 - .github/docker-compose/nwaku.yml | 6 ++++ .github/workflows/ci.yml | 27 +++++++-------- Makefile | 4 +++ examples/basic-relay/main.go | 2 +- examples/chat2/chat.go | 16 ++++----- examples/chat2/exec.go | 4 +-- waku/persistence/store.go | 4 ++- waku/v2/protocol/envelope_test.go | 2 +- waku/v2/protocol/pb/utils_test.go | 12 +++---- waku/v2/protocol/store/client_test.go | 44 ++++++++++++------------- waku/v2/protocol/store/pb/validation.go | 13 +++----- 12 files changed, 70 insertions(+), 65 deletions(-) delete mode 100644 .github/.env create mode 100644 .github/docker-compose/nwaku.yml diff --git a/.github/.env b/.github/.env deleted file mode 100644 index 8789b588c..000000000 --- a/.github/.env +++ /dev/null @@ -1 +0,0 @@ -go_version=1.20 diff --git a/.github/docker-compose/nwaku.yml b/.github/docker-compose/nwaku.yml new file mode 100644 index 000000000..b71001a3e --- /dev/null +++ b/.github/docker-compose/nwaku.yml @@ -0,0 +1,6 @@ +services: + nwaku: + image: "quay.io/wakuorg/nwaku-pr:2632-rln-v2" + command: ["--relay", "--store", "--nodekey=1122334455667788990011223344556677889900112233445566778899001122"] + ports: + - "60000" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2254a822e..690b3da9b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -52,13 +52,6 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - - uses: xom9ikk/dotenv@v2 - with: - path: ".github/" - - run: | - echo "go_version=${{ env.GO_VERSION }}" >> $GITHUB_OUTPUT - - run: | VERSION=$(cat ./VERSION) echo "waku_version=$VERSION" >> $GITHUB_OUTPUT @@ -73,9 +66,9 @@ jobs: uses: actions/checkout@v3 - name: Install Go - uses: actions/setup-go@v4 + uses: actions/setup-go@v5 with: - go-version: ${{ needs.env.outputs.go_version }} + go-version-file: 'go.mod' cache: false - name: Execute golangci-lint @@ -112,9 +105,9 @@ jobs: key: ${{ runner.os }}-vendor-modules-${{ steps.submodules.outputs.hash }} - name: Install Go - uses: actions/setup-go@v4 + uses: actions/setup-go@v5 with: - go-version: ${{ needs.env.outputs.go_version }} + go-version-file: 'go.mod' cache: false - name: Build binary @@ -154,9 +147,9 @@ jobs: key: ${{ runner.os }}-vendor-modules-${{ steps.submodules.outputs.hash }} - name: Install Go - uses: actions/setup-go@v4 + uses: actions/setup-go@v5 with: - go-version: ${{ needs.env.outputs.go_version }} + go-version-file: 'go.mod' cache: false - name: "Run tests" @@ -166,3 +159,11 @@ jobs: run: | docker compose -f .github/docker-compose/ganache.yml up -d make test-onchain${{ matrix.tests == 'test-with-race' && '-with-race' || '' }} + + - name: "Run storev3 tests" + run: | + docker compose -f .github/docker-compose/nwaku.yml up -d + NWAKU_HOST=$(docker-compose -f .github/docker-compose/nwaku.yml port nwaku 60000) + NWAKU_PORT=$(echo $NWAKU_HOST | cut -d ":" -f 2) + sleep 5 + make test-storev3 TEST_STOREV3_NODE="/ip4/127.0.0.1/tcp/${NWAKU_PORT}/p2p/16Uiu2HAmMGhfSTUzKbsjMWxc6T1X4wiTWSF1bEWSLjAukCm7KiHV" diff --git a/Makefile b/Makefile index daf97d1e8..df2f096e6 100644 --- a/Makefile +++ b/Makefile @@ -219,3 +219,7 @@ test-postgres: test-postgres-with-race: ${GOCMD} test -race -p 1 -v -count 1 -tags="${PG_BUILD_TAGS}" github.com/waku-org/go-waku/waku/persistence/... + +TEST_STOREV3_NODE ?= +test-storev3: + TEST_STOREV3_NODE=${TEST_STOREV3_NODE} ${GOCMD} test -p 1 -v -count 1 -tags="${BUILD_TAGS} include_storev3_tests" github.com/waku-org/go-waku/waku/v2/protocol/store/... \ No newline at end of file diff --git a/examples/basic-relay/main.go b/examples/basic-relay/main.go index 86aef53d7..4536701ca 100644 --- a/examples/basic-relay/main.go +++ b/examples/basic-relay/main.go @@ -191,7 +191,7 @@ func write(ctx context.Context, wakuNode *node.WakuNode, contentTopic string, ms if err != nil { log.Error("Error sending a message", zap.Error(err)) } - log.Info("Published msg,", zap.String("data", string(msg.Payload)), logging.HexBytes("hash", hash)) + log.Info("Published msg,", zap.String("data", string(msg.Payload)), logging.HexBytes("hash", hash.Bytes())) } func writeLoop(ctx context.Context, wakuNode *node.WakuNode, contentTopic string) { diff --git a/examples/chat2/chat.go b/examples/chat2/chat.go index 6a65cea4d..48c39d4fd 100644 --- a/examples/chat2/chat.go +++ b/examples/chat2/chat.go @@ -17,11 +17,11 @@ import ( "github.com/waku-org/go-waku/waku/v2/payload" "github.com/waku-org/go-waku/waku/v2/protocol" "github.com/waku-org/go-waku/waku/v2/protocol/filter" + "github.com/waku-org/go-waku/waku/v2/protocol/legacy_store" "github.com/waku-org/go-waku/waku/v2/protocol/lightpush" wpb "github.com/waku-org/go-waku/waku/v2/protocol/pb" "github.com/waku-org/go-waku/waku/v2/protocol/relay" wrln "github.com/waku-org/go-waku/waku/v2/protocol/rln" - "github.com/waku-org/go-waku/waku/v2/protocol/store" "github.com/waku-org/go-waku/waku/v2/utils" "google.golang.org/protobuf/proto" ) @@ -368,10 +368,10 @@ func (c *Chat) retrieveHistory(connectionWg *sync.WaitGroup) { return } - var storeOpt store.HistoryRequestOption + var storeOpt legacy_store.HistoryRequestOption if c.options.Store.Node == nil { c.ui.InfoMessage("No store node configured. Choosing one at random...") - storeOpt = store.WithAutomaticPeerSelection() + storeOpt = legacy_store.WithAutomaticPeerSelection() } else { peerID, err := (*c.options.Store.Node).ValueForProtocol(multiaddr.P_P2P) if err != nil { @@ -383,7 +383,7 @@ func (c *Chat) retrieveHistory(connectionWg *sync.WaitGroup) { c.ui.ErrorMessage(err) return } - storeOpt = store.WithPeer(pID) + storeOpt = legacy_store.WithPeer(pID) c.ui.InfoMessage(fmt.Sprintf("Querying historic messages from %s", peerID)) } @@ -391,14 +391,14 @@ func (c *Chat) retrieveHistory(connectionWg *sync.WaitGroup) { tCtx, cancel := context.WithTimeout(c.ctx, 10*time.Second) defer cancel() - q := store.Query{ + q := legacy_store.Query{ ContentTopics: []string{options.ContentTopic}, } - response, err := c.node.Store().Query(tCtx, q, - store.WithAutomaticRequestID(), + response, err := c.node.LegacyStore().Query(tCtx, q, + legacy_store.WithAutomaticRequestID(), storeOpt, - store.WithPaging(false, 100)) + legacy_store.WithPaging(false, 100)) if err != nil { c.ui.ErrorMessage(fmt.Errorf("could not query storenode: %w", err)) diff --git a/examples/chat2/exec.go b/examples/chat2/exec.go index 266b03151..2882960c6 100644 --- a/examples/chat2/exec.go +++ b/examples/chat2/exec.go @@ -12,9 +12,9 @@ import ( "github.com/waku-org/go-waku/waku/v2/node" "github.com/waku-org/go-waku/waku/v2/peerstore" "github.com/waku-org/go-waku/waku/v2/protocol/filter" + "github.com/waku-org/go-waku/waku/v2/protocol/legacy_store" "github.com/waku-org/go-waku/waku/v2/protocol/lightpush" "github.com/waku-org/go-waku/waku/v2/protocol/pb" - "github.com/waku-org/go-waku/waku/v2/protocol/store" ) func execute(options Options) { @@ -77,7 +77,7 @@ func execute(options Options) { return } - err = addPeer(wakuNode, options.Store.Node, options.Relay.Topics.Value(), store.StoreID_v20beta4) + err = addPeer(wakuNode, options.Store.Node, options.Relay.Topics.Value(), legacy_store.StoreID_v20beta4) if err != nil { fmt.Println(err.Error()) return diff --git a/waku/persistence/store.go b/waku/persistence/store.go index 9e245c6e3..10540c7ce 100644 --- a/waku/persistence/store.go +++ b/waku/persistence/store.go @@ -317,8 +317,10 @@ func (d *DBStore) Put(env *protocol.Envelope) error { storedAt = env.Index().ReceiverTime } + hash := env.Hash() + start := time.Now() - _, err = stmt.Exec(env.Index().Digest, env.Hash(), storedAt, env.Message().GetTimestamp(), env.Message().ContentTopic, env.PubsubTopic(), env.Message().Payload, env.Message().GetVersion()) + _, err = stmt.Exec(env.Index().Digest, hash[:], storedAt, env.Message().GetTimestamp(), env.Message().ContentTopic, env.PubsubTopic(), env.Message().Payload, env.Message().GetVersion()) if err != nil { return err } diff --git a/waku/v2/protocol/envelope_test.go b/waku/v2/protocol/envelope_test.go index b95426b43..84a963dfc 100644 --- a/waku/v2/protocol/envelope_test.go +++ b/waku/v2/protocol/envelope_test.go @@ -24,7 +24,7 @@ func TestEnvelope(t *testing.T) { require.Equal( t, - []byte{0x91, 0x0, 0xe4, 0xa5, 0xcf, 0xf7, 0x19, 0x27, 0x49, 0x81, 0x66, 0xb3, 0xdf, 0xc7, 0xa6, 0x31, 0xf0, 0x87, 0xc7, 0x29, 0xb4, 0x28, 0x83, 0xb9, 0x5c, 0x31, 0x25, 0x33, 0x3, 0xc9, 0x7, 0x95}, + pb.ToMessageHash([]byte{0x91, 0x0, 0xe4, 0xa5, 0xcf, 0xf7, 0x19, 0x27, 0x49, 0x81, 0x66, 0xb3, 0xdf, 0xc7, 0xa6, 0x31, 0xf0, 0x87, 0xc7, 0x29, 0xb4, 0x28, 0x83, 0xb9, 0x5c, 0x31, 0x25, 0x33, 0x3, 0xc9, 0x7, 0x95}), hash, ) } diff --git a/waku/v2/protocol/pb/utils_test.go b/waku/v2/protocol/pb/utils_test.go index a273d5041..133c0f6ee 100644 --- a/waku/v2/protocol/pb/utils_test.go +++ b/waku/v2/protocol/pb/utils_test.go @@ -23,7 +23,7 @@ func TestEnvelopeHash(t *testing.T) { expected := []byte{0xb6, 0x59, 0x60, 0x7f, 0x2a, 0xae, 0x18, 0x84, 0x8d, 0xca, 0xa7, 0xd5, 0x1c, 0xb3, 0x7e, 0x6c, 0xc6, 0xfc, 0x33, 0x40, 0x2c, 0x70, 0x4f, 0xf0, 0xc0, 0x16, 0x33, 0x7d, 0x83, 0xad, 0x61, 0x50} result := msg.Hash("test") - require.Equal(t, expected, result) + require.Equal(t, ToMessageHash(expected), result) } func TestEmptyMeta(t *testing.T) { @@ -38,7 +38,7 @@ func TestEmptyMeta(t *testing.T) { messageHash := msg.Hash(pubsubTopic) - require.Equal(t, "f0183c2e370e473ff471bbe1028d0d8a940949c02f3007a1ccd21fed356852a0", messageHash.String()) + require.Equal(t, "0xf0183c2e370e473ff471bbe1028d0d8a940949c02f3007a1ccd21fed356852a0", messageHash.String()) } func Test13ByteMeta(t *testing.T) { @@ -52,7 +52,7 @@ func Test13ByteMeta(t *testing.T) { messageHash := msg.Hash(pubsubTopic) - require.Equal(t, "f673cd2c9c973d685b52ca74c2559e001733a3a31a49ffc7b6e8713decba5a55", messageHash.String()) + require.Equal(t, "0xf673cd2c9c973d685b52ca74c2559e001733a3a31a49ffc7b6e8713decba5a55", messageHash.String()) } func TestZeroLenPayload(t *testing.T) { @@ -66,7 +66,7 @@ func TestZeroLenPayload(t *testing.T) { messageHash := msg.Hash(pubsubTopic) - require.Equal(t, "978ccc9a665029f9829d42d84e3a49ad3a4791cce53fb5a8b581ef43ad6b4d2f", messageHash.String()) + require.Equal(t, "0x978ccc9a665029f9829d42d84e3a49ad3a4791cce53fb5a8b581ef43ad6b4d2f", messageHash.String()) } func TestHashWithTimestamp(t *testing.T) { @@ -78,11 +78,11 @@ func TestHashWithTimestamp(t *testing.T) { msg.Version = proto.Uint32(1) messageHash := msg.Hash(pubsubTopic) - require.Equal(t, "58e2fc032a82c4adeb967a8b87086d0d6fb304912f120d4404e6236add8f1f56", messageHash.String()) + require.Equal(t, "0x58e2fc032a82c4adeb967a8b87086d0d6fb304912f120d4404e6236add8f1f56", messageHash.String()) msg.Timestamp = proto.Int64(123456789123456789) messageHash = msg.Hash(pubsubTopic) - require.Equal(t, "978ccc9a665029f9829d42d84e3a49ad3a4791cce53fb5a8b581ef43ad6b4d2f", messageHash.String()) + require.Equal(t, "0x978ccc9a665029f9829d42d84e3a49ad3a4791cce53fb5a8b581ef43ad6b4d2f", messageHash.String()) } func TestIntToBytes(t *testing.T) { diff --git a/waku/v2/protocol/store/client_test.go b/waku/v2/protocol/store/client_test.go index c81cac2ff..41e5962f3 100644 --- a/waku/v2/protocol/store/client_test.go +++ b/waku/v2/protocol/store/client_test.go @@ -1,7 +1,7 @@ -package store +//go:build include_storev3_tests +// +build include_storev3_tests -//111go:build include_storev3_tests -// 111+build include_storev3_tests +package store import ( "context" @@ -43,7 +43,7 @@ func TestStoreClient(t *testing.T) { err = wakuRelay.Start(context.Background()) require.NoError(t, err) - pm := peermanager.NewPeerManager(5, 5, utils.Logger()) + pm := peermanager.NewPeerManager(5, 5, nil, utils.Logger()) pm.SetHost(host) err = pm.SubscribeToRelayEvtBus(wakuRelay.Events()) require.NoError(t, err) @@ -189,14 +189,15 @@ func TestStoreClient(t *testing.T) { require.Len(t, response.messages, 5) require.Empty(t, response.Cursor()) - // Invalid cursors should return an empty response - // TODO: nwaku is returning values even with invalid cursors - /* - response, err = wakuStore.Query(ctx, FilterCriteria{ContentFilter: protocol.NewContentFilter(relay.DefaultWakuTopic, "test"), TimeStart: startTime, TimeEnd: endTime}, WithCursor([]byte{1, 2, 3, 4, 5, 6})) - require.NoError(t, err) - require.Len(t, response.messages, 0) - require.Empty(t, response.Cursor()) - */ + // Invalid cursors should fail + _, err = wakuStore.Query(ctx, FilterCriteria{ContentFilter: protocol.NewContentFilter(relay.DefaultWakuTopic, "test"), TimeStart: startTime, TimeEnd: endTime}, WithCursor([]byte{1, 2, 3, 4, 5, 6})) + require.Error(t, err) + + // Inexistent cursors should return an empty response + response, err = wakuStore.Query(ctx, FilterCriteria{ContentFilter: protocol.NewContentFilter(relay.DefaultWakuTopic, "test"), TimeStart: startTime, TimeEnd: endTime}, WithCursor(make([]byte, 32))) // Requesting cursor 0x00...00 + require.NoError(t, err) + require.Len(t, response.messages, 0) + require.Empty(t, response.Cursor()) // Handle temporal history query with an invalid time window _, err = wakuStore.Query(ctx, FilterCriteria{ContentFilter: protocol.NewContentFilter(relay.DefaultWakuTopic, "test"), TimeStart: endTime, TimeEnd: startTime}) @@ -209,16 +210,13 @@ func TestStoreClient(t *testing.T) { require.Empty(t, response.Cursor()) // Should not include data - // TODO: nwaku is returning the data always - /* - response, err = wakuStore.Request(ctx, MessageHashCriteria{MessageHashes: []pb.MessageHash{messages[0].Hash(relay.DefaultWakuTopic)}}, IncludeData(false), WithPeer(storenode.ID)) - require.NoError(t, err) - require.Len(t, response.messages, 1) - require.Nil(t, response.messages[0].Message) + response, err = wakuStore.Request(ctx, MessageHashCriteria{MessageHashes: []pb.MessageHash{messages[0].Hash(relay.DefaultWakuTopic)}}, IncludeData(false), WithPeer(storenode.ID)) + require.NoError(t, err) + require.Len(t, response.messages, 1) + require.Nil(t, response.messages[0].Message) - response, err = wakuStore.Request(ctx, FilterCriteria{ContentFilter: protocol.NewContentFilter(relay.DefaultWakuTopic, "test")}, IncludeData(false)) - require.NoError(t, err) - require.GreaterOrEqual(t, response.messages, 1) - require.Nil(t, response.messages[0].Message) - */ + response, err = wakuStore.Request(ctx, FilterCriteria{ContentFilter: protocol.NewContentFilter(relay.DefaultWakuTopic, "test")}, IncludeData(false)) + require.NoError(t, err) + require.GreaterOrEqual(t, response.messages, 1) + require.Nil(t, response.messages[0].Message) } diff --git a/waku/v2/protocol/store/pb/validation.go b/waku/v2/protocol/store/pb/validation.go index 53f90fb34..40bdfade6 100644 --- a/waku/v2/protocol/store/pb/validation.go +++ b/waku/v2/protocol/store/pb/validation.go @@ -16,7 +16,6 @@ var ( errMissingPubsubTopic = errors.New("missing PubsubTopic field") errMissingContentTopics = errors.New("missing ContentTopics field") errMissingStatusCode = errors.New("missing StatusCode field") - errMissingMessage = errors.New("missing Message field") errInvalidTimeRange = errors.New("invalid time range") errInvalidMessageHash = errors.New("invalid message hash") ) @@ -61,11 +60,7 @@ func (x *StoreQueryRequest) Validate() error { } func (x *StoreQueryResponse) Validate(requestID string) error { - if x.RequestId == "" { - return errMissingRequestID - } - - if x.RequestId != requestID { + if x.RequestId != "" && x.RequestId != requestID { return errRequestIDMismatch } @@ -87,9 +82,9 @@ func (x *WakuMessageKeyValue) Validate() error { return errInvalidMessageHash } - if x.Message == nil { - return errMissingMessage - } else { + if x.Message != nil { return x.Message.Validate() } + + return nil }