From 38e71518dcb96cb4523c6159a4280de59333bcf6 Mon Sep 17 00:00:00 2001 From: Andrii Slisarchuk Date: Tue, 19 Nov 2024 13:42:45 +0200 Subject: [PATCH 01/67] Implemented pending transaction trigger --- .../backend/backend_stream_transactions.go | 51 +++++++++++++------ .../backend_stream_transactions_test.go | 2 +- 2 files changed, 37 insertions(+), 16 deletions(-) diff --git a/engine/access/rpc/backend/backend_stream_transactions.go b/engine/access/rpc/backend/backend_stream_transactions.go index a82b365240e..b7a8efb2762 100644 --- a/engine/access/rpc/backend/backend_stream_transactions.go +++ b/engine/access/rpc/backend/backend_stream_transactions.go @@ -4,7 +4,7 @@ import ( "context" "errors" "fmt" - + "go.uber.org/atomic" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -39,6 +39,7 @@ type TransactionSubscriptionMetadata struct { blockWithTx *flow.Header txExecuted bool eventEncodingVersion entities.EventEncodingVersion + triggerFirstPending *atomic.Bool } // SubscribeTransactionStatuses subscribes to transaction status changes starting from the transaction reference block ID. @@ -57,11 +58,12 @@ func (b *backendSubscribeTransactions) SubscribeTransactionStatuses( TransactionResult: &access.TransactionResult{ TransactionID: tx.ID(), BlockID: flow.ZeroID, - Status: flow.TransactionStatusUnknown, + Status: flow.TransactionStatusPending, }, txReferenceBlockID: tx.ReferenceBlockID, blockWithTx: nil, eventEncodingVersion: requiredEventEncodingVersion, + triggerFirstPending: atomic.NewBool(true), } return b.subscriptionHandler.Subscribe(ctx, nextHeight, b.getTransactionStatusResponse(&txInfo)) @@ -76,6 +78,13 @@ func (b *backendSubscribeTransactions) getTransactionStatusResponse(txInfo *Tran return nil, err } + // The status of the first pending transaction should be returned immediately, as the transaction has already been sent. + // This should occur only once for each subscription. + if txInfo.triggerFirstPending.Load() { + txInfo.triggerFirstPending.Toggle() + return b.generateResultsWithMissingStatuses(txInfo, flow.TransactionStatusUnknown) + } + // If the transaction status already reported the final status, return with no data available if txInfo.Status == flow.TransactionStatusSealed || txInfo.Status == flow.TransactionStatusExpired { return nil, fmt.Errorf("transaction final status %s was already reported: %w", txInfo.Status.String(), subscription.ErrEndOfData) @@ -120,19 +129,8 @@ func (b *backendSubscribeTransactions) getTransactionStatusResponse(txInfo *Tran } // If block with transaction was not found, get transaction status to check if it different from last status - if txInfo.blockWithTx == nil { - txInfo.Status, err = b.txLocalDataProvider.DeriveUnknownTransactionStatus(txInfo.txReferenceBlockID) - } else if txInfo.Status == prevTxStatus { - // When a block with the transaction is available, it is possible to receive a new transaction status while - // searching for the transaction result. Otherwise, it remains unchanged. So, if the old and new transaction - // statuses are the same, the current transaction status should be retrieved. - txInfo.Status, err = b.txLocalDataProvider.DeriveTransactionStatus(txInfo.blockWithTx.Height, txInfo.txExecuted) - } - if err != nil { - if !errors.Is(err, state.ErrUnknownSnapshotReference) { - irrecoverable.Throw(ctx, err) - } - return nil, rpc.ConvertStorageError(err) + if txInfo.Status, err = b.getTransactionStatus(ctx, txInfo, prevTxStatus); err != nil { + return nil, err } // If the old and new transaction statuses are still the same, the status change should not be reported, so @@ -145,6 +143,29 @@ func (b *backendSubscribeTransactions) getTransactionStatusResponse(txInfo *Tran } } +func (b *backendSubscribeTransactions) getTransactionStatus(ctx context.Context, txInfo *TransactionSubscriptionMetadata, prevTxStatus flow.TransactionStatus) (flow.TransactionStatus, error) { + txStatus := prevTxStatus + var err error + + if txInfo.blockWithTx == nil { + txStatus, err = b.txLocalDataProvider.DeriveUnknownTransactionStatus(txInfo.txReferenceBlockID) + } else if txInfo.Status == prevTxStatus { + // When a block with the transaction is available, it is possible to receive a new transaction status while + // searching for the transaction result. Otherwise, it remains unchanged. So, if the old and new transaction + // statuses are the same, the current transaction status should be retrieved. + txStatus, err = b.txLocalDataProvider.DeriveTransactionStatus(txInfo.blockWithTx.Height, txInfo.txExecuted) + } + + if err != nil { + if !errors.Is(err, state.ErrUnknownSnapshotReference) { + irrecoverable.Throw(ctx, err) + } + return flow.TransactionStatusUnknown, rpc.ConvertStorageError(err) + } + + return txStatus, nil +} + // generateResultsWithMissingStatuses checks if the current result differs from the previous result by more than one step. // If yes, it generates results for the missing transaction statuses. This is done because the subscription should send // responses for each of the statuses in the transaction lifecycle, and the message should be sent in the order of transaction statuses. diff --git a/engine/access/rpc/backend/backend_stream_transactions_test.go b/engine/access/rpc/backend/backend_stream_transactions_test.go index 24cdf601f17..1d6faaa0526 100644 --- a/engine/access/rpc/backend/backend_stream_transactions_test.go +++ b/engine/access/rpc/backend/backend_stream_transactions_test.go @@ -310,7 +310,7 @@ func (s *TransactionStatusSuite) TestSubscribeTransactionStatusHappyCase() { result := txResults[0] assert.Equal(s.T(), txId, result.TransactionID) assert.Equal(s.T(), expectedTxStatus, result.Status) - }, time.Second, fmt.Sprintf("timed out waiting for transaction info:\n\t- txID: %x\n\t- blockID: %x", txId, s.finalizedBlock.ID())) + }, 1000*time.Second, fmt.Sprintf("timed out waiting for transaction info:\n\t- txID: %x\n\t- blockID: %x", txId, s.finalizedBlock.ID())) } // 1. Subscribe to transaction status and receive the first message with pending status From 2ac2742a76ad8929421d1819138a1394d55d7c17 Mon Sep 17 00:00:00 2001 From: Andrii Slisarchuk Date: Tue, 19 Nov 2024 13:56:21 +0200 Subject: [PATCH 02/67] fixed tests --- engine/access/rpc/backend/backend_stream_transactions.go | 4 ++-- .../access/rpc/backend/backend_stream_transactions_test.go | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/engine/access/rpc/backend/backend_stream_transactions.go b/engine/access/rpc/backend/backend_stream_transactions.go index b7a8efb2762..3bd455dd66d 100644 --- a/engine/access/rpc/backend/backend_stream_transactions.go +++ b/engine/access/rpc/backend/backend_stream_transactions.go @@ -144,12 +144,12 @@ func (b *backendSubscribeTransactions) getTransactionStatusResponse(txInfo *Tran } func (b *backendSubscribeTransactions) getTransactionStatus(ctx context.Context, txInfo *TransactionSubscriptionMetadata, prevTxStatus flow.TransactionStatus) (flow.TransactionStatus, error) { - txStatus := prevTxStatus + txStatus := txInfo.Status var err error if txInfo.blockWithTx == nil { txStatus, err = b.txLocalDataProvider.DeriveUnknownTransactionStatus(txInfo.txReferenceBlockID) - } else if txInfo.Status == prevTxStatus { + } else if txStatus == prevTxStatus { // When a block with the transaction is available, it is possible to receive a new transaction status while // searching for the transaction result. Otherwise, it remains unchanged. So, if the old and new transaction // statuses are the same, the current transaction status should be retrieved. diff --git a/engine/access/rpc/backend/backend_stream_transactions_test.go b/engine/access/rpc/backend/backend_stream_transactions_test.go index 1d6faaa0526..6d6f98ae1d3 100644 --- a/engine/access/rpc/backend/backend_stream_transactions_test.go +++ b/engine/access/rpc/backend/backend_stream_transactions_test.go @@ -148,7 +148,7 @@ func (s *TransactionStatusSuite) SetupTest() { require.NoError(s.T(), err) s.blocks.On("ByHeight", mock.AnythingOfType("uint64")).Return(mocks.StorageMapGetter(s.blockMap)) - s.state.On("Final").Return(s.finalSnapshot, nil) + s.state.On("Final").Return(s.finalSnapshot, nil).Maybe() s.state.On("AtBlockID", mock.AnythingOfType("flow.Identifier")).Return(func(blockID flow.Identifier) protocolint.Snapshot { s.tempSnapshot.On("Head").Unset() s.tempSnapshot.On("Head").Return(func() *flow.Header { @@ -162,12 +162,12 @@ func (s *TransactionStatusSuite) SetupTest() { }, nil) return s.tempSnapshot - }, nil) + }, nil).Maybe() s.finalSnapshot.On("Head").Return(func() *flow.Header { finalizedHeader := s.finalizedBlock.Header return finalizedHeader - }, nil) + }, nil).Maybe() s.blockTracker.On("GetStartHeightFromBlockID", mock.Anything).Return(func(_ flow.Identifier) (uint64, error) { finalizedHeader := s.finalizedBlock.Header From 8eee89a1631a0968684c65f9a955098cba3392b9 Mon Sep 17 00:00:00 2001 From: Andrii Slisarchuk Date: Tue, 19 Nov 2024 14:05:37 +0200 Subject: [PATCH 03/67] Added documentation --- .../rpc/backend/backend_stream_transactions.go | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/engine/access/rpc/backend/backend_stream_transactions.go b/engine/access/rpc/backend/backend_stream_transactions.go index 3bd455dd66d..36163ea30a1 100644 --- a/engine/access/rpc/backend/backend_stream_transactions.go +++ b/engine/access/rpc/backend/backend_stream_transactions.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "go.uber.org/atomic" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -39,7 +40,7 @@ type TransactionSubscriptionMetadata struct { blockWithTx *flow.Header txExecuted bool eventEncodingVersion entities.EventEncodingVersion - triggerFirstPending *atomic.Bool + shouldTriggerPending *atomic.Bool } // SubscribeTransactionStatuses subscribes to transaction status changes starting from the transaction reference block ID. @@ -63,7 +64,7 @@ func (b *backendSubscribeTransactions) SubscribeTransactionStatuses( txReferenceBlockID: tx.ReferenceBlockID, blockWithTx: nil, eventEncodingVersion: requiredEventEncodingVersion, - triggerFirstPending: atomic.NewBool(true), + shouldTriggerPending: atomic.NewBool(true), } return b.subscriptionHandler.Subscribe(ctx, nextHeight, b.getTransactionStatusResponse(&txInfo)) @@ -80,8 +81,8 @@ func (b *backendSubscribeTransactions) getTransactionStatusResponse(txInfo *Tran // The status of the first pending transaction should be returned immediately, as the transaction has already been sent. // This should occur only once for each subscription. - if txInfo.triggerFirstPending.Load() { - txInfo.triggerFirstPending.Toggle() + if txInfo.shouldTriggerPending.Load() { + txInfo.shouldTriggerPending.Toggle() return b.generateResultsWithMissingStatuses(txInfo, flow.TransactionStatusUnknown) } @@ -143,6 +144,11 @@ func (b *backendSubscribeTransactions) getTransactionStatusResponse(txInfo *Tran } } +// getTransactionStatus determines the current status of a transaction based on its metadata +// and previous status. It derives the transaction status by analyzing the transaction's +// execution block, if available, or its reference block. +// +// No errors expected during normal operations. func (b *backendSubscribeTransactions) getTransactionStatus(ctx context.Context, txInfo *TransactionSubscriptionMetadata, prevTxStatus flow.TransactionStatus) (flow.TransactionStatus, error) { txStatus := txInfo.Status var err error From 6766a380daa0ea81719f20a7a5a5ed16f05e7abf Mon Sep 17 00:00:00 2001 From: Andrii Slisarchuk Date: Tue, 19 Nov 2024 14:06:39 +0200 Subject: [PATCH 04/67] revert unnecessary changes --- engine/access/rpc/backend/backend_stream_transactions_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/engine/access/rpc/backend/backend_stream_transactions_test.go b/engine/access/rpc/backend/backend_stream_transactions_test.go index 6d6f98ae1d3..4ae2af7161a 100644 --- a/engine/access/rpc/backend/backend_stream_transactions_test.go +++ b/engine/access/rpc/backend/backend_stream_transactions_test.go @@ -310,7 +310,7 @@ func (s *TransactionStatusSuite) TestSubscribeTransactionStatusHappyCase() { result := txResults[0] assert.Equal(s.T(), txId, result.TransactionID) assert.Equal(s.T(), expectedTxStatus, result.Status) - }, 1000*time.Second, fmt.Sprintf("timed out waiting for transaction info:\n\t- txID: %x\n\t- blockID: %x", txId, s.finalizedBlock.ID())) + }, time.Second, fmt.Sprintf("timed out waiting for transaction info:\n\t- txID: %x\n\t- blockID: %x", txId, s.finalizedBlock.ID())) } // 1. Subscribe to transaction status and receive the first message with pending status From fed835a831ebb7cb6771f62e354b3baa26e4b937 Mon Sep 17 00:00:00 2001 From: Andrii Date: Fri, 22 Nov 2024 16:15:49 +0200 Subject: [PATCH 05/67] Added skeleton for events data provider --- .../data_providers/events_provider.go | 146 ++++++++++++++++++ .../rest/websockets/data_providers/factory.go | 7 +- .../rest/websockets/models/event_models.go | 8 + 3 files changed, 158 insertions(+), 3 deletions(-) create mode 100644 engine/access/rest/websockets/data_providers/events_provider.go create mode 100644 engine/access/rest/websockets/models/event_models.go diff --git a/engine/access/rest/websockets/data_providers/events_provider.go b/engine/access/rest/websockets/data_providers/events_provider.go new file mode 100644 index 00000000000..2bc8612194a --- /dev/null +++ b/engine/access/rest/websockets/data_providers/events_provider.go @@ -0,0 +1,146 @@ +package data_providers + +import ( + "context" + "fmt" + + "github.com/rs/zerolog" + + "github.com/onflow/flow-go/engine/access/rest/common/parser" + "github.com/onflow/flow-go/engine/access/rest/http/request" + "github.com/onflow/flow-go/engine/access/rest/util" + "github.com/onflow/flow-go/engine/access/rest/websockets/models" + "github.com/onflow/flow-go/engine/access/state_stream" + "github.com/onflow/flow-go/engine/access/subscription" + "github.com/onflow/flow-go/model/flow" +) + +// EventsArguments contains the arguments required for subscribing to events +type EventsArguments struct { + StartBlockID flow.Identifier // ID of the block to start subscription from + StartBlockHeight uint64 // Height of the block to start subscription from + Filter state_stream.EventFilter // Filter applied to events for a given subscription +} + +// EventsDataProvider is responsible for providing events +type EventsDataProvider struct { + *BaseDataProviderImpl + + logger zerolog.Logger + args EventsArguments + stateStreamApi state_stream.API +} + +var _ DataProvider = (*EventsDataProvider)(nil) + +// NewEventsDataProvider creates a new instance of EventsDataProvider. +func NewEventsDataProvider( + ctx context.Context, + logger zerolog.Logger, + stateStreamApi state_stream.API, + topic string, + arguments map[string]string, + send chan<- interface{}, +) (*EventsDataProvider, error) { + p := &EventsDataProvider{ + logger: logger.With().Str("component", "events-data-provider").Logger(), + stateStreamApi: stateStreamApi, + } + + // Initialize arguments passed to the provider. + var err error + p.args, err = ParseEventsArguments(arguments) + if err != nil { + return nil, fmt.Errorf("invalid arguments for events data provider: %w", err) + } + + subCtx, cancel := context.WithCancel(ctx) + + // Set up a subscription to events based on arguments. + sub := p.createSubscription(subCtx) + + p.BaseDataProviderImpl = NewBaseDataProviderImpl( + cancel, + topic, + send, + sub, + ) + + return p, nil +} + +// Run starts processing the subscription for events and handles responses. +// +// No errors are expected during normal operations. +func (p *EventsDataProvider) Run() error { + return subscription.HandleSubscription(p.subscription, p.handleResponse(p.send)) +} + +// createSubscription creates a new subscription using the specified input arguments. +func (p *EventsDataProvider) createSubscription(ctx context.Context) subscription.Subscription { + if p.args.StartBlockID != flow.ZeroID && p.args.StartBlockHeight != request.EmptyHeight { + return p.stateStreamApi.SubscribeEvents(ctx, p.args.StartBlockID, p.args.StartBlockHeight, p.args.Filter) + } + + if p.args.StartBlockID != flow.ZeroID { + return p.stateStreamApi.SubscribeEventsFromStartBlockID(ctx, p.args.StartBlockID, p.args.Filter) + } + + if p.args.StartBlockHeight != request.EmptyHeight { + return p.stateStreamApi.SubscribeEventsFromStartHeight(ctx, p.args.StartBlockHeight, p.args.Filter) + } + + return p.stateStreamApi.SubscribeEventsFromLatest(ctx, p.args.Filter) +} + +// handleResponse processes an event and sends the formatted response. +// +// No errors are expected during normal operations. +func (p *EventsDataProvider) handleResponse(send chan<- interface{}) func(*flow.Event) error { + return func(event *flow.Event) error { + send <- &models.EventResponse{ + Event: event, + } + + return nil + } +} + +// ParseEventsArguments validates and initializes the events arguments. +func ParseEventsArguments(arguments map[string]string) (EventsArguments, error) { + var args EventsArguments + + // Parse + if eventStatusIn, ok := arguments["event_filter"]; ok { + eventFilter := parser.ParseEventFilter(eventStatusIn) + if err != nil { + return args, err + } + args.Filter = eventFilter + } else { + return args, fmt.Errorf("'event_filter' must be provided") + } + + // Parse 'start_block_id' if provided + if startBlockIDIn, ok := arguments["start_block_id"]; ok { + var startBlockID parser.ID + err := startBlockID.Parse(startBlockIDIn) + if err != nil { + return args, err + } + args.StartBlockID = startBlockID.Flow() + } + + // Parse 'start_block_height' if provided + if startBlockHeightIn, ok := arguments["start_block_height"]; ok { + var err error + args.StartBlockHeight, err = util.ToUint64(startBlockHeightIn) + if err != nil { + return args, fmt.Errorf("invalid 'start_block_height': %w", err) + } + } else { + args.StartBlockHeight = request.EmptyHeight + } + + return args, nil +} diff --git a/engine/access/rest/websockets/data_providers/factory.go b/engine/access/rest/websockets/data_providers/factory.go index 6ec8dd3185a..a92b689382d 100644 --- a/engine/access/rest/websockets/data_providers/factory.go +++ b/engine/access/rest/websockets/data_providers/factory.go @@ -73,9 +73,10 @@ func (s *DataProviderFactory) NewDataProvider( return NewBlockHeadersDataProvider(ctx, s.logger, s.accessApi, topic, arguments, ch) case BlockDigestsTopic: return NewBlockDigestsDataProvider(ctx, s.logger, s.accessApi, topic, arguments, ch) - // TODO: Implemented handlers for each topic should be added in respective case - case EventsTopic, - AccountStatusesTopic, + case EventsTopic: + return NewEventsDataProvider(ctx, s.logger, s.stateStreamApi, topic, arguments, ch) + // TODO: Implemented handlers for each topic should be added in respective case + case AccountStatusesTopic, TransactionStatusesTopic: return nil, fmt.Errorf("topic \"%s\" not implemented yet", topic) default: diff --git a/engine/access/rest/websockets/models/event_models.go b/engine/access/rest/websockets/models/event_models.go new file mode 100644 index 00000000000..9142c13a7a8 --- /dev/null +++ b/engine/access/rest/websockets/models/event_models.go @@ -0,0 +1,8 @@ +package models + +import "github.com/onflow/flow-go/model/flow" + +// EventResponse is the response message for 'events' topic. +type EventResponse struct { + Event *flow.Event `json:"event"` +} From 9c10d24c77b8958d0948b53d3d752e684359de92 Mon Sep 17 00:00:00 2001 From: Andrii Date: Fri, 22 Nov 2024 17:33:05 +0200 Subject: [PATCH 06/67] Added initializing of events filter, added missing data to factory --- .../data_providers/events_provider.go | 56 +++++++++++++------ .../data_providers/events_provider_test.go | 1 + .../rest/websockets/data_providers/factory.go | 14 +++-- 3 files changed, 50 insertions(+), 21 deletions(-) create mode 100644 engine/access/rest/websockets/data_providers/events_provider_test.go diff --git a/engine/access/rest/websockets/data_providers/events_provider.go b/engine/access/rest/websockets/data_providers/events_provider.go index 2bc8612194a..4da33bd4e64 100644 --- a/engine/access/rest/websockets/data_providers/events_provider.go +++ b/engine/access/rest/websockets/data_providers/events_provider.go @@ -3,6 +3,7 @@ package data_providers import ( "context" "fmt" + "strings" "github.com/rs/zerolog" @@ -38,6 +39,8 @@ func NewEventsDataProvider( ctx context.Context, logger zerolog.Logger, stateStreamApi state_stream.API, + chain flow.Chain, + eventFilterConfig state_stream.EventFilterConfig, topic string, arguments map[string]string, send chan<- interface{}, @@ -49,7 +52,7 @@ func NewEventsDataProvider( // Initialize arguments passed to the provider. var err error - p.args, err = ParseEventsArguments(arguments) + p.args, err = ParseEventsArguments(arguments, chain, eventFilterConfig) if err != nil { return nil, fmt.Errorf("invalid arguments for events data provider: %w", err) } @@ -60,8 +63,8 @@ func NewEventsDataProvider( sub := p.createSubscription(subCtx) p.BaseDataProviderImpl = NewBaseDataProviderImpl( - cancel, topic, + cancel, send, sub, ) @@ -78,10 +81,6 @@ func (p *EventsDataProvider) Run() error { // createSubscription creates a new subscription using the specified input arguments. func (p *EventsDataProvider) createSubscription(ctx context.Context) subscription.Subscription { - if p.args.StartBlockID != flow.ZeroID && p.args.StartBlockHeight != request.EmptyHeight { - return p.stateStreamApi.SubscribeEvents(ctx, p.args.StartBlockID, p.args.StartBlockHeight, p.args.Filter) - } - if p.args.StartBlockID != flow.ZeroID { return p.stateStreamApi.SubscribeEventsFromStartBlockID(ctx, p.args.StartBlockID, p.args.Filter) } @@ -107,24 +106,48 @@ func (p *EventsDataProvider) handleResponse(send chan<- interface{}) func(*flow. } // ParseEventsArguments validates and initializes the events arguments. -func ParseEventsArguments(arguments map[string]string) (EventsArguments, error) { +func ParseEventsArguments( + arguments map[string]string, + chain flow.Chain, + eventFilterConfig state_stream.EventFilterConfig, +) (EventsArguments, error) { var args EventsArguments - // Parse - if eventStatusIn, ok := arguments["event_filter"]; ok { - eventFilter := parser.ParseEventFilter(eventStatusIn) - if err != nil { - return args, err + // Parse 'event_types' as []string{} + var eventTypes []string + if eventTypesIn, ok := arguments["event_types"]; ok { + if eventTypesIn != "" { + eventTypes = strings.Split(eventTypesIn, ",") } - args.Filter = eventFilter - } else { - return args, fmt.Errorf("'event_filter' must be provided") } + // Parse 'addresses' as []string{} + var addresses []string + if addressesIn, ok := arguments["addresses"]; ok { + if addressesIn != "" { + addresses = strings.Split(addressesIn, ",") + } + } + + // Parse 'contracts' as []string{} + var contracts []string + if contractsIn, ok := arguments["contracts"]; ok { + if contractsIn != "" { + contracts = strings.Split(contractsIn, ",") + } + } + + // Initialize the event filter with the parsed arguments + filter, err := state_stream.NewEventFilter(eventFilterConfig, chain, eventTypes, addresses, contracts) + if err != nil { + return args, err + } + args.Filter = filter + // Parse 'start_block_id' if provided if startBlockIDIn, ok := arguments["start_block_id"]; ok { var startBlockID parser.ID - err := startBlockID.Parse(startBlockIDIn) + err = startBlockID.Parse(startBlockIDIn) if err != nil { return args, err } @@ -133,7 +156,6 @@ func ParseEventsArguments(arguments map[string]string) (EventsArguments, error) // Parse 'start_block_height' if provided if startBlockHeightIn, ok := arguments["start_block_height"]; ok { - var err error args.StartBlockHeight, err = util.ToUint64(startBlockHeightIn) if err != nil { return args, fmt.Errorf("invalid 'start_block_height': %w", err) diff --git a/engine/access/rest/websockets/data_providers/events_provider_test.go b/engine/access/rest/websockets/data_providers/events_provider_test.go new file mode 100644 index 00000000000..06387b46331 --- /dev/null +++ b/engine/access/rest/websockets/data_providers/events_provider_test.go @@ -0,0 +1 @@ +package data_providers diff --git a/engine/access/rest/websockets/data_providers/factory.go b/engine/access/rest/websockets/data_providers/factory.go index 465d407c431..4b45bbcccc1 100644 --- a/engine/access/rest/websockets/data_providers/factory.go +++ b/engine/access/rest/websockets/data_providers/factory.go @@ -3,6 +3,7 @@ package data_providers import ( "context" "fmt" + "github.com/onflow/flow-go/model/flow" "github.com/rs/zerolog" @@ -42,6 +43,9 @@ type DataProviderFactoryImpl struct { stateStreamApi state_stream.API accessApi access.API + + chain flow.Chain + eventFilterConfig state_stream.EventFilterConfig } // NewDataProviderFactory creates a new DataProviderFactory @@ -55,11 +59,13 @@ func NewDataProviderFactory( logger zerolog.Logger, stateStreamApi state_stream.API, accessApi access.API, + eventFilterConfig state_stream.EventFilterConfig, ) *DataProviderFactoryImpl { return &DataProviderFactoryImpl{ - logger: logger, - stateStreamApi: stateStreamApi, - accessApi: accessApi, + logger: logger, + stateStreamApi: stateStreamApi, + accessApi: accessApi, + eventFilterConfig: eventFilterConfig, } } @@ -87,7 +93,7 @@ func (s *DataProviderFactoryImpl) NewDataProvider( case BlockDigestsTopic: return NewBlockDigestsDataProvider(ctx, s.logger, s.accessApi, topic, arguments, ch) case EventsTopic: - return NewEventsDataProvider(ctx, s.logger, s.stateStreamApi, topic, arguments, ch) + return NewEventsDataProvider(ctx, s.logger, s.stateStreamApi, s.chain, s.eventFilterConfig, topic, arguments, ch) // TODO: Implemented handlers for each topic should be added in respective case case AccountStatusesTopic, TransactionStatusesTopic: From 9547b5d59cca97dfb74458f7b7e90d096e7305d2 Mon Sep 17 00:00:00 2001 From: Andrii Date: Fri, 22 Nov 2024 17:49:18 +0200 Subject: [PATCH 07/67] Fixed factory test --- engine/access/rest/server.go | 2 +- engine/access/rest/websockets/data_providers/factory.go | 4 +++- engine/access/rest/websockets/data_providers/factory_test.go | 5 ++++- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/engine/access/rest/server.go b/engine/access/rest/server.go index 4f0e2260ae5..71304db22a1 100644 --- a/engine/access/rest/server.go +++ b/engine/access/rest/server.go @@ -51,7 +51,7 @@ func NewServer(serverAPI access.API, builder.AddLegacyWebsocketsRoutes(stateStreamApi, chain, stateStreamConfig, config.MaxRequestSize) } - dataProviderFactory := dp.NewDataProviderFactory(logger, stateStreamApi, serverAPI) + dataProviderFactory := dp.NewDataProviderFactory(logger, stateStreamApi, serverAPI, chain, stateStreamConfig.EventFilterConfig) builder.AddWebsocketsRoute(chain, wsConfig, config.MaxRequestSize, dataProviderFactory) c := cors.New(cors.Options{ diff --git a/engine/access/rest/websockets/data_providers/factory.go b/engine/access/rest/websockets/data_providers/factory.go index 4b45bbcccc1..a0dd6ae5bbd 100644 --- a/engine/access/rest/websockets/data_providers/factory.go +++ b/engine/access/rest/websockets/data_providers/factory.go @@ -3,12 +3,12 @@ package data_providers import ( "context" "fmt" - "github.com/onflow/flow-go/model/flow" "github.com/rs/zerolog" "github.com/onflow/flow-go/access" "github.com/onflow/flow-go/engine/access/state_stream" + "github.com/onflow/flow-go/model/flow" ) // Constants defining various topic names used to specify different types of @@ -59,12 +59,14 @@ func NewDataProviderFactory( logger zerolog.Logger, stateStreamApi state_stream.API, accessApi access.API, + chain flow.Chain, eventFilterConfig state_stream.EventFilterConfig, ) *DataProviderFactoryImpl { return &DataProviderFactoryImpl{ logger: logger, stateStreamApi: stateStreamApi, accessApi: accessApi, + chain: chain, eventFilterConfig: eventFilterConfig, } } diff --git a/engine/access/rest/websockets/data_providers/factory_test.go b/engine/access/rest/websockets/data_providers/factory_test.go index ce4b16e97f6..1f8a7d488be 100644 --- a/engine/access/rest/websockets/data_providers/factory_test.go +++ b/engine/access/rest/websockets/data_providers/factory_test.go @@ -10,6 +10,7 @@ import ( accessmock "github.com/onflow/flow-go/access/mock" "github.com/onflow/flow-go/engine/access/rest/common/parser" + "github.com/onflow/flow-go/engine/access/state_stream" statestreammock "github.com/onflow/flow-go/engine/access/state_stream/mock" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/utils/unittest" @@ -42,7 +43,9 @@ func (s *DataProviderFactorySuite) SetupTest() { s.ctx = context.Background() s.ch = make(chan interface{}) - s.factory = NewDataProviderFactory(log, s.stateStreamApi, s.accessApi) + chain := flow.Testnet.Chain() + + s.factory = NewDataProviderFactory(log, s.stateStreamApi, s.accessApi, chain, state_stream.DefaultEventFilterConfig) s.Require().NotNil(s.factory) } From 0f34ae160f4f5ac1269a266dc0def2f348c89d7b Mon Sep 17 00:00:00 2001 From: Andrii Date: Mon, 25 Nov 2024 16:09:28 +0200 Subject: [PATCH 08/67] Added test skeleton for testing invalid arguments --- .../data_providers/events_provider.go | 71 ++++++------ .../data_providers/events_provider_test.go | 108 ++++++++++++++++++ 2 files changed, 146 insertions(+), 33 deletions(-) diff --git a/engine/access/rest/websockets/data_providers/events_provider.go b/engine/access/rest/websockets/data_providers/events_provider.go index 4da33bd4e64..6cf0f4daea1 100644 --- a/engine/access/rest/websockets/data_providers/events_provider.go +++ b/engine/access/rest/websockets/data_providers/events_provider.go @@ -113,50 +113,30 @@ func ParseEventsArguments( ) (EventsArguments, error) { var args EventsArguments - // Parse 'event_types' as []string{} - var eventTypes []string - if eventTypesIn, ok := arguments["event_types"]; ok { - if eventTypesIn != "" { - eventTypes = strings.Split(eventTypesIn, ",") - } - } + // Check for mutual exclusivity of start_block_id and start_block_height early + _, hasStartBlockID := arguments["start_block_id"] + _, hasStartBlockHeight := arguments["start_block_height"] - // Parse 'addresses' as []string{} - var addresses []string - if addressesIn, ok := arguments["addresses"]; ok { - if addressesIn != "" { - addresses = strings.Split(addressesIn, ",") - } + if hasStartBlockID && hasStartBlockHeight { + return args, fmt.Errorf("can only provide either 'start_block_id' or 'start_block_height'") } - // Parse 'contracts' as []string{} - var contracts []string - if contractsIn, ok := arguments["contracts"]; ok { - if contractsIn != "" { - contracts = strings.Split(contractsIn, ",") - } - } - - // Initialize the event filter with the parsed arguments - filter, err := state_stream.NewEventFilter(eventFilterConfig, chain, eventTypes, addresses, contracts) - if err != nil { - return args, err - } - args.Filter = filter - // Parse 'start_block_id' if provided - if startBlockIDIn, ok := arguments["start_block_id"]; ok { + if hasStartBlockID { var startBlockID parser.ID - err = startBlockID.Parse(startBlockIDIn) + err := startBlockID.Parse(arguments["start_block_id"]) if err != nil { - return args, err + return args, fmt.Errorf("invalid 'start_block_id': %w", err) } args.StartBlockID = startBlockID.Flow() + } else { + args.StartBlockID = flow.ZeroID } // Parse 'start_block_height' if provided - if startBlockHeightIn, ok := arguments["start_block_height"]; ok { - args.StartBlockHeight, err = util.ToUint64(startBlockHeightIn) + if hasStartBlockHeight { + var err error + args.StartBlockHeight, err = util.ToUint64(arguments["start_block_height"]) if err != nil { return args, fmt.Errorf("invalid 'start_block_height': %w", err) } @@ -164,5 +144,30 @@ func ParseEventsArguments( args.StartBlockHeight = request.EmptyHeight } + // Parse 'event_types' as []string{} + var eventTypes []string + if eventTypesIn, ok := arguments["event_types"]; ok && eventTypesIn != "" { + eventTypes = strings.Split(eventTypesIn, ",") + } + + // Parse 'addresses' as []string{} + var addresses []string + if addressesIn, ok := arguments["addresses"]; ok && addressesIn != "" { + addresses = strings.Split(addressesIn, ",") + } + + // Parse 'contracts' as []string{} + var contracts []string + if contractsIn, ok := arguments["contracts"]; ok && contractsIn != "" { + contracts = strings.Split(contractsIn, ",") + } + + // Initialize the event filter with the parsed arguments + filter, err := state_stream.NewEventFilter(eventFilterConfig, chain, eventTypes, addresses, contracts) + if err != nil { + return args, fmt.Errorf("failed to create event filter: %w", err) + } + args.Filter = filter + return args, nil } diff --git a/engine/access/rest/websockets/data_providers/events_provider_test.go b/engine/access/rest/websockets/data_providers/events_provider_test.go index 06387b46331..d3da592fcb9 100644 --- a/engine/access/rest/websockets/data_providers/events_provider_test.go +++ b/engine/access/rest/websockets/data_providers/events_provider_test.go @@ -1 +1,109 @@ package data_providers + +import ( + "context" + "fmt" + "testing" + + "github.com/rs/zerolog" + "github.com/stretchr/testify/suite" + + "github.com/onflow/flow-go/engine/access/state_stream" + ssmock "github.com/onflow/flow-go/engine/access/state_stream/mock" + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/utils/unittest" +) + +// EventsProviderSuite is a test suite for testing the events providers functionality. +type EventsProviderSuite struct { + suite.Suite + + log zerolog.Logger + api *ssmock.API + + chain flow.Chain + rootBlock flow.Block + finalizedBlock *flow.Header +} + +func TestEventsProviderSuite(t *testing.T) { + suite.Run(t, new(EventsProviderSuite)) +} + +func (s *EventsProviderSuite) SetupTest() { + s.log = unittest.Logger() + s.api = ssmock.NewAPI(s.T()) + + s.chain = flow.Testnet.Chain() + + s.rootBlock = unittest.BlockFixture() + s.rootBlock.Header.Height = 0 +} + +// invalidArgumentsTestCases returns a list of test cases with invalid argument combinations +// for testing the behavior of events data providers. Each test case includes a name, +// a set of input arguments, and the expected error message that should be returned. +// +// The test cases cover scenarios such as: +// 1. Supplying both 'start_block_id' and 'start_block_height' simultaneously, which is not allowed. +// 2. Providing invalid 'start_block_id' value. +// 3. Providing invalid 'start_block_height' value. +func (s *EventsProviderSuite) invalidArgumentsTestCases() []testErrType { + return []testErrType{ + { + name: "provide both 'start_block_id' and 'start_block_height' arguments", + arguments: map[string]string{ + "start_block_id": s.rootBlock.ID().String(), + "start_block_height": fmt.Sprintf("%d", s.rootBlock.Header.Height), + }, + expectedErrorMsg: "can only provide either 'start_block_id' or 'start_block_height'", + }, + { + name: "invalid 'start_block_id' argument", + arguments: map[string]string{ + "start_block_id": "invalid_block_id", + }, + expectedErrorMsg: "invalid ID format", + }, + { + name: "invalid 'start_block_height' argument", + arguments: map[string]string{ + "start_block_height": "-1", + }, + expectedErrorMsg: "value must be an unsigned 64 bit integer", + }, + } +} + +// TestEventsDataProvider_InvalidArguments tests the behavior of the event data provider +// when invalid arguments are provided. It verifies that appropriate errors are returned +// for missing or conflicting arguments. +// This test covers the test cases: +// 1. Providing both 'start_block_id' and 'start_block_height' simultaneously. +// 2. Invalid 'start_block_id' argument. +// 3. Invalid 'start_block_height' argument. +func (s *EventsProviderSuite) TestEventsDataProvider_InvalidArguments() { + ctx := context.Background() + send := make(chan interface{}) + + topic := EventsTopic + + for _, test := range s.invalidArgumentsTestCases() { + s.Run(test.name, func() { + provider, err := NewEventsDataProvider( + ctx, + s.log, + s.api, + s.chain, + state_stream.DefaultEventFilterConfig, + topic, + test.arguments, + send) + s.Require().Nil(provider) + s.Require().Error(err) + s.Require().Contains(err.Error(), test.expectedErrorMsg) + }) + } +} + +// TODO: add tests for responses after the WebsocketController is ready From 411f9e5ee5bef0ee7df6612c26c8c08b7eeec999 Mon Sep 17 00:00:00 2001 From: Andrii Date: Tue, 26 Nov 2024 18:40:05 +0200 Subject: [PATCH 09/67] Added test for messageIndex check --- .../data_providers/events_provider.go | 14 +++- .../data_providers/events_provider_test.go | 74 ++++++++++++++++++- .../rest/websockets/models/event_models.go | 3 +- 3 files changed, 88 insertions(+), 3 deletions(-) diff --git a/engine/access/rest/websockets/data_providers/events_provider.go b/engine/access/rest/websockets/data_providers/events_provider.go index 6cf0f4daea1..f4e36bb5c11 100644 --- a/engine/access/rest/websockets/data_providers/events_provider.go +++ b/engine/access/rest/websockets/data_providers/events_provider.go @@ -3,9 +3,12 @@ package data_providers import ( "context" "fmt" + "strconv" "strings" "github.com/rs/zerolog" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" "github.com/onflow/flow-go/engine/access/rest/common/parser" "github.com/onflow/flow-go/engine/access/rest/http/request" @@ -14,6 +17,7 @@ import ( "github.com/onflow/flow-go/engine/access/state_stream" "github.com/onflow/flow-go/engine/access/subscription" "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module/counters" ) // EventsArguments contains the arguments required for subscribing to events @@ -96,9 +100,17 @@ func (p *EventsDataProvider) createSubscription(ctx context.Context) subscriptio // // No errors are expected during normal operations. func (p *EventsDataProvider) handleResponse(send chan<- interface{}) func(*flow.Event) error { + messageIndex := counters.NewMonotonousCounter(0) + return func(event *flow.Event) error { + if ok := messageIndex.Set(messageIndex.Value() + 1); !ok { + return status.Errorf(codes.Internal, "message index already incremented to %d", messageIndex.Value()) + } + index := messageIndex.Value() + send <- &models.EventResponse{ - Event: event, + Event: event, + MessageIndex: strconv.FormatUint(index, 10), } return nil diff --git a/engine/access/rest/websockets/data_providers/events_provider_test.go b/engine/access/rest/websockets/data_providers/events_provider_test.go index d3da592fcb9..8ee8867dc54 100644 --- a/engine/access/rest/websockets/data_providers/events_provider_test.go +++ b/engine/access/rest/websockets/data_providers/events_provider_test.go @@ -3,11 +3,14 @@ package data_providers import ( "context" "fmt" + "strconv" "testing" "github.com/rs/zerolog" + "github.com/stretchr/testify/mock" "github.com/stretchr/testify/suite" + "github.com/onflow/flow-go/engine/access/rest/websockets/models" "github.com/onflow/flow-go/engine/access/state_stream" ssmock "github.com/onflow/flow-go/engine/access/state_stream/mock" "github.com/onflow/flow-go/model/flow" @@ -106,4 +109,73 @@ func (s *EventsProviderSuite) TestEventsDataProvider_InvalidArguments() { } } -// TODO: add tests for responses after the WebsocketController is ready +func (s *EventsProviderSuite) TestMessageIndexEventProviderResponse_HappyPath() { + ctx := context.Background() + send := make(chan interface{}, 10) + topic := EventsTopic + eventsCount := 4 + + // Create a channel to simulate the subscription's event channel + eventChan := make(chan interface{}) + + // Create a mock subscription and mock the channel + sub := ssmock.NewSubscription(s.T()) + sub.On("Channel").Return((<-chan interface{})(eventChan)) + sub.On("Err").Return(nil) + + s.api.On("SubscribeEventsFromStartBlockID", mock.Anything, mock.Anything, mock.Anything).Return(sub) + + arguments := + map[string]string{ + "start_block_id": s.rootBlock.ID().String(), + } + + // Create the EventsDataProvider instance + provider, err := NewEventsDataProvider( + ctx, + s.log, + s.api, + s.chain, + state_stream.DefaultEventFilterConfig, + topic, + arguments, + send) + s.Require().NotNil(provider) + s.Require().NoError(err) + + // Run the provider in a separate goroutine to simulate subscription processing + go func() { + err = provider.Run() + s.Require().NoError(err) + }() + + // Simulate emitting events to the event channel + go func() { + defer close(eventChan) // Close the channel when done + + for i := 0; i < eventsCount; i++ { + eventChan <- &flow.Event{ + Type: "flow.AccountCreated", + } + } + }() + + // Collect responses + var responses []*models.EventResponse + for i := 0; i < eventsCount; i++ { + res := <-send + eventRes, ok := res.(*models.EventResponse) + s.Require().True(ok, "Expected *models.EventResponse, got %T", res) + responses = append(responses, eventRes) + } + + // Verifying that indices are strictly increasing + for i := 1; i < len(responses); i++ { + prevIndex, _ := strconv.Atoi(responses[i-1].MessageIndex) + currentIndex, _ := strconv.Atoi(responses[i].MessageIndex) + s.Require().Equal(prevIndex+1, currentIndex, "Expected MessageIndex to increment by 1") + } + + // Ensure the provider is properly closed after the test + provider.Close() +} diff --git a/engine/access/rest/websockets/models/event_models.go b/engine/access/rest/websockets/models/event_models.go index 9142c13a7a8..0569ebaf4ae 100644 --- a/engine/access/rest/websockets/models/event_models.go +++ b/engine/access/rest/websockets/models/event_models.go @@ -4,5 +4,6 @@ import "github.com/onflow/flow-go/model/flow" // EventResponse is the response message for 'events' topic. type EventResponse struct { - Event *flow.Event `json:"event"` + Event *flow.Event `json:"event"` + MessageIndex string `json:"message_index"` } From 636740a10eceab345bfa82ddda3349ecb04bbad9 Mon Sep 17 00:00:00 2001 From: Andrii Date: Thu, 28 Nov 2024 12:52:49 +0200 Subject: [PATCH 10/67] Added check for a valid event types in parse function --- .../{http/request => common/parser}/event_type.go | 2 +- engine/access/rest/http/request/get_events.go | 2 +- .../rest/websockets/data_providers/events_provider.go | 11 ++++++----- .../websockets/legacy/request/subscribe_events.go | 2 +- 4 files changed, 9 insertions(+), 8 deletions(-) rename engine/access/rest/{http/request => common/parser}/event_type.go (98%) diff --git a/engine/access/rest/http/request/event_type.go b/engine/access/rest/common/parser/event_type.go similarity index 98% rename from engine/access/rest/http/request/event_type.go rename to engine/access/rest/common/parser/event_type.go index c3f425d81c8..f1ba7ca1acb 100644 --- a/engine/access/rest/http/request/event_type.go +++ b/engine/access/rest/common/parser/event_type.go @@ -1,4 +1,4 @@ -package request +package parser import ( "fmt" diff --git a/engine/access/rest/http/request/get_events.go b/engine/access/rest/http/request/get_events.go index c864cf24a47..dee55f98ded 100644 --- a/engine/access/rest/http/request/get_events.go +++ b/engine/access/rest/http/request/get_events.go @@ -71,7 +71,7 @@ func (g *GetEvents) Parse(rawType string, rawStart string, rawEnd string, rawBlo if rawType == "" { return fmt.Errorf("event type must be provided") } - var eventType EventType + var eventType parser.EventType err = eventType.Parse(rawType) if err != nil { return err diff --git a/engine/access/rest/websockets/data_providers/events_provider.go b/engine/access/rest/websockets/data_providers/events_provider.go index f4e36bb5c11..240ede85b71 100644 --- a/engine/access/rest/websockets/data_providers/events_provider.go +++ b/engine/access/rest/websockets/data_providers/events_provider.go @@ -141,8 +141,6 @@ func ParseEventsArguments( return args, fmt.Errorf("invalid 'start_block_id': %w", err) } args.StartBlockID = startBlockID.Flow() - } else { - args.StartBlockID = flow.ZeroID } // Parse 'start_block_height' if provided @@ -157,9 +155,12 @@ func ParseEventsArguments( } // Parse 'event_types' as []string{} - var eventTypes []string + var eventTypes parser.EventTypes if eventTypesIn, ok := arguments["event_types"]; ok && eventTypesIn != "" { - eventTypes = strings.Split(eventTypesIn, ",") + err := eventTypes.Parse(strings.Split(eventTypesIn, ",")) + if err != nil { + return args, fmt.Errorf("invalid 'event_types': %w", err) + } } // Parse 'addresses' as []string{} @@ -175,7 +176,7 @@ func ParseEventsArguments( } // Initialize the event filter with the parsed arguments - filter, err := state_stream.NewEventFilter(eventFilterConfig, chain, eventTypes, addresses, contracts) + filter, err := state_stream.NewEventFilter(eventFilterConfig, chain, eventTypes.Flow(), addresses, contracts) if err != nil { return args, fmt.Errorf("failed to create event filter: %w", err) } diff --git a/engine/access/rest/websockets/legacy/request/subscribe_events.go b/engine/access/rest/websockets/legacy/request/subscribe_events.go index 1110d3582d4..9e53e7c5fca 100644 --- a/engine/access/rest/websockets/legacy/request/subscribe_events.go +++ b/engine/access/rest/websockets/legacy/request/subscribe_events.go @@ -81,7 +81,7 @@ func (g *SubscribeEvents) Parse( g.StartHeight = 0 } - var eventTypes request.EventTypes + var eventTypes parser.EventTypes err = eventTypes.Parse(rawTypes) if err != nil { return err From 588688eaa19e6f972cd29c336a3a6ab1210dbf5b Mon Sep 17 00:00:00 2001 From: Andrii Date: Thu, 28 Nov 2024 13:04:41 +0200 Subject: [PATCH 11/67] Changed type of arguments for consistency --- .../access/rest/websockets/data_providers/events_provider.go | 4 ++-- .../rest/websockets/data_providers/events_provider_test.go | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/engine/access/rest/websockets/data_providers/events_provider.go b/engine/access/rest/websockets/data_providers/events_provider.go index 240ede85b71..94856bab0ec 100644 --- a/engine/access/rest/websockets/data_providers/events_provider.go +++ b/engine/access/rest/websockets/data_providers/events_provider.go @@ -46,7 +46,7 @@ func NewEventsDataProvider( chain flow.Chain, eventFilterConfig state_stream.EventFilterConfig, topic string, - arguments map[string]string, + arguments models.Arguments, send chan<- interface{}, ) (*EventsDataProvider, error) { p := &EventsDataProvider{ @@ -119,7 +119,7 @@ func (p *EventsDataProvider) handleResponse(send chan<- interface{}) func(*flow. // ParseEventsArguments validates and initializes the events arguments. func ParseEventsArguments( - arguments map[string]string, + arguments models.Arguments, chain flow.Chain, eventFilterConfig state_stream.EventFilterConfig, ) (EventsArguments, error) { diff --git a/engine/access/rest/websockets/data_providers/events_provider_test.go b/engine/access/rest/websockets/data_providers/events_provider_test.go index 8ee8867dc54..063d54b4452 100644 --- a/engine/access/rest/websockets/data_providers/events_provider_test.go +++ b/engine/access/rest/websockets/data_providers/events_provider_test.go @@ -55,7 +55,7 @@ func (s *EventsProviderSuite) invalidArgumentsTestCases() []testErrType { return []testErrType{ { name: "provide both 'start_block_id' and 'start_block_height' arguments", - arguments: map[string]string{ + arguments: models.Arguments{ "start_block_id": s.rootBlock.ID().String(), "start_block_height": fmt.Sprintf("%d", s.rootBlock.Header.Height), }, @@ -109,6 +109,7 @@ func (s *EventsProviderSuite) TestEventsDataProvider_InvalidArguments() { } } +// TestMessageIndexEventProviderResponse_HappyPath tests that MessageIndex values in response are strictly increasing. func (s *EventsProviderSuite) TestMessageIndexEventProviderResponse_HappyPath() { ctx := context.Background() send := make(chan interface{}, 10) From b537a5f3e2e4d11caa814898f13943b36fad26db Mon Sep 17 00:00:00 2001 From: Andrii Date: Thu, 28 Nov 2024 13:08:22 +0200 Subject: [PATCH 12/67] Added test case for event provider in factory_test --- .../rest/websockets/data_providers/factory_test.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/engine/access/rest/websockets/data_providers/factory_test.go b/engine/access/rest/websockets/data_providers/factory_test.go index 7989d6c193c..6d39d1c924c 100644 --- a/engine/access/rest/websockets/data_providers/factory_test.go +++ b/engine/access/rest/websockets/data_providers/factory_test.go @@ -104,6 +104,17 @@ func (s *DataProviderFactorySuite) TestSupportedTopics() { s.accessApi.AssertExpectations(s.T()) }, }, + { + name: "events topic", + topic: EventsTopic, + arguments: models.Arguments{}, + setupSubscription: func() { + s.setupSubscription(s.stateStreamApi.On("SubscribeEventsFromLatest", mock.Anything, mock.Anything)) + }, + assertExpectations: func() { + s.stateStreamApi.AssertExpectations(s.T()) + }, + }, } for _, test := range testCases { From 4f63403c9a95ad2d5ab5d5d614bf88ff31c78b92 Mon Sep 17 00:00:00 2001 From: Andrii Date: Fri, 29 Nov 2024 18:32:27 +0200 Subject: [PATCH 13/67] Added implementations for account provider functions --- .../account_statuses_provider.go | 180 ++++++++++++++++++ .../account_statuses_provider_test.go | 1 + .../rest/websockets/models/account_models.go | 11 ++ 3 files changed, 192 insertions(+) create mode 100644 engine/access/rest/websockets/data_providers/account_statuses_provider.go create mode 100644 engine/access/rest/websockets/data_providers/account_statuses_provider_test.go create mode 100644 engine/access/rest/websockets/models/account_models.go diff --git a/engine/access/rest/websockets/data_providers/account_statuses_provider.go b/engine/access/rest/websockets/data_providers/account_statuses_provider.go new file mode 100644 index 00000000000..52b45ace9f6 --- /dev/null +++ b/engine/access/rest/websockets/data_providers/account_statuses_provider.go @@ -0,0 +1,180 @@ +package data_providers + +import ( + "context" + "fmt" + "strconv" + "strings" + + "github.com/rs/zerolog" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + "github.com/onflow/flow-go/engine/access/rest/common/parser" + "github.com/onflow/flow-go/engine/access/rest/http/request" + "github.com/onflow/flow-go/engine/access/rest/util" + "github.com/onflow/flow-go/engine/access/rest/websockets/models" + "github.com/onflow/flow-go/engine/access/state_stream" + "github.com/onflow/flow-go/engine/access/state_stream/backend" + "github.com/onflow/flow-go/engine/access/subscription" + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module/counters" +) + +type AccountStatusesArguments struct { + StartBlockID flow.Identifier // ID of the block to start subscription from + StartBlockHeight uint64 // Height of the block to start subscription from + Filter state_stream.AccountStatusFilter // Filter applied to events for a given subscription +} + +type AccountStatusesDataProvider struct { + *BaseDataProviderImpl + + logger zerolog.Logger + args AccountStatusesArguments + stateStreamApi state_stream.API +} + +var _ DataProvider = (*AccountStatusesDataProvider)(nil) + +// NewAccountStatusesDataProvider creates a new instance of AccountStatusesDataProvider. +func NewAccountStatusesDataProvider( + ctx context.Context, + logger zerolog.Logger, + stateStreamApi state_stream.API, + chain flow.Chain, + eventFilterConfig state_stream.EventFilterConfig, + topic string, + arguments models.Arguments, + send chan<- interface{}, +) (*AccountStatusesDataProvider, error) { + p := &AccountStatusesDataProvider{ + logger: logger.With().Str("component", "account-statuses-data-provider").Logger(), + stateStreamApi: stateStreamApi, + } + + var err error + p.args, err = ParseAccountStatusesArguments(arguments, chain, eventFilterConfig) + if err != nil { + return nil, fmt.Errorf("invalid arguments for account statuses data provider: %w", err) + } + + subCtx, cancel := context.WithCancel(ctx) + + // Set up a subscription to events based on arguments. + sub := p.createSubscription(subCtx) + + p.BaseDataProviderImpl = NewBaseDataProviderImpl( + topic, + cancel, + send, + sub, + ) + + return p, nil +} + +// Run starts processing the subscription for events and handles responses. +// +// No errors are expected during normal operations. +func (p *AccountStatusesDataProvider) Run() error { + return subscription.HandleSubscription(p.subscription, p.handleResponse(p.send)) +} + +// createSubscription creates a new subscription using the specified input arguments. +func (p *AccountStatusesDataProvider) createSubscription(ctx context.Context) subscription.Subscription { + if p.args.StartBlockID != flow.ZeroID { + return p.stateStreamApi.SubscribeAccountStatusesFromStartBlockID(ctx, p.args.StartBlockID, p.args.Filter) + } + + if p.args.StartBlockHeight != request.EmptyHeight { + return p.stateStreamApi.SubscribeAccountStatusesFromStartHeight(ctx, p.args.StartBlockHeight, p.args.Filter) + } + + return p.stateStreamApi.SubscribeAccountStatusesFromLatestBlock(ctx, p.args.Filter) +} + +// handleResponse processes an account statuses and sends the formatted response. +// +// No errors are expected during normal operations. +func (p *AccountStatusesDataProvider) handleResponse(send chan<- interface{}) func(accountStatusesResponse *backend.AccountStatusesResponse) error { + messageIndex := counters.NewMonotonousCounter(0) + + return func(accountStatusesResponse *backend.AccountStatusesResponse) error { + if ok := messageIndex.Set(messageIndex.Value() + 1); !ok { + return status.Errorf(codes.Internal, "message index already incremented to %d", messageIndex.Value()) + } + index := messageIndex.Value() + + send <- &models.AccountStatusesResponse{ + BlockID: accountStatusesResponse.BlockID.String(), + Height: strconv.FormatUint(accountStatusesResponse.Height, 10), + AccountEvents: accountStatusesResponse.AccountEvents, + MessageIndex: strconv.FormatUint(index, 10), + } + + return nil + } +} + +// ParseAccountStatusesArguments validates and initializes the account statuses arguments. +func ParseAccountStatusesArguments( + arguments models.Arguments, + chain flow.Chain, + eventFilterConfig state_stream.EventFilterConfig, +) (AccountStatusesArguments, error) { + var args AccountStatusesArguments + + // Check for mutual exclusivity of start_block_id and start_block_height early + _, hasStartBlockID := arguments["start_block_id"] + _, hasStartBlockHeight := arguments["start_block_height"] + + if hasStartBlockID && hasStartBlockHeight { + return args, fmt.Errorf("can only provide either 'start_block_id' or 'start_block_height'") + } + + // Parse 'start_block_id' if provided + if hasStartBlockID { + var startBlockID parser.ID + err := startBlockID.Parse(arguments["start_block_id"]) + if err != nil { + return args, fmt.Errorf("invalid 'start_block_id': %w", err) + } + args.StartBlockID = startBlockID.Flow() + } + + // Parse 'start_block_height' if provided + if hasStartBlockHeight { + var err error + args.StartBlockHeight, err = util.ToUint64(arguments["start_block_height"]) + if err != nil { + return args, fmt.Errorf("invalid 'start_block_height': %w", err) + } + } else { + args.StartBlockHeight = request.EmptyHeight + } + + // Parse 'event_types' as []string{} + var eventTypes parser.EventTypes + if eventTypesIn, ok := arguments["event_types"]; ok && eventTypesIn != "" { + err := eventTypes.Parse(strings.Split(eventTypesIn, ",")) + if err != nil { + return args, fmt.Errorf("invalid 'event_types': %w", err) + } + } + + // Parse 'accountAddresses' as []string{} + var accountAddresses []string + if addressesIn, ok := arguments["accountAddresses"]; ok && addressesIn != "" { + accountAddresses = strings.Split(addressesIn, ",") + } + + // Initialize the event filter with the parsed arguments + filter, err := state_stream.NewAccountStatusFilter(eventFilterConfig, chain, eventTypes.Flow(), accountAddresses) + if err != nil { + return args, fmt.Errorf("failed to create event filter: %w", err) + } + args.Filter = filter + + return args, nil +} diff --git a/engine/access/rest/websockets/data_providers/account_statuses_provider_test.go b/engine/access/rest/websockets/data_providers/account_statuses_provider_test.go new file mode 100644 index 00000000000..06387b46331 --- /dev/null +++ b/engine/access/rest/websockets/data_providers/account_statuses_provider_test.go @@ -0,0 +1 @@ +package data_providers diff --git a/engine/access/rest/websockets/models/account_models.go b/engine/access/rest/websockets/models/account_models.go new file mode 100644 index 00000000000..712f7a1be6a --- /dev/null +++ b/engine/access/rest/websockets/models/account_models.go @@ -0,0 +1,11 @@ +package models + +import "github.com/onflow/flow-go/model/flow" + +// AccountStatusesResponse is the response message for 'events' topic. +type AccountStatusesResponse struct { + BlockID string `json:"blockID"` + Height string `json:"height"` + AccountEvents map[string]flow.EventsList `json:"account_events"` + MessageIndex string `json:"message_index"` +} From dca9a253070314e2eff5faeac9ac883569c3f6bb Mon Sep 17 00:00:00 2001 From: Andrii Date: Fri, 29 Nov 2024 18:43:27 +0200 Subject: [PATCH 14/67] Fixed remarks --- .../data_providers/events_provider.go | 25 ++++++++----------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/engine/access/rest/websockets/data_providers/events_provider.go b/engine/access/rest/websockets/data_providers/events_provider.go index 94856bab0ec..9ce7b50f03f 100644 --- a/engine/access/rest/websockets/data_providers/events_provider.go +++ b/engine/access/rest/websockets/data_providers/events_provider.go @@ -29,10 +29,9 @@ type EventsArguments struct { // EventsDataProvider is responsible for providing events type EventsDataProvider struct { - *BaseDataProviderImpl + *baseDataProvider logger zerolog.Logger - args EventsArguments stateStreamApi state_stream.API } @@ -55,22 +54,18 @@ func NewEventsDataProvider( } // Initialize arguments passed to the provider. - var err error - p.args, err = ParseEventsArguments(arguments, chain, eventFilterConfig) + eventArgs, err := ParseEventsArguments(arguments, chain, eventFilterConfig) if err != nil { return nil, fmt.Errorf("invalid arguments for events data provider: %w", err) } subCtx, cancel := context.WithCancel(ctx) - // Set up a subscription to events based on arguments. - sub := p.createSubscription(subCtx) - - p.BaseDataProviderImpl = NewBaseDataProviderImpl( + p.baseDataProvider = newBaseDataProvider( topic, cancel, send, - sub, + p.createSubscription(subCtx, eventArgs), // Set up a subscription to events based on arguments. ) return p, nil @@ -84,16 +79,16 @@ func (p *EventsDataProvider) Run() error { } // createSubscription creates a new subscription using the specified input arguments. -func (p *EventsDataProvider) createSubscription(ctx context.Context) subscription.Subscription { - if p.args.StartBlockID != flow.ZeroID { - return p.stateStreamApi.SubscribeEventsFromStartBlockID(ctx, p.args.StartBlockID, p.args.Filter) +func (p *EventsDataProvider) createSubscription(ctx context.Context, args EventsArguments) subscription.Subscription { + if args.StartBlockID != flow.ZeroID { + return p.stateStreamApi.SubscribeEventsFromStartBlockID(ctx, args.StartBlockID, args.Filter) } - if p.args.StartBlockHeight != request.EmptyHeight { - return p.stateStreamApi.SubscribeEventsFromStartHeight(ctx, p.args.StartBlockHeight, p.args.Filter) + if args.StartBlockHeight != request.EmptyHeight { + return p.stateStreamApi.SubscribeEventsFromStartHeight(ctx, args.StartBlockHeight, args.Filter) } - return p.stateStreamApi.SubscribeEventsFromLatest(ctx, p.args.Filter) + return p.stateStreamApi.SubscribeEventsFromLatest(ctx, args.Filter) } // handleResponse processes an event and sends the formatted response. From 9624894bdfb252dc2031784ad99719af6b4c402e Mon Sep 17 00:00:00 2001 From: Andrii Date: Fri, 29 Nov 2024 20:07:37 +0200 Subject: [PATCH 15/67] Added test for invalid arguments and for message index --- .../account_statuses_provider.go | 25 +-- .../account_statuses_provider_test.go | 180 ++++++++++++++++++ .../rest/websockets/data_providers/factory.go | 5 +- .../websockets/data_providers/factory_test.go | 11 ++ 4 files changed, 204 insertions(+), 17 deletions(-) diff --git a/engine/access/rest/websockets/data_providers/account_statuses_provider.go b/engine/access/rest/websockets/data_providers/account_statuses_provider.go index 52b45ace9f6..3e91abff03c 100644 --- a/engine/access/rest/websockets/data_providers/account_statuses_provider.go +++ b/engine/access/rest/websockets/data_providers/account_statuses_provider.go @@ -28,10 +28,9 @@ type AccountStatusesArguments struct { } type AccountStatusesDataProvider struct { - *BaseDataProviderImpl + *baseDataProvider logger zerolog.Logger - args AccountStatusesArguments stateStreamApi state_stream.API } @@ -53,22 +52,18 @@ func NewAccountStatusesDataProvider( stateStreamApi: stateStreamApi, } - var err error - p.args, err = ParseAccountStatusesArguments(arguments, chain, eventFilterConfig) + accountStatusesArgs, err := ParseAccountStatusesArguments(arguments, chain, eventFilterConfig) if err != nil { return nil, fmt.Errorf("invalid arguments for account statuses data provider: %w", err) } subCtx, cancel := context.WithCancel(ctx) - // Set up a subscription to events based on arguments. - sub := p.createSubscription(subCtx) - - p.BaseDataProviderImpl = NewBaseDataProviderImpl( + p.baseDataProvider = newBaseDataProvider( topic, cancel, send, - sub, + p.createSubscription(subCtx, accountStatusesArgs), // Set up a subscription to events based on arguments. ) return p, nil @@ -82,16 +77,16 @@ func (p *AccountStatusesDataProvider) Run() error { } // createSubscription creates a new subscription using the specified input arguments. -func (p *AccountStatusesDataProvider) createSubscription(ctx context.Context) subscription.Subscription { - if p.args.StartBlockID != flow.ZeroID { - return p.stateStreamApi.SubscribeAccountStatusesFromStartBlockID(ctx, p.args.StartBlockID, p.args.Filter) +func (p *AccountStatusesDataProvider) createSubscription(ctx context.Context, args AccountStatusesArguments) subscription.Subscription { + if args.StartBlockID != flow.ZeroID { + return p.stateStreamApi.SubscribeAccountStatusesFromStartBlockID(ctx, args.StartBlockID, args.Filter) } - if p.args.StartBlockHeight != request.EmptyHeight { - return p.stateStreamApi.SubscribeAccountStatusesFromStartHeight(ctx, p.args.StartBlockHeight, p.args.Filter) + if args.StartBlockHeight != request.EmptyHeight { + return p.stateStreamApi.SubscribeAccountStatusesFromStartHeight(ctx, args.StartBlockHeight, args.Filter) } - return p.stateStreamApi.SubscribeAccountStatusesFromLatestBlock(ctx, p.args.Filter) + return p.stateStreamApi.SubscribeAccountStatusesFromLatestBlock(ctx, args.Filter) } // handleResponse processes an account statuses and sends the formatted response. diff --git a/engine/access/rest/websockets/data_providers/account_statuses_provider_test.go b/engine/access/rest/websockets/data_providers/account_statuses_provider_test.go index 06387b46331..0034270f75f 100644 --- a/engine/access/rest/websockets/data_providers/account_statuses_provider_test.go +++ b/engine/access/rest/websockets/data_providers/account_statuses_provider_test.go @@ -1 +1,181 @@ package data_providers + +import ( + "context" + "fmt" + "github.com/onflow/flow-go/engine/access/state_stream/backend" + "strconv" + "testing" + + "github.com/rs/zerolog" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/suite" + + "github.com/onflow/flow-go/engine/access/rest/websockets/models" + "github.com/onflow/flow-go/engine/access/state_stream" + ssmock "github.com/onflow/flow-go/engine/access/state_stream/mock" + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/utils/unittest" +) + +// AccountStatusesProviderSuite is a test suite for testing the account statuses providers functionality. +type AccountStatusesProviderSuite struct { + suite.Suite + + log zerolog.Logger + api *ssmock.API + + chain flow.Chain + rootBlock flow.Block + finalizedBlock *flow.Header +} + +func TestNewAccountStatusesDataProvider(t *testing.T) { + suite.Run(t, new(AccountStatusesProviderSuite)) +} + +func (s *AccountStatusesProviderSuite) SetupTest() { + s.log = unittest.Logger() + s.api = ssmock.NewAPI(s.T()) + + s.chain = flow.Testnet.Chain() + + s.rootBlock = unittest.BlockFixture() + s.rootBlock.Header.Height = 0 +} + +// invalidArgumentsTestCases returns a list of test cases with invalid argument combinations +// for testing the behavior of account statuses data providers. Each test case includes a name, +// a set of input arguments, and the expected error message that should be returned. +// +// The test cases cover scenarios such as: +// 1. Supplying both 'start_block_id' and 'start_block_height' simultaneously, which is not allowed. +// 2. Providing invalid 'start_block_id' value. +// 3. Providing invalid 'start_block_height' value. +func (s *AccountStatusesProviderSuite) invalidArgumentsTestCases() []testErrType { + return []testErrType{ + { + name: "provide both 'start_block_id' and 'start_block_height' arguments", + arguments: models.Arguments{ + "start_block_id": s.rootBlock.ID().String(), + "start_block_height": fmt.Sprintf("%d", s.rootBlock.Header.Height), + }, + expectedErrorMsg: "can only provide either 'start_block_id' or 'start_block_height'", + }, + { + name: "invalid 'start_block_id' argument", + arguments: map[string]string{ + "start_block_id": "invalid_block_id", + }, + expectedErrorMsg: "invalid ID format", + }, + { + name: "invalid 'start_block_height' argument", + arguments: map[string]string{ + "start_block_height": "-1", + }, + expectedErrorMsg: "value must be an unsigned 64 bit integer", + }, + } +} + +// TestAccountStatusesDataProvider_InvalidArguments tests the behavior of the account statuses data provider +// when invalid arguments are provided. It verifies that appropriate errors are returned +// for missing or conflicting arguments. +// This test covers the test cases: +// 1. Providing both 'start_block_id' and 'start_block_height' simultaneously. +// 2. Invalid 'start_block_id' argument. +// 3. Invalid 'start_block_height' argument. +func (s *AccountStatusesProviderSuite) TestAccountStatusesDataProvider_InvalidArguments() { + ctx := context.Background() + send := make(chan interface{}) + + topic := AccountStatusesTopic + + for _, test := range s.invalidArgumentsTestCases() { + s.Run(test.name, func() { + provider, err := NewAccountStatusesDataProvider( + ctx, + s.log, + s.api, + s.chain, + state_stream.DefaultEventFilterConfig, + topic, + test.arguments, + send) + s.Require().Nil(provider) + s.Require().Error(err) + s.Require().Contains(err.Error(), test.expectedErrorMsg) + }) + } +} + +// TestMessageIndexAccountStatusesProviderResponse_HappyPath tests that MessageIndex values in response are strictly increasing. +func (s *AccountStatusesProviderSuite) TestMessageIndexAccountStatusesProviderResponse_HappyPath() { + ctx := context.Background() + send := make(chan interface{}, 10) + topic := AccountStatusesTopic + accountStatusesCount := 4 + + // Create a channel to simulate the subscription's account statuses channel + accountStatusesChan := make(chan interface{}) + + // Create a mock subscription and mock the channel + sub := ssmock.NewSubscription(s.T()) + sub.On("Channel").Return((<-chan interface{})(accountStatusesChan)) + sub.On("Err").Return(nil) + + s.api.On("SubscribeAccountStatusesFromStartBlockID", mock.Anything, mock.Anything, mock.Anything).Return(sub) + + arguments := + map[string]string{ + "start_block_id": s.rootBlock.ID().String(), + } + + // Create the AccountStatusesDataProvider instance + provider, err := NewAccountStatusesDataProvider( + ctx, + s.log, + s.api, + s.chain, + state_stream.DefaultEventFilterConfig, + topic, + arguments, + send) + s.Require().NotNil(provider) + s.Require().NoError(err) + + // Run the provider in a separate goroutine to simulate subscription processing + go func() { + err = provider.Run() + s.Require().NoError(err) + }() + + // Simulate emitting data to the account statuses channel + go func() { + defer close(accountStatusesChan) // Close the channel when done + + for i := 0; i < accountStatusesCount; i++ { + accountStatusesChan <- &backend.AccountStatusesResponse{} + } + }() + + // Collect responses + var responses []*models.AccountStatusesResponse + for i := 0; i < accountStatusesCount; i++ { + res := <-send + accountStatusesRes, ok := res.(*models.AccountStatusesResponse) + s.Require().True(ok, "Expected *models.AccountStatusesResponse, got %T", res) + responses = append(responses, accountStatusesRes) + } + + // Verifying that indices are strictly increasing + for i := 1; i < len(responses); i++ { + prevIndex, _ := strconv.Atoi(responses[i-1].MessageIndex) + currentIndex, _ := strconv.Atoi(responses[i].MessageIndex) + s.Require().Equal(prevIndex+1, currentIndex, "Expected MessageIndex to increment by 1") + } + + // Ensure the provider is properly closed after the test + provider.Close() +} diff --git a/engine/access/rest/websockets/data_providers/factory.go b/engine/access/rest/websockets/data_providers/factory.go index 60af84784a4..6d1689c3977 100644 --- a/engine/access/rest/websockets/data_providers/factory.go +++ b/engine/access/rest/websockets/data_providers/factory.go @@ -97,9 +97,10 @@ func (s *DataProviderFactoryImpl) NewDataProvider( return NewBlockDigestsDataProvider(ctx, s.logger, s.accessApi, topic, arguments, ch) case EventsTopic: return NewEventsDataProvider(ctx, s.logger, s.stateStreamApi, s.chain, s.eventFilterConfig, topic, arguments, ch) + case AccountStatusesTopic: + return NewAccountStatusesDataProvider(ctx, s.logger, s.stateStreamApi, s.chain, s.eventFilterConfig, topic, arguments, ch) + case TransactionStatusesTopic: // TODO: Implemented handlers for each topic should be added in respective case - case AccountStatusesTopic, - TransactionStatusesTopic: return nil, fmt.Errorf(`topic "%s" not implemented yet`, topic) default: return nil, fmt.Errorf("unsupported topic \"%s\"", topic) diff --git a/engine/access/rest/websockets/data_providers/factory_test.go b/engine/access/rest/websockets/data_providers/factory_test.go index 6d39d1c924c..d7be970772c 100644 --- a/engine/access/rest/websockets/data_providers/factory_test.go +++ b/engine/access/rest/websockets/data_providers/factory_test.go @@ -115,6 +115,17 @@ func (s *DataProviderFactorySuite) TestSupportedTopics() { s.stateStreamApi.AssertExpectations(s.T()) }, }, + { + name: "account statuses topic", + topic: AccountStatusesTopic, + arguments: models.Arguments{}, + setupSubscription: func() { + s.setupSubscription(s.stateStreamApi.On("SubscribeAccountStatusesFromLatestBlock", mock.Anything, mock.Anything)) + }, + assertExpectations: func() { + s.stateStreamApi.AssertExpectations(s.T()) + }, + }, } for _, test := range testCases { From 309b148bd60e5628ae08a710400d901aa3449e5f Mon Sep 17 00:00:00 2001 From: Andrii Date: Mon, 2 Dec 2024 14:29:34 +0200 Subject: [PATCH 16/67] Added check for starting index value --- .../access/rest/websockets/data_providers/events_provider.go | 4 ++-- .../rest/websockets/data_providers/events_provider_test.go | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/engine/access/rest/websockets/data_providers/events_provider.go b/engine/access/rest/websockets/data_providers/events_provider.go index 9ce7b50f03f..581f6e79242 100644 --- a/engine/access/rest/websockets/data_providers/events_provider.go +++ b/engine/access/rest/websockets/data_providers/events_provider.go @@ -95,13 +95,13 @@ func (p *EventsDataProvider) createSubscription(ctx context.Context, args Events // // No errors are expected during normal operations. func (p *EventsDataProvider) handleResponse(send chan<- interface{}) func(*flow.Event) error { - messageIndex := counters.NewMonotonousCounter(0) + messageIndex := counters.NewMonotonousCounter(1) return func(event *flow.Event) error { + index := messageIndex.Value() if ok := messageIndex.Set(messageIndex.Value() + 1); !ok { return status.Errorf(codes.Internal, "message index already incremented to %d", messageIndex.Value()) } - index := messageIndex.Value() send <- &models.EventResponse{ Event: event, diff --git a/engine/access/rest/websockets/data_providers/events_provider_test.go b/engine/access/rest/websockets/data_providers/events_provider_test.go index 063d54b4452..d771f1fe58d 100644 --- a/engine/access/rest/websockets/data_providers/events_provider_test.go +++ b/engine/access/rest/websockets/data_providers/events_provider_test.go @@ -170,6 +170,9 @@ func (s *EventsProviderSuite) TestMessageIndexEventProviderResponse_HappyPath() responses = append(responses, eventRes) } + // Verifying that indices are starting from 1 + s.Require().Equal("1", responses[0].MessageIndex, "Expected MessageIndex to start with 1") + // Verifying that indices are strictly increasing for i := 1; i < len(responses); i++ { prevIndex, _ := strconv.Atoi(responses[i-1].MessageIndex) From 47a4c19470dcd23f16ce57c742a932eda6f76838 Mon Sep 17 00:00:00 2001 From: Andrii Date: Mon, 2 Dec 2024 15:10:49 +0200 Subject: [PATCH 17/67] Added check for msgIndex --- .../websockets/data_providers/account_statuses_provider.go | 4 ++-- .../data_providers/account_statuses_provider_test.go | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/engine/access/rest/websockets/data_providers/account_statuses_provider.go b/engine/access/rest/websockets/data_providers/account_statuses_provider.go index 3e91abff03c..1656fb415a7 100644 --- a/engine/access/rest/websockets/data_providers/account_statuses_provider.go +++ b/engine/access/rest/websockets/data_providers/account_statuses_provider.go @@ -93,13 +93,13 @@ func (p *AccountStatusesDataProvider) createSubscription(ctx context.Context, ar // // No errors are expected during normal operations. func (p *AccountStatusesDataProvider) handleResponse(send chan<- interface{}) func(accountStatusesResponse *backend.AccountStatusesResponse) error { - messageIndex := counters.NewMonotonousCounter(0) + messageIndex := counters.NewMonotonousCounter(1) return func(accountStatusesResponse *backend.AccountStatusesResponse) error { + index := messageIndex.Value() if ok := messageIndex.Set(messageIndex.Value() + 1); !ok { return status.Errorf(codes.Internal, "message index already incremented to %d", messageIndex.Value()) } - index := messageIndex.Value() send <- &models.AccountStatusesResponse{ BlockID: accountStatusesResponse.BlockID.String(), diff --git a/engine/access/rest/websockets/data_providers/account_statuses_provider_test.go b/engine/access/rest/websockets/data_providers/account_statuses_provider_test.go index 0034270f75f..b0d939ef8a8 100644 --- a/engine/access/rest/websockets/data_providers/account_statuses_provider_test.go +++ b/engine/access/rest/websockets/data_providers/account_statuses_provider_test.go @@ -169,6 +169,9 @@ func (s *AccountStatusesProviderSuite) TestMessageIndexAccountStatusesProviderRe responses = append(responses, accountStatusesRes) } + // Verifying that indices are starting from 1 + s.Require().Equal("1", responses[0].MessageIndex, "Expected MessageIndex to start with 1") + // Verifying that indices are strictly increasing for i := 1; i < len(responses); i++ { prevIndex, _ := strconv.Atoi(responses[i-1].MessageIndex) From 8ffe02381327bc126e09d3291b24f16c84596e8c Mon Sep 17 00:00:00 2001 From: Andrii Date: Mon, 2 Dec 2024 17:40:47 +0200 Subject: [PATCH 18/67] changed handleResponse to generic --- .../data_providers/blocks_provider_test.go | 3 +- .../data_providers/events_provider.go | 38 ++++++++----------- 2 files changed, 18 insertions(+), 23 deletions(-) diff --git a/engine/access/rest/websockets/data_providers/blocks_provider_test.go b/engine/access/rest/websockets/data_providers/blocks_provider_test.go index 6f46d27ccfe..a886a1474f1 100644 --- a/engine/access/rest/websockets/data_providers/blocks_provider_test.go +++ b/engine/access/rest/websockets/data_providers/blocks_provider_test.go @@ -15,6 +15,7 @@ import ( accessmock "github.com/onflow/flow-go/access/mock" "github.com/onflow/flow-go/engine/access/rest/common/parser" "github.com/onflow/flow-go/engine/access/rest/websockets/models" + "github.com/onflow/flow-go/engine/access/state_stream" statestreamsmock "github.com/onflow/flow-go/engine/access/state_stream/mock" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/utils/unittest" @@ -73,7 +74,7 @@ func (s *BlocksProviderSuite) SetupTest() { } s.finalizedBlock = parent - s.factory = NewDataProviderFactory(s.log, nil, s.api) + s.factory = NewDataProviderFactory(s.log, nil, s.api, flow.Testnet.Chain(), state_stream.DefaultEventFilterConfig) s.Require().NotNil(s.factory) } diff --git a/engine/access/rest/websockets/data_providers/events_provider.go b/engine/access/rest/websockets/data_providers/events_provider.go index 581f6e79242..9c787338ebe 100644 --- a/engine/access/rest/websockets/data_providers/events_provider.go +++ b/engine/access/rest/websockets/data_providers/events_provider.go @@ -75,7 +75,22 @@ func NewEventsDataProvider( // // No errors are expected during normal operations. func (p *EventsDataProvider) Run() error { - return subscription.HandleSubscription(p.subscription, p.handleResponse(p.send)) + messageIndex := counters.NewMonotonousCounter(1) + + return subscription.HandleSubscription( + p.subscription, + subscription.HandleResponse(p.send, func(event *flow.Event) (interface{}, error) { + index := messageIndex.Value() + if ok := messageIndex.Set(messageIndex.Value() + 1); !ok { + return nil, status.Errorf(codes.Internal, "message index already incremented to %d", messageIndex.Value()) + } + + return &models.EventResponse{ + Event: event, + MessageIndex: strconv.FormatUint(index, 10), + }, nil + })) + } // createSubscription creates a new subscription using the specified input arguments. @@ -91,27 +106,6 @@ func (p *EventsDataProvider) createSubscription(ctx context.Context, args Events return p.stateStreamApi.SubscribeEventsFromLatest(ctx, args.Filter) } -// handleResponse processes an event and sends the formatted response. -// -// No errors are expected during normal operations. -func (p *EventsDataProvider) handleResponse(send chan<- interface{}) func(*flow.Event) error { - messageIndex := counters.NewMonotonousCounter(1) - - return func(event *flow.Event) error { - index := messageIndex.Value() - if ok := messageIndex.Set(messageIndex.Value() + 1); !ok { - return status.Errorf(codes.Internal, "message index already incremented to %d", messageIndex.Value()) - } - - send <- &models.EventResponse{ - Event: event, - MessageIndex: strconv.FormatUint(index, 10), - } - - return nil - } -} - // ParseEventsArguments validates and initializes the events arguments. func ParseEventsArguments( arguments models.Arguments, From 13844055f7f76a6f7715c4f59291b249a13ec083 Mon Sep 17 00:00:00 2001 From: Andrii Date: Tue, 3 Dec 2024 19:25:03 +0200 Subject: [PATCH 19/67] Added happy path for testing all subscribe methods --- .../data_providers/events_provider_test.go | 143 ++++++++++++++++++ 1 file changed, 143 insertions(+) diff --git a/engine/access/rest/websockets/data_providers/events_provider_test.go b/engine/access/rest/websockets/data_providers/events_provider_test.go index d771f1fe58d..48b967e4f40 100644 --- a/engine/access/rest/websockets/data_providers/events_provider_test.go +++ b/engine/access/rest/websockets/data_providers/events_provider_test.go @@ -3,8 +3,10 @@ package data_providers import ( "context" "fmt" + "github.com/stretchr/testify/require" "strconv" "testing" + "time" "github.com/rs/zerolog" "github.com/stretchr/testify/mock" @@ -27,6 +29,8 @@ type EventsProviderSuite struct { chain flow.Chain rootBlock flow.Block finalizedBlock *flow.Header + + factory *DataProviderFactoryImpl } func TestEventsProviderSuite(t *testing.T) { @@ -41,6 +45,145 @@ func (s *EventsProviderSuite) SetupTest() { s.rootBlock = unittest.BlockFixture() s.rootBlock.Header.Height = 0 + + s.factory = NewDataProviderFactory(s.log, s.api, nil, flow.Testnet.Chain(), state_stream.DefaultEventFilterConfig) + s.Require().NotNil(s.factory) +} + +// subscribeEventsDataProviderTestCases generates test cases for events data providers. +func (s *EventsProviderSuite) subscribeEventsDataProviderTestCases() []testType { + return []testType{ + { + name: "SubscribeBlocksFromStartBlockID happy path", + arguments: models.Arguments{ + "start_block_id": s.rootBlock.ID().String(), + }, + setupBackend: func(sub *ssmock.Subscription) { + s.api.On( + "SubscribeEventsFromStartBlockID", + mock.Anything, + s.rootBlock.ID(), + mock.Anything, + ).Return(sub).Once() + }, + }, + { + name: "SubscribeEventsFromStartHeight happy path", + arguments: models.Arguments{ + "start_block_height": strconv.FormatUint(s.rootBlock.Header.Height, 10), + }, + setupBackend: func(sub *ssmock.Subscription) { + s.api.On( + "SubscribeEventsFromStartHeight", + mock.Anything, + s.rootBlock.Header.Height, + mock.Anything, + ).Return(sub).Once() + }, + }, + { + name: "SubscribeEventsFromLatest happy path", + arguments: models.Arguments{}, + setupBackend: func(sub *ssmock.Subscription) { + s.api.On( + "SubscribeEventsFromLatest", + mock.Anything, + mock.Anything, + ).Return(sub).Once() + }, + }, + } +} + +// TestEventsDataProvider_HappyPath tests the behavior of the events data provider +// when it is configured correctly and operating under normal conditions. It +// validates that events are correctly streamed to the channel and ensures +// no unexpected errors occur. +func (s *EventsProviderSuite) TestEventsDataProvider_HappyPath() { + s.testHappyPath( + EventsTopic, + s.subscribeEventsDataProviderTestCases(), + s.requireEvents, + ) +} + +// testHappyPath tests a variety of scenarios for data providers in +// happy path scenarios. This function runs parameterized test cases that +// simulate various configurations and verifies that the data provider operates +// as expected without encountering errors. +// +// Arguments: +// - topic: The topic associated with the data provider. +// - tests: A slice of test cases to run, each specifying setup and validation logic. +// - requireFn: A function to validate the output received in the send channel. +func (s *EventsProviderSuite) testHappyPath( + topic string, + tests []testType, + requireFn func(interface{}, *flow.Event), +) { + expectedEvents := []flow.Event{ + unittest.EventFixture(flow.EventAccountCreated, 0, 0, unittest.IdentifierFixture(), 0), + unittest.EventFixture(flow.EventAccountUpdated, 0, 0, unittest.IdentifierFixture(), 0), + unittest.EventFixture(flow.EventAccountCreated, 0, 0, unittest.IdentifierFixture(), 0), + unittest.EventFixture(flow.EventAccountUpdated, 0, 0, unittest.IdentifierFixture(), 0), + } + + for _, test := range tests { + s.Run(test.name, func() { + ctx := context.Background() + send := make(chan interface{}, 10) + + // Create a channel to simulate the subscription's data channel + eventChan := make(chan interface{}) + + // // Create a mock subscription and mock the channel + sub := ssmock.NewSubscription(s.T()) + sub.On("Channel").Return((<-chan interface{})(eventChan)) + sub.On("Err").Return(nil) + test.setupBackend(sub) + + // Create the data provider instance + provider, err := s.factory.NewDataProvider(ctx, topic, test.arguments, send) + s.Require().NotNil(provider) + s.Require().NoError(err) + + // Run the provider in a separate goroutine + go func() { + err = provider.Run() + s.Require().NoError(err) + }() + + // Simulate emitting data to the events channel + go func() { + defer close(eventChan) + + for i := 0; i < len(expectedEvents); i++ { + eventChan <- &expectedEvents[i] + } + }() + + // Collect responses + for _, e := range expectedEvents { + unittest.RequireReturnsBefore(s.T(), func() { + v, ok := <-send + s.Require().True(ok, "channel closed while waiting for event %v: err: %v", e.ID(), sub.Err()) + + requireFn(v, &e) + }, time.Second, fmt.Sprintf("timed out waiting for event %v ", e.ID())) + } + + // Ensure the provider is properly closed after the test + provider.Close() + }) + } +} + +// requireEvents ensures that the received event information matches the expected data. +func (s *EventsProviderSuite) requireEvents(v interface{}, expectedEvent *flow.Event) { + actualResponse, ok := v.(*models.EventResponse) + require.True(s.T(), ok, "Expected *models.EventResponse, got %T", v) + + s.Require().Equal(expectedEvent, actualResponse.Event) } // invalidArgumentsTestCases returns a list of test cases with invalid argument combinations From 57a7c0f37af43cec6f6379a3ee480c760639d9f6 Mon Sep 17 00:00:00 2001 From: Andrii Date: Tue, 3 Dec 2024 20:45:29 +0200 Subject: [PATCH 20/67] Linted --- .../rest/websockets/data_providers/events_provider_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/engine/access/rest/websockets/data_providers/events_provider_test.go b/engine/access/rest/websockets/data_providers/events_provider_test.go index 48b967e4f40..66e1b05f3bd 100644 --- a/engine/access/rest/websockets/data_providers/events_provider_test.go +++ b/engine/access/rest/websockets/data_providers/events_provider_test.go @@ -3,13 +3,13 @@ package data_providers import ( "context" "fmt" - "github.com/stretchr/testify/require" "strconv" "testing" "time" "github.com/rs/zerolog" "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" "github.com/onflow/flow-go/engine/access/rest/websockets/models" From e076072a4469e51b8bddc029d8111e6d673ea8c5 Mon Sep 17 00:00:00 2001 From: Andrii Date: Wed, 4 Dec 2024 12:41:43 +0200 Subject: [PATCH 21/67] Changed order of params --- .../access/rest/websockets/data_providers/events_provider.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/engine/access/rest/websockets/data_providers/events_provider.go b/engine/access/rest/websockets/data_providers/events_provider.go index 9c787338ebe..71a7ef7e67c 100644 --- a/engine/access/rest/websockets/data_providers/events_provider.go +++ b/engine/access/rest/websockets/data_providers/events_provider.go @@ -42,11 +42,11 @@ func NewEventsDataProvider( ctx context.Context, logger zerolog.Logger, stateStreamApi state_stream.API, - chain flow.Chain, - eventFilterConfig state_stream.EventFilterConfig, topic string, arguments models.Arguments, send chan<- interface{}, + chain flow.Chain, + eventFilterConfig state_stream.EventFilterConfig, ) (*EventsDataProvider, error) { p := &EventsDataProvider{ logger: logger.With().Str("component", "events-data-provider").Logger(), From 89bc4c23ef9de2747beff8e69686928d94b7ed1b Mon Sep 17 00:00:00 2001 From: Andrii Date: Wed, 4 Dec 2024 12:46:20 +0200 Subject: [PATCH 22/67] Fixed issues with params order --- .../data_providers/events_provider_test.go | 12 ++++++------ .../access/rest/websockets/data_providers/factory.go | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/engine/access/rest/websockets/data_providers/events_provider_test.go b/engine/access/rest/websockets/data_providers/events_provider_test.go index 66e1b05f3bd..7e5973be34a 100644 --- a/engine/access/rest/websockets/data_providers/events_provider_test.go +++ b/engine/access/rest/websockets/data_providers/events_provider_test.go @@ -240,11 +240,11 @@ func (s *EventsProviderSuite) TestEventsDataProvider_InvalidArguments() { ctx, s.log, s.api, - s.chain, - state_stream.DefaultEventFilterConfig, topic, test.arguments, - send) + send, + s.chain, + state_stream.DefaultEventFilterConfig) s.Require().Nil(provider) s.Require().Error(err) s.Require().Contains(err.Error(), test.expectedErrorMsg) @@ -279,11 +279,11 @@ func (s *EventsProviderSuite) TestMessageIndexEventProviderResponse_HappyPath() ctx, s.log, s.api, - s.chain, - state_stream.DefaultEventFilterConfig, topic, arguments, - send) + send, + s.chain, + state_stream.DefaultEventFilterConfig) s.Require().NotNil(provider) s.Require().NoError(err) diff --git a/engine/access/rest/websockets/data_providers/factory.go b/engine/access/rest/websockets/data_providers/factory.go index 60af84784a4..fc764e4c7fe 100644 --- a/engine/access/rest/websockets/data_providers/factory.go +++ b/engine/access/rest/websockets/data_providers/factory.go @@ -96,7 +96,7 @@ func (s *DataProviderFactoryImpl) NewDataProvider( case BlockDigestsTopic: return NewBlockDigestsDataProvider(ctx, s.logger, s.accessApi, topic, arguments, ch) case EventsTopic: - return NewEventsDataProvider(ctx, s.logger, s.stateStreamApi, s.chain, s.eventFilterConfig, topic, arguments, ch) + return NewEventsDataProvider(ctx, s.logger, s.stateStreamApi, topic, arguments, ch, s.chain, s.eventFilterConfig) // TODO: Implemented handlers for each topic should be added in respective case case AccountStatusesTopic, TransactionStatusesTopic: From a1d7aa7c5bfc89c50490ae25e36f94f4acf61ce0 Mon Sep 17 00:00:00 2001 From: Andrii Date: Wed, 4 Dec 2024 12:49:59 +0200 Subject: [PATCH 23/67] Refactored parse function --- .../rest/websockets/data_providers/events_provider.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/engine/access/rest/websockets/data_providers/events_provider.go b/engine/access/rest/websockets/data_providers/events_provider.go index 71a7ef7e67c..f85430aaa7c 100644 --- a/engine/access/rest/websockets/data_providers/events_provider.go +++ b/engine/access/rest/websockets/data_providers/events_provider.go @@ -115,8 +115,8 @@ func ParseEventsArguments( var args EventsArguments // Check for mutual exclusivity of start_block_id and start_block_height early - _, hasStartBlockID := arguments["start_block_id"] - _, hasStartBlockHeight := arguments["start_block_height"] + startBlockIDIn, hasStartBlockID := arguments["start_block_id"] + startBlockHeightIn, hasStartBlockHeight := arguments["start_block_height"] if hasStartBlockID && hasStartBlockHeight { return args, fmt.Errorf("can only provide either 'start_block_id' or 'start_block_height'") @@ -125,7 +125,7 @@ func ParseEventsArguments( // Parse 'start_block_id' if provided if hasStartBlockID { var startBlockID parser.ID - err := startBlockID.Parse(arguments["start_block_id"]) + err := startBlockID.Parse(startBlockIDIn) if err != nil { return args, fmt.Errorf("invalid 'start_block_id': %w", err) } @@ -135,7 +135,7 @@ func ParseEventsArguments( // Parse 'start_block_height' if provided if hasStartBlockHeight { var err error - args.StartBlockHeight, err = util.ToUint64(arguments["start_block_height"]) + args.StartBlockHeight, err = util.ToUint64(startBlockHeightIn) if err != nil { return args, fmt.Errorf("invalid 'start_block_height': %w", err) } From 8d9e0902d25b72b80ec304b8fb7eac28139f6671 Mon Sep 17 00:00:00 2001 From: Andrii Date: Wed, 4 Dec 2024 13:19:56 +0200 Subject: [PATCH 24/67] Using json arrays instead of comma separeted lists --- .../data_providers/events_provider.go | 20 +++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/engine/access/rest/websockets/data_providers/events_provider.go b/engine/access/rest/websockets/data_providers/events_provider.go index f85430aaa7c..f1b3bc7876b 100644 --- a/engine/access/rest/websockets/data_providers/events_provider.go +++ b/engine/access/rest/websockets/data_providers/events_provider.go @@ -2,6 +2,7 @@ package data_providers import ( "context" + "encoding/json" "fmt" "strconv" "strings" @@ -143,10 +144,15 @@ func ParseEventsArguments( args.StartBlockHeight = request.EmptyHeight } - // Parse 'event_types' as []string{} var eventTypes parser.EventTypes + // Parse 'event_types' as []string{} if eventTypesIn, ok := arguments["event_types"]; ok && eventTypesIn != "" { - err := eventTypes.Parse(strings.Split(eventTypesIn, ",")) + err := json.Unmarshal([]byte(eventTypesIn), &eventTypes) // Expect a JSON array + if err != nil { + return args, fmt.Errorf("could not parse 'event_types': %w", err) + } + + err = eventTypes.Parse(strings.Split(eventTypesIn, ",")) if err != nil { return args, fmt.Errorf("invalid 'event_types': %w", err) } @@ -155,13 +161,19 @@ func ParseEventsArguments( // Parse 'addresses' as []string{} var addresses []string if addressesIn, ok := arguments["addresses"]; ok && addressesIn != "" { - addresses = strings.Split(addressesIn, ",") + err := json.Unmarshal([]byte(addressesIn), &addresses) // Expect a JSON array + if err != nil { + return args, fmt.Errorf("could not parse 'addresses': %w", err) + } } // Parse 'contracts' as []string{} var contracts []string if contractsIn, ok := arguments["contracts"]; ok && contractsIn != "" { - contracts = strings.Split(contractsIn, ",") + err := json.Unmarshal([]byte(contractsIn), &contracts) // Expect a JSON array + if err != nil { + return args, fmt.Errorf("could not parse 'contracts': %w", err) + } } // Initialize the event filter with the parsed arguments From 416ff58d728c22755824602737d5c235a34057dd Mon Sep 17 00:00:00 2001 From: Andrii Date: Wed, 4 Dec 2024 19:28:18 +0200 Subject: [PATCH 25/67] Added heartbeat handling in handleResponse, updated type of expected response, updated tests --- engine/access/rest/server.go | 8 ++- .../data_providers/blocks_provider_test.go | 8 ++- .../data_providers/events_provider.go | 48 ++++++++++++----- .../data_providers/events_provider_test.go | 53 +++++++++++++------ .../rest/websockets/data_providers/factory.go | 5 +- .../websockets/data_providers/factory_test.go | 9 +++- .../rest/websockets/models/event_models.go | 13 +++-- 7 files changed, 107 insertions(+), 37 deletions(-) diff --git a/engine/access/rest/server.go b/engine/access/rest/server.go index 71304db22a1..c45919725b2 100644 --- a/engine/access/rest/server.go +++ b/engine/access/rest/server.go @@ -51,7 +51,13 @@ func NewServer(serverAPI access.API, builder.AddLegacyWebsocketsRoutes(stateStreamApi, chain, stateStreamConfig, config.MaxRequestSize) } - dataProviderFactory := dp.NewDataProviderFactory(logger, stateStreamApi, serverAPI, chain, stateStreamConfig.EventFilterConfig) + dataProviderFactory := dp.NewDataProviderFactory( + logger, + stateStreamApi, + serverAPI, + chain, + stateStreamConfig.EventFilterConfig, + stateStreamConfig.HeartbeatInterval) builder.AddWebsocketsRoute(chain, wsConfig, config.MaxRequestSize, dataProviderFactory) c := cors.New(cors.Options{ diff --git a/engine/access/rest/websockets/data_providers/blocks_provider_test.go b/engine/access/rest/websockets/data_providers/blocks_provider_test.go index 23523f647ee..c82ec6d47d0 100644 --- a/engine/access/rest/websockets/data_providers/blocks_provider_test.go +++ b/engine/access/rest/websockets/data_providers/blocks_provider_test.go @@ -17,6 +17,7 @@ import ( "github.com/onflow/flow-go/engine/access/rest/websockets/models" "github.com/onflow/flow-go/engine/access/state_stream" statestreamsmock "github.com/onflow/flow-go/engine/access/state_stream/mock" + "github.com/onflow/flow-go/engine/access/subscription" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/utils/unittest" ) @@ -74,7 +75,12 @@ func (s *BlocksProviderSuite) SetupTest() { } s.finalizedBlock = parent - s.factory = NewDataProviderFactory(s.log, nil, s.api, flow.Testnet.Chain(), state_stream.DefaultEventFilterConfig) + s.factory = NewDataProviderFactory(s.log, + nil, + s.api, + flow.Testnet.Chain(), + state_stream.DefaultEventFilterConfig, + subscription.DefaultHeartbeatInterval) s.Require().NotNil(s.factory) } diff --git a/engine/access/rest/websockets/data_providers/events_provider.go b/engine/access/rest/websockets/data_providers/events_provider.go index f1b3bc7876b..822b22dad3b 100644 --- a/engine/access/rest/websockets/data_providers/events_provider.go +++ b/engine/access/rest/websockets/data_providers/events_provider.go @@ -16,6 +16,7 @@ import ( "github.com/onflow/flow-go/engine/access/rest/util" "github.com/onflow/flow-go/engine/access/rest/websockets/models" "github.com/onflow/flow-go/engine/access/state_stream" + "github.com/onflow/flow-go/engine/access/state_stream/backend" "github.com/onflow/flow-go/engine/access/subscription" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module/counters" @@ -34,6 +35,8 @@ type EventsDataProvider struct { logger zerolog.Logger stateStreamApi state_stream.API + + heartbeatInterval uint64 } var _ DataProvider = (*EventsDataProvider)(nil) @@ -48,10 +51,12 @@ func NewEventsDataProvider( send chan<- interface{}, chain flow.Chain, eventFilterConfig state_stream.EventFilterConfig, + heartbeatInterval uint64, ) (*EventsDataProvider, error) { p := &EventsDataProvider{ - logger: logger.With().Str("component", "events-data-provider").Logger(), - stateStreamApi: stateStreamApi, + logger: logger.With().Str("component", "events-data-provider").Logger(), + stateStreamApi: stateStreamApi, + heartbeatInterval: heartbeatInterval, } // Initialize arguments passed to the provider. @@ -76,22 +81,39 @@ func NewEventsDataProvider( // // No errors are expected during normal operations. func (p *EventsDataProvider) Run() error { + return subscription.HandleSubscription(p.subscription, p.handleResponse(p.send)) +} + +func (p *EventsDataProvider) handleResponse(send chan<- interface{}) func(eventsResponse *backend.EventsResponse) error { + blocksSinceLastMessage := uint64(0) messageIndex := counters.NewMonotonousCounter(1) - return subscription.HandleSubscription( - p.subscription, - subscription.HandleResponse(p.send, func(event *flow.Event) (interface{}, error) { - index := messageIndex.Value() - if ok := messageIndex.Set(messageIndex.Value() + 1); !ok { - return nil, status.Errorf(codes.Internal, "message index already incremented to %d", messageIndex.Value()) + return func(eventsResponse *backend.EventsResponse) error { + // check if there are any events in the response. if not, do not send a message unless the last + // response was more than HeartbeatInterval blocks ago + if len(eventsResponse.Events) == 0 { + blocksSinceLastMessage++ + if blocksSinceLastMessage < p.heartbeatInterval { + return nil } + blocksSinceLastMessage = 0 + } - return &models.EventResponse{ - Event: event, - MessageIndex: strconv.FormatUint(index, 10), - }, nil - })) + index := messageIndex.Value() + if ok := messageIndex.Set(messageIndex.Value() + 1); !ok { + return status.Errorf(codes.Internal, "message index already incremented to %d", messageIndex.Value()) + } + + send <- &models.EventResponse{ + BlockId: eventsResponse.BlockID.String(), + BlockHeight: strconv.FormatUint(eventsResponse.Height, 10), + BlockTimestamp: eventsResponse.BlockTimestamp, + Events: eventsResponse.Events, + MessageIndex: strconv.FormatUint(index, 10), + } + return nil + } } // createSubscription creates a new subscription using the specified input arguments. diff --git a/engine/access/rest/websockets/data_providers/events_provider_test.go b/engine/access/rest/websockets/data_providers/events_provider_test.go index 7e5973be34a..fe7ccba6011 100644 --- a/engine/access/rest/websockets/data_providers/events_provider_test.go +++ b/engine/access/rest/websockets/data_providers/events_provider_test.go @@ -5,7 +5,6 @@ import ( "fmt" "strconv" "testing" - "time" "github.com/rs/zerolog" "github.com/stretchr/testify/mock" @@ -14,7 +13,9 @@ import ( "github.com/onflow/flow-go/engine/access/rest/websockets/models" "github.com/onflow/flow-go/engine/access/state_stream" + "github.com/onflow/flow-go/engine/access/state_stream/backend" ssmock "github.com/onflow/flow-go/engine/access/state_stream/mock" + "github.com/onflow/flow-go/engine/access/subscription" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/utils/unittest" ) @@ -46,7 +47,13 @@ func (s *EventsProviderSuite) SetupTest() { s.rootBlock = unittest.BlockFixture() s.rootBlock.Header.Height = 0 - s.factory = NewDataProviderFactory(s.log, s.api, nil, flow.Testnet.Chain(), state_stream.DefaultEventFilterConfig) + s.factory = NewDataProviderFactory( + s.log, + s.api, + nil, + flow.Testnet.Chain(), + state_stream.DefaultEventFilterConfig, + subscription.DefaultHeartbeatInterval) s.Require().NotNil(s.factory) } @@ -119,7 +126,7 @@ func (s *EventsProviderSuite) TestEventsDataProvider_HappyPath() { func (s *EventsProviderSuite) testHappyPath( topic string, tests []testType, - requireFn func(interface{}, *flow.Event), + requireFn func(interface{}, *backend.EventsResponse), ) { expectedEvents := []flow.Event{ unittest.EventFixture(flow.EventAccountCreated, 0, 0, unittest.IdentifierFixture(), 0), @@ -128,6 +135,18 @@ func (s *EventsProviderSuite) testHappyPath( unittest.EventFixture(flow.EventAccountUpdated, 0, 0, unittest.IdentifierFixture(), 0), } + var expectedEventsResponses []backend.EventsResponse + + for i := 0; i < len(expectedEvents); i++ { + expectedEventsResponses = append(expectedEventsResponses, backend.EventsResponse{ + Height: s.rootBlock.Header.Height, + BlockID: s.rootBlock.ID(), + Events: expectedEvents, + BlockTimestamp: s.rootBlock.Header.Timestamp, + }) + + } + for _, test := range tests { s.Run(test.name, func() { ctx := context.Background() @@ -157,19 +176,17 @@ func (s *EventsProviderSuite) testHappyPath( go func() { defer close(eventChan) - for i := 0; i < len(expectedEvents); i++ { - eventChan <- &expectedEvents[i] + for i := 0; i < len(expectedEventsResponses); i++ { + eventChan <- &expectedEventsResponses[i] } }() // Collect responses - for _, e := range expectedEvents { - unittest.RequireReturnsBefore(s.T(), func() { - v, ok := <-send - s.Require().True(ok, "channel closed while waiting for event %v: err: %v", e.ID(), sub.Err()) + for _, e := range expectedEventsResponses { + v, ok := <-send + s.Require().True(ok, "channel closed while waiting for event %v: err: %v", e.BlockID, sub.Err()) - requireFn(v, &e) - }, time.Second, fmt.Sprintf("timed out waiting for event %v ", e.ID())) + requireFn(v, &e) } // Ensure the provider is properly closed after the test @@ -179,11 +196,11 @@ func (s *EventsProviderSuite) testHappyPath( } // requireEvents ensures that the received event information matches the expected data. -func (s *EventsProviderSuite) requireEvents(v interface{}, expectedEvent *flow.Event) { +func (s *EventsProviderSuite) requireEvents(v interface{}, expectedEventsResponse *backend.EventsResponse) { actualResponse, ok := v.(*models.EventResponse) require.True(s.T(), ok, "Expected *models.EventResponse, got %T", v) - s.Require().Equal(expectedEvent, actualResponse.Event) + s.Require().ElementsMatch(expectedEventsResponse.Events, actualResponse.Events) } // invalidArgumentsTestCases returns a list of test cases with invalid argument combinations @@ -244,7 +261,8 @@ func (s *EventsProviderSuite) TestEventsDataProvider_InvalidArguments() { test.arguments, send, s.chain, - state_stream.DefaultEventFilterConfig) + state_stream.DefaultEventFilterConfig, + subscription.DefaultHeartbeatInterval) s.Require().Nil(provider) s.Require().Error(err) s.Require().Contains(err.Error(), test.expectedErrorMsg) @@ -283,7 +301,8 @@ func (s *EventsProviderSuite) TestMessageIndexEventProviderResponse_HappyPath() arguments, send, s.chain, - state_stream.DefaultEventFilterConfig) + state_stream.DefaultEventFilterConfig, + subscription.DefaultHeartbeatInterval) s.Require().NotNil(provider) s.Require().NoError(err) @@ -298,8 +317,8 @@ func (s *EventsProviderSuite) TestMessageIndexEventProviderResponse_HappyPath() defer close(eventChan) // Close the channel when done for i := 0; i < eventsCount; i++ { - eventChan <- &flow.Event{ - Type: "flow.AccountCreated", + eventChan <- &backend.EventsResponse{ + Height: s.rootBlock.Header.Height, } } }() diff --git a/engine/access/rest/websockets/data_providers/factory.go b/engine/access/rest/websockets/data_providers/factory.go index fc764e4c7fe..e39a6643d4e 100644 --- a/engine/access/rest/websockets/data_providers/factory.go +++ b/engine/access/rest/websockets/data_providers/factory.go @@ -47,6 +47,7 @@ type DataProviderFactoryImpl struct { chain flow.Chain eventFilterConfig state_stream.EventFilterConfig + heartbeatInterval uint64 } // NewDataProviderFactory creates a new DataProviderFactory @@ -62,6 +63,7 @@ func NewDataProviderFactory( accessApi access.API, chain flow.Chain, eventFilterConfig state_stream.EventFilterConfig, + heartbeatInterval uint64, ) *DataProviderFactoryImpl { return &DataProviderFactoryImpl{ logger: logger, @@ -69,6 +71,7 @@ func NewDataProviderFactory( accessApi: accessApi, chain: chain, eventFilterConfig: eventFilterConfig, + heartbeatInterval: heartbeatInterval, } } @@ -96,7 +99,7 @@ func (s *DataProviderFactoryImpl) NewDataProvider( case BlockDigestsTopic: return NewBlockDigestsDataProvider(ctx, s.logger, s.accessApi, topic, arguments, ch) case EventsTopic: - return NewEventsDataProvider(ctx, s.logger, s.stateStreamApi, topic, arguments, ch, s.chain, s.eventFilterConfig) + return NewEventsDataProvider(ctx, s.logger, s.stateStreamApi, topic, arguments, ch, s.chain, s.eventFilterConfig, s.heartbeatInterval) // TODO: Implemented handlers for each topic should be added in respective case case AccountStatusesTopic, TransactionStatusesTopic: diff --git a/engine/access/rest/websockets/data_providers/factory_test.go b/engine/access/rest/websockets/data_providers/factory_test.go index 9ea21f70270..38618a5c29a 100644 --- a/engine/access/rest/websockets/data_providers/factory_test.go +++ b/engine/access/rest/websockets/data_providers/factory_test.go @@ -13,6 +13,7 @@ import ( "github.com/onflow/flow-go/engine/access/rest/websockets/models" "github.com/onflow/flow-go/engine/access/state_stream" statestreammock "github.com/onflow/flow-go/engine/access/state_stream/mock" + "github.com/onflow/flow-go/engine/access/subscription" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/utils/unittest" ) @@ -46,7 +47,13 @@ func (s *DataProviderFactorySuite) SetupTest() { chain := flow.Testnet.Chain() - s.factory = NewDataProviderFactory(log, s.stateStreamApi, s.accessApi, chain, state_stream.DefaultEventFilterConfig) + s.factory = NewDataProviderFactory( + log, + s.stateStreamApi, + s.accessApi, + chain, + state_stream.DefaultEventFilterConfig, + subscription.DefaultHeartbeatInterval) s.Require().NotNil(s.factory) } diff --git a/engine/access/rest/websockets/models/event_models.go b/engine/access/rest/websockets/models/event_models.go index 0569ebaf4ae..48d085d9b85 100644 --- a/engine/access/rest/websockets/models/event_models.go +++ b/engine/access/rest/websockets/models/event_models.go @@ -1,9 +1,16 @@ package models -import "github.com/onflow/flow-go/model/flow" +import ( + "time" + + "github.com/onflow/flow-go/model/flow" +) // EventResponse is the response message for 'events' topic. type EventResponse struct { - Event *flow.Event `json:"event"` - MessageIndex string `json:"message_index"` + BlockId string `json:"block_id"` + BlockHeight string `json:"block_height"` + BlockTimestamp time.Time `json:"block_timestamp"` + Events []flow.Event `json:"events"` + MessageIndex string `json:"message_index"` } From 179a6646c1f821aaf39a9aa4ed4abbfa5a32bd59 Mon Sep 17 00:00:00 2001 From: Andrii Date: Thu, 5 Dec 2024 15:38:42 +0200 Subject: [PATCH 26/67] Fixed small remarks --- .../websockets/data_providers/blocks_provider_test.go | 3 ++- .../rest/websockets/data_providers/events_provider.go | 11 ++++------- .../websockets/data_providers/events_provider_test.go | 3 ++- .../rest/websockets/data_providers/factory_test.go | 3 ++- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/engine/access/rest/websockets/data_providers/blocks_provider_test.go b/engine/access/rest/websockets/data_providers/blocks_provider_test.go index c82ec6d47d0..51cf9e63e44 100644 --- a/engine/access/rest/websockets/data_providers/blocks_provider_test.go +++ b/engine/access/rest/websockets/data_providers/blocks_provider_test.go @@ -75,7 +75,8 @@ func (s *BlocksProviderSuite) SetupTest() { } s.finalizedBlock = parent - s.factory = NewDataProviderFactory(s.log, + s.factory = NewDataProviderFactory( + s.log, nil, s.api, flow.Testnet.Chain(), diff --git a/engine/access/rest/websockets/data_providers/events_provider.go b/engine/access/rest/websockets/data_providers/events_provider.go index 822b22dad3b..6305d614bde 100644 --- a/engine/access/rest/websockets/data_providers/events_provider.go +++ b/engine/access/rest/websockets/data_providers/events_provider.go @@ -8,8 +8,6 @@ import ( "strings" "github.com/rs/zerolog" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" "github.com/onflow/flow-go/engine/access/rest/common/parser" "github.com/onflow/flow-go/engine/access/rest/http/request" @@ -81,10 +79,10 @@ func NewEventsDataProvider( // // No errors are expected during normal operations. func (p *EventsDataProvider) Run() error { - return subscription.HandleSubscription(p.subscription, p.handleResponse(p.send)) + return subscription.HandleSubscription(p.subscription, p.handleResponse()) } -func (p *EventsDataProvider) handleResponse(send chan<- interface{}) func(eventsResponse *backend.EventsResponse) error { +func (p *EventsDataProvider) handleResponse() func(eventsResponse *backend.EventsResponse) error { blocksSinceLastMessage := uint64(0) messageIndex := counters.NewMonotonousCounter(1) @@ -101,10 +99,10 @@ func (p *EventsDataProvider) handleResponse(send chan<- interface{}) func(events index := messageIndex.Value() if ok := messageIndex.Set(messageIndex.Value() + 1); !ok { - return status.Errorf(codes.Internal, "message index already incremented to %d", messageIndex.Value()) + return fmt.Errorf("message index already incremented to: %d", messageIndex.Value()) } - send <- &models.EventResponse{ + p.send <- &models.EventResponse{ BlockId: eventsResponse.BlockID.String(), BlockHeight: strconv.FormatUint(eventsResponse.Height, 10), BlockTimestamp: eventsResponse.BlockTimestamp, @@ -167,7 +165,6 @@ func ParseEventsArguments( } var eventTypes parser.EventTypes - // Parse 'event_types' as []string{} if eventTypesIn, ok := arguments["event_types"]; ok && eventTypesIn != "" { err := json.Unmarshal([]byte(eventTypesIn), &eventTypes) // Expect a JSON array if err != nil { diff --git a/engine/access/rest/websockets/data_providers/events_provider_test.go b/engine/access/rest/websockets/data_providers/events_provider_test.go index fe7ccba6011..bc928a601cf 100644 --- a/engine/access/rest/websockets/data_providers/events_provider_test.go +++ b/engine/access/rest/websockets/data_providers/events_provider_test.go @@ -262,7 +262,8 @@ func (s *EventsProviderSuite) TestEventsDataProvider_InvalidArguments() { send, s.chain, state_stream.DefaultEventFilterConfig, - subscription.DefaultHeartbeatInterval) + subscription.DefaultHeartbeatInterval, + ) s.Require().Nil(provider) s.Require().Error(err) s.Require().Contains(err.Error(), test.expectedErrorMsg) diff --git a/engine/access/rest/websockets/data_providers/factory_test.go b/engine/access/rest/websockets/data_providers/factory_test.go index 38618a5c29a..3323e3cc258 100644 --- a/engine/access/rest/websockets/data_providers/factory_test.go +++ b/engine/access/rest/websockets/data_providers/factory_test.go @@ -53,7 +53,8 @@ func (s *DataProviderFactorySuite) SetupTest() { s.accessApi, chain, state_stream.DefaultEventFilterConfig, - subscription.DefaultHeartbeatInterval) + subscription.DefaultHeartbeatInterval, + ) s.Require().NotNil(s.factory) } From e0aa808ee8fff8c180ba94c8be418cbc7442af0e Mon Sep 17 00:00:00 2001 From: Andrii Date: Thu, 5 Dec 2024 15:43:44 +0200 Subject: [PATCH 27/67] Made parse function private --- .../rest/websockets/data_providers/events_provider.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/engine/access/rest/websockets/data_providers/events_provider.go b/engine/access/rest/websockets/data_providers/events_provider.go index 6305d614bde..acd719f4f06 100644 --- a/engine/access/rest/websockets/data_providers/events_provider.go +++ b/engine/access/rest/websockets/data_providers/events_provider.go @@ -58,7 +58,7 @@ func NewEventsDataProvider( } // Initialize arguments passed to the provider. - eventArgs, err := ParseEventsArguments(arguments, chain, eventFilterConfig) + eventArgs, err := parseEventsArguments(arguments, chain, eventFilterConfig) if err != nil { return nil, fmt.Errorf("invalid arguments for events data provider: %w", err) } @@ -127,8 +127,8 @@ func (p *EventsDataProvider) createSubscription(ctx context.Context, args Events return p.stateStreamApi.SubscribeEventsFromLatest(ctx, args.Filter) } -// ParseEventsArguments validates and initializes the events arguments. -func ParseEventsArguments( +// parseEventsArguments validates and initializes the events arguments. +func parseEventsArguments( arguments models.Arguments, chain flow.Chain, eventFilterConfig state_stream.EventFilterConfig, From 19bf3c9b9c1580bf06bd71d153d45af8cbac5711 Mon Sep 17 00:00:00 2001 From: Andrii Date: Thu, 5 Dec 2024 19:16:05 +0200 Subject: [PATCH 28/67] Changed Arguments type and refactored code --- .../data_providers/blocks_provider.go | 23 ++++++++---- .../data_providers/events_provider.go | 35 +++++++++++++++---- .../data_providers/events_provider_test.go | 6 ++-- .../rest/websockets/models/subscribe.go | 4 ++- 4 files changed, 51 insertions(+), 17 deletions(-) diff --git a/engine/access/rest/websockets/data_providers/blocks_provider.go b/engine/access/rest/websockets/data_providers/blocks_provider.go index 72cfaa6f554..28e0a9a03d2 100644 --- a/engine/access/rest/websockets/data_providers/blocks_provider.go +++ b/engine/access/rest/websockets/data_providers/blocks_provider.go @@ -96,7 +96,11 @@ func ParseBlocksArguments(arguments models.Arguments) (BlocksArguments, error) { // Parse 'block_status' if blockStatusIn, ok := arguments["block_status"]; ok { - blockStatus, err := parser.ParseBlockStatus(blockStatusIn) + result, ok := blockStatusIn.(string) + if !ok { + return args, fmt.Errorf("'block_status' must be string") + } + blockStatus, err := parser.ParseBlockStatus(result) if err != nil { return args, err } @@ -113,24 +117,31 @@ func ParseBlocksArguments(arguments models.Arguments) (BlocksArguments, error) { return args, fmt.Errorf("can only provide either 'start_block_id' or 'start_block_height'") } - // Parse 'start_block_id' if provided if hasStartBlockID { + result, ok := startBlockIDIn.(string) + if !ok { + return args, fmt.Errorf("'start_block_id' must be a string") + } var startBlockID parser.ID - err := startBlockID.Parse(startBlockIDIn) + err := startBlockID.Parse(result) if err != nil { return args, err } args.StartBlockID = startBlockID.Flow() } - // Parse 'start_block_height' if provided if hasStartBlockHeight { - var err error - args.StartBlockHeight, err = util.ToUint64(startBlockHeightIn) + result, ok := startBlockHeightIn.(string) + if !ok { + return args, fmt.Errorf("'start_block_height' must be a string") + } + startBlockHeight, err := util.ToUint64(result) if err != nil { return args, fmt.Errorf("invalid 'start_block_height': %w", err) } + args.StartBlockHeight = startBlockHeight } else { + // Default value if 'start_block_height' is not provided args.StartBlockHeight = request.EmptyHeight } diff --git a/engine/access/rest/websockets/data_providers/events_provider.go b/engine/access/rest/websockets/data_providers/events_provider.go index acd719f4f06..6edf203bf2a 100644 --- a/engine/access/rest/websockets/data_providers/events_provider.go +++ b/engine/access/rest/websockets/data_providers/events_provider.go @@ -145,8 +145,12 @@ func parseEventsArguments( // Parse 'start_block_id' if provided if hasStartBlockID { + result, ok := startBlockIDIn.(string) + if !ok { + return args, fmt.Errorf("'start_block_id' must be a string") + } var startBlockID parser.ID - err := startBlockID.Parse(startBlockIDIn) + err := startBlockID.Parse(result) if err != nil { return args, fmt.Errorf("invalid 'start_block_id': %w", err) } @@ -155,23 +159,32 @@ func parseEventsArguments( // Parse 'start_block_height' if provided if hasStartBlockHeight { - var err error - args.StartBlockHeight, err = util.ToUint64(startBlockHeightIn) + result, ok := startBlockHeightIn.(string) + if !ok { + return args, fmt.Errorf("'start_block_height' must be a string") + } + startBlockHeight, err := util.ToUint64(result) if err != nil { return args, fmt.Errorf("invalid 'start_block_height': %w", err) } + args.StartBlockHeight = startBlockHeight } else { args.StartBlockHeight = request.EmptyHeight } + // Parse 'event_types' as a JSON array var eventTypes parser.EventTypes if eventTypesIn, ok := arguments["event_types"]; ok && eventTypesIn != "" { - err := json.Unmarshal([]byte(eventTypesIn), &eventTypes) // Expect a JSON array + result, ok := eventTypesIn.(string) + if !ok { + return args, fmt.Errorf("'event_types' must be a string") + } + err := json.Unmarshal([]byte(result), &eventTypes) // Expect a JSON array if err != nil { return args, fmt.Errorf("could not parse 'event_types': %w", err) } - err = eventTypes.Parse(strings.Split(eventTypesIn, ",")) + err = eventTypes.Parse(strings.Split(result, ",")) if err != nil { return args, fmt.Errorf("invalid 'event_types': %w", err) } @@ -180,7 +193,11 @@ func parseEventsArguments( // Parse 'addresses' as []string{} var addresses []string if addressesIn, ok := arguments["addresses"]; ok && addressesIn != "" { - err := json.Unmarshal([]byte(addressesIn), &addresses) // Expect a JSON array + result, ok := addressesIn.(string) + if !ok { + return args, fmt.Errorf("'addresses' must be a string") + } + err := json.Unmarshal([]byte(result), &addresses) // Expect a JSON array if err != nil { return args, fmt.Errorf("could not parse 'addresses': %w", err) } @@ -189,7 +206,11 @@ func parseEventsArguments( // Parse 'contracts' as []string{} var contracts []string if contractsIn, ok := arguments["contracts"]; ok && contractsIn != "" { - err := json.Unmarshal([]byte(contractsIn), &contracts) // Expect a JSON array + result, ok := contractsIn.(string) + if !ok { + return args, fmt.Errorf("'contracts' must be a string") + } + err := json.Unmarshal([]byte(result), &contracts) // Expect a JSON array if err != nil { return args, fmt.Errorf("could not parse 'contracts': %w", err) } diff --git a/engine/access/rest/websockets/data_providers/events_provider_test.go b/engine/access/rest/websockets/data_providers/events_provider_test.go index bc928a601cf..8721f407946 100644 --- a/engine/access/rest/websockets/data_providers/events_provider_test.go +++ b/engine/access/rest/websockets/data_providers/events_provider_test.go @@ -223,14 +223,14 @@ func (s *EventsProviderSuite) invalidArgumentsTestCases() []testErrType { }, { name: "invalid 'start_block_id' argument", - arguments: map[string]string{ + arguments: map[string]interface{}{ "start_block_id": "invalid_block_id", }, expectedErrorMsg: "invalid ID format", }, { name: "invalid 'start_block_height' argument", - arguments: map[string]string{ + arguments: map[string]interface{}{ "start_block_height": "-1", }, expectedErrorMsg: "value must be an unsigned 64 bit integer", @@ -289,7 +289,7 @@ func (s *EventsProviderSuite) TestMessageIndexEventProviderResponse_HappyPath() s.api.On("SubscribeEventsFromStartBlockID", mock.Anything, mock.Anything, mock.Anything).Return(sub) arguments := - map[string]string{ + map[string]interface{}{ "start_block_id": s.rootBlock.ID().String(), } diff --git a/engine/access/rest/websockets/models/subscribe.go b/engine/access/rest/websockets/models/subscribe.go index 95ad17e3708..85d28f6d8ab 100644 --- a/engine/access/rest/websockets/models/subscribe.go +++ b/engine/access/rest/websockets/models/subscribe.go @@ -1,6 +1,8 @@ package models -type Arguments map[string]string +//type Arguments map[string]string + +type Arguments map[string]interface{} // SubscribeMessageRequest represents a request to subscribe to a topic. type SubscribeMessageRequest struct { From 8d0543d6e8d69cd4058028741aecae2d8acc4643 Mon Sep 17 00:00:00 2001 From: Andrii Date: Thu, 5 Dec 2024 19:16:30 +0200 Subject: [PATCH 29/67] Removed comment --- engine/access/rest/websockets/models/subscribe.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/engine/access/rest/websockets/models/subscribe.go b/engine/access/rest/websockets/models/subscribe.go index 85d28f6d8ab..03b37aee5f1 100644 --- a/engine/access/rest/websockets/models/subscribe.go +++ b/engine/access/rest/websockets/models/subscribe.go @@ -1,7 +1,5 @@ package models -//type Arguments map[string]string - type Arguments map[string]interface{} // SubscribeMessageRequest represents a request to subscribe to a topic. From d6fc9dfe6aff11f655dea036ad495f8b138f8985 Mon Sep 17 00:00:00 2001 From: Andrii Date: Fri, 6 Dec 2024 15:56:02 +0200 Subject: [PATCH 30/67] Fixed parse function for event privoder --- .../data_providers/events_provider.go | 27 +++++-------------- .../data_providers/events_provider_test.go | 1 + 2 files changed, 7 insertions(+), 21 deletions(-) diff --git a/engine/access/rest/websockets/data_providers/events_provider.go b/engine/access/rest/websockets/data_providers/events_provider.go index 6edf203bf2a..dfbeb5c7c44 100644 --- a/engine/access/rest/websockets/data_providers/events_provider.go +++ b/engine/access/rest/websockets/data_providers/events_provider.go @@ -2,12 +2,9 @@ package data_providers import ( "context" - "encoding/json" "fmt" - "strconv" - "strings" - "github.com/rs/zerolog" + "strconv" "github.com/onflow/flow-go/engine/access/rest/common/parser" "github.com/onflow/flow-go/engine/access/rest/http/request" @@ -175,16 +172,12 @@ func parseEventsArguments( // Parse 'event_types' as a JSON array var eventTypes parser.EventTypes if eventTypesIn, ok := arguments["event_types"]; ok && eventTypesIn != "" { - result, ok := eventTypesIn.(string) + result, ok := eventTypesIn.([]string) if !ok { - return args, fmt.Errorf("'event_types' must be a string") - } - err := json.Unmarshal([]byte(result), &eventTypes) // Expect a JSON array - if err != nil { - return args, fmt.Errorf("could not parse 'event_types': %w", err) + return args, fmt.Errorf("'event_types' must be an array of string") } - err = eventTypes.Parse(strings.Split(result, ",")) + err := eventTypes.Parse(result) if err != nil { return args, fmt.Errorf("invalid 'event_types': %w", err) } @@ -193,27 +186,19 @@ func parseEventsArguments( // Parse 'addresses' as []string{} var addresses []string if addressesIn, ok := arguments["addresses"]; ok && addressesIn != "" { - result, ok := addressesIn.(string) + addresses, ok = addressesIn.([]string) if !ok { return args, fmt.Errorf("'addresses' must be a string") } - err := json.Unmarshal([]byte(result), &addresses) // Expect a JSON array - if err != nil { - return args, fmt.Errorf("could not parse 'addresses': %w", err) - } } // Parse 'contracts' as []string{} var contracts []string if contractsIn, ok := arguments["contracts"]; ok && contractsIn != "" { - result, ok := contractsIn.(string) + contracts, ok = contractsIn.([]string) if !ok { return args, fmt.Errorf("'contracts' must be a string") } - err := json.Unmarshal([]byte(result), &contracts) // Expect a JSON array - if err != nil { - return args, fmt.Errorf("could not parse 'contracts': %w", err) - } } // Initialize the event filter with the parsed arguments diff --git a/engine/access/rest/websockets/data_providers/events_provider_test.go b/engine/access/rest/websockets/data_providers/events_provider_test.go index 8721f407946..c1728eef4a9 100644 --- a/engine/access/rest/websockets/data_providers/events_provider_test.go +++ b/engine/access/rest/websockets/data_providers/events_provider_test.go @@ -64,6 +64,7 @@ func (s *EventsProviderSuite) subscribeEventsDataProviderTestCases() []testType name: "SubscribeBlocksFromStartBlockID happy path", arguments: models.Arguments{ "start_block_id": s.rootBlock.ID().String(), + "event_types": []string{"flow.AccountCreated", "flow.AccountUpdated"}, }, setupBackend: func(sub *ssmock.Subscription) { s.api.On( From 1bb4112033e66943fe4d8efcb706bea51f805543 Mon Sep 17 00:00:00 2001 From: Andrii Date: Mon, 9 Dec 2024 12:20:20 +0200 Subject: [PATCH 31/67] Linted --- .../access/rest/websockets/data_providers/events_provider.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/engine/access/rest/websockets/data_providers/events_provider.go b/engine/access/rest/websockets/data_providers/events_provider.go index dfbeb5c7c44..9431ca9a4dc 100644 --- a/engine/access/rest/websockets/data_providers/events_provider.go +++ b/engine/access/rest/websockets/data_providers/events_provider.go @@ -3,9 +3,10 @@ package data_providers import ( "context" "fmt" - "github.com/rs/zerolog" "strconv" + "github.com/rs/zerolog" + "github.com/onflow/flow-go/engine/access/rest/common/parser" "github.com/onflow/flow-go/engine/access/rest/http/request" "github.com/onflow/flow-go/engine/access/rest/util" From f81c794c176e09ed9ab40bf4bc2f5aa8156ba1ef Mon Sep 17 00:00:00 2001 From: Andrii Date: Mon, 9 Dec 2024 13:28:14 +0200 Subject: [PATCH 32/67] Changed parse args function, added hearbeat for hadnling --- .../account_statuses_provider.go | 73 +++++++++++++------ .../data_providers/events_provider.go | 7 +- 2 files changed, 57 insertions(+), 23 deletions(-) diff --git a/engine/access/rest/websockets/data_providers/account_statuses_provider.go b/engine/access/rest/websockets/data_providers/account_statuses_provider.go index 1656fb415a7..8261e77a1ef 100644 --- a/engine/access/rest/websockets/data_providers/account_statuses_provider.go +++ b/engine/access/rest/websockets/data_providers/account_statuses_provider.go @@ -3,12 +3,10 @@ package data_providers import ( "context" "fmt" - "strconv" - "strings" - "github.com/rs/zerolog" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" + "strconv" "github.com/onflow/flow-go/engine/access/rest/common/parser" "github.com/onflow/flow-go/engine/access/rest/http/request" @@ -32,6 +30,8 @@ type AccountStatusesDataProvider struct { logger zerolog.Logger stateStreamApi state_stream.API + + heartbeatInterval uint64 } var _ DataProvider = (*AccountStatusesDataProvider)(nil) @@ -41,17 +41,20 @@ func NewAccountStatusesDataProvider( ctx context.Context, logger zerolog.Logger, stateStreamApi state_stream.API, - chain flow.Chain, - eventFilterConfig state_stream.EventFilterConfig, topic string, arguments models.Arguments, send chan<- interface{}, + chain flow.Chain, + eventFilterConfig state_stream.EventFilterConfig, + heartbeatInterval uint64, ) (*AccountStatusesDataProvider, error) { p := &AccountStatusesDataProvider{ - logger: logger.With().Str("component", "account-statuses-data-provider").Logger(), - stateStreamApi: stateStreamApi, + logger: logger.With().Str("component", "account-statuses-data-provider").Logger(), + stateStreamApi: stateStreamApi, + heartbeatInterval: heartbeatInterval, } + // Initialize arguments passed to the provider. accountStatusesArgs, err := ParseAccountStatusesArguments(arguments, chain, eventFilterConfig) if err != nil { return nil, fmt.Errorf("invalid arguments for account statuses data provider: %w", err) @@ -73,7 +76,7 @@ func NewAccountStatusesDataProvider( // // No errors are expected during normal operations. func (p *AccountStatusesDataProvider) Run() error { - return subscription.HandleSubscription(p.subscription, p.handleResponse(p.send)) + return subscription.HandleSubscription(p.subscription, p.handleResponse()) } // createSubscription creates a new subscription using the specified input arguments. @@ -92,16 +95,27 @@ func (p *AccountStatusesDataProvider) createSubscription(ctx context.Context, ar // handleResponse processes an account statuses and sends the formatted response. // // No errors are expected during normal operations. -func (p *AccountStatusesDataProvider) handleResponse(send chan<- interface{}) func(accountStatusesResponse *backend.AccountStatusesResponse) error { +func (p *AccountStatusesDataProvider) handleResponse() func(accountStatusesResponse *backend.AccountStatusesResponse) error { + blocksSinceLastMessage := uint64(0) messageIndex := counters.NewMonotonousCounter(1) return func(accountStatusesResponse *backend.AccountStatusesResponse) error { + // check if there are any events in the response. if not, do not send a message unless the last + // response was more than HeartbeatInterval blocks ago + if len(accountStatusesResponse.AccountEvents) == 0 { + blocksSinceLastMessage++ + if blocksSinceLastMessage < p.heartbeatInterval { + return nil + } + blocksSinceLastMessage = 0 + } + index := messageIndex.Value() if ok := messageIndex.Set(messageIndex.Value() + 1); !ok { return status.Errorf(codes.Internal, "message index already incremented to %d", messageIndex.Value()) } - send <- &models.AccountStatusesResponse{ + p.send <- &models.AccountStatusesResponse{ BlockID: accountStatusesResponse.BlockID.String(), Height: strconv.FormatUint(accountStatusesResponse.Height, 10), AccountEvents: accountStatusesResponse.AccountEvents, @@ -112,8 +126,8 @@ func (p *AccountStatusesDataProvider) handleResponse(send chan<- interface{}) fu } } -// ParseAccountStatusesArguments validates and initializes the account statuses arguments. -func ParseAccountStatusesArguments( +// parseAccountStatusesArguments validates and initializes the account statuses arguments. +func parseAccountStatusesArguments( arguments models.Arguments, chain flow.Chain, eventFilterConfig state_stream.EventFilterConfig, @@ -121,8 +135,8 @@ func ParseAccountStatusesArguments( var args AccountStatusesArguments // Check for mutual exclusivity of start_block_id and start_block_height early - _, hasStartBlockID := arguments["start_block_id"] - _, hasStartBlockHeight := arguments["start_block_height"] + startBlockIDIn, hasStartBlockID := arguments["start_block_id"] + startBlockHeightIn, hasStartBlockHeight := arguments["start_block_height"] if hasStartBlockID && hasStartBlockHeight { return args, fmt.Errorf("can only provide either 'start_block_id' or 'start_block_height'") @@ -130,29 +144,43 @@ func ParseAccountStatusesArguments( // Parse 'start_block_id' if provided if hasStartBlockID { + result, ok := startBlockIDIn.(string) + if !ok { + return args, fmt.Errorf("'start_block_id' must be a string") + } var startBlockID parser.ID - err := startBlockID.Parse(arguments["start_block_id"]) + err := startBlockID.Parse(result) if err != nil { return args, fmt.Errorf("invalid 'start_block_id': %w", err) } args.StartBlockID = startBlockID.Flow() } + // Parse 'start_block_height' if provided // Parse 'start_block_height' if provided if hasStartBlockHeight { - var err error - args.StartBlockHeight, err = util.ToUint64(arguments["start_block_height"]) + result, ok := startBlockHeightIn.(string) + if !ok { + return args, fmt.Errorf("'start_block_height' must be a string") + } + startBlockHeight, err := util.ToUint64(result) if err != nil { return args, fmt.Errorf("invalid 'start_block_height': %w", err) } + args.StartBlockHeight = startBlockHeight } else { args.StartBlockHeight = request.EmptyHeight } - // Parse 'event_types' as []string{} + // Parse 'event_types' as a JSON array var eventTypes parser.EventTypes if eventTypesIn, ok := arguments["event_types"]; ok && eventTypesIn != "" { - err := eventTypes.Parse(strings.Split(eventTypesIn, ",")) + result, ok := eventTypesIn.([]string) + if !ok { + return args, fmt.Errorf("'event_types' must be an array of string") + } + + err := eventTypes.Parse(result) if err != nil { return args, fmt.Errorf("invalid 'event_types': %w", err) } @@ -160,8 +188,11 @@ func ParseAccountStatusesArguments( // Parse 'accountAddresses' as []string{} var accountAddresses []string - if addressesIn, ok := arguments["accountAddresses"]; ok && addressesIn != "" { - accountAddresses = strings.Split(addressesIn, ",") + if accountAddressesIn, ok := arguments["account_addresses"]; ok && accountAddressesIn != "" { + accountAddresses, ok = accountAddressesIn.([]string) + if !ok { + return args, fmt.Errorf("'account_addresses' must be an array of string") + } } // Initialize the event filter with the parsed arguments diff --git a/engine/access/rest/websockets/data_providers/events_provider.go b/engine/access/rest/websockets/data_providers/events_provider.go index 9431ca9a4dc..6b62f45ffac 100644 --- a/engine/access/rest/websockets/data_providers/events_provider.go +++ b/engine/access/rest/websockets/data_providers/events_provider.go @@ -80,6 +80,9 @@ func (p *EventsDataProvider) Run() error { return subscription.HandleSubscription(p.subscription, p.handleResponse()) } +// handleResponse processes events and sends the formatted response. +// +// No errors are expected during normal operations. func (p *EventsDataProvider) handleResponse() func(eventsResponse *backend.EventsResponse) error { blocksSinceLastMessage := uint64(0) messageIndex := counters.NewMonotonousCounter(1) @@ -189,7 +192,7 @@ func parseEventsArguments( if addressesIn, ok := arguments["addresses"]; ok && addressesIn != "" { addresses, ok = addressesIn.([]string) if !ok { - return args, fmt.Errorf("'addresses' must be a string") + return args, fmt.Errorf("'addresses' must be an array of string") } } @@ -198,7 +201,7 @@ func parseEventsArguments( if contractsIn, ok := arguments["contracts"]; ok && contractsIn != "" { contracts, ok = contractsIn.([]string) if !ok { - return args, fmt.Errorf("'contracts' must be a string") + return args, fmt.Errorf("'contracts' must be an array of string") } } From 50fd2897631007e004ccde193fca7b5d7bd64ee1 Mon Sep 17 00:00:00 2001 From: Andrii Date: Mon, 9 Dec 2024 13:29:38 +0200 Subject: [PATCH 33/67] Fixed error msg --- .../access/rest/websockets/data_providers/events_provider.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/engine/access/rest/websockets/data_providers/events_provider.go b/engine/access/rest/websockets/data_providers/events_provider.go index 9431ca9a4dc..b3cbb41d635 100644 --- a/engine/access/rest/websockets/data_providers/events_provider.go +++ b/engine/access/rest/websockets/data_providers/events_provider.go @@ -189,7 +189,7 @@ func parseEventsArguments( if addressesIn, ok := arguments["addresses"]; ok && addressesIn != "" { addresses, ok = addressesIn.([]string) if !ok { - return args, fmt.Errorf("'addresses' must be a string") + return args, fmt.Errorf("'addresses' must be an array of string") } } @@ -198,7 +198,7 @@ func parseEventsArguments( if contractsIn, ok := arguments["contracts"]; ok && contractsIn != "" { contracts, ok = contractsIn.([]string) if !ok { - return args, fmt.Errorf("'contracts' must be a string") + return args, fmt.Errorf("'contracts' must be an array of string") } } From 8ced75a2d44fee9cd6edc813c4447b509c949ef5 Mon Sep 17 00:00:00 2001 From: Andrii Slisarchuk Date: Mon, 9 Dec 2024 16:46:00 +0200 Subject: [PATCH 34/67] Implemented subsscribe tx statuses. Make refactoring. --- access/api.go | 7 +- access/handler.go | 7 +- access/mock/api.go | 30 ++++++-- engine/access/rpc/backend/backend.go | 1 + .../backend/backend_stream_transactions.go | 75 +++++++++++++++---- .../backend_stream_transactions_test.go | 13 +++- 6 files changed, 103 insertions(+), 30 deletions(-) diff --git a/access/api.go b/access/api.go index adeb7284c10..76126c38828 100644 --- a/access/api.go +++ b/access/api.go @@ -206,7 +206,12 @@ type API interface { // SubscribeTransactionStatuses streams transaction statuses starting from the reference block saved in the // transaction itself until the block containing the transaction becomes sealed or expired. When the transaction // status becomes TransactionStatusSealed or TransactionStatusExpired, the subscription will automatically shut down. - SubscribeTransactionStatuses(ctx context.Context, tx *flow.TransactionBody, requiredEventEncodingVersion entities.EventEncodingVersion) subscription.Subscription + SubscribeTransactionStatuses(ctx context.Context, txID flow.Identifier, blockID flow.Identifier, requiredEventEncodingVersion entities.EventEncodingVersion) subscription.Subscription + + // SendAndSubscribeTransactionStatuses streams transaction statuses starting from the reference block saved in the + // transaction itself until the block containing the transaction becomes sealed or expired. When the transaction + // status becomes TransactionStatusSealed or TransactionStatusExpired, the subscription will automatically shut down. + SendAndSubscribeTransactionStatuses(ctx context.Context, tx *flow.TransactionBody, requiredEventEncodingVersion entities.EventEncodingVersion) subscription.Subscription } // TODO: Combine this with flow.TransactionResult? diff --git a/access/handler.go b/access/handler.go index b974e7034fc..bcf401a2884 100644 --- a/access/handler.go +++ b/access/handler.go @@ -1425,12 +1425,7 @@ func (h *Handler) SendAndSubscribeTransactionStatuses( return status.Error(codes.InvalidArgument, err.Error()) } - err = h.api.SendTransaction(ctx, &tx) - if err != nil { - return err - } - - sub := h.api.SubscribeTransactionStatuses(ctx, &tx, request.GetEventEncodingVersion()) + sub := h.api.SendAndSubscribeTransactionStatuses(ctx, &tx, request.GetEventEncodingVersion()) messageIndex := counters.NewMonotonousCounter(0) return subscription.HandleRPCSubscription(sub, func(txResults []*TransactionResult) error { diff --git a/access/mock/api.go b/access/mock/api.go index eaaf6c428f2..274cc688a5a 100644 --- a/access/mock/api.go +++ b/access/mock/api.go @@ -1145,6 +1145,26 @@ func (_m *API) Ping(ctx context.Context) error { return r0 } +// SendAndSubscribeTransactionStatuses provides a mock function with given fields: ctx, tx, requiredEventEncodingVersion +func (_m *API) SendAndSubscribeTransactionStatuses(ctx context.Context, tx *flow.TransactionBody, requiredEventEncodingVersion entities.EventEncodingVersion) subscription.Subscription { + ret := _m.Called(ctx, tx, requiredEventEncodingVersion) + + if len(ret) == 0 { + panic("no return value specified for SendAndSubscribeTransactionStatuses") + } + + var r0 subscription.Subscription + if rf, ok := ret.Get(0).(func(context.Context, *flow.TransactionBody, entities.EventEncodingVersion) subscription.Subscription); ok { + r0 = rf(ctx, tx, requiredEventEncodingVersion) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(subscription.Subscription) + } + } + + return r0 +} + // SendTransaction provides a mock function with given fields: ctx, tx func (_m *API) SendTransaction(ctx context.Context, tx *flow.TransactionBody) error { ret := _m.Called(ctx, tx) @@ -1343,17 +1363,17 @@ func (_m *API) SubscribeBlocksFromStartHeight(ctx context.Context, startHeight u return r0 } -// SubscribeTransactionStatuses provides a mock function with given fields: ctx, tx, requiredEventEncodingVersion -func (_m *API) SubscribeTransactionStatuses(ctx context.Context, tx *flow.TransactionBody, requiredEventEncodingVersion entities.EventEncodingVersion) subscription.Subscription { - ret := _m.Called(ctx, tx, requiredEventEncodingVersion) +// SubscribeTransactionStatuses provides a mock function with given fields: ctx, txID, blockID, requiredEventEncodingVersion +func (_m *API) SubscribeTransactionStatuses(ctx context.Context, txID flow.Identifier, blockID flow.Identifier, requiredEventEncodingVersion entities.EventEncodingVersion) subscription.Subscription { + ret := _m.Called(ctx, txID, blockID, requiredEventEncodingVersion) if len(ret) == 0 { panic("no return value specified for SubscribeTransactionStatuses") } var r0 subscription.Subscription - if rf, ok := ret.Get(0).(func(context.Context, *flow.TransactionBody, entities.EventEncodingVersion) subscription.Subscription); ok { - r0 = rf(ctx, tx, requiredEventEncodingVersion) + if rf, ok := ret.Get(0).(func(context.Context, flow.Identifier, flow.Identifier, entities.EventEncodingVersion) subscription.Subscription); ok { + r0 = rf(ctx, txID, blockID, requiredEventEncodingVersion) } else { if ret.Get(0) != nil { r0 = ret.Get(0).(subscription.Subscription) diff --git a/engine/access/rpc/backend/backend.go b/engine/access/rpc/backend/backend.go index d4666af9529..5ed2232b87b 100644 --- a/engine/access/rpc/backend/backend.go +++ b/engine/access/rpc/backend/backend.go @@ -260,6 +260,7 @@ func New(params Params) (*Backend, error) { executionResults: params.ExecutionResults, subscriptionHandler: params.SubscriptionHandler, blockTracker: params.BlockTracker, + sendTransaction: b.SendTransaction, } retry.SetBackend(b) diff --git a/engine/access/rpc/backend/backend_stream_transactions.go b/engine/access/rpc/backend/backend_stream_transactions.go index 36163ea30a1..a1cebfba7c9 100644 --- a/engine/access/rpc/backend/backend_stream_transactions.go +++ b/engine/access/rpc/backend/backend_stream_transactions.go @@ -5,7 +5,6 @@ import ( "errors" "fmt" - "go.uber.org/atomic" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -22,6 +21,8 @@ import ( "github.com/onflow/flow/protobuf/go/flow/entities" ) +type sendTransaction func(ctx context.Context, tx *flow.TransactionBody) error + // backendSubscribeTransactions handles transaction subscriptions. type backendSubscribeTransactions struct { txLocalDataProvider *TransactionsLocalDataProvider @@ -31,31 +32,37 @@ type backendSubscribeTransactions struct { subscriptionHandler *subscription.SubscriptionHandler blockTracker subscription.BlockTracker + sendTransaction sendTransaction } -// TransactionSubscriptionMetadata holds data representing the status state for each transaction subscription. -type TransactionSubscriptionMetadata struct { +// transactionSubscriptionMetadata holds data representing the status state for each transaction subscription. +type transactionSubscriptionMetadata struct { *access.TransactionResult txReferenceBlockID flow.Identifier blockWithTx *flow.Header txExecuted bool eventEncodingVersion entities.EventEncodingVersion - shouldTriggerPending *atomic.Bool + shouldTriggerPending bool } -// SubscribeTransactionStatuses subscribes to transaction status changes starting from the transaction reference block ID. -// If invalid tx parameters will be supplied SubscribeTransactionStatuses will return a failed subscription. -func (b *backendSubscribeTransactions) SubscribeTransactionStatuses( +// SendAndSubscribeTransactionStatuses subscribes to transaction status changes starting from the transaction reference block ID. +// If invalid tx parameters will be supplied SendAndSubscribeTransactionStatuses will return a failed subscription. +func (b *backendSubscribeTransactions) SendAndSubscribeTransactionStatuses( ctx context.Context, tx *flow.TransactionBody, requiredEventEncodingVersion entities.EventEncodingVersion, ) subscription.Subscription { + err := b.sendTransaction(ctx, tx) + if err != nil { + return subscription.NewFailedSubscription(err, "transaction sending failed") + } + nextHeight, err := b.blockTracker.GetStartHeightFromBlockID(tx.ReferenceBlockID) if err != nil { return subscription.NewFailedSubscription(err, "could not get start height") } - txInfo := TransactionSubscriptionMetadata{ + txInfo := transactionSubscriptionMetadata{ TransactionResult: &access.TransactionResult{ TransactionID: tx.ID(), BlockID: flow.ZeroID, @@ -64,7 +71,43 @@ func (b *backendSubscribeTransactions) SubscribeTransactionStatuses( txReferenceBlockID: tx.ReferenceBlockID, blockWithTx: nil, eventEncodingVersion: requiredEventEncodingVersion, - shouldTriggerPending: atomic.NewBool(true), + shouldTriggerPending: true, + } + + return b.subscriptionHandler.Subscribe(ctx, nextHeight, b.getTransactionStatusResponse(&txInfo)) +} + +// SubscribeTransactionStatuses subscribes to transaction status changes starting from the transaction reference block ID. +// If invalid tx parameters will be supplied SubscribeTransactionStatuses will return a failed subscription. +func (b *backendSubscribeTransactions) SubscribeTransactionStatuses( + ctx context.Context, + txID flow.Identifier, + blockID flow.Identifier, + requiredEventEncodingVersion entities.EventEncodingVersion, +) subscription.Subscription { + if blockID == flow.ZeroID { + header, err := b.txLocalDataProvider.state.Sealed().Head() + if err != nil { + return subscription.NewFailedSubscription(err, "could not get latest block") + } + blockID = header.ID() + } + + nextHeight, err := b.blockTracker.GetStartHeightFromBlockID(blockID) + if err != nil { + return subscription.NewFailedSubscription(err, "could not get start height") + } + + txInfo := transactionSubscriptionMetadata{ + TransactionResult: &access.TransactionResult{ + TransactionID: txID, + BlockID: flow.ZeroID, + Status: flow.TransactionStatusUnknown, + }, + txReferenceBlockID: blockID, + blockWithTx: nil, + eventEncodingVersion: requiredEventEncodingVersion, + shouldTriggerPending: false, } return b.subscriptionHandler.Subscribe(ctx, nextHeight, b.getTransactionStatusResponse(&txInfo)) @@ -72,7 +115,7 @@ func (b *backendSubscribeTransactions) SubscribeTransactionStatuses( // getTransactionStatusResponse returns a callback function that produces transaction status // subscription responses based on new blocks. -func (b *backendSubscribeTransactions) getTransactionStatusResponse(txInfo *TransactionSubscriptionMetadata) func(context.Context, uint64) (interface{}, error) { +func (b *backendSubscribeTransactions) getTransactionStatusResponse(txInfo *transactionSubscriptionMetadata) func(context.Context, uint64) (interface{}, error) { return func(ctx context.Context, height uint64) (interface{}, error) { err := b.checkBlockReady(height) if err != nil { @@ -81,8 +124,8 @@ func (b *backendSubscribeTransactions) getTransactionStatusResponse(txInfo *Tran // The status of the first pending transaction should be returned immediately, as the transaction has already been sent. // This should occur only once for each subscription. - if txInfo.shouldTriggerPending.Load() { - txInfo.shouldTriggerPending.Toggle() + if txInfo.shouldTriggerPending { + txInfo.shouldTriggerPending = false return b.generateResultsWithMissingStatuses(txInfo, flow.TransactionStatusUnknown) } @@ -149,7 +192,7 @@ func (b *backendSubscribeTransactions) getTransactionStatusResponse(txInfo *Tran // execution block, if available, or its reference block. // // No errors expected during normal operations. -func (b *backendSubscribeTransactions) getTransactionStatus(ctx context.Context, txInfo *TransactionSubscriptionMetadata, prevTxStatus flow.TransactionStatus) (flow.TransactionStatus, error) { +func (b *backendSubscribeTransactions) getTransactionStatus(ctx context.Context, txInfo *transactionSubscriptionMetadata, prevTxStatus flow.TransactionStatus) (flow.TransactionStatus, error) { txStatus := txInfo.Status var err error @@ -180,7 +223,7 @@ func (b *backendSubscribeTransactions) getTransactionStatus(ctx context.Context, // 2. pending(1) -> expired(5) // No errors expected during normal operations. func (b *backendSubscribeTransactions) generateResultsWithMissingStatuses( - txInfo *TransactionSubscriptionMetadata, + txInfo *transactionSubscriptionMetadata, prevTxStatus flow.TransactionStatus, ) ([]*access.TransactionResult, error) { // If the previous status is pending and the new status is expired, which is the last status, return its result. @@ -255,7 +298,7 @@ func (b *backendSubscribeTransactions) checkBlockReady(height uint64) error { // - codes.Internal when other errors occur during block or collection lookup func (b *backendSubscribeTransactions) searchForTransactionBlockInfo( height uint64, - txInfo *TransactionSubscriptionMetadata, + txInfo *transactionSubscriptionMetadata, ) (*flow.Header, flow.Identifier, uint64, flow.Identifier, error) { block, err := b.txLocalDataProvider.blocks.ByHeight(height) if err != nil { @@ -279,7 +322,7 @@ func (b *backendSubscribeTransactions) searchForTransactionBlockInfo( // - codes.Internal if an internal error occurs while retrieving execution result. func (b *backendSubscribeTransactions) searchForTransactionResult( ctx context.Context, - txInfo *TransactionSubscriptionMetadata, + txInfo *transactionSubscriptionMetadata, ) (*access.TransactionResult, error) { _, err := b.executionResults.ByBlockID(txInfo.BlockID) if err != nil { diff --git a/engine/access/rpc/backend/backend_stream_transactions_test.go b/engine/access/rpc/backend/backend_stream_transactions_test.go index 4ae2af7161a..c4f7056f9b2 100644 --- a/engine/access/rpc/backend/backend_stream_transactions_test.go +++ b/engine/access/rpc/backend/backend_stream_transactions_test.go @@ -34,6 +34,7 @@ import ( storagemock "github.com/onflow/flow-go/storage/mock" "github.com/onflow/flow-go/utils/unittest" "github.com/onflow/flow-go/utils/unittest/mocks" + accessproto "github.com/onflow/flow/protobuf/go/flow/access" "github.com/onflow/flow/protobuf/go/flow/entities" ) @@ -121,6 +122,14 @@ func (s *TransactionStatusSuite) SetupTest() { s.blockTracker = subscriptionmock.NewBlockTracker(s.T()) s.resultsMap = map[flow.Identifier]*flow.ExecutionResult{} + s.colClient.On( + "SendTransaction", + mock.Anything, + mock.Anything, + ).Return(&accessproto.SendTransactionResponse{}, nil).Maybe() + + s.transactions.On("Store", mock.Anything).Return(nil).Maybe() + // generate blockCount consecutive blocks with associated seal, result and execution data s.rootBlock = unittest.BlockFixture() rootResult := unittest.ExecutionResultFixture(unittest.WithBlock(&s.rootBlock)) @@ -314,7 +323,7 @@ func (s *TransactionStatusSuite) TestSubscribeTransactionStatusHappyCase() { } // 1. Subscribe to transaction status and receive the first message with pending status - sub := s.backend.SubscribeTransactionStatuses(ctx, &transaction.TransactionBody, entities.EventEncodingVersion_CCF_V0) + sub := s.backend.SendAndSubscribeTransactionStatuses(ctx, &transaction.TransactionBody, entities.EventEncodingVersion_CCF_V0) checkNewSubscriptionMessage(sub, flow.TransactionStatusPending) // 2. Make transaction reference block sealed, and add a new finalized block that includes the transaction @@ -380,7 +389,7 @@ func (s *TransactionStatusSuite) TestSubscribeTransactionStatusExpired() { } // Subscribe to transaction status and receive the first message with pending status - sub := s.backend.SubscribeTransactionStatuses(ctx, &transaction.TransactionBody, entities.EventEncodingVersion_CCF_V0) + sub := s.backend.SendAndSubscribeTransactionStatuses(ctx, &transaction.TransactionBody, entities.EventEncodingVersion_CCF_V0) checkNewSubscriptionMessage(sub, flow.TransactionStatusPending) // Generate 600 blocks without transaction included and check, that transaction still pending From ff1a95f4d02fc4aa91d54f61bc7f3cde7ba2bb6c Mon Sep 17 00:00:00 2001 From: Andrii Slisarchuk Date: Mon, 9 Dec 2024 17:01:06 +0200 Subject: [PATCH 35/67] linted --- cmd/util/cmd/run-script/cmd.go | 11 ++++++++++- .../rpc/backend/backend_stream_transactions_test.go | 3 ++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/cmd/util/cmd/run-script/cmd.go b/cmd/util/cmd/run-script/cmd.go index 171f97e76b7..d66aa341d93 100644 --- a/cmd/util/cmd/run-script/cmd.go +++ b/cmd/util/cmd/run-script/cmd.go @@ -532,7 +532,16 @@ func (*api) SubscribeBlockDigestsFromLatest( return nil } -func (*api) SubscribeTransactionStatuses( +func (a *api) SubscribeTransactionStatuses( + _ context.Context, + _ flow.Identifier, + _ flow.Identifier, + _ entities.EventEncodingVersion, +) subscription.Subscription { + return nil +} + +func (a *api) SendAndSubscribeTransactionStatuses( _ context.Context, _ *flow.TransactionBody, _ entities.EventEncodingVersion, diff --git a/engine/access/rpc/backend/backend_stream_transactions_test.go b/engine/access/rpc/backend/backend_stream_transactions_test.go index c4f7056f9b2..47b494e87b3 100644 --- a/engine/access/rpc/backend/backend_stream_transactions_test.go +++ b/engine/access/rpc/backend/backend_stream_transactions_test.go @@ -14,6 +14,8 @@ import ( "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" + accessproto "github.com/onflow/flow/protobuf/go/flow/access" + accessapi "github.com/onflow/flow-go/access" "github.com/onflow/flow-go/engine" "github.com/onflow/flow-go/engine/access/index" @@ -34,7 +36,6 @@ import ( storagemock "github.com/onflow/flow-go/storage/mock" "github.com/onflow/flow-go/utils/unittest" "github.com/onflow/flow-go/utils/unittest/mocks" - accessproto "github.com/onflow/flow/protobuf/go/flow/access" "github.com/onflow/flow/protobuf/go/flow/entities" ) From 37836f9c6b323d0c0ceec2ea090b5b4a1cdf095f Mon Sep 17 00:00:00 2001 From: Andrii Date: Mon, 9 Dec 2024 18:18:40 +0200 Subject: [PATCH 36/67] Added happy path cases, fixed remarks from event provider PR --- .../account_statuses_provider.go | 7 +- .../account_statuses_provider_test.go | 181 +++++++++++++++--- .../data_providers/events_provider_test.go | 32 ++-- .../rest/websockets/data_providers/factory.go | 2 +- 4 files changed, 171 insertions(+), 51 deletions(-) diff --git a/engine/access/rest/websockets/data_providers/account_statuses_provider.go b/engine/access/rest/websockets/data_providers/account_statuses_provider.go index 8261e77a1ef..1a3aee203c9 100644 --- a/engine/access/rest/websockets/data_providers/account_statuses_provider.go +++ b/engine/access/rest/websockets/data_providers/account_statuses_provider.go @@ -3,10 +3,11 @@ package data_providers import ( "context" "fmt" + "strconv" + "github.com/rs/zerolog" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" - "strconv" "github.com/onflow/flow-go/engine/access/rest/common/parser" "github.com/onflow/flow-go/engine/access/rest/http/request" @@ -55,7 +56,7 @@ func NewAccountStatusesDataProvider( } // Initialize arguments passed to the provider. - accountStatusesArgs, err := ParseAccountStatusesArguments(arguments, chain, eventFilterConfig) + accountStatusesArgs, err := parseAccountStatusesArguments(arguments, chain, eventFilterConfig) if err != nil { return nil, fmt.Errorf("invalid arguments for account statuses data provider: %w", err) } @@ -66,7 +67,7 @@ func NewAccountStatusesDataProvider( topic, cancel, send, - p.createSubscription(subCtx, accountStatusesArgs), // Set up a subscription to events based on arguments. + p.createSubscription(subCtx, accountStatusesArgs), // Set up a subscription to account statuses based on arguments. ) return p, nil diff --git a/engine/access/rest/websockets/data_providers/account_statuses_provider_test.go b/engine/access/rest/websockets/data_providers/account_statuses_provider_test.go index 5a15949bfad..aeadd68f649 100644 --- a/engine/access/rest/websockets/data_providers/account_statuses_provider_test.go +++ b/engine/access/rest/websockets/data_providers/account_statuses_provider_test.go @@ -2,18 +2,19 @@ package data_providers import ( "context" - "fmt" "strconv" "testing" "github.com/rs/zerolog" "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" "github.com/onflow/flow-go/engine/access/rest/websockets/models" "github.com/onflow/flow-go/engine/access/state_stream" "github.com/onflow/flow-go/engine/access/state_stream/backend" ssmock "github.com/onflow/flow-go/engine/access/state_stream/mock" + "github.com/onflow/flow-go/engine/access/subscription" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/utils/unittest" ) @@ -28,6 +29,8 @@ type AccountStatusesProviderSuite struct { chain flow.Chain rootBlock flow.Block finalizedBlock *flow.Header + + factory *DataProviderFactoryImpl } func TestNewAccountStatusesDataProvider(t *testing.T) { @@ -42,43 +45,155 @@ func (s *AccountStatusesProviderSuite) SetupTest() { s.rootBlock = unittest.BlockFixture() s.rootBlock.Header.Height = 0 + + s.factory = NewDataProviderFactory( + s.log, + s.api, + nil, + flow.Testnet.Chain(), + state_stream.DefaultEventFilterConfig, + subscription.DefaultHeartbeatInterval) + s.Require().NotNil(s.factory) +} + +// TestAccountStatusesDataProvider_HappyPath tests the behavior of the account statuses data provider +// when it is configured correctly and operating under normal conditions. It +// validates that events are correctly streamed to the channel and ensures +// no unexpected errors occur. +func (s *AccountStatusesProviderSuite) TestAccountStatusesDataProvider_HappyPath() { + s.testHappyPath( + AccountStatusesTopic, + s.subscribeAccountStatusesDataProviderTestCases(), + s.requireAccountStatuses, + ) +} + +func (s *AccountStatusesProviderSuite) testHappyPath( + topic string, + tests []testType, + requireFn func(interface{}, *backend.AccountStatusesResponse), +) { + expectedEvents := []flow.Event{ + unittest.EventFixture(state_stream.CoreEventAccountCreated, 0, 0, unittest.IdentifierFixture(), 0), + unittest.EventFixture(state_stream.CoreEventAccountKeyAdded, 0, 0, unittest.IdentifierFixture(), 0), + } + + var expectedAccountStatusesResponses []backend.AccountStatusesResponse + + for i := 0; i < len(expectedEvents); i++ { + expectedAccountStatusesResponses = append(expectedAccountStatusesResponses, backend.AccountStatusesResponse{ + Height: s.rootBlock.Header.Height, + BlockID: s.rootBlock.ID(), + AccountEvents: map[string]flow.EventsList{ + unittest.RandomAddressFixture().String(): expectedEvents, + }, + }) + } + + for _, test := range tests { + s.Run(test.name, func() { + ctx := context.Background() + send := make(chan interface{}, 10) + + // Create a channel to simulate the subscription's data channel + accStatusesChan := make(chan interface{}) + + // // Create a mock subscription and mock the channel + sub := ssmock.NewSubscription(s.T()) + sub.On("Channel").Return((<-chan interface{})(accStatusesChan)) + sub.On("Err").Return(nil) + test.setupBackend(sub) + + // Create the data provider instance + provider, err := s.factory.NewDataProvider(ctx, topic, test.arguments, send) + s.Require().NotNil(provider) + s.Require().NoError(err) + + // Run the provider in a separate goroutine + go func() { + err = provider.Run() + s.Require().NoError(err) + }() + + // Simulate emitting data to the events channel + go func() { + defer close(accStatusesChan) + + for i := 0; i < len(expectedAccountStatusesResponses); i++ { + accStatusesChan <- &expectedAccountStatusesResponses[i] + } + }() + + // Collect responses + for _, e := range expectedAccountStatusesResponses { + v, ok := <-send + s.Require().True(ok, "channel closed while waiting for event %v: err: %v", e.BlockID, sub.Err()) + + requireFn(v, &e) + } + + // Ensure the provider is properly closed after the test + provider.Close() + }) + } } -// invalidArgumentsTestCases returns a list of test cases with invalid argument combinations -// for testing the behavior of account statuses data providers. Each test case includes a name, -// a set of input arguments, and the expected error message that should be returned. -// -// The test cases cover scenarios such as: -// 1. Supplying both 'start_block_id' and 'start_block_height' simultaneously, which is not allowed. -// 2. Providing invalid 'start_block_id' value. -// 3. Providing invalid 'start_block_height' value. -func (s *AccountStatusesProviderSuite) invalidArgumentsTestCases() []testErrType { - return []testErrType{ +func (s *AccountStatusesProviderSuite) subscribeAccountStatusesDataProviderTestCases() []testType { + return []testType{ { - name: "provide both 'start_block_id' and 'start_block_height' arguments", + name: "SubscribeAccountStatusesFromStartBlockID happy path", arguments: models.Arguments{ - "start_block_id": s.rootBlock.ID().String(), - "start_block_height": fmt.Sprintf("%d", s.rootBlock.Header.Height), + "start_block_id": s.rootBlock.ID().String(), + "event_types": []string{"flow.AccountCreated", "flow.AccountKeyAdded"}, + }, + setupBackend: func(sub *ssmock.Subscription) { + s.api.On( + "SubscribeAccountStatusesFromStartBlockID", + mock.Anything, + s.rootBlock.ID(), + mock.Anything, + ).Return(sub).Once() }, - expectedErrorMsg: "can only provide either 'start_block_id' or 'start_block_height'", }, { - name: "invalid 'start_block_id' argument", - arguments: map[string]string{ - "start_block_id": "invalid_block_id", + name: "SubscribeAccountStatusesFromStartHeight happy path", + arguments: models.Arguments{ + "start_block_height": strconv.FormatUint(s.rootBlock.Header.Height, 10), + }, + setupBackend: func(sub *ssmock.Subscription) { + s.api.On( + "SubscribeAccountStatusesFromStartHeight", + mock.Anything, + s.rootBlock.Header.Height, + mock.Anything, + ).Return(sub).Once() }, - expectedErrorMsg: "invalid ID format", }, { - name: "invalid 'start_block_height' argument", - arguments: map[string]string{ - "start_block_height": "-1", + name: "SubscribeAccountStatusesFromLatestBlock happy path", + arguments: models.Arguments{}, + setupBackend: func(sub *ssmock.Subscription) { + s.api.On( + "SubscribeAccountStatusesFromLatestBlock", + mock.Anything, + mock.Anything, + ).Return(sub).Once() }, - expectedErrorMsg: "value must be an unsigned 64 bit integer", }, } } +// requireAccountStatuses ensures that the received account statuses information matches the expected data. +func (s *AccountStatusesProviderSuite) requireAccountStatuses( + v interface{}, + expectedAccountStatusesResponse *backend.AccountStatusesResponse, +) { + _, ok := v.(*models.AccountStatusesResponse) + require.True(s.T(), ok, "Expected *models.AccountStatusesResponse, got %T", v) + + //s.Require().ElementsMatch(expectedAccountStatusesResponse.AccountEvents, actualResponse.AccountEvents) +} + // TestAccountStatusesDataProvider_InvalidArguments tests the behavior of the account statuses data provider // when invalid arguments are provided. It verifies that appropriate errors are returned // for missing or conflicting arguments. @@ -92,17 +207,19 @@ func (s *AccountStatusesProviderSuite) TestAccountStatusesDataProvider_InvalidAr topic := AccountStatusesTopic - for _, test := range s.invalidArgumentsTestCases() { + for _, test := range invalidArgumentsTestCases() { s.Run(test.name, func() { provider, err := NewAccountStatusesDataProvider( ctx, s.log, s.api, - s.chain, - state_stream.DefaultEventFilterConfig, topic, test.arguments, - send) + send, + s.chain, + state_stream.DefaultEventFilterConfig, + subscription.DefaultHeartbeatInterval, + ) s.Require().Nil(provider) s.Require().Error(err) s.Require().Contains(err.Error(), test.expectedErrorMsg) @@ -128,7 +245,7 @@ func (s *AccountStatusesProviderSuite) TestMessageIndexAccountStatusesProviderRe s.api.On("SubscribeAccountStatusesFromStartBlockID", mock.Anything, mock.Anything, mock.Anything).Return(sub) arguments := - map[string]string{ + map[string]interface{}{ "start_block_id": s.rootBlock.ID().String(), } @@ -137,11 +254,13 @@ func (s *AccountStatusesProviderSuite) TestMessageIndexAccountStatusesProviderRe ctx, s.log, s.api, - s.chain, - state_stream.DefaultEventFilterConfig, topic, arguments, - send) + send, + s.chain, + state_stream.DefaultEventFilterConfig, + subscription.DefaultHeartbeatInterval, + ) s.Require().NotNil(provider) s.Require().NoError(err) diff --git a/engine/access/rest/websockets/data_providers/events_provider_test.go b/engine/access/rest/websockets/data_providers/events_provider_test.go index c1728eef4a9..336744d24c7 100644 --- a/engine/access/rest/websockets/data_providers/events_provider_test.go +++ b/engine/access/rest/websockets/data_providers/events_provider_test.go @@ -57,6 +57,18 @@ func (s *EventsProviderSuite) SetupTest() { s.Require().NotNil(s.factory) } +// TestEventsDataProvider_HappyPath tests the behavior of the events data provider +// when it is configured correctly and operating under normal conditions. It +// validates that events are correctly streamed to the channel and ensures +// no unexpected errors occur. +func (s *EventsProviderSuite) TestEventsDataProvider_HappyPath() { + s.testHappyPath( + EventsTopic, + s.subscribeEventsDataProviderTestCases(), + s.requireEvents, + ) +} + // subscribeEventsDataProviderTestCases generates test cases for events data providers. func (s *EventsProviderSuite) subscribeEventsDataProviderTestCases() []testType { return []testType{ @@ -103,18 +115,6 @@ func (s *EventsProviderSuite) subscribeEventsDataProviderTestCases() []testType } } -// TestEventsDataProvider_HappyPath tests the behavior of the events data provider -// when it is configured correctly and operating under normal conditions. It -// validates that events are correctly streamed to the channel and ensures -// no unexpected errors occur. -func (s *EventsProviderSuite) TestEventsDataProvider_HappyPath() { - s.testHappyPath( - EventsTopic, - s.subscribeEventsDataProviderTestCases(), - s.requireEvents, - ) -} - // testHappyPath tests a variety of scenarios for data providers in // happy path scenarios. This function runs parameterized test cases that // simulate various configurations and verifies that the data provider operates @@ -212,13 +212,13 @@ func (s *EventsProviderSuite) requireEvents(v interface{}, expectedEventsRespons // 1. Supplying both 'start_block_id' and 'start_block_height' simultaneously, which is not allowed. // 2. Providing invalid 'start_block_id' value. // 3. Providing invalid 'start_block_height' value. -func (s *EventsProviderSuite) invalidArgumentsTestCases() []testErrType { +func invalidArgumentsTestCases() []testErrType { return []testErrType{ { name: "provide both 'start_block_id' and 'start_block_height' arguments", arguments: models.Arguments{ - "start_block_id": s.rootBlock.ID().String(), - "start_block_height": fmt.Sprintf("%d", s.rootBlock.Header.Height), + "start_block_id": unittest.BlockFixture().ID().String(), + "start_block_height": fmt.Sprintf("%d", unittest.BlockFixture().Header.Height), }, expectedErrorMsg: "can only provide either 'start_block_id' or 'start_block_height'", }, @@ -252,7 +252,7 @@ func (s *EventsProviderSuite) TestEventsDataProvider_InvalidArguments() { topic := EventsTopic - for _, test := range s.invalidArgumentsTestCases() { + for _, test := range invalidArgumentsTestCases() { s.Run(test.name, func() { provider, err := NewEventsDataProvider( ctx, diff --git a/engine/access/rest/websockets/data_providers/factory.go b/engine/access/rest/websockets/data_providers/factory.go index 4c014d3a3b5..26aade4e090 100644 --- a/engine/access/rest/websockets/data_providers/factory.go +++ b/engine/access/rest/websockets/data_providers/factory.go @@ -101,7 +101,7 @@ func (s *DataProviderFactoryImpl) NewDataProvider( case EventsTopic: return NewEventsDataProvider(ctx, s.logger, s.stateStreamApi, topic, arguments, ch, s.chain, s.eventFilterConfig, s.heartbeatInterval) case AccountStatusesTopic: - return NewAccountStatusesDataProvider(ctx, s.logger, s.stateStreamApi, s.chain, s.eventFilterConfig, topic, arguments, ch) + return NewAccountStatusesDataProvider(ctx, s.logger, s.stateStreamApi, topic, arguments, ch, s.chain, s.eventFilterConfig, s.heartbeatInterval) case TransactionStatusesTopic: // TODO: Implemented handlers for each topic should be added in respective case return nil, fmt.Errorf(`topic "%s" not implemented yet`, topic) From 815419b0a1f570202e9def6218d4877e60975888 Mon Sep 17 00:00:00 2001 From: Andrii Slisarchuk Date: Tue, 10 Dec 2024 16:57:18 +0200 Subject: [PATCH 37/67] Clean up code. Added comments --- access/api.go | 28 ++++-- .../backend/backend_stream_transactions.go | 92 +++++++++++-------- 2 files changed, 76 insertions(+), 44 deletions(-) diff --git a/access/api.go b/access/api.go index 76126c38828..63db940414a 100644 --- a/access/api.go +++ b/access/api.go @@ -203,14 +203,28 @@ type API interface { // // If invalid parameters will be supplied SubscribeBlockDigestsFromLatest will return a failed subscription. SubscribeBlockDigestsFromLatest(ctx context.Context, blockStatus flow.BlockStatus) subscription.Subscription - // SubscribeTransactionStatuses streams transaction statuses starting from the reference block saved in the - // transaction itself until the block containing the transaction becomes sealed or expired. When the transaction - // status becomes TransactionStatusSealed or TransactionStatusExpired, the subscription will automatically shut down. + // SubscribeTransactionStatuses subscribes to transaction status updates for a given transaction ID. + // Monitoring begins from the specified block ID or the latest block if no block ID is provided. + // The subscription streams status updates until the transaction reaches a final state (TransactionStatusSealed or TransactionStatusExpired). + // When the transaction reaches one of these final statuses, the subscription will automatically terminate. + // + // Parameters: + // - ctx: The context to manage the subscription's lifecycle, including cancellation. + // - txID: The unique identifier of the transaction to monitor. + // - blockID: The block ID from which to start monitoring. If set to flow.ZeroID, monitoring starts from the latest block. + // - requiredEventEncodingVersion: The version of event encoding required for the subscription. SubscribeTransactionStatuses(ctx context.Context, txID flow.Identifier, blockID flow.Identifier, requiredEventEncodingVersion entities.EventEncodingVersion) subscription.Subscription - - // SendAndSubscribeTransactionStatuses streams transaction statuses starting from the reference block saved in the - // transaction itself until the block containing the transaction becomes sealed or expired. When the transaction - // status becomes TransactionStatusSealed or TransactionStatusExpired, the subscription will automatically shut down. + // SendAndSubscribeTransactionStatuses sends a transaction to the execution node and subscribes to its status updates. + // Monitoring begins from the reference block saved in the transaction itself and streams status updates until the transaction + // reaches a final state (TransactionStatusSealed or TransactionStatusExpired). Once a final status is reached, the subscription + // automatically terminates. + // + // Parameters: + // - ctx: The context to manage the transaction sending and subscription lifecycle, including cancellation. + // - tx: The transaction body to be sent and monitored. + // - requiredEventEncodingVersion: The version of event encoding required for the subscription. + // + // If the transaction cannot be sent, the subscription will fail and return a failed subscription. SendAndSubscribeTransactionStatuses(ctx context.Context, tx *flow.TransactionBody, requiredEventEncodingVersion entities.EventEncodingVersion) subscription.Subscription } diff --git a/engine/access/rpc/backend/backend_stream_transactions.go b/engine/access/rpc/backend/backend_stream_transactions.go index a1cebfba7c9..718d41ec5de 100644 --- a/engine/access/rpc/backend/backend_stream_transactions.go +++ b/engine/access/rpc/backend/backend_stream_transactions.go @@ -21,6 +21,7 @@ import ( "github.com/onflow/flow/protobuf/go/flow/entities" ) +// sendTransaction defines a function type for sending a transaction. type sendTransaction func(ctx context.Context, tx *flow.TransactionBody) error // backendSubscribeTransactions handles transaction subscriptions. @@ -45,69 +46,79 @@ type transactionSubscriptionMetadata struct { shouldTriggerPending bool } -// SendAndSubscribeTransactionStatuses subscribes to transaction status changes starting from the transaction reference block ID. -// If invalid tx parameters will be supplied SendAndSubscribeTransactionStatuses will return a failed subscription. +// SendAndSubscribeTransactionStatuses sends a transaction and subscribes to its status updates. +// It starts monitoring the status from the transaction's reference block ID. +// If the transaction cannot be sent or an error occurs during subscription creation, a failed subscription is returned. func (b *backendSubscribeTransactions) SendAndSubscribeTransactionStatuses( ctx context.Context, tx *flow.TransactionBody, requiredEventEncodingVersion entities.EventEncodingVersion, ) subscription.Subscription { - err := b.sendTransaction(ctx, tx) - if err != nil { - return subscription.NewFailedSubscription(err, "transaction sending failed") - } - - nextHeight, err := b.blockTracker.GetStartHeightFromBlockID(tx.ReferenceBlockID) - if err != nil { - return subscription.NewFailedSubscription(err, "could not get start height") + if err := b.sendTransaction(ctx, tx); err != nil { + b.log.Error().Err(err).Str("tx_id", tx.ID().String()).Msg("failed to send transaction") + return subscription.NewFailedSubscription(err, "failed to send transaction") } - txInfo := transactionSubscriptionMetadata{ - TransactionResult: &access.TransactionResult{ - TransactionID: tx.ID(), - BlockID: flow.ZeroID, - Status: flow.TransactionStatusPending, - }, - txReferenceBlockID: tx.ReferenceBlockID, - blockWithTx: nil, - eventEncodingVersion: requiredEventEncodingVersion, - shouldTriggerPending: true, - } - - return b.subscriptionHandler.Subscribe(ctx, nextHeight, b.getTransactionStatusResponse(&txInfo)) + return b.createSubscription(ctx, tx.ID(), tx.ReferenceBlockID, tx.ReferenceBlockID, requiredEventEncodingVersion, true) } -// SubscribeTransactionStatuses subscribes to transaction status changes starting from the transaction reference block ID. -// If invalid tx parameters will be supplied SubscribeTransactionStatuses will return a failed subscription. +// SubscribeTransactionStatuses subscribes to the status updates of a transaction. +// Monitoring starts from the specified block ID or the latest block if no block ID is provided. +// If the block ID cannot be determined or an error occurs during subscription creation, a failed subscription is returned. func (b *backendSubscribeTransactions) SubscribeTransactionStatuses( ctx context.Context, txID flow.Identifier, blockID flow.Identifier, requiredEventEncodingVersion entities.EventEncodingVersion, ) subscription.Subscription { + // if no block ID provided, get latest block ID if blockID == flow.ZeroID { header, err := b.txLocalDataProvider.state.Sealed().Head() if err != nil { - return subscription.NewFailedSubscription(err, "could not get latest block") + b.log.Error().Err(err).Msg("failed to retrieve latest block") + return subscription.NewFailedSubscription(err, "failed to retrieve latest block") } blockID = header.ID() } + return b.createSubscription(ctx, txID, blockID, flow.ZeroID, requiredEventEncodingVersion, false) +} + +// createSubscription initializes a subscription for monitoring a transaction's status. +// If the start height cannot be determined, a failed subscription is returned. +func (b *backendSubscribeTransactions) createSubscription( + ctx context.Context, + txID flow.Identifier, + blockID flow.Identifier, + referenceBlockID flow.Identifier, + requiredEventEncodingVersion entities.EventEncodingVersion, + shouldTriggerPending bool, +) subscription.Subscription { + // Get height to start subscription from nextHeight, err := b.blockTracker.GetStartHeightFromBlockID(blockID) if err != nil { - return subscription.NewFailedSubscription(err, "could not get start height") + b.log.Error().Err(err).Str("block_id", blockID.String()).Msg("failed to get start height") + return subscription.NewFailedSubscription(err, "failed to get start height") + } + + // choose initial transaction status + initialStatus := flow.TransactionStatusUnknown + if shouldTriggerPending { + // The status of the first pending transaction should be returned immediately, as the transaction has already been sent. + // This should occur only once for each subscription. + initialStatus = flow.TransactionStatusPending } txInfo := transactionSubscriptionMetadata{ TransactionResult: &access.TransactionResult{ TransactionID: txID, BlockID: flow.ZeroID, - Status: flow.TransactionStatusUnknown, + Status: initialStatus, }, - txReferenceBlockID: blockID, + txReferenceBlockID: referenceBlockID, blockWithTx: nil, eventEncodingVersion: requiredEventEncodingVersion, - shouldTriggerPending: false, + shouldTriggerPending: shouldTriggerPending, } return b.subscriptionHandler.Subscribe(ctx, nextHeight, b.getTransactionStatusResponse(&txInfo)) @@ -122,16 +133,12 @@ func (b *backendSubscribeTransactions) getTransactionStatusResponse(txInfo *tran return nil, err } - // The status of the first pending transaction should be returned immediately, as the transaction has already been sent. - // This should occur only once for each subscription. if txInfo.shouldTriggerPending { - txInfo.shouldTriggerPending = false - return b.generateResultsWithMissingStatuses(txInfo, flow.TransactionStatusUnknown) + return b.handlePendingStatus(txInfo) } - // If the transaction status already reported the final status, return with no data available - if txInfo.Status == flow.TransactionStatusSealed || txInfo.Status == flow.TransactionStatusExpired { - return nil, fmt.Errorf("transaction final status %s was already reported: %w", txInfo.Status.String(), subscription.ErrEndOfData) + if b.isTransactionFinalStatus(txInfo) { + return nil, fmt.Errorf("transaction final status %s already reported: %w", txInfo.Status.String(), subscription.ErrEndOfData) } // If on this step transaction block not available, search for it. @@ -187,6 +194,17 @@ func (b *backendSubscribeTransactions) getTransactionStatusResponse(txInfo *tran } } +// handlePendingStatus handles the initial pending status for a transaction. +func (b *backendSubscribeTransactions) handlePendingStatus(txInfo *transactionSubscriptionMetadata) (interface{}, error) { + txInfo.shouldTriggerPending = false + return b.generateResultsWithMissingStatuses(txInfo, flow.TransactionStatusUnknown) +} + +// isTransactionFinalStatus checks if a transaction has reached a final state (Sealed or Expired). +func (b *backendSubscribeTransactions) isTransactionFinalStatus(txInfo *transactionSubscriptionMetadata) bool { + return txInfo.Status == flow.TransactionStatusSealed || txInfo.Status == flow.TransactionStatusExpired +} + // getTransactionStatus determines the current status of a transaction based on its metadata // and previous status. It derives the transaction status by analyzing the transaction's // execution block, if available, or its reference block. From 8432297a7b76df487a273abe62d704a1be3e997a Mon Sep 17 00:00:00 2001 From: Andrii Date: Wed, 11 Dec 2024 12:39:06 +0200 Subject: [PATCH 38/67] Fixed account statuses test --- .../account_statuses_provider_test.go | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/engine/access/rest/websockets/data_providers/account_statuses_provider_test.go b/engine/access/rest/websockets/data_providers/account_statuses_provider_test.go index aeadd68f649..98aabf50551 100644 --- a/engine/access/rest/websockets/data_providers/account_statuses_provider_test.go +++ b/engine/access/rest/websockets/data_providers/account_statuses_provider_test.go @@ -188,10 +188,19 @@ func (s *AccountStatusesProviderSuite) requireAccountStatuses( v interface{}, expectedAccountStatusesResponse *backend.AccountStatusesResponse, ) { - _, ok := v.(*models.AccountStatusesResponse) + actualResponse, ok := v.(*models.AccountStatusesResponse) require.True(s.T(), ok, "Expected *models.AccountStatusesResponse, got %T", v) - //s.Require().ElementsMatch(expectedAccountStatusesResponse.AccountEvents, actualResponse.AccountEvents) + require.Equal(s.T(), expectedAccountStatusesResponse.BlockID.String(), actualResponse.BlockID) + require.Equal(s.T(), len(expectedAccountStatusesResponse.AccountEvents), len(actualResponse.AccountEvents)) + + for key, expectedEvents := range expectedAccountStatusesResponse.AccountEvents { + actualEvents, ok := actualResponse.AccountEvents[key] + require.True(s.T(), ok, "Missing key in actual AccountEvents: %s", key) + + s.Require().Equal(expectedEvents, actualEvents, "Mismatch for key: %s", key) + } + } // TestAccountStatusesDataProvider_InvalidArguments tests the behavior of the account statuses data provider From 3ba47e8ce114b0200bd2664590e281a81f32e88d Mon Sep 17 00:00:00 2001 From: Andrii Date: Wed, 11 Dec 2024 14:06:55 +0200 Subject: [PATCH 39/67] Refacored events provider tests to use generic testHappyPath function --- .../block_digests_provider_test.go | 16 ++- .../block_headers_provider_test.go | 16 ++- .../data_providers/blocks_provider_test.go | 50 ++++--- .../data_providers/events_provider_test.go | 122 ++++++------------ 4 files changed, 90 insertions(+), 114 deletions(-) diff --git a/engine/access/rest/websockets/data_providers/block_digests_provider_test.go b/engine/access/rest/websockets/data_providers/block_digests_provider_test.go index 476edf77111..975716c74af 100644 --- a/engine/access/rest/websockets/data_providers/block_digests_provider_test.go +++ b/engine/access/rest/websockets/data_providers/block_digests_provider_test.go @@ -106,20 +106,26 @@ func (s *BlockDigestsProviderSuite) validBlockDigestsArgumentsTestCases() []test // validates that block digests are correctly streamed to the channel and ensures // no unexpected errors occur. func (s *BlockDigestsProviderSuite) TestBlockDigestsDataProvider_HappyPath() { - s.testHappyPath( + testHappyPath( + s.T(), BlockDigestsTopic, + s.factory, s.validBlockDigestsArgumentsTestCases(), - func(dataChan chan interface{}, blocks []*flow.Block) { - for _, block := range blocks { + func(dataChan chan interface{}) { + for _, block := range s.blocks { dataChan <- flow.NewBlockDigest(block.Header.ID(), block.Header.Height, block.Header.Timestamp) } }, - s.requireBlockDigests, + s.blocks, + s.requireBlockDigest, ) } // requireBlockHeaders ensures that the received block header information matches the expected data. -func (s *BlocksProviderSuite) requireBlockDigests(v interface{}, expectedBlock *flow.Block) { +func (s *BlocksProviderSuite) requireBlockDigest(v interface{}, expected interface{}) { + expectedBlock, ok := expected.(*flow.Block) + require.True(s.T(), ok, "unexpected type: %T", v) + actualResponse, ok := v.(*models.BlockDigestMessageResponse) require.True(s.T(), ok, "unexpected response type: %T", v) diff --git a/engine/access/rest/websockets/data_providers/block_headers_provider_test.go b/engine/access/rest/websockets/data_providers/block_headers_provider_test.go index 57c262d8795..b929a46d076 100644 --- a/engine/access/rest/websockets/data_providers/block_headers_provider_test.go +++ b/engine/access/rest/websockets/data_providers/block_headers_provider_test.go @@ -106,20 +106,26 @@ func (s *BlockHeadersProviderSuite) validBlockHeadersArgumentsTestCases() []test // validates that block headers are correctly streamed to the channel and ensures // no unexpected errors occur. func (s *BlockHeadersProviderSuite) TestBlockHeadersDataProvider_HappyPath() { - s.testHappyPath( + testHappyPath( + s.T(), BlockHeadersTopic, + s.factory, s.validBlockHeadersArgumentsTestCases(), - func(dataChan chan interface{}, blocks []*flow.Block) { - for _, block := range blocks { + func(dataChan chan interface{}) { + for _, block := range s.blocks { dataChan <- block.Header } }, - s.requireBlockHeaders, + s.blocks, + s.requireBlockHeader, ) } // requireBlockHeaders ensures that the received block header information matches the expected data. -func (s *BlockHeadersProviderSuite) requireBlockHeaders(v interface{}, expectedBlock *flow.Block) { +func (s *BlockHeadersProviderSuite) requireBlockHeader(v interface{}, expected interface{}) { + expectedBlock, ok := expected.(*flow.Block) + require.True(s.T(), ok, "unexpected type: %T", v) + actualResponse, ok := v.(*models.BlockHeaderMessageResponse) require.True(s.T(), ok, "unexpected response type: %T", v) diff --git a/engine/access/rest/websockets/data_providers/blocks_provider_test.go b/engine/access/rest/websockets/data_providers/blocks_provider_test.go index 51cf9e63e44..49092109659 100644 --- a/engine/access/rest/websockets/data_providers/blocks_provider_test.go +++ b/engine/access/rest/websockets/data_providers/blocks_provider_test.go @@ -197,44 +197,54 @@ func (s *BlocksProviderSuite) validBlockArgumentsTestCases() []testType { // validates that blocks are correctly streamed to the channel and ensures // no unexpected errors occur. func (s *BlocksProviderSuite) TestBlocksDataProvider_HappyPath() { - s.testHappyPath( + testHappyPath( + s.T(), BlocksTopic, + s.factory, s.validBlockArgumentsTestCases(), - func(dataChan chan interface{}, blocks []*flow.Block) { - for _, block := range blocks { + func(dataChan chan interface{}) { + for _, block := range s.blocks { dataChan <- block } }, + s.blocks, s.requireBlock, ) } // requireBlocks ensures that the received block information matches the expected data. -func (s *BlocksProviderSuite) requireBlock(v interface{}, expectedBlock *flow.Block) { +func (s *BlocksProviderSuite) requireBlock(v interface{}, expected interface{}) { + expectedBlock, ok := expected.(*flow.Block) + require.True(s.T(), ok, "unexpected type: %T", v) + actualResponse, ok := v.(*models.BlockMessageResponse) require.True(s.T(), ok, "unexpected response type: %T", v) s.Require().Equal(expectedBlock, actualResponse.Block) } -// testHappyPath tests a variety of scenarios for data providers in +// TestHappyPath tests a variety of scenarios for data providers in // happy path scenarios. This function runs parameterized test cases that // simulate various configurations and verifies that the data provider operates // as expected without encountering errors. // +// TODO: update arguments // Arguments: // - topic: The topic associated with the data provider. // - tests: A slice of test cases to run, each specifying setup and validation logic. // - sendData: A function to simulate emitting data into the subscription's data channel. // - requireFn: A function to validate the output received in the send channel. -func (s *BlocksProviderSuite) testHappyPath( +func testHappyPath[T any]( + t *testing.T, topic string, + factory *DataProviderFactoryImpl, tests []testType, - sendData func(chan interface{}, []*flow.Block), - requireFn func(interface{}, *flow.Block), + sendData func(chan interface{}), + expectedResponses []T, + requireFn func(interface{}, interface{}), ) { for _, test := range tests { - s.Run(test.name, func() { + t.Run(test.name, func(t *testing.T) { ctx := context.Background() send := make(chan interface{}, 10) @@ -242,36 +252,36 @@ func (s *BlocksProviderSuite) testHappyPath( dataChan := make(chan interface{}) // Create a mock subscription and mock the channel - sub := statestreamsmock.NewSubscription(s.T()) + sub := statestreamsmock.NewSubscription(t) sub.On("Channel").Return((<-chan interface{})(dataChan)) sub.On("Err").Return(nil) test.setupBackend(sub) // Create the data provider instance - provider, err := s.factory.NewDataProvider(ctx, topic, test.arguments, send) - s.Require().NotNil(provider) - s.Require().NoError(err) + provider, err := factory.NewDataProvider(ctx, topic, test.arguments, send) + require.NotNil(t, provider) + require.NoError(t, err) // Run the provider in a separate goroutine go func() { err = provider.Run() - s.Require().NoError(err) + require.NoError(t, err) }() // Simulate emitting data to the data channel go func() { defer close(dataChan) - sendData(dataChan, s.blocks) + sendData(dataChan) }() // Collect responses - for _, b := range s.blocks { - unittest.RequireReturnsBefore(s.T(), func() { + for i, expected := range expectedResponses { + unittest.RequireReturnsBefore(t, func() { v, ok := <-send - s.Require().True(ok, "channel closed while waiting for block %x %v: err: %v", b.Header.Height, b.ID(), sub.Err()) + require.True(t, ok, "channel closed while waiting for response %v: err: %v", expected, sub.Err()) - requireFn(v, b) - }, time.Second, fmt.Sprintf("timed out waiting for block %d %v", b.Header.Height, b.ID())) + requireFn(v, expected) + }, time.Second, fmt.Sprintf("timed out waiting for response %d %v", i, expected)) } // Ensure the provider is properly closed after the test diff --git a/engine/access/rest/websockets/data_providers/events_provider_test.go b/engine/access/rest/websockets/data_providers/events_provider_test.go index 336744d24c7..ed179e2c463 100644 --- a/engine/access/rest/websockets/data_providers/events_provider_test.go +++ b/engine/access/rest/websockets/data_providers/events_provider_test.go @@ -62,11 +62,43 @@ func (s *EventsProviderSuite) SetupTest() { // validates that events are correctly streamed to the channel and ensures // no unexpected errors occur. func (s *EventsProviderSuite) TestEventsDataProvider_HappyPath() { - s.testHappyPath( + + expectedEvents := []flow.Event{ + unittest.EventFixture(flow.EventAccountCreated, 0, 0, unittest.IdentifierFixture(), 0), + unittest.EventFixture(flow.EventAccountUpdated, 0, 0, unittest.IdentifierFixture(), 0), + unittest.EventFixture(flow.EventAccountCreated, 0, 0, unittest.IdentifierFixture(), 0), + unittest.EventFixture(flow.EventAccountUpdated, 0, 0, unittest.IdentifierFixture(), 0), + } + + var expectedEventsResponses []backend.EventsResponse + for i := 0; i < len(expectedEvents); i++ { + expectedEventsResponses = append(expectedEventsResponses, backend.EventsResponse{ + Height: s.rootBlock.Header.Height, + BlockID: s.rootBlock.ID(), + Events: expectedEvents, + BlockTimestamp: s.rootBlock.Header.Timestamp, + }) + + } + + testHappyPath( + s.T(), EventsTopic, + s.factory, s.subscribeEventsDataProviderTestCases(), + func(dataChan chan interface{}) { + //for _, block := range s.blocks { + // dataChan <- block + //} + + for i := 0; i < len(expectedEventsResponses); i++ { + dataChan <- &expectedEventsResponses[i] + } + }, + expectedEventsResponses, s.requireEvents, ) + } // subscribeEventsDataProviderTestCases generates test cases for events data providers. @@ -115,93 +147,15 @@ func (s *EventsProviderSuite) subscribeEventsDataProviderTestCases() []testType } } -// testHappyPath tests a variety of scenarios for data providers in -// happy path scenarios. This function runs parameterized test cases that -// simulate various configurations and verifies that the data provider operates -// as expected without encountering errors. -// -// Arguments: -// - topic: The topic associated with the data provider. -// - tests: A slice of test cases to run, each specifying setup and validation logic. -// - requireFn: A function to validate the output received in the send channel. -func (s *EventsProviderSuite) testHappyPath( - topic string, - tests []testType, - requireFn func(interface{}, *backend.EventsResponse), -) { - expectedEvents := []flow.Event{ - unittest.EventFixture(flow.EventAccountCreated, 0, 0, unittest.IdentifierFixture(), 0), - unittest.EventFixture(flow.EventAccountUpdated, 0, 0, unittest.IdentifierFixture(), 0), - unittest.EventFixture(flow.EventAccountCreated, 0, 0, unittest.IdentifierFixture(), 0), - unittest.EventFixture(flow.EventAccountUpdated, 0, 0, unittest.IdentifierFixture(), 0), - } - - var expectedEventsResponses []backend.EventsResponse - - for i := 0; i < len(expectedEvents); i++ { - expectedEventsResponses = append(expectedEventsResponses, backend.EventsResponse{ - Height: s.rootBlock.Header.Height, - BlockID: s.rootBlock.ID(), - Events: expectedEvents, - BlockTimestamp: s.rootBlock.Header.Timestamp, - }) - - } - - for _, test := range tests { - s.Run(test.name, func() { - ctx := context.Background() - send := make(chan interface{}, 10) - - // Create a channel to simulate the subscription's data channel - eventChan := make(chan interface{}) - - // // Create a mock subscription and mock the channel - sub := ssmock.NewSubscription(s.T()) - sub.On("Channel").Return((<-chan interface{})(eventChan)) - sub.On("Err").Return(nil) - test.setupBackend(sub) - - // Create the data provider instance - provider, err := s.factory.NewDataProvider(ctx, topic, test.arguments, send) - s.Require().NotNil(provider) - s.Require().NoError(err) - - // Run the provider in a separate goroutine - go func() { - err = provider.Run() - s.Require().NoError(err) - }() - - // Simulate emitting data to the events channel - go func() { - defer close(eventChan) - - for i := 0; i < len(expectedEventsResponses); i++ { - eventChan <- &expectedEventsResponses[i] - } - }() - - // Collect responses - for _, e := range expectedEventsResponses { - v, ok := <-send - s.Require().True(ok, "channel closed while waiting for event %v: err: %v", e.BlockID, sub.Err()) - - requireFn(v, &e) - } - - // Ensure the provider is properly closed after the test - provider.Close() - }) - } -} - // requireEvents ensures that the received event information matches the expected data. -func (s *EventsProviderSuite) requireEvents(v interface{}, expectedEventsResponse *backend.EventsResponse) { +func (s *EventsProviderSuite) requireEvents(v interface{}, expectedEventsResponse interface{}) { + expectedResponse, ok := expectedEventsResponse.(backend.EventsResponse) + require.True(s.T(), ok, "unexpected type: %T", expectedEventsResponse) + actualResponse, ok := v.(*models.EventResponse) require.True(s.T(), ok, "Expected *models.EventResponse, got %T", v) - s.Require().ElementsMatch(expectedEventsResponse.Events, actualResponse.Events) + s.Require().ElementsMatch(expectedResponse.Events, actualResponse.Events) } // invalidArgumentsTestCases returns a list of test cases with invalid argument combinations From 85e446465563467d75ac5913c750afe7cbe6625c Mon Sep 17 00:00:00 2001 From: Andrii Date: Wed, 11 Dec 2024 14:17:08 +0200 Subject: [PATCH 40/67] Refactored account statuses test --- .../account_statuses_provider_test.go | 75 ++++--------------- .../data_providers/events_provider_test.go | 13 +--- 2 files changed, 20 insertions(+), 68 deletions(-) diff --git a/engine/access/rest/websockets/data_providers/account_statuses_provider_test.go b/engine/access/rest/websockets/data_providers/account_statuses_provider_test.go index 98aabf50551..3c54f2a7e15 100644 --- a/engine/access/rest/websockets/data_providers/account_statuses_provider_test.go +++ b/engine/access/rest/websockets/data_providers/account_statuses_provider_test.go @@ -61,25 +61,13 @@ func (s *AccountStatusesProviderSuite) SetupTest() { // validates that events are correctly streamed to the channel and ensures // no unexpected errors occur. func (s *AccountStatusesProviderSuite) TestAccountStatusesDataProvider_HappyPath() { - s.testHappyPath( - AccountStatusesTopic, - s.subscribeAccountStatusesDataProviderTestCases(), - s.requireAccountStatuses, - ) -} -func (s *AccountStatusesProviderSuite) testHappyPath( - topic string, - tests []testType, - requireFn func(interface{}, *backend.AccountStatusesResponse), -) { expectedEvents := []flow.Event{ unittest.EventFixture(state_stream.CoreEventAccountCreated, 0, 0, unittest.IdentifierFixture(), 0), unittest.EventFixture(state_stream.CoreEventAccountKeyAdded, 0, 0, unittest.IdentifierFixture(), 0), } var expectedAccountStatusesResponses []backend.AccountStatusesResponse - for i := 0; i < len(expectedEvents); i++ { expectedAccountStatusesResponses = append(expectedAccountStatusesResponses, backend.AccountStatusesResponse{ Height: s.rootBlock.Header.Height, @@ -90,52 +78,19 @@ func (s *AccountStatusesProviderSuite) testHappyPath( }) } - for _, test := range tests { - s.Run(test.name, func() { - ctx := context.Background() - send := make(chan interface{}, 10) - - // Create a channel to simulate the subscription's data channel - accStatusesChan := make(chan interface{}) - - // // Create a mock subscription and mock the channel - sub := ssmock.NewSubscription(s.T()) - sub.On("Channel").Return((<-chan interface{})(accStatusesChan)) - sub.On("Err").Return(nil) - test.setupBackend(sub) - - // Create the data provider instance - provider, err := s.factory.NewDataProvider(ctx, topic, test.arguments, send) - s.Require().NotNil(provider) - s.Require().NoError(err) - - // Run the provider in a separate goroutine - go func() { - err = provider.Run() - s.Require().NoError(err) - }() - - // Simulate emitting data to the events channel - go func() { - defer close(accStatusesChan) - - for i := 0; i < len(expectedAccountStatusesResponses); i++ { - accStatusesChan <- &expectedAccountStatusesResponses[i] - } - }() - - // Collect responses - for _, e := range expectedAccountStatusesResponses { - v, ok := <-send - s.Require().True(ok, "channel closed while waiting for event %v: err: %v", e.BlockID, sub.Err()) - - requireFn(v, &e) + testHappyPath( + s.T(), + AccountStatusesTopic, + s.factory, + s.subscribeAccountStatusesDataProviderTestCases(), + func(dataChan chan interface{}) { + for i := 0; i < len(expectedAccountStatusesResponses); i++ { + dataChan <- &expectedAccountStatusesResponses[i] } - - // Ensure the provider is properly closed after the test - provider.Close() - }) - } + }, + expectedAccountStatusesResponses, + s.requireAccountStatuses, + ) } func (s *AccountStatusesProviderSuite) subscribeAccountStatusesDataProviderTestCases() []testType { @@ -186,8 +141,11 @@ func (s *AccountStatusesProviderSuite) subscribeAccountStatusesDataProviderTestC // requireAccountStatuses ensures that the received account statuses information matches the expected data. func (s *AccountStatusesProviderSuite) requireAccountStatuses( v interface{}, - expectedAccountStatusesResponse *backend.AccountStatusesResponse, + expectedResponse interface{}, ) { + expectedAccountStatusesResponse, ok := expectedResponse.(backend.AccountStatusesResponse) + require.True(s.T(), ok, "unexpected type: %T", expectedResponse) + actualResponse, ok := v.(*models.AccountStatusesResponse) require.True(s.T(), ok, "Expected *models.AccountStatusesResponse, got %T", v) @@ -200,7 +158,6 @@ func (s *AccountStatusesProviderSuite) requireAccountStatuses( s.Require().Equal(expectedEvents, actualEvents, "Mismatch for key: %s", key) } - } // TestAccountStatusesDataProvider_InvalidArguments tests the behavior of the account statuses data provider diff --git a/engine/access/rest/websockets/data_providers/events_provider_test.go b/engine/access/rest/websockets/data_providers/events_provider_test.go index ed179e2c463..5f600077a4d 100644 --- a/engine/access/rest/websockets/data_providers/events_provider_test.go +++ b/engine/access/rest/websockets/data_providers/events_provider_test.go @@ -87,10 +87,6 @@ func (s *EventsProviderSuite) TestEventsDataProvider_HappyPath() { s.factory, s.subscribeEventsDataProviderTestCases(), func(dataChan chan interface{}) { - //for _, block := range s.blocks { - // dataChan <- block - //} - for i := 0; i < len(expectedEventsResponses); i++ { dataChan <- &expectedEventsResponses[i] } @@ -98,7 +94,6 @@ func (s *EventsProviderSuite) TestEventsDataProvider_HappyPath() { expectedEventsResponses, s.requireEvents, ) - } // subscribeEventsDataProviderTestCases generates test cases for events data providers. @@ -148,14 +143,14 @@ func (s *EventsProviderSuite) subscribeEventsDataProviderTestCases() []testType } // requireEvents ensures that the received event information matches the expected data. -func (s *EventsProviderSuite) requireEvents(v interface{}, expectedEventsResponse interface{}) { - expectedResponse, ok := expectedEventsResponse.(backend.EventsResponse) - require.True(s.T(), ok, "unexpected type: %T", expectedEventsResponse) +func (s *EventsProviderSuite) requireEvents(v interface{}, expectedResponse interface{}) { + expectedEventsResponse, ok := expectedResponse.(backend.EventsResponse) + require.True(s.T(), ok, "unexpected type: %T", expectedResponse) actualResponse, ok := v.(*models.EventResponse) require.True(s.T(), ok, "Expected *models.EventResponse, got %T", v) - s.Require().ElementsMatch(expectedResponse.Events, actualResponse.Events) + s.Require().ElementsMatch(expectedEventsResponse.Events, actualResponse.Events) } // invalidArgumentsTestCases returns a list of test cases with invalid argument combinations From 03e5e42c5fdf56db75e15dc4f7dfa153219ad0d0 Mon Sep 17 00:00:00 2001 From: Andrii Date: Wed, 11 Dec 2024 14:18:56 +0200 Subject: [PATCH 41/67] Decreased expected events count --- .../rest/websockets/data_providers/events_provider_test.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/engine/access/rest/websockets/data_providers/events_provider_test.go b/engine/access/rest/websockets/data_providers/events_provider_test.go index 5f600077a4d..6bbe1f36a44 100644 --- a/engine/access/rest/websockets/data_providers/events_provider_test.go +++ b/engine/access/rest/websockets/data_providers/events_provider_test.go @@ -66,8 +66,6 @@ func (s *EventsProviderSuite) TestEventsDataProvider_HappyPath() { expectedEvents := []flow.Event{ unittest.EventFixture(flow.EventAccountCreated, 0, 0, unittest.IdentifierFixture(), 0), unittest.EventFixture(flow.EventAccountUpdated, 0, 0, unittest.IdentifierFixture(), 0), - unittest.EventFixture(flow.EventAccountCreated, 0, 0, unittest.IdentifierFixture(), 0), - unittest.EventFixture(flow.EventAccountUpdated, 0, 0, unittest.IdentifierFixture(), 0), } var expectedEventsResponses []backend.EventsResponse From 2bbd39916b8c346bd492e9717e59e550b659ec9e Mon Sep 17 00:00:00 2001 From: Andrii Date: Wed, 11 Dec 2024 17:11:43 +0200 Subject: [PATCH 42/67] Added skeleton for tx account data provider --- .../transaction_statuses_provider.go | 93 +++++++++++++++++++ 1 file changed, 93 insertions(+) create mode 100644 engine/access/rest/websockets/data_providers/transaction_statuses_provider.go diff --git a/engine/access/rest/websockets/data_providers/transaction_statuses_provider.go b/engine/access/rest/websockets/data_providers/transaction_statuses_provider.go new file mode 100644 index 00000000000..e17f61b116b --- /dev/null +++ b/engine/access/rest/websockets/data_providers/transaction_statuses_provider.go @@ -0,0 +1,93 @@ +package data_providers + +import ( + "context" + "fmt" + "github.com/onflow/flow/protobuf/go/flow/entities" + + "github.com/rs/zerolog" + + "github.com/onflow/flow-go/access" + "github.com/onflow/flow-go/engine/access/rest/websockets/models" + "github.com/onflow/flow-go/engine/access/state_stream" + "github.com/onflow/flow-go/engine/access/state_stream/backend" + "github.com/onflow/flow-go/engine/access/subscription" + "github.com/onflow/flow-go/model/flow" +) + +type TransactionStatusesArguments struct { + StartBlockID flow.Identifier // ID of the block to start subscription from + txID flow.Identifier // ID of the transaction to monitor. +} + +type TransactionStatusesDataProvider struct { + *baseDataProvider + + logger zerolog.Logger + stateStreamApi state_stream.API + api access.API +} + +var _ DataProvider = (*TransactionStatusesDataProvider)(nil) + +func NewTransactionStatusesDataProvider( + ctx context.Context, + logger zerolog.Logger, + stateStreamApi state_stream.API, + topic string, + arguments models.Arguments, + send chan<- interface{}, + chain flow.Chain, +) (*TransactionStatusesDataProvider, error) { + p := &TransactionStatusesDataProvider{ + logger: logger.With().Str("component", "transaction-statuses-data-provider").Logger(), + stateStreamApi: stateStreamApi, + } + + // Initialize arguments passed to the provider. + txStatusesArgs, err := parseTransactionStatusesArguments(arguments, chain) + if err != nil { + return nil, fmt.Errorf("invalid arguments for tx statuses data provider: %w", err) + } + + subCtx, cancel := context.WithCancel(ctx) + + p.baseDataProvider = newBaseDataProvider( + topic, + cancel, + send, + p.createSubscription(subCtx, txStatusesArgs), // Set up a subscription to tx statuses based on arguments. + ) + + return p, nil +} + +// Run starts processing the subscription for events and handles responses. +// +// No errors are expected during normal operations. +func (p *TransactionStatusesDataProvider) Run() error { + return subscription.HandleSubscription(p.subscription, p.handleResponse()) +} + +// createSubscription creates a new subscription using the specified input arguments. +func (p *TransactionStatusesDataProvider) createSubscription( + ctx context.Context, + args TransactionStatusesArguments, +) subscription.Subscription { + return p.api.SubscribeTransactionStatuses(ctx, args.txID, args.StartBlockID, entities.EventEncodingVersion_JSON_CDC_V0) +} + +// handleResponse processes an account statuses and sends the formatted response. +// +// No errors are expected during normal operations. +func (p *TransactionStatusesDataProvider) handleResponse() func(txStatusesResponse *backend.) { + +} + +// parseAccountStatusesArguments validates and initializes the account statuses arguments. +func parseTransactionStatusesArguments( + arguments models.Arguments, + chain flow.Chain, +) (TransactionStatusesArguments, error) { + +} From de5f077b5f101dd41ce84d11208de1aaa428cd8b Mon Sep 17 00:00:00 2001 From: Andrii Date: Wed, 11 Dec 2024 17:19:19 +0200 Subject: [PATCH 43/67] Added description for missing func params --- .../rest/websockets/data_providers/blocks_provider_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/engine/access/rest/websockets/data_providers/blocks_provider_test.go b/engine/access/rest/websockets/data_providers/blocks_provider_test.go index 49092109659..cf3f6997d3c 100644 --- a/engine/access/rest/websockets/data_providers/blocks_provider_test.go +++ b/engine/access/rest/websockets/data_providers/blocks_provider_test.go @@ -228,11 +228,12 @@ func (s *BlocksProviderSuite) requireBlock(v interface{}, expected interface{}) // simulate various configurations and verifies that the data provider operates // as expected without encountering errors. // -// TODO: update arguments // Arguments: // - topic: The topic associated with the data provider. +// - factory: A factory for creating data provider instance. // - tests: A slice of test cases to run, each specifying setup and validation logic. // - sendData: A function to simulate emitting data into the subscription's data channel. +// - expectedResponses: An expected responses to validate the received output. // - requireFn: A function to validate the output received in the send channel. func testHappyPath[T any]( t *testing.T, From 683f0260513de07a4567e23383aa95c791b81f12 Mon Sep 17 00:00:00 2001 From: Andrii Date: Wed, 11 Dec 2024 18:55:38 +0200 Subject: [PATCH 44/67] Implemented tx statuses provider --- .../account_statuses_provider.go | 1 - .../rest/websockets/data_providers/factory.go | 3 +- .../websockets/data_providers/factory_test.go | 11 +++ .../transaction_statuses_provider.go | 69 +++++++++++++++---- .../websockets/models/tx_statuses_model.go | 11 +++ 5 files changed, 79 insertions(+), 16 deletions(-) create mode 100644 engine/access/rest/websockets/models/tx_statuses_model.go diff --git a/engine/access/rest/websockets/data_providers/account_statuses_provider.go b/engine/access/rest/websockets/data_providers/account_statuses_provider.go index 1a3aee203c9..bf3d77ce823 100644 --- a/engine/access/rest/websockets/data_providers/account_statuses_provider.go +++ b/engine/access/rest/websockets/data_providers/account_statuses_provider.go @@ -157,7 +157,6 @@ func parseAccountStatusesArguments( args.StartBlockID = startBlockID.Flow() } - // Parse 'start_block_height' if provided // Parse 'start_block_height' if provided if hasStartBlockHeight { result, ok := startBlockHeightIn.(string) diff --git a/engine/access/rest/websockets/data_providers/factory.go b/engine/access/rest/websockets/data_providers/factory.go index 26aade4e090..b8897deb988 100644 --- a/engine/access/rest/websockets/data_providers/factory.go +++ b/engine/access/rest/websockets/data_providers/factory.go @@ -103,8 +103,7 @@ func (s *DataProviderFactoryImpl) NewDataProvider( case AccountStatusesTopic: return NewAccountStatusesDataProvider(ctx, s.logger, s.stateStreamApi, topic, arguments, ch, s.chain, s.eventFilterConfig, s.heartbeatInterval) case TransactionStatusesTopic: - // TODO: Implemented handlers for each topic should be added in respective case - return nil, fmt.Errorf(`topic "%s" not implemented yet`, topic) + return NewTransactionStatusesDataProvider(ctx, s.logger, s.accessApi, topic, arguments, ch) default: return nil, fmt.Errorf("unsupported topic \"%s\"", topic) } diff --git a/engine/access/rest/websockets/data_providers/factory_test.go b/engine/access/rest/websockets/data_providers/factory_test.go index 9421cef37f6..674fc6998ea 100644 --- a/engine/access/rest/websockets/data_providers/factory_test.go +++ b/engine/access/rest/websockets/data_providers/factory_test.go @@ -132,6 +132,17 @@ func (s *DataProviderFactorySuite) TestSupportedTopics() { s.stateStreamApi.AssertExpectations(s.T()) }, }, + { + name: "transaction statuses topic", + topic: TransactionStatusesTopic, + arguments: models.Arguments{}, + setupSubscription: func() { + s.setupSubscription(s.accessApi.On("SubscribeTransactionStatuses", mock.Anything, mock.Anything, mock.Anything, mock.Anything)) + }, + assertExpectations: func() { + s.stateStreamApi.AssertExpectations(s.T()) + }, + }, } for _, test := range testCases { diff --git a/engine/access/rest/websockets/data_providers/transaction_statuses_provider.go b/engine/access/rest/websockets/data_providers/transaction_statuses_provider.go index e17f61b116b..6a446282d5e 100644 --- a/engine/access/rest/websockets/data_providers/transaction_statuses_provider.go +++ b/engine/access/rest/websockets/data_providers/transaction_statuses_provider.go @@ -3,16 +3,19 @@ package data_providers import ( "context" "fmt" - "github.com/onflow/flow/protobuf/go/flow/entities" + "strconv" "github.com/rs/zerolog" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" "github.com/onflow/flow-go/access" + "github.com/onflow/flow-go/engine/access/rest/common/parser" "github.com/onflow/flow-go/engine/access/rest/websockets/models" - "github.com/onflow/flow-go/engine/access/state_stream" - "github.com/onflow/flow-go/engine/access/state_stream/backend" "github.com/onflow/flow-go/engine/access/subscription" "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module/counters" + "github.com/onflow/flow/protobuf/go/flow/entities" ) type TransactionStatusesArguments struct { @@ -23,9 +26,8 @@ type TransactionStatusesArguments struct { type TransactionStatusesDataProvider struct { *baseDataProvider - logger zerolog.Logger - stateStreamApi state_stream.API - api access.API + logger zerolog.Logger + api access.API } var _ DataProvider = (*TransactionStatusesDataProvider)(nil) @@ -33,19 +35,18 @@ var _ DataProvider = (*TransactionStatusesDataProvider)(nil) func NewTransactionStatusesDataProvider( ctx context.Context, logger zerolog.Logger, - stateStreamApi state_stream.API, + api access.API, topic string, arguments models.Arguments, send chan<- interface{}, - chain flow.Chain, ) (*TransactionStatusesDataProvider, error) { p := &TransactionStatusesDataProvider{ - logger: logger.With().Str("component", "transaction-statuses-data-provider").Logger(), - stateStreamApi: stateStreamApi, + logger: logger.With().Str("component", "transaction-statuses-data-provider").Logger(), + api: api, } // Initialize arguments passed to the provider. - txStatusesArgs, err := parseTransactionStatusesArguments(arguments, chain) + txStatusesArgs, err := parseTransactionStatusesArguments(arguments) if err != nil { return nil, fmt.Errorf("invalid arguments for tx statuses data provider: %w", err) } @@ -80,14 +81,56 @@ func (p *TransactionStatusesDataProvider) createSubscription( // handleResponse processes an account statuses and sends the formatted response. // // No errors are expected during normal operations. -func (p *TransactionStatusesDataProvider) handleResponse() func(txStatusesResponse *backend.) { +func (p *TransactionStatusesDataProvider) handleResponse() func(txResults []*access.TransactionResult) error { + messageIndex := counters.NewMonotonousCounter(1) + + return func(txResults []*access.TransactionResult) error { + + index := messageIndex.Value() + if ok := messageIndex.Set(messageIndex.Value() + 1); !ok { + return status.Errorf(codes.Internal, "message index already incremented to %d", messageIndex.Value()) + } + + p.send <- models.TransactionStatusesResponse{ + TransactionResults: txResults, + MessageIndex: strconv.FormatUint(index, 10), + } + return nil + } } // parseAccountStatusesArguments validates and initializes the account statuses arguments. func parseTransactionStatusesArguments( arguments models.Arguments, - chain flow.Chain, ) (TransactionStatusesArguments, error) { + var args TransactionStatusesArguments + + if txIDIn, ok := arguments["tx_id"]; ok && txIDIn != "" { + result, ok := txIDIn.(string) + if !ok { + return args, fmt.Errorf("'tx_id' must be a string") + } + var txID parser.ID + err := txID.Parse(result) + if err != nil { + return args, fmt.Errorf("invalid 'tx_id': %w", err) + } + args.txID = txID.Flow() + } + + if startBlockIDIn, ok := arguments["start_block_id"]; ok && startBlockIDIn != "" { + result, ok := startBlockIDIn.(string) + if !ok { + return args, fmt.Errorf("'start_block_id' must be a string") + } + var startBlockID parser.ID + err := startBlockID.Parse(result) + if err != nil { + return args, fmt.Errorf("invalid 'start_block_id': %w", err) + } + args.StartBlockID = startBlockID.Flow() + } + return args, nil } diff --git a/engine/access/rest/websockets/models/tx_statuses_model.go b/engine/access/rest/websockets/models/tx_statuses_model.go new file mode 100644 index 00000000000..55b9d3bdfdd --- /dev/null +++ b/engine/access/rest/websockets/models/tx_statuses_model.go @@ -0,0 +1,11 @@ +package models + +import ( + "github.com/onflow/flow-go/access" +) + +// TransactionStatusesResponse is the response message for 'events' topic. +type TransactionStatusesResponse struct { + TransactionResults []*access.TransactionResult `json:"transaction_results"` + MessageIndex string `json:"message_index"` +} From 3b19cd07fb9aae0a9e7a1e6477263b00da0db778 Mon Sep 17 00:00:00 2001 From: Andrii Slisarchuk Date: Thu, 12 Dec 2024 16:13:35 +0200 Subject: [PATCH 45/67] Separate subscribe tx statuses to 3 function --- access/api.go | 33 +++++++-- access/mock/api.go | 50 ++++++++++++-- cmd/util/cmd/run-script/cmd.go | 18 ++++- .../backend/backend_stream_transactions.go | 67 +++++++++++++------ .../backend_stream_transactions_test.go | 4 +- 5 files changed, 139 insertions(+), 33 deletions(-) diff --git a/access/api.go b/access/api.go index 63db940414a..acd4d6138b8 100644 --- a/access/api.go +++ b/access/api.go @@ -203,17 +203,38 @@ type API interface { // // If invalid parameters will be supplied SubscribeBlockDigestsFromLatest will return a failed subscription. SubscribeBlockDigestsFromLatest(ctx context.Context, blockStatus flow.BlockStatus) subscription.Subscription - // SubscribeTransactionStatuses subscribes to transaction status updates for a given transaction ID. - // Monitoring begins from the specified block ID or the latest block if no block ID is provided. - // The subscription streams status updates until the transaction reaches a final state (TransactionStatusSealed or TransactionStatusExpired). - // When the transaction reaches one of these final statuses, the subscription will automatically terminate. + // SubscribeTransactionStatusesFromStartBlockID subscribes to transaction status updates for a given transaction ID. + // Monitoring begins from the specified block ID. The subscription streams status updates until the transaction + // reaches a final state (TransactionStatusSealed or TransactionStatusExpired). When the transaction reaches one of + // these final statuses, the subscription will automatically terminate. + // + // Parameters: + // - ctx: The context to manage the subscription's lifecycle, including cancellation. + // - txID: The identifier of the transaction to monitor. + // - startBlockID: The block ID from which to start monitoring. + // - requiredEventEncodingVersion: The version of event encoding required for the subscription. + SubscribeTransactionStatusesFromStartBlockID(ctx context.Context, txID flow.Identifier, startBlockID flow.Identifier, requiredEventEncodingVersion entities.EventEncodingVersion) subscription.Subscription + // SubscribeTransactionStatusesFromStartHeight subscribes to transaction status updates for a given transaction ID. + // Monitoring begins from the specified block height. The subscription streams status updates until the transaction + // reaches a final state (TransactionStatusSealed or TransactionStatusExpired). When the transaction reaches one of + // these final statuses, the subscription will automatically terminate. + // + // Parameters: + // - ctx: The context to manage the subscription's lifecycle, including cancellation. + // - txID: The unique identifier of the transaction to monitor. + // - startHeight: The block height from which to start monitoring. + // - requiredEventEncodingVersion: The version of event encoding required for the subscription. + SubscribeTransactionStatusesFromStartHeight(ctx context.Context, txID flow.Identifier, startHeight uint64, requiredEventEncodingVersion entities.EventEncodingVersion) subscription.Subscription + // SubscribeTransactionStatusesFromLatest subscribes to transaction status updates for a given transaction ID. + // Monitoring begins from the latest block. The subscription streams status updates until the transaction + // reaches a final state (TransactionStatusSealed or TransactionStatusExpired). When the transaction reaches one of + // these final statuses, the subscription will automatically terminate. // // Parameters: // - ctx: The context to manage the subscription's lifecycle, including cancellation. // - txID: The unique identifier of the transaction to monitor. - // - blockID: The block ID from which to start monitoring. If set to flow.ZeroID, monitoring starts from the latest block. // - requiredEventEncodingVersion: The version of event encoding required for the subscription. - SubscribeTransactionStatuses(ctx context.Context, txID flow.Identifier, blockID flow.Identifier, requiredEventEncodingVersion entities.EventEncodingVersion) subscription.Subscription + SubscribeTransactionStatusesFromLatest(ctx context.Context, txID flow.Identifier, requiredEventEncodingVersion entities.EventEncodingVersion) subscription.Subscription // SendAndSubscribeTransactionStatuses sends a transaction to the execution node and subscribes to its status updates. // Monitoring begins from the reference block saved in the transaction itself and streams status updates until the transaction // reaches a final state (TransactionStatusSealed or TransactionStatusExpired). Once a final status is reached, the subscription diff --git a/access/mock/api.go b/access/mock/api.go index 274cc688a5a..13c35b293d3 100644 --- a/access/mock/api.go +++ b/access/mock/api.go @@ -1363,17 +1363,57 @@ func (_m *API) SubscribeBlocksFromStartHeight(ctx context.Context, startHeight u return r0 } -// SubscribeTransactionStatuses provides a mock function with given fields: ctx, txID, blockID, requiredEventEncodingVersion -func (_m *API) SubscribeTransactionStatuses(ctx context.Context, txID flow.Identifier, blockID flow.Identifier, requiredEventEncodingVersion entities.EventEncodingVersion) subscription.Subscription { - ret := _m.Called(ctx, txID, blockID, requiredEventEncodingVersion) +// SubscribeTransactionStatusesFromLatest provides a mock function with given fields: ctx, txID, requiredEventEncodingVersion +func (_m *API) SubscribeTransactionStatusesFromLatest(ctx context.Context, txID flow.Identifier, requiredEventEncodingVersion entities.EventEncodingVersion) subscription.Subscription { + ret := _m.Called(ctx, txID, requiredEventEncodingVersion) if len(ret) == 0 { - panic("no return value specified for SubscribeTransactionStatuses") + panic("no return value specified for SubscribeTransactionStatusesFromLatest") + } + + var r0 subscription.Subscription + if rf, ok := ret.Get(0).(func(context.Context, flow.Identifier, entities.EventEncodingVersion) subscription.Subscription); ok { + r0 = rf(ctx, txID, requiredEventEncodingVersion) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(subscription.Subscription) + } + } + + return r0 +} + +// SubscribeTransactionStatusesFromStartBlockID provides a mock function with given fields: ctx, txID, startBlockID, requiredEventEncodingVersion +func (_m *API) SubscribeTransactionStatusesFromStartBlockID(ctx context.Context, txID flow.Identifier, startBlockID flow.Identifier, requiredEventEncodingVersion entities.EventEncodingVersion) subscription.Subscription { + ret := _m.Called(ctx, txID, startBlockID, requiredEventEncodingVersion) + + if len(ret) == 0 { + panic("no return value specified for SubscribeTransactionStatusesFromStartBlockID") } var r0 subscription.Subscription if rf, ok := ret.Get(0).(func(context.Context, flow.Identifier, flow.Identifier, entities.EventEncodingVersion) subscription.Subscription); ok { - r0 = rf(ctx, txID, blockID, requiredEventEncodingVersion) + r0 = rf(ctx, txID, startBlockID, requiredEventEncodingVersion) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(subscription.Subscription) + } + } + + return r0 +} + +// SubscribeTransactionStatusesFromStartHeight provides a mock function with given fields: ctx, txID, startHeight, requiredEventEncodingVersion +func (_m *API) SubscribeTransactionStatusesFromStartHeight(ctx context.Context, txID flow.Identifier, startHeight uint64, requiredEventEncodingVersion entities.EventEncodingVersion) subscription.Subscription { + ret := _m.Called(ctx, txID, startHeight, requiredEventEncodingVersion) + + if len(ret) == 0 { + panic("no return value specified for SubscribeTransactionStatusesFromStartHeight") + } + + var r0 subscription.Subscription + if rf, ok := ret.Get(0).(func(context.Context, flow.Identifier, uint64, entities.EventEncodingVersion) subscription.Subscription); ok { + r0 = rf(ctx, txID, startHeight, requiredEventEncodingVersion) } else { if ret.Get(0) != nil { r0 = ret.Get(0).(subscription.Subscription) diff --git a/cmd/util/cmd/run-script/cmd.go b/cmd/util/cmd/run-script/cmd.go index d66aa341d93..59646d0687a 100644 --- a/cmd/util/cmd/run-script/cmd.go +++ b/cmd/util/cmd/run-script/cmd.go @@ -532,7 +532,7 @@ func (*api) SubscribeBlockDigestsFromLatest( return nil } -func (a *api) SubscribeTransactionStatuses( +func (a *api) SubscribeTransactionStatusesFromStartBlockID( _ context.Context, _ flow.Identifier, _ flow.Identifier, @@ -541,6 +541,22 @@ func (a *api) SubscribeTransactionStatuses( return nil } +func (a *api) SubscribeTransactionStatusesFromStartHeight( + _ context.Context, + _ flow.Identifier, + _ uint64, + _ entities.EventEncodingVersion, +) subscription.Subscription { + return nil +} + +func (a *api) SubscribeTransactionStatusesFromLatest( + _ context.Context, + _ flow.Identifier, + _ entities.EventEncodingVersion, +) subscription.Subscription { + return nil +} func (a *api) SendAndSubscribeTransactionStatuses( _ context.Context, _ *flow.TransactionBody, diff --git a/engine/access/rpc/backend/backend_stream_transactions.go b/engine/access/rpc/backend/backend_stream_transactions.go index 718d41ec5de..ae6678e0a8b 100644 --- a/engine/access/rpc/backend/backend_stream_transactions.go +++ b/engine/access/rpc/backend/backend_stream_transactions.go @@ -59,29 +59,48 @@ func (b *backendSubscribeTransactions) SendAndSubscribeTransactionStatuses( return subscription.NewFailedSubscription(err, "failed to send transaction") } - return b.createSubscription(ctx, tx.ID(), tx.ReferenceBlockID, tx.ReferenceBlockID, requiredEventEncodingVersion, true) + return b.createSubscription(ctx, tx.ID(), tx.ReferenceBlockID, 0, tx.ReferenceBlockID, requiredEventEncodingVersion, true) } -// SubscribeTransactionStatuses subscribes to the status updates of a transaction. -// Monitoring starts from the specified block ID or the latest block if no block ID is provided. +// SubscribeTransactionStatusesFromStartHeight subscribes to the status updates of a transaction. +// Monitoring starts from the specified block height. +// If the block height cannot be determined or an error occurs during subscription creation, a failed subscription is returned. +func (b *backendSubscribeTransactions) SubscribeTransactionStatusesFromStartHeight( + ctx context.Context, + txID flow.Identifier, + startHeight uint64, + requiredEventEncodingVersion entities.EventEncodingVersion, +) subscription.Subscription { + return b.createSubscription(ctx, txID, flow.ZeroID, startHeight, flow.ZeroID, requiredEventEncodingVersion, false) +} + +// SubscribeTransactionStatusesFromStartBlockID subscribes to the status updates of a transaction. +// Monitoring starts from the specified block ID. // If the block ID cannot be determined or an error occurs during subscription creation, a failed subscription is returned. -func (b *backendSubscribeTransactions) SubscribeTransactionStatuses( +func (b *backendSubscribeTransactions) SubscribeTransactionStatusesFromStartBlockID( ctx context.Context, txID flow.Identifier, - blockID flow.Identifier, + startBlockID flow.Identifier, requiredEventEncodingVersion entities.EventEncodingVersion, ) subscription.Subscription { - // if no block ID provided, get latest block ID - if blockID == flow.ZeroID { - header, err := b.txLocalDataProvider.state.Sealed().Head() - if err != nil { - b.log.Error().Err(err).Msg("failed to retrieve latest block") - return subscription.NewFailedSubscription(err, "failed to retrieve latest block") - } - blockID = header.ID() + return b.createSubscription(ctx, txID, startBlockID, 0, flow.ZeroID, requiredEventEncodingVersion, false) +} + +// SubscribeTransactionStatusesFromLatest subscribes to the status updates of a transaction. +// Monitoring starts from the latest block. +// If the block cannot be retrieved or an error occurs during subscription creation, a failed subscription is returned. +func (b *backendSubscribeTransactions) SubscribeTransactionStatusesFromLatest( + ctx context.Context, + txID flow.Identifier, + requiredEventEncodingVersion entities.EventEncodingVersion, +) subscription.Subscription { + header, err := b.txLocalDataProvider.state.Sealed().Head() + if err != nil { + b.log.Error().Err(err).Msg("failed to retrieve latest block") + return subscription.NewFailedSubscription(err, "failed to retrieve latest block") } - return b.createSubscription(ctx, txID, blockID, flow.ZeroID, requiredEventEncodingVersion, false) + return b.createSubscription(ctx, txID, header.ID(), 0, flow.ZeroID, requiredEventEncodingVersion, false) } // createSubscription initializes a subscription for monitoring a transaction's status. @@ -89,16 +108,26 @@ func (b *backendSubscribeTransactions) SubscribeTransactionStatuses( func (b *backendSubscribeTransactions) createSubscription( ctx context.Context, txID flow.Identifier, - blockID flow.Identifier, + startBlockID flow.Identifier, + startBlockHeight uint64, referenceBlockID flow.Identifier, requiredEventEncodingVersion entities.EventEncodingVersion, shouldTriggerPending bool, ) subscription.Subscription { + var nextHeight uint64 + var err error + // Get height to start subscription from - nextHeight, err := b.blockTracker.GetStartHeightFromBlockID(blockID) - if err != nil { - b.log.Error().Err(err).Str("block_id", blockID.String()).Msg("failed to get start height") - return subscription.NewFailedSubscription(err, "failed to get start height") + if startBlockID == flow.ZeroID { + if nextHeight, err = b.blockTracker.GetStartHeightFromHeight(startBlockHeight); err != nil { + b.log.Error().Err(err).Uint64("block_height", startBlockHeight).Msg("failed to get start height") + return subscription.NewFailedSubscription(err, "failed to get start height") + } + } else { + if nextHeight, err = b.blockTracker.GetStartHeightFromBlockID(startBlockID); err != nil { + b.log.Error().Err(err).Str("block_id", startBlockID.String()).Msg("failed to get start height") + return subscription.NewFailedSubscription(err, "failed to get start height") + } } // choose initial transaction status diff --git a/engine/access/rpc/backend/backend_stream_transactions_test.go b/engine/access/rpc/backend/backend_stream_transactions_test.go index 47b494e87b3..52049a4b0ef 100644 --- a/engine/access/rpc/backend/backend_stream_transactions_test.go +++ b/engine/access/rpc/backend/backend_stream_transactions_test.go @@ -245,7 +245,7 @@ func (s *TransactionStatusSuite) addNewFinalizedBlock(parent *flow.Header, notif } } -// TestSubscribeTransactionStatusHappyCase tests the functionality of the SubscribeTransactionStatuses method in the Backend. +// TestSubscribeTransactionStatusHappyCase tests the functionality of the SubscribeTransactionStatusesFromStartBlockID method in the Backend. // It covers the emulation of transaction stages from pending to sealed, and receiving status updates. func (s *TransactionStatusSuite) TestSubscribeTransactionStatusHappyCase() { ctx, cancel := context.WithCancel(context.Background()) @@ -359,7 +359,7 @@ func (s *TransactionStatusSuite) TestSubscribeTransactionStatusHappyCase() { }, 100*time.Millisecond, "timed out waiting for subscription to shutdown") } -// TestSubscribeTransactionStatusExpired tests the functionality of the SubscribeTransactionStatuses method in the Backend +// TestSubscribeTransactionStatusExpired tests the functionality of the SubscribeTransactionStatusesFromStartBlockID method in the Backend // when transaction become expired func (s *TransactionStatusSuite) TestSubscribeTransactionStatusExpired() { ctx, cancel := context.WithCancel(context.Background()) From 699b243af7417e427cecea51a3453bfc65df0a73 Mon Sep 17 00:00:00 2001 From: Andrii Date: Fri, 13 Dec 2024 11:09:32 +0200 Subject: [PATCH 46/67] Added invalid params test --- .../transaction_statuses_provider_test.go | 161 ++++++++++++++++++ 1 file changed, 161 insertions(+) create mode 100644 engine/access/rest/websockets/data_providers/transaction_statuses_provider_test.go diff --git a/engine/access/rest/websockets/data_providers/transaction_statuses_provider_test.go b/engine/access/rest/websockets/data_providers/transaction_statuses_provider_test.go new file mode 100644 index 00000000000..7eac0ec850b --- /dev/null +++ b/engine/access/rest/websockets/data_providers/transaction_statuses_provider_test.go @@ -0,0 +1,161 @@ +package data_providers + +import ( + "context" + "github.com/onflow/flow-go/access" + accessmock "github.com/onflow/flow-go/access/mock" + "github.com/onflow/flow-go/engine/access/rest/websockets/models" + ssmock "github.com/onflow/flow-go/engine/access/state_stream/mock" + "github.com/onflow/flow/protobuf/go/flow/entities" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + "testing" + + "github.com/rs/zerolog" + "github.com/stretchr/testify/suite" + + "github.com/onflow/flow-go/engine/access/state_stream" + "github.com/onflow/flow-go/engine/access/subscription" + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/utils/unittest" +) + +type TransactionStatusesProviderSuite struct { + suite.Suite + + log zerolog.Logger + api access.API + + chain flow.Chain + rootBlock flow.Block + finalizedBlock *flow.Header + + factory *DataProviderFactoryImpl +} + +func TestNewTransactionStatusesDataProvider(t *testing.T) { + suite.Run(t, new(TransactionStatusesProviderSuite)) +} + +func (s *TransactionStatusesProviderSuite) SetupTest() { + s.log = unittest.Logger() + s.api = accessmock.NewAPI(s.T()) + + s.chain = flow.Testnet.Chain() + + s.rootBlock = unittest.BlockFixture() + s.rootBlock.Header.Height = 0 + + s.factory = NewDataProviderFactory( + s.log, + nil, + s.api, + flow.Testnet.Chain(), + state_stream.DefaultEventFilterConfig, + subscription.DefaultHeartbeatInterval) + s.Require().NotNil(s.factory) +} + +func (s *TransactionStatusesProviderSuite) TestTransactionStatusesDataProvider_HappyPath() { + + testHappyPath( + s.T(), + AccountStatusesTopic, + s.factory, + s.subscribeTransactionStatusesDataProviderTestCases(), + func(dataChan chan interface{}) { + for i := 0; i < len(expectedAccountStatusesResponses); i++ { + dataChan <- &expectedAccountStatusesResponses[i] + } + }, + expectedAccountStatusesResponses, + s.requireAccountStatuses, + ) + +} + +func (s *TransactionStatusesProviderSuite) subscribeTransactionStatusesDataProviderTestCases() []testType { + return []testType{ + { + name: "SubscribeTransactionStatuses happy path", + arguments: models.Arguments{ + "start_block_id": s.rootBlock.ID().String(), + "event_types": []string{"flow.AccountCreated", "flow.AccountKeyAdded"}, + }, + setupBackend: func(sub *ssmock.Subscription) { + s.api.On( + "SubscribeTransactionStatuses", + mock.Anything, + mock.Anything, + s.rootBlock.ID(), + entities.EventEncodingVersion_JSON_CDC_V0, + ).Return(sub).Once() + }, + }, + } +} + +// requireAccountStatuses ensures that the received account statuses information matches the expected data. +func (s *AccountStatusesProviderSuite) requireTransactionStatuses( + v interface{}, + expectedResponse interface{}, +) { + expectedTransactionStatusesResponse, ok := expectedResponse.([]access.TransactionResult) + require.True(s.T(), ok, "unexpected type: %T", expectedResponse) + +} + +// TestAccountStatusesDataProvider_InvalidArguments tests the behavior of the transaction statuses data provider +// when invalid arguments are provided. It verifies that appropriate errors are returned +// for missing or conflicting arguments. +// This test covers the test cases: +// 1. Invalid 'tx_id' argument. +// 2. Invalid 'start_block_id' argument. +func (s *TransactionStatusesProviderSuite) TestAccountStatusesDataProvider_InvalidArguments() { + ctx := context.Background() + send := make(chan interface{}) + + topic := TransactionStatusesTopic + + for _, test := range invalidTransactionStatusesArgumentsTestCases() { + s.Run(test.name, func() { + provider, err := NewTransactionStatusesDataProvider( + ctx, + s.log, + s.api, + topic, + test.arguments, + send, + ) + s.Require().Nil(provider) + s.Require().Error(err) + s.Require().Contains(err.Error(), test.expectedErrorMsg) + }) + } +} + +// invalidTransactionStatusesArgumentsTestCases returns a list of test cases with invalid argument combinations +// for testing the behavior of transaction statuses data providers. Each test case includes a name, +// a set of input arguments, and the expected error message that should be returned. +// +// The test cases cover scenarios such as: +// 1. Providing invalid 'tx_id' value. +// 2. Providing invalid 'start_block_id' value. +func invalidTransactionStatusesArgumentsTestCases() []testErrType { + return []testErrType{ + { + name: "invalid 'tx_id' argument", + arguments: map[string]interface{}{ + "tx_id": "invalid_tx_id", + }, + expectedErrorMsg: "invalid ID format", + }, + { + name: "invalid 'start_block_id' argument", + arguments: map[string]interface{}{ + "start_block_id": "invalid_block_id", + }, + expectedErrorMsg: "invalid ID format", + }, + } +} From e83930e28f4354a0a8d618d04a9acce2243730b6 Mon Sep 17 00:00:00 2001 From: Andrii Date: Fri, 13 Dec 2024 11:54:25 +0200 Subject: [PATCH 47/67] Refactored parse functions --- .../websockets/data_providers/account_statuses_provider.go | 6 +++--- .../rest/websockets/data_providers/blocks_provider.go | 4 ++-- .../rest/websockets/data_providers/events_provider.go | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/engine/access/rest/websockets/data_providers/account_statuses_provider.go b/engine/access/rest/websockets/data_providers/account_statuses_provider.go index 1a3aee203c9..8b5a5114a6f 100644 --- a/engine/access/rest/websockets/data_providers/account_statuses_provider.go +++ b/engine/access/rest/websockets/data_providers/account_statuses_provider.go @@ -134,6 +134,7 @@ func parseAccountStatusesArguments( eventFilterConfig state_stream.EventFilterConfig, ) (AccountStatusesArguments, error) { var args AccountStatusesArguments + //var err error // Check for mutual exclusivity of start_block_id and start_block_height early startBlockIDIn, hasStartBlockID := arguments["start_block_id"] @@ -157,18 +158,17 @@ func parseAccountStatusesArguments( args.StartBlockID = startBlockID.Flow() } - // Parse 'start_block_height' if provided // Parse 'start_block_height' if provided if hasStartBlockHeight { result, ok := startBlockHeightIn.(string) if !ok { return args, fmt.Errorf("'start_block_height' must be a string") } - startBlockHeight, err := util.ToUint64(result) + var err error + args.StartBlockHeight, err = util.ToUint64(result) if err != nil { return args, fmt.Errorf("invalid 'start_block_height': %w", err) } - args.StartBlockHeight = startBlockHeight } else { args.StartBlockHeight = request.EmptyHeight } diff --git a/engine/access/rest/websockets/data_providers/blocks_provider.go b/engine/access/rest/websockets/data_providers/blocks_provider.go index 28e0a9a03d2..5b49f20870d 100644 --- a/engine/access/rest/websockets/data_providers/blocks_provider.go +++ b/engine/access/rest/websockets/data_providers/blocks_provider.go @@ -135,11 +135,11 @@ func ParseBlocksArguments(arguments models.Arguments) (BlocksArguments, error) { if !ok { return args, fmt.Errorf("'start_block_height' must be a string") } - startBlockHeight, err := util.ToUint64(result) + var err error + args.StartBlockHeight, err = util.ToUint64(result) if err != nil { return args, fmt.Errorf("invalid 'start_block_height': %w", err) } - args.StartBlockHeight = startBlockHeight } else { // Default value if 'start_block_height' is not provided args.StartBlockHeight = request.EmptyHeight diff --git a/engine/access/rest/websockets/data_providers/events_provider.go b/engine/access/rest/websockets/data_providers/events_provider.go index 6b62f45ffac..f3b7bfe3668 100644 --- a/engine/access/rest/websockets/data_providers/events_provider.go +++ b/engine/access/rest/websockets/data_providers/events_provider.go @@ -164,11 +164,11 @@ func parseEventsArguments( if !ok { return args, fmt.Errorf("'start_block_height' must be a string") } - startBlockHeight, err := util.ToUint64(result) + var err error + args.StartBlockHeight, err = util.ToUint64(result) if err != nil { return args, fmt.Errorf("invalid 'start_block_height': %w", err) } - args.StartBlockHeight = startBlockHeight } else { args.StartBlockHeight = request.EmptyHeight } From 5c9f16cd259c70ce0dc7cfca5f62dc306ad80257 Mon Sep 17 00:00:00 2001 From: Andrii Date: Fri, 13 Dec 2024 11:56:25 +0200 Subject: [PATCH 48/67] Refactored args filter init --- .../websockets/data_providers/account_statuses_provider.go | 5 ++--- .../access/rest/websockets/data_providers/events_provider.go | 5 ++--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/engine/access/rest/websockets/data_providers/account_statuses_provider.go b/engine/access/rest/websockets/data_providers/account_statuses_provider.go index 8b5a5114a6f..9851ad9eadc 100644 --- a/engine/access/rest/websockets/data_providers/account_statuses_provider.go +++ b/engine/access/rest/websockets/data_providers/account_statuses_provider.go @@ -159,12 +159,12 @@ func parseAccountStatusesArguments( } // Parse 'start_block_height' if provided + var err error if hasStartBlockHeight { result, ok := startBlockHeightIn.(string) if !ok { return args, fmt.Errorf("'start_block_height' must be a string") } - var err error args.StartBlockHeight, err = util.ToUint64(result) if err != nil { return args, fmt.Errorf("invalid 'start_block_height': %w", err) @@ -197,11 +197,10 @@ func parseAccountStatusesArguments( } // Initialize the event filter with the parsed arguments - filter, err := state_stream.NewAccountStatusFilter(eventFilterConfig, chain, eventTypes.Flow(), accountAddresses) + args.Filter, err = state_stream.NewAccountStatusFilter(eventFilterConfig, chain, eventTypes.Flow(), accountAddresses) if err != nil { return args, fmt.Errorf("failed to create event filter: %w", err) } - args.Filter = filter return args, nil } diff --git a/engine/access/rest/websockets/data_providers/events_provider.go b/engine/access/rest/websockets/data_providers/events_provider.go index f3b7bfe3668..90cbe7df93a 100644 --- a/engine/access/rest/websockets/data_providers/events_provider.go +++ b/engine/access/rest/websockets/data_providers/events_provider.go @@ -159,12 +159,12 @@ func parseEventsArguments( } // Parse 'start_block_height' if provided + var err error if hasStartBlockHeight { result, ok := startBlockHeightIn.(string) if !ok { return args, fmt.Errorf("'start_block_height' must be a string") } - var err error args.StartBlockHeight, err = util.ToUint64(result) if err != nil { return args, fmt.Errorf("invalid 'start_block_height': %w", err) @@ -206,11 +206,10 @@ func parseEventsArguments( } // Initialize the event filter with the parsed arguments - filter, err := state_stream.NewEventFilter(eventFilterConfig, chain, eventTypes.Flow(), addresses, contracts) + args.Filter, err = state_stream.NewEventFilter(eventFilterConfig, chain, eventTypes.Flow(), addresses, contracts) if err != nil { return args, fmt.Errorf("failed to create event filter: %w", err) } - args.Filter = filter return args, nil } From c002841c0efac61b751acdff21d04633c3720d16 Mon Sep 17 00:00:00 2001 From: Andrii Date: Fri, 13 Dec 2024 12:12:03 +0200 Subject: [PATCH 49/67] Created separate file for generic testHappyPath method and for testType struct --- .../account_statuses_provider.go | 9 +- .../account_statuses_provider_test.go | 2 +- .../data_providers/block_digests_provider.go | 2 +- .../data_providers/block_headers_provider.go | 2 +- .../data_providers/blocks_provider.go | 8 +- .../data_providers/blocks_provider_test.go | 151 +++++++++--------- .../data_providers/events_provider.go | 10 +- .../data_providers/events_provider_test.go | 2 +- .../rest/websockets/data_providers/util.go | 95 +++++++++++ 9 files changed, 185 insertions(+), 96 deletions(-) create mode 100644 engine/access/rest/websockets/data_providers/util.go diff --git a/engine/access/rest/websockets/data_providers/account_statuses_provider.go b/engine/access/rest/websockets/data_providers/account_statuses_provider.go index 9851ad9eadc..94e984ae32c 100644 --- a/engine/access/rest/websockets/data_providers/account_statuses_provider.go +++ b/engine/access/rest/websockets/data_providers/account_statuses_provider.go @@ -20,7 +20,8 @@ import ( "github.com/onflow/flow-go/module/counters" ) -type AccountStatusesArguments struct { +// accountStatusesArguments contains the arguments required for subscribing to account statuses +type accountStatusesArguments struct { StartBlockID flow.Identifier // ID of the block to start subscription from StartBlockHeight uint64 // Height of the block to start subscription from Filter state_stream.AccountStatusFilter // Filter applied to events for a given subscription @@ -81,7 +82,7 @@ func (p *AccountStatusesDataProvider) Run() error { } // createSubscription creates a new subscription using the specified input arguments. -func (p *AccountStatusesDataProvider) createSubscription(ctx context.Context, args AccountStatusesArguments) subscription.Subscription { +func (p *AccountStatusesDataProvider) createSubscription(ctx context.Context, args accountStatusesArguments) subscription.Subscription { if args.StartBlockID != flow.ZeroID { return p.stateStreamApi.SubscribeAccountStatusesFromStartBlockID(ctx, args.StartBlockID, args.Filter) } @@ -132,8 +133,8 @@ func parseAccountStatusesArguments( arguments models.Arguments, chain flow.Chain, eventFilterConfig state_stream.EventFilterConfig, -) (AccountStatusesArguments, error) { - var args AccountStatusesArguments +) (accountStatusesArguments, error) { + var args accountStatusesArguments //var err error // Check for mutual exclusivity of start_block_id and start_block_height early diff --git a/engine/access/rest/websockets/data_providers/account_statuses_provider_test.go b/engine/access/rest/websockets/data_providers/account_statuses_provider_test.go index 3c54f2a7e15..1bef0e1fad0 100644 --- a/engine/access/rest/websockets/data_providers/account_statuses_provider_test.go +++ b/engine/access/rest/websockets/data_providers/account_statuses_provider_test.go @@ -50,7 +50,7 @@ func (s *AccountStatusesProviderSuite) SetupTest() { s.log, s.api, nil, - flow.Testnet.Chain(), + s.chain, state_stream.DefaultEventFilterConfig, subscription.DefaultHeartbeatInterval) s.Require().NotNil(s.factory) diff --git a/engine/access/rest/websockets/data_providers/block_digests_provider.go b/engine/access/rest/websockets/data_providers/block_digests_provider.go index 1fa3f7a6dc7..80307be6b64 100644 --- a/engine/access/rest/websockets/data_providers/block_digests_provider.go +++ b/engine/access/rest/websockets/data_providers/block_digests_provider.go @@ -69,7 +69,7 @@ func (p *BlockDigestsDataProvider) Run() error { } // createSubscription creates a new subscription using the specified input arguments. -func (p *BlockDigestsDataProvider) createSubscription(ctx context.Context, args BlocksArguments) subscription.Subscription { +func (p *BlockDigestsDataProvider) createSubscription(ctx context.Context, args blocksArguments) subscription.Subscription { if args.StartBlockID != flow.ZeroID { return p.api.SubscribeBlockDigestsFromStartBlockID(ctx, args.StartBlockID, args.BlockStatus) } diff --git a/engine/access/rest/websockets/data_providers/block_headers_provider.go b/engine/access/rest/websockets/data_providers/block_headers_provider.go index 4f9e29e2428..4fddeb499f2 100644 --- a/engine/access/rest/websockets/data_providers/block_headers_provider.go +++ b/engine/access/rest/websockets/data_providers/block_headers_provider.go @@ -69,7 +69,7 @@ func (p *BlockHeadersDataProvider) Run() error { } // createSubscription creates a new subscription using the specified input arguments. -func (p *BlockHeadersDataProvider) createSubscription(ctx context.Context, args BlocksArguments) subscription.Subscription { +func (p *BlockHeadersDataProvider) createSubscription(ctx context.Context, args blocksArguments) subscription.Subscription { if args.StartBlockID != flow.ZeroID { return p.api.SubscribeBlockHeadersFromStartBlockID(ctx, args.StartBlockID, args.BlockStatus) } diff --git a/engine/access/rest/websockets/data_providers/blocks_provider.go b/engine/access/rest/websockets/data_providers/blocks_provider.go index 5b49f20870d..89791397b5b 100644 --- a/engine/access/rest/websockets/data_providers/blocks_provider.go +++ b/engine/access/rest/websockets/data_providers/blocks_provider.go @@ -16,7 +16,7 @@ import ( ) // BlocksArguments contains the arguments required for subscribing to blocks / block headers / block digests -type BlocksArguments struct { +type blocksArguments struct { StartBlockID flow.Identifier // ID of the block to start subscription from StartBlockHeight uint64 // Height of the block to start subscription from BlockStatus flow.BlockStatus // Status of blocks to subscribe to @@ -78,7 +78,7 @@ func (p *BlocksDataProvider) Run() error { } // createSubscription creates a new subscription using the specified input arguments. -func (p *BlocksDataProvider) createSubscription(ctx context.Context, args BlocksArguments) subscription.Subscription { +func (p *BlocksDataProvider) createSubscription(ctx context.Context, args blocksArguments) subscription.Subscription { if args.StartBlockID != flow.ZeroID { return p.api.SubscribeBlocksFromStartBlockID(ctx, args.StartBlockID, args.BlockStatus) } @@ -91,8 +91,8 @@ func (p *BlocksDataProvider) createSubscription(ctx context.Context, args Blocks } // ParseBlocksArguments validates and initializes the blocks arguments. -func ParseBlocksArguments(arguments models.Arguments) (BlocksArguments, error) { - var args BlocksArguments +func ParseBlocksArguments(arguments models.Arguments) (blocksArguments, error) { + var args blocksArguments // Parse 'block_status' if blockStatusIn, ok := arguments["block_status"]; ok { diff --git a/engine/access/rest/websockets/data_providers/blocks_provider_test.go b/engine/access/rest/websockets/data_providers/blocks_provider_test.go index cf3f6997d3c..f029b39667d 100644 --- a/engine/access/rest/websockets/data_providers/blocks_provider_test.go +++ b/engine/access/rest/websockets/data_providers/blocks_provider_test.go @@ -5,7 +5,6 @@ import ( "fmt" "strconv" "testing" - "time" "github.com/rs/zerolog" "github.com/stretchr/testify/mock" @@ -24,18 +23,12 @@ import ( const unknownBlockStatus = "unknown_block_status" -type testErrType struct { - name string - arguments models.Arguments - expectedErrorMsg string -} - -// testType represents a valid test scenario for subscribing -type testType struct { - name string - arguments models.Arguments - setupBackend func(sub *statestreamsmock.Subscription) -} +//// testType represents a valid test scenario for subscribing +//type testType struct { +// name string +// arguments models.Arguments +// setupBackend func(sub *statestreamsmock.Subscription) +//} // BlocksProviderSuite is a test suite for testing the block providers functionality. type BlocksProviderSuite struct { @@ -223,70 +216,70 @@ func (s *BlocksProviderSuite) requireBlock(v interface{}, expected interface{}) s.Require().Equal(expectedBlock, actualResponse.Block) } -// TestHappyPath tests a variety of scenarios for data providers in -// happy path scenarios. This function runs parameterized test cases that -// simulate various configurations and verifies that the data provider operates -// as expected without encountering errors. +//// testHappyPath tests a variety of scenarios for data providers in +//// happy path scenarios. This function runs parameterized test cases that +//// simulate various configurations and verifies that the data provider operates +//// as expected without encountering errors. +//// +//// Arguments: +//// - topic: The topic associated with the data provider. +//// - factory: A factory for creating data provider instance. +//// - tests: A slice of test cases to run, each specifying setup and validation logic. +//// - sendData: A function to simulate emitting data into the subscription's data channel. +//// - expectedResponses: An expected responses to validate the received output. +//// - requireFn: A function to validate the output received in the send channel. +//func testHappyPath[T any]( +// t *testing.T, +// topic string, +// factory *DataProviderFactoryImpl, +// tests []testType, +// sendData func(chan interface{}), +// expectedResponses []T, +// requireFn func(interface{}, interface{}), +//) { +// for _, test := range tests { +// t.Run(test.name, func(t *testing.T) { +// ctx := context.Background() +// send := make(chan interface{}, 10) // -// Arguments: -// - topic: The topic associated with the data provider. -// - factory: A factory for creating data provider instance. -// - tests: A slice of test cases to run, each specifying setup and validation logic. -// - sendData: A function to simulate emitting data into the subscription's data channel. -// - expectedResponses: An expected responses to validate the received output. -// - requireFn: A function to validate the output received in the send channel. -func testHappyPath[T any]( - t *testing.T, - topic string, - factory *DataProviderFactoryImpl, - tests []testType, - sendData func(chan interface{}), - expectedResponses []T, - requireFn func(interface{}, interface{}), -) { - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - ctx := context.Background() - send := make(chan interface{}, 10) - - // Create a channel to simulate the subscription's data channel - dataChan := make(chan interface{}) - - // Create a mock subscription and mock the channel - sub := statestreamsmock.NewSubscription(t) - sub.On("Channel").Return((<-chan interface{})(dataChan)) - sub.On("Err").Return(nil) - test.setupBackend(sub) - - // Create the data provider instance - provider, err := factory.NewDataProvider(ctx, topic, test.arguments, send) - require.NotNil(t, provider) - require.NoError(t, err) - - // Run the provider in a separate goroutine - go func() { - err = provider.Run() - require.NoError(t, err) - }() - - // Simulate emitting data to the data channel - go func() { - defer close(dataChan) - sendData(dataChan) - }() - - // Collect responses - for i, expected := range expectedResponses { - unittest.RequireReturnsBefore(t, func() { - v, ok := <-send - require.True(t, ok, "channel closed while waiting for response %v: err: %v", expected, sub.Err()) - - requireFn(v, expected) - }, time.Second, fmt.Sprintf("timed out waiting for response %d %v", i, expected)) - } - - // Ensure the provider is properly closed after the test - provider.Close() - }) - } -} +// // Create a channel to simulate the subscription's data channel +// dataChan := make(chan interface{}) +// +// // Create a mock subscription and mock the channel +// sub := statestreamsmock.NewSubscription(t) +// sub.On("Channel").Return((<-chan interface{})(dataChan)) +// sub.On("Err").Return(nil) +// test.setupBackend(sub) +// +// // Create the data provider instance +// provider, err := factory.NewDataProvider(ctx, topic, test.arguments, send) +// require.NotNil(t, provider) +// require.NoError(t, err) +// +// // Run the provider in a separate goroutine +// go func() { +// err = provider.Run() +// require.NoError(t, err) +// }() +// +// // Simulate emitting data to the data channel +// go func() { +// defer close(dataChan) +// sendData(dataChan) +// }() +// +// // Collect responses +// for i, expected := range expectedResponses { +// unittest.RequireReturnsBefore(t, func() { +// v, ok := <-send +// require.True(t, ok, "channel closed while waiting for response %v: err: %v", expected, sub.Err()) +// +// requireFn(v, expected) +// }, time.Second, fmt.Sprintf("timed out waiting for response %d %v", i, expected)) +// } +// +// // Ensure the provider is properly closed after the test +// provider.Close() +// }) +// } +//} diff --git a/engine/access/rest/websockets/data_providers/events_provider.go b/engine/access/rest/websockets/data_providers/events_provider.go index 90cbe7df93a..b6854c93b4d 100644 --- a/engine/access/rest/websockets/data_providers/events_provider.go +++ b/engine/access/rest/websockets/data_providers/events_provider.go @@ -18,8 +18,8 @@ import ( "github.com/onflow/flow-go/module/counters" ) -// EventsArguments contains the arguments required for subscribing to events -type EventsArguments struct { +// eventsArguments contains the arguments required for subscribing to events +type eventsArguments struct { StartBlockID flow.Identifier // ID of the block to start subscription from StartBlockHeight uint64 // Height of the block to start subscription from Filter state_stream.EventFilter // Filter applied to events for a given subscription @@ -116,7 +116,7 @@ func (p *EventsDataProvider) handleResponse() func(eventsResponse *backend.Event } // createSubscription creates a new subscription using the specified input arguments. -func (p *EventsDataProvider) createSubscription(ctx context.Context, args EventsArguments) subscription.Subscription { +func (p *EventsDataProvider) createSubscription(ctx context.Context, args eventsArguments) subscription.Subscription { if args.StartBlockID != flow.ZeroID { return p.stateStreamApi.SubscribeEventsFromStartBlockID(ctx, args.StartBlockID, args.Filter) } @@ -133,8 +133,8 @@ func parseEventsArguments( arguments models.Arguments, chain flow.Chain, eventFilterConfig state_stream.EventFilterConfig, -) (EventsArguments, error) { - var args EventsArguments +) (eventsArguments, error) { + var args eventsArguments // Check for mutual exclusivity of start_block_id and start_block_height early startBlockIDIn, hasStartBlockID := arguments["start_block_id"] diff --git a/engine/access/rest/websockets/data_providers/events_provider_test.go b/engine/access/rest/websockets/data_providers/events_provider_test.go index 6bbe1f36a44..214781f8885 100644 --- a/engine/access/rest/websockets/data_providers/events_provider_test.go +++ b/engine/access/rest/websockets/data_providers/events_provider_test.go @@ -51,7 +51,7 @@ func (s *EventsProviderSuite) SetupTest() { s.log, s.api, nil, - flow.Testnet.Chain(), + s.chain, state_stream.DefaultEventFilterConfig, subscription.DefaultHeartbeatInterval) s.Require().NotNil(s.factory) diff --git a/engine/access/rest/websockets/data_providers/util.go b/engine/access/rest/websockets/data_providers/util.go new file mode 100644 index 00000000000..8ddcd54f840 --- /dev/null +++ b/engine/access/rest/websockets/data_providers/util.go @@ -0,0 +1,95 @@ +package data_providers + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/onflow/flow-go/engine/access/rest/websockets/models" + statestreamsmock "github.com/onflow/flow-go/engine/access/state_stream/mock" + "github.com/onflow/flow-go/utils/unittest" +) + +// testType represents a valid test scenario for subscribing +type testType struct { + name string + arguments models.Arguments + setupBackend func(sub *statestreamsmock.Subscription) +} + +type testErrType struct { + name string + arguments models.Arguments + expectedErrorMsg string +} + +// testHappyPath tests a variety of scenarios for data providers in +// happy path scenarios. This function runs parameterized test cases that +// simulate various configurations and verifies that the data provider operates +// as expected without encountering errors. +// +// Arguments: +// - topic: The topic associated with the data provider. +// - factory: A factory for creating data provider instance. +// - tests: A slice of test cases to run, each specifying setup and validation logic. +// - sendData: A function to simulate emitting data into the subscription's data channel. +// - expectedResponses: An expected responses to validate the received output. +// - requireFn: A function to validate the output received in the send channel. +func testHappyPath[T any]( + t *testing.T, + topic string, + factory *DataProviderFactoryImpl, + tests []testType, + sendData func(chan interface{}), + expectedResponses []T, + requireFn func(interface{}, interface{}), +) { + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + ctx := context.Background() + send := make(chan interface{}, 10) + + // Create a channel to simulate the subscription's data channel + dataChan := make(chan interface{}) + + // Create a mock subscription and mock the channel + sub := statestreamsmock.NewSubscription(t) + sub.On("Channel").Return((<-chan interface{})(dataChan)) + sub.On("Err").Return(nil) + test.setupBackend(sub) + + // Create the data provider instance + provider, err := factory.NewDataProvider(ctx, topic, test.arguments, send) + require.NotNil(t, provider) + require.NoError(t, err) + + // Run the provider in a separate goroutine + go func() { + err = provider.Run() + require.NoError(t, err) + }() + + // Simulate emitting data to the data channel + go func() { + defer close(dataChan) + sendData(dataChan) + }() + + // Collect responses + for i, expected := range expectedResponses { + unittest.RequireReturnsBefore(t, func() { + v, ok := <-send + require.True(t, ok, "channel closed while waiting for response %v: err: %v", expected, sub.Err()) + + requireFn(v, expected) + }, time.Second, fmt.Sprintf("timed out waiting for response %d %v", i, expected)) + } + + // Ensure the provider is properly closed after the test + provider.Close() + }) + } +} From 9ed92071d5822c83879b244e7d9bba597e0f7c71 Mon Sep 17 00:00:00 2001 From: Andrii Date: Fri, 13 Dec 2024 13:26:11 +0200 Subject: [PATCH 50/67] Changed parse args function, added invalid args testcases --- .../account_statuses_provider.go | 1 - .../transaction_statuses_provider.go | 54 +++++++++++++++---- .../transaction_statuses_provider_test.go | 51 +++++++++++------- 3 files changed, 77 insertions(+), 29 deletions(-) diff --git a/engine/access/rest/websockets/data_providers/account_statuses_provider.go b/engine/access/rest/websockets/data_providers/account_statuses_provider.go index 94e984ae32c..09fca79bd70 100644 --- a/engine/access/rest/websockets/data_providers/account_statuses_provider.go +++ b/engine/access/rest/websockets/data_providers/account_statuses_provider.go @@ -135,7 +135,6 @@ func parseAccountStatusesArguments( eventFilterConfig state_stream.EventFilterConfig, ) (accountStatusesArguments, error) { var args accountStatusesArguments - //var err error // Check for mutual exclusivity of start_block_id and start_block_height early startBlockIDIn, hasStartBlockID := arguments["start_block_id"] diff --git a/engine/access/rest/websockets/data_providers/transaction_statuses_provider.go b/engine/access/rest/websockets/data_providers/transaction_statuses_provider.go index 6a446282d5e..e2b151b64f0 100644 --- a/engine/access/rest/websockets/data_providers/transaction_statuses_provider.go +++ b/engine/access/rest/websockets/data_providers/transaction_statuses_provider.go @@ -3,6 +3,8 @@ package data_providers import ( "context" "fmt" + "github.com/onflow/flow-go/engine/access/rest/http/request" + "github.com/onflow/flow-go/engine/access/rest/util" "strconv" "github.com/rs/zerolog" @@ -18,9 +20,11 @@ import ( "github.com/onflow/flow/protobuf/go/flow/entities" ) -type TransactionStatusesArguments struct { - StartBlockID flow.Identifier // ID of the block to start subscription from - txID flow.Identifier // ID of the transaction to monitor. +// transactionStatusesArguments contains the arguments required for subscribing to transaction statuses +type transactionStatusesArguments struct { + TxID flow.Identifier // ID of the transaction to monitor. + StartBlockID flow.Identifier // ID of the block to start subscription from + StartBlockHeight uint64 // Height of the block to start subscription from } type TransactionStatusesDataProvider struct { @@ -73,9 +77,17 @@ func (p *TransactionStatusesDataProvider) Run() error { // createSubscription creates a new subscription using the specified input arguments. func (p *TransactionStatusesDataProvider) createSubscription( ctx context.Context, - args TransactionStatusesArguments, + args transactionStatusesArguments, ) subscription.Subscription { - return p.api.SubscribeTransactionStatuses(ctx, args.txID, args.StartBlockID, entities.EventEncodingVersion_JSON_CDC_V0) + if args.StartBlockID != flow.ZeroID { + return p.api.SubscribeTransactionStatusesFromStartBlockID(ctx, args.TxID, args.StartBlockID, entities.EventEncodingVersion_JSON_CDC_V0) + } + + if args.StartBlockHeight != request.EmptyHeight { + return p.api.SubscribeTransactionStatusesFromStartHeight(ctx, args.TxID, args.StartBlockHeight, entities.EventEncodingVersion_JSON_CDC_V0) + } + + return p.api.SubscribeTransactionStatusesFromLatest(ctx, args.TxID, entities.EventEncodingVersion_JSON_CDC_V0) } // handleResponse processes an account statuses and sends the formatted response. @@ -103,8 +115,16 @@ func (p *TransactionStatusesDataProvider) handleResponse() func(txResults []*acc // parseAccountStatusesArguments validates and initializes the account statuses arguments. func parseTransactionStatusesArguments( arguments models.Arguments, -) (TransactionStatusesArguments, error) { - var args TransactionStatusesArguments +) (transactionStatusesArguments, error) { + var args transactionStatusesArguments + + // Check for mutual exclusivity of start_block_id and start_block_height early + startBlockIDIn, hasStartBlockID := arguments["start_block_id"] + startBlockHeightIn, hasStartBlockHeight := arguments["start_block_height"] + + if hasStartBlockID && hasStartBlockHeight { + return args, fmt.Errorf("can only provide either 'start_block_id' or 'start_block_height'") + } if txIDIn, ok := arguments["tx_id"]; ok && txIDIn != "" { result, ok := txIDIn.(string) @@ -116,10 +136,11 @@ func parseTransactionStatusesArguments( if err != nil { return args, fmt.Errorf("invalid 'tx_id': %w", err) } - args.txID = txID.Flow() + args.TxID = txID.Flow() } - if startBlockIDIn, ok := arguments["start_block_id"]; ok && startBlockIDIn != "" { + // Parse 'start_block_id' if provided + if hasStartBlockID { result, ok := startBlockIDIn.(string) if !ok { return args, fmt.Errorf("'start_block_id' must be a string") @@ -132,5 +153,20 @@ func parseTransactionStatusesArguments( args.StartBlockID = startBlockID.Flow() } + // Parse 'start_block_height' if provided + var err error + if hasStartBlockHeight { + result, ok := startBlockHeightIn.(string) + if !ok { + return args, fmt.Errorf("'start_block_height' must be a string") + } + args.StartBlockHeight, err = util.ToUint64(result) + if err != nil { + return args, fmt.Errorf("invalid 'start_block_height': %w", err) + } + } else { + args.StartBlockHeight = request.EmptyHeight + } + return args, nil } diff --git a/engine/access/rest/websockets/data_providers/transaction_statuses_provider_test.go b/engine/access/rest/websockets/data_providers/transaction_statuses_provider_test.go index 3b0b486db4f..153fb67da83 100644 --- a/engine/access/rest/websockets/data_providers/transaction_statuses_provider_test.go +++ b/engine/access/rest/websockets/data_providers/transaction_statuses_provider_test.go @@ -2,6 +2,7 @@ package data_providers import ( "context" + "fmt" "testing" "github.com/rs/zerolog" @@ -11,12 +12,9 @@ import ( accessmock "github.com/onflow/flow-go/access/mock" "github.com/onflow/flow-go/engine/access/rest/websockets/models" "github.com/onflow/flow-go/engine/access/state_stream" - ssmock "github.com/onflow/flow-go/engine/access/state_stream/mock" "github.com/onflow/flow-go/engine/access/subscription" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/utils/unittest" - "github.com/onflow/flow/protobuf/go/flow/entities" - "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" ) @@ -76,22 +74,22 @@ func (s *TransactionStatusesProviderSuite) TestTransactionStatusesDataProvider_H func (s *TransactionStatusesProviderSuite) subscribeTransactionStatusesDataProviderTestCases() []testType { return []testType{ - { - name: "SubscribeTransactionStatuses happy path", - arguments: models.Arguments{ - "start_block_id": s.rootBlock.ID().String(), - "event_types": []string{"flow.AccountCreated", "flow.AccountKeyAdded"}, - }, - setupBackend: func(sub *ssmock.Subscription) { - s.api.On( - "SubscribeTransactionStatuses", - mock.Anything, - mock.Anything, - s.rootBlock.ID(), - entities.EventEncodingVersion_JSON_CDC_V0, - ).Return(sub).Once() - }, - }, + //{ + // name: "SubscribeTransactionStatuses happy path", + // arguments: models.Arguments{ + // "start_block_id": s.rootBlock.ID().String(), + // "event_types": []string{"flow.AccountCreated", "flow.AccountKeyAdded"}, + // }, + // setupBackend: func(sub *ssmock.Subscription) { + // s.api.On( + // "SubscribeTransactionStatuses", + // mock.Anything, + // mock.Anything, + // s.rootBlock.ID(), + // entities.EventEncodingVersion_JSON_CDC_V0, + // ).Return(sub).Once() + // }, + //}, } } @@ -143,6 +141,14 @@ func (s *TransactionStatusesProviderSuite) TestAccountStatusesDataProvider_Inval // 2. Providing invalid 'start_block_id' value. func invalidTransactionStatusesArgumentsTestCases() []testErrType { return []testErrType{ + { + name: "provide both 'start_block_id' and 'start_block_height' arguments", + arguments: models.Arguments{ + "start_block_id": unittest.BlockFixture().ID().String(), + "start_block_height": fmt.Sprintf("%d", unittest.BlockFixture().Header.Height), + }, + expectedErrorMsg: "can only provide either 'start_block_id' or 'start_block_height'", + }, { name: "invalid 'tx_id' argument", arguments: map[string]interface{}{ @@ -157,5 +163,12 @@ func invalidTransactionStatusesArgumentsTestCases() []testErrType { }, expectedErrorMsg: "invalid ID format", }, + { + name: "invalid 'start_block_height' argument", + arguments: map[string]interface{}{ + "start_block_height": "-1", + }, + expectedErrorMsg: "value must be an unsigned 64 bit integer", + }, } } From d660c7fe3b0522d9d40c7ec6f01a7cb2a1ce5c6e Mon Sep 17 00:00:00 2001 From: Andrii Date: Fri, 13 Dec 2024 13:33:22 +0200 Subject: [PATCH 51/67] Linted --- .../data_providers/transaction_statuses_provider.go | 5 +++-- .../data_providers/transaction_statuses_provider_test.go | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/engine/access/rest/websockets/data_providers/transaction_statuses_provider.go b/engine/access/rest/websockets/data_providers/transaction_statuses_provider.go index e2b151b64f0..1562f738b80 100644 --- a/engine/access/rest/websockets/data_providers/transaction_statuses_provider.go +++ b/engine/access/rest/websockets/data_providers/transaction_statuses_provider.go @@ -3,8 +3,6 @@ package data_providers import ( "context" "fmt" - "github.com/onflow/flow-go/engine/access/rest/http/request" - "github.com/onflow/flow-go/engine/access/rest/util" "strconv" "github.com/rs/zerolog" @@ -13,10 +11,13 @@ import ( "github.com/onflow/flow-go/access" "github.com/onflow/flow-go/engine/access/rest/common/parser" + "github.com/onflow/flow-go/engine/access/rest/http/request" + "github.com/onflow/flow-go/engine/access/rest/util" "github.com/onflow/flow-go/engine/access/rest/websockets/models" "github.com/onflow/flow-go/engine/access/subscription" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module/counters" + "github.com/onflow/flow/protobuf/go/flow/entities" ) diff --git a/engine/access/rest/websockets/data_providers/transaction_statuses_provider_test.go b/engine/access/rest/websockets/data_providers/transaction_statuses_provider_test.go index 153fb67da83..4e2c18e81f9 100644 --- a/engine/access/rest/websockets/data_providers/transaction_statuses_provider_test.go +++ b/engine/access/rest/websockets/data_providers/transaction_statuses_provider_test.go @@ -6,6 +6,7 @@ import ( "testing" "github.com/rs/zerolog" + "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" "github.com/onflow/flow-go/access" @@ -15,7 +16,6 @@ import ( "github.com/onflow/flow-go/engine/access/subscription" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/utils/unittest" - "github.com/stretchr/testify/require" ) type TransactionStatusesProviderSuite struct { From 3126cd9c2e64eaeae44a86a2a179bbb25a0827f5 Mon Sep 17 00:00:00 2001 From: Andrii Date: Fri, 13 Dec 2024 13:40:28 +0200 Subject: [PATCH 52/67] Changed api in test to mock api --- .../websockets/data_providers/factory_test.go | 2 +- .../transaction_statuses_provider_test.go | 64 ++++++++++++++----- 2 files changed, 48 insertions(+), 18 deletions(-) diff --git a/engine/access/rest/websockets/data_providers/factory_test.go b/engine/access/rest/websockets/data_providers/factory_test.go index 674fc6998ea..33389c3997e 100644 --- a/engine/access/rest/websockets/data_providers/factory_test.go +++ b/engine/access/rest/websockets/data_providers/factory_test.go @@ -137,7 +137,7 @@ func (s *DataProviderFactorySuite) TestSupportedTopics() { topic: TransactionStatusesTopic, arguments: models.Arguments{}, setupSubscription: func() { - s.setupSubscription(s.accessApi.On("SubscribeTransactionStatuses", mock.Anything, mock.Anything, mock.Anything, mock.Anything)) + s.setupSubscription(s.accessApi.On("SubscribeTransactionStatusesFromLatest", mock.Anything, mock.Anything, mock.Anything)) }, assertExpectations: func() { s.stateStreamApi.AssertExpectations(s.T()) diff --git a/engine/access/rest/websockets/data_providers/transaction_statuses_provider_test.go b/engine/access/rest/websockets/data_providers/transaction_statuses_provider_test.go index 4e2c18e81f9..c57a7dff685 100644 --- a/engine/access/rest/websockets/data_providers/transaction_statuses_provider_test.go +++ b/engine/access/rest/websockets/data_providers/transaction_statuses_provider_test.go @@ -3,9 +3,12 @@ package data_providers import ( "context" "fmt" + "github.com/onflow/flow/protobuf/go/flow/entities" + "strconv" "testing" "github.com/rs/zerolog" + "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" @@ -13,6 +16,7 @@ import ( accessmock "github.com/onflow/flow-go/access/mock" "github.com/onflow/flow-go/engine/access/rest/websockets/models" "github.com/onflow/flow-go/engine/access/state_stream" + ssmock "github.com/onflow/flow-go/engine/access/state_stream/mock" "github.com/onflow/flow-go/engine/access/subscription" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/utils/unittest" @@ -22,7 +26,7 @@ type TransactionStatusesProviderSuite struct { suite.Suite log zerolog.Logger - api access.API + api *accessmock.API chain flow.Chain rootBlock flow.Block @@ -74,22 +78,48 @@ func (s *TransactionStatusesProviderSuite) TestTransactionStatusesDataProvider_H func (s *TransactionStatusesProviderSuite) subscribeTransactionStatusesDataProviderTestCases() []testType { return []testType{ - //{ - // name: "SubscribeTransactionStatuses happy path", - // arguments: models.Arguments{ - // "start_block_id": s.rootBlock.ID().String(), - // "event_types": []string{"flow.AccountCreated", "flow.AccountKeyAdded"}, - // }, - // setupBackend: func(sub *ssmock.Subscription) { - // s.api.On( - // "SubscribeTransactionStatuses", - // mock.Anything, - // mock.Anything, - // s.rootBlock.ID(), - // entities.EventEncodingVersion_JSON_CDC_V0, - // ).Return(sub).Once() - // }, - //}, + { + name: "SubscribeAccountStatusesFromStartBlockID happy path", + arguments: models.Arguments{ + "start_block_id": s.rootBlock.ID().String(), + "event_types": []string{"flow.AccountCreated", "flow.AccountKeyAdded"}, + }, + setupBackend: func(sub *ssmock.Subscription) { + s.api.On( + "SubscribeTransactionStatusesFromStartBlockID", + mock.Anything, + s.rootBlock.ID(), + mock.Anything, + entities.EventEncodingVersion_JSON_CDC_V0, + ).Return(sub).Once() + }, + }, + { + name: "SubscribeAccountStatusesFromStartHeight happy path", + arguments: models.Arguments{ + "start_block_height": strconv.FormatUint(s.rootBlock.Header.Height, 10), + }, + setupBackend: func(sub *ssmock.Subscription) { + s.api.On( + "SubscribeTransactionStatusesFromStartHeight", + mock.Anything, + s.rootBlock.Header.Height, + mock.Anything, + entities.EventEncodingVersion_JSON_CDC_V0, + ).Return(sub).Once() + }, + }, + { + name: "SubscribeAccountStatusesFromLatestBlock happy path", + arguments: models.Arguments{}, + setupBackend: func(sub *ssmock.Subscription) { + s.api.On( + "SubscribeTransactionStatusesFromLatest", + mock.Anything, + entities.EventEncodingVersion_JSON_CDC_V0, + ).Return(sub).Once() + }, + }, } } From 2f0142d6e8c42fe01bb148bc9fea3e158dbecd1e Mon Sep 17 00:00:00 2001 From: Andrii Date: Fri, 13 Dec 2024 13:41:29 +0200 Subject: [PATCH 53/67] Removed commented code --- .../account_statuses_provider.go | 1 - .../data_providers/blocks_provider_test.go | 75 ------------------- 2 files changed, 76 deletions(-) diff --git a/engine/access/rest/websockets/data_providers/account_statuses_provider.go b/engine/access/rest/websockets/data_providers/account_statuses_provider.go index 94e984ae32c..09fca79bd70 100644 --- a/engine/access/rest/websockets/data_providers/account_statuses_provider.go +++ b/engine/access/rest/websockets/data_providers/account_statuses_provider.go @@ -135,7 +135,6 @@ func parseAccountStatusesArguments( eventFilterConfig state_stream.EventFilterConfig, ) (accountStatusesArguments, error) { var args accountStatusesArguments - //var err error // Check for mutual exclusivity of start_block_id and start_block_height early startBlockIDIn, hasStartBlockID := arguments["start_block_id"] diff --git a/engine/access/rest/websockets/data_providers/blocks_provider_test.go b/engine/access/rest/websockets/data_providers/blocks_provider_test.go index f029b39667d..85136ae5819 100644 --- a/engine/access/rest/websockets/data_providers/blocks_provider_test.go +++ b/engine/access/rest/websockets/data_providers/blocks_provider_test.go @@ -23,13 +23,6 @@ import ( const unknownBlockStatus = "unknown_block_status" -//// testType represents a valid test scenario for subscribing -//type testType struct { -// name string -// arguments models.Arguments -// setupBackend func(sub *statestreamsmock.Subscription) -//} - // BlocksProviderSuite is a test suite for testing the block providers functionality. type BlocksProviderSuite struct { suite.Suite @@ -215,71 +208,3 @@ func (s *BlocksProviderSuite) requireBlock(v interface{}, expected interface{}) s.Require().Equal(expectedBlock, actualResponse.Block) } - -//// testHappyPath tests a variety of scenarios for data providers in -//// happy path scenarios. This function runs parameterized test cases that -//// simulate various configurations and verifies that the data provider operates -//// as expected without encountering errors. -//// -//// Arguments: -//// - topic: The topic associated with the data provider. -//// - factory: A factory for creating data provider instance. -//// - tests: A slice of test cases to run, each specifying setup and validation logic. -//// - sendData: A function to simulate emitting data into the subscription's data channel. -//// - expectedResponses: An expected responses to validate the received output. -//// - requireFn: A function to validate the output received in the send channel. -//func testHappyPath[T any]( -// t *testing.T, -// topic string, -// factory *DataProviderFactoryImpl, -// tests []testType, -// sendData func(chan interface{}), -// expectedResponses []T, -// requireFn func(interface{}, interface{}), -//) { -// for _, test := range tests { -// t.Run(test.name, func(t *testing.T) { -// ctx := context.Background() -// send := make(chan interface{}, 10) -// -// // Create a channel to simulate the subscription's data channel -// dataChan := make(chan interface{}) -// -// // Create a mock subscription and mock the channel -// sub := statestreamsmock.NewSubscription(t) -// sub.On("Channel").Return((<-chan interface{})(dataChan)) -// sub.On("Err").Return(nil) -// test.setupBackend(sub) -// -// // Create the data provider instance -// provider, err := factory.NewDataProvider(ctx, topic, test.arguments, send) -// require.NotNil(t, provider) -// require.NoError(t, err) -// -// // Run the provider in a separate goroutine -// go func() { -// err = provider.Run() -// require.NoError(t, err) -// }() -// -// // Simulate emitting data to the data channel -// go func() { -// defer close(dataChan) -// sendData(dataChan) -// }() -// -// // Collect responses -// for i, expected := range expectedResponses { -// unittest.RequireReturnsBefore(t, func() { -// v, ok := <-send -// require.True(t, ok, "channel closed while waiting for response %v: err: %v", expected, sub.Err()) -// -// requireFn(v, expected) -// }, time.Second, fmt.Sprintf("timed out waiting for response %d %v", i, expected)) -// } -// -// // Ensure the provider is properly closed after the test -// provider.Close() -// }) -// } -//} From 2ec38177d83b1aea6b51da95cac83a81e2cee3f2 Mon Sep 17 00:00:00 2001 From: Andrii Date: Tue, 17 Dec 2024 13:15:51 +0200 Subject: [PATCH 54/67] Linted --- .../data_providers/events_provider_test.go | 1 - .../transaction_statuses_provider.go | 2 +- .../transaction_statuses_provider_test.go | 72 +++++++++++++------ 3 files changed, 50 insertions(+), 25 deletions(-) diff --git a/engine/access/rest/websockets/data_providers/events_provider_test.go b/engine/access/rest/websockets/data_providers/events_provider_test.go index 214781f8885..ddfcd7d9fce 100644 --- a/engine/access/rest/websockets/data_providers/events_provider_test.go +++ b/engine/access/rest/websockets/data_providers/events_provider_test.go @@ -76,7 +76,6 @@ func (s *EventsProviderSuite) TestEventsDataProvider_HappyPath() { Events: expectedEvents, BlockTimestamp: s.rootBlock.Header.Timestamp, }) - } testHappyPath( diff --git a/engine/access/rest/websockets/data_providers/transaction_statuses_provider.go b/engine/access/rest/websockets/data_providers/transaction_statuses_provider.go index 1562f738b80..2dfa62fbb88 100644 --- a/engine/access/rest/websockets/data_providers/transaction_statuses_provider.go +++ b/engine/access/rest/websockets/data_providers/transaction_statuses_provider.go @@ -104,7 +104,7 @@ func (p *TransactionStatusesDataProvider) handleResponse() func(txResults []*acc return status.Errorf(codes.Internal, "message index already incremented to %d", messageIndex.Value()) } - p.send <- models.TransactionStatusesResponse{ + p.send <- &models.TransactionStatusesResponse{ TransactionResults: txResults, MessageIndex: strconv.FormatUint(index, 10), } diff --git a/engine/access/rest/websockets/data_providers/transaction_statuses_provider_test.go b/engine/access/rest/websockets/data_providers/transaction_statuses_provider_test.go index c57a7dff685..ba37340ef41 100644 --- a/engine/access/rest/websockets/data_providers/transaction_statuses_provider_test.go +++ b/engine/access/rest/websockets/data_providers/transaction_statuses_provider_test.go @@ -3,7 +3,6 @@ package data_providers import ( "context" "fmt" - "github.com/onflow/flow/protobuf/go/flow/entities" "strconv" "testing" @@ -20,6 +19,8 @@ import ( "github.com/onflow/flow-go/engine/access/subscription" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/utils/unittest" + + "github.com/onflow/flow/protobuf/go/flow/entities" ) type TransactionStatusesProviderSuite struct { @@ -59,43 +60,63 @@ func (s *TransactionStatusesProviderSuite) SetupTest() { } func (s *TransactionStatusesProviderSuite) TestTransactionStatusesDataProvider_HappyPath() { + id := unittest.IdentifierFixture() + cid := unittest.IdentifierFixture() + txr := access.TransactionResult{ + Status: flow.TransactionStatusSealed, + StatusCode: 10, + Events: []flow.Event{ + unittest.EventFixture(flow.EventAccountCreated, 1, 0, id, 200), + }, + ErrorMessage: "", + BlockID: s.rootBlock.ID(), + CollectionID: cid, + BlockHeight: s.rootBlock.Header.Height, + } + + var expectedTxStatusesResponses [][]*access.TransactionResult + var expectedTxResultsResponses []*access.TransactionResult - //testHappyPath( - // s.T(), - // AccountStatusesTopic, - // s.factory, - // s.subscribeTransactionStatusesDataProviderTestCases(), - // func(dataChan chan interface{}) { - // for i := 0; i < len(expectedAccountStatusesResponses); i++ { - // dataChan <- &expectedAccountStatusesResponses[i] - // } - // }, - // expectedAccountStatusesResponses, - // s.requireAccountStatuses, - //) + for i := 0; i < 2; i++ { + expectedTxResultsResponses = append(expectedTxResultsResponses, &txr) + expectedTxStatusesResponses = append(expectedTxStatusesResponses, expectedTxResultsResponses) + } + + testHappyPath( + s.T(), + TransactionStatusesTopic, + s.factory, + s.subscribeTransactionStatusesDataProviderTestCases(), + func(dataChan chan interface{}) { + for i := 0; i < len(expectedTxStatusesResponses); i++ { + dataChan <- expectedTxStatusesResponses[i] + } + }, + expectedTxStatusesResponses, + s.requireTransactionStatuses, + ) } func (s *TransactionStatusesProviderSuite) subscribeTransactionStatusesDataProviderTestCases() []testType { return []testType{ { - name: "SubscribeAccountStatusesFromStartBlockID happy path", + name: "SubscribeTransactionStatusesFromStartBlockID happy path", arguments: models.Arguments{ "start_block_id": s.rootBlock.ID().String(), - "event_types": []string{"flow.AccountCreated", "flow.AccountKeyAdded"}, }, setupBackend: func(sub *ssmock.Subscription) { s.api.On( "SubscribeTransactionStatusesFromStartBlockID", mock.Anything, - s.rootBlock.ID(), mock.Anything, + s.rootBlock.ID(), entities.EventEncodingVersion_JSON_CDC_V0, ).Return(sub).Once() }, }, { - name: "SubscribeAccountStatusesFromStartHeight happy path", + name: "SubscribeTransactionStatusesFromStartHeight happy path", arguments: models.Arguments{ "start_block_height": strconv.FormatUint(s.rootBlock.Header.Height, 10), }, @@ -103,19 +124,20 @@ func (s *TransactionStatusesProviderSuite) subscribeTransactionStatusesDataProvi s.api.On( "SubscribeTransactionStatusesFromStartHeight", mock.Anything, - s.rootBlock.Header.Height, mock.Anything, + s.rootBlock.Header.Height, entities.EventEncodingVersion_JSON_CDC_V0, ).Return(sub).Once() }, }, { - name: "SubscribeAccountStatusesFromLatestBlock happy path", + name: "SubscribeTransactionStatusesFromLatest happy path", arguments: models.Arguments{}, setupBackend: func(sub *ssmock.Subscription) { s.api.On( "SubscribeTransactionStatusesFromLatest", mock.Anything, + mock.Anything, entities.EventEncodingVersion_JSON_CDC_V0, ).Return(sub).Once() }, @@ -123,14 +145,18 @@ func (s *TransactionStatusesProviderSuite) subscribeTransactionStatusesDataProvi } } -// requireAccountStatuses ensures that the received account statuses information matches the expected data. -func (s *AccountStatusesProviderSuite) requireTransactionStatuses( +// requireTransactionStatuses ensures that the received transaction statuses information matches the expected data. +func (s *TransactionStatusesProviderSuite) requireTransactionStatuses( v interface{}, expectedResponse interface{}, ) { - _, ok := expectedResponse.([]access.TransactionResult) + expectedAccountStatusesResponse, ok := expectedResponse.([]*access.TransactionResult) require.True(s.T(), ok, "unexpected type: %T", expectedResponse) + actualResponse, ok := v.(*models.TransactionStatusesResponse) + require.True(s.T(), ok, "Expected *models.AccountStatusesResponse, got %T", v) + + s.Require().ElementsMatch(expectedAccountStatusesResponse, actualResponse.TransactionResults) } // TestAccountStatusesDataProvider_InvalidArguments tests the behavior of the transaction statuses data provider From a7a6f76cae3adb9dccf1cc5f51ad42c2bc90c17c Mon Sep 17 00:00:00 2001 From: Andrii Date: Tue, 17 Dec 2024 13:47:56 +0200 Subject: [PATCH 55/67] Fixed type in the log msg --- .../data_providers/transaction_statuses_provider_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/engine/access/rest/websockets/data_providers/transaction_statuses_provider_test.go b/engine/access/rest/websockets/data_providers/transaction_statuses_provider_test.go index ba37340ef41..723e866ca74 100644 --- a/engine/access/rest/websockets/data_providers/transaction_statuses_provider_test.go +++ b/engine/access/rest/websockets/data_providers/transaction_statuses_provider_test.go @@ -154,7 +154,7 @@ func (s *TransactionStatusesProviderSuite) requireTransactionStatuses( require.True(s.T(), ok, "unexpected type: %T", expectedResponse) actualResponse, ok := v.(*models.TransactionStatusesResponse) - require.True(s.T(), ok, "Expected *models.AccountStatusesResponse, got %T", v) + require.True(s.T(), ok, "Expected *models.TransactionStatusesResponse, got %T", v) s.Require().ElementsMatch(expectedAccountStatusesResponse, actualResponse.TransactionResults) } From e34d7877d03762f6c5b9537e6e0d294f9d4a19ea Mon Sep 17 00:00:00 2001 From: Andrii Date: Wed, 18 Dec 2024 13:49:42 +0200 Subject: [PATCH 56/67] Added new topic for send and subcribe tx statuses --- .../rest/websockets/data_providers/factory.go | 13 +- .../send_transaction_statuses_provider.go | 146 ++++++++++++++++++ .../rest/websockets/data_providers/util.go | 1 + 3 files changed, 154 insertions(+), 6 deletions(-) create mode 100644 engine/access/rest/websockets/data_providers/send_transaction_statuses_provider.go diff --git a/engine/access/rest/websockets/data_providers/factory.go b/engine/access/rest/websockets/data_providers/factory.go index b8897deb988..aa9b6920f67 100644 --- a/engine/access/rest/websockets/data_providers/factory.go +++ b/engine/access/rest/websockets/data_providers/factory.go @@ -15,12 +15,13 @@ import ( // Constants defining various topic names used to specify different types of // data providers. const ( - EventsTopic = "events" - AccountStatusesTopic = "account_statuses" - BlocksTopic = "blocks" - BlockHeadersTopic = "block_headers" - BlockDigestsTopic = "block_digests" - TransactionStatusesTopic = "transaction_statuses" + EventsTopic = "events" + AccountStatusesTopic = "account_statuses" + BlocksTopic = "blocks" + BlockHeadersTopic = "block_headers" + BlockDigestsTopic = "block_digests" + TransactionStatusesTopic = "transaction_statuses" + SendTransactionStatusesTopic = "send_transaction_statuses" ) // DataProviderFactory defines an interface for creating data providers diff --git a/engine/access/rest/websockets/data_providers/send_transaction_statuses_provider.go b/engine/access/rest/websockets/data_providers/send_transaction_statuses_provider.go new file mode 100644 index 00000000000..2ccbc3c7f2a --- /dev/null +++ b/engine/access/rest/websockets/data_providers/send_transaction_statuses_provider.go @@ -0,0 +1,146 @@ +package data_providers + +import ( + "context" + "fmt" + "github.com/onflow/flow-go/engine/access/rest/common/parser" + "github.com/onflow/flow-go/engine/access/subscription" + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module/counters" + "github.com/onflow/flow/protobuf/go/flow/entities" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "strconv" + + "github.com/rs/zerolog" + + "github.com/onflow/flow-go/access" + "github.com/onflow/flow-go/engine/access/rest/websockets/models" +) + +// sendTransactionStatusesArguments contains the arguments required for sending tx and subscribing to transaction statuses +type sendTransactionStatusesArguments struct { + Transaction flow.TransactionBody // The transaction body to be sent and monitored. +} + +type SendTransactionStatusesDataProvider struct { + *baseDataProvider + + logger zerolog.Logger + api access.API +} + +var _ DataProvider = (*SendTransactionStatusesDataProvider)(nil) + +func NewSendTransactionStatusesDataProvider( + ctx context.Context, + logger zerolog.Logger, + api access.API, + topic string, + arguments models.Arguments, + send chan<- interface{}, +) (*SendTransactionStatusesDataProvider, error) { + p := &SendTransactionStatusesDataProvider{ + logger: logger.With().Str("component", "send-transaction-statuses-data-provider").Logger(), + api: api, + } + + // Initialize arguments passed to the provider. + sendTxStatusesArgs, err := parseSendTransactionStatusesArguments(arguments) + if err != nil { + return nil, fmt.Errorf("invalid arguments for send tx statuses data provider: %w", err) + } + + subCtx, cancel := context.WithCancel(ctx) + + p.baseDataProvider = newBaseDataProvider( + topic, + cancel, + send, + p.createSubscription(subCtx, sendTxStatusesArgs), // Set up a subscription to tx statuses based on arguments. + ) + + return p, nil +} + +// Run starts processing the subscription for events and handles responses. +// +// No errors are expected during normal operations. +func (p *SendTransactionStatusesDataProvider) Run() error { + return subscription.HandleSubscription(p.subscription, p.handleResponse()) +} + +// createSubscription creates a new subscription using the specified input arguments. +func (p *SendTransactionStatusesDataProvider) createSubscription( + ctx context.Context, + args sendTransactionStatusesArguments, +) subscription.Subscription { + return p.api.SendAndSubscribeTransactionStatuses(ctx, &args.Transaction, entities.EventEncodingVersion_JSON_CDC_V0) +} + +// handleResponse processes an account statuses and sends the formatted response. +// +// No errors are expected during normal operations. +func (p *SendTransactionStatusesDataProvider) handleResponse() func(txResults []*access.TransactionResult) error { + + messageIndex := counters.NewMonotonousCounter(1) + + return func(txResults []*access.TransactionResult) error { + + index := messageIndex.Value() + if ok := messageIndex.Set(messageIndex.Value() + 1); !ok { + return status.Errorf(codes.Internal, "message index already incremented to %d", messageIndex.Value()) + } + + p.send <- &models.TransactionStatusesResponse{ + TransactionResults: txResults, + MessageIndex: strconv.FormatUint(index, 10), + } + + return nil + } + +} + +// parseAccountStatusesArguments validates and initializes the account statuses arguments. +func parseSendTransactionStatusesArguments( + arguments models.Arguments, +) (sendTransactionStatusesArguments, error) { + var args sendTransactionStatusesArguments + var tx flow.TransactionBody + + if scriptIn, ok := arguments["script"]; ok && scriptIn != "" { + script, ok := scriptIn.([]byte) + if !ok { + return args, fmt.Errorf("'script' must be a byte array") + } + + tx.Script = script + } + + if argumentsIn, ok := arguments["arguments"]; ok && argumentsIn != "" { + argumentsData, ok := argumentsIn.([][]byte) + if !ok { + return args, fmt.Errorf("'arguments' must be a [][]byte type") + } + + tx.Arguments = argumentsData + } + + if referenceBlockIDIn, ok := arguments["reference_block_id"]; ok && referenceBlockIDIn != "" { + result, ok := referenceBlockIDIn.(string) + if !ok { + return args, fmt.Errorf("'reference_block_id' must be a string") + } + + var referenceBlockID parser.ID + err := referenceBlockID.Parse(result) + if err != nil { + return args, fmt.Errorf("invalid 'reference_block_id': %w", err) + } + + tx.ReferenceBlockID = referenceBlockID.Flow() + } + + return args, nil +} diff --git a/engine/access/rest/websockets/data_providers/util.go b/engine/access/rest/websockets/data_providers/util.go index 8ddcd54f840..8ade7c127fc 100644 --- a/engine/access/rest/websockets/data_providers/util.go +++ b/engine/access/rest/websockets/data_providers/util.go @@ -20,6 +20,7 @@ type testType struct { setupBackend func(sub *statestreamsmock.Subscription) } +// testErrType represents an error cases for subscribing type testErrType struct { name string arguments models.Arguments From 13c4102109f4b0cfca0a8d7bc3b5a979c6ce2e5b Mon Sep 17 00:00:00 2001 From: Andrii Date: Wed, 18 Dec 2024 13:50:37 +0200 Subject: [PATCH 57/67] Renamed util filke for tests --- .../rest/websockets/data_providers/{util.go => utittest.go} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename engine/access/rest/websockets/data_providers/{util.go => utittest.go} (100%) diff --git a/engine/access/rest/websockets/data_providers/util.go b/engine/access/rest/websockets/data_providers/utittest.go similarity index 100% rename from engine/access/rest/websockets/data_providers/util.go rename to engine/access/rest/websockets/data_providers/utittest.go From 7b8ee773e4ec1a51863dcc8cb7ed530b9a2e51c7 Mon Sep 17 00:00:00 2001 From: Andrii Date: Wed, 18 Dec 2024 15:27:29 +0200 Subject: [PATCH 58/67] Initializing msg index by 0 --- .../websockets/data_providers/account_statuses_provider.go | 4 ++-- .../access/rest/websockets/data_providers/events_provider.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/engine/access/rest/websockets/data_providers/account_statuses_provider.go b/engine/access/rest/websockets/data_providers/account_statuses_provider.go index 09fca79bd70..d9e968c6ae0 100644 --- a/engine/access/rest/websockets/data_providers/account_statuses_provider.go +++ b/engine/access/rest/websockets/data_providers/account_statuses_provider.go @@ -99,7 +99,7 @@ func (p *AccountStatusesDataProvider) createSubscription(ctx context.Context, ar // No errors are expected during normal operations. func (p *AccountStatusesDataProvider) handleResponse() func(accountStatusesResponse *backend.AccountStatusesResponse) error { blocksSinceLastMessage := uint64(0) - messageIndex := counters.NewMonotonousCounter(1) + messageIndex := counters.NewMonotonousCounter(0) return func(accountStatusesResponse *backend.AccountStatusesResponse) error { // check if there are any events in the response. if not, do not send a message unless the last @@ -112,10 +112,10 @@ func (p *AccountStatusesDataProvider) handleResponse() func(accountStatusesRespo blocksSinceLastMessage = 0 } - index := messageIndex.Value() if ok := messageIndex.Set(messageIndex.Value() + 1); !ok { return status.Errorf(codes.Internal, "message index already incremented to %d", messageIndex.Value()) } + index := messageIndex.Value() p.send <- &models.AccountStatusesResponse{ BlockID: accountStatusesResponse.BlockID.String(), diff --git a/engine/access/rest/websockets/data_providers/events_provider.go b/engine/access/rest/websockets/data_providers/events_provider.go index b6854c93b4d..cb83fd3248e 100644 --- a/engine/access/rest/websockets/data_providers/events_provider.go +++ b/engine/access/rest/websockets/data_providers/events_provider.go @@ -85,7 +85,7 @@ func (p *EventsDataProvider) Run() error { // No errors are expected during normal operations. func (p *EventsDataProvider) handleResponse() func(eventsResponse *backend.EventsResponse) error { blocksSinceLastMessage := uint64(0) - messageIndex := counters.NewMonotonousCounter(1) + messageIndex := counters.NewMonotonousCounter(0) return func(eventsResponse *backend.EventsResponse) error { // check if there are any events in the response. if not, do not send a message unless the last @@ -98,10 +98,10 @@ func (p *EventsDataProvider) handleResponse() func(eventsResponse *backend.Event blocksSinceLastMessage = 0 } - index := messageIndex.Value() if ok := messageIndex.Set(messageIndex.Value() + 1); !ok { return fmt.Errorf("message index already incremented to: %d", messageIndex.Value()) } + index := messageIndex.Value() p.send <- &models.EventResponse{ BlockId: eventsResponse.BlockID.String(), From dd23b790cbb153e3232bd3507a72d058c5cea58e Mon Sep 17 00:00:00 2001 From: Andrii Date: Wed, 18 Dec 2024 15:38:15 +0200 Subject: [PATCH 59/67] Changed type of msgIndex in model to uint64: --- .../websockets/data_providers/account_statuses_provider.go | 2 +- .../data_providers/account_statuses_provider_test.go | 6 +++--- .../rest/websockets/data_providers/events_provider.go | 2 +- .../rest/websockets/data_providers/events_provider_test.go | 6 +++--- engine/access/rest/websockets/models/account_models.go | 2 +- engine/access/rest/websockets/models/event_models.go | 2 +- 6 files changed, 10 insertions(+), 10 deletions(-) diff --git a/engine/access/rest/websockets/data_providers/account_statuses_provider.go b/engine/access/rest/websockets/data_providers/account_statuses_provider.go index d9e968c6ae0..e1549bb65fb 100644 --- a/engine/access/rest/websockets/data_providers/account_statuses_provider.go +++ b/engine/access/rest/websockets/data_providers/account_statuses_provider.go @@ -121,7 +121,7 @@ func (p *AccountStatusesDataProvider) handleResponse() func(accountStatusesRespo BlockID: accountStatusesResponse.BlockID.String(), Height: strconv.FormatUint(accountStatusesResponse.Height, 10), AccountEvents: accountStatusesResponse.AccountEvents, - MessageIndex: strconv.FormatUint(index, 10), + MessageIndex: index, } return nil diff --git a/engine/access/rest/websockets/data_providers/account_statuses_provider_test.go b/engine/access/rest/websockets/data_providers/account_statuses_provider_test.go index 1bef0e1fad0..269f9de41ec 100644 --- a/engine/access/rest/websockets/data_providers/account_statuses_provider_test.go +++ b/engine/access/rest/websockets/data_providers/account_statuses_provider_test.go @@ -255,12 +255,12 @@ func (s *AccountStatusesProviderSuite) TestMessageIndexAccountStatusesProviderRe } // Verifying that indices are starting from 1 - s.Require().Equal("1", responses[0].MessageIndex, "Expected MessageIndex to start with 1") + s.Require().Equal(uint64(1), responses[0].MessageIndex, "Expected MessageIndex to start with 1") // Verifying that indices are strictly increasing for i := 1; i < len(responses); i++ { - prevIndex, _ := strconv.Atoi(responses[i-1].MessageIndex) - currentIndex, _ := strconv.Atoi(responses[i].MessageIndex) + prevIndex := responses[i-1].MessageIndex + currentIndex := responses[i].MessageIndex s.Require().Equal(prevIndex+1, currentIndex, "Expected MessageIndex to increment by 1") } diff --git a/engine/access/rest/websockets/data_providers/events_provider.go b/engine/access/rest/websockets/data_providers/events_provider.go index cb83fd3248e..947da07cd0b 100644 --- a/engine/access/rest/websockets/data_providers/events_provider.go +++ b/engine/access/rest/websockets/data_providers/events_provider.go @@ -108,7 +108,7 @@ func (p *EventsDataProvider) handleResponse() func(eventsResponse *backend.Event BlockHeight: strconv.FormatUint(eventsResponse.Height, 10), BlockTimestamp: eventsResponse.BlockTimestamp, Events: eventsResponse.Events, - MessageIndex: strconv.FormatUint(index, 10), + MessageIndex: index, } return nil diff --git a/engine/access/rest/websockets/data_providers/events_provider_test.go b/engine/access/rest/websockets/data_providers/events_provider_test.go index 214781f8885..c3aed834c8b 100644 --- a/engine/access/rest/websockets/data_providers/events_provider_test.go +++ b/engine/access/rest/websockets/data_providers/events_provider_test.go @@ -282,12 +282,12 @@ func (s *EventsProviderSuite) TestMessageIndexEventProviderResponse_HappyPath() } // Verifying that indices are starting from 1 - s.Require().Equal("1", responses[0].MessageIndex, "Expected MessageIndex to start with 1") + s.Require().Equal(uint64(1), responses[0].MessageIndex, "Expected MessageIndex to start with 1") // Verifying that indices are strictly increasing for i := 1; i < len(responses); i++ { - prevIndex, _ := strconv.Atoi(responses[i-1].MessageIndex) - currentIndex, _ := strconv.Atoi(responses[i].MessageIndex) + prevIndex := responses[i-1].MessageIndex + currentIndex := responses[i].MessageIndex s.Require().Equal(prevIndex+1, currentIndex, "Expected MessageIndex to increment by 1") } diff --git a/engine/access/rest/websockets/models/account_models.go b/engine/access/rest/websockets/models/account_models.go index 712f7a1be6a..fdb6826b4f1 100644 --- a/engine/access/rest/websockets/models/account_models.go +++ b/engine/access/rest/websockets/models/account_models.go @@ -7,5 +7,5 @@ type AccountStatusesResponse struct { BlockID string `json:"blockID"` Height string `json:"height"` AccountEvents map[string]flow.EventsList `json:"account_events"` - MessageIndex string `json:"message_index"` + MessageIndex uint64 `json:"message_index"` } diff --git a/engine/access/rest/websockets/models/event_models.go b/engine/access/rest/websockets/models/event_models.go index 48d085d9b85..0659cbc6937 100644 --- a/engine/access/rest/websockets/models/event_models.go +++ b/engine/access/rest/websockets/models/event_models.go @@ -12,5 +12,5 @@ type EventResponse struct { BlockHeight string `json:"block_height"` BlockTimestamp time.Time `json:"block_timestamp"` Events []flow.Event `json:"events"` - MessageIndex string `json:"message_index"` + MessageIndex uint64 `json:"message_index"` } From bf28e2fb38c132c95e1571114c33976c54948edb Mon Sep 17 00:00:00 2001 From: Andrii Date: Wed, 18 Dec 2024 17:27:03 +0200 Subject: [PATCH 60/67] Refactored parse arguments function --- .../account_statuses_provider.go | 42 +++---------------- .../data_providers/blocks_provider.go | 37 ++++++++++------ .../data_providers/events_provider.go | 42 +++---------------- 3 files changed, 36 insertions(+), 85 deletions(-) diff --git a/engine/access/rest/websockets/data_providers/account_statuses_provider.go b/engine/access/rest/websockets/data_providers/account_statuses_provider.go index e1549bb65fb..effd81b1ad4 100644 --- a/engine/access/rest/websockets/data_providers/account_statuses_provider.go +++ b/engine/access/rest/websockets/data_providers/account_statuses_provider.go @@ -11,7 +11,6 @@ import ( "github.com/onflow/flow-go/engine/access/rest/common/parser" "github.com/onflow/flow-go/engine/access/rest/http/request" - "github.com/onflow/flow-go/engine/access/rest/util" "github.com/onflow/flow-go/engine/access/rest/websockets/models" "github.com/onflow/flow-go/engine/access/state_stream" "github.com/onflow/flow-go/engine/access/state_stream/backend" @@ -136,42 +135,13 @@ func parseAccountStatusesArguments( ) (accountStatusesArguments, error) { var args accountStatusesArguments - // Check for mutual exclusivity of start_block_id and start_block_height early - startBlockIDIn, hasStartBlockID := arguments["start_block_id"] - startBlockHeightIn, hasStartBlockHeight := arguments["start_block_height"] - - if hasStartBlockID && hasStartBlockHeight { - return args, fmt.Errorf("can only provide either 'start_block_id' or 'start_block_height'") - } - - // Parse 'start_block_id' if provided - if hasStartBlockID { - result, ok := startBlockIDIn.(string) - if !ok { - return args, fmt.Errorf("'start_block_id' must be a string") - } - var startBlockID parser.ID - err := startBlockID.Parse(result) - if err != nil { - return args, fmt.Errorf("invalid 'start_block_id': %w", err) - } - args.StartBlockID = startBlockID.Flow() - } - - // Parse 'start_block_height' if provided - var err error - if hasStartBlockHeight { - result, ok := startBlockHeightIn.(string) - if !ok { - return args, fmt.Errorf("'start_block_height' must be a string") - } - args.StartBlockHeight, err = util.ToUint64(result) - if err != nil { - return args, fmt.Errorf("invalid 'start_block_height': %w", err) - } - } else { - args.StartBlockHeight = request.EmptyHeight + // Parse block arguments + startBlockID, startBlockHeight, err := ParseStartBlock(arguments) + if err != nil { + return args, err } + args.StartBlockID = startBlockID + args.StartBlockHeight = startBlockHeight // Parse 'event_types' as a JSON array var eventTypes parser.EventTypes diff --git a/engine/access/rest/websockets/data_providers/blocks_provider.go b/engine/access/rest/websockets/data_providers/blocks_provider.go index 89791397b5b..6c09c4a623a 100644 --- a/engine/access/rest/websockets/data_providers/blocks_provider.go +++ b/engine/access/rest/websockets/data_providers/blocks_provider.go @@ -109,41 +109,52 @@ func ParseBlocksArguments(arguments models.Arguments) (blocksArguments, error) { return args, fmt.Errorf("'block_status' must be provided") } + // Parse block arguments + startBlockID, startBlockHeight, err := ParseStartBlock(arguments) + if err != nil { + return args, err + } + args.StartBlockID = startBlockID + args.StartBlockHeight = startBlockHeight + + return args, nil +} + +func ParseStartBlock(arguments models.Arguments) (flow.Identifier, uint64, error) { startBlockIDIn, hasStartBlockID := arguments["start_block_id"] startBlockHeightIn, hasStartBlockHeight := arguments["start_block_height"] - // Ensure only one of start_block_id or start_block_height is provided + // Check for mutual exclusivity of start_block_id and start_block_height early if hasStartBlockID && hasStartBlockHeight { - return args, fmt.Errorf("can only provide either 'start_block_id' or 'start_block_height'") + return flow.ZeroID, 0, fmt.Errorf("can only provide either 'start_block_id' or 'start_block_height'") } + // Parse 'start_block_id' if hasStartBlockID { result, ok := startBlockIDIn.(string) if !ok { - return args, fmt.Errorf("'start_block_id' must be a string") + return flow.ZeroID, request.EmptyHeight, fmt.Errorf("'start_block_id' must be a string") } var startBlockID parser.ID err := startBlockID.Parse(result) if err != nil { - return args, err + return flow.ZeroID, request.EmptyHeight, fmt.Errorf("invalid 'start_block_id': %w", err) } - args.StartBlockID = startBlockID.Flow() + return startBlockID.Flow(), request.EmptyHeight, nil } + // Parse 'start_block_height' if hasStartBlockHeight { result, ok := startBlockHeightIn.(string) if !ok { - return args, fmt.Errorf("'start_block_height' must be a string") + return flow.ZeroID, 0, fmt.Errorf("'start_block_height' must be a string") } - var err error - args.StartBlockHeight, err = util.ToUint64(result) + startBlockHeight, err := util.ToUint64(result) if err != nil { - return args, fmt.Errorf("invalid 'start_block_height': %w", err) + return flow.ZeroID, request.EmptyHeight, fmt.Errorf("invalid 'start_block_height': %w", err) } - } else { - // Default value if 'start_block_height' is not provided - args.StartBlockHeight = request.EmptyHeight + return flow.ZeroID, startBlockHeight, nil } - return args, nil + return flow.ZeroID, request.EmptyHeight, nil } diff --git a/engine/access/rest/websockets/data_providers/events_provider.go b/engine/access/rest/websockets/data_providers/events_provider.go index 947da07cd0b..7b3397387b3 100644 --- a/engine/access/rest/websockets/data_providers/events_provider.go +++ b/engine/access/rest/websockets/data_providers/events_provider.go @@ -9,7 +9,6 @@ import ( "github.com/onflow/flow-go/engine/access/rest/common/parser" "github.com/onflow/flow-go/engine/access/rest/http/request" - "github.com/onflow/flow-go/engine/access/rest/util" "github.com/onflow/flow-go/engine/access/rest/websockets/models" "github.com/onflow/flow-go/engine/access/state_stream" "github.com/onflow/flow-go/engine/access/state_stream/backend" @@ -136,42 +135,13 @@ func parseEventsArguments( ) (eventsArguments, error) { var args eventsArguments - // Check for mutual exclusivity of start_block_id and start_block_height early - startBlockIDIn, hasStartBlockID := arguments["start_block_id"] - startBlockHeightIn, hasStartBlockHeight := arguments["start_block_height"] - - if hasStartBlockID && hasStartBlockHeight { - return args, fmt.Errorf("can only provide either 'start_block_id' or 'start_block_height'") - } - - // Parse 'start_block_id' if provided - if hasStartBlockID { - result, ok := startBlockIDIn.(string) - if !ok { - return args, fmt.Errorf("'start_block_id' must be a string") - } - var startBlockID parser.ID - err := startBlockID.Parse(result) - if err != nil { - return args, fmt.Errorf("invalid 'start_block_id': %w", err) - } - args.StartBlockID = startBlockID.Flow() - } - - // Parse 'start_block_height' if provided - var err error - if hasStartBlockHeight { - result, ok := startBlockHeightIn.(string) - if !ok { - return args, fmt.Errorf("'start_block_height' must be a string") - } - args.StartBlockHeight, err = util.ToUint64(result) - if err != nil { - return args, fmt.Errorf("invalid 'start_block_height': %w", err) - } - } else { - args.StartBlockHeight = request.EmptyHeight + // Parse block arguments + startBlockID, startBlockHeight, err := ParseStartBlock(arguments) + if err != nil { + return args, err } + args.StartBlockID = startBlockID + args.StartBlockHeight = startBlockHeight // Parse 'event_types' as a JSON array var eventTypes parser.EventTypes From 907c708b482a39370d9b4b93731cf9910142d0c0 Mon Sep 17 00:00:00 2001 From: Andrii Date: Thu, 19 Dec 2024 11:51:12 +0200 Subject: [PATCH 61/67] Fixed msg index start from 0 --- .../websockets/data_providers/account_statuses_provider.go | 2 +- .../data_providers/account_statuses_provider_test.go | 4 ++-- .../access/rest/websockets/data_providers/events_provider.go | 2 +- .../rest/websockets/data_providers/events_provider_test.go | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/engine/access/rest/websockets/data_providers/account_statuses_provider.go b/engine/access/rest/websockets/data_providers/account_statuses_provider.go index effd81b1ad4..396dcbc7b9a 100644 --- a/engine/access/rest/websockets/data_providers/account_statuses_provider.go +++ b/engine/access/rest/websockets/data_providers/account_statuses_provider.go @@ -111,10 +111,10 @@ func (p *AccountStatusesDataProvider) handleResponse() func(accountStatusesRespo blocksSinceLastMessage = 0 } + index := messageIndex.Value() if ok := messageIndex.Set(messageIndex.Value() + 1); !ok { return status.Errorf(codes.Internal, "message index already incremented to %d", messageIndex.Value()) } - index := messageIndex.Value() p.send <- &models.AccountStatusesResponse{ BlockID: accountStatusesResponse.BlockID.String(), diff --git a/engine/access/rest/websockets/data_providers/account_statuses_provider_test.go b/engine/access/rest/websockets/data_providers/account_statuses_provider_test.go index 269f9de41ec..8f689ca034a 100644 --- a/engine/access/rest/websockets/data_providers/account_statuses_provider_test.go +++ b/engine/access/rest/websockets/data_providers/account_statuses_provider_test.go @@ -254,8 +254,8 @@ func (s *AccountStatusesProviderSuite) TestMessageIndexAccountStatusesProviderRe responses = append(responses, accountStatusesRes) } - // Verifying that indices are starting from 1 - s.Require().Equal(uint64(1), responses[0].MessageIndex, "Expected MessageIndex to start with 1") + // Verifying that indices are starting from 0 + s.Require().Equal(uint64(0), responses[0].MessageIndex, "Expected MessageIndex to start with 0") // Verifying that indices are strictly increasing for i := 1; i < len(responses); i++ { diff --git a/engine/access/rest/websockets/data_providers/events_provider.go b/engine/access/rest/websockets/data_providers/events_provider.go index 7b3397387b3..318e8081d2c 100644 --- a/engine/access/rest/websockets/data_providers/events_provider.go +++ b/engine/access/rest/websockets/data_providers/events_provider.go @@ -97,10 +97,10 @@ func (p *EventsDataProvider) handleResponse() func(eventsResponse *backend.Event blocksSinceLastMessage = 0 } + index := messageIndex.Value() if ok := messageIndex.Set(messageIndex.Value() + 1); !ok { return fmt.Errorf("message index already incremented to: %d", messageIndex.Value()) } - index := messageIndex.Value() p.send <- &models.EventResponse{ BlockId: eventsResponse.BlockID.String(), diff --git a/engine/access/rest/websockets/data_providers/events_provider_test.go b/engine/access/rest/websockets/data_providers/events_provider_test.go index c3aed834c8b..2f7912f0ceb 100644 --- a/engine/access/rest/websockets/data_providers/events_provider_test.go +++ b/engine/access/rest/websockets/data_providers/events_provider_test.go @@ -282,7 +282,7 @@ func (s *EventsProviderSuite) TestMessageIndexEventProviderResponse_HappyPath() } // Verifying that indices are starting from 1 - s.Require().Equal(uint64(1), responses[0].MessageIndex, "Expected MessageIndex to start with 1") + s.Require().Equal(uint64(0), responses[0].MessageIndex, "Expected MessageIndex to start with 0") // Verifying that indices are strictly increasing for i := 1; i < len(responses); i++ { From 08f4b29f5c7c4b790b62e67b3c0cbe07c2690fc4 Mon Sep 17 00:00:00 2001 From: Andrii Date: Thu, 19 Dec 2024 13:08:58 +0200 Subject: [PATCH 62/67] Refactored tx statuses parse function --- .../transaction_statuses_provider.go | 42 +++---------------- 1 file changed, 6 insertions(+), 36 deletions(-) diff --git a/engine/access/rest/websockets/data_providers/transaction_statuses_provider.go b/engine/access/rest/websockets/data_providers/transaction_statuses_provider.go index 2dfa62fbb88..c73a193da4b 100644 --- a/engine/access/rest/websockets/data_providers/transaction_statuses_provider.go +++ b/engine/access/rest/websockets/data_providers/transaction_statuses_provider.go @@ -12,7 +12,6 @@ import ( "github.com/onflow/flow-go/access" "github.com/onflow/flow-go/engine/access/rest/common/parser" "github.com/onflow/flow-go/engine/access/rest/http/request" - "github.com/onflow/flow-go/engine/access/rest/util" "github.com/onflow/flow-go/engine/access/rest/websockets/models" "github.com/onflow/flow-go/engine/access/subscription" "github.com/onflow/flow-go/model/flow" @@ -119,13 +118,13 @@ func parseTransactionStatusesArguments( ) (transactionStatusesArguments, error) { var args transactionStatusesArguments - // Check for mutual exclusivity of start_block_id and start_block_height early - startBlockIDIn, hasStartBlockID := arguments["start_block_id"] - startBlockHeightIn, hasStartBlockHeight := arguments["start_block_height"] - - if hasStartBlockID && hasStartBlockHeight { - return args, fmt.Errorf("can only provide either 'start_block_id' or 'start_block_height'") + // Parse block arguments + startBlockID, startBlockHeight, err := ParseStartBlock(arguments) + if err != nil { + return args, err } + args.StartBlockID = startBlockID + args.StartBlockHeight = startBlockHeight if txIDIn, ok := arguments["tx_id"]; ok && txIDIn != "" { result, ok := txIDIn.(string) @@ -140,34 +139,5 @@ func parseTransactionStatusesArguments( args.TxID = txID.Flow() } - // Parse 'start_block_id' if provided - if hasStartBlockID { - result, ok := startBlockIDIn.(string) - if !ok { - return args, fmt.Errorf("'start_block_id' must be a string") - } - var startBlockID parser.ID - err := startBlockID.Parse(result) - if err != nil { - return args, fmt.Errorf("invalid 'start_block_id': %w", err) - } - args.StartBlockID = startBlockID.Flow() - } - - // Parse 'start_block_height' if provided - var err error - if hasStartBlockHeight { - result, ok := startBlockHeightIn.(string) - if !ok { - return args, fmt.Errorf("'start_block_height' must be a string") - } - args.StartBlockHeight, err = util.ToUint64(result) - if err != nil { - return args, fmt.Errorf("invalid 'start_block_height': %w", err) - } - } else { - args.StartBlockHeight = request.EmptyHeight - } - return args, nil } From 80c97a6864f02edbb63eca70ce2a8d2f55fb4510 Mon Sep 17 00:00:00 2001 From: Andrii Date: Thu, 19 Dec 2024 14:17:29 +0200 Subject: [PATCH 63/67] Added test for tx statuses data provider for msgIndex increment --- .../send_transaction_statuses_provider.go | 22 +++--- .../transaction_statuses_provider.go | 5 +- .../transaction_statuses_provider_test.go | 79 +++++++++++++++++++ .../websockets/models/tx_statuses_model.go | 2 +- 4 files changed, 93 insertions(+), 15 deletions(-) diff --git a/engine/access/rest/websockets/data_providers/send_transaction_statuses_provider.go b/engine/access/rest/websockets/data_providers/send_transaction_statuses_provider.go index 2ccbc3c7f2a..4a661874d0e 100644 --- a/engine/access/rest/websockets/data_providers/send_transaction_statuses_provider.go +++ b/engine/access/rest/websockets/data_providers/send_transaction_statuses_provider.go @@ -3,19 +3,19 @@ package data_providers import ( "context" "fmt" - "github.com/onflow/flow-go/engine/access/rest/common/parser" - "github.com/onflow/flow-go/engine/access/subscription" - "github.com/onflow/flow-go/model/flow" - "github.com/onflow/flow-go/module/counters" - "github.com/onflow/flow/protobuf/go/flow/entities" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" - "strconv" "github.com/rs/zerolog" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" "github.com/onflow/flow-go/access" + "github.com/onflow/flow-go/engine/access/rest/common/parser" "github.com/onflow/flow-go/engine/access/rest/websockets/models" + "github.com/onflow/flow-go/engine/access/subscription" + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module/counters" + + "github.com/onflow/flow/protobuf/go/flow/entities" ) // sendTransactionStatusesArguments contains the arguments required for sending tx and subscribing to transaction statuses @@ -83,18 +83,18 @@ func (p *SendTransactionStatusesDataProvider) createSubscription( // No errors are expected during normal operations. func (p *SendTransactionStatusesDataProvider) handleResponse() func(txResults []*access.TransactionResult) error { - messageIndex := counters.NewMonotonousCounter(1) + messageIndex := counters.NewMonotonousCounter(0) return func(txResults []*access.TransactionResult) error { - index := messageIndex.Value() if ok := messageIndex.Set(messageIndex.Value() + 1); !ok { return status.Errorf(codes.Internal, "message index already incremented to %d", messageIndex.Value()) } + index := messageIndex.Value() p.send <- &models.TransactionStatusesResponse{ TransactionResults: txResults, - MessageIndex: strconv.FormatUint(index, 10), + MessageIndex: index, } return nil diff --git a/engine/access/rest/websockets/data_providers/transaction_statuses_provider.go b/engine/access/rest/websockets/data_providers/transaction_statuses_provider.go index c73a193da4b..7e5e993dd23 100644 --- a/engine/access/rest/websockets/data_providers/transaction_statuses_provider.go +++ b/engine/access/rest/websockets/data_providers/transaction_statuses_provider.go @@ -3,7 +3,6 @@ package data_providers import ( "context" "fmt" - "strconv" "github.com/rs/zerolog" "google.golang.org/grpc/codes" @@ -94,7 +93,7 @@ func (p *TransactionStatusesDataProvider) createSubscription( // // No errors are expected during normal operations. func (p *TransactionStatusesDataProvider) handleResponse() func(txResults []*access.TransactionResult) error { - messageIndex := counters.NewMonotonousCounter(1) + messageIndex := counters.NewMonotonousCounter(0) return func(txResults []*access.TransactionResult) error { @@ -105,7 +104,7 @@ func (p *TransactionStatusesDataProvider) handleResponse() func(txResults []*acc p.send <- &models.TransactionStatusesResponse{ TransactionResults: txResults, - MessageIndex: strconv.FormatUint(index, 10), + MessageIndex: index, } return nil diff --git a/engine/access/rest/websockets/data_providers/transaction_statuses_provider_test.go b/engine/access/rest/websockets/data_providers/transaction_statuses_provider_test.go index 723e866ca74..11b36d14cb1 100644 --- a/engine/access/rest/websockets/data_providers/transaction_statuses_provider_test.go +++ b/engine/access/rest/websockets/data_providers/transaction_statuses_provider_test.go @@ -228,3 +228,82 @@ func invalidTransactionStatusesArgumentsTestCases() []testErrType { }, } } + +// TestMessageIndexTransactionStatusesProviderResponse_HappyPath tests that MessageIndex values in response are strictly increasing. +func (s *TransactionStatusesProviderSuite) TestMessageIndexTransactionStatusesProviderResponse_HappyPath() { + ctx := context.Background() + send := make(chan interface{}, 10) + topic := TransactionStatusesTopic + txStatusesCount := 4 + + // Create a channel to simulate the subscription's account statuses channel + txStatusesChan := make(chan interface{}) + + // Create a mock subscription and mock the channel + sub := ssmock.NewSubscription(s.T()) + sub.On("Channel").Return((<-chan interface{})(txStatusesChan)) + sub.On("Err").Return(nil) + + s.api.On( + "SubscribeTransactionStatusesFromStartBlockID", + mock.Anything, + mock.Anything, + mock.Anything, + entities.EventEncodingVersion_JSON_CDC_V0, + ).Return(sub) + + arguments := + map[string]interface{}{ + "start_block_id": s.rootBlock.ID().String(), + } + + // Create the TransactionStatusesDataProvider instance + provider, err := NewTransactionStatusesDataProvider( + ctx, + s.log, + s.api, + topic, + arguments, + send, + ) + s.Require().NotNil(provider) + s.Require().NoError(err) + + // Run the provider in a separate goroutine to simulate subscription processing + go func() { + err = provider.Run() + s.Require().NoError(err) + }() + + // Simulate emitting data to the еч statuses channel + go func() { + defer close(txStatusesChan) // Close the channel when done + + for i := 0; i < txStatusesCount; i++ { + txStatusesChan <- []*access.TransactionResult{} + } + }() + + // Collect responses + var responses []*models.TransactionStatusesResponse + for i := 0; i < txStatusesCount; i++ { + res := <-send + txStatusesRes, ok := res.(*models.TransactionStatusesResponse) + s.Require().True(ok, "Expected *models.TransactionStatusesResponse, got %T", res) + responses = append(responses, txStatusesRes) + } + + // Verifying that indices are starting from 0 + s.Require().Equal(uint64(0), responses[0].MessageIndex, "Expected MessageIndex to start with 0") + + // Verifying that indices are strictly increasing + for i := 1; i < len(responses); i++ { + prevIndex := responses[i-1].MessageIndex + currentIndex := responses[i].MessageIndex + s.Require().Equal(prevIndex+1, currentIndex, "Expected MessageIndex to increment by 1") + } + + // Ensure the provider is properly closed after the test + provider.Close() + +} diff --git a/engine/access/rest/websockets/models/tx_statuses_model.go b/engine/access/rest/websockets/models/tx_statuses_model.go index 55b9d3bdfdd..32754a06603 100644 --- a/engine/access/rest/websockets/models/tx_statuses_model.go +++ b/engine/access/rest/websockets/models/tx_statuses_model.go @@ -7,5 +7,5 @@ import ( // TransactionStatusesResponse is the response message for 'events' topic. type TransactionStatusesResponse struct { TransactionResults []*access.TransactionResult `json:"transaction_results"` - MessageIndex string `json:"message_index"` + MessageIndex uint64 `json:"message_index"` } From 317eb0e105109810c5b7117faaad38fc8c8cc9fd Mon Sep 17 00:00:00 2001 From: Andrii Date: Thu, 19 Dec 2024 17:02:16 +0200 Subject: [PATCH 64/67] Implemented parse function for send and susubdcribe tx statuses provider --- .../rest/websockets/data_providers/factory.go | 2 + .../send_transaction_statuses_provider.go | 97 ++++++++++++++++++- 2 files changed, 95 insertions(+), 4 deletions(-) diff --git a/engine/access/rest/websockets/data_providers/factory.go b/engine/access/rest/websockets/data_providers/factory.go index aa9b6920f67..ff23708d337 100644 --- a/engine/access/rest/websockets/data_providers/factory.go +++ b/engine/access/rest/websockets/data_providers/factory.go @@ -105,6 +105,8 @@ func (s *DataProviderFactoryImpl) NewDataProvider( return NewAccountStatusesDataProvider(ctx, s.logger, s.stateStreamApi, topic, arguments, ch, s.chain, s.eventFilterConfig, s.heartbeatInterval) case TransactionStatusesTopic: return NewTransactionStatusesDataProvider(ctx, s.logger, s.accessApi, topic, arguments, ch) + case SendTransactionStatusesTopic: + return NewSendTransactionStatusesDataProvider(ctx, s.logger, s.accessApi, topic, arguments, ch) default: return nil, fmt.Errorf("unsupported topic \"%s\"", topic) } diff --git a/engine/access/rest/websockets/data_providers/send_transaction_statuses_provider.go b/engine/access/rest/websockets/data_providers/send_transaction_statuses_provider.go index 4a661874d0e..8dc2a5e8dbd 100644 --- a/engine/access/rest/websockets/data_providers/send_transaction_statuses_provider.go +++ b/engine/access/rest/websockets/data_providers/send_transaction_statuses_provider.go @@ -10,6 +10,7 @@ import ( "github.com/onflow/flow-go/access" "github.com/onflow/flow-go/engine/access/rest/common/parser" + "github.com/onflow/flow-go/engine/access/rest/util" "github.com/onflow/flow-go/engine/access/rest/websockets/models" "github.com/onflow/flow-go/engine/access/subscription" "github.com/onflow/flow-go/model/flow" @@ -110,18 +111,33 @@ func parseSendTransactionStatusesArguments( var tx flow.TransactionBody if scriptIn, ok := arguments["script"]; ok && scriptIn != "" { - script, ok := scriptIn.([]byte) + result, ok := scriptIn.(string) if !ok { - return args, fmt.Errorf("'script' must be a byte array") + return args, fmt.Errorf("'script' must be a string") + } + + script, err := util.FromBase64(result) + if err != nil { + return args, fmt.Errorf("invalid 'script': %w", err) } tx.Script = script } if argumentsIn, ok := arguments["arguments"]; ok && argumentsIn != "" { - argumentsData, ok := argumentsIn.([][]byte) + result, ok := argumentsIn.([]string) if !ok { - return args, fmt.Errorf("'arguments' must be a [][]byte type") + return args, fmt.Errorf("'arguments' must be a []string type") + } + + var argumentsData [][]byte + for _, arg := range result { + argument, err := util.FromBase64(arg) + if err != nil { + return args, fmt.Errorf("invalid 'arguments': %w", err) + } + + argumentsData = append(argumentsData, argument) } tx.Arguments = argumentsData @@ -142,5 +158,78 @@ func parseSendTransactionStatusesArguments( tx.ReferenceBlockID = referenceBlockID.Flow() } + if gasLimitIn, ok := arguments["gas_limit"]; ok && gasLimitIn != "" { + result, ok := gasLimitIn.(string) + if !ok { + return args, fmt.Errorf("'gas_limit' must be a string") + } + + gasLimit, err := util.ToUint64(result) + if err != nil { + return args, fmt.Errorf("invalid 'gas_limit': %w", err) + } + tx.GasLimit = gasLimit + } + + if payerIn, ok := arguments["payer"]; ok && payerIn != "" { + result, ok := payerIn.(string) + if !ok { + return args, fmt.Errorf("'payerIn' must be a string") + } + + payerAddr, err := flow.StringToAddress(result) + if err != nil { + return args, fmt.Errorf("invalid 'payer': %w", err) + } + tx.Payer = payerAddr + } + + if proposalKeyIn, ok := arguments["proposal_key"]; ok && proposalKeyIn != "" { + proposalKey, ok := proposalKeyIn.(flow.ProposalKey) + if !ok { + return args, fmt.Errorf("'proposal_key' must be a object (ProposalKey)") + } + + tx.ProposalKey = proposalKey + } + + if authorizersIn, ok := arguments["authorizers"]; ok && authorizersIn != "" { + result, ok := authorizersIn.([]string) + if !ok { + return args, fmt.Errorf("'authorizers' must be a []string type") + } + + var authorizersData []flow.Address + for _, auth := range result { + authorizer, err := flow.StringToAddress(auth) + if err != nil { + return args, fmt.Errorf("invalid 'authorizers': %w", err) + } + + authorizersData = append(authorizersData, authorizer) + } + + tx.Authorizers = authorizersData + } + + if payloadSignaturesIn, ok := arguments["payload_signatures"]; ok && payloadSignaturesIn != "" { + payloadSignatures, ok := payloadSignaturesIn.([]flow.TransactionSignature) + if !ok { + return args, fmt.Errorf("'payload_signatures' must be an array of objects (TransactionSignature)") + } + + tx.PayloadSignatures = payloadSignatures + } + + if envelopeSignaturesIn, ok := arguments["envelope_signatures"]; ok && envelopeSignaturesIn != "" { + envelopeSignatures, ok := envelopeSignaturesIn.([]flow.TransactionSignature) + if !ok { + return args, fmt.Errorf("'payload_signatures' must be an array of objects (TransactionSignature)") + } + + tx.EnvelopeSignatures = envelopeSignatures + } + args.Transaction = tx + return args, nil } From 679ef2fb97bbb3be041d6c8d4515a41eeb778896 Mon Sep 17 00:00:00 2001 From: Andrii Date: Mon, 23 Dec 2024 12:58:33 +0200 Subject: [PATCH 65/67] Fixed typos and added missing invalid cases to godoc --- .../send_transaction_statuses_provider_test.go | 1 + .../transaction_statuses_provider_test.go | 12 +++++++----- 2 files changed, 8 insertions(+), 5 deletions(-) create mode 100644 engine/access/rest/websockets/data_providers/send_transaction_statuses_provider_test.go diff --git a/engine/access/rest/websockets/data_providers/send_transaction_statuses_provider_test.go b/engine/access/rest/websockets/data_providers/send_transaction_statuses_provider_test.go new file mode 100644 index 00000000000..06387b46331 --- /dev/null +++ b/engine/access/rest/websockets/data_providers/send_transaction_statuses_provider_test.go @@ -0,0 +1 @@ +package data_providers diff --git a/engine/access/rest/websockets/data_providers/transaction_statuses_provider_test.go b/engine/access/rest/websockets/data_providers/transaction_statuses_provider_test.go index 11b36d14cb1..68e79e3b978 100644 --- a/engine/access/rest/websockets/data_providers/transaction_statuses_provider_test.go +++ b/engine/access/rest/websockets/data_providers/transaction_statuses_provider_test.go @@ -53,7 +53,7 @@ func (s *TransactionStatusesProviderSuite) SetupTest() { s.log, nil, s.api, - flow.Testnet.Chain(), + s.chain, state_stream.DefaultEventFilterConfig, subscription.DefaultHeartbeatInterval) s.Require().NotNil(s.factory) @@ -159,13 +159,13 @@ func (s *TransactionStatusesProviderSuite) requireTransactionStatuses( s.Require().ElementsMatch(expectedAccountStatusesResponse, actualResponse.TransactionResults) } -// TestAccountStatusesDataProvider_InvalidArguments tests the behavior of the transaction statuses data provider +// TestTransactionStatusesDataProvider_InvalidArguments tests the behavior of the transaction statuses data provider // when invalid arguments are provided. It verifies that appropriate errors are returned // for missing or conflicting arguments. // This test covers the test cases: // 1. Invalid 'tx_id' argument. // 2. Invalid 'start_block_id' argument. -func (s *TransactionStatusesProviderSuite) TestAccountStatusesDataProvider_InvalidArguments() { +func (s *TransactionStatusesProviderSuite) TestTransactionStatusesDataProvider_InvalidArguments() { ctx := context.Background() send := make(chan interface{}) @@ -193,8 +193,10 @@ func (s *TransactionStatusesProviderSuite) TestAccountStatusesDataProvider_Inval // a set of input arguments, and the expected error message that should be returned. // // The test cases cover scenarios such as: -// 1. Providing invalid 'tx_id' value. -// 2. Providing invalid 'start_block_id' value. +// 1. Providing both 'start_block_id' and 'start_block_height' simultaneously. +// 2. Providing invalid 'tx_id' value. +// 3. Providing invalid 'start_block_id' value. +// 4. Invalid 'start_block_id' argument. func invalidTransactionStatusesArgumentsTestCases() []testErrType { return []testErrType{ { From 3e828b6b87f9e5c856661c0d25ff2d6abda83aa8 Mon Sep 17 00:00:00 2001 From: Andrii Date: Mon, 23 Dec 2024 13:01:09 +0200 Subject: [PATCH 66/67] fixed small remarks --- engine/access/rest/server.go | 3 ++- .../websockets/data_providers/transaction_statuses_provider.go | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/engine/access/rest/server.go b/engine/access/rest/server.go index c45919725b2..98643f19638 100644 --- a/engine/access/rest/server.go +++ b/engine/access/rest/server.go @@ -57,7 +57,8 @@ func NewServer(serverAPI access.API, serverAPI, chain, stateStreamConfig.EventFilterConfig, - stateStreamConfig.HeartbeatInterval) + stateStreamConfig.HeartbeatInterval, + ) builder.AddWebsocketsRoute(chain, wsConfig, config.MaxRequestSize, dataProviderFactory) c := cors.New(cors.Options{ diff --git a/engine/access/rest/websockets/data_providers/transaction_statuses_provider.go b/engine/access/rest/websockets/data_providers/transaction_statuses_provider.go index 7e5e993dd23..3e6fa0cc928 100644 --- a/engine/access/rest/websockets/data_providers/transaction_statuses_provider.go +++ b/engine/access/rest/websockets/data_providers/transaction_statuses_provider.go @@ -26,6 +26,7 @@ type transactionStatusesArguments struct { StartBlockHeight uint64 // Height of the block to start subscription from } +// TransactionStatusesDataProvider is responsible for providing tx statuses type TransactionStatusesDataProvider struct { *baseDataProvider From 71bac4979b8205926f3ea75539ffeb95370b0c96 Mon Sep 17 00:00:00 2001 From: Andrii Date: Mon, 23 Dec 2024 16:31:12 +0200 Subject: [PATCH 67/67] Added tests for send and subscribe data provider --- .../websockets/data_providers/factory_test.go | 11 + .../send_transaction_statuses_provider.go | 2 +- ...send_transaction_statuses_provider_test.go | 232 ++++++++++++++++++ .../transaction_statuses_provider_test.go | 60 +++-- 4 files changed, 278 insertions(+), 27 deletions(-) diff --git a/engine/access/rest/websockets/data_providers/factory_test.go b/engine/access/rest/websockets/data_providers/factory_test.go index 33389c3997e..f18455b7edd 100644 --- a/engine/access/rest/websockets/data_providers/factory_test.go +++ b/engine/access/rest/websockets/data_providers/factory_test.go @@ -143,6 +143,17 @@ func (s *DataProviderFactorySuite) TestSupportedTopics() { s.stateStreamApi.AssertExpectations(s.T()) }, }, + { + name: "send transaction statuses topic", + topic: SendTransactionStatusesTopic, + arguments: models.Arguments{}, + setupSubscription: func() { + s.setupSubscription(s.accessApi.On("SendAndSubscribeTransactionStatuses", mock.Anything, mock.Anything, mock.Anything)) + }, + assertExpectations: func() { + s.stateStreamApi.AssertExpectations(s.T()) + }, + }, } for _, test := range testCases { diff --git a/engine/access/rest/websockets/data_providers/send_transaction_statuses_provider.go b/engine/access/rest/websockets/data_providers/send_transaction_statuses_provider.go index 8dc2a5e8dbd..c2ad0ca5937 100644 --- a/engine/access/rest/websockets/data_providers/send_transaction_statuses_provider.go +++ b/engine/access/rest/websockets/data_providers/send_transaction_statuses_provider.go @@ -224,7 +224,7 @@ func parseSendTransactionStatusesArguments( if envelopeSignaturesIn, ok := arguments["envelope_signatures"]; ok && envelopeSignaturesIn != "" { envelopeSignatures, ok := envelopeSignaturesIn.([]flow.TransactionSignature) if !ok { - return args, fmt.Errorf("'payload_signatures' must be an array of objects (TransactionSignature)") + return args, fmt.Errorf("'envelope_signatures' must be an array of objects (TransactionSignature)") } tx.EnvelopeSignatures = envelopeSignatures diff --git a/engine/access/rest/websockets/data_providers/send_transaction_statuses_provider_test.go b/engine/access/rest/websockets/data_providers/send_transaction_statuses_provider_test.go index 06387b46331..ea617265d8f 100644 --- a/engine/access/rest/websockets/data_providers/send_transaction_statuses_provider_test.go +++ b/engine/access/rest/websockets/data_providers/send_transaction_statuses_provider_test.go @@ -1 +1,233 @@ package data_providers + +import ( + "context" + "testing" + + "github.com/rs/zerolog" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/suite" + + accessmock "github.com/onflow/flow-go/access/mock" + "github.com/onflow/flow-go/engine/access/rest/websockets/models" + "github.com/onflow/flow-go/engine/access/state_stream" + ssmock "github.com/onflow/flow-go/engine/access/state_stream/mock" + "github.com/onflow/flow-go/engine/access/subscription" + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/utils/unittest" + + "github.com/onflow/flow/protobuf/go/flow/entities" +) + +type SendTransactionStatusesProviderSuite struct { + suite.Suite + + log zerolog.Logger + api *accessmock.API + + chain flow.Chain + rootBlock flow.Block + finalizedBlock *flow.Header + + factory *DataProviderFactoryImpl +} + +func TestNewSendTransactionStatusesDataProvider(t *testing.T) { + suite.Run(t, new(SendTransactionStatusesProviderSuite)) +} + +func (s *SendTransactionStatusesProviderSuite) SetupTest() { + s.log = unittest.Logger() + s.api = accessmock.NewAPI(s.T()) + + s.chain = flow.Testnet.Chain() + + s.rootBlock = unittest.BlockFixture() + s.rootBlock.Header.Height = 0 + + s.factory = NewDataProviderFactory( + s.log, + nil, + s.api, + s.chain, + state_stream.DefaultEventFilterConfig, + subscription.DefaultHeartbeatInterval) + s.Require().NotNil(s.factory) +} + +// TestSendTransactionStatusesDataProvider_HappyPath tests the behavior of the send transaction statuses data provider +// when it is configured correctly and operating under normal conditions. It +// validates that tx statuses are correctly streamed to the channel and ensures +// no unexpected errors occur. +func (s *TransactionStatusesProviderSuite) TestSendTransactionStatusesDataProvider_HappyPath() { + + sendTxStatutesTestCases := []testType{ + { + name: "SubscribeTransactionStatusesFromStartBlockID happy path", + arguments: models.Arguments{ + "start_block_id": s.rootBlock.ID().String(), + }, + setupBackend: func(sub *ssmock.Subscription) { + s.api.On( + "SendAndSubscribeTransactionStatuses", + mock.Anything, + mock.Anything, + entities.EventEncodingVersion_JSON_CDC_V0, + ).Return(sub).Once() + }, + }, + } + + expectedResponse := expectedTransactionStatusesResponse(s.rootBlock) + + testHappyPath( + s.T(), + SendTransactionStatusesTopic, + s.factory, + sendTxStatutesTestCases, + func(dataChan chan interface{}) { + for i := 0; i < len(expectedResponse); i++ { + dataChan <- expectedResponse[i] + } + }, + expectedResponse, + s.requireTransactionStatuses, + ) + +} + +// TestSendTransactionStatusesDataProvider_InvalidArguments tests the behavior of the send transaction statuses data provider +// when invalid arguments are provided. It verifies that appropriate errors are returned +// for missing or conflicting arguments. +// This test covers the test cases: +// 1. Invalid 'script' type. +// 2. Invalid 'script' value. +// 3. Invalid 'arguments' type. +// 4. Invalid 'arguments' value. +// 5. Invalid 'reference_block_id' value. +// 6. Invalid 'gas_limit' value. +// 7. Invalid 'payer' value. +// 8. Invalid 'proposal_key' value. +// 9. Invalid 'authorizers' value. +// 10. Invalid 'payload_signatures' value. +// 11. Invalid 'envelope_signatures' value. +func (s *SendTransactionStatusesProviderSuite) TestSendTransactionStatusesDataProvider_InvalidArguments() { + ctx := context.Background() + send := make(chan interface{}) + + topic := SendTransactionStatusesTopic + + for _, test := range invalidSendTransactionStatusesArgumentsTestCases() { + s.Run(test.name, func() { + provider, err := NewSendTransactionStatusesDataProvider( + ctx, + s.log, + s.api, + topic, + test.arguments, + send, + ) + s.Require().Nil(provider) + s.Require().Error(err) + s.Require().Contains(err.Error(), test.expectedErrorMsg) + }) + } +} + +// invalidSendTransactionStatusesArgumentsTestCases returns a list of test cases with invalid argument combinations +// for testing the behavior of send transaction statuses data providers. Each test case includes a name, +// a set of input arguments, and the expected error message that should be returned. +// +// The test cases cover scenarios such as: +// 1. Providing invalid 'script' type. +// 2. Providing invalid 'script' value. +// 3. Providing invalid 'arguments' type. +// 4. Providing invalid 'arguments' value. +// 5. Providing invalid 'reference_block_id' value. +// 6. Providing invalid 'gas_limit' value. +// 7. Providing invalid 'payer' value. +// 8. Providing invalid 'proposal_key' value. +// 9. Providing invalid 'authorizers' value. +// 10. Providing invalid 'payload_signatures' value. +// 11. Providing invalid 'envelope_signatures' value. +func invalidSendTransactionStatusesArgumentsTestCases() []testErrType { + return []testErrType{ + { + name: "invalid 'script' argument type", + arguments: map[string]interface{}{ + "script": 0, + }, + expectedErrorMsg: "'script' must be a string", + }, + { + name: "invalid 'script' argument", + arguments: map[string]interface{}{ + "script": "invalid_script", + }, + expectedErrorMsg: "invalid 'script': illegal base64 data ", + }, + { + name: "invalid 'arguments' type", + arguments: map[string]interface{}{ + "arguments": 0, + }, + expectedErrorMsg: "'arguments' must be a []string type", + }, + { + name: "invalid 'arguments' argument", + arguments: map[string]interface{}{ + "arguments": []string{"invalid_base64_1", "invalid_base64_2"}, + }, + expectedErrorMsg: "invalid 'arguments'", + }, + { + name: "invalid 'reference_block_id' argument", + arguments: map[string]interface{}{ + "reference_block_id": "invalid_reference_block_id", + }, + expectedErrorMsg: "invalid ID format", + }, + { + name: "invalid 'gas_limit' argument", + arguments: map[string]interface{}{ + "gas_limit": "-1", + }, + expectedErrorMsg: "value must be an unsigned 64 bit integer", + }, + { + name: "invalid 'payer' argument", + arguments: map[string]interface{}{ + "payer": "invalid_payer", + }, + expectedErrorMsg: "invalid 'payer': can not decode hex string", + }, + { + name: "invalid 'proposal_key' argument", + arguments: map[string]interface{}{ + "proposal_key": "invalid ProposalKey object", + }, + expectedErrorMsg: "'proposal_key' must be a object (ProposalKey)", + }, + { + name: "invalid 'authorizers' argument", + arguments: map[string]interface{}{ + "authorizers": []string{"invalid_base64_1", "invalid_base64_2"}, + }, + expectedErrorMsg: "invalid 'authorizers': can not decode hex string", + }, + { + name: "invalid 'payload_signatures' argument", + arguments: map[string]interface{}{ + "payload_signatures": "invalid TransactionSignature array", + }, + expectedErrorMsg: "'payload_signatures' must be an array of objects (TransactionSignature)", + }, + { + name: "invalid 'envelope_signatures' argument", + arguments: map[string]interface{}{ + "envelope_signatures": "invalid TransactionSignature array", + }, + expectedErrorMsg: "'envelope_signatures' must be an array of objects (TransactionSignature)", + }, + } +} diff --git a/engine/access/rest/websockets/data_providers/transaction_statuses_provider_test.go b/engine/access/rest/websockets/data_providers/transaction_statuses_provider_test.go index 68e79e3b978..bfb81f82f81 100644 --- a/engine/access/rest/websockets/data_providers/transaction_statuses_provider_test.go +++ b/engine/access/rest/websockets/data_providers/transaction_statuses_provider_test.go @@ -59,28 +59,12 @@ func (s *TransactionStatusesProviderSuite) SetupTest() { s.Require().NotNil(s.factory) } +// TestTransactionStatusesDataProvider_HappyPath tests the behavior of the transaction statuses data provider +// when it is configured correctly and operating under normal conditions. It +// validates that tx statuses are correctly streamed to the channel and ensures +// no unexpected errors occur. func (s *TransactionStatusesProviderSuite) TestTransactionStatusesDataProvider_HappyPath() { - id := unittest.IdentifierFixture() - cid := unittest.IdentifierFixture() - txr := access.TransactionResult{ - Status: flow.TransactionStatusSealed, - StatusCode: 10, - Events: []flow.Event{ - unittest.EventFixture(flow.EventAccountCreated, 1, 0, id, 200), - }, - ErrorMessage: "", - BlockID: s.rootBlock.ID(), - CollectionID: cid, - BlockHeight: s.rootBlock.Header.Height, - } - - var expectedTxStatusesResponses [][]*access.TransactionResult - var expectedTxResultsResponses []*access.TransactionResult - - for i := 0; i < 2; i++ { - expectedTxResultsResponses = append(expectedTxResultsResponses, &txr) - expectedTxStatusesResponses = append(expectedTxStatusesResponses, expectedTxResultsResponses) - } + expectedResponse := expectedTransactionStatusesResponse(s.rootBlock) testHappyPath( s.T(), @@ -88,14 +72,13 @@ func (s *TransactionStatusesProviderSuite) TestTransactionStatusesDataProvider_H s.factory, s.subscribeTransactionStatusesDataProviderTestCases(), func(dataChan chan interface{}) { - for i := 0; i < len(expectedTxStatusesResponses); i++ { - dataChan <- expectedTxStatusesResponses[i] + for i := 0; i < len(expectedResponse); i++ { + dataChan <- expectedResponse[i] } }, - expectedTxStatusesResponses, + expectedResponse, s.requireTransactionStatuses, ) - } func (s *TransactionStatusesProviderSuite) subscribeTransactionStatusesDataProviderTestCases() []testType { @@ -195,7 +178,7 @@ func (s *TransactionStatusesProviderSuite) TestTransactionStatusesDataProvider_I // The test cases cover scenarios such as: // 1. Providing both 'start_block_id' and 'start_block_height' simultaneously. // 2. Providing invalid 'tx_id' value. -// 3. Providing invalid 'start_block_id' value. +// 3. Providing invalid 'start_block_id' value. // 4. Invalid 'start_block_id' argument. func invalidTransactionStatusesArgumentsTestCases() []testErrType { return []testErrType{ @@ -307,5 +290,30 @@ func (s *TransactionStatusesProviderSuite) TestMessageIndexTransactionStatusesPr // Ensure the provider is properly closed after the test provider.Close() +} + +func expectedTransactionStatusesResponse(block flow.Block) [][]*access.TransactionResult { + id := unittest.IdentifierFixture() + cid := unittest.IdentifierFixture() + txr := access.TransactionResult{ + Status: flow.TransactionStatusSealed, + StatusCode: 10, + Events: []flow.Event{ + unittest.EventFixture(flow.EventAccountCreated, 1, 0, id, 200), + }, + ErrorMessage: "", + BlockID: block.ID(), + CollectionID: cid, + BlockHeight: block.Header.Height, + } + + var expectedTxStatusesResponses [][]*access.TransactionResult + var expectedTxResultsResponses []*access.TransactionResult + + for i := 0; i < 2; i++ { + expectedTxResultsResponses = append(expectedTxResultsResponses, &txr) + expectedTxStatusesResponses = append(expectedTxStatusesResponses, expectedTxResultsResponses) + } + return expectedTxStatusesResponses }