Merge branch 'master' into fxamacker/separate-node-type

.github/workflows/ci.yml

@@ -35,7 +35,7 @@ jobs:
       uses: golangci/golangci-lint-action@v2
       with:
         # Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version.
-        version: v1.41
+        version: v1.45
         args: -v --build-tags relic
         # https://github.com/golangci/golangci-lint-action/issues/244
         skip-pkg-cache: true

.golangci.yml

@@ -28,3 +28,8 @@ issues:
     - path: _test\.go # disable some linters on test files
       linters:
         - unused
+    # typecheck currently not handling the way we do function inheritance well
+    # disabling for now
+    - path: 'cmd/access/node_build/*' 
+      linters:
+        - typecheck

Makefile

@@ -66,7 +66,7 @@ install-mock-generators:
 
 .PHONY: install-tools
 install-tools: crypto_setup_tests crypto_setup_gopath check-go-version install-mock-generators
-	curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b ${GOPATH}/bin v1.41.1; \
+	curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b ${GOPATH}/bin v1.45.2; \
 	cd ${GOPATH}; \
 	GO111MODULE=on go install github.com/golang/protobuf/protoc-gen-go@v1.3.2; \
 	GO111MODULE=on go install github.com/uber/prototool/cmd/prototool@v1.9.0; \

access/api.go

@@ -27,8 +27,10 @@ type API interface {
 
 	SendTransaction(ctx context.Context, tx *flow.TransactionBody) error
 	GetTransaction(ctx context.Context, id flow.Identifier) (*flow.TransactionBody, error)
+	GetTransactionsByBlockID(ctx context.Context, blockID flow.Identifier) ([]*flow.TransactionBody, error)
 	GetTransactionResult(ctx context.Context, id flow.Identifier) (*TransactionResult, error)
 	GetTransactionResultByIndex(ctx context.Context, blockID flow.Identifier, index uint32) (*TransactionResult, error)
+	GetTransactionResultsByBlockID(ctx context.Context, blockID flow.Identifier) ([]*TransactionResult, error)
 
 	GetAccount(ctx context.Context, address flow.Address) (*flow.Account, error)
 	GetAccountAtLatestBlock(ctx context.Context, address flow.Address) (*flow.Account, error)
@@ -49,31 +51,48 @@ type API interface {
 
 // TODO: Combine this with flow.TransactionResult?
 type TransactionResult struct {
-	Status       flow.TransactionStatus
-	StatusCode   uint
-	Events       []flow.Event
-	ErrorMessage string
-	BlockID      flow.Identifier
+	Status        flow.TransactionStatus
+	StatusCode    uint
+	Events        []flow.Event
+	ErrorMessage  string
+	BlockID       flow.Identifier
+	TransactionID flow.Identifier
+	CollectionID  flow.Identifier
 }
 
 func TransactionResultToMessage(result *TransactionResult) *access.TransactionResultResponse {
 	return &access.TransactionResultResponse{
-		Status:       entities.TransactionStatus(result.Status),
-		StatusCode:   uint32(result.StatusCode),
-		ErrorMessage: result.ErrorMessage,
-		Events:       convert.EventsToMessages(result.Events),
-		BlockId:      result.BlockID[:],
+		Status:        entities.TransactionStatus(result.Status),
+		StatusCode:    uint32(result.StatusCode),
+		ErrorMessage:  result.ErrorMessage,
+		Events:        convert.EventsToMessages(result.Events),
+		BlockId:       result.BlockID[:],
+		TransactionId: result.TransactionID[:],
+		CollectionId:  result.CollectionID[:],
+	}
+}
+
+func TransactionResultsToMessage(results []*TransactionResult) *access.TransactionResultsResponse {
+	messages := make([]*access.TransactionResultResponse, len(results))
+	for i, result := range results {
+		messages[i] = TransactionResultToMessage(result)
+	}
+
+	return &access.TransactionResultsResponse{
+		TransactionResults: messages,
 	}
 }
 
 func MessageToTransactionResult(message *access.TransactionResultResponse) *TransactionResult {
 
 	return &TransactionResult{
-		Status:       flow.TransactionStatus(message.Status),
-		StatusCode:   uint(message.StatusCode),
-		ErrorMessage: message.ErrorMessage,
-		Events:       convert.MessagesToEvents(message.Events),
-		BlockID:      flow.HashToID(message.BlockId),
+		Status:        flow.TransactionStatus(message.Status),
+		StatusCode:    uint(message.StatusCode),
+		ErrorMessage:  message.ErrorMessage,
+		Events:        convert.MessagesToEvents(message.Events),
+		BlockID:       flow.HashToID(message.BlockId),
+		TransactionID: flow.HashToID(message.TransactionId),
+		CollectionID:  flow.HashToID(message.CollectionId),
 	}
 }
 

access/handler.go

@@ -221,6 +221,42 @@ func (h *Handler) GetTransactionResult(
 	return TransactionResultToMessage(result), nil
 }
 
+func (h *Handler) GetTransactionResultsByBlockID(
+	ctx context.Context,
+	req *access.GetTransactionsByBlockIDRequest,
+) (*access.TransactionResultsResponse, error) {
+	id, err := convert.BlockID(req.GetBlockId())
+	if err != nil {
+		return nil, err
+	}
+
+	results, err := h.api.GetTransactionResultsByBlockID(ctx, id)
+	if err != nil {
+		return nil, err
+	}
+
+	return TransactionResultsToMessage(results), nil
+}
+
+func (h *Handler) GetTransactionsByBlockID(
+	ctx context.Context,
+	req *access.GetTransactionsByBlockIDRequest,
+) (*access.TransactionsResponse, error) {
+	id, err := convert.BlockID(req.GetBlockId())
+	if err != nil {
+		return nil, err
+	}
+
+	transactions, err := h.api.GetTransactionsByBlockID(ctx, id)
+	if err != nil {
+		return nil, err
+	}
+
+	return &access.TransactionsResponse{
+		Transactions: convert.TransactionsToMessages(transactions),
+	}, nil
+}
+
 // GetTransactionResultByIndex gets a transaction at a specific index for in a block that is executed,
 // pending or finalized transactions return errors
 func (h *Handler) GetTransactionResultByIndex(

cmd/access/node_builder/access_node_builder.go

@@ -12,6 +12,7 @@ import (
 	"github.com/spf13/pflag"
 
 	"github.com/onflow/flow-go/crypto"
+	"github.com/onflow/flow-go/module/compliance"
 
 	"github.com/onflow/flow-go/cmd"
 	"github.com/onflow/flow-go/consensus"
@@ -309,6 +310,7 @@ func (builder *FlowAccessNodeBuilder) buildFollowerEngine() *FlowAccessNodeBuild
 			builder.FollowerCore,
 			builder.SyncCore,
 			node.Tracer,
+			compliance.WithSkipNewProposalsThreshold(builder.ComplianceConfig.SkipNewProposalsThreshold),
 		)
 		if err != nil {
 			return nil, fmt.Errorf("could not create follower engine: %w", err)

cmd/access/node_builder/upstream_connector.go

@@ -55,14 +55,10 @@ func (connector *upstreamConnector) Ready() <-chan struct{} {
 				defer wg.Done()
 				lg := connector.logger.With().Str("bootstrap_node", id.NodeID.String()).Logger()
 
-				fibRetry, err := retry.NewFibonacci(connector.retryInitialTimeout)
-				if err != nil {
-					lg.Err(err).Msg("cannot create retry mechanism")
-					return
-				}
-				cappedFibRetry := retry.WithMaxRetries(connector.maxRetries, fibRetry)
+				backoff := retry.NewFibonacci(connector.retryInitialTimeout)
+				backoff = retry.WithMaxRetries(connector.maxRetries, backoff)
 
-				if err = retry.Do(workerCtx, cappedFibRetry, func(ctx context.Context) error {
+				if err := retry.Do(workerCtx, backoff, func(ctx context.Context) error {
 					return retry.RetryableError(connector.connect(ctx, id))
 				}); err != nil {
 					lg.Err(err).Msg("failed to connect")

cmd/bootstrap/cmd/constraints.go

@@ -50,12 +50,12 @@ func checkConstraints(partnerNodes, internalNodes []model.NodeInfo) {
 			if _, exists := internals.ByNodeID(node.NodeID); exists {
 				clusterInternalCount++
 			}
-			if clusterInternalCount <= clusterPartnerCount*2 {
-				log.Fatal().Msgf(
-					"will not bootstrap configuration without Byzantine majority within cluster: "+
-						"(partners=%d, internals=%d, min_internals=%d)",
-					clusterPartnerCount, clusterInternalCount, clusterPartnerCount*2+1)
-			}
+		}
+		if clusterInternalCount <= clusterPartnerCount*2 {
+			log.Fatal().Msgf(
+				"will not bootstrap configuration without Byzantine majority within cluster: "+
+					"(partners=%d, internals=%d, min_internals=%d)",
+				clusterPartnerCount, clusterInternalCount, clusterPartnerCount*2+1)
 		}
 		partnerCOLCount += clusterPartnerCount
 		internalCOLCount += clusterInternalCount

cmd/bootstrap/cmd/partner_infos.go

@@ -116,7 +116,7 @@ func getFlowClient() *client.Client {
 		insecureClient = false
 	}
 
-	config, err := common.NewFlowClientConfig(flagANAddress, strings.TrimPrefix(flagANNetworkKey, "0x"), insecureClient)
+	config, err := common.NewFlowClientConfig(flagANAddress, strings.TrimPrefix(flagANNetworkKey, "0x"), flow.ZeroID, insecureClient)
 	if err != nil {
 		log.Fatal().Err(err).Msgf("could not get flow client config with address (%s) and network key (%s)", flagANAddress, flagANNetworkKey)
 	}

cmd/bootstrap/transit/cmd/version.go

@@ -8,8 +8,6 @@ import (
 	"github.com/onflow/flow-go/cmd/build"
 )
 
-var ()
-
 // versionCmd prints the current versioning information about the Transit CLI
 var versionCmd = &cobra.Command{
 	Use:   "version",

cmd/collection/main.go

@@ -8,6 +8,7 @@ import (
 
 	"github.com/onflow/flow-go/cmd/util/cmd/common"
 	"github.com/onflow/flow-go/model/bootstrap"
+	modulecompliance "github.com/onflow/flow-go/module/compliance"
 	"github.com/onflow/flow-go/module/mempool/herocache"
 
 	"github.com/onflow/flow-go-sdk/client"
@@ -68,9 +69,10 @@ func main() {
 		startupTimeString                      string
 		startupTime                            time.Time
 
-		followerState protocol.MutableState
-		ingestConf    = ingest.DefaultConfig()
-		rpcConf       rpc.Config
+		followerState           protocol.MutableState
+		ingestConf              = ingest.DefaultConfig()
+		rpcConf                 rpc.Config
+		clusterComplianceConfig modulecompliance.Config
 
 		pools                   *epochpool.TransactionPools // epoch-scoped transaction pools
 		followerBuffer          *buffer.PendingBlocks       // pending block cache for follower
@@ -140,6 +142,8 @@ func main() {
 			"additional fraction of replica timeout that the primary will wait for votes")
 		flags.DurationVar(&blockRateDelay, "block-rate-delay", 250*time.Millisecond,
 			"the delay to broadcast block proposal in order to control block production rate")
+		flags.Uint64Var(&clusterComplianceConfig.SkipNewProposalsThreshold,
+			"cluster-compliance-skip-proposals-threshold", modulecompliance.DefaultConfig().SkipNewProposalsThreshold, "threshold at which new proposals are discarded rather than cached, if their height is this much above local finalized height (cluster compliance engine)")
 		flags.StringVar(&startupTimeString, "hotstuff-startup-time", cmd.NotSet, "specifies date and time (in ISO 8601 format) after which the consensus participant may enter the first view (e.g (e.g 1996-04-24T15:04:05-07:00))")
 
 		// epoch qc contract flags
@@ -306,6 +310,7 @@ func main() {
 				followerCore,
 				mainChainSyncCore,
 				node.Tracer,
+				modulecompliance.WithSkipNewProposalsThreshold(node.ComplianceConfig.SkipNewProposalsThreshold),
 			)
 			if err != nil {
 				return nil, fmt.Errorf("could not create follower engine: %w", err)
@@ -425,6 +430,7 @@ func main() {
 				node.Metrics.Mempool,
 				node.State,
 				node.Storage.Transactions,
+				modulecompliance.WithSkipNewProposalsThreshold(clusterComplianceConfig.SkipNewProposalsThreshold),
 			)
 			if err != nil {
 				return nil, err
@@ -526,7 +532,7 @@ func main() {
 }
 
 // createQCContractClient creates QC contract client
-func createQCContractClient(node *cmd.NodeConfig, machineAccountInfo *bootstrap.NodeMachineAccountInfo, flowClient *client.Client) (module.QCContractClient, error) {
+func createQCContractClient(node *cmd.NodeConfig, machineAccountInfo *bootstrap.NodeMachineAccountInfo, flowClient *client.Client, anID flow.Identifier) (module.QCContractClient, error) {
 
 	var qcContractClient module.QCContractClient
 
@@ -544,7 +550,7 @@ func createQCContractClient(node *cmd.NodeConfig, machineAccountInfo *bootstrap.
 	txSigner := sdkcrypto.NewInMemorySigner(sk, machineAccountInfo.HashAlgorithm)
 
 	// create actual qc contract client, all flags and machine account info file found
-	qcContractClient = epochs.NewQCContractClient(node.Logger, flowClient, node.Me.NodeID(), machineAccountInfo.Address, machineAccountInfo.KeyIndex, qcContractAddress, txSigner)
+	qcContractClient = epochs.NewQCContractClient(node.Logger, flowClient, anID, node.Me.NodeID(), machineAccountInfo.Address, machineAccountInfo.KeyIndex, qcContractAddress, txSigner)
 
 	return qcContractClient, nil
 }
@@ -559,7 +565,7 @@ func createQCContractClients(node *cmd.NodeConfig, machineAccountInfo *bootstrap
 			return nil, fmt.Errorf("failed to create flow client for qc contract client with options: %s %w", flowClientOpts, err)
 		}
 
-		qcClient, err := createQCContractClient(node, machineAccountInfo, flowClient)
+		qcClient, err := createQCContractClient(node, machineAccountInfo, flowClient, opt.AccessNodeID)
 		if err != nil {
 			return nil, fmt.Errorf("failed to create qc contract client with flow client options: %s %w", flowClientOpts, err)
 		}