Merge branch 'master' into fxamacker/reuse-mtrie-state-for-checkpoint…

…ing-2

.github/workflows/flaky-test-debug.yml

@@ -0,0 +1,45 @@
+name: Flaky Test Debug
+
+on:
+  schedule:
+    - cron: '0 */1 * * *' # every 1 hour
+  push:
+    branches:
+      - '**/*flaky-test*'
+env:
+  GO_VERSION: 1.18
+
+jobs:
+  integration-test:
+    name: Integration Tests
+    strategy:
+      fail-fast: false
+      matrix:
+        test-category:
+          - integration-epochs
+    env:
+      TEST_CATEGORY: ${{ matrix.test-category }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Setup Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - name: Checkout repo
+        uses: actions/checkout@v2
+      - name: Build relic
+        run: make crypto_setup_gopath
+      - name: Docker build
+        run: make docker-build-flow
+      - name: Run tests
+        if: github.actor != 'bors[bot]'
+        run: ./tools/test_monitor/run-tests.sh
+      - name: Run tests (Bors)
+        if: github.actor == 'bors[bot]'
+        uses: nick-invision/retry@v2
+        with:
+          timeout_minutes: 15
+          max_attempts: 2
+          command: ./tools/test_monitor/run-tests.sh
+
+

CODEOWNERS

@@ -37,7 +37,7 @@
 /cmd/bootstrap/** @vishalchangrani
 
 # Performance Stream
-/integration/localnet/** @SaveTheRbtz @simonhf @Kay-Zee @zhangchiqing @pattyshack
-/integration/loader/** @SaveTheRbtz @simonhf @Kay-Zee @pattyshack
-/integration/benchmark/** @SaveTheRbtz @simonhf @Kay-Zee @pattyshack
-/integration/utils/** @SaveTheRbtz @simonhf @Kay-Zee @pattyshack
+/integration/localnet/** @SaveTheRbtz @simonhf @Kay-Zee @zhangchiqing @pattyshack @AndrewM-SDET
+/integration/loader/** @SaveTheRbtz @simonhf @Kay-Zee @pattyshack @AndrewM-SDET
+/integration/benchmark/** @SaveTheRbtz @simonhf @Kay-Zee @pattyshack @AndrewM-SDET
+/integration/utils/** @SaveTheRbtz @simonhf @Kay-Zee @pattyshack @AndrewM-SDET

Makefile

@@ -53,24 +53,24 @@ cmd/util/util:
 .PHONY: unittest-main
 unittest-main:
 	# test all packages with Relic library enabled
-	GO111MODULE=on go test -coverprofile=$(COVER_PROFILE) -covermode=atomic $(if $(JSON_OUTPUT),-json,) $(if $(NUM_RUNS),-count $(NUM_RUNS),) --tags relic ./...
+	go test -coverprofile=$(COVER_PROFILE) -covermode=atomic $(if $(JSON_OUTPUT),-json,) $(if $(NUM_RUNS),-count $(NUM_RUNS),) --tags relic ./...
 
 .PHONY: install-mock-generators
 install-mock-generators:
 	cd ${GOPATH}; \
-    GO111MODULE=on go install github.com/vektra/mockery/v2@v2.13.0; \
-    GO111MODULE=on go install github.com/golang/mock/mockgen@v1.3.1;
+    go install github.com/vektra/mockery/v2@v2.13.0; \
+    go install github.com/golang/mock/mockgen@v1.3.1;
 
 ############################################################################################
 
 .PHONY: install-tools
 install-tools: crypto_setup_tests crypto_setup_gopath check-go-version install-mock-generators
 	curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b ${GOPATH}/bin v1.46.2; \
 	cd ${GOPATH}; \
-	GO111MODULE=on go install github.com/golang/protobuf/protoc-gen-go@v1.3.2; \
-	GO111MODULE=on go install github.com/uber/prototool/cmd/prototool@v1.9.0; \
-	GO111MODULE=on go install github.com/gogo/protobuf/protoc-gen-gofast@latest; \
-	GO111MODULE=on go install golang.org/x/tools/cmd/stringer@master;
+	go install github.com/golang/protobuf/protoc-gen-go@v1.3.2; \
+	go install github.com/uber/prototool/cmd/prototool@v1.9.0; \
+	go install github.com/gogo/protobuf/protoc-gen-gofast@latest; \
+	go install golang.org/x/tools/cmd/stringer@master;
 
 .PHONY: unittest
 unittest: unittest-main
@@ -80,7 +80,7 @@ unittest: unittest-main
 .PHONY: emulator-build
 emulator-build:
 	# test the fvm package compiles with Relic library disabled (required for the emulator build)
-	cd ./fvm && GO111MODULE=on go test ./... -run=NoTestHasThisPrefix
+	cd ./fvm && go test ./... -run=NoTestHasThisPrefix
 
 .PHONY: test
 test: generate-mocks emulator-build unittest
@@ -118,46 +118,46 @@ generate-proto:
 
 .PHONY: generate-mocks
 generate-mocks:
-	GO111MODULE=on mockery --name '(Connector|PingInfoProvider)' --dir=network/p2p --case=underscore --output="./network/mocknetwork" --outpkg="mocknetwork"
-	GO111MODULE=on mockgen -destination=storage/mocks/storage.go -package=mocks github.com/onflow/flow-go/storage Blocks,Headers,Payloads,Collections,Commits,Events,ServiceEvents,TransactionResults
-	GO111MODULE=on mockgen -destination=module/mocks/network.go -package=mocks github.com/onflow/flow-go/module Local,Requester
-	GO111MODULE=on mockgen -destination=network/mocknetwork/engine.go -package=mocknetwork github.com/onflow/flow-go/network Engine
-	GO111MODULE=on mockgen -destination=network/mocknetwork/mock_network.go -package=mocknetwork github.com/onflow/flow-go/network Network
-	GO111MODULE=on mockery --name '(ExecutionDataService|ExecutionDataCIDCache|ExecutionDataRequester)' --dir=module/state_synchronization --case=underscore --output="./module/state_synchronization/mock" --outpkg="state_synchronization"
-	GO111MODULE=on mockery --name 'ExecutionState' --dir=engine/execution/state --case=underscore --output="engine/execution/state/mock" --outpkg="mock"
-	GO111MODULE=on mockery --name 'BlockComputer' --dir=engine/execution/computation/computer --case=underscore --output="engine/execution/computation/computer/mock" --outpkg="mock"
-	GO111MODULE=on mockery --name 'ComputationManager' --dir=engine/execution/computation --case=underscore --output="engine/execution/computation/mock" --outpkg="mock"
-	GO111MODULE=on mockery --name 'EpochComponentsFactory' --dir=engine/collection/epochmgr --case=underscore --output="engine/collection/epochmgr/mock" --outpkg="mock"
-	GO111MODULE=on mockery --name 'Backend' --dir=engine/collection/rpc --case=underscore --output="engine/collection/rpc/mock" --outpkg="mock"
-	GO111MODULE=on mockery --name 'ProviderEngine' --dir=engine/execution/provider --case=underscore --output="engine/execution/provider/mock" --outpkg="mock"
-	(cd ./crypto && GO111MODULE=on mockery --name 'PublicKey' --case=underscore --output="../module/mock" --outpkg="mock")
-	GO111MODULE=on mockery --name '.*' --dir=state/cluster --case=underscore --output="state/cluster/mock" --outpkg="mock"
-	GO111MODULE=on mockery --name '.*' --dir=module --case=underscore --tags="relic" --output="./module/mock" --outpkg="mock"
-	GO111MODULE=on mockery --name '.*' --dir=module/mempool --case=underscore --output="./module/mempool/mock" --outpkg="mempool"
-	GO111MODULE=on mockery --name '.*' --dir=module/component --case=underscore --output="./module/component/mock" --outpkg="component"
-	GO111MODULE=on mockery --name '.*' --dir=network --case=underscore --output="./network/mocknetwork" --outpkg="mocknetwork"
-	GO111MODULE=on mockery --name '.*' --dir=storage --case=underscore --output="./storage/mock" --outpkg="mock"
-	GO111MODULE=on mockery --name '.*' --dir="state/protocol" --case=underscore --output="state/protocol/mock" --outpkg="mock"
-	GO111MODULE=on mockery --name '.*' --dir="state/protocol/events" --case=underscore --output="./state/protocol/events/mock" --outpkg="mock"
-	GO111MODULE=on mockery --name '.*' --dir=engine/execution/computation/computer --case=underscore --output="./engine/execution/computation/computer/mock" --outpkg="mock"
-	GO111MODULE=on mockery --name '.*' --dir=engine/execution/state --case=underscore --output="./engine/execution/state/mock" --outpkg="mock"
-	GO111MODULE=on mockery --name '.*' --dir=engine/consensus --case=underscore --output="./engine/consensus/mock" --outpkg="mock"
-	GO111MODULE=on mockery --name '.*' --dir=engine/consensus/approvals --case=underscore --output="./engine/consensus/approvals/mock" --outpkg="mock"
-	GO111MODULE=on mockery --name '.*' --dir=fvm --case=underscore --output="./fvm/mock" --outpkg="mock"
-	GO111MODULE=on mockery --name '.*' --dir=fvm/state --case=underscore --output="./fvm/mock/state" --outpkg="mock"
-	GO111MODULE=on mockery --name '.*' --dir=ledger --case=underscore --output="./ledger/mock" --outpkg="mock"
-	GO111MODULE=on mockery --name 'SubscriptionManager' --dir=network/ --case=underscore --output="./network/mocknetwork" --outpkg="mocknetwork"
-	GO111MODULE=on mockery --name 'Vertex' --dir="./module/forest" --case=underscore --output="./module/forest/mock" --outpkg="mock"
-	GO111MODULE=on mockery --name '.*' --dir="./consensus/hotstuff" --case=underscore --output="./consensus/hotstuff/mocks" --outpkg="mocks"
-	GO111MODULE=on mockery --name '.*' --dir="./engine/access/wrapper" --case=underscore --output="./engine/access/mock" --outpkg="mock"
-	GO111MODULE=on mockery --name 'API' --dir="./access" --case=underscore --output="./access/mock" --outpkg="mock"
-	GO111MODULE=on mockery --name 'ConnectionFactory' --dir="./engine/access/rpc/backend" --case=underscore --output="./engine/access/rpc/backend/mock" --outpkg="mock"
-	GO111MODULE=on mockery --name 'IngestRPC' --dir="./engine/execution/ingestion" --case=underscore --tags relic --output="./engine/execution/ingestion/mock" --outpkg="mock"
-	GO111MODULE=on mockery --name '.*' --dir=model/fingerprint --case=underscore --output="./model/fingerprint/mock" --outpkg="mock"
-	GO111MODULE=on mockery --name 'ExecForkActor' --structname 'ExecForkActorMock' --dir=module/mempool/consensus/mock/ --case=underscore --output="./module/mempool/consensus/mock/" --outpkg="mock"
-	GO111MODULE=on mockery --name '.*' --dir=engine/verification/fetcher/ --case=underscore --output="./engine/verification/fetcher/mock" --outpkg="mockfetcher"
-	GO111MODULE=on mockery --name '.*' --dir=insecure/ --case=underscore --output="./insecure/mock"  --outpkg="mockinsecure"
-	GO111MODULE=on mockery --name '.*' --dir=./cmd/util/ledger/reporters --case=underscore --output="./cmd/util/ledger/reporters/mock" --outpkg="mock"
+	mockery --name '(Connector|PingInfoProvider)' --dir=network/p2p --case=underscore --output="./network/mocknetwork" --outpkg="mocknetwork"
+	mockgen -destination=storage/mocks/storage.go -package=mocks github.com/onflow/flow-go/storage Blocks,Headers,Payloads,Collections,Commits,Events,ServiceEvents,TransactionResults
+	mockgen -destination=module/mocks/network.go -package=mocks github.com/onflow/flow-go/module Local,Requester
+	mockgen -destination=network/mocknetwork/engine.go -package=mocknetwork github.com/onflow/flow-go/network Engine
+	mockgen -destination=network/mocknetwork/mock_network.go -package=mocknetwork github.com/onflow/flow-go/network Network
+	mockery --name '(ExecutionDataService|ExecutionDataCIDCache|ExecutionDataRequester)' --dir=module/state_synchronization --case=underscore --output="./module/state_synchronization/mock" --outpkg="state_synchronization"
+	mockery --name 'ExecutionState' --dir=engine/execution/state --case=underscore --output="engine/execution/state/mock" --outpkg="mock"
+	mockery --name 'BlockComputer' --dir=engine/execution/computation/computer --case=underscore --output="engine/execution/computation/computer/mock" --outpkg="mock"
+	mockery --name 'ComputationManager' --dir=engine/execution/computation --case=underscore --output="engine/execution/computation/mock" --outpkg="mock"
+	mockery --name 'EpochComponentsFactory' --dir=engine/collection/epochmgr --case=underscore --output="engine/collection/epochmgr/mock" --outpkg="mock"
+	mockery --name 'Backend' --dir=engine/collection/rpc --case=underscore --output="engine/collection/rpc/mock" --outpkg="mock"
+	mockery --name 'ProviderEngine' --dir=engine/execution/provider --case=underscore --output="engine/execution/provider/mock" --outpkg="mock"
+	(cd ./crypto && mockery --name 'PublicKey' --case=underscore --output="../module/mock" --outpkg="mock")
+	mockery --name '.*' --dir=state/cluster --case=underscore --output="state/cluster/mock" --outpkg="mock"
+	mockery --name '.*' --dir=module --case=underscore --tags="relic" --output="./module/mock" --outpkg="mock"
+	mockery --name '.*' --dir=module/mempool --case=underscore --output="./module/mempool/mock" --outpkg="mempool"
+	mockery --name '.*' --dir=module/component --case=underscore --output="./module/component/mock" --outpkg="component"
+	mockery --name '.*' --dir=network --case=underscore --output="./network/mocknetwork" --outpkg="mocknetwork"
+	mockery --name '.*' --dir=storage --case=underscore --output="./storage/mock" --outpkg="mock"
+	mockery --name '.*' --dir="state/protocol" --case=underscore --output="state/protocol/mock" --outpkg="mock"
+	mockery --name '.*' --dir="state/protocol/events" --case=underscore --output="./state/protocol/events/mock" --outpkg="mock"
+	mockery --name '.*' --dir=engine/execution/computation/computer --case=underscore --output="./engine/execution/computation/computer/mock" --outpkg="mock"
+	mockery --name '.*' --dir=engine/execution/state --case=underscore --output="./engine/execution/state/mock" --outpkg="mock"
+	mockery --name '.*' --dir=engine/consensus --case=underscore --output="./engine/consensus/mock" --outpkg="mock"
+	mockery --name '.*' --dir=engine/consensus/approvals --case=underscore --output="./engine/consensus/approvals/mock" --outpkg="mock"
+	mockery --name '.*' --dir=fvm --case=underscore --output="./fvm/mock" --outpkg="mock"
+	mockery --name '.*' --dir=fvm/state --case=underscore --output="./fvm/mock/state" --outpkg="mock"
+	mockery --name '.*' --dir=ledger --case=underscore --output="./ledger/mock" --outpkg="mock"
+	mockery --name 'SubscriptionManager' --dir=network/ --case=underscore --output="./network/mocknetwork" --outpkg="mocknetwork"
+	mockery --name 'Vertex' --dir="./module/forest" --case=underscore --output="./module/forest/mock" --outpkg="mock"
+	mockery --name '.*' --dir="./consensus/hotstuff" --case=underscore --output="./consensus/hotstuff/mocks" --outpkg="mocks"
+	mockery --name '.*' --dir="./engine/access/wrapper" --case=underscore --output="./engine/access/mock" --outpkg="mock"
+	mockery --name 'API' --dir="./access" --case=underscore --output="./access/mock" --outpkg="mock"
+	mockery --name 'ConnectionFactory' --dir="./engine/access/rpc/backend" --case=underscore --output="./engine/access/rpc/backend/mock" --outpkg="mock"
+	mockery --name 'IngestRPC' --dir="./engine/execution/ingestion" --case=underscore --tags relic --output="./engine/execution/ingestion/mock" --outpkg="mock"
+	mockery --name '.*' --dir=model/fingerprint --case=underscore --output="./model/fingerprint/mock" --outpkg="mock"
+	mockery --name 'ExecForkActor' --structname 'ExecForkActorMock' --dir=module/mempool/consensus/mock/ --case=underscore --output="./module/mempool/consensus/mock/" --outpkg="mock"
+	mockery --name '.*' --dir=engine/verification/fetcher/ --case=underscore --output="./engine/verification/fetcher/mock" --outpkg="mockfetcher"
+	mockery --name '.*' --dir=insecure/ --case=underscore --output="./insecure/mock"  --outpkg="mockinsecure"
+	mockery --name '.*' --dir=./cmd/util/ledger/reporters --case=underscore --output="./cmd/util/ledger/reporters/mock" --outpkg="mock"
 
 # this ensures there is no unused dependency being added by accident
 .PHONY: tidy
@@ -170,12 +170,12 @@ tidy:
 
 .PHONY: lint
 lint: tidy
-	# GO111MODULE=on revive -config revive.toml -exclude storage/ledger/trie ./...
+	# revive -config revive.toml -exclude storage/ledger/trie ./...
 	golangci-lint run -v --build-tags relic ./...
 
 .PHONY: fix-lint
 fix-lint:
-	# GO111MODULE=on revive -config revive.toml -exclude storage/ledger/trie ./...
+	# revive -config revive.toml -exclude storage/ledger/trie ./...
 	golangci-lint run -v --build-tags relic --fix ./...
 
 # Runs unit tests, SKIP FOR NOW linter, coverage

README.md

@@ -85,11 +85,10 @@ The following table lists all work streams and links to their home directory and
 
 At this point, you should be ready to build, test, and run Flow! 🎉
 
-Note: if there is error about "relic" or "crypto", trying force removing the relic build and reinstall the tools again:
+Note: Whenever the crypto module version imported by "go.mod" is updated to a version that was never locally imported before, the crypto dependency needs to be set-up. If not, you should notice errors about "relic" or "crypto". Run the following command to set-up the new module version:
 
 ```bash
-rm -rf crypto/relic
-make install-tools
+make crypto_setup_gopath
 ```
 
 ## Development Workflow