From 9a2f643b6c0f0ef9d1e2a1481cd3346c2e7c1fdf Mon Sep 17 00:00:00 2001 From: Daniel Olshansky Date: Wed, 30 Nov 2022 13:34:23 -0800 Subject: [PATCH] [Persistence] First Implementation of the StateHash (#284) (#285) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Description The first implementation of the ## Issue Fixes #284 with follow up work in #361. ## Type of change Please mark the relevant option(s): - [x] New feature, functionality or library - [x] Bug fix - [x] Code health or cleanup - [x] Major breaking change - [x] Documentation - [ ] Other ## List of changes ### Persistence - Core Changes for SMT * Introduced & defined for `block_persistence.proto` * A persistence specific protobuf for the Block stored in the BlockStore * On `Commit`, prepare and store a persistence block in the KV Store, SQL Store * Replace `IndexTransactions` (plural) to `IndexTransaction` (singular) * Maintaining a list of StateTrees using Celestia’s SMT and badger as the KV store to compute the state hash * Implemented `ComputeStateHash` to update the global state based on: * Validators * Applications * Servicers * Fisherman * Accounts * Pools * Transactions * Added a placeholder for `params` and `flags` * Added a benchmarking and a determinism test suite to validate this ### Persistence - General module changes * Implemented `GetAccountsUpdated`, `GetPoolsUpdated` and `GetActorsUpdated` functions * Removed `GetPrevAppHash` and `indexTransactions` functions * Removed `blockProtoBytes` and `txResults` from the local state and added `quorumCert` * Consolidate all `resetContext` related operations into a single function * Implemented `ReleaseWriteContext` * Implemented ability to `ClearAllState` and `ResetToGenesis` for debugging & testing purposes * Added unit tests for all of the supporting SQL functions implemented * Some improvements in unit test preparation & cleanup (limited to this PR's functionality) ### Persistence - KVStore changes * Renamed `Put` to `Set` * Embedded `smt.MapStore` in the interface containing `Get`, `Set` and `Delete` * Implemented `Delete` * Modified `GetAll` to return both `keys` and `values` * Turned off badger logging options since it’s noisy ### Persistence - Module Interface changes * Removed `GetPrevHash` and just using `GetBlockHash` instead * Removed `blockProtoBz` from `SetProposalBlock` interface * Removed `GetLatestBlockTxs` and `SetLatestTxResults` in exchange for `IndexTransaction` * Removed `SetTxResults` * Renamed `UpdateAppHash` to `ComputeStateHash` * Removed some getters related to the proposal block (`GetBlockTxs`, `GetBlockHash`, etc…) ### Consensus * Propagate `highPrepareQC` if available to the block being created * Remove `blockProtoBytes` from propagation in `SetProposalBlock` * Guarantee that `writeContext` is released when refreshing the `utilityContext` * Use `GetBlockHash(height)` instead of `GetPrevAppHash` to be more explicit * Use the real `quorumCert` when preparing a new block ### Configs * Updated the test generator to produce deterministic keys * Added `trees_store_dir` to persistence configs * Updated `LocalNet` configs to have an empty `tx_indexer_path` and `trees_store_dir` ### Makefile changes * Added `db_cli_node` * Added `db_show_schemas` * Added `test_persistence_state_hash` * Added `benchmark_persistence_state_hash` ### Debug * `ResetToGenesis` - Added the ability to reset the state to genesis * `ClearState` - Added the ability to clear the state completely (height 0 without genesis data) ## Testing **New:** - [x] `make benchmark_persistence_state_hash` - [x] `make test_persistence_state_hash` - [x] Iteration 3 demo was done using this PR: https://drive.google.com/file/d/1IOrzq-XJP04BJjyqPPpPu873aSfwrnur/view?usp=sharing **Existing:** - [x] `make develop_test` - [x] [LocalNet](https://github.com/pokt-network/pocket/blob/main/docs/development/README.md) w/ all of the steps outlined in the `README` ## Required Checklist - [x] I have performed a self-review of my own code - [x] I have commented my code, particularly in hard-to-understand areas - [x] I have tested my changes using the available tooling - [x] I have updated the corresponding CHANGELOG ### If Applicable Checklist - [ ] I have updated the corresponding README(s); local and/or global - [x] I have added tests that prove my fix is effective or that my feature works - [ ] I have added, or updated, [mermaid.js](https://mermaid-js.github.io) diagrams in the corresponding README(s) - [ ] I have added, or updated, documentation and [mermaid.js](https://mermaid-js.github.io) diagrams in `shared/docs/*` if I updated `shared/*`README(s) --- Co-authored-by: Alessandro De Blasis Co-authored-by: Irving A.J. Rivas Z. Co-authored-by: Andrew Nguyen Co-authored-by: Daniel Olshansky Co-authored-by: Jason You Co-authored-by: Dmitry Knyazev --- .github/workflows/main.yml | 33 +- Makefile | 58 +++- app/client/cli/debug.go | 7 +- build/config/config1.json | 4 +- build/config/config2.json | 4 +- build/config/config3.json | 4 +- build/config/config4.json | 4 +- build/sql/show_all_schemas.sql | 1 + consensus/block.go | 6 + consensus/consensus_tests/utils_test.go | 10 +- consensus/debugging.go | 2 +- consensus/doc/CHANGELOG.md | 9 + consensus/helpers.go | 6 +- consensus/hotstuff_leader.go | 31 +- consensus/hotstuff_replica.go | 12 +- consensus/module.go | 5 +- consensus/pacemaker.go | 1 - consensus/state_sync/state_sync.go | 2 +- go.mod | 5 +- go.sum | 6 +- p2p/raintree/peers_manager_test.go | 8 +- persistence/CHANGELOG.md | 39 +++ persistence/account.go | 37 ++- persistence/application.go | 4 + persistence/block.go | 98 +++--- persistence/context.go | 92 ++++-- persistence/db.go | 90 +---- persistence/debug.go | 125 +++++-- persistence/docs/PROTOCOL_STATE_HASH.md | 138 ++++++++ persistence/docs/README.md | 4 +- persistence/genesis.go | 29 +- persistence/indexer/indexer.go | 14 +- persistence/kvstore/kvstore.go | 54 ++- persistence/module.go | 68 ++-- persistence/proto/block_persistence.proto | 14 + persistence/proto/persistence_config.proto | 1 + persistence/shared_sql.go | 50 ++- persistence/state.go | 363 +++++++++++++++++++++ persistence/test/account_test.go | 102 +++++- persistence/test/application_test.go | 7 + persistence/test/benchmark_state_test.go | 174 ++++++++++ persistence/test/block_test.go | 25 ++ persistence/test/fisherman_test.go | 7 + persistence/test/generic_test.go | 55 +++- persistence/test/module_test.go | 24 +- persistence/test/service_node_test.go | 7 + persistence/test/setup_test.go | 75 +++-- persistence/test/state_test.go | 237 ++++++++++++++ persistence/test/validator_test.go | 9 +- persistence/types/account.go | 16 + persistence/types/base_actor.go | 5 + persistence/types/protocol_actor.go | 3 + persistence/types/shared_sql.go | 5 + runtime/test_artifacts/generator.go | 64 +++- runtime/test_artifacts/util.go | 8 +- shared/CHANGELOG.md | 15 +- shared/crypto/ed25519.go | 6 + shared/docs/PROTOCOL_STATE_HASH.md | 8 +- shared/messaging/envelope_test.go | 2 +- shared/messaging/proto/debug_message.proto | 18 +- shared/modules/doc/CHANGELOG.md | 9 + shared/modules/persistence_module.go | 32 +- shared/modules/types.go | 1 + shared/modules/utility_module.go | 1 + shared/node.go | 3 + utility/block.go | 57 ++-- utility/context.go | 5 +- utility/doc/CHANGELOG.md | 4 + utility/test/actor_test.go | 58 +++- utility/test/block_test.go | 12 +- utility/test/module_test.go | 57 ++-- utility/types/message_test.go | 2 +- 72 files changed, 2066 insertions(+), 485 deletions(-) create mode 100644 build/sql/show_all_schemas.sql create mode 100644 persistence/docs/PROTOCOL_STATE_HASH.md create mode 100644 persistence/proto/block_persistence.proto create mode 100644 persistence/state.go create mode 100644 persistence/test/benchmark_state_test.go create mode 100644 persistence/test/block_test.go create mode 100644 persistence/test/state_test.go diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 0dbaded3e..a13fac5aa 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -50,14 +50,34 @@ jobs: run: make install_cli_deps - name: generate protobufs, RPC server, RPC client and mocks run: make protogen_local && make mockgen && make generate_rpc_openapi - - name: run all tests - run: make test_all_with_json + - name: Create coverage report and run tests + # Not utilizing makefile target here to make use of pipefail bash feature. + run: | + set -euo pipefail + go test -p 1 -json ./... -covermode=count -coverprofile=coverage.out 2>&1 | tee test_results.json + - name: Output test failures + # Makes it easier to find failed tests so no need to scroll through the whole log. + if: ${{ failure() && env.TARGET_GOLANG_VERSION == matrix.go }} + run: cat test_results.json | jq 'select(.Action == "fail")' + - name: Upload test results + if: ${{ always() && env.TARGET_GOLANG_VERSION == matrix.go }} + uses: actions/upload-artifact@v3 + with: + name: test-results + path: | + test_results.json - name: Annotate tests on GitHub # Only annotate if the test failed on target version to avoid duplicated annotations on GitHub. if: ${{ always() && env.TARGET_GOLANG_VERSION == matrix.go }} uses: guyarb/golang-test-annotations@v0.5.1 with: test-results: test_results.json + - name: Prepare code coverage report + if: ${{ always() && env.TARGET_GOLANG_VERSION == matrix.go }} + run: go tool cover -func=coverage.out -o=coverage.out + - name: Upload coverage to Codecov + if: ${{ always() && env.TARGET_GOLANG_VERSION == matrix.go }} + uses: codecov/codecov-action@v3 - name: Run golangci-lint # Only run if the test failed on target version to avoid duplicated annotations on GitHub. if: ${{ always() && env.TARGET_GOLANG_VERSION == matrix.go }} @@ -65,18 +85,15 @@ jobs: with: # only-new-issues: true args: --issues-exit-code=0 # TODO: Remove this once we fix all the issues. - - name: create coverage report - if: ${{ always() && env.TARGET_GOLANG_VERSION == matrix.go }} - run: make test_all_with_coverage - - name: Upload coverage to Codecov - if: ${{ always() && env.TARGET_GOLANG_VERSION == matrix.go }} - uses: codecov/codecov-action@v3 # TODO(@okdas): reuse artifacts built by the previous job instead # of going through the build process in container build job again # - figure out how to handle musl/alpine case if we want to support it build-images: runs-on: ubuntu-latest + needs: test-multiple-go-versions + # Until we have developer environments, we don't need the images built on other that main branches. + if: github.ref == 'refs/heads/main' strategy: matrix: # Build dev & prod images diff --git a/Makefile b/Makefile index cd8a09c2b..9ae99f784 100644 --- a/Makefile +++ b/Makefile @@ -8,6 +8,8 @@ CWD ?= CURRENT_WORKING_DIRECTIONRY_NOT_SUPPLIED # seconds, and fail if any additional messages are received. EXTRA_MSG_FAIL ?= false +# IMPROVE: Add `-shuffle=on` to the `go test` command to randomize the order in which tests are run. + # An easy way to turn off verbose test output for some of the test targets. For example # `$ make test_persistence` by default enables verbose testing # `VERBOSE_TEST="" make test_persistence` is an easy way to run the same tests without verbose output @@ -162,6 +164,14 @@ db_cli: echo "View schema by running 'SELECT schema_name FROM information_schema.schemata;'" docker exec -it pocket-db bash -c "psql -U postgres" +psqlSchema ?= node1 + +.PHONY: db_cli_node +## Open a CLI to the local containerized postgres instance for a specific node +db_cli_node: + echo "View all avialable tables by running \dt" + docker exec -it pocket-db bash -c "PGOPTIONS=--search_path=${psqlSchema} psql -U postgres" + .PHONY: db_drop ## Drop all schemas used for LocalNet development matching `node%` db_drop: docker_check @@ -177,6 +187,11 @@ db_bench_init: docker_check db_bench: docker_check docker exec -it pocket-db bash -c "pgbench -U postgres -d postgres" +.PHONY: db_show_schemas +## Show all the node schemas in the local SQL DB +db_show_schemas: docker_check + docker exec -it pocket-db bash -c "psql -U postgres -d postgres -a -f /tmp/scripts/show_all_schemas.sql" + .PHONY: db_admin ## Helper to access to postgres admin GUI interface db_admin: @@ -252,7 +267,7 @@ protogen_local: go_protoc-go-inject-tag protoc --go_opt=paths=source_relative -I=./p2p/types/proto --go_out=./p2p/types ./p2p/types/proto/*.proto --experimental_allow_proto3_optional protoc --go_opt=paths=source_relative -I=./telemetry/proto --go_out=./telemetry ./telemetry/proto/*.proto --experimental_allow_proto3_optional protoc --go_opt=paths=source_relative -I=./logger/proto --go_out=./logger ./logger/proto/*.proto --experimental_allow_proto3_optional - protoc --go_opt=paths=source_relative -I=./rpc/types/proto --go_out=./rpc/types ./rpc/types/proto/*.proto --experimental_allow_proto3_optional + protoc --go_opt=paths=source_relative -I=./rpc/types/proto --go_out=./rpc/types ./rpc/types/proto/*.proto --experimental_allow_proto3_optional echo "View generated proto files by running: make protogen_show" .PHONY: protogen_docker_m1 @@ -290,16 +305,10 @@ generate_cli_commands_docs: test_all: # generate_mocks go test -p 1 -count=1 ./... -.PHONY: test_all_with_json -## Run all go unit tests, output results in json file -test_all_with_json: generate_rpc_openapi # generate_mocks - go test -p 1 -json ./... > test_results.json - -.PHONY: test_all_with_coverage -## Run all go unit tests, output results & coverage into files -test_all_with_coverage: generate_rpc_openapi # generate_mocks - go test -p 1 -v ./... -covermode=count -coverprofile=coverage.out - go tool cover -func=coverage.out -o=coverage.out +.PHONY: test_all_with_json_coverage +## Run all go unit tests, output results & coverage into json & coverage files +test_all_with_json_coverage: generate_rpc_openapi # generate_mocks + go test -p 1 -json ./... -covermode=count -coverprofile=coverage.out | tee test_results.json | jq .PHONY: test_race ## Identify all unit tests that may result in race conditions @@ -317,9 +326,9 @@ test_shared: # generate_mocks go test ${VERBOSE_TEST} -p 1 ./shared/... .PHONY: test_consensus -## Run all go unit tests in the Consensus module +## Run all go unit tests in the consensus module test_consensus: # mockgen - go test ${VERBOSE_TEST} ./consensus/... + go test ${VERBOSE_TEST} -count=1 ./consensus/... .PHONY: test_consensus_concurrent_tests ## Run unit tests in the consensus module that could be prone to race conditions (#192) @@ -354,6 +363,11 @@ test_sortition: test_persistence: go test ${VERBOSE_TEST} -p 1 -count=1 ./persistence/... +.PHONY: test_persistence_state_hash +## Run all go unit tests in the Persistence module related to the state hash +test_persistence_state_hash: + go test ${VERBOSE_TEST} -run TestStateHash -count=1 ./persistence/... + .PHONY: test_p2p ## Run all p2p test_p2p: @@ -362,22 +376,29 @@ test_p2p: .PHONY: test_p2p_raintree ## Run all p2p raintree related tests test_p2p_raintree: - go test -run RainTreeNetwork -v -count=1 ./p2p/... + go test ${VERBOSE_TEST} -run RainTreeNetwork -count=1 ./p2p/... .PHONY: test_p2p_raintree_addrbook ## Run all p2p raintree addr book related tests test_p2p_raintree_addrbook: - go test -run RainTreeAddrBook -v -count=1 ./p2p/... + go test ${VERBOSE_TEST} -run RainTreeAddrBook -count=1 ./p2p/... + +# TIP: For benchmarks, consider appending `-run=^#` to avoid running unit tests in the same package + +.PHONY: benchmark_persistence_state_hash +## Benchmark the state hash computation +benchmark_persistence_state_hash: + go test ${VERBOSE_TEST} -cpu 1,2 -benchtime=1s -benchmem -bench=. -run BenchmarkStateHash -count=1 ./persistence/... .PHONY: benchmark_sortition ## Benchmark the Sortition library benchmark_sortition: - go test ${VERBOSE_TEST} ./consensus/leader_election/sortition -bench=. + go test ${VERBOSE_TEST} -bench=. -run ^# ./consensus/leader_election/sortition .PHONY: benchmark_p2p_addrbook ## Benchmark all P2P addr book related tests benchmark_p2p_addrbook: - go test -bench=. -run BenchmarkAddrBook -v -count=1 ./p2p/... + go test ${VERBOSE_TEST} -bench=. -run BenchmarkAddrBook -count=1 ./p2p/... ### Inspired by @goldinguy_ in this post: https://goldin.io/blog/stop-using-todo ### # TODO - General Purpose catch-all. @@ -391,10 +412,11 @@ benchmark_p2p_addrbook: # REFACTOR - Similar to TECHDEBT, but will require a substantial rewrite and change across the codebase # CONSIDERATION - A comment that involves extra work but was thoughts / considered as part of some implementation # CONSOLIDATE - We likely have similar implementations/types of the same thing, and we should consolidate them. +# ADDTEST - Add more tests for a specific code section # DEPRECATE - Code that should be removed in the future # DISCUSS_IN_THIS_COMMIT - SHOULD NEVER BE COMMITTED TO MASTER. It is a way for the reviewer of a PR to start / reply to a discussion. # TODO_IN_THIS_COMMIT - SHOULD NEVER BE COMMITTED TO MASTER. It is a way to start the review process while non-critical changes are still in progress -TODO_KEYWORDS = -e "TODO" -e "TECHDEBT" -e "IMPROVE" -e "DISCUSS" -e "INCOMPLETE" -e "INVESTIGATE" -e "CLEANUP" -e "HACK" -e "REFACTOR" -e "CONSIDERATION" -e "TODO_IN_THIS_COMMIT" -e "DISCUSS_IN_THIS_COMMIT" -e "CONSOLIDATE" -e "DEPRECATE" +TODO_KEYWORDS = -e "TODO" -e "TECHDEBT" -e "IMPROVE" -e "DISCUSS" -e "INCOMPLETE" -e "INVESTIGATE" -e "CLEANUP" -e "HACK" -e "REFACTOR" -e "CONSIDERATION" -e "TODO_IN_THIS_COMMIT" -e "DISCUSS_IN_THIS_COMMIT" -e "CONSOLIDATE" -e "DEPRECATE" -e "ADDTEST" # How do I use TODOs? # 1. : ; diff --git a/app/client/cli/debug.go b/app/client/cli/debug.go index bcf52c2c7..2aeb64a2d 100644 --- a/app/client/cli/debug.go +++ b/app/client/cli/debug.go @@ -23,6 +23,7 @@ import ( "google.golang.org/protobuf/types/known/anypb" ) +// TECHDEBT: Lowercase variables / constants that do not need to be exported. const ( PromptResetToGenesis string = "ResetToGenesis" PromptPrintNodeState string = "PrintNodeState" @@ -207,10 +208,10 @@ func initDebug(remoteCLIURL string) { logger.Global.Fatal().Err(err).Msg("Failed to create logger module") } loggerMod := loggerM.(modules.LoggerModule) - - rpcM, err := rpc.Create(runtimeMgr) + + rpcM, err := rpc.Create(runtimeMgr) if err != nil { - logger.Global.Fatal().Err(err).Msg("Failed to create rpc module") + logger.Global.Fatal().Err(err).Msg("Failed to create rpc module") } rpcMod := rpcM.(modules.RPCModule) diff --git a/build/config/config1.json b/build/config/config1.json index 708c19a16..359328cb5 100644 --- a/build/config/config1.json +++ b/build/config/config1.json @@ -19,7 +19,9 @@ "persistence": { "postgres_url": "postgres://postgres:postgres@pocket-db:5432/postgres", "node_schema": "node1", - "block_store_path": "/var/blockstore" + "block_store_path": "/var/blockstore", + "tx_indexer_path": "", + "trees_store_dir": "/var/trees" }, "p2p": { "consensus_port": 8080, diff --git a/build/config/config2.json b/build/config/config2.json index d32475f78..3b60ffa62 100644 --- a/build/config/config2.json +++ b/build/config/config2.json @@ -19,7 +19,9 @@ "persistence": { "postgres_url": "postgres://postgres:postgres@pocket-db:5432/postgres", "node_schema": "node2", - "block_store_path": "/var/blockstore" + "block_store_path": "/var/blockstore", + "tx_indexer_path": "", + "trees_store_dir": "/var/trees" }, "p2p": { "consensus_port": 8080, diff --git a/build/config/config3.json b/build/config/config3.json index 1d2a05016..54140b350 100644 --- a/build/config/config3.json +++ b/build/config/config3.json @@ -19,7 +19,9 @@ "persistence": { "postgres_url": "postgres://postgres:postgres@pocket-db:5432/postgres", "node_schema": "node3", - "block_store_path": "/var/blockstore" + "block_store_path": "/var/blockstore", + "tx_indexer_path": "", + "trees_store_dir": "/var/trees" }, "p2p": { "consensus_port": 8080, diff --git a/build/config/config4.json b/build/config/config4.json index 261b65eb4..333ebad2e 100644 --- a/build/config/config4.json +++ b/build/config/config4.json @@ -19,7 +19,9 @@ "persistence": { "postgres_url": "postgres://postgres:postgres@pocket-db:5432/postgres", "node_schema": "node4", - "block_store_path": "/var/blockstore" + "block_store_path": "/var/blockstore", + "tx_indexer_path": "", + "trees_store_dir": "/var/trees" }, "p2p": { "consensus_port": 8080, diff --git a/build/sql/show_all_schemas.sql b/build/sql/show_all_schemas.sql new file mode 100644 index 000000000..b50e4fb4b --- /dev/null +++ b/build/sql/show_all_schemas.sql @@ -0,0 +1 @@ +SELECT schema_name FROM information_schema.schemata; \ No newline at end of file diff --git a/consensus/block.go b/consensus/block.go index 3ad4f327e..53aa4e8af 100644 --- a/consensus/block.go +++ b/consensus/block.go @@ -63,6 +63,12 @@ func (m *consensusModule) refreshUtilityContext() error { m.utilityContext = nil } + // Only one write context can exist at a time, and the utility context needs to instantiate + // a new one to modify the state. + if err := m.GetBus().GetPersistenceModule().ReleaseWriteContext(); err != nil { + log.Printf("[WARN] Error releasing persistence write context: %v\n", err) + } + utilityContext, err := m.GetBus().GetUtilityModule().NewContext(int64(m.Height)) if err != nil { return err diff --git a/consensus/consensus_tests/utils_test.go b/consensus/consensus_tests/utils_test.go index 3b0aab2fc..8fe54c90b 100644 --- a/consensus/consensus_tests/utils_test.go +++ b/consensus/consensus_tests/utils_test.go @@ -141,8 +141,8 @@ func CreateTestConsensusPocketNode( utilityMock := baseUtilityMock(t, testChannel) telemetryMock := baseTelemetryMock(t, testChannel) loggerMock := baseLoggerMock(t, testChannel) - rpcMock := baseRpcMock(t, testChannel) - + rpcMock := baseRpcMock(t, testChannel) + bus, err := shared.CreateBus(runtimeMgr, persistenceMock, p2pMock, utilityMock, consensusMod.(modules.ConsensusModule), telemetryMock, loggerMock, rpcMock) require.NoError(t, err) @@ -157,6 +157,7 @@ func CreateTestConsensusPocketNode( return pocketNode } +// CLEANUP: Reduce package scope visibility in the consensus test module func StartAllTestPocketNodes(t *testing.T, pocketNodes IdToNodeMapping) { for _, pocketNode := range pocketNodes { go pocketNode.Start() @@ -307,6 +308,7 @@ func basePersistenceMock(t *testing.T, _ modules.EventsChannel) *modulesMock.Moc persistenceMock.EXPECT().Start().Return(nil).AnyTimes() persistenceMock.EXPECT().SetBus(gomock.Any()).Return().AnyTimes() persistenceMock.EXPECT().NewReadContext(int64(-1)).Return(persistenceContextMock, nil).AnyTimes() + persistenceMock.EXPECT().ReleaseWriteContext().Return(nil).AnyTimes() // The persistence context should usually be accessed via the utility module within the context // of the consensus module. This one is only used when loading the initial consensus module @@ -363,7 +365,7 @@ func baseUtilityContextMock(t *testing.T) *modulesMock.MockUtilityContext { utilityContextMock := modulesMock.NewMockUtilityContext(ctrl) persistenceContextMock := modulesMock.NewMockPersistenceRWContext(ctrl) persistenceContextMock.EXPECT().SetProposalBlock(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes() - persistenceContextMock.EXPECT().GetPrevAppHash().Return("", nil).AnyTimes() + persistenceContextMock.EXPECT().GetBlockHash(gomock.Any()).Return([]byte(""), nil).AnyTimes() utilityContextMock.EXPECT(). CreateAndApplyProposalBlock(gomock.Any(), maxTxBytes). @@ -377,6 +379,8 @@ func baseUtilityContextMock(t *testing.T) *modulesMock.MockUtilityContext { utilityContextMock.EXPECT().Release().Return(nil).AnyTimes() utilityContextMock.EXPECT().GetPersistenceContext().Return(persistenceContextMock).AnyTimes() + persistenceContextMock.EXPECT().Release().Return(nil).AnyTimes() + return utilityContextMock } diff --git a/consensus/debugging.go b/consensus/debugging.go index 4cf015ebf..7db4f63d9 100644 --- a/consensus/debugging.go +++ b/consensus/debugging.go @@ -48,7 +48,7 @@ func (m *consensusModule) resetToGenesis(_ *messaging.DebugMessage) { m.clearLeader() m.clearMessagesPool() m.GetBus().GetPersistenceModule().HandleDebugMessage(&messaging.DebugMessage{ - Action: messaging.DebugMessageAction_DEBUG_CLEAR_STATE, + Action: messaging.DebugMessageAction_DEBUG_PERSISTENCE_RESET_TO_GENESIS, Message: nil, }) m.GetBus().GetPersistenceModule().Start() // reload genesis state diff --git a/consensus/doc/CHANGELOG.md b/consensus/doc/CHANGELOG.md index b3da7171e..074b7e69b 100644 --- a/consensus/doc/CHANGELOG.md +++ b/consensus/doc/CHANGELOG.md @@ -7,7 +7,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.0.0.10] - 2022-11-30 + +- Propagate `highPrepareQC` if available to the block being created +- Remove `blockProtoBytes` from propagation in `SetProposalBlock` +- Guarantee that write context is released when refreshing the utility context +- Use `GetBlockHash(height)` instead of `GetPrevAppHash` to be more explicit +- Use the real `quorumCert` when preparing a new block + ## [0.0.0.9] - 2022-11-30 + - Added state sync interfaces and diagrams ## [0.0.0.8] - 2022-11-15 diff --git a/consensus/helpers.go b/consensus/helpers.go index 9ff77f182..3eb8c1e49 100644 --- a/consensus/helpers.go +++ b/consensus/helpers.go @@ -5,12 +5,10 @@ import ( "encoding/base64" "log" - "github.com/pokt-network/pocket/shared/codec" - - "google.golang.org/protobuf/proto" - typesCons "github.com/pokt-network/pocket/consensus/types" + "github.com/pokt-network/pocket/shared/codec" cryptoPocket "github.com/pokt-network/pocket/shared/crypto" + "google.golang.org/protobuf/proto" ) // These constants and variables are wrappers around the autogenerated protobuf types and were diff --git a/consensus/hotstuff_leader.go b/consensus/hotstuff_leader.go index 324639187..6c749e6a0 100644 --- a/consensus/hotstuff_leader.go +++ b/consensus/hotstuff_leader.go @@ -4,10 +4,9 @@ import ( "encoding/hex" "unsafe" - "github.com/pokt-network/pocket/shared/codec" - consensusTelemetry "github.com/pokt-network/pocket/consensus/telemetry" typesCons "github.com/pokt-network/pocket/consensus/types" + "github.com/pokt-network/pocket/shared/codec" ) type HotstuffLeaderMessageHandler struct{} @@ -55,8 +54,7 @@ func (handler *HotstuffLeaderMessageHandler) HandleNewRoundMessage(m *consensusM // TODO: Add test to make sure same block is not applied twice if round is interrupted after being 'Applied'. // TODO: Add more unit tests for these checks... if m.shouldPrepareNewBlock(highPrepareQC) { - // Leader prepares a new block if `highPrepareQC` is not applicable - block, err := m.prepareAndApplyBlock() + block, err := m.prepareAndApplyBlock(highPrepareQC) if err != nil { m.nodeLogError(typesCons.ErrPrepareBlock.Error(), err) m.paceMaker.InterruptRound() @@ -334,7 +332,7 @@ func (m *consensusModule) tempIndexHotstuffMessage(msg *typesCons.HotstuffMessag // This is a helper function intended to be called by a leader/validator during a view change // to prepare a new block that is applied to the new underlying context. -func (m *consensusModule) prepareAndApplyBlock() (*typesCons.Block, error) { +func (m *consensusModule) prepareAndApplyBlock(qc *typesCons.QuorumCertificate) (*typesCons.Block, error) { if m.isReplica() { return nil, typesCons.ErrReplicaPrepareBlock } @@ -350,7 +348,13 @@ func (m *consensusModule) prepareAndApplyBlock() (*typesCons.Block, error) { persistenceContext := m.utilityContext.GetPersistenceContext() - prevAppHash, err := persistenceContext.GetPrevAppHash() + // CONSOLIDATE: Last/Prev & AppHash/StateHash/BlockHash + prevAppHash, err := persistenceContext.GetBlockHash(int64(m.Height) - 1) + if err != nil { + return nil, err + } + + qcBytes, err := codec.GetCodec().Marshal(qc) if err != nil { return nil, err } @@ -360,30 +364,25 @@ func (m *consensusModule) prepareAndApplyBlock() (*typesCons.Block, error) { Height: int64(m.Height), Hash: hex.EncodeToString(appHash), NumTxs: uint32(len(txs)), - LastBlockHash: prevAppHash, // IMRPROVE: this should be a block hash not the appHash + LastBlockHash: hex.EncodeToString(prevAppHash), ProposerAddress: m.privateKey.Address().Bytes(), - QuorumCertificate: []byte("HACK: Temporary placeholder"), + QuorumCertificate: qcBytes, } block := &typesCons.Block{ BlockHeader: blockHeader, Transactions: txs, } - cdc := codec.GetCodec() - blockProtoBz, err := cdc.Marshal(block) - if err != nil { - return nil, err - } - // Set the proposal block in the persistence context - if err = persistenceContext.SetProposalBlock(blockHeader.Hash, blockProtoBz, blockHeader.ProposerAddress, block.Transactions); err != nil { + if err = persistenceContext.SetProposalBlock(blockHeader.Hash, blockHeader.ProposerAddress, blockHeader.QuorumCertificate, block.Transactions); err != nil { return nil, err } return block, nil } -// Return true if this node, the leader, should prepare a new block +// Return true if this node, the leader, should prepare a new block. +// ADDTEST: Add more tests for all the different scenarios here func (m *consensusModule) shouldPrepareNewBlock(highPrepareQC *typesCons.QuorumCertificate) bool { if highPrepareQC == nil { m.nodeLog("Preparing a new block - no highPrepareQC found") diff --git a/consensus/hotstuff_replica.go b/consensus/hotstuff_replica.go index 0f57c88d8..9373f9549 100644 --- a/consensus/hotstuff_replica.go +++ b/consensus/hotstuff_replica.go @@ -7,7 +7,6 @@ import ( consensusTelemetry "github.com/pokt-network/pocket/consensus/telemetry" "github.com/pokt-network/pocket/consensus/types" typesCons "github.com/pokt-network/pocket/consensus/types" - "github.com/pokt-network/pocket/shared/codec" ) type HotstuffReplicaMessageHandler struct{} @@ -227,13 +226,10 @@ func (m *consensusModule) validateProposal(msg *typesCons.HotstuffMessage) error // This helper applies the block metadata to the utility & persistence layers func (m *consensusModule) applyBlock(block *typesCons.Block) error { - blockProtoBz, err := codec.GetCodec().Marshal(block) - if err != nil { - return err - } persistenceContext := m.utilityContext.GetPersistenceContext() + blockHeader := block.BlockHeader // Set the proposal block in the persistence context - if err = persistenceContext.SetProposalBlock(block.BlockHeader.Hash, blockProtoBz, block.BlockHeader.ProposerAddress, block.Transactions); err != nil { + if err := persistenceContext.SetProposalBlock(blockHeader.Hash, blockHeader.ProposerAddress, blockHeader.QuorumCertificate, block.Transactions); err != nil { return err } @@ -244,8 +240,8 @@ func (m *consensusModule) applyBlock(block *typesCons.Block) error { } // CONSOLIDATE: Terminology of `appHash` and `stateHash` - if block.BlockHeader.Hash != hex.EncodeToString(appHash) { - return typesCons.ErrInvalidAppHash(block.BlockHeader.Hash, hex.EncodeToString(appHash)) + if appHashString := hex.EncodeToString(appHash); blockHeader.Hash != appHashString { + return typesCons.ErrInvalidAppHash(blockHeader.Hash, appHashString) } return nil diff --git a/consensus/module.go b/consensus/module.go index f8b52d1f4..ecab90edd 100644 --- a/consensus/module.go +++ b/consensus/module.go @@ -7,13 +7,12 @@ import ( "sync" "github.com/pokt-network/pocket/consensus/leader_election" + consensusTelemetry "github.com/pokt-network/pocket/consensus/telemetry" typesCons "github.com/pokt-network/pocket/consensus/types" "github.com/pokt-network/pocket/shared/codec" cryptoPocket "github.com/pokt-network/pocket/shared/crypto" - "google.golang.org/protobuf/types/known/anypb" - - consensusTelemetry "github.com/pokt-network/pocket/consensus/telemetry" "github.com/pokt-network/pocket/shared/modules" + "google.golang.org/protobuf/types/known/anypb" ) const ( diff --git a/consensus/pacemaker.go b/consensus/pacemaker.go index 45a6839e7..049df6516 100644 --- a/consensus/pacemaker.go +++ b/consensus/pacemaker.go @@ -8,7 +8,6 @@ import ( consensusTelemetry "github.com/pokt-network/pocket/consensus/telemetry" typesCons "github.com/pokt-network/pocket/consensus/types" - "github.com/pokt-network/pocket/shared/modules" ) diff --git a/consensus/state_sync/state_sync.go b/consensus/state_sync/state_sync.go index 5ca419853..fac579bee 100644 --- a/consensus/state_sync/state_sync.go +++ b/consensus/state_sync/state_sync.go @@ -6,7 +6,7 @@ import ( "github.com/pokt-network/pocket/shared/modules" ) -TODO(#362): Update the interface so it can be easily integrated with the app specific bus +// TODO(#362): Update the interface so it can be easily integrated with the app specific bus type StateSyncModule interface { modules.Module diff --git a/go.mod b/go.mod index 7291e0203..b0f541362 100644 --- a/go.mod +++ b/go.mod @@ -20,8 +20,9 @@ require ( require ( github.com/benbjohnson/clock v1.3.0 + github.com/celestiaorg/smt v0.2.1-0.20220414134126-dba215ccb884 github.com/dgraph-io/badger/v3 v3.2103.2 - github.com/getkin/kin-openapi v0.106.0 + github.com/getkin/kin-openapi v0.107.0 github.com/jackc/pgconn v1.13.0 github.com/jordanorelli/lexnum v0.0.0-20141216151731-460eeb125754 github.com/labstack/echo/v4 v4.9.1 @@ -45,7 +46,6 @@ require ( github.com/docker/go-connections v0.4.0 // indirect github.com/docker/go-units v0.5.0 // indirect github.com/dustin/go-humanize v1.0.0 // indirect - github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/glog v1.0.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.2 @@ -84,6 +84,7 @@ require ( github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/go-openapi/jsonpointer v0.19.5 // indirect github.com/go-openapi/swag v0.19.5 // indirect + github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt v3.2.2+incompatible // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/invopop/yaml v0.1.0 // indirect diff --git a/go.sum b/go.sum index 5c4af8d35..71f1933cc 100644 --- a/go.sum +++ b/go.sum @@ -64,6 +64,8 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/celestiaorg/smt v0.2.1-0.20220414134126-dba215ccb884 h1:iRNKw2WmAbVgGMNYzDH5Y2yY3+jyxwEK9Hc5pwIjZAE= +github.com/celestiaorg/smt v0.2.1-0.20220414134126-dba215ccb884/go.mod h1:/sdYDakowo/XaxS2Fl7CBqtuf/O2uTqF2zmAUFAtAiw= github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= @@ -128,8 +130,8 @@ github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= -github.com/getkin/kin-openapi v0.106.0 h1:hrqfqJPAvWvuO/V0lCr/xyQOq4Gy21mcr28JJOSRcEI= -github.com/getkin/kin-openapi v0.106.0/go.mod h1:9Dhr+FasATJZjS4iOLvB0hkaxgYdulrNYm2e9epLWOo= +github.com/getkin/kin-openapi v0.107.0 h1:bxhL6QArW7BXQj8NjXfIJQy680NsMKd25nwhvpCXchg= +github.com/getkin/kin-openapi v0.107.0/go.mod h1:9Dhr+FasATJZjS4iOLvB0hkaxgYdulrNYm2e9epLWOo= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= diff --git a/p2p/raintree/peers_manager_test.go b/p2p/raintree/peers_manager_test.go index 4d37bbf71..b836c16c8 100644 --- a/p2p/raintree/peers_manager_test.go +++ b/p2p/raintree/peers_manager_test.go @@ -100,7 +100,7 @@ func BenchmarkAddrBookUpdates(b *testing.B) { // the test will add this arbitrary number of addresses after the initial initialization (done via NewRainTreeNetwork) // this is to add extra subsequent work that -should- grow linearly and it's actually going to test AddressBook updates // not simply initializations. - numAddressessToBeAdded := 1000 + numAddressesToBeAdded := 1000 for _, testCase := range testCases { n := testCase.numNodes @@ -115,7 +115,7 @@ func BenchmarkAddrBookUpdates(b *testing.B) { require.Equal(b, n, len(peersManagerStateView.addrBookMap)) require.Equal(b, testCase.numExpectedLevels, int(peersManagerStateView.maxNumLevels)) - for i := 0; i < numAddressessToBeAdded; i++ { + for i := 0; i < numAddressesToBeAdded; i++ { newAddr, err := crypto.GenerateAddress() require.NoError(b, err) network.AddPeerToAddrBook(&types.NetworkPeer{Address: newAddr}) @@ -123,8 +123,8 @@ func BenchmarkAddrBookUpdates(b *testing.B) { peersManagerStateView = network.peersManager.getNetworkView() - require.Equal(b, n+numAddressessToBeAdded, len(peersManagerStateView.addrList)) - require.Equal(b, n+numAddressessToBeAdded, len(peersManagerStateView.addrBookMap)) + require.Equal(b, n+numAddressesToBeAdded, len(peersManagerStateView.addrList)) + require.Equal(b, n+numAddressesToBeAdded, len(peersManagerStateView.addrBookMap)) }) } } diff --git a/persistence/CHANGELOG.md b/persistence/CHANGELOG.md index a22f7a916..fb184e4f3 100644 --- a/persistence/CHANGELOG.md +++ b/persistence/CHANGELOG.md @@ -9,6 +9,45 @@ TODO: consolidate `persistence/docs/CHANGELOG` and `persistence/CHANGELOG.md` ## [Unreleased] +## [0.0.0.9] - 2022-11-30 + +Core StateHash changes + +- Introduced & defined for `block_persistence.proto` + - A persistence specific protobuf for the Block stored in the BlockStore +- On `Commit`, prepare and store a persistence block in the KV Store, SQL Store +- Replace `IndexTransactions` (plural) to `IndexTransaction` (singular) +- Maintaining a list of StateTrees using Celestia’s SMT and badger as the KV store to compute the state hash +- Implemented `ComputeStateHash` to update the global state based on: + - Validators + - Applications + - Servicers + - Fisherman + - Accounts + - Pools + - Transactions + - Added a placeholder for `params` and `flags` +- Added a benchmarking and a determinism test suite to validate this + +Supporting StateHash changes + +- Implemented `GetAccountsUpdated`, `GetPoolsUpdated` and `GetActorsUpdated` functions +- Removed `GetPrevAppHash` and `indexTransactions` functions +- Removed `blockProtoBytes` and `txResults` from the local state and added `quorumCert` +- Consolidate all `resetContext` related operations into a single function +- Implemented `ReleaseWriteContext` +- Implemented ability to `ClearAllState` and `ResetToGenesis` for debugging & testing purposes +- Added unit tests for all of the supporting SQL functions implemented +- Some improvements in unit test preparation & cleanup (limited to this PR's functionality) + +KVStore changes + +- Renamed `Put` to `Set` +- Embedded `smt.MapStore` in the interface containing `Get`, `Set` and `Delete` +- Implemented `Delete` +- Modified `GetAll` to return both `keys` and `values` +- Turned off badger logging options since it’s noisy + ## [0.0.0.8] - 2022-11-15 - Rename `GetBlockHash` to `GetBlockHashAtHeight` diff --git a/persistence/account.go b/persistence/account.go index 75737413a..63229f4f7 100644 --- a/persistence/account.go +++ b/persistence/account.go @@ -68,6 +68,10 @@ func (p PostgresContext) SetAccountAmount(address []byte, amount string) error { return nil } +func (p PostgresContext) GetAccountsUpdated(height int64) (accounts []*types.Account, err error) { + return p.getPoolOrAccUpdatedInternal(types.GetAccountsUpdatedAtHeightQuery(height)) +} + func (p *PostgresContext) operationAccountAmount(address []byte, deltaAmount string, op func(*big.Int, *big.Int) error) error { return p.operationPoolOrAccAmount(hex.EncodeToString(address), deltaAmount, op, p.getAccountAmountStr, types.InsertAccountAmountQuery) } @@ -142,7 +146,38 @@ func (p *PostgresContext) operationPoolAmount(name string, amount string, op fun return p.operationPoolOrAccAmount(name, amount, op, p.GetPoolAmount, types.InsertPoolAmountQuery) } -func (p *PostgresContext) operationPoolOrAccAmount(name, amount string, +func (p PostgresContext) GetPoolsUpdated(height int64) ([]*types.Account, error) { + return p.getPoolOrAccUpdatedInternal(types.GetPoolsUpdatedAtHeightQuery(height)) +} + +// Joint Pool & Account Helpers + +// Shared logic between `getPoolsUpdated` & `getAccountsUpdated` to keep explicit external interfaces +func (p *PostgresContext) getPoolOrAccUpdatedInternal(query string) (accounts []*types.Account, err error) { + ctx, tx, err := p.getCtxAndTx() + if err != nil { + return + } + + rows, err := tx.Query(ctx, query) + if err != nil { + return nil, err + } + defer rows.Close() + + for rows.Next() { + account := new(types.Account) + if err = rows.Scan(&account.Address, &account.Amount); err != nil { + return nil, err + } + accounts = append(accounts, account) + } + + return +} + +func (p *PostgresContext) operationPoolOrAccAmount( + name, amount string, op func(*big.Int, *big.Int) error, getAmount func(string, int64) (string, error), insert func(name, amount string, height int64) string) error { diff --git a/persistence/application.go b/persistence/application.go index 1c8518009..414286550 100644 --- a/persistence/application.go +++ b/persistence/application.go @@ -2,6 +2,7 @@ package persistence import ( "encoding/hex" + "github.com/pokt-network/pocket/persistence/types" "github.com/pokt-network/pocket/shared/modules" ) @@ -12,6 +13,9 @@ func (p PostgresContext) GetAppExists(address []byte, height int64) (exists bool func (p PostgresContext) GetApp(address []byte, height int64) (operator, publicKey, stakedTokens, maxRelays, outputAddress string, pauseHeight, unstakingHeight int64, chains []string, err error) { actor, err := p.getActor(types.ApplicationActor, address, height) + if err != nil { + return + } operator = actor.Address publicKey = actor.PublicKey stakedTokens = actor.StakedAmount diff --git a/persistence/block.go b/persistence/block.go index d3c1b9125..5452f3e39 100644 --- a/persistence/block.go +++ b/persistence/block.go @@ -3,13 +3,13 @@ package persistence import ( "encoding/binary" "encoding/hex" - "fmt" "github.com/pokt-network/pocket/persistence/kvstore" "github.com/pokt-network/pocket/persistence/types" + "github.com/pokt-network/pocket/shared/codec" ) -// OPTIMIZE(team): get from blockstore or keep in memory +// OPTIMIZE: evaluate if it's faster to get this from the blockstore (or cache) than the SQL engine func (p PostgresContext) GetLatestBlockHeight() (latestHeight uint64, err error) { ctx, tx, err := p.getCtxAndTx() if err != nil { @@ -20,8 +20,8 @@ func (p PostgresContext) GetLatestBlockHeight() (latestHeight uint64, err error) return } -// OPTIMIZE(team): get from blockstore or keep in cache/memory -func (p PostgresContext) GetBlockHashAtHeight(height int64) ([]byte, error) { +// OPTIMIZE: evaluate if it's faster to get this from the blockstore (or cache) than the SQL engine +func (p PostgresContext) GetBlockHash(height int64) ([]byte, error) { ctx, tx, err := p.getCtxAndTx() if err != nil { return nil, err @@ -40,21 +40,6 @@ func (p PostgresContext) GetHeight() (int64, error) { return p.Height, nil } -func (p PostgresContext) GetPrevAppHash() (string, error) { - height, err := p.GetHeight() - if err != nil { - return "", err - } - if height <= 1 { - return "TODO: get from genesis", nil - } - block, err := p.blockstore.Get(heightToBytes(height - 1)) - if err != nil { - return "", fmt.Errorf("error getting block hash for height %d even though it's in the database: %s", height, err) - } - return hex.EncodeToString(block), nil // TODO(#284): Return `block.Hash` instead of the hex encoded representation of the blockBz -} - func (p PostgresContext) TransactionExists(transactionHash string) (bool, error) { hash, err := hex.DecodeString(transactionHash) if err != nil { @@ -71,64 +56,67 @@ func (p PostgresContext) TransactionExists(transactionHash string) (bool, error) return true, err } -func (p PostgresContext) indexTransactions() error { - // TODO: store in batch - for _, txResult := range p.GetTxResults() { - if err := p.txIndexer.Index(txResult); err != nil { - return err - } - } - return nil -} - // DISCUSS: this might be retrieved from the block store - temporarily we will access it directly from the module // following the pattern of the Consensus Module prior to pocket/issue-#315 // TODO(#284): Remove blockProtoBytes from the interface -func (p *PostgresContext) SetProposalBlock(blockHash string, blockProtoBytes, proposerAddr []byte, transactions [][]byte) error { +func (p *PostgresContext) SetProposalBlock(blockHash string, proposerAddr, quorumCert []byte, transactions [][]byte) error { p.blockHash = blockHash - p.blockProtoBytes = blockProtoBytes + p.quorumCert = quorumCert p.proposerAddr = proposerAddr p.blockTxs = transactions return nil } -// TEMPORARY: Including two functions for the SQL and KV Store as an interim solution -// until we include the schema as part of the SQL Store because persistence -// currently has no access to the protobuf schema which is the source of truth. -// TODO: atomic operations needed here - inherited pattern from consensus module -func (p PostgresContext) storeBlock(quorumCert []byte) error { - if p.blockProtoBytes == nil { - // IMPROVE/CLEANUP: HACK - currently tests call Commit() on the same height and it throws a - // ERROR: duplicate key value violates unique constraint "block_pkey", because it attempts to - // store a block at height 0 for each test. We need a cleanup function to clear the block table - // each iteration - return nil +// Creates a block protobuf object using the schema defined in the persistence module +func (p *PostgresContext) prepareBlock(quorumCert []byte) (*types.Block, error) { + var prevHash []byte + if p.Height == 0 { + prevHash = []byte("") + } else { + var err error + prevHash, err = p.GetBlockHash(p.Height - 1) + if err != nil { + return nil, err + } } - // INVESTIGATE: Note that we are writing this directly to the blockStore. Depending on how - // the use of the PostgresContext evolves, we may need to write this to `ContextStore` and copy - // over to `BlockStore` when the block is committed. - if err := p.blockstore.Put(heightToBytes(p.Height), p.blockProtoBytes); err != nil { - return err + + txsHash, err := p.getTxsHash() + if err != nil { + return nil, err } - // Store in SQL Store - if err := p.InsertBlock(uint64(p.Height), p.blockHash, p.proposerAddr, quorumCert); err != nil { - return err + + block := &types.Block{ + Height: uint64(p.Height), + StateHash: p.blockHash, + PrevStateHash: hex.EncodeToString(prevHash), + ProposerAddress: p.proposerAddr, + QuorumCertificate: quorumCert, + TransactionsHash: txsHash, } - // Store transactions in indexer - return p.indexTransactions() + + return block, nil } -func (p PostgresContext) InsertBlock(height uint64, hash string, proposerAddr []byte, quorumCert []byte) error { +// Inserts the block into the postgres database +func (p *PostgresContext) insertBlock(block *types.Block) error { ctx, tx, err := p.getCtxAndTx() if err != nil { return err } - _, err = tx.Exec(ctx, types.InsertBlockQuery(height, hash, proposerAddr, quorumCert)) + _, err = tx.Exec(ctx, types.InsertBlockQuery(block.Height, block.StateHash, block.ProposerAddress, block.QuorumCertificate)) return err } -// CLEANUP: Should this be moved to a shared directory? +// Stores the block in the key-value store +func (p PostgresContext) storeBlock(block *types.Block) error { + blockBz, err := codec.GetCodec().Marshal(block) + if err != nil { + return err + } + return p.blockStore.Set(heightToBytes(p.Height), blockBz) +} + func heightToBytes(height int64) []byte { heightBytes := make([]byte, 8) binary.LittleEndian.PutUint64(heightBytes, uint64(height)) diff --git a/persistence/context.go b/persistence/context.go index 499cfebed..764979da5 100644 --- a/persistence/context.go +++ b/persistence/context.go @@ -1,8 +1,12 @@ package persistence +// TECHDEBT: Look into whether the receivers of `PostgresContext` could/should be pointers? + import ( "context" "log" + + "github.com/pokt-network/pocket/shared/modules" ) func (p PostgresContext) UpdateAppHash() ([]byte, error) { @@ -14,55 +18,57 @@ func (p PostgresContext) NewSavePoint(bytes []byte) error { return nil } +// TECHDEBT(#327): Guarantee atomicity betweens `prepareBlock`, `insertBlock` and `storeBlock` for save points & rollbacks. func (p PostgresContext) RollbackToSavePoint(bytes []byte) error { log.Println("TODO: RollbackToSavePoint not fully implemented") return p.GetTx().Rollback(context.TODO()) } -func (p PostgresContext) AppHash() ([]byte, error) { - log.Println("TODO: AppHash not implemented") - return []byte("A real app hash, I am not"), nil -} - -func (p *PostgresContext) Reset() error { - p.txResults = nil - p.blockHash = "" - p.proposerAddr = nil - p.blockProtoBytes = nil - p.blockTxs = nil - return nil +func (p *PostgresContext) ComputeAppHash() ([]byte, error) { + // IMPROVE(#361): Guarantee the integrity of the state + // Full details in the thread from the PR review: https://github.com/pokt-network/pocket/pull/285#discussion_r1018471719 + return p.updateMerkleTrees() } +// TECHDEBT(#327): Make sure these operations are atomic func (p PostgresContext) Commit(quorumCert []byte) error { - log.Printf("About to commit context at height %d.\n", p.Height) + log.Printf("About to commit block & context at height %d.\n", p.Height) - ctx := context.TODO() - if err := p.GetTx().Commit(context.TODO()); err != nil { + // Create a persistence block proto + block, err := p.prepareBlock(quorumCert) + if err != nil { return err } - if err := p.storeBlock(quorumCert); err != nil { + + // Store block in the KV store + if err := p.storeBlock(block); err != nil { return err } - if err := p.conn.Close(ctx); err != nil { - log.Println("[TODO][ERROR] Implement connection pooling. Error when closing DB connecting...", err) + + // Insert the block into the SQL DB + if err := p.insertBlock(block); err != nil { + return err } - if err := p.Reset(); err != nil { + + // Commit the SQL transaction + ctx := context.TODO() + if err := p.GetTx().Commit(ctx); err != nil { return err } + if err := p.conn.Close(ctx); err != nil { + log.Println("[TODO][ERROR] Implement connection pooling. Error when closing DB connecting...", err) + } + return nil } func (p PostgresContext) Release() error { log.Printf("About to release context at height %d.\n", p.Height) - ctx := context.TODO() if err := p.GetTx().Rollback(ctx); err != nil { return err } - if err := p.conn.Close(ctx); err != nil { - log.Println("[TODO][ERROR] Implement connection pooling. Error when closing DB connecting...", err) - } - if err := p.Reset(); err != nil { + if err := p.resetContext(); err != nil { return err } return nil @@ -70,6 +76,42 @@ func (p PostgresContext) Release() error { func (p PostgresContext) Close() error { log.Printf("About to close context at height %d.\n", p.Height) - return p.conn.Close(context.TODO()) } + +// INVESTIGATE(#361): Revisit if is used correctly in the context of the lifecycle of a persistenceContext and a utilityContext +func (p PostgresContext) IndexTransaction(txResult modules.TxResult) error { + return p.txIndexer.Index(txResult) +} + +func (p *PostgresContext) resetContext() (err error) { + if p == nil { + return nil + } + + p.blockHash = "" + p.quorumCert = nil + p.proposerAddr = nil + p.blockTxs = nil + + tx := p.GetTx() + if p.tx == nil { + return nil + } + + conn := tx.Conn() + if conn == nil { + return nil + } + + if !conn.IsClosed() { + if err := conn.Close(context.TODO()); err != nil { + return err + } + } + + p.conn = nil + p.tx = nil + + return err +} diff --git a/persistence/db.go b/persistence/db.go index 7adde2595..2f0d7cb1b 100644 --- a/persistence/db.go +++ b/persistence/db.go @@ -6,13 +6,11 @@ import ( "fmt" "log" - "github.com/pokt-network/pocket/persistence/indexer" - - "github.com/pokt-network/pocket/persistence/types" - "github.com/jackc/pgconn" "github.com/jackc/pgx/v4" + "github.com/pokt-network/pocket/persistence/indexer" "github.com/pokt-network/pocket/persistence/kvstore" + "github.com/pokt-network/pocket/persistence/types" "github.com/pokt-network/pocket/shared/modules" ) @@ -39,18 +37,22 @@ var protocolActorSchemas = []types.ProtocolActorSchema{ var _ modules.PersistenceRWContext = &PostgresContext{} type PostgresContext struct { - Height int64 // TODO(olshansky): `Height` is only externalized for testing purposes. Replace with helpers... - conn *pgx.Conn - tx pgx.Tx - blockstore kvstore.KVStore + Height int64 // TODO: `Height` is only externalized for testing purposes. Replace with helpers... + conn *pgx.Conn + tx pgx.Tx + + // TECHDEBT(#361): These three values are pointers to objects maintained by the PersistenceModule, + // so there should be a better way to access them (via the bus?) rather than embedding here. + blockStore kvstore.KVStore txIndexer indexer.TxIndexer - // DISCUSS(#284): this might be retrieved from the block store - temporarily we will access it directly from the module - // following the pattern of the Consensus Module prior to pocket/issue-#315 - proposerAddr []byte - blockProtoBytes []byte - blockHash string - blockTxs [][]byte - txResults []modules.TxResult // Not indexed by `txIndexer` until commit. + stateTrees *stateTrees + + // DISCUSS(#361): Could/should we move these to the utilityContext? + // IMPROVE: Could/should we rename these to proposalXX? + proposerAddr []byte + quorumCert []byte + blockHash string // CONSOLIDATE: blockHash / appHash / stateHash + blockTxs [][]byte } func (pg *PostgresContext) getCtxAndTx() (context.Context, pgx.Tx, error) { @@ -91,26 +93,10 @@ func (p PostgresContext) GetProposerAddr() []byte { return p.proposerAddr } -func (p PostgresContext) GetBlockProtoBytes() []byte { - return p.blockProtoBytes -} - -func (p PostgresContext) GetBlockHash() string { - return p.blockHash -} - func (p PostgresContext) GetBlockTxs() [][]byte { return p.blockTxs } -func (p PostgresContext) GetTxResults() []modules.TxResult { - return p.txResults -} - -func (p *PostgresContext) SetTxResults(txResults []modules.TxResult) { - p.txResults = txResults -} - // TECHDEBT: Implement proper connection pooling func connectToDatabase(postgresUrl string, schema string) (*pgx.Conn, error) { ctx := context.TODO() @@ -217,45 +203,3 @@ func initializeBlockTables(ctx context.Context, db *pgx.Conn) error { } return nil } - -// Exposed for testing purposes only -func (p PostgresContext) DebugClearAll() error { - ctx, tx, err := p.getCtxAndTx() - if err != nil { - return err - } - - clearTx, err := tx.Begin(ctx) // creates a pseudo-nested transaction - if err != nil { - return err - } - - for _, actor := range protocolActorSchemas { - if _, err = clearTx.Exec(ctx, actor.ClearAllQuery()); err != nil { - return err - } - if actor.GetChainsTableName() != "" { - if _, err = clearTx.Exec(ctx, actor.ClearAllChainsQuery()); err != nil { - return err - } - } - } - - if _, err = tx.Exec(ctx, types.ClearAllGovParamsQuery()); err != nil { - return err - } - - if _, err = tx.Exec(ctx, types.ClearAllGovFlagsQuery()); err != nil { - return err - } - - if _, err = tx.Exec(ctx, types.ClearAllBlocksQuery()); err != nil { - return err - } - - if err = clearTx.Commit(ctx); err != nil { - return err - } - - return nil -} diff --git a/persistence/debug.go b/persistence/debug.go index 3c4c9582e..2019a66df 100644 --- a/persistence/debug.go +++ b/persistence/debug.go @@ -1,58 +1,141 @@ package persistence import ( + "crypto/sha256" "log" - typesCons "github.com/pokt-network/pocket/consensus/types" + "github.com/celestiaorg/smt" "github.com/pokt-network/pocket/persistence/types" "github.com/pokt-network/pocket/shared/codec" "github.com/pokt-network/pocket/shared/messaging" ) +// A list of functions to clear data from the DB not associated with protocol actors +var nonActorClearFunctions = []func() string{ + types.ClearAllAccounts, + types.ClearAllPools, + types.ClearAllGovParamsQuery, + types.ClearAllGovFlagsQuery, + types.ClearAllBlocksQuery, +} + func (m *persistenceModule) HandleDebugMessage(debugMessage *messaging.DebugMessage) error { switch debugMessage.Action { case messaging.DebugMessageAction_DEBUG_SHOW_LATEST_BLOCK_IN_STORE: m.showLatestBlockInStore(debugMessage) - case messaging.DebugMessageAction_DEBUG_CLEAR_STATE: - m.clearState(debugMessage) + // Clears all the state (SQL DB, KV Stores, Trees, etc) to nothing + case messaging.DebugMessageAction_DEBUG_PERSISTENCE_CLEAR_STATE: + if err := m.clearAllState(debugMessage); err != nil { + return err + } + // Resets all the state (SQL DB, KV Stores, Trees, etc) to the tate specified in the genesis file provided + case messaging.DebugMessageAction_DEBUG_PERSISTENCE_RESET_TO_GENESIS: + if err := m.clearAllState(debugMessage); err != nil { + return err + } g := m.genesisState.(*types.PersistenceGenesisState) - m.populateGenesisState(g) + m.populateGenesisState(g) // fatal if there's an error default: log.Printf("Debug message not handled by persistence module: %s \n", debugMessage.Message) } return nil } -// TODO(olshansky): Create a shared interface `Block` to avoid the use of typesCons here. +// IMPROVE: Add an iterator to the `kvstore` and use that instead func (m *persistenceModule) showLatestBlockInStore(_ *messaging.DebugMessage) { - // TODO: Add an iterator to the `kvstore` and use that instead - height := m.GetBus().GetConsensusModule().CurrentHeight() - 1 // -1 because we want the latest committed height + height := m.GetBus().GetConsensusModule().CurrentHeight() - 1 blockBytes, err := m.GetBlockStore().Get(heightToBytes(int64(height))) if err != nil { log.Printf("Error getting block %d from block store: %s \n", height, err) return } - codec := codec.GetCodec() - block := &typesCons.Block{} - codec.Unmarshal(blockBytes, block) - log.Printf("Block at height %d with %d transactions: %+v \n", height, len(block.Transactions), block) + block := &types.Block{} + if err := codec.GetCodec().Unmarshal(blockBytes, block); err != nil { + log.Printf("Error decoding block %d from block store: %s \n", height, err) + return + } + + log.Printf("Block at height %d: %+v \n", height, block) } -func (m *persistenceModule) clearState(_ *messaging.DebugMessage) { - context, err := m.NewRWContext(-1) +// TECHDEBT: Make sure this is atomic +func (m *persistenceModule) clearAllState(_ *messaging.DebugMessage) error { + ctx, err := m.NewRWContext(-1) if err != nil { - log.Printf("Error creating new context: %s \n", err) - return + return err } - defer context.Commit(nil) + postgresCtx := ctx.(*PostgresContext) - if err := context.(*PostgresContext).DebugClearAll(); err != nil { - log.Printf("Error clearing state: %s \n", err) - return + // Clear the SQL DB + if err := postgresCtx.clearAllSQLState(); err != nil { + return err + } + + // Release the SQL context + if err := m.ReleaseWriteContext(); err != nil { + return err } + + // Clear the BlockStore if err := m.blockStore.ClearAll(); err != nil { - log.Printf("Error clearing block store: %s \n", err) - return + return err + } + + // Clear all the Trees + if err := postgresCtx.clearAllTreeState(); err != nil { + return err + } + + log.Println("Cleared all the state") + return nil +} + +func (p *PostgresContext) clearAllSQLState() error { + ctx, clearTx, err := p.getCtxAndTx() + if err != nil { + return err } + + for _, actor := range protocolActorSchemas { + if _, err = clearTx.Exec(ctx, actor.ClearAllQuery()); err != nil { + return err + } + if actor.GetChainsTableName() != "" { + if _, err = clearTx.Exec(ctx, actor.ClearAllChainsQuery()); err != nil { + return err + } + } + } + + for _, clearFn := range nonActorClearFunctions { + if _, err := clearTx.Exec(ctx, clearFn()); err != nil { + return err + } + } + + if err = clearTx.Commit(ctx); err != nil { + return err + } + + return nil +} + +func (p *PostgresContext) clearAllTreeState() error { + for treeType := merkleTree(0); treeType < numMerkleTrees; treeType++ { + valueStore := p.stateTrees.valueStores[treeType] + nodeStore := p.stateTrees.nodeStores[treeType] + + if err := valueStore.ClearAll(); err != nil { + return err + } + if err := nodeStore.ClearAll(); err != nil { + return err + } + + // Needed in order to make sure the root is re-set correctly after clearing + p.stateTrees.merkleTrees[treeType] = smt.NewSparseMerkleTree(valueStore, nodeStore, sha256.New()) + } + + return nil } diff --git a/persistence/docs/PROTOCOL_STATE_HASH.md b/persistence/docs/PROTOCOL_STATE_HASH.md new file mode 100644 index 000000000..e6e0ba069 --- /dev/null +++ b/persistence/docs/PROTOCOL_STATE_HASH.md @@ -0,0 +1,138 @@ +# State Hash + +This document describes the `Persistence` module's internal implementation of how the state hash is computed. Specifically, it defines the **'Compute State Hash'** flow in the shared architectural state hash flow defined [here](../../shared/docs/PROTOCOL_STATE_HASH.md). + +Alternative implementation of the persistence module are free to choose their own **State Storage** engines (SQL, KV stores, etc) or their own **State Commitment** paradigms (Merkle Trees, Vector Commitments, etc), but the output hash **must** remain identical. + +- [Introduction](#introduction) +- [Data Types](#data-types) + - [Infrastructural Components](#infrastructural-components) + - [Block Proto](#block-proto) + - [Trees](#trees) +- [Compute State Hash](#compute-state-hash) +- [Store Block (Commit)](#store-block-commit) +- [Failed Commitments](#failed-commitments) + +## Introduction + +The state hash is a single 256 bit digest that takes a snapshot of the world state at any committed height. It is needed to guarantee and prove the integrity of the world state, and is what's referenced in every block header when building any _blockchain_. + +This document defines how Pocket V1 takes a snapshot of its world state. An introduction to the requirements, types and uses of hashes in blockchain systems is outside the scope of this document. + +## Data Types + +### Infrastructural Components + +| Component | Data Type | Implementation Options - Examples | Implementation Selected - Current | Example | Use Case | +| --------------------- | ------------------------------------- | ------------------------------------------------------ | --------------------------------- | ------------------- | -------------------------------------------------------------------------------- | +| Data Tables | SQL Database / Engine | MySQL, SQLite, PostgreSQL | PostgresSQL | Validator SQL Table | Validating & updating information when applying a transaction | +| Merkle Trees | Merkle Trie backed by Key-Value Store | Celestia's SMT, Libra's JMT, Cosmos' IAVL, Verkle Tree | Celestia's SMT | Fisherman Trie | Maintains the state of all account based trees | +| Blocks | Serialization Codec | Amino, Protobuf, Thrift, Avro | Protobuf | Block protobuf | Serialized and inserted into the Block Store | +| Objects (e.g. Actors) | Serialization Codec | Amino, Protobuf, Thrift, Avro | Protobuf | Servicer protobuf | Serialized and inserted into the corresponding Tree | +| Block Store | Key Value Store | LevelDB, BadgerDB, RocksDB, BoltDB | BadgerDb | Block Store | Maintains a key-value store of the blockchain blocks | +| Transaction Indexer | Key Value Store | LevelDB, BadgerDB, RocksDB, BoltDB | BadgerDB | Tx Indexer | Indexes transactions in different ways for fast queries, presence checks, etc... | + +### Block Proto + +The block protobuf that is serialized and store in the block store can be found in `persistence/proto/block_persistence.proto`. This proto contains the `stateHash` along with the corresponding height. + +### Trees + +An individual Merkle Tree is created for each type of actor, record or data type. Each of these is backed by its own key-value store. + +Note that the order in which the trees are defined (found in `persistence/state.go`) is important since it determines how the state hash is computed. _TODO(#361): Consider specifying the oder in a `.proto` `enum` rather than a `.go` `iota`._ + +**Actor Merkle Trees**: + +- Applications +- Validators +- Fisherman +- ServiceNodes + +**Account Merkle Trees**: + +- Accounts +- Pools + +**Data Merkle Trees** + +- Transactions +- Parameters +- Flags + +## Compute State Hash + +_Note: `GetRecordsUpdatedAtHeight` is an abstraction for retrieving all the records from the corresponding SQL tables depending on the type of record (Actors, Transactions, Params, etc...)_ + +This flow shows the interaction between the `PostgresDB` and `MerkleTrees` listed above to compute the state hash. Assuming the process of applying a proposal block to the current context (i.e. the uncommitted SQL state) is done, the following steps compute the hash of the new world state. + +1. Loop over all of the merkle tree types +2. Use `GetRecordsUpdatedAtHeight` to retrieve all the records updated at the context's height +3. Serialize each record using the corresponding underlying protobuf +4. Insert the serialized record into the corresponding tree (which is back by a key-value store) +5. Compute the root hash of each tree +6. Aggregate all the root hashes by concatenating them together +7. Compute the new `stateHash` by taking a hash of the concatenated hash list + +```mermaid +sequenceDiagram + participant P as Persistence + participant PSQL as Persistence (SQL Store) + participant PKV as Persistence (Key-Value Store) + + loop for each merkle tree type + P->>+PSQL: GetRecordsUpdatedAtHeight(height, recordType) + PSQL->>-P: records + loop for each state tree + P->>+PKV: Update(addr, serialize(record)) + PKV->>-P: result, err_code + end + P->>+PKV: GetRoot() + PKV->>-P: rootHash + end + + P->>P: stateHash = hash(concat(rootHashes)) + activate P + deactivate P +``` + +_IMPORTANT: The order in which the `rootHashes` are concatenated is based on the definition in which the trees are ordered in within `state.go`._ + +## Store Block (Commit) + +When the `Commit(quorumCert)` function is invoked, the current context is committed to disk. The `PersistenceContext` does the following: + +1. Read data from the persistence context's in-memory state +2. Prepare a instance of the `Block` proto & serialize it +3. Insert the `Block` into the `BlockStore` +4. Insert the `Block` into the SQL Store +5. Commit the context's SQL transaction to disk + +```mermaid +sequenceDiagram + participant P as Persistence + participant PSQL as Persistence (SQL Store) + participant PKV as Persistence (Key-Value Store) + + P->>P: prepare & serialize block proto + activate P + deactivate P + + %% Insert into the SQL store + P->>+PSQL: Insert(height, block) + PSQL->>-P: result, err_code + + %% Insert into the Block Store (i.e. Key-Value store) + P->>+PKV: Put(height, block) + PKV->>-P: result, err_code + + %% Commit the SQL transaction + P->>+PSQL: Commit(SQL Tx to disk) + PSQL->>-P: result, err_code +``` + +_TODO: If an error occurs at any step, all of the operations must be reverted in an atomic manner._ + +## Failed Commitments + +TODO: Failed commitments and the implementation of rollbacks is tracked in #327 and #329. diff --git a/persistence/docs/README.md b/persistence/docs/README.md index f87efd229..06ba8707a 100644 --- a/persistence/docs/README.md +++ b/persistence/docs/README.md @@ -30,7 +30,9 @@ The persistence specific configuration within a node's `config.json` looks like "persistence": { "postgres_url": "postgres://postgres:postgres@pocket-db:5432/postgres", "schema": "node1", - "block_store_path": "/var/blockstore" + "block_store_path": "/var/blockstore", + "tx_indexer_path": "", + "trees_store_dir": "/var/trees" } ``` diff --git a/persistence/genesis.go b/persistence/genesis.go index 544d298a0..ba384eafd 100644 --- a/persistence/genesis.go +++ b/persistence/genesis.go @@ -10,9 +10,7 @@ import ( "github.com/pokt-network/pocket/shared/modules" ) -// TODO(andrew): generalize with the `actors interface` - -// WARNING: This function crashes the process if there is an error populating the genesis state. +// CONSIDERATION: Should this return an error and let the caller decide if it should log a fatal error? func (m *persistenceModule) populateGenesisState(state modules.PersistenceGenesisState) { log.Println("Populating genesis state...") @@ -31,15 +29,11 @@ func (m *persistenceModule) populateGenesisState(state modules.PersistenceGenesi return nil } - log.Println("Populating genesis state...") rwContext, err := m.NewRWContext(0) if err != nil { log.Fatalf("an error occurred creating the rwContext for the genesis state: %s", err.Error()) } - if err != nil { - log.Fatalf("an error occurred creating the rwContext for the genesis state: %s", err.Error()) - } for _, acc := range state.GetAccs() { addrBz, err := hex.DecodeString(acc.GetAddress()) if err != nil { @@ -150,8 +144,19 @@ func (m *persistenceModule) populateGenesisState(state modules.PersistenceGenesi log.Fatalf("an error occurred initializing flags: %s", err.Error()) } - if err = rwContext.Commit([]byte("TODO(#284): Genesis QC")); err != nil { - log.Fatalf("an error occurred during commit() on genesis state %s ", err.Error()) + // Updates all the merkle trees + appHash, err := rwContext.ComputeAppHash() + if err != nil { + log.Fatalf("an error occurred updating the app hash during genesis: %s", err.Error()) + } + + if err := rwContext.SetProposalBlock(hex.EncodeToString(appHash), nil, nil, nil); err != nil { + log.Fatalf("an error occurred setting the proposal block during genesis: %s", err.Error()) + } + + // This update the DB, blockstore, and commits the state + if err = rwContext.Commit(nil); err != nil { + log.Fatalf("error committing genesis state to DB %s ", err.Error()) } } @@ -221,11 +226,11 @@ func (p PostgresContext) GetAllApps(height int64) (apps []modules.Actor, err err } rows.Close() for _, actor := range actors { - actor, err = p.getChainsForActor(ctx, tx, types.ApplicationActor, actor, height) + actorWithChains, err := p.getChainsForActor(ctx, tx, types.ApplicationActor, actor, height) if err != nil { - return + return nil, err } - apps = append(apps, actor) + apps = append(apps, actorWithChains) } return } diff --git a/persistence/indexer/indexer.go b/persistence/indexer/indexer.go index e02547897..95808fa7b 100644 --- a/persistence/indexer/indexer.go +++ b/persistence/indexer/indexer.go @@ -5,6 +5,7 @@ package indexer import ( "encoding/hex" "fmt" + "github.com/pokt-network/pocket/shared/codec" shared "github.com/pokt-network/pocket/shared/modules" @@ -24,7 +25,7 @@ type TxIndexer interface { // `GetByHash` returns the transaction specified by the hash if indexed or nil otherwise GetByHash(hash []byte) (shared.TxResult, error) - // `GetByHeight` returns all transactions specified by height or nil if there are no transactions at that height + // `GetByHeight` returns all transactions specified by height or nil if there are no transactions at that height; may be ordered descending/ascending GetByHeight(height int64, descending bool) ([]shared.TxResult, error) // `GetBySender` returns all transactions signed by *sender*; may be ordered descending/ascending @@ -84,6 +85,7 @@ func NewTxIndexer(databasePath string) (TxIndexer, error) { if databasePath == "" { return NewMemTxIndexer() } + db, err := kvstore.NewKVStore(databasePath) return &txIndexer{ db: db, @@ -144,7 +146,7 @@ func (indexer *txIndexer) Close() error { // kv helper functions func (indexer *txIndexer) getAll(prefix []byte, descending bool) (result []shared.TxResult, err error) { - hashKeys, err := indexer.db.GetAll(prefix, descending) + _, hashKeys, err := indexer.db.GetAll(prefix, descending) if err != nil { return nil, err } @@ -171,22 +173,22 @@ func (indexer *txIndexer) get(key []byte) (shared.TxResult, error) { func (indexer *txIndexer) indexByHash(hash, bz []byte) (hashKey []byte, err error) { key := indexer.hashKey(hash) - return key, indexer.db.Put(key, bz) + return key, indexer.db.Set(key, bz) } func (indexer *txIndexer) indexByHeightAndIndex(height int64, index int32, bz []byte) error { - return indexer.db.Put(indexer.heightAndIndexKey(height, index), bz) + return indexer.db.Set(indexer.heightAndIndexKey(height, index), bz) } func (indexer *txIndexer) indexBySender(sender string, bz []byte) error { - return indexer.db.Put(indexer.senderKey(sender), bz) + return indexer.db.Set(indexer.senderKey(sender), bz) } func (indexer *txIndexer) indexByRecipient(recipient string, bz []byte) error { if recipient == "" { return nil } - return indexer.db.Put(indexer.recipientKey(recipient), bz) + return indexer.db.Set(indexer.recipientKey(recipient), bz) } // key helper functions diff --git a/persistence/kvstore/kvstore.go b/persistence/kvstore/kvstore.go index d086ab75e..f0a2d7976 100644 --- a/persistence/kvstore/kvstore.go +++ b/persistence/kvstore/kvstore.go @@ -1,22 +1,23 @@ package kvstore import ( + "errors" "log" + "github.com/celestiaorg/smt" badger "github.com/dgraph-io/badger/v3" ) -// CLEANUP: move this structure to a shared module type KVStore interface { + smt.MapStore // Get, Set, Delete + // Lifecycle methods Stop() error // Accessors // TODO: Add a proper iterator interface - Put(key []byte, value []byte) error - Get(key []byte) ([]byte, error) // TODO: Add pagination for `GetAll` - GetAll(prefixKey []byte, descending bool) ([][]byte, error) + GetAll(prefixKey []byte, descending bool) (keys [][]byte, values [][]byte, err error) Exists(key []byte) (bool, error) ClearAll() error } @@ -26,28 +27,34 @@ const ( ) var _ KVStore = &badgerKVStore{} +var _ smt.MapStore = &badgerKVStore{} + +var ( + ErrKVStoreExists = errors.New("kvstore already exists") + ErrKVStoreNotExists = errors.New("kvstore does not exist") +) type badgerKVStore struct { db *badger.DB } func NewKVStore(path string) (KVStore, error) { - db, err := badger.Open(badger.DefaultOptions(path)) + db, err := badger.Open(badgerOptions(path)) if err != nil { return nil, err } - return badgerKVStore{db: db}, nil + return &badgerKVStore{db: db}, nil } func NewMemKVStore() KVStore { - db, err := badger.Open(badger.DefaultOptions("").WithInMemory(true)) + db, err := badger.Open(badgerOptions("").WithInMemory(true)) if err != nil { log.Fatal(err) } - return badgerKVStore{db: db} + return &badgerKVStore{db: db} } -func (store badgerKVStore) Put(key []byte, value []byte) error { +func (store *badgerKVStore) Set(key, value []byte) error { tx := store.db.NewTransaction(true) defer tx.Discard() @@ -59,7 +66,7 @@ func (store badgerKVStore) Put(key []byte, value []byte) error { return tx.Commit() } -func (store badgerKVStore) Get(key []byte) ([]byte, error) { +func (store *badgerKVStore) Get(key []byte) ([]byte, error) { tx := store.db.NewTransaction(false) defer tx.Discard() @@ -80,8 +87,16 @@ func (store badgerKVStore) Get(key []byte) ([]byte, error) { return value, nil } -func (store badgerKVStore) GetAll(prefix []byte, descending bool) (values [][]byte, err error) { +func (store *badgerKVStore) Delete(key []byte) error { + tx := store.db.NewTransaction(true) + defer tx.Discard() + + return tx.Delete(key) +} + +func (store *badgerKVStore) GetAll(prefix []byte, descending bool) (keys [][]byte, values [][]byte, err error) { // INVESTIGATE: research `badger.views` for further improvements and optimizations + // Reference https://pkg.go.dev/github.com/dgraph-io/badger#readme-prefix-scans txn := store.db.NewTransaction(false) defer txn.Discard() @@ -94,11 +109,15 @@ func (store badgerKVStore) GetAll(prefix []byte, descending bool) (values [][]by it := txn.NewIterator(opt) defer it.Close() + keys = make([][]byte, 0) + values = make([][]byte, 0) + for it.Seek(prefix); it.Valid(); it.Next() { item := it.Item() err = item.Value(func(v []byte) error { b := make([]byte, len(v)) copy(b, v) + keys = append(keys, item.Key()) values = append(values, b) return nil }) @@ -109,7 +128,7 @@ func (store badgerKVStore) GetAll(prefix []byte, descending bool) (values [][]by return } -func (store badgerKVStore) Exists(key []byte) (bool, error) { +func (store *badgerKVStore) Exists(key []byte) (bool, error) { val, err := store.Get(key) if err != nil { return false, err @@ -117,11 +136,11 @@ func (store badgerKVStore) Exists(key []byte) (bool, error) { return val != nil, nil } -func (store badgerKVStore) ClearAll() error { +func (store *badgerKVStore) ClearAll() error { return store.db.DropAll() } -func (store badgerKVStore) Stop() error { +func (store *badgerKVStore) Stop() error { return store.db.Close() } @@ -141,3 +160,10 @@ func prefixEndBytes(prefix []byte) []byte { end[len(end)-1]++ return end } + +// TODO: Propagate persistence configurations to badger +func badgerOptions(path string) badger.Options { + opts := badger.DefaultOptions(path) + opts.Logger = nil // disable badger's logger since it's very noisy + return opts +} diff --git a/persistence/module.go b/persistence/module.go index 1d442c0c7..535020641 100644 --- a/persistence/module.go +++ b/persistence/module.go @@ -5,12 +5,10 @@ import ( "fmt" "log" - "github.com/pokt-network/pocket/persistence/indexer" - - "github.com/pokt-network/pocket/persistence/types" - "github.com/jackc/pgx/v4" + "github.com/pokt-network/pocket/persistence/indexer" "github.com/pokt-network/pocket/persistence/kvstore" + "github.com/pokt-network/pocket/persistence/types" "github.com/pokt-network/pocket/shared/modules" ) @@ -30,8 +28,9 @@ type persistenceModule struct { config modules.PersistenceConfig genesisState modules.PersistenceGenesisState - blockStore kvstore.KVStore // INVESTIGATE: We may need to create a custom `BlockStore` package in the future + blockStore kvstore.KVStore txIndexer indexer.TxIndexer + stateTrees *stateTrees // TECHDEBT: Need to implement context pooling (for writes), timeouts (for read & writes), etc... writeContext *PostgresContext // only one write context is allowed at a time @@ -70,6 +69,7 @@ func (*persistenceModule) Create(runtimeMgr modules.RuntimeMgr) (modules.Module, } conn.Close(context.TODO()) + // TODO: Follow the same pattern as txIndexer below for initializing the blockStore blockStore, err := initializeBlockStore(persistenceCfg.GetBlockStorePath()) if err != nil { return nil, err @@ -80,26 +80,35 @@ func (*persistenceModule) Create(runtimeMgr modules.RuntimeMgr) (modules.Module, return nil, err } + stateTrees, err := newStateTrees(persistenceCfg.GetTreesStoreDir()) + if err != nil { + return nil, err + } + m = &persistenceModule{ bus: nil, config: persistenceCfg, genesisState: persistenceGenesis, - blockStore: blockStore, - txIndexer: txIndexer, + + blockStore: blockStore, + txIndexer: txIndexer, + stateTrees: stateTrees, + writeContext: nil, } + // TECHDEBT: reconsider if this is the best place to call `populateGenesisState`. Note that + // this forces the genesis state to be reloaded on every node startup until state + // sync is implemented. // Determine if we should hydrate the genesis db or use the current state of the DB attached if shouldHydrateGenesis, err := m.shouldHydrateGenesisDb(); err != nil { return nil, err } else if shouldHydrateGenesis { - // TECHDEBT: reconsider if this is the best place to call `populateGenesisState`. Note that - // this forces the genesis state to be reloaded on every node startup until state sync is - // implemented. - // NOTE: `populateGenesisState` does not return an error but logs a fatal error if there's a problem - m.populateGenesisState(persistenceGenesis) + m.populateGenesisState(persistenceGenesis) // fatal if there's an error } else { - log.Println("Loading state from previous state...") + // This configurations will connect to the SQL database and key-value stores specified + // in the configurations and connected to those. + log.Println("Loading state from disk...") } return m, nil @@ -158,11 +167,17 @@ func (m *persistenceModule) NewRWContext(height int64) (modules.PersistenceRWCon } m.writeContext = &PostgresContext{ - Height: height, - conn: conn, - tx: tx, - blockstore: m.blockStore, + Height: height, + conn: conn, + tx: tx, + + blockStore: m.blockStore, txIndexer: m.txIndexer, + stateTrees: m.stateTrees, + + proposerAddr: nil, + quorumCert: nil, + blockHash: "", } return m.writeContext, nil @@ -187,13 +202,19 @@ func (m *persistenceModule) NewReadContext(height int64) (modules.PersistenceRea Height: height, conn: conn, tx: tx, - blockstore: m.blockStore, + blockStore: m.blockStore, txIndexer: m.txIndexer, }, nil } func (m *persistenceModule) ReleaseWriteContext() error { - panic("TODO(#284): Implement proper write context release.") + if m.writeContext != nil { + if err := m.writeContext.resetContext(); err != nil { + log.Println("[TODO][ERROR] Error releasing write context...", err) + } + m.writeContext = nil + } + return nil } func (m *persistenceModule) GetBlockStore() kvstore.KVStore { @@ -220,8 +241,15 @@ func (m *persistenceModule) shouldHydrateGenesisDb() (bool, error) { } defer checkContext.Close() - if _, err = checkContext.GetLatestBlockHeight(); err != nil { + blockHeight, err := checkContext.GetLatestBlockHeight() + if err != nil { + return true, nil + } + + if blockHeight == 0 { + m.clearAllState(nil) return true, nil } + return false, nil } diff --git a/persistence/proto/block_persistence.proto b/persistence/proto/block_persistence.proto new file mode 100644 index 000000000..4be306137 --- /dev/null +++ b/persistence/proto/block_persistence.proto @@ -0,0 +1,14 @@ +syntax = "proto3"; +package persistence; + +option go_package = "github.com/pokt-network/pocket/persistence/types"; + +message Block { + uint64 height = 1; + string stateHash = 2; + string prevStateHash = 3; // The stateHash of the block at height-1 + bytes proposerAddress = 4; // The proposer of this block + bytes quorumCertificate = 5; // The quorum certificate containing signature from 2/3+ validators at height + // INVESTIGATE(#361): Decide if we need `transactionsHash` given that it is captured in the `transactionsTree`. + bytes transactionsHash = 6; // The hash of all the transactions in the block +} \ No newline at end of file diff --git a/persistence/proto/persistence_config.proto b/persistence/proto/persistence_config.proto index 6215bb14c..ffe70c3b3 100644 --- a/persistence/proto/persistence_config.proto +++ b/persistence/proto/persistence_config.proto @@ -8,4 +8,5 @@ message PersistenceConfig { string node_schema = 2; string block_store_path = 3; string tx_indexer_path = 4; + string trees_store_dir = 5; } \ No newline at end of file diff --git a/persistence/shared_sql.go b/persistence/shared_sql.go index 089b8ab16..49c54770b 100644 --- a/persistence/shared_sql.go +++ b/persistence/shared_sql.go @@ -44,6 +44,44 @@ func (p *PostgresContext) GetExists(actorSchema types.ProtocolActorSchema, addre return } +func (p *PostgresContext) GetActorsUpdated(actorSchema types.ProtocolActorSchema, height int64) (actors []*types.Actor, err error) { + ctx, tx, err := p.getCtxAndTx() + if err != nil { + return + } + + rows, err := tx.Query(ctx, actorSchema.GetUpdatedAtHeightQuery(height)) + if err != nil { + return nil, err + } + defer rows.Close() + + addrs := make([][]byte, 0) + for rows.Next() { + var addr string + if err = rows.Scan(&addr); err != nil { + return nil, err + } + addrBz, err := hex.DecodeString(addr) + if err != nil { + return nil, err + } + addrs = append(addrs, addrBz) + } + rows.Close() + + actors = make([]*types.Actor, len(addrs)) + for i, addr := range addrs { + actor, err := p.getActor(actorSchema, addr, height) + if err != nil { + return nil, err + } + actors[i] = actor + } + + return +} + func (p *PostgresContext) getActor(actorSchema types.ProtocolActorSchema, address []byte, height int64) (actor *types.Actor, err error) { ctx, tx, err := p.getCtxAndTx() if err != nil { @@ -60,8 +98,13 @@ func (p *PostgresContext) getActor(actorSchema types.ProtocolActorSchema, addres func (p *PostgresContext) getActorFromRow(row pgx.Row) (actor *types.Actor, height int64, err error) { actor = new(types.Actor) err = row.Scan( - &actor.Address, &actor.PublicKey, &actor.StakedAmount, &actor.GenericParam, - &actor.Output, &actor.PausedHeight, &actor.UnstakingHeight, + &actor.Address, + &actor.PublicKey, + &actor.StakedAmount, + &actor.GenericParam, + &actor.Output, + &actor.PausedHeight, + &actor.UnstakingHeight, &height) return } @@ -71,7 +114,8 @@ func (p *PostgresContext) getChainsForActor( tx pgx.Tx, actorSchema types.ProtocolActorSchema, actor *types.Actor, - height int64) (a *types.Actor, err error) { + height int64, +) (a *types.Actor, err error) { if actorSchema.GetChainsTableName() == "" { return actor, nil } diff --git a/persistence/state.go b/persistence/state.go new file mode 100644 index 000000000..f2cc65417 --- /dev/null +++ b/persistence/state.go @@ -0,0 +1,363 @@ +package persistence + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "fmt" + "log" + + "github.com/celestiaorg/smt" + "github.com/pokt-network/pocket/persistence/kvstore" + "github.com/pokt-network/pocket/persistence/types" + "github.com/pokt-network/pocket/shared/crypto" + "google.golang.org/protobuf/proto" +) + +type merkleTree float64 + +type stateTrees struct { + merkleTrees map[merkleTree]*smt.SparseMerkleTree + + // nodeStores & valueStore are part of the SMT, but references are kept below for convenience + // and debugging purposes + nodeStores map[merkleTree]kvstore.KVStore + valueStores map[merkleTree]kvstore.KVStore +} + +// A list of Merkle Trees used to maintain the state hash. +const ( + // IMPORTANT: The order in which these trees are defined is important and strict. It implicitly + // defines the index of the root hash each independent as they are concatenated together + // to generate the state hash. + + // Actor Merkle Trees + appMerkleTree merkleTree = iota + valMerkleTree + fishMerkleTree + serviceNodeMerkleTree + + // Account Merkle Trees + accountMerkleTree + poolMerkleTree + + // Data Merkle Trees + transactionsMerkleTree + paramsMerkleTree + flagsMerkleTree + + // Used for iteration purposes only; see https://stackoverflow.com/a/64178235/768439 as a reference + numMerkleTrees +) + +const ( + // IMPORTANT: The order, ascending, is critical since it defines the integrity of `transactionsHash`. + // If this changes, the `transactionsHash`` in the block will differ, rendering it invalid. + txsOrderInBlockHashDescending = false +) + +var merkleTreeToString = map[merkleTree]string{ + appMerkleTree: "app", + valMerkleTree: "val", + fishMerkleTree: "fish", + serviceNodeMerkleTree: "serviceNode", + + accountMerkleTree: "account", + poolMerkleTree: "pool", + + transactionsMerkleTree: "transactions", + paramsMerkleTree: "params", + flagsMerkleTree: "flags", +} + +var actorTypeToMerkleTreeName = map[types.ActorType]merkleTree{ + types.ActorType_App: appMerkleTree, + types.ActorType_Val: valMerkleTree, + types.ActorType_Fish: fishMerkleTree, + types.ActorType_Node: serviceNodeMerkleTree, +} + +var actorTypeToSchemaName = map[types.ActorType]types.ProtocolActorSchema{ + types.ActorType_App: types.ApplicationActor, + types.ActorType_Val: types.ValidatorActor, + types.ActorType_Fish: types.FishermanActor, + types.ActorType_Node: types.ServiceNodeActor, +} + +var merkleTreeToActorTypeName = map[merkleTree]types.ActorType{ + appMerkleTree: types.ActorType_App, + valMerkleTree: types.ActorType_Val, + fishMerkleTree: types.ActorType_Fish, + serviceNodeMerkleTree: types.ActorType_Node, +} + +func newStateTrees(treesStoreDir string) (*stateTrees, error) { + if treesStoreDir == "" { + return newMemStateTrees() + } + + stateTrees := &stateTrees{ + merkleTrees: make(map[merkleTree]*smt.SparseMerkleTree, int(numMerkleTrees)), + nodeStores: make(map[merkleTree]kvstore.KVStore, int(numMerkleTrees)), + valueStores: make(map[merkleTree]kvstore.KVStore, int(numMerkleTrees)), + } + + for tree := merkleTree(0); tree < numMerkleTrees; tree++ { + nodeStore, err := kvstore.NewKVStore(fmt.Sprintf("%s/%s_nodes", treesStoreDir, merkleTreeToString[tree])) + if err != nil { + return nil, err + } + valueStore, err := kvstore.NewKVStore(fmt.Sprintf("%s/%s_values", treesStoreDir, merkleTreeToString[tree])) + if err != nil { + return nil, err + } + stateTrees.nodeStores[tree] = nodeStore + stateTrees.valueStores[tree] = valueStore + stateTrees.merkleTrees[tree] = smt.NewSparseMerkleTree(nodeStore, valueStore, sha256.New()) + } + return stateTrees, nil +} + +func newMemStateTrees() (*stateTrees, error) { + stateTrees := &stateTrees{ + merkleTrees: make(map[merkleTree]*smt.SparseMerkleTree, int(numMerkleTrees)), + nodeStores: make(map[merkleTree]kvstore.KVStore, int(numMerkleTrees)), + valueStores: make(map[merkleTree]kvstore.KVStore, int(numMerkleTrees)), + } + for tree := merkleTree(0); tree < numMerkleTrees; tree++ { + nodeStore := kvstore.NewMemKVStore() // For testing, `smt.NewSimpleMap()` can be used as well + valueStore := kvstore.NewMemKVStore() + stateTrees.nodeStores[tree] = nodeStore + stateTrees.valueStores[tree] = valueStore + stateTrees.merkleTrees[tree] = smt.NewSparseMerkleTree(nodeStore, valueStore, sha256.New()) + } + return stateTrees, nil +} + +func (p *PostgresContext) updateMerkleTrees() ([]byte, error) { + // Update all the merkle trees + for treeType := merkleTree(0); treeType < numMerkleTrees; treeType++ { + switch treeType { + // Actor Merkle Trees + case appMerkleTree: + fallthrough + case valMerkleTree: + fallthrough + case fishMerkleTree: + fallthrough + case serviceNodeMerkleTree: + actorType, ok := merkleTreeToActorTypeName[treeType] + if !ok { + return nil, fmt.Errorf("no actor type found for merkle tree: %v\n", treeType) + } + if err := p.updateActorsTree(actorType); err != nil { + return nil, err + } + + // Account Merkle Trees + case accountMerkleTree: + if err := p.updateAccountTrees(); err != nil { + return nil, err + } + case poolMerkleTree: + if err := p.updatePoolTrees(); err != nil { + return nil, err + } + + // Data Merkle Trees + case transactionsMerkleTree: + if err := p.updateTransactionsTree(); err != nil { + return nil, err + } + case paramsMerkleTree: + if err := p.updateParamsTree(); err != nil { + return nil, err + } + case flagsMerkleTree: + if err := p.updateFlagsTree(); err != nil { + return nil, err + } + + // Default + default: + log.Fatalf("Not handled yet in state commitment update. Merkle tree #{%v}\n", treeType) + } + } + + return p.getStateHash(), nil +} + +func (p *PostgresContext) getStateHash() []byte { + // Get the root of each Merkle Tree + roots := make([][]byte, 0) + for tree := merkleTree(0); tree < numMerkleTrees; tree++ { + roots = append(roots, p.stateTrees.merkleTrees[tree].Root()) + } + + // Get the state hash + rootsConcat := bytes.Join(roots, []byte{}) + stateHash := sha256.Sum256(rootsConcat) + + // Convert the array to a slice and return it + return stateHash[:] +} + +// Transactions Hash Helpers + +// Returns a digest (a single hash) of all the transactions included in the block. +// This allows separating the integrity of the transactions from their storage. +func (p PostgresContext) getTxsHash() (txs []byte, err error) { + txResults, err := p.txIndexer.GetByHeight(p.Height, txsOrderInBlockHashDescending) + if err != nil { + return nil, err + } + + for _, txResult := range txResults { + txHash, err := txResult.Hash() + if err != nil { + return nil, err + } + txs = append(txs, txHash...) + } + + return crypto.SHA3Hash(txs), nil +} + +// Actor Tree Helpers + +func (p *PostgresContext) updateActorsTree(actorType types.ActorType) error { + actors, err := p.getActorsUpdatedAtHeight(actorType, p.Height) + if err != nil { + return err + } + + for _, actor := range actors { + bzAddr, err := hex.DecodeString(actor.GetAddress()) + if err != nil { + return err + } + + actorBz, err := proto.Marshal(actor) + if err != nil { + return err + } + + merkleTreeName, ok := actorTypeToMerkleTreeName[actorType] + if !ok { + return fmt.Errorf("no merkle tree found for actor type: %s", actorType) + } + if _, err := p.stateTrees.merkleTrees[merkleTreeName].Update(bzAddr, actorBz); err != nil { + return err + } + } + + return nil +} + +func (p *PostgresContext) getActorsUpdatedAtHeight(actorType types.ActorType, height int64) (actors []*types.Actor, err error) { + actorSchema, ok := actorTypeToSchemaName[actorType] + if !ok { + return nil, fmt.Errorf("no schema found for actor type: %s", actorType) + } + + schemaActors, err := p.GetActorsUpdated(actorSchema, height) + if err != nil { + return nil, err + } + + actors = make([]*types.Actor, len(schemaActors)) + for i, schemaActor := range schemaActors { + actor := &types.Actor{ + ActorType: actorType, + Address: schemaActor.Address, + PublicKey: schemaActor.PublicKey, + Chains: schemaActor.Chains, + GenericParam: schemaActor.GenericParam, + StakedAmount: schemaActor.StakedAmount, + PausedHeight: schemaActor.PausedHeight, + UnstakingHeight: schemaActor.UnstakingHeight, + Output: schemaActor.Output, + } + actors[i] = actor + } + return +} + +// Account Tree Helpers + +func (p *PostgresContext) updateAccountTrees() error { + accounts, err := p.GetAccountsUpdated(p.Height) + if err != nil { + return err + } + + for _, account := range accounts { + bzAddr, err := hex.DecodeString(account.GetAddress()) + if err != nil { + return err + } + + accBz, err := proto.Marshal(account) + if err != nil { + return err + } + + if _, err := p.stateTrees.merkleTrees[accountMerkleTree].Update(bzAddr, accBz); err != nil { + return err + } + } + + return nil +} + +func (p *PostgresContext) updatePoolTrees() error { + pools, err := p.GetPoolsUpdated(p.Height) + if err != nil { + return err + } + + for _, pool := range pools { + bzAddr := []byte(pool.GetAddress()) + accBz, err := proto.Marshal(pool) + if err != nil { + return err + } + + if _, err := p.stateTrees.merkleTrees[poolMerkleTree].Update(bzAddr, accBz); err != nil { + return err + } + } + + return nil +} + +// Data Tree Helpers + +func (p *PostgresContext) updateTransactionsTree() error { + txResults, err := p.txIndexer.GetByHeight(p.Height, false) + if err != nil { + return err + } + + for _, txResult := range txResults { + txHash, err := txResult.Hash() + if err != nil { + return err + } + if _, err := p.stateTrees.merkleTrees[transactionsMerkleTree].Update(txHash, txResult.GetTx()); err != nil { + return err + } + } + + return nil +} + +func (p *PostgresContext) updateParamsTree() error { + // TODO(#361): Create a core starter task to implement this + return nil +} + +func (p *PostgresContext) updateFlagsTree() error { + // TODO(#361): Create a core starter task to implement this + return nil +} diff --git a/persistence/test/account_test.go b/persistence/test/account_test.go index 96ed1365c..fd679c1ab 100644 --- a/persistence/test/account_test.go +++ b/persistence/test/account_test.go @@ -20,7 +20,7 @@ import ( // TODO(andrew): Find all places where we import twice and update the imports appropriately. func FuzzAccountAmount(f *testing.F) { - db := NewFuzzTestPostgresContext(f, 0) + db := NewTestPostgresContext(f, 0) operations := []string{ "AddAmount", "SubAmount", @@ -92,7 +92,6 @@ func TestDefaultNonExistentAccountAmount(t *testing.T) { db := NewTestPostgresContext(t, 0) addr, err := crypto.GenerateAddress() require.NoError(t, err) - accountAmount, err := db.GetAccountAmount(addr, db.Height) require.NoError(t, err) require.Equal(t, "0", accountAmount) @@ -142,6 +141,54 @@ func TestAddAccountAmount(t *testing.T) { require.Equal(t, expectedAccountAmount, accountAmount, "unexpected amount after add") } +func TestAccountsUpdatedAtHeight(t *testing.T) { + db := NewTestPostgresContext(t, 0) + numAccsInTestGenesis := 8 + + // Check num accounts in genesis + accs, err := db.GetAccountsUpdated(0) + require.NoError(t, err) + require.Equal(t, numAccsInTestGenesis, len(accs)) + + // Insert a new account at height 0 + _, err = createAndInsertNewAccount(db) + require.NoError(t, err) + + // Verify that num accounts incremented by 1 + accs, err = db.GetAccountsUpdated(0) + require.NoError(t, err) + require.Equal(t, numAccsInTestGenesis+1, len(accs)) + + // Close context at height 0 without committing new account + require.NoError(t, db.Close()) + // start a new context at height 1 + db = NewTestPostgresContext(t, 1) + + // Verify that num accounts at height 0 is genesis because the new one was not committed + accs, err = db.GetAccountsUpdated(0) + require.NoError(t, err) + require.Equal(t, numAccsInTestGenesis, len(accs)) + + // Insert a new account at height 1 + _, err = createAndInsertNewAccount(db) + require.NoError(t, err) + + // Verify that num accounts updated height 1 is 1 + accs, err = db.GetAccountsUpdated(1) + require.NoError(t, err) + require.Equal(t, 1, len(accs)) + + // Commit & close the context at height 1 + require.NoError(t, db.Commit(nil)) + // start a new context at height 2 + db = NewTestPostgresContext(t, 2) + + // Verify only 1 account was updated at height 1 + accs, err = db.GetAccountsUpdated(1) + require.NoError(t, err) + require.Equal(t, 1, len(accs)) +} + func TestSubAccountAmount(t *testing.T) { db := NewTestPostgresContext(t, 0) account := newTestAccount(t) @@ -165,7 +212,7 @@ func TestSubAccountAmount(t *testing.T) { } func FuzzPoolAmount(f *testing.F) { - db := NewFuzzTestPostgresContext(f, 0) + db := NewTestPostgresContext(f, 0) operations := []string{ "AddAmount", "SubAmount", @@ -301,7 +348,6 @@ func TestGetAllAccounts(t *testing.T) { } else { return db.AddAccountAmount(addr, "10") } - } getAllActorsTest(t, db, db.GetAllAccounts, createAndInsertNewAccount, updateAccount, 8) @@ -317,6 +363,54 @@ func TestGetAllPools(t *testing.T) { getAllActorsTest(t, db, db.GetAllPools, createAndInsertNewPool, updatePool, 7) } +func TestPoolsUpdatedAtHeight(t *testing.T) { + db := NewTestPostgresContext(t, 0) + numPoolsInTestGenesis := 7 + + // Check num Pools in genesis + accs, err := db.GetPoolsUpdated(0) + require.NoError(t, err) + require.Equal(t, numPoolsInTestGenesis, len(accs)) + + // Insert a new Pool at height 0 + _, err = createAndInsertNewPool(db) + require.NoError(t, err) + + // Verify that num Pools incremented by 1 + accs, err = db.GetPoolsUpdated(0) + require.NoError(t, err) + require.Equal(t, numPoolsInTestGenesis+1, len(accs)) + + // Close context at height 0 without committing new Pool + require.NoError(t, db.Close()) + // start a new context at height 1 + db = NewTestPostgresContext(t, 1) + + // Verify that num Pools at height 0 is genesis because the new one was not committed + accs, err = db.GetPoolsUpdated(0) + require.NoError(t, err) + require.Equal(t, numPoolsInTestGenesis, len(accs)) + + // Insert a new Pool at height 1 + _, err = createAndInsertNewPool(db) + require.NoError(t, err) + + // Verify that num Pools updated height 1 is 1 + accs, err = db.GetPoolsUpdated(1) + require.NoError(t, err) + require.Equal(t, 1, len(accs)) + + // Commit & close the context at height 1 + require.NoError(t, db.Commit(nil)) + // start a new context at height 2 + db = NewTestPostgresContext(t, 2) + + // Verify only 1 Pool was updated at height 1 + accs, err = db.GetPoolsUpdated(1) + require.NoError(t, err) + require.Equal(t, 1, len(accs)) +} + // --- Helpers --- func createAndInsertNewAccount(db *persistence.PostgresContext) (modules.Account, error) { diff --git a/persistence/test/application_test.go b/persistence/test/application_test.go index c1be72291..87357a91d 100644 --- a/persistence/test/application_test.go +++ b/persistence/test/application_test.go @@ -19,6 +19,13 @@ func FuzzApplication(f *testing.F) { types.ApplicationActor) } +func TestGetApplicationsUpdatedAtHeight(t *testing.T) { + getApplicationsUpdatedFunc := func(db *persistence.PostgresContext, height int64) ([]*types.Actor, error) { + return db.GetActorsUpdated(types.ApplicationActor, height) + } + getAllActorsUpdatedAtHeightTest(t, createAndInsertDefaultTestApp, getApplicationsUpdatedFunc, 1) +} + func TestInsertAppAndExists(t *testing.T) { db := NewTestPostgresContext(t, 0) diff --git a/persistence/test/benchmark_state_test.go b/persistence/test/benchmark_state_test.go new file mode 100644 index 000000000..b4ce68f02 --- /dev/null +++ b/persistence/test/benchmark_state_test.go @@ -0,0 +1,174 @@ +package test + +import ( + "encoding/hex" + "fmt" + "io/ioutil" + "log" + "math/rand" + "os" + "reflect" + "regexp" + "strconv" + "testing" + + "github.com/pokt-network/pocket/persistence" + "github.com/pokt-network/pocket/persistence/indexer" + "github.com/pokt-network/pocket/shared/modules" +) + +const ( + maxStringAmount = 1000000000000000000 +) + +var isModifierRe = regexp.MustCompile(`^(Insert|Set|Add|Subtract)`) // Add Update? + +// INVESTIGATE: This benchmark can be used to experiment with different Merkle Tree implementations +// and key-value stores. +// IMPROVE(#361): Improve the output of this benchmark to be more informative and human readable. +func BenchmarkStateHash(b *testing.B) { + log.SetOutput(ioutil.Discard) + defer log.SetOutput(os.Stderr) + + clearAllState() + b.Cleanup(clearAllState) + + // NOTE: The idiomatic way to run Go benchmarks is to use `b.N` and the `-benchtime` flag, + // to specify how long the benchmark should take. However, the code below is non-idiomatic + // since our goal is to test a specific we use a fixed number of iterations + testCases := []struct { + numHeights int64 + numTxPerHeight int + numOpsPerTx int + }{ + {1, 1, 1}, + {1, 1, 10}, + {1, 10, 10}, + + {10, 1, 1}, + {10, 1, 10}, + {10, 10, 10}, + + // This takes a VERY long time to run + // {100, 1, 1}, + // {100, 1, 100}, + // {100, 100, 100}, + } + + for _, testCase := range testCases { + numHeights := testCase.numHeights + numTxPerHeight := testCase.numTxPerHeight + numOpsPerTx := testCase.numOpsPerTx + + // Since this is a benchmark, errors are not + b.Run(fmt.Sprintf("height=%d;txs=%d,ops=%d", numHeights, numTxPerHeight, numOpsPerTx), func(b *testing.B) { + for height := int64(0); height < numHeights; height++ { + db := NewTestPostgresContext(b, height) + for txIdx := 0; txIdx < numTxPerHeight; txIdx++ { + for opIdx := 0; opIdx < numOpsPerTx; opIdx++ { + callRandomDatabaseModifierFunc(db, false) + } + db.IndexTransaction(modules.TxResult(getRandomTxResult(height))) + } + db.ComputeAppHash() + db.Commit([]byte("placeholder")) + db.Release() + } + }) + } +} + +// Calls a random database modifier function on the given persistence context +func callRandomDatabaseModifierFunc( + p *persistence.PostgresContext, + mustSucceed bool, +) (string, []reflect.Value, error) { + t := reflect.TypeOf(modules.PersistenceWriteContext(p)) + numMethods := t.NumMethod() + + // Select a random method and loops until a successful invocation takes place +MethodLoop: + for { + method := t.Method(rand.Intn(numMethods)) + methodName := method.Name + numArgs := method.Type.NumIn() + + // Preliminary filter to determine which functions we're interested in trying to call + if !isModifierRe.MatchString(methodName) { + continue + } + + // Build a random set of arguments to pass to the function being called + var callArgs []reflect.Value + for i := 1; i < numArgs; i++ { + var v reflect.Value + arg := method.Type.In(i) + switch arg.Kind() { + case reflect.String: + // String values in modifier functions are usually amounts + v = reflect.ValueOf(getRandomIntString(maxStringAmount)) + case reflect.Slice: + switch arg.Elem().Kind() { + case reflect.Uint8: + v = reflect.ValueOf([]uint8{0}) + case reflect.String: + v = reflect.ValueOf([]string{"abc"}) + default: + continue MethodLoop // IMPROVE: Slices of other types not supported yet + } + case reflect.Bool: + v = reflect.ValueOf(rand.Intn(2) == 1) + case reflect.Uint8: + v = reflect.ValueOf(uint8(rand.Intn(2 ^ 8 - 1))) + case reflect.Int32: + v = reflect.ValueOf(rand.Int31()) + case reflect.Int64: + v = reflect.ValueOf(rand.Int63()) + case reflect.Int: + v = reflect.ValueOf(rand.Int()) + case reflect.Pointer: + fallthrough + default: + continue MethodLoop // IMPROVE: Other types not supported yet + } + callArgs = append(callArgs, v) + } + res := reflect.ValueOf(p).MethodByName(method.Name).Call(callArgs) + var err error + if v := res[0].Interface(); v != nil { + if mustSucceed { + continue MethodLoop + } + err = v.(error) + } + return methodName, callArgs, err + } +} + +func getRandomTxResult(height int64) *indexer.TxRes { + return &indexer.TxRes{ + Tx: getRandomBytes(50), + Height: height, + Index: 0, + ResultCode: 0, + Error: "TODO", + SignerAddr: "TODO", + RecipientAddr: "TODO", + MessageType: "TODO", + } +} + +func getRandomIntString(n int) string { + return strconv.Itoa(rand.Intn(n)) +} + +// NOTE: This is not current used but was added for future usage. +func getRandomString(numChars int64) string { + return string(getRandomBytes(numChars)) +} + +func getRandomBytes(numBytes int64) []byte { + bz := make([]byte, numBytes) + rand.Read(bz) + return []byte(hex.EncodeToString(bz)) +} diff --git a/persistence/test/block_test.go b/persistence/test/block_test.go new file mode 100644 index 000000000..bd92779c6 --- /dev/null +++ b/persistence/test/block_test.go @@ -0,0 +1,25 @@ +package test + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestGetBlockStateHash(t *testing.T) { + db := NewTestPostgresContext(t, 0) + + // Cannot get prev hash at height 0 + appHash, err := db.GetBlockHash(0) + require.NoError(t, err) + require.NotEmpty(t, appHash) + + // Cannot get a hash at height 1 since it doesn't exist + appHash, err = db.GetBlockHash(1) + require.Error(t, err) + + // Cannot get a hash at height 10 since it doesn't exist + appHash, err = db.GetBlockHash(10) + require.Error(t, err) + +} diff --git a/persistence/test/fisherman_test.go b/persistence/test/fisherman_test.go index 11859fde6..a22f7f9fe 100644 --- a/persistence/test/fisherman_test.go +++ b/persistence/test/fisherman_test.go @@ -26,6 +26,13 @@ func TestGetSetFishermanStakeAmount(t *testing.T) { getTestGetSetStakeAmountTest(t, db, createAndInsertDefaultTestFisherman, db.GetFishermanStakeAmount, db.SetFishermanStakeAmount, 1) } +func TestGetFishermanUpdatedAtHeight(t *testing.T) { + getFishermanUpdatedFunc := func(db *persistence.PostgresContext, height int64) ([]*types.Actor, error) { + return db.GetActorsUpdated(types.FishermanActor, height) + } + getAllActorsUpdatedAtHeightTest(t, createAndInsertDefaultTestFisherman, getFishermanUpdatedFunc, 1) +} + func TestInsertFishermanAndExists(t *testing.T) { db := NewTestPostgresContext(t, 0) diff --git a/persistence/test/generic_test.go b/persistence/test/generic_test.go index c58d15889..cb4555f04 100644 --- a/persistence/test/generic_test.go +++ b/persistence/test/generic_test.go @@ -2,10 +2,11 @@ package test import ( "encoding/hex" - "github.com/pokt-network/pocket/persistence/types" "reflect" "testing" + "github.com/pokt-network/pocket/persistence/types" + "github.com/pokt-network/pocket/persistence" "github.com/stretchr/testify/require" ) @@ -150,6 +151,58 @@ func getTestGetSetStakeAmountTest[T any]( require.Equal(t, newStakeAmount, stakeAmountAfter, "unexpected status") } +func getAllActorsUpdatedAtHeightTest[T any]( + t *testing.T, + createAndInsertTestActor func(*persistence.PostgresContext) (*T, error), + getActorsUpdated func(*persistence.PostgresContext, int64) ([]*T, error), + numActorsInTestGenesis int, +) { + db := NewTestPostgresContext(t, 0) + + // Check num actors in genesis + accs, err := getActorsUpdated(db, 0) + require.NoError(t, err) + require.Equal(t, numActorsInTestGenesis, len(accs)) + + // Insert a new actor at height 0 + _, err = createAndInsertTestActor(db) + require.NoError(t, err) + + // Verify that num actors incremented by 1 + accs, err = getActorsUpdated(db, 0) + require.NoError(t, err) + require.Equal(t, numActorsInTestGenesis+1, len(accs)) + + // Close context at height 0 without committing new Pool + require.NoError(t, db.Close()) + // start a new context at height 1 + db = NewTestPostgresContext(t, 1) + + // Verify that num actors at height 0 is genesis because the new one was not committed + accs, err = getActorsUpdated(db, 0) + require.NoError(t, err) + require.Equal(t, numActorsInTestGenesis, len(accs)) + + // Insert a new actor at height 1 + _, err = createAndInsertTestActor(db) + require.NoError(t, err) + + // Verify that num actors updated height 1 is 1 + accs, err = getActorsUpdated(db, 1) + require.NoError(t, err) + require.Equal(t, 1, len(accs)) + + // Commit & close the context at height 1 + require.NoError(t, db.Commit(nil)) + // start a new context at height 2 + db = NewTestPostgresContext(t, 2) + + // Verify only 1 actor was updated at height 1 + accs, err = getActorsUpdated(db, 1) + require.NoError(t, err) + require.Equal(t, 1, len(accs)) +} + func getActorValues(_ types.ProtocolActorSchema, actorValue reflect.Value) *types.Actor { chains := make([]string, 0) if actorValue.FieldByName("Chains").Kind() != 0 { diff --git a/persistence/test/module_test.go b/persistence/test/module_test.go index a0c54c45e..02b2ed0c7 100644 --- a/persistence/test/module_test.go +++ b/persistence/test/module_test.go @@ -7,20 +7,20 @@ import ( ) func TestPersistenceContextParallelReadWrite(t *testing.T) { - t.Cleanup(func() { - require.NoError(t, testPersistenceMod.NewWriteContext().Release()) - }) + prepareAndCleanContext(t) + // variables for testing poolName := "fake" poolAddress := []byte("address") originalAmount := "15" modifiedAmount := "10" + quorumCert := []byte("quorumCert") // setup a write context, insert a pool and commit it context, err := testPersistenceMod.NewRWContext(0) require.NoError(t, err) require.NoError(t, context.InsertPool(poolName, poolAddress, originalAmount)) - require.NoError(t, context.Commit(nil)) + require.NoError(t, context.Commit(quorumCert)) // verify the insert in the previously committed context worked contextA, err := testPersistenceMod.NewRWContext(0) @@ -49,9 +49,8 @@ func TestPersistenceContextParallelReadWrite(t *testing.T) { } func TestPersistenceContextTwoWritesErrors(t *testing.T) { - t.Cleanup(func() { - require.NoError(t, testPersistenceMod.NewWriteContext().Release()) - }) + prepareAndCleanContext(t) + // Opening up first write context succeeds _, err := testPersistenceMod.NewRWContext(0) require.NoError(t, err) @@ -66,6 +65,8 @@ func TestPersistenceContextTwoWritesErrors(t *testing.T) { } func TestPersistenceContextSequentialWrites(t *testing.T) { + prepareAndCleanContext(t) + // Opening up first write context succeeds writeContext1, err := testPersistenceMod.NewRWContext(0) require.NoError(t, err) @@ -89,6 +90,8 @@ func TestPersistenceContextSequentialWrites(t *testing.T) { } func TestPersistenceContextMultipleParallelReads(t *testing.T) { + prepareAndCleanContext(t) + // Opening up first read context succeeds readContext1, err := testPersistenceMod.NewReadContext(0) require.NoError(t, err) @@ -105,3 +108,10 @@ func TestPersistenceContextMultipleParallelReads(t *testing.T) { require.NoError(t, readContext2.Close()) require.NoError(t, readContext3.Close()) } + +func prepareAndCleanContext(t *testing.T) { + // Cleanup context after the test + t.Cleanup(clearAllState) + + clearAllState() +} diff --git a/persistence/test/service_node_test.go b/persistence/test/service_node_test.go index 51c1e25c9..4e3f7e28b 100644 --- a/persistence/test/service_node_test.go +++ b/persistence/test/service_node_test.go @@ -24,6 +24,13 @@ func TestGetSetServiceNodeStakeAmount(t *testing.T) { getTestGetSetStakeAmountTest(t, db, createAndInsertDefaultTestServiceNode, db.GetServiceNodeStakeAmount, db.SetServiceNodeStakeAmount, 1) } +func TestGetServiceNodeUpdatedAtHeight(t *testing.T) { + getServiceNodeUpdatedFunc := func(db *persistence.PostgresContext, height int64) ([]*types.Actor, error) { + return db.GetActorsUpdated(types.ServiceNodeActor, height) + } + getAllActorsUpdatedAtHeightTest(t, createAndInsertDefaultTestServiceNode, getServiceNodeUpdatedFunc, 1) +} + func TestInsertServiceNodeAndExists(t *testing.T) { db := NewTestPostgresContext(t, 0) diff --git a/persistence/test/setup_test.go b/persistence/test/setup_test.go index 290c79453..fbd0a0c79 100644 --- a/persistence/test/setup_test.go +++ b/persistence/test/setup_test.go @@ -16,6 +16,7 @@ import ( "github.com/pokt-network/pocket/runtime" "github.com/pokt-network/pocket/runtime/test_artifacts" "github.com/pokt-network/pocket/shared/converters" + "github.com/pokt-network/pocket/shared/messaging" "github.com/pokt-network/pocket/shared/modules" "github.com/stretchr/testify/require" "golang.org/x/exp/slices" @@ -58,22 +59,8 @@ func TestMain(m *testing.M) { os.Exit(exitCode) } -func NewTestPostgresContext(t *testing.T, height int64) *persistence.PostgresContext { - ctx, err := testPersistenceMod.NewRWContext(height) - require.NoError(t, err) - - db, ok := ctx.(*persistence.PostgresContext) - require.True(t, ok) - - t.Cleanup(func() { - require.NoError(t, db.Release()) - require.NoError(t, db.ResetContext()) - }) - - return db -} - -func NewFuzzTestPostgresContext(f *testing.F, height int64) *persistence.PostgresContext { +// IMPROVE: Look into returning `testPersistenceMod` to avoid exposing underlying abstraction. +func NewTestPostgresContext(t testing.TB, height int64) *persistence.PostgresContext { ctx, err := testPersistenceMod.NewRWContext(height) if err != nil { log.Fatalf("Error creating new context: %v\n", err) @@ -84,24 +71,25 @@ func NewFuzzTestPostgresContext(f *testing.F, height int64) *persistence.Postgre log.Fatalf("Error casting RW context to Postgres context") } - f.Cleanup(func() { - if err := db.Release(); err != nil { - f.FailNow() - } - if err := db.ResetContext(); err != nil { - f.FailNow() - } - }) + // TECHDEBT: This should not be part of `NewTestPostgresContext`. It causes unnecessary resets + // if we call `NewTestPostgresContext` more than once in a single test. + t.Cleanup(resetStateToGenesis) return db } -// TODO(andrew): Take in `t testing.T` as a parameter and error if there's an issue +// TODO(olshansky): Take in `t testing.T` as a parameter and error if there's an issue func newTestPersistenceModule(databaseUrl string) modules.PersistenceModule { + // HACK: See `runtime/test_artifacts/generator.go` for why we're doing this to get deterministic key generation. + os.Setenv(test_artifacts.PrivateKeySeedEnv, "42") + defer os.Unsetenv(test_artifacts.PrivateKeySeedEnv) + cfg := runtime.NewConfig(&runtime.BaseConfig{}, runtime.WithPersistenceConfig(&types.PersistenceConfig{ PostgresUrl: databaseUrl, NodeSchema: testSchema, BlockStorePath: "", + TxIndexerPath: "", + TreesStoreDir: "", })) genesisState, _ := test_artifacts.NewGenesisState(5, 1, 1, 1) runtimeCfg := runtime.NewManager(cfg, genesisState) @@ -118,12 +106,11 @@ func fuzzSingleProtocolActor( f *testing.F, newTestActor func() (*types.Actor, error), getTestActor func(db *persistence.PostgresContext, address string) (*types.Actor, error), - protocolActorSchema types.ProtocolActorSchema) { - - db := NewFuzzTestPostgresContext(f, 0) - - err := db.DebugClearAll() - require.NoError(f, err) + protocolActorSchema types.ProtocolActorSchema, +) { + // Clear the genesis state. + clearAllState() + db := NewTestPostgresContext(f, 0) actor, err := newTestActor() require.NoError(f, err) @@ -326,3 +313,29 @@ func getRandomBigIntString() string { func setRandomSeed() { rand.Seed(time.Now().UnixNano()) } + +// This is necessary for unit tests that are dependant on a baseline genesis state +func resetStateToGenesis() { + if err := testPersistenceMod.ReleaseWriteContext(); err != nil { + log.Fatalf("Error releasing write context: %v\n", err) + } + if err := testPersistenceMod.HandleDebugMessage(&messaging.DebugMessage{ + Action: messaging.DebugMessageAction_DEBUG_PERSISTENCE_RESET_TO_GENESIS, + Message: nil, + }); err != nil { + log.Fatalf("Error clearing state: %v\n", err) + } +} + +// This is necessary for unit tests that are dependant on a completely clear state when starting +func clearAllState() { + if err := testPersistenceMod.ReleaseWriteContext(); err != nil { + log.Fatalf("Error releasing write context: %v\n", err) + } + if err := testPersistenceMod.HandleDebugMessage(&messaging.DebugMessage{ + Action: messaging.DebugMessageAction_DEBUG_PERSISTENCE_CLEAR_STATE, + Message: nil, + }); err != nil { + log.Fatalf("Error clearing state: %v\n", err) + } +} diff --git a/persistence/test/state_test.go b/persistence/test/state_test.go new file mode 100644 index 000000000..f344f46ae --- /dev/null +++ b/persistence/test/state_test.go @@ -0,0 +1,237 @@ +package test + +import ( + "encoding/binary" + "encoding/hex" + "fmt" + "reflect" + "strconv" + "testing" + + "github.com/pokt-network/pocket/persistence/indexer" + "github.com/pokt-network/pocket/persistence/types" + "github.com/pokt-network/pocket/shared/codec" + "github.com/pokt-network/pocket/shared/modules" + "github.com/stretchr/testify/require" +) + +const ( + txBytesRandSeed = "42" + txBytesSize = 42 + + proposerBytesSize = 10 + quorumCertBytesSize = 10 + + // This value is arbitrarily selected, but needs to be a constant to guarantee deterministic tests. + initialStakeAmount = 42 +) + +type TestReplayableOperation struct { + methodName string + args []reflect.Value +} +type TestReplayableTransaction struct { + operations []*TestReplayableOperation + txResult modules.TxResult +} + +type TestReplayableBlock struct { + height int64 + txs []*TestReplayableTransaction + hash []byte + proposer []byte + quorumCert []byte +} + +func TestStateHash_DeterministicStateWhenUpdatingAppStake(t *testing.T) { + // These hashes were determined manually by running the test, but hardcoded to guarantee + // that the business logic doesn't change and that they remain deterministic. Anytime the business + // logic changes, these hashes will need to be updated based on the test output. + encodedAppHash := []string{ + "b076081d48f6652d2302c974f20e5371b4728c7950735f6617aac7b6be62f581", + "171af2b820d2a65861c4e63f0cdd9c8bdde4798e6ace28c47d0e83467848ab02", + "b168dff3a83215f12093e548aa22cdf907fbfdb1e12d217ffbb4a07beca065f1", + } + + stakeAmount := initialStakeAmount + for i := 0; i < len(encodedAppHash); i++ { + // Get the context at the new height and retrieve one of the apps + height := int64(i + 1) + heightBz := heightToBytes(height) + expectedAppHash := encodedAppHash[i] + + db := NewTestPostgresContext(t, height) + + apps, err := db.GetAllApps(height) + require.NoError(t, err) + app := apps[0] + + addrBz, err := hex.DecodeString(app.GetAddress()) + require.NoError(t, err) + + // Update the app's stake + stakeAmount += 1 // change the stake amount + stakeAmountStr := strconv.Itoa(stakeAmount) + err = db.SetAppStakeAmount(addrBz, stakeAmountStr) + require.NoError(t, err) + + txBz := []byte("a tx, i am, which set the app stake amount to " + stakeAmountStr) + txResult := indexer.TxRes{ + Tx: txBz, + Height: height, + Index: 0, + ResultCode: 0, + Error: "TODO", + SignerAddr: "TODO", + RecipientAddr: "TODO", + MessageType: "TODO", + } + + err = db.IndexTransaction(modules.TxResult(&txResult)) + require.NoError(t, err) + + // Update the state hash + appHash, err := db.ComputeAppHash() + require.NoError(t, err) + require.Equal(t, expectedAppHash, hex.EncodeToString(appHash)) + + // Commit the transactions above + proposer := []byte("placeholderProposer") + quorumCert := []byte("placeholderQuorumCert") + + db.SetProposalBlock(hex.EncodeToString(appHash), proposer, quorumCert, [][]byte{txBz}) + + err = db.Commit(quorumCert) + require.NoError(t, err) + + // Retrieve the block + blockBz, err := testPersistenceMod.GetBlockStore().Get(heightBz) + require.NoError(t, err) + + // Verify the block contents + var block types.Block + err = codec.GetCodec().Unmarshal(blockBz, &block) + require.NoError(t, err) + require.Equal(t, expectedAppHash, block.StateHash) // verify block hash + if i > 0 { + require.Equal(t, encodedAppHash[i-1], block.PrevStateHash) // verify chain chain + } + } +} + +// This unit test generates random transactions and creates random state changes, but checks +// that replaying them will result in the same state hash, guaranteeing the integrity of the +// state hash. +func TestStateHash_ReplayingRandomTransactionsIsDeterministic(t *testing.T) { + testCases := []struct { + numHeights int64 + numTxsPerHeight int + numOpsPerTx int + numReplays int + }{ + {1, 2, 1, 3}, + {10, 2, 5, 5}, + } + + for _, testCase := range testCases { + numHeights := testCase.numHeights + numTxsPerHeight := testCase.numTxsPerHeight + numOpsPerTx := testCase.numOpsPerTx + numReplays := testCase.numReplays + + t.Run(fmt.Sprintf("ReplayingRandomTransactionsIsDeterministic(%d;%d,%d,%d", numHeights, numTxsPerHeight, numOpsPerTx, numReplays), func(t *testing.T) { + t.Cleanup(clearAllState) + clearAllState() + + replayableBlocks := make([]*TestReplayableBlock, numHeights) + + for height := int64(0); height < int64(numHeights); height++ { + db := NewTestPostgresContext(t, height) + replayableTxs := make([]*TestReplayableTransaction, numTxsPerHeight) + + for txIdx := 0; txIdx < numTxsPerHeight; txIdx++ { + replayableOps := make([]*TestReplayableOperation, numOpsPerTx) + + for opIdx := 0; opIdx < numOpsPerTx; opIdx++ { + methodName, args, err := callRandomDatabaseModifierFunc(db, true) + require.NoError(t, err) + + replayableOps[opIdx] = &TestReplayableOperation{ + methodName: methodName, + args: args, + } + } + + txResult := modules.TxResult(getRandomTxResult(height)) + err := db.IndexTransaction(txResult) + require.NoError(t, err) + + replayableTxs[txIdx] = &TestReplayableTransaction{ + operations: replayableOps, + txResult: txResult, + } + } + + appHash, err := db.ComputeAppHash() + require.NoError(t, err) + + proposer := getRandomBytes(proposerBytesSize) + quorumCert := getRandomBytes(quorumCertBytesSize) + + err = db.Commit(quorumCert) + require.NoError(t, err) + + replayableBlocks[height] = &TestReplayableBlock{ + height: height, + txs: replayableTxs, + hash: appHash, + proposer: proposer, + quorumCert: quorumCert, + } + } + + for i := 0; i < numReplays; i++ { + t.Run("verify block", func(t *testing.T) { + verifyReplayableBlocks(t, replayableBlocks) + }) + } + }) + } +} + +func TestStateHash_TreeUpdatesAreIdempotent(t *testing.T) { + // ADDTEST(#361): Create an issue dedicated to increasing the test coverage for state hashes +} + +func TestStateHash_TreeUpdatesNegativeTestCase(t *testing.T) { + // ADDTEST(#361): Create an issue dedicated to increasing the test coverage for state hashes +} + +func verifyReplayableBlocks(t *testing.T, replayableBlocks []*TestReplayableBlock) { + t.Cleanup(clearAllState) + clearAllState() + + for _, block := range replayableBlocks { + db := NewTestPostgresContext(t, block.height) + + for _, tx := range block.txs { + for _, op := range tx.operations { + require.Nil(t, reflect.ValueOf(db).MethodByName(op.methodName).Call(op.args)[0].Interface()) + } + require.NoError(t, db.IndexTransaction(tx.txResult)) + } + + appHash, err := db.ComputeAppHash() + require.NoError(t, err) + require.Equal(t, block.hash, appHash) + + err = db.Commit(block.quorumCert) + require.NoError(t, err) + } +} + +func heightToBytes(height int64) []byte { + heightBytes := make([]byte, 8) + binary.LittleEndian.PutUint64(heightBytes, uint64(height)) + return heightBytes +} diff --git a/persistence/test/validator_test.go b/persistence/test/validator_test.go index f5d5c1d71..86ce0db18 100644 --- a/persistence/test/validator_test.go +++ b/persistence/test/validator_test.go @@ -24,6 +24,13 @@ func TestGetSetValidatorStakeAmount(t *testing.T) { getTestGetSetStakeAmountTest(t, db, createAndInsertDefaultTestValidator, db.GetValidatorStakeAmount, db.SetValidatorStakeAmount, 1) } +func TestGetValidatorUpdatedAtHeight(t *testing.T) { + getValidatorsUpdatedFunc := func(db *persistence.PostgresContext, height int64) ([]*types.Actor, error) { + return db.GetActorsUpdated(types.ValidatorActor, height) + } + getAllActorsUpdatedAtHeightTest(t, createAndInsertDefaultTestValidator, getValidatorsUpdatedFunc, 5) +} + func TestInsertValidatorAndExists(t *testing.T) { db := NewTestPostgresContext(t, 0) @@ -50,7 +57,7 @@ func TestInsertValidatorAndExists(t *testing.T) { exists, err = db.GetValidatorExists(addrBz2, 0) require.NoError(t, err) - require.False(t, exists, "actor that should not exist at previous height validatorears to") + require.False(t, exists, "actor that should not exist at previous height does") exists, err = db.GetValidatorExists(addrBz2, 1) require.NoError(t, err) require.True(t, exists, "actor that should exist at current height does not") diff --git a/persistence/types/account.go b/persistence/types/account.go index 9f90f5526..8e2feea9a 100644 --- a/persistence/types/account.go +++ b/persistence/types/account.go @@ -79,3 +79,19 @@ func SelectPools(height int64, tableName string) string { ORDER BY name, height DESC `, tableName, height) } + +func GetAccountsUpdatedAtHeightQuery(height int64) string { + return SelectAtHeight(fmt.Sprintf("%s,%s", AddressCol, BalanceCol), height, AccountTableName) +} + +func GetPoolsUpdatedAtHeightQuery(height int64) string { + return SelectAtHeight(fmt.Sprintf("%s,%s", NameCol, BalanceCol), height, PoolTableName) +} + +func ClearAllAccounts() string { + return fmt.Sprintf(`DELETE FROM %s`, AccountTableName) +} + +func ClearAllPools() string { + return fmt.Sprintf(`DELETE FROM %s`, PoolTableName) +} diff --git a/persistence/types/base_actor.go b/persistence/types/base_actor.go index 7c1134ec3..a21f65e62 100644 --- a/persistence/types/base_actor.go +++ b/persistence/types/base_actor.go @@ -1,5 +1,6 @@ package types +// REFACTOR: Move schema related functions to a separate sub-package import "github.com/pokt-network/pocket/shared/modules" var _ ProtocolActorSchema = &BaseProtocolActorSchema{} @@ -41,6 +42,10 @@ func (actor *BaseProtocolActorSchema) GetChainsTableSchema() string { return protocolActorChainsTableSchema(actor.chainsHeightConstraintName) } +func (actor *BaseProtocolActorSchema) GetUpdatedAtHeightQuery(height int64) string { + return SelectAtHeight(AddressCol, height, actor.tableName) +} + func (actor *BaseProtocolActorSchema) GetQuery(address string, height int64) string { return Select(AllColsSelector, address, height, actor.tableName) } diff --git a/persistence/types/protocol_actor.go b/persistence/types/protocol_actor.go index 1052dc3d6..709457b6c 100644 --- a/persistence/types/protocol_actor.go +++ b/persistence/types/protocol_actor.go @@ -15,6 +15,9 @@ type ProtocolActorSchema interface { GetActorSpecificColName() string /*** Read/Get Queries ***/ + + // Returns a query to retrieve the addresses of all the Actors updated at that specific height + GetUpdatedAtHeightQuery(height int64) string // Returns a query to retrieve all of a single Actor's attributes. GetQuery(address string, height int64) string // Returns all actors at that height diff --git a/persistence/types/shared_sql.go b/persistence/types/shared_sql.go index b99e36a2c..a0b04820a 100644 --- a/persistence/types/shared_sql.go +++ b/persistence/types/shared_sql.go @@ -69,6 +69,11 @@ func protocolActorChainsTableSchema(constraintName string) string { )`, AddressCol, ChainIDCol, HeightCol, DefaultBigInt, constraintName, AddressCol, ChainIDCol, HeightCol) } +func SelectAtHeight(selector string, height int64, tableName string) string { + return fmt.Sprintf(`SELECT %s FROM %s WHERE height=%d`, + selector, tableName, height) +} + func Select(selector, address string, height int64, tableName string) string { return fmt.Sprintf(`SELECT %s FROM %s WHERE address='%s' AND height<=%d ORDER BY height DESC LIMIT 1`, selector, tableName, address, height) diff --git a/runtime/test_artifacts/generator.go b/runtime/test_artifacts/generator.go index f0f6b1b37..313e4028f 100644 --- a/runtime/test_artifacts/generator.go +++ b/runtime/test_artifacts/generator.go @@ -2,7 +2,11 @@ package test_artifacts // Cross module imports are okay because this is only used for testing and not business logic import ( + "bytes" + "encoding/binary" "fmt" + "math/rand" + "os" "strconv" typesCons "github.com/pokt-network/pocket/consensus/types" @@ -18,13 +22,31 @@ import ( "google.golang.org/protobuf/types/known/timestamppb" ) -// INVESTIGATE: It seems improperly scoped that the modules have to have shared 'testing' code -// It might be an inevitability to have shared testing code, but would like more eyes on it. -// Look for opportunities to make testing completely modular +// HACK: This is a hack used to enable deterministic key generation via an environment variable. +// In order to avoid this, `NewGenesisState` and all downstream functions would need to be +// refactored. Alternatively, the seed would need to be passed via the runtime manager. +// To avoid these large scale changes, this is a temporary approach to enable deterministic +// key generation. +// IMPROVE(#361): Design a better way to generate deterministic keys for testing. +const PrivateKeySeedEnv = "DEFAULT_PRIVATE_KEY_SEED" + +var privateKeySeed int + +// Intentionally not using `init` in case the caller sets this before `NewGenesisState` is called.s +func loadPrivateKeySeed() { + privateKeySeedEnvValue := os.Getenv(PrivateKeySeedEnv) + if seedInt, err := strconv.Atoi(privateKeySeedEnvValue); err == nil { + privateKeySeed = seedInt + } else { + rand.Seed(timestamppb.Now().Seconds) + privateKeySeed = rand.Int() + } +} -// TODO (Team) this is meant to be a **temporary** replacement for the recently deprecated -// 'genesis config' option. We need to implement a real suite soon! +// IMPROVE: Generate a proper genesis suite in the future. func NewGenesisState(numValidators, numServiceNodes, numApplications, numFisherman int) (modules.GenesisState, []string) { + loadPrivateKeySeed() + apps, appsPrivateKeys := NewActors(types.ActorType_App, numApplications) vals, validatorPrivateKeys := NewActors(types.ActorType_Validator, numValidators) serviceNodes, snPrivateKeys := NewActors(types.ActorType_ServiceNode, numServiceNodes) @@ -48,6 +70,7 @@ func NewGenesisState(numValidators, numServiceNodes, numApplications, numFisherm }, ) + // TODO: Generalize this to all actors and not just validators return genesisState, validatorPrivateKeys } @@ -116,7 +139,7 @@ func NewPools() (pools []modules.Account) { // TODO (Team) in the real testing s func NewAccounts(n int, privateKeys ...string) (accounts []modules.Account) { for i := 0; i < n; i++ { - _, _, addr := GenerateNewKeysStrings() + _, _, addr := generateNewKeysStrings() if privateKeys != nil { pk, _ := crypto.NewPrivateKey(privateKeys[i]) addr = pk.Address().String() @@ -150,7 +173,7 @@ func getServiceUrl(n int) string { } func NewDefaultActor(actorType int32, genericParam string) (actor modules.Actor, privateKey string) { - privKey, pubKey, addr := GenerateNewKeysStrings() + privKey, pubKey, addr := generateNewKeysStrings() chains := defaults.DefaultChains if actorType == int32(typesPers.ActorType_Val) { chains = nil @@ -170,17 +193,22 @@ func NewDefaultActor(actorType int32, genericParam string) (actor modules.Actor, }, privKey } -func GenerateNewKeys() (privateKey crypto.PrivateKey, publicKey crypto.PublicKey, address crypto.Address) { - privateKey, _ = crypto.GeneratePrivateKey() - publicKey = privateKey.PublicKey() - address = publicKey.Address() - return -} +// TECHDEBT: This function has the side effect of incrementing the global variable `privateKeySeed` +// in order to guarantee unique keys, but that are still deterministic for testing purposes. +func generateNewKeysStrings() (privateKey, publicKey, address string) { + privateKeySeed += 1 // Different on every call but deterministic + cryptoSeed := make([]byte, crypto.SeedSize) + binary.LittleEndian.PutUint32(cryptoSeed, uint32(privateKeySeed)) + + reader := bytes.NewReader(cryptoSeed) + privateKeyBz, err := crypto.GeneratePrivateKeyWithReader(reader) + if err != nil { + panic(err) + } + + privateKey = privateKeyBz.String() + publicKey = privateKeyBz.PublicKey().String() + address = privateKeyBz.PublicKey().Address().String() -func GenerateNewKeysStrings() (privateKey, publicKey, address string) { - privKey, pubKey, addr := GenerateNewKeys() - privateKey = privKey.String() - publicKey = pubKey.String() - address = addr.String() return } diff --git a/runtime/test_artifacts/util.go b/runtime/test_artifacts/util.go index b23252a45..381adc193 100644 --- a/runtime/test_artifacts/util.go +++ b/runtime/test_artifacts/util.go @@ -65,7 +65,7 @@ func SetupPostgresDocker() (*dockertest.Pool, *dockertest.Resource, string) { } }() - resource.Expire(120) // Tell docker to hard kill the container in 120 seconds + resource.Expire(1200) // Tell docker to hard kill the container in 20 minutes poolRetryChan := make(chan struct{}, 1) retryConnectFn := func() error { @@ -95,7 +95,5 @@ func CleanupPostgresDocker(_ *testing.M, pool *dockertest.Pool, resource *docker } } -// TODO(drewsky): Remove this in favor of a golang specific solution -func CleanupTest(u utility.UtilityContext) { - u.Context.Release() -} +// CLEANUP: Remove this since it's no longer used or necessary but make sure remote tests are still passing +func CleanupTest(u utility.UtilityContext) {} diff --git a/shared/CHANGELOG.md b/shared/CHANGELOG.md index 02d7d1b2c..eb02de134 100644 --- a/shared/CHANGELOG.md +++ b/shared/CHANGELOG.md @@ -7,7 +7,20 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] -## [0.0.3] - 2022-11-14 +## [0.0.0.4] - 2022-11-30 + +Debug: + +- `ResetToGenesis` - Added the ability to reset the state to genesis +- `ClearState` - Added the ability to clear the state completely (height 0 without genesis data) + +Configs: + +- Updated the test generator to produce deterministic keys +- Added `trees_store_dir` to persistence configs +- Updated `LocalNet` configs to have an empty `tx_indexer_path` and `trees_store_dir` + +## [0.0.0.3] - 2022-11-14 ### [#353](https://github.com/pokt-network/pocket/pull/353) Remove topic from messaging diff --git a/shared/crypto/ed25519.go b/shared/crypto/ed25519.go index 97b0c385b..6967f2763 100644 --- a/shared/crypto/ed25519.go +++ b/shared/crypto/ed25519.go @@ -5,6 +5,7 @@ import ( "crypto/sha256" "encoding/hex" "encoding/json" + "io" ) const ( @@ -55,6 +56,11 @@ func GeneratePrivateKey() (PrivateKey, error) { return Ed25519PrivateKey(pk), err } +func GeneratePrivateKeyWithReader(rand io.Reader) (PrivateKey, error) { + _, pk, err := ed25519.GenerateKey(rand) + return Ed25519PrivateKey(pk), err +} + func NewPrivateKeyFromBytes(bz []byte) (PrivateKey, error) { bzLen := len(bz) if bzLen != ed25519.PrivateKeySize { diff --git a/shared/docs/PROTOCOL_STATE_HASH.md b/shared/docs/PROTOCOL_STATE_HASH.md index f66368437..1eccdc387 100644 --- a/shared/docs/PROTOCOL_STATE_HASH.md +++ b/shared/docs/PROTOCOL_STATE_HASH.md @@ -148,7 +148,7 @@ sequenceDiagram 4. Loop over all transactions proposed 5. Check if the transaction has already been applied to the local state 6. Perform the CRUD operation(s) corresponding to each transaction -7. The persistence module's internal implementation for ['Update State Hash'](../../persistence/docs/PROTOCOL_STATE_HASH.md) must be triggered +7. The persistence module's internal implementation for ['Compute State Hash'](../../persistence/docs/PROTOCOL_STATE_HASH.md) must be triggered 8. Validate that the local state hash computed is the same as that proposed ```mermaid @@ -170,12 +170,12 @@ sequenceDiagram end end %% TODO: Consolidate AppHash and StateHash - U->>+P: UpdateAppHash() + U->>+P: ComputeAppHash() P->>P: Internal Implementation - Note over P: Update State Hash + Note over P: Compute State Hash P->>-U: stateHash U->>C: stateHash - %% Validate the updated hash + %% Validate the computed hash C->>C: Compare local hash
against proposed hash ``` diff --git a/shared/messaging/envelope_test.go b/shared/messaging/envelope_test.go index ebc7efd44..fc748e94d 100644 --- a/shared/messaging/envelope_test.go +++ b/shared/messaging/envelope_test.go @@ -8,7 +8,7 @@ import ( ) func Test_UnpackMessage_Roundtrip(t *testing.T) { - someMsg := &DebugMessage{Action: DebugMessageAction_DEBUG_CLEAR_STATE} + someMsg := &DebugMessage{Action: DebugMessageAction_DEBUG_PERSISTENCE_CLEAR_STATE} packedMsg, err := PackMessage(someMsg) require.NoError(t, err) diff --git a/shared/messaging/proto/debug_message.proto b/shared/messaging/proto/debug_message.proto index ab026c35d..a6668824f 100644 --- a/shared/messaging/proto/debug_message.proto +++ b/shared/messaging/proto/debug_message.proto @@ -7,13 +7,17 @@ import "google/protobuf/any.proto"; option go_package = "github.com/pokt-network/pocket/shared/messaging"; enum DebugMessageAction { - DEBUG_ACTION_UNKNOWN = 0; - DEBUG_CONSENSUS_RESET_TO_GENESIS = 1; - DEBUG_CONSENSUS_PRINT_NODE_STATE = 2; - DEBUG_CONSENSUS_TRIGGER_NEXT_VIEW = 3; - DEBUG_CONSENSUS_TOGGLE_PACE_MAKER_MODE = 4; // toggle between manual and automatic - DEBUG_SHOW_LATEST_BLOCK_IN_STORE = 5; // toggle between manual and automatic - DEBUG_CLEAR_STATE = 6; + DEBUG_ACTION_UNKNOWN = 0; + + DEBUG_CONSENSUS_RESET_TO_GENESIS = 1; + DEBUG_CONSENSUS_PRINT_NODE_STATE = 2; + DEBUG_CONSENSUS_TRIGGER_NEXT_VIEW = 3; + DEBUG_CONSENSUS_TOGGLE_PACE_MAKER_MODE = 4; // toggle between manual and automatic + + DEBUG_SHOW_LATEST_BLOCK_IN_STORE = 5; + + DEBUG_PERSISTENCE_CLEAR_STATE = 6; + DEBUG_PERSISTENCE_RESET_TO_GENESIS = 7; } message DebugMessage { diff --git a/shared/modules/doc/CHANGELOG.md b/shared/modules/doc/CHANGELOG.md index 4312d25c9..2576ddb0b 100644 --- a/shared/modules/doc/CHANGELOG.md +++ b/shared/modules/doc/CHANGELOG.md @@ -7,6 +7,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.0.0.4] - 2022-11-30 + +- Removed `GetPrevHash` and just using `GetBlockHash` instead +- Removed `blockProtoBz` from `SetProposalBlock` interface +- Removed `GetLatestBlockTxs` and `SetLatestTxResults` in exchange for `IndexTransaction` +- Removed `SetTxResults` +- Renamed `UpdateAppHash` to `ComputeStateHash` +- Removed some getters related to the proposal block (`GetBlockTxs`, `GetBlockHash`, etc…) + ## [0.0.0.3] - 2022-11-15 PersistenceModule diff --git a/shared/modules/persistence_module.go b/shared/modules/persistence_module.go index e3fc5e580..af10ac3de 100644 --- a/shared/modules/persistence_module.go +++ b/shared/modules/persistence_module.go @@ -58,12 +58,10 @@ type PersistenceWriteContext interface { // Indexer Operations // Block Operations - // DISCUSS_IN_THIS_COMMIT: Can this function be removed ? If so, could we remove `TxResult` from the public facing interface given that we set transactions in `SetProposalBlock`? - SetTxResults(txResults []TxResult) - // TODO(#284): Remove `blockProtoBytes` - SetProposalBlock(blockHash string, blockProtoBytes, proposerAddr []byte, transactions [][]byte) error - // Store the block into persistence - UpdateAppHash() ([]byte, error) + SetProposalBlock(blockHash string, proposerAddr []byte, quorumCert []byte, transactions [][]byte) error + GetBlockTxs() [][]byte // Returns the transactions set by `SetProposalBlock` + ComputeAppHash() ([]byte, error) // Update the merkle trees, computes the new state hash, and returns in + IndexTransaction(txResult TxResult) error // TODO(#361): Look into an approach to remove `TxResult` from shared interfaces // Pool Operations AddPoolAmount(name string, amount string) error @@ -120,22 +118,16 @@ type PersistenceWriteContext interface { } type PersistenceReadContext interface { - GetHeight() (int64, error) - - // Closes the read context - Close() error + // Context Operations + GetHeight() (int64, error) // Returns the height of the context + Close() error // Closes the read context // CONSOLIDATE: BlockHash / AppHash / StateHash // Block Queries - GetPrevAppHash() (string, error) // hash from the previous block relative to the context height - GetLatestBlockHeight() (uint64, error) - GetBlockHashAtHeight(height int64) ([]byte, error) - GetBlocksPerSession(height int64) (int, error) - GetProposerAddr() []byte - GetBlockProtoBytes() []byte - GetBlockHash() string - GetBlockTxs() [][]byte - + GetLatestBlockHeight() (uint64, error) // Returns the height of the latest block in the persistence layer + GetBlockHash(height int64) ([]byte, error) // Returns the app hash corresponding to the height provided + GetProposerAddr() []byte // Returns the proposer set via `SetProposalBlock` + GetBlocksPerSession(height int64) (int, error) // TECHDEBT(#286): Deprecate this method // Indexer Queries TransactionExists(transactionHash string) (bool, error) @@ -169,7 +161,7 @@ type PersistenceReadContext interface { GetServiceNodePauseHeightIfExists(address []byte, height int64) (int64, error) GetServiceNodeOutputAddress(operator []byte, height int64) (output []byte, err error) GetServiceNodeCount(chain string, height int64) (int, error) - GetServiceNodesPerSessionAt(height int64) (int, error) + GetServiceNodesPerSessionAt(height int64) (int, error) // TECHDEBT(#286): Deprecate this method // Fisherman Queries GetAllFishermen(height int64) ([]Actor, error) diff --git a/shared/modules/types.go b/shared/modules/types.go index 41f295b33..1b9a4b909 100644 --- a/shared/modules/types.go +++ b/shared/modules/types.go @@ -47,6 +47,7 @@ type PersistenceConfig interface { GetNodeSchema() string GetBlockStorePath() string GetTxIndexerPath() string + GetTreesStoreDir() string } type P2PConfig interface { diff --git a/shared/modules/utility_module.go b/shared/modules/utility_module.go index 67705638f..816ab4967 100644 --- a/shared/modules/utility_module.go +++ b/shared/modules/utility_module.go @@ -24,6 +24,7 @@ type UtilityContext interface { ApplyBlock() (appHash []byte, err error) // Context operations + Release() error // Releases the utility context and any underlying contexts it references Commit(quorumCert []byte) error // State commitment of the current context GetPersistenceContext() PersistenceRWContext diff --git a/shared/node.go b/shared/node.go index 2d7d072ef..822d75ee8 100644 --- a/shared/node.go +++ b/shared/node.go @@ -170,6 +170,7 @@ func (node *Node) handleEvent(message *messaging.PocketEnvelope) error { } func (node *Node) handleDebugMessage(message *messaging.PocketEnvelope) error { + // Consensus Debug debugMessage, err := messaging.UnpackMessage[*messaging.DebugMessage](message) if err != nil { return err @@ -183,8 +184,10 @@ func (node *Node) handleDebugMessage(message *messaging.PocketEnvelope) error { fallthrough case messaging.DebugMessageAction_DEBUG_CONSENSUS_TOGGLE_PACE_MAKER_MODE: return node.GetBus().GetConsensusModule().HandleDebugMessage(debugMessage) + // Persistence Debug case messaging.DebugMessageAction_DEBUG_SHOW_LATEST_BLOCK_IN_STORE: return node.GetBus().GetPersistenceModule().HandleDebugMessage(debugMessage) + // Default Debug default: log.Printf("Debug message: %s \n", debugMessage.Message) } diff --git a/utility/block.go b/utility/block.go index a1f7f7394..fa30aed2f 100644 --- a/utility/block.go +++ b/utility/block.go @@ -1,6 +1,7 @@ package utility import ( + "log" "math/big" "github.com/pokt-network/pocket/shared/modules" @@ -25,11 +26,11 @@ func (u *UtilityContext) CreateAndApplyProposalBlock(proposer []byte, maxTransac if err != nil { return nil, nil, err } + // begin block lifecycle phase if err := u.BeginBlock(lastBlockByzantineVals); err != nil { return nil, nil, err } transactions := make([][]byte, 0) - txResults := make([]modules.TxResult, 0) totalTxsSizeInBytes := 0 txIndex := 0 for u.Mempool.Size() != typesUtil.ZeroInt { @@ -49,7 +50,7 @@ func (u *UtilityContext) CreateAndApplyProposalBlock(proposer []byte, maxTransac if err != nil { return nil, nil, err } - txTxsSizeInBytes -= txTxsSizeInBytes + totalTxsSizeInBytes -= txTxsSizeInBytes break // we've reached our max } txResult, err := u.ApplyTransaction(txIndex, transaction) @@ -61,31 +62,39 @@ func (u *UtilityContext) CreateAndApplyProposalBlock(proposer []byte, maxTransac totalTxsSizeInBytes -= txTxsSizeInBytes continue } + if err := u.Context.IndexTransaction(txResult); err != nil { + log.Fatalf("TODO(#327): We can apply the transaction but not index it. Crash the process for now: %v\n", err) + } + transactions = append(transactions, txBytes) - txResults = append(txResults, txResult) txIndex++ } + if err := u.EndBlock(proposer); err != nil { return nil, nil, err } - u.GetPersistenceContext().SetTxResults(txResults) // return the app hash (consensus module will get the validator set directly) - appHash, err := u.GetAppHash() + appHash, err := u.Context.ComputeAppHash() + if err != nil { + log.Fatalf("Updating the app hash failed: %v. TODO: Look into roll-backing the entire commit...\n", err) + } + return appHash, transactions, err } // TODO: Make sure to call `utility.CheckTransaction`, which calls `persistence.TransactionExists` // CLEANUP: code re-use ApplyBlock() for CreateAndApplyBlock() func (u *UtilityContext) ApplyBlock() (appHash []byte, err error) { - var txResults []modules.TxResult lastByzantineValidators, err := u.GetLastBlockByzantineValidators() if err != nil { return nil, err } + // begin block lifecycle phase if err := u.BeginBlock(lastByzantineValidators); err != nil { return nil, err } + // deliver txs lifecycle phase for index, transactionProtoBytes := range u.GetPersistenceContext().GetBlockTxs() { tx, err := typesUtil.TransactionFromBytes(transactionProtoBytes) @@ -95,31 +104,40 @@ func (u *UtilityContext) ApplyBlock() (appHash []byte, err error) { if err := tx.ValidateBasic(); err != nil { return nil, err } + // TODO(#346): Currently, the pattern is allowing nil err with an error transaction... + // Should we terminate applyBlock immediately if there's an invalid transaction? + // Or wait until the entire lifecycle is over to evaluate an 'invalid' block + // Validate and apply the transaction to the Postgres database - // DISCUSS: currently, the pattern is allowing nil err with an error transaction... - // Should we terminate applyBlock immediately if there's an invalid transaction? - // Or wait until the entire lifecycle is over to evaluate an 'invalid' block txResult, err := u.ApplyTransaction(index, tx) if err != nil { return nil, err } - // Add the transaction result to the array - txResults = append(txResults, txResult) - // TODO: if found, remove transaction from mempool + if err := u.Context.IndexTransaction(txResult); err != nil { + log.Fatalf("TODO(#327): We can apply the transaction but not index it. Crash the process for now: %v\n", err) + } + + // TODO: if found, remove transaction from mempool. // DISCUSS: What if the context is rolled back or cancelled. Do we add it back to the mempool? // if err := u.Mempool.DeleteTransaction(transaction); err != nil { // return nil, err // } } + // end block lifecycle phase if err := u.EndBlock(u.GetPersistenceContext().GetProposerAddr()); err != nil { return nil, err } - u.GetPersistenceContext().SetTxResults(txResults) // return the app hash (consensus module will get the validator set directly) - appHash, err = u.GetAppHash() - return + appHash, err = u.Context.ComputeAppHash() + if err != nil { + log.Fatalf("Updating the app hash failed: %v. TODO: Look into roll-backing the entire commit...\n", err) + return nil, typesUtil.ErrAppHash(err) + } + + // return the app hash; consensus module will get the validator set directly + return appHash, nil } func (u *UtilityContext) BeginBlock(previousBlockByzantineValidators [][]byte) typesUtil.Error { @@ -145,15 +163,6 @@ func (u *UtilityContext) EndBlock(proposer []byte) typesUtil.Error { return nil } -func (u *UtilityContext) GetAppHash() ([]byte, typesUtil.Error) { - // Get the root hash of the merkle state tree for state consensus integrity - appHash, er := u.Context.UpdateAppHash() - if er != nil { - return nil, typesUtil.ErrAppHash(er) - } - return appHash, nil -} - // HandleByzantineValidators handles the validators who either didn't sign at all or disagreed with the 2/3+ majority func (u *UtilityContext) HandleByzantineValidators(lastBlockByzantineValidators [][]byte) typesUtil.Error { latestBlockHeight, err := u.GetLatestBlockHeight() diff --git a/utility/context.go b/utility/context.go index c1dc7c39d..c213661d8 100644 --- a/utility/context.go +++ b/utility/context.go @@ -11,11 +11,14 @@ import ( type UtilityContext struct { LatestHeight int64 Mempool typesUtil.Mempool - Context *Context // IMPROVE: Consider renaming to PersistenceContext + Context *Context // IMPROVE: Rename to `persistenceContext` or `storeContext` or `reversibleContext`? } +// IMPROVE: Consider renaming to `persistenceContext` or `storeContext`? type Context struct { + // CLEANUP: Since `Context` embeds `PersistenceRWContext`, we don't need to do `u.Context.PersistenceRWContext`, but can call `u.Context` directly modules.PersistenceRWContext + // TODO(#327): `SavePoints`` have not been implemented yet SavePointsM map[string]struct{} SavePoints [][]byte } diff --git a/utility/doc/CHANGELOG.md b/utility/doc/CHANGELOG.md index 551eec1c8..cea3e3c52 100644 --- a/utility/doc/CHANGELOG.md +++ b/utility/doc/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.0.0.11] - 2022-11-30 + +- Minor lifecycle changes needed to supported the implementation of `ComputeAppHash` as a replacement for `GetAppHash` in #285 + ## [0.0.0.10] - 2022-11-15 - Propagating the `quorumCertificate` appropriately on block commit diff --git a/utility/test/actor_test.go b/utility/test/actor_test.go index f8064fa1e..25215b6a6 100644 --- a/utility/test/actor_test.go +++ b/utility/test/actor_test.go @@ -57,6 +57,7 @@ func TestUtilityContext_HandleMessageStake(t *testing.T) { require.Equal(t, defaults.DefaultStakeAmountString, actor.GetStakedAmount(), "incorrect actor stake amount") require.Equal(t, typesUtil.HeightNotUsed, actor.GetUnstakingHeight(), "incorrect actor unstaking height") require.Equal(t, outputAddress.String(), actor.GetOutput(), "incorrect actor output address") + test_artifacts.CleanupTest(ctx) }) } @@ -67,8 +68,10 @@ func TestUtilityContext_HandleMessageEditStake(t *testing.T) { t.Run(fmt.Sprintf("%s.HandleMessageEditStake", actorType.String()), func(t *testing.T) { ctx := NewTestingUtilityContext(t, 0) actor := getFirstActor(t, ctx, actorType) + addrBz, err := hex.DecodeString(actor.GetAddress()) require.NoError(t, err) + msg := &typesUtil.MessageEditStake{ Address: addrBz, Chains: defaults.DefaultChains, @@ -76,7 +79,6 @@ func TestUtilityContext_HandleMessageEditStake(t *testing.T) { Signer: addrBz, ActorType: actorType, } - msgChainsEdited := proto.Clone(msg).(*typesUtil.MessageEditStake) msgChainsEdited.Chains = defaultTestingChainsEdited @@ -98,7 +100,6 @@ func TestUtilityContext_HandleMessageEditStake(t *testing.T) { err = ctx.HandleEditStakeMessage(msgAmountEdited) require.NoError(t, err, "handle edit stake message") - actor = getActorByAddr(t, ctx, addrBz, actorType) test_artifacts.CleanupTest(ctx) }) } @@ -107,8 +108,8 @@ func TestUtilityContext_HandleMessageEditStake(t *testing.T) { func TestUtilityContext_HandleMessageUnpause(t *testing.T) { for _, actorType := range actorTypes { t.Run(fmt.Sprintf("%s.HandleMessageUnpause", actorType.String()), func(t *testing.T) { - ctx := NewTestingUtilityContext(t, 1) + var err error switch actorType { case typesUtil.ActorType_Validator: @@ -127,6 +128,7 @@ func TestUtilityContext_HandleMessageUnpause(t *testing.T) { actor := getFirstActor(t, ctx, actorType) addrBz, err := hex.DecodeString(actor.GetAddress()) require.NoError(t, err) + err = ctx.SetActorPauseHeight(actorType, addrBz, 1) require.NoError(t, err, "error setting pause height") @@ -144,6 +146,7 @@ func TestUtilityContext_HandleMessageUnpause(t *testing.T) { actor = getActorByAddr(t, ctx, addrBz, actorType) require.Equal(t, int64(-1), actor.GetPausedHeight()) + test_artifacts.CleanupTest(ctx) }) } @@ -153,6 +156,7 @@ func TestUtilityContext_HandleMessageUnstake(t *testing.T) { for _, actorType := range actorTypes { t.Run(fmt.Sprintf("%s.HandleMessageUnstake", actorType.String()), func(t *testing.T) { ctx := NewTestingUtilityContext(t, 1) + var err error switch actorType { case typesUtil.ActorType_App: @@ -171,6 +175,7 @@ func TestUtilityContext_HandleMessageUnstake(t *testing.T) { actor := getFirstActor(t, ctx, actorType) addrBz, err := hex.DecodeString(actor.GetAddress()) require.NoError(t, err) + msg := &typesUtil.MessageUnstake{ Address: addrBz, Signer: addrBz, @@ -182,6 +187,7 @@ func TestUtilityContext_HandleMessageUnstake(t *testing.T) { actor = getActorByAddr(t, ctx, addrBz, actorType) require.Equal(t, defaultUnstaking, actor.GetUnstakingHeight(), "actor should be unstaking") + test_artifacts.CleanupTest(ctx) }) } @@ -191,10 +197,11 @@ func TestUtilityContext_BeginUnstakingMaxPaused(t *testing.T) { for _, actorType := range actorTypes { t.Run(fmt.Sprintf("%s.BeginUnstakingMaxPaused", actorType.String()), func(t *testing.T) { ctx := NewTestingUtilityContext(t, 1) - actor := getFirstActor(t, ctx, actorType) + addrBz, err := hex.DecodeString(actor.GetAddress()) require.NoError(t, err) + switch actorType { case typesUtil.ActorType_App: err = ctx.Context.SetParam(typesUtil.AppMaxPauseBlocksParamName, 0) @@ -218,6 +225,7 @@ func TestUtilityContext_BeginUnstakingMaxPaused(t *testing.T) { status, err := ctx.GetActorStatus(actorType, addrBz) require.NoError(t, err) require.Equal(t, int32(typesUtil.StakeStatus_Unstaking), status, "actor should be unstaking") + test_artifacts.CleanupTest(ctx) }) } @@ -254,8 +262,8 @@ func TestUtilityContext_CalculateUnstakingHeight(t *testing.T) { unstakingHeight, err := ctx.GetUnstakingHeight(actorType) require.NoError(t, err) - require.Equal(t, unstakingBlocks, unstakingHeight, "unexpected unstaking height") + test_artifacts.CleanupTest(ctx) }) } @@ -269,8 +277,10 @@ func TestUtilityContext_GetExists(t *testing.T) { actor := getFirstActor(t, ctx, actorType) randAddr, err := crypto.GenerateAddress() require.NoError(t, err) + addrBz, err := hex.DecodeString(actor.GetAddress()) require.NoError(t, err) + exists, err := ctx.GetActorExists(actorType, addrBz) require.NoError(t, err) require.True(t, exists, "actor that should exist does not") @@ -278,6 +288,7 @@ func TestUtilityContext_GetExists(t *testing.T) { exists, err = ctx.GetActorExists(actorType, randAddr) require.NoError(t, err) require.False(t, exists, "actor that shouldn't exist does") + test_artifacts.CleanupTest(ctx) }) } @@ -291,10 +302,11 @@ func TestUtilityContext_GetOutputAddress(t *testing.T) { actor := getFirstActor(t, ctx, actorType) addrBz, err := hex.DecodeString(actor.GetAddress()) require.NoError(t, err) + outputAddress, err := ctx.GetActorOutputAddress(actorType, addrBz) require.NoError(t, err) - require.Equal(t, actor.GetOutput(), hex.EncodeToString(outputAddress), "unexpected output address") + test_artifacts.CleanupTest(ctx) }) } @@ -304,11 +316,12 @@ func TestUtilityContext_GetPauseHeightIfExists(t *testing.T) { for _, actorType := range actorTypes { t.Run(fmt.Sprintf("%s.GetPauseHeightIfExists", actorType.String()), func(t *testing.T) { ctx := NewTestingUtilityContext(t, 0) - pauseHeight := int64(100) actor := getFirstActor(t, ctx, actorType) + addrBz, err := hex.DecodeString(actor.GetAddress()) require.NoError(t, err) + err = ctx.SetActorPauseHeight(actorType, addrBz, pauseHeight) require.NoError(t, err, "error setting actor pause height") @@ -321,6 +334,7 @@ func TestUtilityContext_GetPauseHeightIfExists(t *testing.T) { _, err = ctx.GetPauseHeight(actorType, randAddr) require.Error(t, err, "non existent actor should error") + test_artifacts.CleanupTest(ctx) }) } @@ -330,22 +344,24 @@ func TestUtilityContext_GetMessageEditStakeSignerCandidates(t *testing.T) { for _, actorType := range actorTypes { t.Run(fmt.Sprintf("%s.GetMessageEditStakeSignerCandidates", actorType.String()), func(t *testing.T) { ctx := NewTestingUtilityContext(t, 0) - actor := getFirstActor(t, ctx, actorType) + addrBz, err := hex.DecodeString(actor.GetAddress()) require.NoError(t, err) + msgEditStake := &typesUtil.MessageEditStake{ Address: addrBz, Chains: defaults.DefaultChains, Amount: defaults.DefaultStakeAmountString, ActorType: actorType, } - candidates, err := ctx.GetMessageEditStakeSignerCandidates(msgEditStake) require.NoError(t, err) + require.Equal(t, len(candidates), 2, "unexpected number of candidates") require.Equal(t, actor.GetOutput(), hex.EncodeToString(candidates[0]), "incorrect output candidate") require.Equal(t, actor.GetAddress(), hex.EncodeToString(candidates[1]), "incorrect addr candidate") + test_artifacts.CleanupTest(ctx) }) } @@ -355,20 +371,22 @@ func TestUtilityContext_GetMessageUnpauseSignerCandidates(t *testing.T) { for _, actorType := range actorTypes { t.Run(fmt.Sprintf("%s.GetMessageUnpauseSignerCandidates", actorType.String()), func(t *testing.T) { ctx := NewTestingUtilityContext(t, 0) - actor := getFirstActor(t, ctx, actorType) + addrBz, err := hex.DecodeString(actor.GetAddress()) require.NoError(t, err) + msg := &typesUtil.MessageUnpause{ Address: addrBz, ActorType: actorType, } - candidates, err := ctx.GetMessageUnpauseSignerCandidates(msg) require.NoError(t, err) + require.Equal(t, len(candidates), 2, "unexpected number of candidates") require.Equal(t, actor.GetOutput(), hex.EncodeToString(candidates[0]), "incorrect output candidate") require.Equal(t, actor.GetAddress(), hex.EncodeToString(candidates[1]), "incorrect addr candidate") + test_artifacts.CleanupTest(ctx) }) } @@ -378,19 +396,22 @@ func TestUtilityContext_GetMessageUnstakeSignerCandidates(t *testing.T) { for _, actorType := range actorTypes { t.Run(fmt.Sprintf("%s.GetMessageUnstakeSignerCandidates", actorType.String()), func(t *testing.T) { ctx := NewTestingUtilityContext(t, 0) - actor := getFirstActor(t, ctx, actorType) + addrBz, err := hex.DecodeString(actor.GetAddress()) require.NoError(t, err) + msg := &typesUtil.MessageUnstake{ Address: addrBz, ActorType: actorType, } candidates, err := ctx.GetMessageUnstakeSignerCandidates(msg) require.NoError(t, err) + require.Equal(t, len(candidates), 2, "unexpected number of candidates") require.Equal(t, actor.GetOutput(), hex.EncodeToString(candidates[0]), "incorrect output candidate") require.Equal(t, actor.GetAddress(), hex.EncodeToString(candidates[1]), "incorrect addr candidate") + test_artifacts.CleanupTest(ctx) }) } @@ -403,8 +424,10 @@ func TestUtilityContext_UnstakePausedBefore(t *testing.T) { actor := getFirstActor(t, ctx, actorType) require.Equal(t, actor.GetUnstakingHeight(), int64(-1), "wrong starting status") + addrBz, err := hex.DecodeString(actor.GetAddress()) require.NoError(t, err) + err = ctx.SetActorPauseHeight(actorType, addrBz, 0) require.NoError(t, err, "error setting actor pause height") @@ -447,6 +470,7 @@ func TestUtilityContext_UnstakePausedBefore(t *testing.T) { } require.NoError(t, err, "error getting unstaking blocks") require.Equal(t, unstakingBlocks+1, actor.GetUnstakingHeight(), "incorrect unstaking height") + test_artifacts.CleanupTest(ctx) }) } @@ -457,7 +481,7 @@ func TestUtilityContext_UnstakeActorsThatAreReady(t *testing.T) { t.Run(fmt.Sprintf("%s.UnstakeActorsThatAreReady", actorType.String()), func(t *testing.T) { ctx := NewTestingUtilityContext(t, 1) - poolName := "" + var poolName string var err1, err2 error switch actorType { case typesUtil.ActorType_App: @@ -479,11 +503,11 @@ func TestUtilityContext_UnstakeActorsThatAreReady(t *testing.T) { default: t.Fatalf("unexpected actor type %s", actorType.String()) } + require.NoError(t, err1, "error setting unstaking blocks") + require.NoError(t, err2, "error setting max pause blocks") err := ctx.SetPoolAmount(poolName, big.NewInt(math.MaxInt64)) require.NoError(t, err) - require.NoError(t, err1, "error setting unstaking blocks") - require.NoError(t, err2, "error setting max pause blocks") actors := getAllTestingActors(t, ctx, actorType) for _, actor := range actors { @@ -498,7 +522,6 @@ func TestUtilityContext_UnstakeActorsThatAreReady(t *testing.T) { require.NoError(t, err, "error setting actor pause before") accountAmountsBefore := make([]*big.Int, 0) - for _, actor := range actors { // get the output address account amount before the 'unstake' outputAddressString := actor.GetOutput() @@ -521,12 +544,15 @@ func TestUtilityContext_UnstakeActorsThatAreReady(t *testing.T) { outputAddressString := actor.GetOutput() outputAddress, err := hex.DecodeString(outputAddressString) require.NoError(t, err) + outputAccountAmount, err := ctx.GetAccountAmount(outputAddress) require.NoError(t, err) + // ensure the stake amount went to the output address outputAccountAmountDelta := new(big.Int).Sub(outputAccountAmount, accountAmountsBefore[i]) require.Equal(t, outputAccountAmountDelta, defaults.DefaultStakeAmount) } + // ensure the staking pool is `# of readyToUnstake actors * default stake` less than before the unstake poolAmountAfter, err := ctx.GetPoolAmount(poolName) require.NoError(t, err) diff --git a/utility/test/block_test.go b/utility/test/block_test.go index dd6fc86f3..5fb1ab5fa 100644 --- a/utility/test/block_test.go +++ b/utility/test/block_test.go @@ -25,7 +25,7 @@ func TestUtilityContext_ApplyBlock(t *testing.T) { require.NoError(t, er) proposerBeforeBalance, err := ctx.GetAccountAmount(addrBz) require.NoError(t, err) - er = ctx.GetPersistenceContext().SetProposalBlock("", nil, addrBz, [][]byte{txBz}) + er = ctx.GetPersistenceContext().SetProposalBlock("", addrBz, nil, [][]byte{txBz}) require.NoError(t, er) // apply block _, er = ctx.ApplyBlock() @@ -73,7 +73,7 @@ func TestUtilityContext_BeginBlock(t *testing.T) { require.NoError(t, err) addrBz, er := hex.DecodeString(proposer.GetAddress()) require.NoError(t, er) - er = ctx.GetPersistenceContext().SetProposalBlock("", nil, addrBz, [][]byte{txBz}) + er = ctx.GetPersistenceContext().SetProposalBlock("", addrBz, nil, [][]byte{txBz}) require.NoError(t, er) // apply block _, er = ctx.ApplyBlock() @@ -108,8 +108,10 @@ func TestUtilityContext_BeginUnstakingMaxPausedActors(t *testing.T) { t.Fatalf("unexpected actor type %s", actorType.String()) } require.NoError(t, err) + addrBz, er := hex.DecodeString(actor.GetAddress()) require.NoError(t, er) + err = ctx.SetActorPauseHeight(actorType, addrBz, 0) require.NoError(t, err) @@ -136,7 +138,7 @@ func TestUtilityContext_EndBlock(t *testing.T) { require.NoError(t, er) proposerBeforeBalance, err := ctx.GetAccountAmount(addrBz) require.NoError(t, err) - er = ctx.GetPersistenceContext().SetProposalBlock("", nil, addrBz, [][]byte{txBz}) + er = ctx.GetPersistenceContext().SetProposalBlock("", addrBz, nil, [][]byte{txBz}) require.NoError(t, er) // apply block _, er = ctx.ApplyBlock() @@ -166,6 +168,7 @@ func TestUtilityContext_UnstakeValidatorsActorsThatAreReady(t *testing.T) { for _, actorType := range actorTypes { t.Run(fmt.Sprintf("%s.UnstakeValidatorsActorsThatAreReady", actorType.String()), func(t *testing.T) { ctx := NewTestingUtilityContext(t, 1) + var poolName string switch actorType { case typesUtil.ActorType_App: @@ -179,8 +182,8 @@ func TestUtilityContext_UnstakeValidatorsActorsThatAreReady(t *testing.T) { default: t.Fatalf("unexpected actor type %s", actorType.String()) } - ctx.SetPoolAmount(poolName, big.NewInt(math.MaxInt64)) + err := ctx.Context.SetParam(typesUtil.AppUnstakingBlocksParamName, 0) require.NoError(t, err) @@ -207,7 +210,6 @@ func TestUtilityContext_UnstakeValidatorsActorsThatAreReady(t *testing.T) { // TODO: We need to better define what 'deleted' really is in the postgres world. // We might not need to 'unstakeActorsThatAreReady' if we are already filtering by unstakingHeight - test_artifacts.CleanupTest(ctx) }) } diff --git a/utility/test/module_test.go b/utility/test/module_test.go index 347cd55fc..2ad166bdf 100644 --- a/utility/test/module_test.go +++ b/utility/test/module_test.go @@ -2,16 +2,18 @@ package test import ( "encoding/hex" + "log" "math/big" "os" "testing" - "github.com/golang/mock/gomock" "github.com/pokt-network/pocket/persistence" + "github.com/pokt-network/pocket/persistence/types" + "github.com/pokt-network/pocket/runtime" "github.com/pokt-network/pocket/runtime/defaults" "github.com/pokt-network/pocket/runtime/test_artifacts" + "github.com/pokt-network/pocket/shared/messaging" "github.com/pokt-network/pocket/shared/modules" - mock_modules "github.com/pokt-network/pocket/shared/modules/mocks" "github.com/pokt-network/pocket/utility" utilTypes "github.com/pokt-network/pocket/utility/types" "github.com/stretchr/testify/require" @@ -34,7 +36,7 @@ var ( testMessageSendType = "MessageSend" ) -var persistenceDbUrl string +var testPersistenceMod modules.PersistenceModule // initialized in TestMain var actorTypes = []utilTypes.ActorType{ utilTypes.ActorType_App, utilTypes.ActorType_ServiceNode, @@ -48,20 +50,25 @@ func NewTestingMempool(_ *testing.T) utilTypes.Mempool { func TestMain(m *testing.M) { pool, resource, dbUrl := test_artifacts.SetupPostgresDocker() - persistenceDbUrl = dbUrl + testPersistenceMod = newTestPersistenceModule(dbUrl) exitCode := m.Run() test_artifacts.CleanupPostgresDocker(m, pool, resource) os.Exit(exitCode) } func NewTestingUtilityContext(t *testing.T, height int64) utility.UtilityContext { - testPersistenceMod := newTestPersistenceModule(t, persistenceDbUrl) - persistenceContext, err := testPersistenceMod.NewRWContext(height) require.NoError(t, err) + // TECHDEBT: Move the internal of cleanup into a separate function and call this in the + // beginning of every test. This (the current implementation) is an issue because if we call + // `NewTestingUtilityContext` more than once in a single test, we create unnecessary calls to clean. t.Cleanup(func() { - persistenceContext.Release() + require.NoError(t, testPersistenceMod.ReleaseWriteContext()) + require.NoError(t, testPersistenceMod.HandleDebugMessage(&messaging.DebugMessage{ + Action: messaging.DebugMessageAction_DEBUG_PERSISTENCE_RESET_TO_GENESIS, + Message: nil, + })) }) return utility.UtilityContext{ @@ -75,30 +82,22 @@ func NewTestingUtilityContext(t *testing.T, height int64) utility.UtilityContext } } -func newTestPersistenceModule(t *testing.T, databaseUrl string) modules.PersistenceModule { - ctrl := gomock.NewController(t) - - mockPersistenceConfig := mock_modules.NewMockPersistenceConfig(ctrl) - mockPersistenceConfig.EXPECT().GetPostgresUrl().Return(databaseUrl).AnyTimes() - mockPersistenceConfig.EXPECT().GetNodeSchema().Return(testSchema).AnyTimes() - mockPersistenceConfig.EXPECT().GetBlockStorePath().Return("").AnyTimes() - mockPersistenceConfig.EXPECT().GetTxIndexerPath().Return("").AnyTimes() - - mockRuntimeConfig := mock_modules.NewMockConfig(ctrl) - mockRuntimeConfig.EXPECT().GetPersistenceConfig().Return(mockPersistenceConfig).AnyTimes() - - mockRuntimeMgr := mock_modules.NewMockRuntimeMgr(ctrl) - mockRuntimeMgr.EXPECT().GetConfig().Return(mockRuntimeConfig).AnyTimes() - +// TODO(olshansky): Take in `t testing.T` as a parameter and error if there's an issue +func newTestPersistenceModule(databaseUrl string) modules.PersistenceModule { + cfg := runtime.NewConfig(&runtime.BaseConfig{}, runtime.WithPersistenceConfig(&types.PersistenceConfig{ + PostgresUrl: databaseUrl, + NodeSchema: testSchema, + BlockStorePath: "", + TxIndexerPath: "", + TreesStoreDir: "", + })) genesisState, _ := test_artifacts.NewGenesisState(5, 1, 1, 1) - mockRuntimeMgr.EXPECT().GetGenesis().Return(genesisState).AnyTimes() - - persistenceMod, err := persistence.Create(mockRuntimeMgr) - require.NoError(t, err) - - err = persistenceMod.Start() - require.NoError(t, err) + runtimeCfg := runtime.NewManager(cfg, genesisState) + persistenceMod, err := persistence.Create(runtimeCfg) + if err != nil { + log.Fatalf("Error creating persistence module: %s", err) + } return persistenceMod.(modules.PersistenceModule) } diff --git a/utility/types/message_test.go b/utility/types/message_test.go index 0702a9bf6..bf487e64b 100644 --- a/utility/types/message_test.go +++ b/utility/types/message_test.go @@ -1,10 +1,10 @@ package types import ( - "github.com/pokt-network/pocket/shared/codec" "math/big" "testing" + "github.com/pokt-network/pocket/shared/codec" "github.com/pokt-network/pocket/shared/crypto" "github.com/stretchr/testify/require" "google.golang.org/protobuf/proto"