[Persistence] First iteration of a PostgreSQL based Persistence Schema (

#73)

## Objective

Foundational iteration of PostgreSQL based persistence module implementation.

## Origin Document

#68

## Type of change

New major module implementation.

### Persistence-related core changes:
- List of actors / interfaces with MVP implementation:
    - Applications
    - Fisherman
    - ServiceNode
    - Accounts
    - Pools
- List of actors / interfaces with partial MVP implementation:
    - Validator
    - Gov params
- List of actors / interfaces with minorimplementation:
    - Block
- SQL Schema definition of the actors above
- SQL Query implementation for common actor persistence functionality
- PostgresContext implementation of the actors actors above
- Base infrastructure for fuzz testing

Non-persistence “fly-by” changes
- Updates to the PrePersistence module and utility module with breaking changes
- A few minor improvements/additions to the Makefile
- TODOs & comment cleanups throughout the codebase

## How Has This Been Tested?

### Unit Tests

```
make test_persistence
make test_all
```

### LocalNet
Ran a basic LocalNet following the instructions in the [development README](docs/development/README.md).


Co-authored-by: Daniel Olshansky <olshansky.daniel@gmail.com>
Co-authored-by: Andrew Nguyen <andrewnguyen@Andrews-MacBook-Pro-2.local>
Co-authored-by: Andrew Nguyen <andrewnguyen@Andrews-MBP-2.lan>
Co-authored-by: Daniel Olshansky <olshansky@pokt.network>

Makefile

@@ -8,6 +8,11 @@ CWD ?= CURRENT_WORKING_DIRECTIONRY_NOT_SUPPLIED
 #		                        seconds, and fail if any additional messages are received.
 EXTRA_MSG_FAIL ?= false
 
+# An easy way to turn off verbose test output for some of the test targets. For example
+#  `$ make test_persistence` by default enables verbose testing
+#  `VERBOSE_TEST="" make test_persistence` is an easy way to run the same tests without verbose output
+VERBOSE_TEST ?= -v
+
 .SILENT:
 
 help:
@@ -22,6 +27,13 @@ help:
 	} \
 	{ lastLine = $$0 }' $(MAKEFILE_LIST)
 
+docker_check:
+	{ \
+	if ! builtin type -P "docker" > /dev/null || ! builtin type -P "docker-compose" > /dev/null; then \
+		echo "Seems like you don't have Docker or docker-compose installed. Make sure you review docs/development/README.md before continuing"; \
+		exit 1; \
+	fi; \
+	}
 
 prompt_user:
 	@echo "Are you sure? [y/N] " && read ans && [ $${ans:-N} = y ]
@@ -34,7 +46,25 @@ go_vet:
 .PHONY: go_staticcheck
 ## Run `go staticcheck` on all files in the current project
 go_staticcheck:
-	@if builtin type -P "staticcheck"; then staticcheck ./... ; else echo "Install with 'go install honnef.co/go/tools/cmd/staticcheck@latest'"; fi
+	{ \
+	if builtin type -P "staticcheck"; then \
+		staticcheck ./...; \
+	else \
+		echo "Install with 'go install honnef.co/go/tools/cmd/staticcheck@latest'"; \
+	fi; \
+	}
+
+.PHONY: go_doc
+## Generate documentation for the current project using `godo`
+go_doc:
+	{ \
+	if builtin type "godoc"; then \
+		echo "Visit http://localhost:6060/pocket"; \
+		godoc -http=localhost:6060  -goroot=${PWD}/..; \
+	else \
+		echo "Install with 'go install golang.org/x/tools/cmd/godoc@latest'"; \
+	fi; \
+	}
 
 .PHONY: go_clean_deps
 ## Runs `go mod tidy` && `go mod vendor`
@@ -48,23 +78,23 @@ build_and_watch:
 
 .PHONY: client_start
 ## Run a client daemon which is only used for debugging purposes
-client_start:
+client_start: docker_check
 	docker-compose -f build/deployments/docker-compose.yaml up -d client
 
 .PHONY: client_connect
 ## Connect to the running client debugging daemon
-client_connect:
+client_connect: docker_check
 	docker exec -it client /bin/bash -c "go run app/client/*.go"
 
 # TODO(olshansky): Need to think of a Pocket related name for `compose_and_watch`, maybe just `pocket_watch`?
 .PHONY: compose_and_watch
 ## Run a localnet composed of 4 consensus validators w/ hot reload & debugging
-compose_and_watch: db_start
+compose_and_watch: docker_check db_start
 	docker-compose -f build/deployments/docker-compose.yaml up --force-recreate node1.consensus node2.consensus node3.consensus node4.consensus
 
 .PHONY: db_start
 ## Start a detached local postgres and admin instance (this is auto-triggered by compose_and_watch)
-db_start:
+db_start: docker_check
 	docker-compose -f build/deployments/docker-compose.yaml up --no-recreate -d db pgadmin
 
 .PHONY: db_cli
@@ -75,37 +105,39 @@ db_cli:
 
 .PHONY: db_drop
 ## Drop all schemas used for LocalNet development matching `node%`
-db_drop:
+db_drop: docker_check
 	docker exec -it pocket-db bash -c "psql -U postgres -d postgres -a -f /tmp/scripts/drop_all_schemas.sql"
 
 .PHONY: db_bench_init
 ## Initialize pgbench on local postgres - needs to be called once after container is created.
-db_bench_init:
+db_bench_init: docker_check
 	docker exec -it pocket-db bash -c "pgbench -i -U postgres -d postgres"
 
 .PHONY: db_bench
 ## Run a local benchmark against the local postgres instance - TODO(olshansky): visualize results
-db_bench:
+db_bench: docker_check
 	docker exec -it pocket-db bash -c "pgbench -U postgres -d postgres"
 
 .PHONY: db_admin
 ## Helper to access to postgres admin GUI interface
 db_admin:
 	echo "Open http://0.0.0.0:5050 and login with 'pgadmin4@pgadmin.org' and 'pgadmin4'.\n The password is 'postgres'"
 
-.PHONY: compose_and_watch
+.PHONY: docker_kill_all
 ## Kill all containers started by the docker-compose file
-docker_kill_all:
+docker_kill_all: docker_check
 	docker-compose -f build/deployments/docker-compose.yaml down
 
 .PHONY: docker_wipe
 ## [WARNING] Remove all the docker containers, images and volumes.
-docker_wipe: prompt_user
+docker_wipe: docker_check prompt_user
 	docker ps -a -q | xargs -r -I {} docker stop {}
 	docker ps -a -q | xargs -r -I {} docker rm {}
 	docker images -q | xargs -r -I {} docker rmi {}
 	docker volume ls -q | xargs -r -I {} docker volume rm {}
 
+# Reference the following for mockgen with 1.18: https://github.com/golang/mock/issues/621
+
 .PHONY: mockgen
 ## Use `mockgen` to generate mocks used for testing purposes of all the modules.
 mockgen:
@@ -134,12 +166,12 @@ test_race: # generate_mocks
 .PHONY: test_utility_module
 ## Run all go utility module unit tests
 test_utility_module: # generate_mocks
-	go test -v ./shared/tests/utility_module/...
+	go test ${VERBOSE_TEST} ./shared/tests/utility_module/...
 
 .PHONY: test_utility_types
 ## Run all go utility types module unit tests
 test_utility_types: # generate_mocks
-	go test -v ./utility/types/...
+	go test ${VERBOSE_TEST} ./utility/types/...
 
 .PHONY: test_shared
 ## Run all go unit tests in the shared module
@@ -149,7 +181,7 @@ test_shared: # generate_mocks
 .PHONY: test_consensus
 ## Run all go unit tests in the Consensus module
 test_consensus: # mockgen
-	go test -v ./consensus/...
+	go test ${VERBOSE_TEST} ./consensus/...
 
 .PHONY: test_pre_persistence
 ## Run all go per persistence unit tests
@@ -159,27 +191,32 @@ test_pre_persistence: # generate_mocks
 .PHONY: test_hotstuff
 ## Run all go unit tests related to hotstuff consensus
 test_hotstuff: # mockgen
-	go test -v ./consensus/consensus_tests -run Hotstuff -failOnExtraMessages=${EXTRA_MSG_FAIL}
+	go test ${VERBOSE_TEST} ./consensus/consensus_tests -run Hotstuff -failOnExtraMessages=${EXTRA_MSG_FAIL}
 
 .PHONY: test_pacemaker
 ## Run all go unit tests related to the hotstuff pacemaker
 test_pacemaker: # mockgen
-	go test -v ./consensus/consensus_tests -run Pacemaker -failOnExtraMessages=${EXTRA_MSG_FAIL}
+	go test ${VERBOSE_TEST} ./consensus/consensus_tests -run Pacemaker -failOnExtraMessages=${EXTRA_MSG_FAIL}
 
 .PHONY: test_vrf
 ## Run all go unit tests in the VRF library
 test_vrf:
-	go test -v ./consensus/leader_election/vrf
+	go test ${VERBOSE_TEST} ./consensus/leader_election/vrf
 
 .PHONY: test_sortition
 ## Run all go unit tests in the Sortition library
 test_sortition:
-	go test -v ./consensus/leader_election/sortition
+	go test ${VERBOSE_TEST} ./consensus/leader_election/sortition
+
+.PHONY: test_persistence
+## Run all go unit tests in the Persistence module
+test_persistence:
+	go test ${VERBOSE_TEST} -p=1 ./persistence/...
 
 .PHONY: benchmark_sortition
 ## Benchmark the Sortition library
 benchmark_sortition:
-	go test -v ./consensus/leader_election/sortition -bench=.
+	go test ${VERBOSE_TEST} ./consensus/leader_election/sortition -bench=.
 
 # TODO(team): Tested locally with `protoc` version `libprotoc 3.19.4`. In the near future, only the Dockerfiles will be used to compile protos.
 
@@ -208,12 +245,12 @@ protogen_local:
 
 .PHONY: protogen_docker_m1
 ## TODO(derrandz): Test, validate & update.
-protogen_docker_m1:
+protogen_docker_m1: docker_check
 	docker build  -t pocket/proto-generator -f ./build/Dockerfile.m1.proto . && docker run --platform=linux/amd64 -it -v $(CWD)/shared:/usr/src/app/shared pocket/proto-generator
 
 .PHONY: protogen_docker
 ## TODO(derrandz): Test, validate & update.
-protogen_docker:
+protogen_docker: docker_check
 	docker build -t pocket/proto-generator -f ./build/Dockerfile.proto . && docker run -it pocket/proto-generator
 
 .PHONY: gofmt
@@ -236,17 +273,17 @@ test_p2p_socket:
 .PHONY: test_p2p_types
 ## Run p2p subcomponents' tests
 test_p2p_types:
-	go test -v -race ./p2p/types
+	go test ${VERBOSE_TEST} -race ./p2p/types
 
 .PHONY: test_p2p
 ## Run all p2p tests
 test_p2p:
-	go test -v -race ./p2p
+	go test ${VERBOSE_TEST} -race ./p2p
 
 .PHONY: test_pre2p
 ## Run all pre2p
 test_pre2p:
-	go test -v -count=1 ./p2p/pre2p/...
+	go test ${VERBOSE_TEST} -count=1 ./p2p/pre2p/...
 
 .PHONY: test_pre2p_addrbook
 ## Run all Pre2P addr book related tests
@@ -258,18 +295,19 @@ test_pre2p_addrbook:
 benchmark_pre2p_addrbook:
 	go test -bench=. -run BenchmarkAddrBook -v -count=1 ./p2p/pre2p/...
 
-# /Users/olshansky/workspace/pocket/pocket/p2p/pre2p/raintree/addrbook_utils_test.go
-# Inspired by: https://goldin.io/blog/stop-using-todo
-# TODO        - General Purpose catch-all.
-# TECHDEBT    - Not a great implementation, but we need to fix it later.
-# IMPROVE     - A nice to have, but not a priority. It's okay if we never get to this.
-# DISCUSS     - Probably requires a lengthy offline discussion to understand next steps.
-# INCOMPLETE  - A change which was out of scope of a specific PR but needed to be documented.
-# INVESTIGATE - TBD what was going on, but needed to continue moving and not get distracted.
-# CLEANUP     - Like TECHDEBT, but not as bad.  It's okay if we never get to this.
-# HACK        - Like TECHDEBT, but much worse. This needs to be prioritized
-# REFACTOR    - Similar to TECHDEBT, but will require a substantial rewrite and change across the codebase
-TODO_KEYWORDS = -e "TODO" -e "TECHDEBT" -e "IMPROVE" -e "DISCUSS" -e "INCOMPLETE" -e "INVESTIGATE" -e "CLEANUP" -e "HACK" -e "REFACTOR"
+### Inspired by @goldinguy_ in this post: https://goldin.io/blog/stop-using-todo ###
+# TODO          - General Purpose catch-all.
+# TECHDEBT      - Not a great implementation, but we need to fix it later.
+# IMPROVE       - A nice to have, but not a priority. It's okay if we never get to this.
+# DISCUSS       - Probably requires a lengthy offline discussion to understand next steps.
+# INCOMPLETE    - A change which was out of scope of a specific PR but needed to be documented.
+# INVESTIGATE   - TBD what was going on, but needed to continue moving and not get distracted.
+# CLEANUP       - Like TECHDEBT, but not as bad.  It's okay if we never get to this.
+# HACK          - Like TECHDEBT, but much worse. This needs to be prioritized
+# REFACTOR      - Similar to TECHDEBT, but will require a substantial rewrite and change across the codebase
+# CONSIDERATION - A comment that involves extra work but was thoughts / considered as part of some implementation
+# INTHISCOMMIT  - SHOULD NEVER BE COMMITTED TO MASTER. It is a way for the review of a PR to start / reply to a discussion.
+TODO_KEYWORDS = -e "TODO" -e "TECHDEBT" -e "IMPROVE" -e "DISCUSS" -e "INCOMPLETE" -e "INVESTIGATE" -e "CLEANUP" -e "HACK" -e "REFACTOR" -e "CONSIDERATION" -e "INTHISCOMMIT"
 
 .PHONY: todo_list
 ## List all the TODOs in the project (excludes vendor and prototype directories)
@@ -280,3 +318,11 @@ todo_list:
 ## Print a count of all the TODOs in the project
 todo_count:
 	grep --exclude-dir={.git,vendor,prototype} -r ${TODO_KEYWORDS} . | wc -l
+
+.PHONY: develop_and_test
+## Run all of the make commands necessary to develop on the project and verify the tests pass
+develop_test:
+		make mockgen && \
+		make protogen_clean && make protogen_local && \
+		make go_clean_deps && \
+		make test_all

README.md

@@ -37,16 +37,17 @@ All the links you'll need are listed below. If you'd like to contribute to the P
 
 - [Shared Architecture](shared/README.md)
 - _Coming Soon: Consensus Architecture_
-- [PrePersistence Architecture](persistence/pre_persistence/README.md)
+- [Persistence Architecture](persistence/README.md)
+- _(Soon to be deprecated)_ [PrePersistence Architecture](persistence/pre_persistence/README.md)
 - [P2P Architecture](p2p/README.md)
 - [Utility Architecture](utility/README.md)
 
 ### Changelogs
 
 - [Consensus Changelog](consensus/CHANGELOG.md)
 - [Utility Changelog](utility/CHANGELOG.md)
+- [Persistence Changelog](persistence/CHANGELOG.md)
 - _Coming Soon: P2P Changelog_
-- _Coming Soon: Persistence Changelog_
 
 ## Support & Contact
 

consensus/consensus_tests/utils_test.go

@@ -216,7 +216,8 @@ func WaitForNetworkConsensusMessages(
 	return waitForNetworkConsensusMessagesInternal(t, testChannel, types.PocketTopic_CONSENSUS_MESSAGE_TOPIC, numMessages, millis, includeFilter, errorMessage)
 }
 
-func waitForNetworkConsensusMessagesInternal( // TODO(olshansky): Translate this to use generics.
+// IMPROVE(olshansky): Translate this to use generics.
+func waitForNetworkConsensusMessagesInternal(
 	_ *testing.T,
 	testChannel modules.EventsChannel,
 	topic types.PocketTopic,

go.mod

@@ -8,6 +8,7 @@ require (
 	github.com/jackc/pgx/v4 v4.15.0
 	github.com/jordanorelli/lexnum v0.0.0-20141216151731-460eeb125754
 	github.com/manifoldco/promptui v0.9.0
+	github.com/ory/dockertest v3.3.5+incompatible
 	github.com/stretchr/testify v1.7.0
 	github.com/syndtr/goleveldb v1.0.0
 	golang.org/x/crypto v0.0.0-20220214200702-86341886e292
@@ -16,10 +17,31 @@ require (
 	google.golang.org/protobuf v1.27.1
 )
 
-require github.com/matryer/resync v0.0.0-20161211202428-d39c09a11215
+require (
+	github.com/iancoleman/strcase v0.2.0
+	github.com/matryer/resync v0.0.0-20161211202428-d39c09a11215
+)
 
 require github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927 // indirect
 
+require (
+	github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
+	github.com/Microsoft/go-winio v0.5.2 // indirect
+	github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 // indirect
+	github.com/cenkalti/backoff v2.2.1+incompatible // indirect
+	github.com/containerd/continuity v0.3.0 // indirect
+	github.com/docker/go-connections v0.4.0 // indirect
+	github.com/docker/go-units v0.4.0 // indirect
+	github.com/gotestyourself/gotestyourself v2.2.0+incompatible // indirect
+	github.com/opencontainers/go-digest v1.0.0 // indirect
+	github.com/opencontainers/image-spec v1.0.2 // indirect
+	github.com/opencontainers/runc v1.1.2 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/sirupsen/logrus v1.8.1 // indirect
+	golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 // indirect
+	gotest.tools v2.2.0+incompatible // indirect
+)
+
 require (
 	filippo.io/edwards25519 v1.0.0-rc.1 // indirect
 	github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e // indirect
@@ -38,7 +60,7 @@ require (
 	github.com/onsi/gomega v1.16.0 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	go.uber.org/atomic v1.9.0
-	golang.org/x/sys v0.0.0-20220209214540-3681064d5158 // indirect
+	golang.org/x/sys v0.0.0-20220405210540-1e041c57c461 // indirect
 	golang.org/x/text v0.3.7 // indirect
 	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
 	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect