Revert "remove all dbs except for goleveldb + memdb"

This reverts commit 26e5cdc.

.github/dependabot.yml

@@ -28,6 +28,11 @@ updates:
       everything-else:
         patterns:
           - "*"
+        exclude-patterns:
+          # NOTE: grocksdb should be updated manually, to match the version
+          # available on Ubuntu's latest LTS release; and updated in conjunction
+          # with the ubuntu version on .github/workflows/db-tests.yml
+          - "github.com/linxGnu/grocksdb"
     open-pull-requests-limit: 10
     pull-request-branch-name:
       separator: "-"

.github/workflows/db-tests.yml

@@ -0,0 +1,54 @@
+name: db-tests
+
+on:
+  pull_request:
+    paths:
+      - "tm2/pkg/db/**.go"
+      - "go.sum"
+      - ".github/workflows/db-tests.yml"
+  push:
+    branches: [ "master" ]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+jobs:
+  test:
+    # NOTE: this job uses a specific version of ubuntu as the version of grocksb
+    # we use is related to the version of RocksDB in ubuntu's repositories.
+    # If updating this to a later ubuntu release, update the grocksdb version
+    # accordingly:
+    # https://github.com/linxGnu/grocksdb/releases
+    # https://pkgs.org/search/?q=rocksdb-dev
+    runs-on: ubuntu-22.04
+    timeout-minutes: 5
+    strategy:
+      fail-fast: false
+      matrix:
+        goversion:
+          - "1.20.x"
+          - "1.21.x"
+    steps:
+      - uses: actions/checkout@v4
+
+      # golang
+      - uses: actions/setup-go@v4
+        with:
+          go-version: ${{ matrix.goversion }}
+
+      - name: install database dependencies
+        run: sudo apt-get install -y libleveldb-dev librocksdb-dev
+
+      - name: Set environment variables for debug mode
+        if: env.ACTIONS_STEP_DEBUG == 'true'
+        run: |
+          export LOG_PATH_DIR=${{ runner.temp }}/logs
+          mkdir -p $LOG_PATH_DIR
+
+          echo "LOG_LEVEL=debug" >> $GITHUB_ENV
+          echo "LOG_PATH_DIR=$LOG_PATH_DIR" >> $GITHUB_ENV
+
+      # test ./pkgs/db
+      - name: test ./tm2/pkg/db
+        run: go test -v ./tm2/pkg/db/...

.github/workflows/lint.yml

@@ -24,7 +24,7 @@ jobs:
       - name: Lint
         uses: golangci/golangci-lint-action@v4
         env:
-          # Don't attempt to compile/resolve C packages.
+          # Don't attempt to compile/resolve C packages (grocksdb, cleveldb).
           CGO_ENABLED: 0
         with:
           # sync with misc/devdeps/go.mod

.github/workflows/tm2.yml

@@ -59,7 +59,7 @@ jobs:
           - _test.pkg.amino
           - _test.pkg.bft
           - _test.pkg.others
-          - _test.pkg.db
+          # _test.pkg.db needs special dependencies -- see db-tests.
     runs-on: ubuntu-latest
     timeout-minutes: 21
     steps:

go.mod

@@ -19,14 +19,17 @@ require (
 	github.com/gorilla/websocket v1.5.1
 	github.com/gotuna/gotuna v0.6.0
 	github.com/jaekwon/testify v1.6.1
+	github.com/jmhodges/levigo v1.0.0
 	github.com/libp2p/go-buffer-pool v0.1.0
+	github.com/linxGnu/grocksdb v1.6.20
 	github.com/mattn/go-runewidth v0.0.15
 	github.com/pelletier/go-toml v1.9.5
 	github.com/peterbourgon/ff/v3 v3.4.0
 	github.com/pmezard/go-difflib v1.0.0
 	github.com/rogpeppe/go-internal v1.11.0
 	github.com/rs/cors v1.10.1
 	github.com/stretchr/testify v1.8.4
+	go.etcd.io/bbolt v1.3.8
 	go.uber.org/multierr v1.10.0
 	go.uber.org/zap v1.26.0
 	go.uber.org/zap/exp v0.2.0

tm2/Makefile

@@ -50,4 +50,8 @@ GOTEST_FLAGS ?= -v -p 1 -timeout=30m
 _test.pkg.others:;  go test $(GOTEST_FLAGS) `go list ./pkg/... | grep -Ev 'pkg/(amino|bft|db|iavl/benchmarks)(/|$$)'`
 _test.pkg.amino:;   go test $(GOTEST_FLAGS) ./pkg/amino/...
 _test.pkg.bft:;     go test $(GOTEST_FLAGS) ./pkg/bft/...
-_test.pkg.db:;      go test $(GOTEST_FLAGS) ./pkg/db/... ./pkg/iavl/benchmarks/...
+
+CGO_ENABLED ?= 0
+_test.pkg.db:
+	@[ -z "$$CGO_ENABLED" ] && echo 'Disabling cgo for database tests. Use CGO_ENABLED=1 to enable it if you have the correct libs' || true
+	CGO_ENABLED=$(CGO_ENABLED) go test $(GOTEST_FLAGS) ./pkg/db/... ./pkg/iavl/benchmarks/...

tm2/pkg/bft/config/config.go

@@ -31,7 +31,9 @@ var (
 )
 
 const (
-	levelDBName = "goleveldb"
+	levelDBName  = "goleveldb"
+	clevelDBName = "cleveldb"
+	boltDBName   = "boltdb"
 )
 
 const (
@@ -241,11 +243,19 @@ type BaseConfig struct {
 	// and verifying their commits
 	FastSyncMode bool `toml:"fast_sync" comment:"If this node is many blocks behind the tip of the chain, FastSync\n allows them to catchup quickly by downloading blocks in parallel\n and verifying their commits"`
 
-	// Database backend: goleveldb
+	// Database backend: goleveldb | cleveldb | boltdb
 	// * goleveldb (github.com/gnolang/goleveldb - most popular implementation)
 	//   - pure go
 	//   - stable
-	DBBackend string `toml:"db_backend" comment:"Database backend: goleveldb\n * goleveldb (github.com/gnolang/goleveldb - most popular implementation)\n  - pure go\n  - stable"`
+	// * cleveldb (uses levigo wrapper)
+	//   - fast
+	//   - requires gcc
+	//   - use cleveldb build tag (go build -tags cleveldb)
+	// * boltdb (uses etcd's fork of bolt - go.etcd.io/bbolt)
+	//   - EXPERIMENTAL
+	//   - may be faster is some use-cases (random reads - indexer)
+	//   - use boltdb build tag (go build -tags boltdb)
+	DBBackend string `toml:"db_backend" comment:"Database backend: goleveldb | cleveldb | boltdb\n * goleveldb (github.com/gnolang/goleveldb - most popular implementation)\n  - pure go\n  - stable\n * cleveldb (uses levigo wrapper)\n  - fast\n  - requires gcc\n  - use cleveldb build tag (go build -tags cleveldb)\n * boltdb (uses etcd's fork of bolt - go.etcd.io/bbolt)\n  - EXPERIMENTAL\n  - may be faster is some use-cases (random reads - indexer)\n  - use boltdb build tag (go build -tags boltdb)"`
 
 	// Database directory
 	DBPath string `toml:"db_dir" comment:"Database directory"`
@@ -355,7 +365,9 @@ func (cfg BaseConfig) ValidateBasic() error {
 	}
 
 	// Verify the DB backend
-	if cfg.DBBackend != levelDBName {
+	if cfg.DBBackend != levelDBName &&
+		cfg.DBBackend != clevelDBName &&
+		cfg.DBBackend != boltDBName {
 		return errInvalidDBBackend
 	}
 

tm2/pkg/db/_all/all.go

@@ -1,7 +1,13 @@
 // Package all imports all available databases. It is useful mostly in tests.
+//
+// cgo databases (rocksdb, cleveldb) will be excluded if CGO_ENABLED=0.
 package all
 
 import (
+	// Keep in sync with list of non-cgo backends.
+	// Add cgo backends in all_cgo.go.
+	_ "github.com/gnolang/gno/tm2/pkg/db/boltdb"
+	_ "github.com/gnolang/gno/tm2/pkg/db/fsdb"
 	_ "github.com/gnolang/gno/tm2/pkg/db/goleveldb"
 	_ "github.com/gnolang/gno/tm2/pkg/db/memdb"
 )

tm2/pkg/db/_all/all_cgo.go

@@ -0,0 +1,10 @@
+//go:build cgo
+
+package all
+
+import (
+	// Keep in sync with list of cgo backends.
+	// Add non-cgo backends in all.go.
+	_ "github.com/gnolang/gno/tm2/pkg/db/cleveldb"
+	_ "github.com/gnolang/gno/tm2/pkg/db/rocksdb"
+)

tm2/pkg/db/_tags/boltdb.go

@@ -0,0 +1,5 @@
+//go:build boltdb
+
+package tags
+
+import _ "github.com/gnolang/gno/tm2/pkg/db/boltdb"

tm2/pkg/db/_tags/cleveldb.go

@@ -0,0 +1,5 @@
+//go:build cleveldb
+
+package tags
+
+import _ "github.com/gnolang/gno/tm2/pkg/db/cleveldb"

tm2/pkg/db/_tags/fsdb.go

@@ -0,0 +1,5 @@
+//go:build fsdb
+
+package tags
+
+import _ "github.com/gnolang/gno/tm2/pkg/db/fsdb"

tm2/pkg/db/_tags/rocksdb.go

@@ -0,0 +1,5 @@
+//go:build rocksdb
+
+package tags
+
+import _ "github.com/gnolang/gno/tm2/pkg/db/rocksdb"