feat(pkg/db): remove cleveldb, fsdb and rocksdb (#1714)

Split from #1602. Context:

> @zivkovicmilos:[^1] Unpopular opinion: we should probably just drop
these DBs entirely from the codebase, they are never gonna be used in
place of LevelDB in the near future
>
> @ajnavarro: IMHO the needed database is defined by the application
needs, not from any other external usecases. Sticking to one of the
existing key/value databases, we can use their special capabilities to
make it more performant for our use case (yes, even if they are all
key/value DBs, they differ in a lot of low-level functionality). \
> I would say to use one and add support in the future for more if
needed.

After a discussion in the review meeting w/ @jaekwon, this PR is now
changed to maintain another reference implementation with boltdb. It
also makes sure that it is selectable, when available, from the tm2 node
configuration.

After this PR, there are no databases in the codebase using cgo.

[^1]: #1602 (comment)

.github/dependabot.yml

@@ -28,11 +28,6 @@ updates:
       everything-else:
         patterns:
           - "*"
-        exclude-patterns:
-          # NOTE: grocksdb should be updated manually, to match the version
-          # available on Ubuntu's latest LTS release; and updated in conjunction
-          # with the ubuntu version on .github/workflows/db-tests.yml
-          - "github.com/linxGnu/grocksdb"
     open-pull-requests-limit: 10
     pull-request-branch-name:
       separator: "-"

.github/workflows/lint_template.yml

@@ -19,8 +19,6 @@ jobs:
           go-version: ${{ inputs.go-version }}
       - name: Checkout code
         uses: actions/checkout@v4
-      - name: Install database dependencies
-        run: sudo apt-get install -y libleveldb-dev librocksdb-dev
       - name: Lint
         uses: golangci/golangci-lint-action@v5
         with:

.github/workflows/test_template.yml

@@ -24,8 +24,6 @@ jobs:
               go-version: ${{ inputs.go-version }}
         - name: Checkout code
           uses: actions/checkout@v4
-        - name: Install database dependencies
-          run: sudo apt-get install -y libleveldb-dev librocksdb-dev
         - name: Go test
           run: go test -coverprofile coverage.out -covermode=atomic -timeout ${{ inputs.tests-timeout }} -v ./...
           working-directory:  ${{ inputs.modulepath }}

.github/workflows/tm2.yml

@@ -5,7 +5,7 @@ on:
     branches:
       - master
   workflow_dispatch:
-  pull_request: 
+  pull_request:
     paths:
       - "tm2/**"
       - ".github/**"
@@ -17,4 +17,4 @@ jobs:
     with:
       modulepath: "tm2"
     secrets:
-      codecov-token: ${{ secrets.CODECOV_TOKEN }}
+      codecov-token: ${{ secrets.CODECOV_TOKEN }}

contribs/gnodev/go.mod

@@ -53,6 +53,7 @@ require (
 	github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 // indirect
 	github.com/zondax/hid v0.9.2 // indirect
 	github.com/zondax/ledger-go v0.14.3 // indirect
+	go.etcd.io/bbolt v1.3.9 // indirect
 	go.opentelemetry.io/otel v1.25.0 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.25.0 // indirect
 	go.opentelemetry.io/otel/metric v1.25.0 // indirect

gno.land/pkg/gnoland/app.go

@@ -20,6 +20,7 @@ import (
 	"github.com/gnolang/gno/tm2/pkg/store/iavl"
 
 	// Only goleveldb is supported for now.
+	_ "github.com/gnolang/gno/tm2/pkg/db/_tags"
 	_ "github.com/gnolang/gno/tm2/pkg/db/goleveldb"
 	"github.com/gnolang/gno/tm2/pkg/db/memdb"
 )

go.mod

@@ -18,9 +18,7 @@ require (
 	github.com/gorilla/mux v1.8.1
 	github.com/gorilla/websocket v1.5.1
 	github.com/gotuna/gotuna v0.6.0
-	github.com/jmhodges/levigo v1.0.0
 	github.com/libp2p/go-buffer-pool v0.1.0
-	github.com/linxGnu/grocksdb v1.6.20
 	github.com/mattn/go-runewidth v0.0.15
 	github.com/pelletier/go-toml v1.9.5
 	github.com/peterbourgon/ff/v3 v3.4.0
@@ -66,7 +64,6 @@ require (
 	github.com/zondax/ledger-go v0.14.3 // indirect
 	go.opentelemetry.io/otel/trace v1.25.0 // indirect
 	go.opentelemetry.io/proto/otlp v1.1.0 // indirect
-	golang.org/x/sync v0.7.0 // indirect
 	golang.org/x/sys v0.18.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20240227224415-6ceb2ff114de // indirect

tm2/Makefile

@@ -62,6 +62,4 @@ _test.flappy:
 _test.pkg.others:;  go test $(GOTEST_FLAGS) `go list ./pkg/... | grep -Ev 'pkg/(amino|bft|db|iavl/benchmarks)(/|$$)'`
 _test.pkg.amino:;   go test $(GOTEST_FLAGS) ./pkg/amino/...
 _test.pkg.bft:;     go test $(GOTEST_FLAGS) ./pkg/bft/...
-
-_test.pkg.db:
-	CGO_ENABLED=$(CGO_ENABLED) go test $(GOTEST_FLAGS) ./pkg/db/... ./pkg/iavl/benchmarks/...
+_test.pkg.db:;      go test $(GOTEST_FLAGS) ./pkg/db/... ./pkg/iavl/benchmarks/...

tm2/pkg/bft/config/config.go

@@ -5,6 +5,7 @@ import (
 	"os"
 	"path/filepath"
 	"regexp"
+	"slices"
 
 	"dario.cat/mergo"
 	abci "github.com/gnolang/gno/tm2/pkg/bft/abci/types"
@@ -256,19 +257,15 @@ type BaseConfig struct {
 	// and verifying their commits
 	FastSyncMode bool `toml:"fast_sync" comment:"If this node is many blocks behind the tip of the chain, FastSync\n allows them to catchup quickly by downloading blocks in parallel\n and verifying their commits"`
 
-	// Database backend: goleveldb | cleveldb | boltdb
+	// Database backend: goleveldb | boltdb
 	// * goleveldb (github.com/syndtr/goleveldb - most popular implementation)
 	//   - pure go
 	//   - stable
-	// * cleveldb (uses levigo wrapper)
-	//   - fast
-	//   - requires gcc
-	//   - use cleveldb build tag (go build -tags cleveldb)
 	// * boltdb (uses etcd's fork of bolt - go.etcd.io/bbolt)
 	//   - EXPERIMENTAL
 	//   - may be faster is some use-cases (random reads - indexer)
 	//   - use boltdb build tag (go build -tags boltdb)
-	DBBackend string `toml:"db_backend" comment:"Database backend: goleveldb | cleveldb | boltdb\n * goleveldb (github.com/syndtr/goleveldb - most popular implementation)\n  - pure go\n  - stable\n * cleveldb (uses levigo wrapper)\n  - fast\n  - requires gcc\n  - use cleveldb build tag (go build -tags cleveldb)\n * boltdb (uses etcd's fork of bolt - go.etcd.io/bbolt)\n  - EXPERIMENTAL\n  - may be faster is some use-cases (random reads - indexer)\n  - use boltdb build tag (go build -tags boltdb)"`
+	DBBackend string `toml:"db_backend" comment:"Database backend: goleveldb | boltdb\n * goleveldb (github.com/syndtr/goleveldb - most popular implementation)\n  - pure go\n  - stable\n* boltdb (uses etcd's fork of bolt - go.etcd.io/bbolt)\n  - EXPERIMENTAL\n  - may be faster is some use-cases (random reads - indexer)\n  - use boltdb build tag (go build -tags boltdb)"`
 
 	// Database directory
 	DBPath string `toml:"db_dir" comment:"Database directory"`
@@ -369,9 +366,10 @@ func (cfg BaseConfig) ValidateBasic() error {
 	}
 
 	// Verify the DB backend
-	if cfg.DBBackend != db.GoLevelDBBackend.String() &&
-		cfg.DBBackend != db.CLevelDBBackend.String() &&
-		cfg.DBBackend != db.BoltDBBackend.String() {
+	// This will reject also any databases that haven't been added with build tags.
+	// always reject memdb, as it shouldn't be used as a real-life database.
+	if cfg.DBBackend == "memdb" ||
+		!slices.Contains(db.BackendList(), db.BackendType(cfg.DBBackend)) {
 		return errInvalidDBBackend
 	}
 

tm2/pkg/bft/config/config_test.go

@@ -6,6 +6,9 @@ import (
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+
+	// allows the default config to have a valid DB
+	_ "github.com/gnolang/gno/tm2/pkg/db/goleveldb"
 )
 
 func TestConfig_LoadOrMakeConfigWithOptions(t *testing.T) {

tm2/pkg/bft/state/state_test.go

@@ -16,9 +16,7 @@ import (
 	"github.com/gnolang/gno/tm2/pkg/bft/types"
 	"github.com/gnolang/gno/tm2/pkg/crypto/ed25519"
 	dbm "github.com/gnolang/gno/tm2/pkg/db"
-	_ "github.com/gnolang/gno/tm2/pkg/db/_tags"
-	_ "github.com/gnolang/gno/tm2/pkg/db/goleveldb"
-	_ "github.com/gnolang/gno/tm2/pkg/db/memdb"
+	_ "github.com/gnolang/gno/tm2/pkg/db/_all"
 	"github.com/gnolang/gno/tm2/pkg/random"
 )
 

tm2/pkg/db/_all/all.go

@@ -1,13 +1,8 @@
 // Package all imports all available databases. It is useful mostly in tests.
-//
-// cgo databases (rocksdb, cleveldb) will be excluded if CGO_ENABLED=0.
 package all
 
 import (
-	// Keep in sync with list of non-cgo backends.
-	// Add cgo backends in all_cgo.go.
 	_ "github.com/gnolang/gno/tm2/pkg/db/boltdb"
-	_ "github.com/gnolang/gno/tm2/pkg/db/fsdb"
 	_ "github.com/gnolang/gno/tm2/pkg/db/goleveldb"
 	_ "github.com/gnolang/gno/tm2/pkg/db/memdb"
 )