Skip to content

Commit

Permalink
use grocksdb to support latest version of rocksdb
Browse files Browse the repository at this point in the history
also add API `NewRocksDBWithRawDB` to support read-only open mode and customize options
  • Loading branch information
yihuang committed Mar 6, 2023
1 parent 816127f commit b3e4fd3
Show file tree
Hide file tree
Showing 5 changed files with 30 additions and 34 deletions.
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,9 @@ require (
github.com/gogo/protobuf v1.3.2
github.com/google/btree v1.1.2
github.com/jmhodges/levigo v1.0.0
github.com/linxGnu/grocksdb v1.7.15
github.com/stretchr/testify v1.8.2
github.com/syndtr/goleveldb v1.0.1-0.20200815110645-5c35d600f0ca
github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c
go.etcd.io/bbolt v1.3.7
google.golang.org/grpc v1.53.0
)
Expand Down
11 changes: 2 additions & 9 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,6 @@ github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghf
github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
github.com/cosmos/gorocksdb v1.1.1 h1:N0OqpEKXgsi2qtDm8T1+AlNMXkTm6s1jowYf7/4pH5I=
github.com/cosmos/gorocksdb v1.1.1/go.mod h1:b/U29r/CtguX3TF7mKG1Jjn4APDqh4wECshxXdiWHpA=
github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
Expand All @@ -21,12 +19,6 @@ github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczC
github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo=
github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c h1:8ISkoahWXwZR41ois5lSJBSVw4D0OV19Ht/JSTzvSv0=
github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c/go.mod h1:Yg+htXGokKKdzcwhuNDwVvN+uBxDGXJ7G/VN1d8fa64=
github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 h1:JWuenKqqX8nojtoVVWjGfOF9635RETekkoH6Cc9SX0A=
github.com/facebookgo/stack v0.0.0-20160209184415-751773369052/go.mod h1:UbMTZqLaRiH3MsBH8va0n7s1pQYcu3uTb8G4tygF4Zg=
github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 h1:7HZCaLC5+BZpmbhCOZJ293Lz68O7PYrF2EzeiFMwCLk=
github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4/go.mod h1:5tD+neXqOorC30/tWg0LCSkrqj/AR6gu8yY8/fpw1q0=
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
Expand Down Expand Up @@ -67,6 +59,8 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/linxGnu/grocksdb v1.7.15 h1:AEhP28lkeAybv5UYNYviYISpR6bJejEnKuYbnWAnxx0=
github.com/linxGnu/grocksdb v1.7.15/go.mod h1:pY55D0o+r8yUYLq70QmhdudxYvoDb9F+9puf4m3/W+U=
github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
Expand Down Expand Up @@ -99,7 +93,6 @@ github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSS
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=
Expand Down
37 changes: 20 additions & 17 deletions rocksdb.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ import (
"path/filepath"
"runtime"

"github.com/tecbot/gorocksdb"
"github.com/linxGnu/grocksdb"
)

func init() {
Expand All @@ -20,10 +20,10 @@ func init() {

// RocksDB is a RocksDB backend.
type RocksDB struct {
db *gorocksdb.DB
ro *gorocksdb.ReadOptions
wo *gorocksdb.WriteOptions
woSync *gorocksdb.WriteOptions
db *grocksdb.DB
ro *grocksdb.ReadOptions
wo *grocksdb.WriteOptions
woSync *grocksdb.WriteOptions
}

var _ DB = (*RocksDB)(nil)
Expand All @@ -32,11 +32,11 @@ func NewRocksDB(name string, dir string) (*RocksDB, error) {
// default rocksdb option, good enough for most cases, including heavy workloads.
// 1GB table cache, 512MB write buffer(may use 50% more on heavy workloads).
// compression: snappy as default, need to -lsnappy to enable.
bbto := gorocksdb.NewDefaultBlockBasedTableOptions()
bbto.SetBlockCache(gorocksdb.NewLRUCache(1 << 30))
bbto.SetFilterPolicy(gorocksdb.NewBloomFilter(10))
bbto := grocksdb.NewDefaultBlockBasedTableOptions()
bbto.SetBlockCache(grocksdb.NewLRUCache(1 << 30))
bbto.SetFilterPolicy(grocksdb.NewBloomFilter(10))

opts := gorocksdb.NewDefaultOptions()
opts := grocksdb.NewDefaultOptions()
opts.SetBlockBasedTableFactory(bbto)
opts.SetCreateIfMissing(true)
opts.IncreaseParallelism(runtime.NumCPU())
Expand All @@ -45,23 +45,26 @@ func NewRocksDB(name string, dir string) (*RocksDB, error) {
return NewRocksDBWithOptions(name, dir, opts)
}

func NewRocksDBWithOptions(name string, dir string, opts *gorocksdb.Options) (*RocksDB, error) {
func NewRocksDBWithOptions(name string, dir string, opts *grocksdb.Options) (*RocksDB, error) {
dbPath := filepath.Join(dir, name+".db")
db, err := gorocksdb.OpenDb(opts, dbPath)
db, err := grocksdb.OpenDb(opts, dbPath)
if err != nil {
return nil, err
}
ro := gorocksdb.NewDefaultReadOptions()
wo := gorocksdb.NewDefaultWriteOptions()
woSync := gorocksdb.NewDefaultWriteOptions()
ro := grocksdb.NewDefaultReadOptions()
wo := grocksdb.NewDefaultWriteOptions()
woSync := grocksdb.NewDefaultWriteOptions()
woSync.SetSync(true)
database := &RocksDB{
return NewRocksDBWithRawDB(db, ro, wo, woSync), nil
}

func NewRocksDBWithRawDB(db *grocksdb.DB, ro *grocksdb.ReadOptions, wo *grocksdb.WriteOptions, woSync *grocksdb.WriteOptions) *RocksDB {
return &RocksDB{
db: db,
ro: ro,
wo: wo,
woSync: woSync,
}
return database, nil
}

// Get implements DB.
Expand Down Expand Up @@ -139,7 +142,7 @@ func (db *RocksDB) DeleteSync(key []byte) error {
return nil
}

func (db *RocksDB) DB() *gorocksdb.DB {
func (db *RocksDB) DB() *grocksdb.DB {
return db.db
}

Expand Down
6 changes: 3 additions & 3 deletions rocksdb_batch.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,19 +3,19 @@

package db

import "github.com/tecbot/gorocksdb"
import "github.com/linxGnu/grocksdb"

type rocksDBBatch struct {
db *RocksDB
batch *gorocksdb.WriteBatch
batch *grocksdb.WriteBatch
}

var _ Batch = (*rocksDBBatch)(nil)

func newRocksDBBatch(db *RocksDB) *rocksDBBatch {
return &rocksDBBatch{
db: db,
batch: gorocksdb.NewWriteBatch(),
batch: grocksdb.NewWriteBatch(),
}
}

Expand Down
8 changes: 4 additions & 4 deletions rocksdb_iterator.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,19 +6,19 @@ package db
import (
"bytes"

"github.com/tecbot/gorocksdb"
"github.com/linxGnu/grocksdb"
)

type rocksDBIterator struct {
source *gorocksdb.Iterator
source *grocksdb.Iterator
start, end []byte
isReverse bool
isInvalid bool
}

var _ Iterator = (*rocksDBIterator)(nil)

func newRocksDBIterator(source *gorocksdb.Iterator, start, end []byte, isReverse bool) *rocksDBIterator {
func newRocksDBIterator(source *grocksdb.Iterator, start, end []byte, isReverse bool) *rocksDBIterator {
if isReverse {
if end == nil {
source.SeekToLast()
Expand Down Expand Up @@ -136,7 +136,7 @@ func (itr *rocksDBIterator) assertIsValid() {
// moveSliceToBytes will free the slice and copy out a go []byte
// This function can be applied on *Slice returned from Key() and Value()
// of an Iterator, because they are marked as freed.
func moveSliceToBytes(s *gorocksdb.Slice) []byte {
func moveSliceToBytes(s *grocksdb.Slice) []byte {
defer s.Free()
if !s.Exists() {
return nil
Expand Down

0 comments on commit b3e4fd3

Please sign in to comment.