-
Notifications
You must be signed in to change notification settings - Fork 66
/
options.go
2890 lines (2567 loc) · 115 KB
/
options.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
package grocksdb
// #include "rocksdb/c.h"
// #include "grocksdb.h"
import "C"
import (
"fmt"
"unsafe"
)
// CompressionType specifies the block compression.
// DB contents are stored in a set of blocks, each of which holds a
// sequence of key,value pairs. Each block may be compressed before
// being stored in a file. The following enum describes which
// compression method (if any) is used to compress a block.
type CompressionType uint
// Compression types.
const (
NoCompression = CompressionType(C.rocksdb_no_compression)
SnappyCompression = CompressionType(C.rocksdb_snappy_compression)
ZLibCompression = CompressionType(C.rocksdb_zlib_compression)
Bz2Compression = CompressionType(C.rocksdb_bz2_compression)
LZ4Compression = CompressionType(C.rocksdb_lz4_compression)
LZ4HCCompression = CompressionType(C.rocksdb_lz4hc_compression)
XpressCompression = CompressionType(C.rocksdb_xpress_compression)
ZSTDCompression = CompressionType(C.rocksdb_zstd_compression)
)
// CompactionStyle specifies the compaction style.
type CompactionStyle uint
// Compaction styles.
const (
LevelCompactionStyle = CompactionStyle(C.rocksdb_level_compaction)
UniversalCompactionStyle = CompactionStyle(C.rocksdb_universal_compaction)
FIFOCompactionStyle = CompactionStyle(C.rocksdb_fifo_compaction)
)
// CompactionAccessPattern specifies the access patern in compaction.
type CompactionAccessPattern uint
// Access patterns for compaction.
const (
NoneCompactionAccessPattern = CompactionAccessPattern(0)
NormalCompactionAccessPattern = CompactionAccessPattern(1)
SequentialCompactionAccessPattern = CompactionAccessPattern(2)
WillneedCompactionAccessPattern = CompactionAccessPattern(3)
)
// InfoLogLevel describes the log level.
type InfoLogLevel uint
// Log leves.
const (
DebugInfoLogLevel = InfoLogLevel(0)
InfoInfoLogLevel = InfoLogLevel(1)
WarnInfoLogLevel = InfoLogLevel(2)
ErrorInfoLogLevel = InfoLogLevel(3)
FatalInfoLogLevel = InfoLogLevel(4)
)
// WALRecoveryMode mode of WAL Recovery.
type WALRecoveryMode int
const (
// TolerateCorruptedTailRecordsRecovery is original levelDB recovery
// We tolerate incomplete record in trailing data on all logs
// Use case : This is legacy behavior
TolerateCorruptedTailRecordsRecovery = WALRecoveryMode(0)
// AbsoluteConsistencyRecovery recover from clean shutdown
// We don't expect to find any corruption in the WAL
// Use case : This is ideal for unit tests and rare applications that
// can require high consistency guarantee
AbsoluteConsistencyRecovery = WALRecoveryMode(1)
// PointInTimeRecovery recover to point-in-time consistency (default)
// We stop the WAL playback on discovering WAL inconsistency
// Use case : Ideal for systems that have disk controller cache like
// hard disk, SSD without super capacitor that store related data
PointInTimeRecovery = WALRecoveryMode(2)
// SkipAnyCorruptedRecordsRecovery recovery after a disaster
// We ignore any corruption in the WAL and try to salvage as much data as
// possible
// Use case : Ideal for last ditch effort to recover data or systems that
// operate with low grade unrelated data
SkipAnyCorruptedRecordsRecovery = WALRecoveryMode(3)
)
// PrepopulateBlob represents strategy for prepopulate warm/hot blobs which are already in memory into
// blob cache at the time of flush.
type PrepopulateBlob int
const (
// PrepopulateBlobDisable disables prepopulate blob cache.
PrepopulateBlobDisable = PrepopulateBlob(0)
// PrepopulateBlobFlushOnly prepopulates blobs during flush only.
PrepopulateBlobFlushOnly = PrepopulateBlob(1)
)
// Options represent all of the available options when opening a database with Open.
type Options struct {
c *C.rocksdb_options_t
env *C.rocksdb_env_t
// Hold references for GC.
bbto *BlockBasedTableOptions
// We keep these so we can free their memory in Destroy.
ccmp *C.rocksdb_comparator_t
cmo *C.rocksdb_mergeoperator_t
cst *C.rocksdb_slicetransform_t
ccf *C.rocksdb_compactionfilter_t
}
// NewDefaultOptions creates the default Options.
func NewDefaultOptions() *Options {
return newNativeOptions(C.rocksdb_options_create())
}
// NewNativeOptions creates a Options object.
func newNativeOptions(c *C.rocksdb_options_t) *Options {
return &Options{c: c}
}
// GetOptionsFromString creates a Options object from existing opt and string.
// If base is nil, a default opt create by NewDefaultOptions will be used as base opt.
func GetOptionsFromString(base *Options, optStr string) (newOpt *Options, err error) {
providedBaseNil := base == nil
if providedBaseNil {
base = NewDefaultOptions()
}
var (
cErr *C.char
cOptStr = C.CString(optStr)
)
newOpt = NewDefaultOptions()
C.rocksdb_get_options_from_string(base.c, cOptStr, newOpt.c, &cErr)
if err = fromCError(cErr); err != nil {
newOpt.Destroy()
}
C.free(unsafe.Pointer(cOptStr))
if providedBaseNil {
base.Destroy()
}
return
}
// Clone the options
func (opts *Options) Clone() *Options {
cloned := *opts
cloned.c = C.rocksdb_options_create_copy(opts.c)
return &cloned
}
// -------------------
// Parameters that affect behavior
// SetCompactionFilter sets the specified compaction filter
// which will be applied on compactions.
//
// Default: nil
func (opts *Options) SetCompactionFilter(value CompactionFilter) {
C.rocksdb_compactionfilter_destroy(opts.ccf)
if nc, ok := value.(*nativeCompactionFilter); ok {
opts.ccf = nc.c
} else {
idx := registerCompactionFilter(value)
opts.ccf = C.gorocksdb_compactionfilter_create(C.uintptr_t(idx))
}
C.rocksdb_options_set_compaction_filter(opts.c, opts.ccf)
}
// SetComparator sets the comparator which define the order of keys in the table.
// This operation is `move`, thus underlying native c-pointer is owned by Options.
// `cmp` is no longer usable.
//
// Default: a comparator that uses lexicographic byte-wise ordering
func (opts *Options) SetComparator(cmp *Comparator) {
cmp_ := unsafe.Pointer(cmp.c)
opts.SetNativeComparator(cmp_)
cmp.c = nil
}
// SetNativeComparator sets the comparator which define the order of keys in the table.
//
// Default: a comparator that uses lexicographic byte-wise ordering
func (opts *Options) SetNativeComparator(cmp unsafe.Pointer) {
C.rocksdb_comparator_destroy(opts.ccmp)
opts.ccmp = (*C.rocksdb_comparator_t)(cmp)
C.rocksdb_options_set_comparator(opts.c, opts.ccmp)
}
// SetMergeOperator sets the merge operator which will be called
// if a merge operations are used.
//
// Default: nil
func (opts *Options) SetMergeOperator(value MergeOperator) {
C.rocksdb_mergeoperator_destroy(opts.cmo)
if nmo, ok := value.(*nativeMergeOperator); ok {
opts.cmo = nmo.c
} else {
idx := registerMergeOperator(value)
opts.cmo = C.gorocksdb_mergeoperator_create(C.uintptr_t(idx))
}
C.rocksdb_options_set_merge_operator(opts.c, opts.cmo)
}
// This is a factory that provides compaction filter objects which allow
// an application to modify/delete a key-value during background compaction.
//
// A new filter will be created on each compaction run. If multithreaded
// compaction is being used, each created CompactionFilter will only be used
// from a single thread and so does not need to be thread-safe.
//
// Default: a factory that doesn't provide any object
// std::shared_ptr<CompactionFilterFactory> compaction_filter_factory;
// TODO: implement in C and Go
// Version TWO of the compaction_filter_factory
// It supports rolling compaction
//
// Default: a factory that doesn't provide any object
// std::shared_ptr<CompactionFilterFactoryV2> compaction_filter_factory_v2;
// TODO: implement in C and Go
// SetCreateIfMissing specifies whether the database
// should be created if it is missing.
// Default: false
func (opts *Options) SetCreateIfMissing(value bool) {
C.rocksdb_options_set_create_if_missing(opts.c, boolToChar(value))
}
// CreateIfMissing checks if create_if_mission option is set
func (opts *Options) CreateIfMissing() bool {
return charToBool(C.rocksdb_options_get_create_if_missing(opts.c))
}
// SetErrorIfExists specifies whether an error should be raised
// if the database already exists.
// Default: false
func (opts *Options) SetErrorIfExists(value bool) {
C.rocksdb_options_set_error_if_exists(opts.c, boolToChar(value))
}
// ErrorIfExists checks if error_if_exist option is set
func (opts *Options) ErrorIfExists() bool {
return charToBool(C.rocksdb_options_get_error_if_exists(opts.c))
}
// SetParanoidChecks enable/disable paranoid checks.
//
// If true, the implementation will do aggressive checking of the
// data it is processing and will stop early if it detects any
// errors. This may have unforeseen ramifications: for example, a
// corruption of one DB entry may cause a large number of entries to
// become unreadable or for the entire DB to become unopenable.
// If any of the writes to the database fails (Put, Delete, Merge, Write),
// the database will switch to read-only mode and fail all other
// Write operations.
// Default: false
func (opts *Options) SetParanoidChecks(value bool) {
C.rocksdb_options_set_paranoid_checks(opts.c, boolToChar(value))
}
// ParanoidChecks checks if paranoid_check option is set
func (opts *Options) ParanoidChecks() bool {
return charToBool(C.rocksdb_options_get_paranoid_checks(opts.c))
}
// SetDBPaths sets the db_paths option.
//
// db_paths is a list of paths where SST files can be put into, with its target size.
// Newer data is placed into paths specified earlier in the vector while
// older data gradually moves to paths specified later in the vector.
//
// For example, you have a flash device with 10GB allocated for the DB,
// as well as a hard drive of 2TB, you should config it to be:
//
// [{"/flash_path", 10GB}, {"/hard_drive", 2TB}]
//
// The system will try to guarantee data under each path is close to but
// not larger than the target size. But current and future file sizes used
// by determining where to place a file are based on best-effort estimation,
// which means there is a chance that the actual size under the directory
// is slightly more than target size under some workloads. User should give
// some buffer room for those cases.
//
// If none of the paths has sufficient room to place a file, the file will
// be placed to the last path anyway, despite to the target size.
//
// Placing newer data to earlier paths is also best-efforts. User should
// expect user files to be placed in higher levels in some extreme cases.
//
// If left empty, only one path will be used, which is db_name passed when
// opening the DB.
//
// Default: empty
func (opts *Options) SetDBPaths(dbpaths []*DBPath) {
if n := len(dbpaths); n > 0 {
cDbpaths := make([]*C.rocksdb_dbpath_t, n)
for i, v := range dbpaths {
cDbpaths[i] = v.c
}
C.rocksdb_options_set_db_paths(opts.c, &cDbpaths[0], C.size_t(n))
}
}
// SetCFPaths sets cf_paths option. cf_paths is a list of paths where SST files
// for this column family can be put into, with its target size.
//
// Similar to db_paths, newer data is placed into paths specified earlier in the
// vector while older data gradually moves to paths specified later in the vector.
//
// Note that, if a path is supplied to multiple column
// families, it would have files and total size from all
// the column families combined. User should provision for the
// total size(from all the column families) in such cases.
//
// If left empty, db_paths will be used.
// Default: empty
func (opts *Options) SetCFPaths(dbpaths []*DBPath) {
if n := len(dbpaths); n > 0 {
cDbpaths := make([]*C.rocksdb_dbpath_t, n)
for i, v := range dbpaths {
cDbpaths[i] = v.c
}
C.rocksdb_options_set_cf_paths(opts.c, &cDbpaths[0], C.size_t(n))
}
}
// SetEnv sets the specified object to interact with the environment,
// e.g. to read/write files, schedule background work, etc.
//
// NOTE: move semantic. Don't use env after calling this function
func (opts *Options) SetEnv(env *Env) {
if opts.env != nil {
C.rocksdb_env_destroy(opts.env)
}
C.rocksdb_options_set_env(opts.c, env.c)
opts.env = env.c
env.c = nil
}
// SetInfoLog sets info logger.
func (opts *Options) SetInfoLog(logger *Logger) {
C.rocksdb_options_set_info_log(opts.c, logger.c)
}
// GetInfoLog gets info logger.
func (opts *Options) GetInfoLog() *Logger {
return &Logger{
c: C.rocksdb_options_get_info_log(opts.c),
}
}
// SetInfoLogLevel sets the info log level.
//
// Default: InfoInfoLogLevel
func (opts *Options) SetInfoLogLevel(value InfoLogLevel) {
C.rocksdb_options_set_info_log_level(opts.c, C.int(value))
}
// GetInfoLogLevel gets the info log level which options hold
func (opts *Options) GetInfoLogLevel() InfoLogLevel {
return InfoLogLevel(C.rocksdb_options_get_info_log_level(opts.c))
}
// IncreaseParallelism sets the parallelism.
//
// By default, RocksDB uses only one background thread for flush and
// compaction. Calling this function will set it up such that total of
// `total_threads` is used. Good value for `total_threads` is the number of
// cores. You almost definitely want to call this function if your system is
// bottlenecked by RocksDB.
func (opts *Options) IncreaseParallelism(totalThreads int) {
C.rocksdb_options_increase_parallelism(opts.c, C.int(totalThreads))
}
// OptimizeForPointLookup optimize the DB for point lookups.
//
// Use this if you don't need to keep the data sorted, i.e. you'll never use
// an iterator, only Put() and Get() API calls
//
// If you use this with rocksdb >= 5.0.2, you must call `SetAllowConcurrentMemtableWrites(false)`
// to avoid an assertion error immediately on opening the db.
func (opts *Options) OptimizeForPointLookup(blockCacheSizeMB uint64) {
C.rocksdb_options_optimize_for_point_lookup(opts.c, C.uint64_t(blockCacheSizeMB))
}
// OptimizeLevelStyleCompaction optimize the DB for leveld compaction.
//
// Default values for some parameters in ColumnFamilyOptions are not
// optimized for heavy workloads and big datasets, which means you might
// observe write stalls under some conditions. As a starting point for tuning
// RocksDB options, use the following two functions:
// * OptimizeLevelStyleCompaction -- optimizes level style compaction
// * OptimizeUniversalStyleCompaction -- optimizes universal style compaction
// Universal style compaction is focused on reducing Write Amplification
// Factor for big data sets, but increases Space Amplification. You can learn
// more about the different styles here:
// https://github.com/facebook/rocksdb/wiki/Rocksdb-Architecture-Guide
// Make sure to also call IncreaseParallelism(), which will provide the
// biggest performance gains.
// Note: we might use more memory than memtable_memory_budget during high
// write rate period
func (opts *Options) OptimizeLevelStyleCompaction(memtableMemoryBudget uint64) {
C.rocksdb_options_optimize_level_style_compaction(opts.c, C.uint64_t(memtableMemoryBudget))
}
// OptimizeUniversalStyleCompaction optimize the DB for universal compaction.
// See note on OptimizeLevelStyleCompaction.
func (opts *Options) OptimizeUniversalStyleCompaction(memtableMemoryBudget uint64) {
C.rocksdb_options_optimize_universal_style_compaction(opts.c, C.uint64_t(memtableMemoryBudget))
}
// SetAllowConcurrentMemtableWrites whether to allow concurrent memtable writes. Conccurent writes are
// not supported by all memtable factories (currently only SkipList memtables).
// As of rocksdb 5.0.2 you must call `SetAllowConcurrentMemtableWrites(false)`
// if you use `OptimizeForPointLookup`.
func (opts *Options) SetAllowConcurrentMemtableWrites(allow bool) {
C.rocksdb_options_set_allow_concurrent_memtable_write(opts.c, boolToChar(allow))
}
// AllowConcurrentMemtableWrites whether to allow concurrent memtable writes. Conccurent writes are
// not supported by all memtable factories (currently only SkipList memtables).
// As of rocksdb 5.0.2 you must call `SetAllowConcurrentMemtableWrites(false)`
// if you use `OptimizeForPointLookup`.
func (opts *Options) AllowConcurrentMemtableWrites() bool {
return charToBool(C.rocksdb_options_get_allow_concurrent_memtable_write(opts.c))
}
// SetWriteBufferSize sets the amount of data to build up in memory
// (backed by an unsorted log on disk) before converting to a sorted on-disk file.
//
// Larger values increase performance, especially during bulk loads.
// Up to max_write_buffer_number write buffers may be held in memory
// at the same time,
// so you may wish to adjust this parameter to control memory usage.
// Also, a larger write buffer will result in a longer recovery time
// the next time the database is opened.
//
// Default: 64MB
func (opts *Options) SetWriteBufferSize(value uint64) {
C.rocksdb_options_set_write_buffer_size(opts.c, C.size_t(value))
}
// GetWriteBufferSize gets write_buffer_size which is set for options
func (opts *Options) GetWriteBufferSize() uint64 {
return uint64(C.rocksdb_options_get_write_buffer_size(opts.c))
}
// SetMaxWriteBufferNumber sets the maximum number of write buffers
// that are built up in memory.
//
// The default is 2, so that when 1 write buffer is being flushed to
// storage, new writes can continue to the other write buffer.
//
// Default: 2
func (opts *Options) SetMaxWriteBufferNumber(value int) {
C.rocksdb_options_set_max_write_buffer_number(opts.c, C.int(value))
}
// GetMaxWriteBufferNumber gets the maximum number of write buffers
// that are built up in memory.
func (opts *Options) GetMaxWriteBufferNumber() int {
return int(C.rocksdb_options_get_max_write_buffer_number(opts.c))
}
// SetMinWriteBufferNumberToMerge sets the minimum number of write buffers
// that will be merged together before writing to storage.
//
// If set to 1, then all write buffers are flushed to L0 as individual files
// and this increases read amplification because a get request has to check
// in all of these files. Also, an in-memory merge may result in writing lesser
// data to storage if there are duplicate records in each of these
// individual write buffers.
//
// Default: 1
func (opts *Options) SetMinWriteBufferNumberToMerge(value int) {
C.rocksdb_options_set_min_write_buffer_number_to_merge(opts.c, C.int(value))
}
// GetMinWriteBufferNumberToMerge gets the minimum number of write buffers
// that will be merged together before writing to storage.
func (opts *Options) GetMinWriteBufferNumberToMerge() int {
return int(C.rocksdb_options_get_min_write_buffer_number_to_merge(opts.c))
}
// SetMaxOpenFiles sets the number of open files that can be used by the DB.
//
// You may need to increase this if your database has a large working set
// (budget one open file per 2MB of working set).
//
// Default: -1 - unlimited
func (opts *Options) SetMaxOpenFiles(value int) {
C.rocksdb_options_set_max_open_files(opts.c, C.int(value))
}
// GetMaxOpenFiles gets the number of open files that can be used by the DB.
func (opts *Options) GetMaxOpenFiles() int {
return int(C.rocksdb_options_get_max_open_files(opts.c))
}
// SetMaxFileOpeningThreads sets the maximum number of file opening threads.
// If max_open_files is -1, DB will open all files on DB::Open(). You can
// use this option to increase the number of threads used to open the files.
//
// Default: 16
func (opts *Options) SetMaxFileOpeningThreads(value int) {
C.rocksdb_options_set_max_file_opening_threads(opts.c, C.int(value))
}
// GetMaxFileOpeningThreads gets the maximum number of file opening threads.
func (opts *Options) GetMaxFileOpeningThreads() int {
return int(C.rocksdb_options_get_max_file_opening_threads(opts.c))
}
// SetMaxTotalWalSize sets the maximum total wal size (in bytes).
// Once write-ahead logs exceed this size, we will start forcing the flush of
// column families whose memtables are backed by the oldest live WAL file
// (i.e. the ones that are causing all the space amplification). If set to 0
// (default), we will dynamically choose the WAL size limit to be
// [sum of all write_buffer_size * max_write_buffer_number] * 4
// Default: 0
func (opts *Options) SetMaxTotalWalSize(value uint64) {
C.rocksdb_options_set_max_total_wal_size(opts.c, C.uint64_t(value))
}
// GetMaxTotalWalSize gets the maximum total wal size (in bytes).
func (opts *Options) GetMaxTotalWalSize() uint64 {
return uint64(C.rocksdb_options_get_max_total_wal_size(opts.c))
}
// SetCompression sets the compression algorithm.
//
// Default: SnappyCompression, which gives lightweight but fast
// compression.
func (opts *Options) SetCompression(value CompressionType) {
C.rocksdb_options_set_compression(opts.c, C.int(value))
}
// GetCompression returns the compression algorithm.
func (opts *Options) GetCompression() CompressionType {
return CompressionType(C.rocksdb_options_get_compression(opts.c))
}
// SetCompressionOptions sets different options for compression algorithms.
func (opts *Options) SetCompressionOptions(value CompressionOptions) {
C.rocksdb_options_set_compression_options(
opts.c,
C.int(value.WindowBits),
C.int(value.Level),
C.int(value.Strategy),
C.int(value.MaxDictBytes),
)
}
// SetBottommostCompression sets the compression algorithm for
// bottommost level.
func (opts *Options) SetBottommostCompression(value CompressionType) {
C.rocksdb_options_set_bottommost_compression(opts.c, C.int(value))
}
// GetBottommostCompression returns the compression algorithm for
// bottommost level.
func (opts *Options) GetBottommostCompression() CompressionType {
return CompressionType(C.rocksdb_options_get_bottommost_compression(opts.c))
}
// SetBottommostCompressionOptions sets different options for compression algorithms, for bottommost.
//
// `enabled` true to use these compression options.
func (opts *Options) SetBottommostCompressionOptions(value CompressionOptions, enabled bool) {
C.rocksdb_options_set_bottommost_compression_options(
opts.c,
C.int(value.WindowBits),
C.int(value.Level),
C.int(value.Strategy),
C.int(value.MaxDictBytes),
boolToChar(enabled),
)
}
// SetCompressionPerLevel sets different compression algorithm per level.
//
// Different levels can have different compression policies. There
// are cases where most lower levels would like to quick compression
// algorithm while the higher levels (which have more data) use
// compression algorithms that have better compression but could
// be slower. This array should have an entry for
// each level of the database. This array overrides the
// value specified in the previous field 'compression'.
func (opts *Options) SetCompressionPerLevel(value []CompressionType) {
if len(value) > 0 {
cLevels := make([]C.int, len(value))
for i, v := range value {
cLevels[i] = C.int(v)
}
C.rocksdb_options_set_compression_per_level(opts.c, &cLevels[0], C.size_t(len(value)))
}
}
// SetCompressionOptionsZstdMaxTrainBytes sets maximum size of training data passed
// to zstd's dictionary trainer. Using zstd's dictionary trainer can achieve even
// better compression ratio improvements than using `max_dict_bytes` alone.
//
// The training data will be used to generate a dictionary of max_dict_bytes.
//
// Default: 0.
func (opts *Options) SetCompressionOptionsZstdMaxTrainBytes(value int) {
C.rocksdb_options_set_compression_options_zstd_max_train_bytes(opts.c, C.int(value))
}
// GetCompressionOptionsZstdMaxTrainBytes gets maximum size of training data passed
// to zstd's dictionary trainer. Using zstd's dictionary trainer can achieve even
// better compression ratio improvements than using `max_dict_bytes` alone.
func (opts *Options) GetCompressionOptionsZstdMaxTrainBytes() int {
return int(C.rocksdb_options_get_compression_options_zstd_max_train_bytes(opts.c))
}
// SetCompressionOptionsZstdDictTrainer uses/not use zstd trainer to generate dictionaries.
// When this option is set to true, zstd_max_train_bytes of training data sampled from
// max_dict_buffer_bytes buffered data will be passed to zstd dictionary trainer to generate a
// dictionary of size max_dict_bytes.
//
// When this option is false, zstd's API ZDICT_finalizeDictionary() will be
// called to generate dictionaries. zstd_max_train_bytes of training sampled
// data will be passed to this API. Using this API should save CPU time on
// dictionary training, but the compression ratio may not be as good as using
// a dictionary trainer.
//
// Default: true
func (opts *Options) SetCompressionOptionsZstdDictTrainer(enabled bool) {
C.rocksdb_options_set_compression_options_use_zstd_dict_trainer(opts.c, boolToChar(enabled))
}
// GetCompressionOptionsZstdDictTrainer returns if zstd dict trainer is used or not.
func (opts *Options) GetCompressionOptionsZstdDictTrainer() bool {
return charToBool(C.rocksdb_options_get_compression_options_use_zstd_dict_trainer(opts.c))
}
// SetCompressionOptionsParallelThreads sets number of threads for
// parallel compression. Parallel compression is enabled only if threads > 1.
//
// This option is valid only when BlockBasedTable is used.
//
// When parallel compression is enabled, SST size file sizes might be
// more inflated compared to the target size, because more data of unknown
// compressed size is in flight when compression is parallelized. To be
// reasonably accurate, this inflation is also estimated by using historical
// compression ratio and current bytes inflight.
//
// Default: 1.
//
// Note: THE FEATURE IS STILL EXPERIMENTAL
func (opts *Options) SetCompressionOptionsParallelThreads(n int) {
C.rocksdb_options_set_compression_options_parallel_threads(opts.c, C.int(n))
}
// GetCompressionOptionsParallelThreads returns number of threads for
// parallel compression. Parallel compression is enabled only if threads > 1.
//
// This option is valid only when BlockBasedTable is used.
// Default: 1.
//
// Note: THE FEATURE IS STILL EXPERIMENTAL
func (opts *Options) GetCompressionOptionsParallelThreads() int {
return int(C.rocksdb_options_get_compression_options_parallel_threads(opts.c))
}
// SetCompressionOptionsMaxDictBufferBytes limits on data buffering when
// gathering samples to build a dictionary. Zero means no limit. When dictionary
// is disabled (`max_dict_bytes == 0`), enabling this limit (`max_dict_buffer_bytes != 0`)
// has no effect.
//
// In compaction, the buffering is limited to the target file size (see
// `target_file_size_base` and `target_file_size_multiplier`) even if this
// setting permits more buffering. Since we cannot determine where the file
// should be cut until data blocks are compressed with dictionary, buffering
// more than the target file size could lead to selecting samples that belong
// to a later output SST.
//
// Limiting too strictly may harm dictionary effectiveness since it forces
// RocksDB to pick samples from the initial portion of the output SST, which
// may not be representative of the whole file. Configuring this limit below
// `zstd_max_train_bytes` (when enabled) can restrict how many samples we can
// pass to the dictionary trainer. Configuring it below `max_dict_bytes` can
// restrict the size of the final dictionary.
//
// Default: 0 (unlimited)
func (opts *Options) SetCompressionOptionsMaxDictBufferBytes(value uint64) {
C.rocksdb_options_set_compression_options_max_dict_buffer_bytes(opts.c, C.uint64_t(value))
}
// GetCompressionOptionsMaxDictBufferBytes returns the limit on data buffering when
// gathering samples to build a dictionary. Zero means no limit. When dictionary
// is disabled (`max_dict_bytes == 0`), enabling this limit (`max_dict_buffer_bytes != 0`)
// has no effect.
func (opts *Options) GetCompressionOptionsMaxDictBufferBytes() uint64 {
return uint64(C.rocksdb_options_get_compression_options_max_dict_buffer_bytes(opts.c))
}
// SetBottommostCompressionOptionsZstdMaxTrainBytes sets maximum size of training data passed
// to zstd's dictionary trainer for bottommost level. Using zstd's dictionary trainer can achieve even
// better compression ratio improvements than using `max_dict_bytes` alone.
//
// `enabled` true to use these compression options.
func (opts *Options) SetBottommostCompressionOptionsZstdMaxTrainBytes(value int, enabled bool) {
C.rocksdb_options_set_bottommost_compression_options_zstd_max_train_bytes(opts.c, C.int(value), boolToChar(enabled))
}
// SetBottommostCompressionOptionsMaxDictBufferBytes limits on data buffering
// when gathering samples to build a dictionary, for bottom most level.
// Zero means no limit. When dictionary is disabled (`max_dict_bytes == 0`),
// enabling this limit (`max_dict_buffer_bytes != 0`) has no effect.
//
// In compaction, the buffering is limited to the target file size (see
// `target_file_size_base` and `target_file_size_multiplier`) even if this
// setting permits more buffering. Since we cannot determine where the file
// should be cut until data blocks are compressed with dictionary, buffering
// more than the target file size could lead to selecting samples that belong
// to a later output SST.
//
// Limiting too strictly may harm dictionary effectiveness since it forces
// RocksDB to pick samples from the initial portion of the output SST, which
// may not be representative of the whole file. Configuring this limit below
// `zstd_max_train_bytes` (when enabled) can restrict how many samples we can
// pass to the dictionary trainer. Configuring it below `max_dict_bytes` can
// restrict the size of the final dictionary.
//
// Default: 0 (unlimited)
func (opts *Options) SetBottommostCompressionOptionsMaxDictBufferBytes(value uint64, enabled bool) {
C.rocksdb_options_set_bottommost_compression_options_max_dict_buffer_bytes(
opts.c,
C.uint64_t(value),
boolToChar(enabled),
)
}
// SetBottommostCompressionOptionsZstdDictTrainer uses/not use zstd trainer to generate dictionaries.
// When this option is set to true, zstd_max_train_bytes of training data sampled from
// max_dict_buffer_bytes buffered data will be passed to zstd dictionary trainer to generate a
// dictionary of size max_dict_bytes.
//
// When this option is false, zstd's API ZDICT_finalizeDictionary() will be
// called to generate dictionaries. zstd_max_train_bytes of training sampled
// data will be passed to this API. Using this API should save CPU time on
// dictionary training, but the compression ratio may not be as good as using
// a dictionary trainer.
//
// Default: true
func (opts *Options) SetBottommostCompressionOptionsZstdDictTrainer(enabled bool) {
c := boolToChar(enabled)
C.rocksdb_options_set_bottommost_compression_options_use_zstd_dict_trainer(opts.c, c, c)
}
// GetBottommostCompressionOptionsZstdDictTrainer returns if zstd dict trainer is used or not.
func (opts *Options) GetBottommostCompressionOptionsZstdDictTrainer() bool {
return charToBool(C.rocksdb_options_get_bottommost_compression_options_use_zstd_dict_trainer(opts.c))
}
// SetMinLevelToCompress sets the start level to use compression.
func (opts *Options) SetMinLevelToCompress(value int) {
C.rocksdb_options_set_min_level_to_compress(opts.c, C.int(value))
}
// SetPrefixExtractor sets the prefic extractor.
//
// If set, use the specified function to determine the
// prefixes for keys. These prefixes will be placed in the filter.
// Depending on the workload, this can reduce the number of read-IOP
// cost for scans when a prefix is passed via ReadOptions to
// db.NewIterator().
//
// Note: move semantic. Don't use slice transform after calling this function.
func (opts *Options) SetPrefixExtractor(value SliceTransform) {
C.rocksdb_slicetransform_destroy(opts.cst)
if nst, ok := value.(*nativeSliceTransform); ok {
opts.cst = nst.c
} else {
idx := registerSliceTransform(value)
opts.cst = C.gorocksdb_slicetransform_create(C.uintptr_t(idx))
}
C.rocksdb_options_set_prefix_extractor(opts.c, opts.cst)
}
// SetNumLevels sets the number of levels for this database.
//
// Default: 7
func (opts *Options) SetNumLevels(value int) {
C.rocksdb_options_set_num_levels(opts.c, C.int(value))
}
// GetNumLevels gets the number of levels.
func (opts *Options) GetNumLevels() int {
return int(C.rocksdb_options_get_num_levels(opts.c))
}
// SetLevel0FileNumCompactionTrigger sets the number of files
// to trigger level-0 compaction.
//
// A value <0 means that level-0 compaction will not be
// triggered by number of files at all.
//
// Default: 2
func (opts *Options) SetLevel0FileNumCompactionTrigger(value int) {
C.rocksdb_options_set_level0_file_num_compaction_trigger(opts.c, C.int(value))
}
// GetLevel0FileNumCompactionTrigger gets the number of files to trigger level-0 compaction.
func (opts *Options) GetLevel0FileNumCompactionTrigger() int {
return int(C.rocksdb_options_get_level0_file_num_compaction_trigger(opts.c))
}
// SetLevel0SlowdownWritesTrigger sets the soft limit on number of level-0 files.
//
// We start slowing down writes at this point.
// A value <0 means that no writing slow down will be triggered by
// number of files in level-0.
//
// Default: 20
func (opts *Options) SetLevel0SlowdownWritesTrigger(value int) {
C.rocksdb_options_set_level0_slowdown_writes_trigger(opts.c, C.int(value))
}
// GetLevel0SlowdownWritesTrigger gets the soft limit on number of level-0 files.
// We start slowing down writes at this point.
func (opts *Options) GetLevel0SlowdownWritesTrigger() int {
return int(C.rocksdb_options_get_level0_slowdown_writes_trigger(opts.c))
}
// SetLevel0StopWritesTrigger sets the maximum number of level-0 files.
// We stop writes at this point.
//
// Default: 36
func (opts *Options) SetLevel0StopWritesTrigger(value int) {
C.rocksdb_options_set_level0_stop_writes_trigger(opts.c, C.int(value))
}
// GetLevel0StopWritesTrigger gets the maximum number of level-0 files.
// We stop writes at this point.
func (opts *Options) GetLevel0StopWritesTrigger() int {
return int(C.rocksdb_options_get_level0_stop_writes_trigger(opts.c))
}
// SetTargetFileSizeBase sets the target file size for compaction.
//
// Target file size is per-file size for level-1.
// Target file size for level L can be calculated by
// target_file_size_base * (target_file_size_multiplier ^ (L-1))
//
// For example, if target_file_size_base is 2MB and
// target_file_size_multiplier is 10, then each file on level-1 will
// be 2MB, and each file on level 2 will be 20MB,
// and each file on level-3 will be 200MB.
//
// Default: 1MB
func (opts *Options) SetTargetFileSizeBase(value uint64) {
C.rocksdb_options_set_target_file_size_base(opts.c, C.uint64_t(value))
}
// GetTargetFileSizeBase gets the target file size base for compaction.
func (opts *Options) GetTargetFileSizeBase() uint64 {
return uint64(C.rocksdb_options_get_target_file_size_base(opts.c))
}
// SetTargetFileSizeMultiplier sets the target file size multiplier for compaction.
//
// Default: 1
func (opts *Options) SetTargetFileSizeMultiplier(value int) {
C.rocksdb_options_set_target_file_size_multiplier(opts.c, C.int(value))
}
// GetTargetFileSizeMultiplier gets the target file size multiplier for compaction.
func (opts *Options) GetTargetFileSizeMultiplier() int {
return int(C.rocksdb_options_get_target_file_size_multiplier(opts.c))
}
// SetMaxBytesForLevelBase sets the maximum total data size for a level.
//
// It is the max total for level-1.
// Maximum number of bytes for level L can be calculated as
// (max_bytes_for_level_base) * (max_bytes_for_level_multiplier ^ (L-1))
//
// For example, if max_bytes_for_level_base is 20MB, and if
// max_bytes_for_level_multiplier is 10, total data size for level-1
// will be 20MB, total file size for level-2 will be 200MB,
// and total file size for level-3 will be 2GB.
//
// Default: 10MB
func (opts *Options) SetMaxBytesForLevelBase(value uint64) {
C.rocksdb_options_set_max_bytes_for_level_base(opts.c, C.uint64_t(value))
}
// GetMaxBytesForLevelBase gets the maximum total data size for a level.
func (opts *Options) GetMaxBytesForLevelBase() uint64 {
return uint64(C.rocksdb_options_get_max_bytes_for_level_base(opts.c))
}
// SetMaxBytesForLevelMultiplier sets the max bytes for level multiplier.
//
// Default: 10
func (opts *Options) SetMaxBytesForLevelMultiplier(value float64) {
C.rocksdb_options_set_max_bytes_for_level_multiplier(opts.c, C.double(value))
}
// GetMaxBytesForLevelMultiplier gets the max bytes for level multiplier.
func (opts *Options) GetMaxBytesForLevelMultiplier() float64 {
return float64(C.rocksdb_options_get_max_bytes_for_level_multiplier(opts.c))
}
// SetLevelCompactionDynamicLevelBytes specifies whether to pick
// target size of each level dynamically.
//
// We will pick a base level b >= 1. L0 will be directly merged into level b,
// instead of always into level 1. Level 1 to b-1 need to be empty.
// We try to pick b and its target size so that
// 1. target size is in the range of
// (max_bytes_for_level_base / max_bytes_for_level_multiplier,
// max_bytes_for_level_base]
// 2. target size of the last level (level num_levels-1) equals to extra size
// of the level.
//
// At the same time max_bytes_for_level_multiplier and
// max_bytes_for_level_multiplier_additional are still satisfied.
//
// With this option on, from an empty DB, we make last level the base level,
// which means merging L0 data into the last level, until it exceeds
// max_bytes_for_level_base. And then we make the second last level to be
// base level, to start to merge L0 data to second last level, with its
// target size to be 1/max_bytes_for_level_multiplier of the last level's
// extra size. After the data accumulates more so that we need to move the
// base level to the third last one, and so on.
//
// For example, assume max_bytes_for_level_multiplier=10, num_levels=6,
// and max_bytes_for_level_base=10MB.
// Target sizes of level 1 to 5 starts with:
// [- - - - 10MB]
// with base level is level. Target sizes of level 1 to 4 are not applicable
// because they will not be used.
// Until the size of Level 5 grows to more than 10MB, say 11MB, we make
// base target to level 4 and now the targets looks like:
// [- - - 1.1MB 11MB]
// While data are accumulated, size targets are tuned based on actual data
// of level 5. When level 5 has 50MB of data, the target is like:
// [- - - 5MB 50MB]
// Until level 5's actual size is more than 100MB, say 101MB. Now if we keep
// level 4 to be the base level, its target size needs to be 10.1MB, which
// doesn't satisfy the target size range. So now we make level 3 the target
// size and the target sizes of the levels look like:
// [- - 1.01MB 10.1MB 101MB]
// In the same way, while level 5 further grows, all levels' targets grow,
// like
// [- - 5MB 50MB 500MB]
// Until level 5 exceeds 1000MB and becomes 1001MB, we make level 2 the
// base level and make levels' target sizes like this:
// [- 1.001MB 10.01MB 100.1MB 1001MB]
// and go on...
//
// By doing it, we give max_bytes_for_level_multiplier a priority against
// max_bytes_for_level_base, for a more predictable LSM tree shape. It is
// useful to limit worse case space amplification.
//
// max_bytes_for_level_multiplier_additional is ignored with this flag on.
//
// Turning this feature on or off for an existing DB can cause unexpected
// LSM tree structure so it's not recommended.
//
// Default: false
func (opts *Options) SetLevelCompactionDynamicLevelBytes(value bool) {
C.rocksdb_options_set_level_compaction_dynamic_level_bytes(opts.c, boolToChar(value))
}
// GetLevelCompactionDynamicLevelBytes checks if level_compaction_dynamic_level_bytes option
// is set.
func (opts *Options) GetLevelCompactionDynamicLevelBytes() bool {
return charToBool(C.rocksdb_options_get_level_compaction_dynamic_level_bytes(opts.c))
}
// SetMaxCompactionBytes sets the maximum number of bytes in all compacted files.
// We try to limit number of bytes in one compaction to be lower than this
// threshold. But it's not guaranteed.
// Value 0 will be sanitized.
//
// Default: result.target_file_size_base * 25