From b129e169a5025ec16bb71ee4cd104d92de55188b Mon Sep 17 00:00:00 2001 From: mrambacher Date: Mon, 16 Oct 2023 14:44:06 -0400 Subject: [PATCH] Add JAVA and C Interface for EnableSpeedbFeatures --- HISTORY.md | 1 + db/c.cc | 70 ++++++++ db/c_test.c | 53 ++++++ include/rocksdb/c.h | 33 ++++ java/CMakeLists.txt | 16 ++ java/Makefile | 18 +- java/rocksjni/options.cc | 169 ++++++++++++++++++ .../java/org/rocksdb/ColumnFamilyOptions.java | 21 +++ .../rocksdb/ColumnFamilyOptionsInterface.java | 22 +++ java/src/main/java/org/rocksdb/DBOptions.java | 21 +++ .../java/org/rocksdb/DBOptionsInterface.java | 22 +++ java/src/main/java/org/rocksdb/Options.java | 21 +++ .../main/java/org/rocksdb/SharedOptions.java | 69 +++++++ .../java/org/rocksdb/SharedOptionsTest.java | 115 ++++++++++++ 14 files changed, 650 insertions(+), 1 deletion(-) create mode 100644 java/src/main/java/org/rocksdb/SharedOptions.java create mode 100644 java/src/test/java/org/rocksdb/SharedOptionsTest.java diff --git a/HISTORY.md b/HISTORY.md index 738aa63535..8d64b1a81f 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -7,6 +7,7 @@ ### Enhancements * Added a kUseBaseAddress flag and GetBaseOffset flag to OptionTypeInfo. If this flag is set and a function is used for processing options, the function is passed the base address of the struct rather than the specific field (#397) +* Enabled speedb features in C and Java (#722) ### Bug Fixes * Stall deadlock consists small cfs (#637). diff --git a/db/c.cc b/db/c.cc index c72f9d8c32..446b4f104c 100644 --- a/db/c.cc +++ b/db/c.cc @@ -121,6 +121,7 @@ using ROCKSDB_NAMESPACE::RateLimiter; using ROCKSDB_NAMESPACE::ReadOptions; using ROCKSDB_NAMESPACE::RestoreOptions; using ROCKSDB_NAMESPACE::SequentialFile; +using ROCKSDB_NAMESPACE::SharedOptions; using ROCKSDB_NAMESPACE::Slice; using ROCKSDB_NAMESPACE::SliceParts; using ROCKSDB_NAMESPACE::SliceTransform; @@ -189,6 +190,9 @@ struct rocksdb_writeoptions_t { struct rocksdb_options_t { Options rep; }; +struct rocksdb_shared_options_t { + SharedOptions rep; +}; struct rocksdb_compactoptions_t { CompactRangeOptions rep; Slice full_history_ts_low; @@ -2828,6 +2832,27 @@ void rocksdb_options_optimize_universal_style_compaction( opt->rep.OptimizeUniversalStyleCompaction(memtable_memory_budget); } +void rocksdb_options_enable_speedb_features(rocksdb_options_t* opt, + rocksdb_shared_options_t* shared) { + opt->rep.EnableSpeedbFeatures(shared->rep); +} + +void rocksdb_options_enable_speedb_features_db( + rocksdb_options_t* db_options, rocksdb_shared_options_t* shared) { + DBOptions& db_options_part = db_options->rep; + db_options_part.EnableSpeedbFeaturesDB(shared->rep); +} + +void rocksdb_options_enable_speedb_features_cf( + rocksdb_options_t* column_family_options, + rocksdb_shared_options_t* shared) { + ColumnFamilyOptions& cf_options_part = column_family_options->rep; + cf_options_part.EnableSpeedbFeaturesCF(shared->rep); +} + +void rocksdb_options_enable_speedb_features_db( + rocksdb_options_t* opt, rocksdb_shared_options_t* shared); + void rocksdb_options_set_allow_ingest_behind(rocksdb_options_t* opt, unsigned char v) { opt->rep.allow_ingest_behind = v; @@ -3975,6 +4000,51 @@ int rocksdb_options_get_wal_compression(rocksdb_options_t* opt) { return opt->rep.wal_compression; } +rocksdb_shared_options_t* rocksdb_shared_options_create( + size_t total_ram_size_bytes, size_t total_threads) { + return new rocksdb_shared_options_t{ + SharedOptions(total_ram_size_bytes, total_threads)}; +} +rocksdb_shared_options_t* rocksdb_shared_options_create_from( + size_t total_ram_size_bytes, size_t total_threads, + size_t delayed_write_rate, size_t bucket_size, int use_merge) { + return new rocksdb_shared_options_t{ + SharedOptions(total_ram_size_bytes, total_threads, delayed_write_rate, + bucket_size, use_merge)}; +} + +void rocksdb_shared_options_destroy(rocksdb_shared_options_t* opt) { + delete opt; +} + +size_t rocksdb_shared_options_get_max_write_buffer_manager_size( + rocksdb_shared_options_t* opt) { + return opt->rep.GetMaxWriteBufferManagerSize(); +} + +size_t rocksdb_shared_options_get_total_threads(rocksdb_shared_options_t* opt) { + return opt->rep.GetTotalThreads(); +} + +size_t rocksdb_shared_options_get_total_ram_size_bytes( + rocksdb_shared_options_t* opt) { + return opt->rep.GetTotalRamSizeBytes(); +} + +size_t rocksdb_shared_options_get_delayed_write_rate( + rocksdb_shared_options_t* opt) { + return opt->rep.GetDelayedWriteRate(); +} + +size_t rocksdb_shared_options_get_bucket_size(rocksdb_shared_options_t* opt) { + return opt->rep.GetBucketSize(); +} + +unsigned char rocksdb_shared_options_is_merge_memtable_supported( + rocksdb_shared_options_t* opt) { + return opt->rep.IsMergeMemtableSupported(); +} + rocksdb_ratelimiter_t* rocksdb_ratelimiter_create(int64_t rate_bytes_per_sec, int64_t refill_period_us, int32_t fairness) { diff --git a/db/c_test.c b/db/c_test.c index 063ed93bb8..2809214d3d 100644 --- a/db/c_test.c +++ b/db/c_test.c @@ -2586,6 +2586,59 @@ int main(int argc, char** argv) { rocksdb_options_destroy(o); } + StartPhase("shared_options"); + { + size_t total_ram_size = 100 * 1024 * 1024 * 1024ul; + size_t total_threads = 8; + size_t delayed_write_rate = 256 * 1024 * 1024ul; + size_t bucket_size = 50000; + rocksdb_shared_options_t* so; + so = rocksdb_shared_options_create(total_ram_size, total_threads); + CheckCondition(total_threads == + rocksdb_shared_options_get_total_threads(so)); + CheckCondition(total_ram_size == + rocksdb_shared_options_get_total_ram_size_bytes(so)); + rocksdb_shared_options_destroy(so); + + so = rocksdb_shared_options_create_from(total_ram_size, total_threads, + delayed_write_rate, bucket_size, 1); + CheckCondition(total_threads == + rocksdb_shared_options_get_total_threads(so)); + CheckCondition(total_ram_size == + rocksdb_shared_options_get_total_ram_size_bytes(so)); + CheckCondition(delayed_write_rate == + rocksdb_shared_options_get_delayed_write_rate(so)); + CheckCondition(bucket_size == rocksdb_shared_options_get_bucket_size(so)); + CheckCondition( + total_ram_size / 4 == + rocksdb_shared_options_get_max_write_buffer_manager_size(so)); + CheckCondition(1 == rocksdb_shared_options_is_merge_memtable_supported(so)); + rocksdb_options_t* opts1 = rocksdb_options_create(); + rocksdb_options_enable_speedb_features(opts1, so); + + CheckCondition((int)total_threads == + rocksdb_options_get_max_background_jobs(opts1)); + CheckCondition(1 << 20 == rocksdb_options_get_bytes_per_sync(opts1)); + CheckCondition(4 == rocksdb_options_get_max_write_buffer_number(opts1)); + + rocksdb_options_t* opts2 = rocksdb_options_create(); + size_t orig_total_threads = rocksdb_options_get_max_background_jobs(opts2); + CheckCondition(total_threads != orig_total_threads); + rocksdb_options_enable_speedb_features_db(opts2, so); + CheckCondition((int)total_threads == + rocksdb_options_get_max_background_jobs(opts2)); + + rocksdb_options_t* opts3 = rocksdb_options_create(); + rocksdb_options_enable_speedb_features_cf(opts3, so); + CheckCondition((int)orig_total_threads == + rocksdb_options_get_max_background_jobs(opts3)); + + rocksdb_options_destroy(opts3); + rocksdb_options_destroy(opts2); + rocksdb_options_destroy(opts1); + rocksdb_shared_options_destroy(so); + } + StartPhase("read_options"); { rocksdb_readoptions_t* ro; diff --git a/include/rocksdb/c.h b/include/rocksdb/c.h index 245ccd5f13..258d0f0771 100644 --- a/include/rocksdb/c.h +++ b/include/rocksdb/c.h @@ -109,6 +109,7 @@ typedef struct rocksdb_iterator_t rocksdb_iterator_t; typedef struct rocksdb_logger_t rocksdb_logger_t; typedef struct rocksdb_mergeoperator_t rocksdb_mergeoperator_t; typedef struct rocksdb_options_t rocksdb_options_t; +typedef struct rocksdb_shared_options_t rocksdb_shared_options_t; typedef struct rocksdb_compactoptions_t rocksdb_compactoptions_t; typedef struct rocksdb_block_based_table_options_t rocksdb_block_based_table_options_t; @@ -1113,6 +1114,12 @@ extern ROCKSDB_LIBRARY_API void rocksdb_options_optimize_for_point_lookup( rocksdb_options_t* opt, uint64_t block_cache_size_mb); extern ROCKSDB_LIBRARY_API void rocksdb_options_optimize_level_style_compaction( rocksdb_options_t* opt, uint64_t memtable_memory_budget); +extern ROCKSDB_LIBRARY_API void rocksdb_options_enable_speedb_features( + rocksdb_options_t* opt, rocksdb_shared_options_t* shared); +extern ROCKSDB_LIBRARY_API void rocksdb_options_enable_speedb_features_db( + rocksdb_options_t* opt, rocksdb_shared_options_t* shared); +extern ROCKSDB_LIBRARY_API void rocksdb_options_enable_speedb_features_cf( + rocksdb_options_t* opt, rocksdb_shared_options_t* shared); extern ROCKSDB_LIBRARY_API void rocksdb_options_optimize_universal_style_compaction( rocksdb_options_t* opt, uint64_t memtable_memory_budget); @@ -1683,6 +1690,32 @@ extern ROCKSDB_LIBRARY_API void rocksdb_options_set_wal_compression( extern ROCKSDB_LIBRARY_API int rocksdb_options_get_wal_compression( rocksdb_options_t* opt); +/* SharedOptions */ +extern ROCKSDB_LIBRARY_API rocksdb_shared_options_t* +rocksdb_shared_options_create(size_t total_ram_size_bytes, + size_t total_threads); +extern ROCKSDB_LIBRARY_API rocksdb_shared_options_t* +rocksdb_shared_options_create_from(size_t total_ram_size_bytes, + size_t total_threads, + size_t delayed_write_rate, + size_t bucket_size, int use_merge); +extern ROCKSDB_LIBRARY_API void rocksdb_shared_options_destroy( + rocksdb_shared_options_t* options); +extern ROCKSDB_LIBRARY_API size_t +rocksdb_shared_options_get_max_write_buffer_manager_size( + rocksdb_shared_options_t* opt); +extern ROCKSDB_LIBRARY_API size_t +rocksdb_shared_options_get_total_threads(rocksdb_shared_options_t* opt); +extern ROCKSDB_LIBRARY_API size_t +rocksdb_shared_options_get_total_ram_size_bytes(rocksdb_shared_options_t* opt); +extern ROCKSDB_LIBRARY_API size_t +rocksdb_shared_options_get_delayed_write_rate(rocksdb_shared_options_t* opt); +extern ROCKSDB_LIBRARY_API size_t +rocksdb_shared_options_get_bucket_size(rocksdb_shared_options_t* options); +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_shared_options_is_merge_memtable_supported( + rocksdb_shared_options_t* options); + /* RateLimiter */ extern ROCKSDB_LIBRARY_API rocksdb_ratelimiter_t* rocksdb_ratelimiter_create( int64_t rate_bytes_per_sec, int64_t refill_period_us, int32_t fairness); diff --git a/java/CMakeLists.txt b/java/CMakeLists.txt index 6b068bd2da..8b7bd26e00 100644 --- a/java/CMakeLists.txt +++ b/java/CMakeLists.txt @@ -1,3 +1,17 @@ +# Copyright (C) 2023 Speedb Ltd. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + cmake_minimum_required(VERSION 3.4) if(${CMAKE_VERSION} VERSION_LESS "3.11.4") @@ -280,6 +294,7 @@ set(JAVA_MAIN_CLASSES src/main/java/org/rocksdb/util/ReverseBytewiseComparator.java src/main/java/org/rocksdb/util/SizeUnit.java src/main/java/org/rocksdb/UInt64AddOperator.java + src/main/java/org/rocksdb/SharedOptions.java ) set(JAVA_TEST_CLASSES @@ -523,6 +538,7 @@ if(${CMAKE_VERSION} VERSION_LESS "3.11.4") org.rocksdb.WriteBatchTestInternalHelper org.rocksdb.WriteBufferManager org.rocksdb.test.TestableEventListener + org.rocksdb.SharedOptions ) create_javah( diff --git a/java/Makefile b/java/Makefile index 5cc3bcdbd4..4326f90e5e 100644 --- a/java/Makefile +++ b/java/Makefile @@ -1,3 +1,17 @@ +# Copyright (C) 2023 Speedb Ltd. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + PROJECT_NAME?=speedb NATIVE_JAVA_CLASSES = \ @@ -92,7 +106,8 @@ NATIVE_JAVA_CLASSES = \ org.rocksdb.WriteOptions\ org.rocksdb.WriteBatchWithIndex\ org.rocksdb.WriteBufferManager\ - org.rocksdb.WBWIRocksIterator + org.rocksdb.WBWIRocksIterator\ + org.rocksdb.SharedOptions NATIVE_JAVA_TEST_CLASSES = \ org.rocksdb.RocksDBExceptionTest\ @@ -174,6 +189,7 @@ JAVA_TESTS = \ org.rocksdb.RocksMemEnvTest\ org.rocksdb.util.SizeUnitTest\ org.rocksdb.SecondaryDBTest\ + org.rocksdb.SharedOptionsTest\ org.rocksdb.SliceTest\ org.rocksdb.SnapshotTest\ org.rocksdb.SstFileManagerTest\ diff --git a/java/rocksjni/options.cc b/java/rocksjni/options.cc index 0d84901c91..5bc214313f 100644 --- a/java/rocksjni/options.cc +++ b/java/rocksjni/options.cc @@ -1,3 +1,17 @@ +// Copyright (C) 2023 Speedb Ltd. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// // Copyright (c) 2011-present, Facebook, Inc. All rights reserved. // This source code is licensed under both the GPLv2 (found in the // COPYING file in the root directory) and Apache 2.0 License @@ -21,6 +35,7 @@ #include "include/org_rocksdb_FlushOptions.h" #include "include/org_rocksdb_Options.h" #include "include/org_rocksdb_ReadOptions.h" +#include "include/org_rocksdb_SharedOptions.h" #include "include/org_rocksdb_WriteOptions.h" #include "rocksdb/comparator.h" #include "rocksdb/convenience.h" @@ -3264,6 +3279,19 @@ void Java_org_rocksdb_Options_optimizeForPointLookup( ->OptimizeForPointLookup(block_cache_size_mb); } +/* + * Class: org_rocksdb_Options + * Method: enableSpeedbFeatures + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_rocksdb_Options_enableSpeedbFeatures( + JNIEnv*, jobject, jlong jhandle, jlong shared_handle) { + auto shared_opts = + reinterpret_cast(shared_handle); + auto opts = reinterpret_cast(jhandle); + opts->EnableSpeedbFeatures(*shared_opts); +} + /* * Class: org_rocksdb_Options * Method: optimizeLevelStyleCompaction @@ -4097,6 +4125,21 @@ void Java_org_rocksdb_ColumnFamilyOptions_optimizeForPointLookup( ->OptimizeForPointLookup(block_cache_size_mb); } +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: enableSpeedbFeatures + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL +Java_org_rocksdb_ColumnFamilyOptions_enableSpeedbFeatures(JNIEnv*, jobject, + jlong jhandle, + jlong shared_handle) { + auto shared_opts = + reinterpret_cast(shared_handle); + auto opts = + reinterpret_cast(jhandle); + opts->EnableSpeedbFeaturesCF(*shared_opts); +} /* * Class: org_rocksdb_ColumnFamilyOptions * Method: optimizeLevelStyleCompaction @@ -5944,6 +5987,19 @@ void Java_org_rocksdb_DBOptions_optimizeForSmallDb(JNIEnv*, jobject, ->OptimizeForSmallDb(); } +/* + * Class: org_rocksdb_DBOptions + * Method: enableSpeedbFeatures + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_rocksdb_DBOptions_enableSpeedbFeatures( + JNIEnv*, jobject, jlong jhandle, jlong shared_handle) { + auto shared_opts = + reinterpret_cast(shared_handle); + auto opts = reinterpret_cast(jhandle); + opts->EnableSpeedbFeaturesDB(*shared_opts); +} + /* * Class: org_rocksdb_DBOptions * Method: setEnv @@ -8740,3 +8796,116 @@ void Java_org_rocksdb_FlushOptions_disposeInternal(JNIEnv*, jobject, assert(flush_opt != nullptr); delete flush_opt; } + +///////////////////////////////////////////////////////////////////// +// ROCKSDB_NAMESPACE::SharedOptions + +/* + * Class: org_rocksdb_SharedOptions + * Method: newSharedOptions + * Signature: (JJJJZ)J + */ +JNIEXPORT jlong JNICALL Java_org_rocksdb_SharedOptions_newSharedOptions__JJJJZ( + JNIEnv*, jclass, jlong capacity, jlong total_threads, + jlong delayed_write_rate, jlong bucket_size, jboolean use_merge) { + auto opts = new ROCKSDB_NAMESPACE::SharedOptions( + capacity, total_threads, delayed_write_rate, bucket_size, use_merge); + return GET_CPLUSPLUS_POINTER(opts); +} + +/* + * Class: org_rocksdb_SharedOptions + * Method: newSharedOptions + * Signature: (JJ)J + */ +JNIEXPORT jlong JNICALL Java_org_rocksdb_SharedOptions_newSharedOptions__JJ( + JNIEnv*, jclass, jlong capacity, jlong total_threads) { + auto opts = new ROCKSDB_NAMESPACE::SharedOptions(capacity, total_threads); + return GET_CPLUSPLUS_POINTER(opts); +} + +/* + * Class: org_rocksdb_SharedOptions + * Method: disposeInternal + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_rocksdb_SharedOptions_disposeInternal( + JNIEnv*, jobject, jlong jhandle) { + auto* opts = reinterpret_cast(jhandle); + assert(opts != nullptr); + delete opts; +} + +/* + * Class: org_rocksdb_SharedOptions + * Method: getTotalThreads + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL +Java_org_rocksdb_SharedOptions_getTotalThreads(JNIEnv*, jclass, jlong jhandle) { + auto* opts = reinterpret_cast(jhandle); + assert(opts != nullptr); + return opts->GetTotalThreads(); +} + +/* + * Class: org_rocksdb_SharedOptions + * Method: getTotalRamSizeBytes + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_rocksdb_SharedOptions_getTotalRamSizeBytes( + JNIEnv*, jclass, jlong jhandle) { + auto* opts = reinterpret_cast(jhandle); + assert(opts != nullptr); + return opts->GetTotalRamSizeBytes(); +} + +/* + * Class: org_rocksdb_SharedOptions + * Method: getDelayedWriteRate + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_rocksdb_SharedOptions_getDelayedWriteRate( + JNIEnv*, jclass, jlong jhandle) { + auto* opts = reinterpret_cast(jhandle); + assert(opts != nullptr); + return opts->GetDelayedWriteRate(); +} + +/* + * Class: org_rocksdb_SharedOptions + * Method: getMaxWriteBufferManagerSize + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL +Java_org_rocksdb_SharedOptions_getMaxWriteBufferManagerSize(JNIEnv*, jclass, + jlong jhandle) { + auto* opts = reinterpret_cast(jhandle); + assert(opts != nullptr); + return opts->GetMaxWriteBufferManagerSize(); +} + +/* + * Class: org_rocksdb_SharedOptions + * Method: getBucketSize + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL +Java_org_rocksdb_SharedOptions_getBucketSize(JNIEnv*, jclass, jlong jhandle) { + auto* opts = reinterpret_cast(jhandle); + assert(opts != nullptr); + return opts->GetBucketSize(); +} + +/* + * Class: org_rocksdb_SharedOptions + * Method: isMergeMemtableSupported + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL +Java_org_rocksdb_SharedOptions_isMergeMemtableSupported(JNIEnv*, jclass, + jlong jhandle) { + auto* opts = reinterpret_cast(jhandle); + assert(opts != nullptr); + return opts->IsMergeMemtableSupported(); +} diff --git a/java/src/main/java/org/rocksdb/ColumnFamilyOptions.java b/java/src/main/java/org/rocksdb/ColumnFamilyOptions.java index 8274ebeea8..5d761f9519 100644 --- a/java/src/main/java/org/rocksdb/ColumnFamilyOptions.java +++ b/java/src/main/java/org/rocksdb/ColumnFamilyOptions.java @@ -1,3 +1,17 @@ +// Copyright (C) 2023 Speedb Ltd. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// // Copyright (c) 2011-present, Facebook, Inc. All rights reserved. // This source code is licensed under both the GPLv2 (found in the // COPYING file in the root directory) and Apache 2.0 License @@ -163,6 +177,12 @@ public ColumnFamilyOptions optimizeForPointLookup( return this; } + @Override + public ColumnFamilyOptions enableSpeedbFeatures(final SharedOptions sharedOptions) { + enableSpeedbFeatures(nativeHandle_, sharedOptions.nativeHandle_); + return this; + } + @Override public ColumnFamilyOptions optimizeLevelStyleCompaction() { optimizeLevelStyleCompaction(nativeHandle_, @@ -1345,6 +1365,7 @@ private static native void oldDefaults( private static native void optimizeForSmallDb(final long handle, final long cacheHandle); private native void optimizeForPointLookup(long handle, long blockCacheSizeMb); + private native void enableSpeedbFeatures(final long handle, final long sharedHandle); private native void optimizeLevelStyleCompaction(long handle, long memtableMemoryBudget); private native void optimizeUniversalStyleCompaction(long handle, diff --git a/java/src/main/java/org/rocksdb/ColumnFamilyOptionsInterface.java b/java/src/main/java/org/rocksdb/ColumnFamilyOptionsInterface.java index 4776773bd8..e3fc4304c5 100644 --- a/java/src/main/java/org/rocksdb/ColumnFamilyOptionsInterface.java +++ b/java/src/main/java/org/rocksdb/ColumnFamilyOptionsInterface.java @@ -1,3 +1,17 @@ +// Copyright (C) 2023 Speedb Ltd. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// // Copyright (c) 2011-present, Facebook, Inc. All rights reserved. // This source code is licensed under both the GPLv2 (found in the // COPYING file in the root directory) and Apache 2.0 License @@ -47,6 +61,14 @@ public interface ColumnFamilyOptionsInterfaceDefault values for some parameters in ColumnFamilyOptions are not * optimized for heavy workloads and big datasets, which means you might diff --git a/java/src/main/java/org/rocksdb/DBOptions.java b/java/src/main/java/org/rocksdb/DBOptions.java index 62ad137ee7..51bf6b3abf 100644 --- a/java/src/main/java/org/rocksdb/DBOptions.java +++ b/java/src/main/java/org/rocksdb/DBOptions.java @@ -1,3 +1,17 @@ +// Copyright (C) 2023 Speedb Ltd. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// // Copyright (c) 2011-present, Facebook, Inc. All rights reserved. // This source code is licensed under both the GPLv2 (found in the // COPYING file in the root directory) and Apache 2.0 License @@ -127,6 +141,12 @@ public DBOptions optimizeForSmallDb() { return this; } + @Override + public DBOptions enableSpeedbFeatures(final SharedOptions sharedOptions) { + enableSpeedbFeatures(nativeHandle_, sharedOptions.nativeHandle_); + return this; + } + @Override public DBOptions setIncreaseParallelism( final int totalThreads) { @@ -1259,6 +1279,7 @@ private DBOptions(final long nativeHandle) { @Override protected final native void disposeInternal(final long handle); private native void optimizeForSmallDb(final long handle); + private native void enableSpeedbFeatures(final long handle, final long sharedHandle); private native void setIncreaseParallelism(long handle, int totalThreads); private native void setCreateIfMissing(long handle, boolean flag); private native boolean createIfMissing(long handle); diff --git a/java/src/main/java/org/rocksdb/DBOptionsInterface.java b/java/src/main/java/org/rocksdb/DBOptionsInterface.java index 326da98d23..6e6540d75d 100644 --- a/java/src/main/java/org/rocksdb/DBOptionsInterface.java +++ b/java/src/main/java/org/rocksdb/DBOptionsInterface.java @@ -1,3 +1,17 @@ +// Copyright (C) 2023 Speedb Ltd. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// // Copyright (c) 2011-present, Facebook, Inc. All rights reserved. // This source code is licensed under both the GPLv2 (found in the // COPYING file in the root directory) and Apache 2.0 License @@ -17,6 +31,14 @@ public interface DBOptionsInterface> { */ T optimizeForSmallDb(); + /** + * Use this to set the options for Speedb optimization + * + * @param sharedOptions Options shared between options + * @return the instance of the current object. + */ + T enableSpeedbFeatures(final SharedOptions sharedOptions); + /** * Use the specified object to interact with the environment, * e.g. to read/write files, schedule background work, etc. diff --git a/java/src/main/java/org/rocksdb/Options.java b/java/src/main/java/org/rocksdb/Options.java index d00b489abc..faa144c8e1 100644 --- a/java/src/main/java/org/rocksdb/Options.java +++ b/java/src/main/java/org/rocksdb/Options.java @@ -1,3 +1,17 @@ +// Copyright (C) 2023 Speedb Ltd. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// // Copyright (c) 2011-present, Facebook, Inc. All rights reserved. // This source code is licensed under both the GPLv2 (found in the // COPYING file in the root directory) and Apache 2.0 License @@ -185,6 +199,12 @@ public Options optimizeForPointLookup(final long blockCacheSizeMb) { return this; } + @Override + public Options enableSpeedbFeatures(final SharedOptions sharedOptions) { + enableSpeedbFeatures(nativeHandle_, sharedOptions.nativeHandle_); + return this; + } + @Override public Options optimizeLevelStyleCompaction() { optimizeLevelStyleCompaction(nativeHandle_, @@ -2352,6 +2372,7 @@ private static native void oldDefaults( private static native void optimizeForSmallDb(final long handle, final long cacheHandle); private native void optimizeForPointLookup(long handle, long blockCacheSizeMb); + private native void enableSpeedbFeatures(final long handle, final long sharedHandle); private native void optimizeLevelStyleCompaction(long handle, long memtableMemoryBudget); private native void optimizeUniversalStyleCompaction(long handle, diff --git a/java/src/main/java/org/rocksdb/SharedOptions.java b/java/src/main/java/org/rocksdb/SharedOptions.java new file mode 100644 index 0000000000..062dd36fa4 --- /dev/null +++ b/java/src/main/java/org/rocksdb/SharedOptions.java @@ -0,0 +1,69 @@ +// Copyright (C) 2022 Speedb Ltd. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.rocksdb; + +import java.util.List; + +public class SharedOptions extends RocksObject { + public SharedOptions(final long capacity, final long total_threads) { + super(newSharedOptions(capacity, total_threads)); + } + + public SharedOptions(final long capacity, final long total_threads, final long delayed_write_rate, + final long bucket_size, final boolean use_merge) { + super(newSharedOptions(capacity, total_threads, delayed_write_rate, bucket_size, use_merge)); + } + + public long getMaxWriteBufferManagerSize() { + assert (isOwningHandle()); + return getMaxWriteBufferManagerSize(nativeHandle_); + } + + public long getTotalThreads() { + assert (isOwningHandle()); + return getTotalThreads(nativeHandle_); + } + + public long getTotalRamSizeBytes() { + assert (isOwningHandle()); + return getTotalRamSizeBytes(nativeHandle_); + } + + public long getDelayedWriteRate() { + assert (isOwningHandle()); + return getDelayedWriteRate(nativeHandle_); + } + + public long getBucketSize() { + assert (isOwningHandle()); + return getBucketSize(nativeHandle_); + } + + public boolean isMergeMemtableSupported() { + assert (isOwningHandle()); + return isMergeMemtableSupported(nativeHandle_); + } + + private native static long newSharedOptions(final long capacity, final long total_threads, + final long delayed_write_rate, final long bucket_size, final boolean use_merge); + private native static long newSharedOptions(final long capacity, final long total_threads); + @Override protected final native void disposeInternal(final long handle); + private native static long getMaxWriteBufferManagerSize(final long handle); + private native static long getTotalThreads(final long handle); + private native static long getTotalRamSizeBytes(final long handle); + private native static long getDelayedWriteRate(final long handle); + private native static long getBucketSize(final long handle); + private native static boolean isMergeMemtableSupported(final long handle); +} diff --git a/java/src/test/java/org/rocksdb/SharedOptionsTest.java b/java/src/test/java/org/rocksdb/SharedOptionsTest.java new file mode 100644 index 0000000000..c16fcc9878 --- /dev/null +++ b/java/src/test/java/org/rocksdb/SharedOptionsTest.java @@ -0,0 +1,115 @@ +// Copyright (C) 2022 Speedb Ltd. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.rocksdb; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.util.Arrays; +import java.util.Random; +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +public class SharedOptionsTest { + @ClassRule + public static final RocksNativeLibraryResource ROCKS_NATIVE_LIBRARY_RESOURCE = + new RocksNativeLibraryResource(); + + @Test + public void shortConstructor() { + long totalRamSize = 100 * 1024 * 1024 * 1024; + long totalThreads = 10; + long defaultDelayedWriteRate = 256 * 1024 * 1024; + long defaultDefaultBucketSize = 1000000; + boolean defaultUseMerge = true; + + try (final SharedOptions so = new SharedOptions(totalRamSize, totalThreads)) { + assertThat(so.getMaxWriteBufferManagerSize()).isEqualTo(totalRamSize / 4); + assertThat(so.getTotalThreads()).isEqualTo(totalThreads); + assertThat(so.getTotalRamSizeBytes()).isEqualTo(totalRamSize); + assertThat(so.getDelayedWriteRate()).isEqualTo(defaultDelayedWriteRate); + assertThat(so.getBucketSize()).isEqualTo(defaultDefaultBucketSize); + assertThat(so.isMergeMemtableSupported()).isEqualTo(defaultUseMerge); + } + } + + @Test + public void altConstructor() { + long totalRamSize = 100 * 1024 * 1024 * 1024; + long totalThreads = 10; + long delayedWriteRate = 512 * 1024 * 1024; + long bucketSize = 50000; + boolean use_merge = false; + + try (final SharedOptions so = new SharedOptions( + totalRamSize, totalThreads, delayedWriteRate, bucketSize, use_merge)) { + assertThat(so.getMaxWriteBufferManagerSize()).isEqualTo(totalRamSize / 4); + assertThat(so.getTotalThreads()).isEqualTo(totalThreads); + assertThat(so.getTotalRamSizeBytes()).isEqualTo(totalRamSize); + assertThat(so.getDelayedWriteRate()).isEqualTo(delayedWriteRate); + assertThat(so.getBucketSize()).isEqualTo(bucketSize); + assertThat(so.isMergeMemtableSupported()).isEqualTo(use_merge); + } + } + + @Test + public void enableSpeedb() { + try (final Options opts = new Options()) { + long totalRamSize = 100 * 1024 * 1024 * 1024; + long totalThreads = 10; + long defaultDelayedWriteRate = 256 * 1024 * 1024; + + try (final SharedOptions so = new SharedOptions(totalRamSize, totalThreads)) { + assertThat(opts.maxBackgroundJobs()).isEqualTo((int) 2); + opts.setDelayedWriteRate(1000); + assertThat(opts.delayedWriteRate()).isEqualTo((long) 1000); + opts.setBytesPerSync(1 << 10); + assertThat(opts.bytesPerSync()).isEqualTo((long) (1 << 10)); + opts.setMaxWriteBufferNumber(3); + assertThat(opts.maxWriteBufferNumber()).isEqualTo(3); + opts.setMinWriteBufferNumberToMerge(10); + assertThat(opts.minWriteBufferNumberToMerge()).isEqualTo(10); + assertThat(opts.writeBufferManager()).isNull(); + + opts.enableSpeedbFeatures(so); + assertThat(opts.maxBackgroundJobs()).isEqualTo((int) totalThreads); + assertThat(opts.delayedWriteRate()).isEqualTo((long) defaultDelayedWriteRate); + assertThat(opts.bytesPerSync()).isEqualTo((long) (1 << 20)); + assertThat(opts.maxWriteBufferNumber()).isEqualTo(4); + assertThat(opts.minWriteBufferNumberToMerge()).isEqualTo(1); + // The following assertion will fail since the JNI assumes a WBM is created + // via a call to setWriteBufferManager() which caches the newly created + // WBM in a member. The accessor just returns that member rather than actually + // obtaining the WBM from the CPP side. + // assertThat(opts.writeBufferManager()).isNotNull(); + + DBOptions db_opts = new DBOptions(); + db_opts.setDelayedWriteRate(1000); + assertThat(db_opts.delayedWriteRate()).isEqualTo((long) 1000); + + db_opts.enableSpeedbFeatures(so); + assertThat(db_opts.delayedWriteRate()).isEqualTo((long) defaultDelayedWriteRate); + + ColumnFamilyOptions cfo = new ColumnFamilyOptions(); + cfo.setMaxWriteBufferNumber(3); + assertThat(cfo.maxWriteBufferNumber()).isEqualTo(3); + + cfo.enableSpeedbFeatures(so); + assertThat(cfo.maxWriteBufferNumber()).isEqualTo(4); + } + } + } +}