support bottom level compression (#3634)

* support bottom level compression * address liuyu85cn's comments
vesoft-inc · Jan 5, 2022 · 40adfab · 40adfab
1 parent b3da209
commit 40adfab
Show file tree

Hide file tree

Showing 3 changed files with 38 additions and 2 deletions.
diff --git a/src/kvstore/RocksEngineConfig.cpp b/src/kvstore/RocksEngineConfig.cpp
@@ -59,7 +59,13 @@ DEFINE_bool(enable_partitioned_index_filter, false, "True for partitioned index
 DEFINE_string(rocksdb_compression,
  "snappy",
  "Compression algorithm used by RocksDB, "
- "options: no,snappy,lz4,lz4hc,zstd,zlib,bzip2");
+ "options: no, snappy, lz4, lz4hc, zstd, zlib, bzip2, xpress");
+
+DEFINE_string(rocksdb_bottommost_compression,
+ "disable",
+ "Specify the bottommost level compression algorithm"
+ "options: no, snappy, lz4, lz4hc, zstd, zlib, bzip2, xpress, disable");
+
 DEFINE_string(rocksdb_compression_per_level,
  "",
  "Specify per level compression algorithm, "
@@ -142,7 +148,9 @@ static const std::unordered_map<std::string, rocksdb::CompressionType> kCompress
  {"lz4hc", rocksdb::kLZ4HCCompression},
  {"zstd", rocksdb::kZSTD},
  {"zlib", rocksdb::kZlibCompression},
- {"bzip2", rocksdb::kBZip2Compression}};
+ {"bzip2", rocksdb::kBZip2Compression},
+ {"xpress", rocksdb::kXpressCompression},
+ {"disable", rocksdb::kDisableCompressionOption}};
 
 static rocksdb::Status initRocksdbCompression(rocksdb::Options& baseOpts) {
  // Set the general compression algorithm
@@ -153,6 +161,13 @@ static rocksdb::Status initRocksdbCompression(rocksdb::Options& baseOpts) {
  return rocksdb::Status::InvalidArgument();
  }
  baseOpts.compression = it->second;
+
+ it = kCompressionTypeMap.find(FLAGS_rocksdb_bottommost_compression);
+ if (it == kCompressionTypeMap.end()) {
+ LOG(ERROR) << "Unsupported compression type: " << FLAGS_rocksdb_bottommost_compression;
+ return rocksdb::Status::InvalidArgument();
+ }
+ baseOpts.bottommost_compression = it->second;
  }
  if (FLAGS_rocksdb_compression_per_level.empty()) {
  return rocksdb::Status::OK();

diff --git a/src/kvstore/RocksEngineConfig.h b/src/kvstore/RocksEngineConfig.h
@@ -47,6 +47,7 @@ DECLARE_string(part_man_type);
 
 DECLARE_string(rocksdb_compression_per_level);
 DECLARE_string(rocksdb_compression);
+DECLARE_string(rocksdb_bottommost_compression);
 
 DECLARE_bool(enable_rocksdb_statistics);
 DECLARE_string(rocksdb_stats_level);

diff --git a/src/kvstore/test/RocksEngineConfigTest.cpp b/src/kvstore/test/RocksEngineConfigTest.cpp
@@ -110,6 +110,26 @@ TEST(RocksEngineConfigTest, CompressionConfigTest) {
  ASSERT_EQ(rocksdb::Status::kInvalidArgument, status.code());
  }
 
+ {
+ FLAGS_rocksdb_compression = "lz4";
+ FLAGS_rocksdb_compression_per_level = "";
+ FLAGS_rocksdb_bottommost_compression = "zstd";
+ rocksdb::Options options;
+ auto status = initRocksdbOptions(options, 1);
+ ASSERT_TRUE(status.ok()) << status.ToString();
+ ASSERT_EQ(rocksdb::kLZ4Compression, options.compression);
+ ASSERT_EQ(rocksdb::kZSTD, options.bottommost_compression);
+
+ rocksdb::DB* db = nullptr;
+ SCOPE_EXIT {
+ delete db;
+ };
+ options.create_if_missing = true;
+ fs::TempDir rootPath("/tmp/RocksDBCompressionConfigTest.XXXXXX");
+ status = rocksdb::DB::Open(options, rootPath.path(), &db);
+ ASSERT_TRUE(status.ok()) << status.ToString();
+ }
+
  {
  FLAGS_rocksdb_compression = "lz4";
  FLAGS_rocksdb_compression_per_level = "no:no::snappy::zstd";