Upgrade rocksdb library (#8532)

* upgrade rocksdb to latest

* removed deprecated statistics (removed in 8.0.0 release)

* tweak some defaults from RocksDb

* turn compression on

* add jemalloc detection

Signed-off-by: Gabriel Fukushima <gabrielfukushima@gmail.com>

---------

Signed-off-by: Gabriel Fukushima <gabrielfukushima@gmail.com>

gradle/versions.gradle

@@ -113,6 +113,8 @@ dependencyManagement {
       entry 'junit-jupiter-params'
     }
 
+    dependency 'net.java.dev.jna:jna:5.14.0'
+
     dependencySet(group: 'org.mockito', version: '5.13.0') {
       entry 'mockito-core'
       entry 'mockito-junit-jupiter'
@@ -123,7 +125,7 @@ dependencyManagement {
       entry 'jmh-generator-annprocess'
     }
     dependency 'org.quartz-scheduler:quartz:2.3.2'
-    dependency 'org.rocksdb:rocksdbjni:7.7.3'
+    dependency 'org.rocksdb:rocksdbjni:9.5.2'
     dependency 'org.fusesource.leveldbjni:leveldbjni-win64:1.8'
     dependency 'org.fusesource.leveldbjni:leveldbjni-win32:1.8'
     dependency 'tech.pegasys:leveldb-native:0.3.1'

infrastructure/io/build.gradle

@@ -1,5 +1,6 @@
 dependencies {
     implementation 'org.apache.tuweni:tuweni-bytes'
+    implementation 'net.java.dev.jna:jna'
     implementation project(':infrastructure:exceptions')
 
     testFixturesImplementation 'commons-io:commons-io'

infrastructure/io/src/main/java/tech/pegasys/teku/infrastructure/io/JemallocDetector.java

@@ -0,0 +1,57 @@
+/*
+ * Copyright Consensys Software Inc., 2024
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+
+package tech.pegasys.teku.infrastructure.io;
+
+import com.sun.jna.Library;
+import com.sun.jna.Native;
+import com.sun.jna.ptr.IntByReference;
+import com.sun.jna.ptr.PointerByReference;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+@SuppressWarnings("JavaCase")
+public class JemallocDetector {
+  private static final Logger LOG = LogManager.getLogger();
+  private static String _jemalloc;
+
+  public static void logJemallocPresence() {
+    try {
+      getJemalloc();
+      if (_jemalloc != null) {
+        LOG.info("jemalloc: " + _jemalloc);
+      }
+    } catch (final Throwable throwable) {
+      LOG.info("jemalloc library not found.");
+    }
+  }
+
+  private static void getJemalloc() {
+    interface JemallocLib extends Library {
+      int mallctl(
+          String property,
+          PointerByReference value,
+          IntByReference len,
+          String newValue,
+          int newLen);
+    }
+
+    final JemallocLib jemallocLib = Native.load("jemalloc", JemallocLib.class);
+
+    PointerByReference pVersion = new PointerByReference();
+    IntByReference pSize = new IntByReference(Native.POINTER_SIZE);
+    jemallocLib.mallctl("version", pVersion, pSize, null, 0);
+
+    _jemalloc = pVersion.getValue().getString(0);
+  }
+}

storage/src/main/java/tech/pegasys/teku/storage/server/kvstore/KvStoreConfiguration.java

@@ -41,20 +41,28 @@
 @SuppressWarnings("FieldCanBeFinal")
 @JsonIgnoreProperties(ignoreUnknown = true)
 public class KvStoreConfiguration {
-  public static final int DEFAULT_MAX_OPEN_FILES = 128;
+  public static final int DEFAULT_MAX_OPEN_FILES = 1024;
 
   public static final int DEFAULT_LEVELDB_BLOCK_SIZE = 4096;
 
-  public static final int DEFAULT_LEVELDB_WRITE_BUFFER_SIZE = 4194304;
+  public static final int DEFAULT_LEVELDB_WRITE_BUFFER_SIZE = 4_194_304;
 
   public static final int DEFAULT_LEVELDB_MAX_OPEN_FILES = 1000;
 
   public static final int DEFAULT_MAX_BACKGROUND_JOBS = 6;
   public static final int DEFAULT_BACKGROUND_THREAD_COUNT = 6;
-  public static final long DEFAULT_CACHE_CAPACITY = 8 << 20;
+  public static final long DEFAULT_CACHE_CAPACITY = 8 << 20; // 8MB
   public static final long DEFAULT_WRITE_BUFFER_CAPACITY = 128 << 20;
   private static final boolean DEFAULT_OPTIMISE_FOR_SMALL_DB = false;
 
+  /** RocksDb number of log files to keep on disk */
+  public static final long NUMBER_OF_LOG_FILES_TO_KEEP = 5;
+
+  /** RocksDb Time to roll a log file (1 day = 3600 * 24 seconds) */
+  public static final long TIME_TO_ROLL_LOG_FILE = 86_400L;
+
+  public static final long ROCKSDB_BLOCK_SIZE = 32_768;
+
   /* --------------- Safe to Change Properties ------------ */
 
   @JsonProperty(value = "maxOpenFiles", access = Access.WRITE_ONLY)
@@ -88,7 +96,7 @@ public class KvStoreConfiguration {
   /* ---------------     Fixed Properties     ------------ */
 
   @JsonProperty("compressionType")
-  private CompressionType compressionType = CompressionType.NO_COMPRESSION;
+  private CompressionType compressionType = CompressionType.LZ4_COMPRESSION;
 
   @JsonProperty("bottomMostCompressionType")
   private CompressionType bottomMostCompressionType = CompressionType.NO_COMPRESSION;

storage/src/main/java/tech/pegasys/teku/storage/server/rocksdb/RocksDbInstanceFactory.java

@@ -14,6 +14,9 @@
 package tech.pegasys.teku.storage.server.rocksdb;
 
 import static com.google.common.base.Preconditions.checkArgument;
+import static tech.pegasys.teku.storage.server.kvstore.KvStoreConfiguration.NUMBER_OF_LOG_FILES_TO_KEEP;
+import static tech.pegasys.teku.storage.server.kvstore.KvStoreConfiguration.ROCKSDB_BLOCK_SIZE;
+import static tech.pegasys.teku.storage.server.kvstore.KvStoreConfiguration.TIME_TO_ROLL_LOG_FILE;
 
 import com.google.common.collect.ImmutableMap;
 import java.util.ArrayList;
@@ -28,6 +31,7 @@
 import org.hyperledger.besu.plugin.services.MetricsSystem;
 import org.hyperledger.besu.plugin.services.metrics.MetricCategory;
 import org.rocksdb.BlockBasedTableConfig;
+import org.rocksdb.BloomFilter;
 import org.rocksdb.Cache;
 import org.rocksdb.ColumnFamilyDescriptor;
 import org.rocksdb.ColumnFamilyHandle;
@@ -137,18 +141,23 @@ private static DBOptions createDBOptions(
     final DBOptions options =
         new DBOptions()
             .setCreateIfMissing(true)
-            .setBytesPerSync(1048576L)
-            .setWalBytesPerSync(1048576L)
             .setIncreaseParallelism(Runtime.getRuntime().availableProcessors())
             .setMaxBackgroundJobs(configuration.getMaxBackgroundJobs())
             .setDbWriteBufferSize(configuration.getWriteBufferCapacity())
             .setMaxOpenFiles(configuration.getMaxOpenFiles())
+            .setBytesPerSync(1_048_576L) // 1MB
+            .setWalBytesPerSync(1_048_576L)
             .setCreateMissingColumnFamilies(true)
+            .setLogFileTimeToRoll(TIME_TO_ROLL_LOG_FILE)
+            .setKeepLogFileNum(NUMBER_OF_LOG_FILES_TO_KEEP)
             .setEnv(Env.getDefault().setBackgroundThreads(configuration.getBackgroundThreadCount()))
             .setStatistics(stats);
+
+    // Java docs suggests this if db is under 1GB, nearly impossible atm
     if (configuration.optimizeForSmallDb()) {
       options.optimizeForSmallDb();
     }
+
     return options;
   }
 
@@ -157,6 +166,7 @@ private static ColumnFamilyOptions createColumnFamilyOptions(
     return new ColumnFamilyOptions()
         .setCompressionType(configuration.getCompressionType())
         .setBottommostCompressionType(configuration.getBottomMostCompressionType())
+        .setTtl(0)
         .setTableFormatConfig(createBlockBasedTableConfig(cache));
   }
 
@@ -175,8 +185,11 @@ private static List<ColumnFamilyDescriptor> createColumnFamilyDescriptors(
 
   private static BlockBasedTableConfig createBlockBasedTableConfig(final Cache cache) {
     return new BlockBasedTableConfig()
+        .setFormatVersion(5)
         .setBlockCache(cache)
+        .setFilterPolicy(new BloomFilter(10, false))
+        .setPartitionFilters(true)
         .setCacheIndexAndFilterBlocks(true)
-        .setFormatVersion(4); // Use the latest format version (only applies to new tables)
+        .setBlockSize(ROCKSDB_BLOCK_SIZE);
   }
 }

storage/src/main/java/tech/pegasys/teku/storage/server/rocksdb/RocksDbStats.java

@@ -52,12 +52,10 @@ public class RocksDbStats implements AutoCloseable {
     TickerType.BLOCK_CACHE_INDEX_HIT,
     TickerType.BLOCK_CACHE_INDEX_ADD,
     TickerType.BLOCK_CACHE_INDEX_BYTES_INSERT,
-    TickerType.BLOCK_CACHE_INDEX_BYTES_EVICT,
     TickerType.BLOCK_CACHE_FILTER_MISS,
     TickerType.BLOCK_CACHE_FILTER_HIT,
     TickerType.BLOCK_CACHE_FILTER_ADD,
     TickerType.BLOCK_CACHE_FILTER_BYTES_INSERT,
-    TickerType.BLOCK_CACHE_FILTER_BYTES_EVICT,
     TickerType.BLOCK_CACHE_DATA_MISS,
     TickerType.BLOCK_CACHE_DATA_HIT,
     TickerType.BLOCK_CACHE_DATA_ADD,
@@ -91,34 +89,25 @@ public class RocksDbStats implements AutoCloseable {
     TickerType.NUMBER_DB_NEXT_FOUND,
     TickerType.NUMBER_DB_PREV_FOUND,
     TickerType.ITER_BYTES_READ,
-    TickerType.NO_FILE_CLOSES,
     TickerType.NO_FILE_OPENS,
     TickerType.NO_FILE_ERRORS,
     // TickerType.STALL_L0_SLOWDOWN_MICROS,
     // TickerType.STALL_MEMTABLE_COMPACTION_MICROS,
     // TickerType.STALL_L0_NUM_FILES_MICROS,
     TickerType.STALL_MICROS,
     TickerType.DB_MUTEX_WAIT_MICROS,
-    TickerType.RATE_LIMIT_DELAY_MILLIS,
-    TickerType.NO_ITERATORS,
     TickerType.NUMBER_MULTIGET_BYTES_READ,
     TickerType.NUMBER_MULTIGET_KEYS_READ,
     TickerType.NUMBER_MULTIGET_CALLS,
-    TickerType.NUMBER_FILTERED_DELETES,
     TickerType.NUMBER_MERGE_FAILURES,
     TickerType.BLOOM_FILTER_PREFIX_CHECKED,
     TickerType.BLOOM_FILTER_PREFIX_USEFUL,
     TickerType.NUMBER_OF_RESEEKS_IN_ITERATION,
     TickerType.GET_UPDATES_SINCE_CALLS,
-    TickerType.BLOCK_CACHE_COMPRESSED_MISS,
-    TickerType.BLOCK_CACHE_COMPRESSED_HIT,
-    TickerType.BLOCK_CACHE_COMPRESSED_ADD,
-    TickerType.BLOCK_CACHE_COMPRESSED_ADD_FAILURES,
     TickerType.WAL_FILE_SYNCED,
     TickerType.WAL_FILE_BYTES,
     TickerType.WRITE_DONE_BY_SELF,
     TickerType.WRITE_DONE_BY_OTHER,
-    TickerType.WRITE_TIMEDOUT,
     TickerType.WRITE_WITH_WAL,
     TickerType.COMPACT_READ_BYTES,
     TickerType.COMPACT_WRITE_BYTES,
@@ -129,7 +118,6 @@ public class RocksDbStats implements AutoCloseable {
     TickerType.NUMBER_SUPERVERSION_CLEANUPS,
     TickerType.NUMBER_BLOCK_COMPRESSED,
     TickerType.NUMBER_BLOCK_DECOMPRESSED,
-    TickerType.NUMBER_BLOCK_NOT_COMPRESSED,
     TickerType.MERGE_OPERATION_TOTAL_TIME,
     TickerType.FILTER_OPERATION_TOTAL_TIME,
     TickerType.ROW_CACHE_HIT,
@@ -156,11 +144,6 @@ public class RocksDbStats implements AutoCloseable {
     HistogramType.READ_BLOCK_COMPACTION_MICROS,
     HistogramType.READ_BLOCK_GET_MICROS,
     HistogramType.WRITE_RAW_BLOCK_MICROS,
-    HistogramType.STALL_L0_SLOWDOWN_COUNT,
-    HistogramType.STALL_MEMTABLE_COMPACTION_COUNT,
-    HistogramType.STALL_L0_NUM_FILES_COUNT,
-    HistogramType.HARD_RATE_LIMIT_DELAY_COUNT,
-    HistogramType.SOFT_RATE_LIMIT_DELAY_COUNT,
     HistogramType.NUM_FILES_IN_SINGLE_COMPACTION,
     HistogramType.DB_SEEK,
     HistogramType.WRITE_STALL,
@@ -169,8 +152,6 @@ public class RocksDbStats implements AutoCloseable {
     HistogramType.BYTES_PER_READ,
     HistogramType.BYTES_PER_WRITE,
     HistogramType.BYTES_PER_MULTIGET,
-    HistogramType.BYTES_COMPRESSED,
-    HistogramType.BYTES_DECOMPRESSED,
     HistogramType.COMPRESSION_TIMES_NANOS,
     HistogramType.DECOMPRESSION_TIMES_NANOS,
     HistogramType.READ_NUM_MERGE_OPERANDS,

teku/src/main/java/tech/pegasys/teku/Teku.java

@@ -23,6 +23,7 @@
 import tech.pegasys.teku.cli.BeaconNodeCommand;
 import tech.pegasys.teku.cli.BeaconNodeCommand.StartAction;
 import tech.pegasys.teku.config.TekuConfiguration;
+import tech.pegasys.teku.infrastructure.io.JemallocDetector;
 import tech.pegasys.teku.infrastructure.logging.LoggingConfigurator;
 
 public final class Teku {
@@ -74,6 +75,7 @@ private static Node start(final TekuConfiguration config, final boolean validato
     if (BlstLoader.INSTANCE.isEmpty()) {
       throw new UnsupportedOperationException("BLS native library unavailable for this platform");
     }
+    JemallocDetector.logJemallocPresence();
 
     node.start();