Database metadata refactor

CHANGELOG.md

@@ -1,5 +1,17 @@
 # Changelog
 
+## 24.2.1-SNAPSHOT
+
+### Breaking Changes
+- RocksDB database metadata format has changed to be more expressive, the migration of an existing metadata file to the new format is automatic at startup. Before performing a downgrade to a previous version it is mandatory to revert to the original format using the subcommand `besu --data-path=/path/to/besu/datadir storage revert-metadata v2-to-v1`.
+
+### Deprecations
+
+### Additions and Improvements
+- RocksDB database metadata refactoring [#6555](https://github.com/hyperledger/besu/pull/6555)
+
+### Bug fixes
+
 ## 24.2.0-SNAPSHOT
 
 ### Breaking Changes

besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java

@@ -138,7 +138,6 @@
 import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueStorageProvider;
 import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueStorageProviderBuilder;
 import org.hyperledger.besu.ethereum.trie.forest.pruner.PrunerConfiguration;
-import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat;
 import org.hyperledger.besu.evm.precompile.AbstractAltBnPrecompiledContract;
 import org.hyperledger.besu.evm.precompile.BigIntegerModularExponentiationPrecompiledContract;
 import org.hyperledger.besu.evm.precompile.KZGPointEvalPrecompiledContract;
@@ -169,6 +168,7 @@
 import org.hyperledger.besu.plugin.services.metrics.MetricCategory;
 import org.hyperledger.besu.plugin.services.metrics.MetricCategoryRegistry;
 import org.hyperledger.besu.plugin.services.securitymodule.SecurityModule;
+import org.hyperledger.besu.plugin.services.storage.DataStorageFormat;
 import org.hyperledger.besu.plugin.services.storage.PrivacyKeyValueStorageFactory;
 import org.hyperledger.besu.plugin.services.storage.rocksdb.RocksDBPlugin;
 import org.hyperledger.besu.plugin.services.txselection.PluginTransactionSelectorFactory;
@@ -2551,8 +2551,8 @@ public Path getDataPath() {
     }
 
     @Override
-    public int getDatabaseVersion() {
-      return dataStorageOptions.toDomainObject().getDataStorageFormat().getDatabaseVersion();
+    public DataStorageFormat getDatabaseFormat() {
+      return dataStorageOptions.toDomainObject().getDataStorageFormat();
     }
   }
 

besu/src/main/java/org/hyperledger/besu/cli/options/stable/DataStorageOptions.java

@@ -25,8 +25,8 @@
 import org.hyperledger.besu.cli.options.CLIOptions;
 import org.hyperledger.besu.cli.util.CommandLineUtils;
 import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration;
-import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat;
 import org.hyperledger.besu.ethereum.worldstate.ImmutableDataStorageConfiguration;
+import org.hyperledger.besu.plugin.services.storage.DataStorageFormat;
 
 import java.util.List;
 

besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/RevertMetadataSubCommand.java

@@ -0,0 +1,135 @@
+/*
+ * Copyright Hyperledger Besu Contributors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+package org.hyperledger.besu.cli.subcommands.storage;
+
+import org.hyperledger.besu.cli.util.VersionProvider;
+import org.hyperledger.besu.plugin.services.storage.DataStorageFormat;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Path;
+import java.util.OptionalInt;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.SerializationFeature;
+import com.fasterxml.jackson.databind.annotation.JsonSerialize;
+import com.fasterxml.jackson.datatype.jdk8.Jdk8Module;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import picocli.CommandLine;
+import picocli.CommandLine.Command;
+import picocli.CommandLine.ParentCommand;
+
+/** The revert metadata to v1 subcommand. */
+@Command(
+    name = "revert-metadata",
+    description = "Revert database metadata to previous format",
+    mixinStandardHelpOptions = true,
+    versionProvider = VersionProvider.class,
+    subcommands = RevertMetadataSubCommand.v2ToV1.class)
+public class RevertMetadataSubCommand implements Runnable {
+  private static final Logger LOG = LoggerFactory.getLogger(RevertMetadataSubCommand.class);
+  private static final String METADATA_FILENAME = "DATABASE_METADATA.json";
+  private static final ObjectMapper MAPPER =
+      new ObjectMapper()
+          .registerModule(new Jdk8Module())
+          .setSerializationInclusion(JsonInclude.Include.NON_ABSENT)
+          .enable(SerializationFeature.INDENT_OUTPUT);
+
+  @SuppressWarnings("unused")
+  @ParentCommand
+  private StorageSubCommand parentCommand;
+
+  @SuppressWarnings("unused")
+  @CommandLine.Spec
+  private CommandLine.Model.CommandSpec spec;
+
+  @Override
+  public void run() {
+    spec.commandLine().usage(System.out);
+  }
+
+  @Command(
+      name = "v2-to-v1",
+      description = "Revert a database metadata v2 format to v1 format",
+      mixinStandardHelpOptions = true,
+      versionProvider = VersionProvider.class)
+  static class v2ToV1 implements Runnable {
+
+    @SuppressWarnings("unused")
+    @CommandLine.Spec
+    private CommandLine.Model.CommandSpec spec;
+
+    @SuppressWarnings("unused")
+    @ParentCommand
+    private RevertMetadataSubCommand parentCommand;
+
+    @Override
+    public void run() {
+
+      final Path dataDir = parentCommand.parentCommand.besuCommand.dataDir();
+
+      final File dbMetadata = dataDir.resolve(METADATA_FILENAME).toFile();
+      if (!dbMetadata.exists()) {
+        String errMsg =
+            String.format(
+                "Could not find database metadata file %s, check your data dir %s",
+                dbMetadata, dataDir);
+        LOG.error(errMsg);
+        throw new IllegalArgumentException(errMsg);
+      }
+      try {
+        final var root = MAPPER.readTree(dbMetadata);
+        if (!root.has("v2")) {
+          String errMsg =
+              String.format("Database metadata file %s is not in v2 format", dbMetadata);
+          LOG.error(errMsg);
+          throw new IllegalArgumentException(errMsg);
+        }
+
+        final var v2Obj = root.get("v2");
+        if (!v2Obj.has("format")) {
+          String errMsg =
+              String.format(
+                  "Database metadata file %s is malformed, \"format\" field not found", dbMetadata);
+          LOG.error(errMsg);
+          throw new IllegalArgumentException(errMsg);
+        }
+
+        final var formatField = v2Obj.get("format").asText();
+        final OptionalInt maybePrivacyVersion =
+            v2Obj.has("privacyVersion")
+                ? OptionalInt.of(v2Obj.get("privacyVersion").asInt())
+                : OptionalInt.empty();
+
+        final DataStorageFormat dataStorageFormat = DataStorageFormat.valueOf(formatField);
+        final int v1Version =
+            switch (dataStorageFormat) {
+              case FOREST -> 1;
+              case BONSAI -> 2;
+            };
+
+        @JsonSerialize
+        record V1(int version, OptionalInt privacyVersion) {}
+
+        MAPPER.writeValue(dbMetadata, new V1(v1Version, maybePrivacyVersion));
+        LOG.info("Successfully reverted database metadata from v2 to v1 in {}", dbMetadata);
+      } catch (IOException ioe) {
+        throw new RuntimeException(ioe);
+      }
+    }
+  }
+}

besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/StorageSubCommand.java

@@ -48,7 +48,8 @@
     subcommands = {
       StorageSubCommand.RevertVariablesStorage.class,
       RocksDbSubCommand.class,
-      TrieLogSubCommand.class
+      TrieLogSubCommand.class,
+      RevertMetadataSubCommand.class
     })
 public class StorageSubCommand implements Runnable {
 

besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommand.java

@@ -28,7 +28,7 @@
 import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage;
 import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogPruner;
 import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration;
-import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat;
+import org.hyperledger.besu.plugin.services.storage.DataStorageFormat;
 
 import java.io.IOException;
 import java.io.PrintWriter;

besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java

@@ -90,14 +90,14 @@
 import org.hyperledger.besu.ethereum.trie.forest.pruner.Pruner;
 import org.hyperledger.besu.ethereum.trie.forest.pruner.PrunerConfiguration;
 import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration;
-import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat;
 import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive;
 import org.hyperledger.besu.ethereum.worldstate.WorldStatePreimageStorage;
 import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage;
 import org.hyperledger.besu.evm.internal.EvmConfiguration;
 import org.hyperledger.besu.metrics.ObservableMetricsSystem;
 import org.hyperledger.besu.plugin.services.MetricsSystem;
 import org.hyperledger.besu.plugin.services.permissioning.NodeMessagePermissioningProvider;
+import org.hyperledger.besu.plugin.services.storage.DataStorageFormat;
 import org.hyperledger.besu.plugin.services.txselection.PluginTransactionSelectorFactory;
 import org.hyperledger.besu.plugin.services.txvalidator.PluginTransactionValidatorFactory;
 

besu/src/test/java/org/hyperledger/besu/RunnerTest.java

@@ -164,7 +164,7 @@ private void syncFromGenesis(final SyncMode mode, final GenesisConfigFile genesi
     final Path dataDirAhead = Files.createTempDirectory(temp, "db-ahead");
     final Path dbAhead = dataDirAhead.resolve("database");
     final int blockCount = 500;
-    final NodeKey aheadDbNodeKey = NodeKeyUtils.createFrom(KeyPairUtil.loadKeyPair(dbAhead));
+    final NodeKey aheadDbNodeKey = NodeKeyUtils.createFrom(KeyPairUtil.loadKeyPair(dataDirAhead));
     final NodeKey behindDbNodeKey = NodeKeyUtils.generate();
     final SynchronizerConfiguration syncConfigAhead =
         SynchronizerConfiguration.builder().syncMode(SyncMode.FULL).build();

besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java

@@ -32,8 +32,8 @@
 import static org.hyperledger.besu.ethereum.p2p.config.DefaultDiscoveryConfiguration.GOERLI_DISCOVERY_URL;
 import static org.hyperledger.besu.ethereum.p2p.config.DefaultDiscoveryConfiguration.MAINNET_BOOTSTRAP_NODES;
 import static org.hyperledger.besu.ethereum.p2p.config.DefaultDiscoveryConfiguration.MAINNET_DISCOVERY_URL;
-import static org.hyperledger.besu.ethereum.worldstate.DataStorageFormat.BONSAI;
 import static org.hyperledger.besu.nat.kubernetes.KubernetesNatManager.DEFAULT_BESU_SERVICE_NAME_FILTER;
+import static org.hyperledger.besu.plugin.services.storage.DataStorageFormat.BONSAI;
 import static org.junit.Assume.assumeThat;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.contains;

besu/src/test/java/org/hyperledger/besu/cli/options/stable/DataStorageOptionsTest.java

@@ -20,8 +20,8 @@
 
 import org.hyperledger.besu.cli.options.AbstractCLIOptionsTest;
 import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration;
-import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat;
 import org.hyperledger.besu.ethereum.worldstate.ImmutableDataStorageConfiguration;
+import org.hyperledger.besu.plugin.services.storage.DataStorageFormat;
 
 import org.junit.jupiter.api.Test;
 

besu/src/test/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelperTest.java

@@ -19,7 +19,7 @@
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.assertj.core.api.Assertions.assertThatThrownBy;
 import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.Unstable.DEFAULT_BONSAI_TRIE_LOG_PRUNING_WINDOW_SIZE;
-import static org.hyperledger.besu.ethereum.worldstate.DataStorageFormat.BONSAI;
+import static org.hyperledger.besu.plugin.services.storage.DataStorageFormat.BONSAI;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.Mockito.spy;
 import static org.mockito.Mockito.when;

besu/src/test/java/org/hyperledger/besu/controller/BesuControllerBuilderTest.java

@@ -48,14 +48,14 @@
 import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldState;
 import org.hyperledger.besu.ethereum.trie.forest.pruner.PrunerConfiguration;
 import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration;
-import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat;
 import org.hyperledger.besu.ethereum.worldstate.ImmutableDataStorageConfiguration;
 import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive;
 import org.hyperledger.besu.ethereum.worldstate.WorldStatePreimageStorage;
 import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage;
 import org.hyperledger.besu.evm.internal.EvmConfiguration;
 import org.hyperledger.besu.metrics.ObservableMetricsSystem;
 import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem;
+import org.hyperledger.besu.plugin.services.storage.DataStorageFormat;
 import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage;
 
 import java.math.BigInteger;

besu/src/test/java/org/hyperledger/besu/services/TraceServiceImplTest.java

@@ -31,7 +31,6 @@
 import org.hyperledger.besu.ethereum.chain.MutableBlockchain;
 import org.hyperledger.besu.ethereum.core.Block;
 import org.hyperledger.besu.ethereum.core.BlockchainSetupUtil;
-import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat;
 import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive;
 import org.hyperledger.besu.evm.log.Log;
 import org.hyperledger.besu.evm.worldstate.WorldView;
@@ -40,6 +39,7 @@
 import org.hyperledger.besu.plugin.data.BlockTraceResult;
 import org.hyperledger.besu.plugin.data.TransactionTraceResult;
 import org.hyperledger.besu.plugin.services.TraceService;
+import org.hyperledger.besu.plugin.services.storage.DataStorageFormat;
 import org.hyperledger.besu.plugin.services.tracer.BlockAwareOperationTracer;
 
 import java.util.HashSet;

ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/graphql/AbstractEthGraphQLHttpServiceTest.java

@@ -34,8 +34,8 @@
 import org.hyperledger.besu.ethereum.mainnet.ValidationResult;
 import org.hyperledger.besu.ethereum.p2p.rlpx.wire.Capability;
 import org.hyperledger.besu.ethereum.transaction.TransactionInvalidReason;
-import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat;
 import org.hyperledger.besu.plugin.data.SyncStatus;
+import org.hyperledger.besu.plugin.services.storage.DataStorageFormat;
 
 import java.nio.file.Path;
 import java.util.Collections;

ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/AbstractJsonRpcHttpServiceTest.java

@@ -45,10 +45,10 @@
 import org.hyperledger.besu.ethereum.p2p.network.P2PNetwork;
 import org.hyperledger.besu.ethereum.p2p.rlpx.wire.Capability;
 import org.hyperledger.besu.ethereum.transaction.TransactionInvalidReason;
-import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat;
 import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem;
 import org.hyperledger.besu.metrics.prometheus.MetricsConfiguration;
 import org.hyperledger.besu.nat.NatService;
+import org.hyperledger.besu.plugin.services.storage.DataStorageFormat;
 import org.hyperledger.besu.testutil.BlockTestUtil.ChainResources;
 
 import java.math.BigInteger;

ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/bonsai/DebugTraceJsonRpcHttpBySpecTest.java

@@ -16,7 +16,7 @@
 
 import org.hyperledger.besu.ethereum.api.jsonrpc.AbstractJsonRpcHttpBySpecTest;
 import org.hyperledger.besu.ethereum.core.BlockchainSetupUtil;
-import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat;
+import org.hyperledger.besu.plugin.services.storage.DataStorageFormat;
 
 import org.junit.jupiter.api.BeforeEach;
 

ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/bonsai/EthByzantiumJsonRpcHttpBySpecTest.java

@@ -16,7 +16,7 @@
 
 import org.hyperledger.besu.ethereum.api.jsonrpc.AbstractJsonRpcHttpBySpecTest;
 import org.hyperledger.besu.ethereum.core.BlockchainSetupUtil;
-import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat;
+import org.hyperledger.besu.plugin.services.storage.DataStorageFormat;
 
 import org.junit.jupiter.api.BeforeEach;
 

ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/bonsai/TraceJsonRpcHttpBySpecTest.java

@@ -16,7 +16,7 @@
 
 import org.hyperledger.besu.ethereum.api.jsonrpc.AbstractJsonRpcHttpBySpecTest;
 import org.hyperledger.besu.ethereum.core.BlockchainSetupUtil;
-import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat;
+import org.hyperledger.besu.plugin.services.storage.DataStorageFormat;
 
 import org.junit.jupiter.api.BeforeEach;
 

ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/forest/DebugTraceJsonRpcHttpBySpecTest.java

@@ -16,7 +16,7 @@
 
 import org.hyperledger.besu.ethereum.api.jsonrpc.AbstractJsonRpcHttpBySpecTest;
 import org.hyperledger.besu.ethereum.core.BlockchainSetupUtil;
-import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat;
+import org.hyperledger.besu.plugin.services.storage.DataStorageFormat;
 
 import org.junit.jupiter.api.BeforeEach;
 

ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/forest/EthByzantiumJsonRpcHttpBySpecTest.java

@@ -16,7 +16,7 @@
 
 import org.hyperledger.besu.ethereum.api.jsonrpc.AbstractJsonRpcHttpBySpecTest;
 import org.hyperledger.besu.ethereum.core.BlockchainSetupUtil;
-import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat;
+import org.hyperledger.besu.plugin.services.storage.DataStorageFormat;
 
 import org.junit.jupiter.api.BeforeEach;
 

ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/forest/TraceJsonRpcHttpBySpecTest.java

@@ -16,7 +16,7 @@
 
 import org.hyperledger.besu.ethereum.api.jsonrpc.AbstractJsonRpcHttpBySpecTest;
 import org.hyperledger.besu.ethereum.core.BlockchainSetupUtil;
-import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat;
+import org.hyperledger.besu.plugin.services.storage.DataStorageFormat;
 
 import org.junit.jupiter.api.BeforeEach;
 

ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/filter/EthJsonRpcHttpServiceTest.java

@@ -21,7 +21,7 @@
 import org.hyperledger.besu.ethereum.core.Block;
 import org.hyperledger.besu.ethereum.core.BlockchainSetupUtil;
 import org.hyperledger.besu.ethereum.core.Transaction;
-import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat;
+import org.hyperledger.besu.plugin.services.storage.DataStorageFormat;
 
 import java.io.IOException;