From 527807969e6ac1b9408e9785903457514a61a201 Mon Sep 17 00:00:00 2001 From: Tsz-Wo Nicholas Sze Date: Thu, 10 Oct 2024 08:15:37 -0700 Subject: [PATCH] HDDS-11557. Simplify DBColumnFamilyDefinition. --- .../DatanodeSchemaOneDBDefinition.java | 6 --- .../DatanodeSchemaThreeDBDefinition.java | 10 ---- .../DatanodeSchemaTwoDBDefinition.java | 11 ---- .../utils/db/DBColumnFamilyDefinition.java | 21 ++------ .../hdds/utils/db/TestDBStoreBuilder.java | 6 +-- .../hdds/scm/metadata/SCMDBDefinition.java | 20 ------- .../hadoop/ozone/om/codec/OMDBDefinition.java | 52 ++++--------------- .../ozone/recon/scm/ReconSCMDBDefinition.java | 2 - .../recon/spi/impl/ReconDBDefinition.java | 12 ----- 9 files changed, 17 insertions(+), 123 deletions(-) diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/metadata/DatanodeSchemaOneDBDefinition.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/metadata/DatanodeSchemaOneDBDefinition.java index 4f54e85da2b..bd1c0fb368a 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/metadata/DatanodeSchemaOneDBDefinition.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/metadata/DatanodeSchemaOneDBDefinition.java @@ -51,27 +51,21 @@ public class DatanodeSchemaOneDBDefinition BLOCK_DATA = new DBColumnFamilyDefinition<>( StringUtils.bytes2String(DEFAULT_COLUMN_FAMILY), - String.class, SchemaOneKeyCodec.get(), - BlockData.class, BlockData.getCodec()); public static final DBColumnFamilyDefinition METADATA = new DBColumnFamilyDefinition<>( StringUtils.bytes2String(DEFAULT_COLUMN_FAMILY), - String.class, SchemaOneKeyCodec.get(), - Long.class, LongCodec.get()); public static final DBColumnFamilyDefinition DELETED_BLOCKS = new DBColumnFamilyDefinition<>( StringUtils.bytes2String(DEFAULT_COLUMN_FAMILY), - String.class, SchemaOneKeyCodec.get(), - ChunkInfoList.class, SchemaOneChunkInfoListCodec.get()); private static final Map>> diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/metadata/DatanodeSchemaThreeDBDefinition.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/metadata/DatanodeSchemaThreeDBDefinition.java index d47446d49b0..10537ca6f2d 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/metadata/DatanodeSchemaThreeDBDefinition.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/metadata/DatanodeSchemaThreeDBDefinition.java @@ -59,45 +59,35 @@ public class DatanodeSchemaThreeDBDefinition BLOCK_DATA = new DBColumnFamilyDefinition<>( "block_data", - String.class, FixedLengthStringCodec.get(), - BlockData.class, BlockData.getCodec()); public static final DBColumnFamilyDefinition METADATA = new DBColumnFamilyDefinition<>( "metadata", - String.class, FixedLengthStringCodec.get(), - Long.class, LongCodec.get()); public static final DBColumnFamilyDefinition DELETE_TRANSACTION = new DBColumnFamilyDefinition<>( "delete_txns", - String.class, FixedLengthStringCodec.get(), - DeletedBlocksTransaction.class, Proto2Codec.get(DeletedBlocksTransaction.getDefaultInstance())); public static final DBColumnFamilyDefinition FINALIZE_BLOCKS = new DBColumnFamilyDefinition<>( "finalize_blocks", - String.class, FixedLengthStringCodec.get(), - Long.class, LongCodec.get()); public static final DBColumnFamilyDefinition LAST_CHUNK_INFO = new DBColumnFamilyDefinition<>( "last_chunk_info", - String.class, FixedLengthStringCodec.get(), - BlockData.class, BlockData.getCodec()); private static String separator = ""; diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/metadata/DatanodeSchemaTwoDBDefinition.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/metadata/DatanodeSchemaTwoDBDefinition.java index b9e7ec7bd5b..bf6b1d0a29c 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/metadata/DatanodeSchemaTwoDBDefinition.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/metadata/DatanodeSchemaTwoDBDefinition.java @@ -18,7 +18,6 @@ package org.apache.hadoop.ozone.container.metadata; import org.apache.hadoop.hdds.conf.ConfigurationSource; -import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos; import org.apache.hadoop.hdds.utils.db.DBColumnFamilyDefinition; import org.apache.hadoop.hdds.utils.db.DBDefinition; import org.apache.hadoop.hdds.utils.db.FixedLengthStringCodec; @@ -44,45 +43,35 @@ public class DatanodeSchemaTwoDBDefinition BLOCK_DATA = new DBColumnFamilyDefinition<>( "block_data", - String.class, StringCodec.get(), - BlockData.class, BlockData.getCodec()); public static final DBColumnFamilyDefinition METADATA = new DBColumnFamilyDefinition<>( "metadata", - String.class, StringCodec.get(), - Long.class, LongCodec.get()); public static final DBColumnFamilyDefinition DELETE_TRANSACTION = new DBColumnFamilyDefinition<>( "delete_txns", - Long.class, LongCodec.get(), - StorageContainerDatanodeProtocolProtos.DeletedBlocksTransaction.class, Proto2Codec.get(DeletedBlocksTransaction.getDefaultInstance())); public static final DBColumnFamilyDefinition FINALIZE_BLOCKS = new DBColumnFamilyDefinition<>( "finalize_blocks", - String.class, FixedLengthStringCodec.get(), - Long.class, LongCodec.get()); public static final DBColumnFamilyDefinition LAST_CHUNK_INFO = new DBColumnFamilyDefinition<>( "last_chunk_info", - String.class, FixedLengthStringCodec.get(), - BlockData.class, BlockData.getCodec()); public DatanodeSchemaTwoDBDefinition(String dbPath, diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBColumnFamilyDefinition.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBColumnFamilyDefinition.java index 653182214b6..a5268e6031c 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBColumnFamilyDefinition.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBColumnFamilyDefinition.java @@ -54,32 +54,21 @@ public class DBColumnFamilyDefinition { private final String tableName; - private final Class keyType; - private final Codec keyCodec; - private final Class valueType; - private final Codec valueCodec; - private ManagedColumnFamilyOptions cfOptions; + private volatile ManagedColumnFamilyOptions cfOptions; - public DBColumnFamilyDefinition( - String tableName, - Class keyType, - Codec keyCodec, - Class valueType, - Codec valueCodec) { + public DBColumnFamilyDefinition(String tableName, Codec keyCodec, Codec valueCodec) { this.tableName = tableName; - this.keyType = keyType; this.keyCodec = keyCodec; - this.valueType = valueType; this.valueCodec = valueCodec; this.cfOptions = null; } public Table getTable(DBStore db) throws IOException { - return db.getTable(tableName, keyType, valueType); + return db.getTable(tableName, getKeyType(), getValueType()); } public String getName() { @@ -87,7 +76,7 @@ public String getName() { } public Class getKeyType() { - return keyType; + return keyCodec.getTypeClass(); } public Codec getKeyCodec() { @@ -95,7 +84,7 @@ public Codec getKeyCodec() { } public Class getValueType() { - return valueType; + return valueCodec.getTypeClass(); } public Codec getValueCodec() { diff --git a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/db/TestDBStoreBuilder.java b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/db/TestDBStoreBuilder.java index aad3e9e12e6..7966afe5045 100644 --- a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/db/TestDBStoreBuilder.java +++ b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/db/TestDBStoreBuilder.java @@ -179,7 +179,7 @@ public void builderWithColumnFamilyOptions(@TempDir Path tempDir) String sampleTableName = "sampleTable"; final DBColumnFamilyDefinition sampleTable = new DBColumnFamilyDefinition<>(sampleTableName, - String.class, StringCodec.get(), Long.class, LongCodec.get()); + StringCodec.get(), LongCodec.get()); final DBDefinition sampleDB = new DBDefinition.WithMap( DBColumnFamilyDefinition.newUnmodifiableMap(sampleTable)) { { @@ -250,8 +250,8 @@ public void testIfAutoCompactionDisabled(boolean disableAutoCompaction, String sampleTableName = "sampleTable"; final DBColumnFamilyDefinition sampleTable = - new DBColumnFamilyDefinition<>(sampleTableName, String.class, - StringCodec.get(), Long.class, LongCodec.get()); + new DBColumnFamilyDefinition<>(sampleTableName, + StringCodec.get(), LongCodec.get()); final DBDefinition sampleDB = new DBDefinition.WithMap( DBColumnFamilyDefinition.newUnmodifiableMap(sampleTable)) { @Override diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/SCMDBDefinition.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/SCMDBDefinition.java index 4a280d2103a..39cf1e53480 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/SCMDBDefinition.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/SCMDBDefinition.java @@ -53,63 +53,49 @@ protected SCMDBDefinition(Map> map) { DELETED_BLOCKS = new DBColumnFamilyDefinition<>( "deletedBlocks", - Long.class, LongCodec.get(), - DeletedBlocksTransaction.class, Proto2Codec.get(DeletedBlocksTransaction.getDefaultInstance())); public static final DBColumnFamilyDefinition VALID_CERTS = new DBColumnFamilyDefinition<>( "validCerts", - BigInteger.class, BigIntegerCodec.get(), - X509Certificate.class, X509CertificateCodec.get()); public static final DBColumnFamilyDefinition VALID_SCM_CERTS = new DBColumnFamilyDefinition<>( "validSCMCerts", - BigInteger.class, BigIntegerCodec.get(), - X509Certificate.class, X509CertificateCodec.get()); public static final DBColumnFamilyDefinition PIPELINES = new DBColumnFamilyDefinition<>( "pipelines", - PipelineID.class, PipelineID.getCodec(), - Pipeline.class, Pipeline.getCodec()); public static final DBColumnFamilyDefinition CONTAINERS = new DBColumnFamilyDefinition<>( "containers", - ContainerID.class, ContainerID.getCodec(), - ContainerInfo.class, ContainerInfo.getCodec()); public static final DBColumnFamilyDefinition TRANSACTIONINFO = new DBColumnFamilyDefinition<>( "scmTransactionInfos", - String.class, StringCodec.get(), - TransactionInfo.class, TransactionInfo.getCodec()); public static final DBColumnFamilyDefinition SEQUENCE_ID = new DBColumnFamilyDefinition<>( "sequenceId", - String.class, StringCodec.get(), - Long.class, LongCodec.get()); public static final DBColumnFamilyDefinition> map) { MOVE = new DBColumnFamilyDefinition<>( "move", - ContainerID.class, ContainerID.getCodec(), - MoveDataNodePair.class, MoveDataNodePair.getCodec()); /** @@ -129,18 +113,14 @@ protected SCMDBDefinition(Map> map) { public static final DBColumnFamilyDefinition META = new DBColumnFamilyDefinition<>( "meta", - String.class, StringCodec.get(), - String.class, StringCodec.get()); public static final DBColumnFamilyDefinition STATEFUL_SERVICE_CONFIG = new DBColumnFamilyDefinition<>( "statefulServiceConfig", - String.class, StringCodec.get(), - ByteString.class, ByteStringCodec.get()); private static final Map> diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/codec/OMDBDefinition.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/codec/OMDBDefinition.java index d48b413a2ae..be57a7b7451 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/codec/OMDBDefinition.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/codec/OMDBDefinition.java @@ -56,139 +56,109 @@ public final class OMDBDefinition extends DBDefinition.WithMap { DELETED_TABLE = new DBColumnFamilyDefinition<>( OmMetadataManagerImpl.DELETED_TABLE, - String.class, StringCodec.get(), - RepeatedOmKeyInfo.class, RepeatedOmKeyInfo.getCodec(true)); public static final DBColumnFamilyDefinition USER_TABLE = new DBColumnFamilyDefinition<>( OmMetadataManagerImpl.USER_TABLE, - String.class, StringCodec.get(), - PersistedUserVolumeInfo.class, Proto2Codec.get(PersistedUserVolumeInfo.getDefaultInstance())); public static final DBColumnFamilyDefinition VOLUME_TABLE = new DBColumnFamilyDefinition<>( OmMetadataManagerImpl.VOLUME_TABLE, - String.class, StringCodec.get(), - OmVolumeArgs.class, OmVolumeArgs.getCodec()); public static final DBColumnFamilyDefinition OPEN_KEY_TABLE = new DBColumnFamilyDefinition<>( OmMetadataManagerImpl.OPEN_KEY_TABLE, - String.class, StringCodec.get(), - OmKeyInfo.class, OmKeyInfo.getCodec(true)); public static final DBColumnFamilyDefinition KEY_TABLE = new DBColumnFamilyDefinition<>( OmMetadataManagerImpl.KEY_TABLE, - String.class, StringCodec.get(), - OmKeyInfo.class, OmKeyInfo.getCodec(true)); public static final DBColumnFamilyDefinition BUCKET_TABLE = new DBColumnFamilyDefinition<>( OmMetadataManagerImpl.BUCKET_TABLE, - String.class, StringCodec.get(), - OmBucketInfo.class, OmBucketInfo.getCodec()); public static final DBColumnFamilyDefinition MULTIPART_INFO_TABLE = new DBColumnFamilyDefinition<>( OmMetadataManagerImpl.MULTIPARTINFO_TABLE, - String.class, StringCodec.get(), - OmMultipartKeyInfo.class, OmMultipartKeyInfo.getCodec()); public static final DBColumnFamilyDefinition PREFIX_TABLE = new DBColumnFamilyDefinition<>( OmMetadataManagerImpl.PREFIX_TABLE, - String.class, StringCodec.get(), - OmPrefixInfo.class, OmPrefixInfo.getCodec()); public static final DBColumnFamilyDefinition DTOKEN_TABLE = new DBColumnFamilyDefinition<>( OmMetadataManagerImpl.DELEGATION_TOKEN_TABLE, - OzoneTokenIdentifier.class, TokenIdentifierCodec.get(), - Long.class, LongCodec.get()); public static final DBColumnFamilyDefinition S3_SECRET_TABLE = new DBColumnFamilyDefinition<>( OmMetadataManagerImpl.S3_SECRET_TABLE, - String.class, StringCodec.get(), - S3SecretValue.class, S3SecretValue.getCodec()); public static final DBColumnFamilyDefinition TRANSACTION_INFO_TABLE = new DBColumnFamilyDefinition<>( OmMetadataManagerImpl.TRANSACTION_INFO_TABLE, - String.class, StringCodec.get(), - TransactionInfo.class, TransactionInfo.getCodec()); public static final DBColumnFamilyDefinition DIRECTORY_TABLE = new DBColumnFamilyDefinition<>( OmMetadataManagerImpl.DIRECTORY_TABLE, - String.class, StringCodec.get(), - OmDirectoryInfo.class, OmDirectoryInfo.getCodec()); public static final DBColumnFamilyDefinition FILE_TABLE = new DBColumnFamilyDefinition<>( OmMetadataManagerImpl.FILE_TABLE, - String.class, StringCodec.get(), - OmKeyInfo.class, OmKeyInfo.getCodec(true)); public static final DBColumnFamilyDefinition OPEN_FILE_TABLE = new DBColumnFamilyDefinition<>( OmMetadataManagerImpl.OPEN_FILE_TABLE, - String.class, StringCodec.get(), - OmKeyInfo.class, OmKeyInfo.getCodec(true)); public static final DBColumnFamilyDefinition DELETED_DIR_TABLE = new DBColumnFamilyDefinition<>(OmMetadataManagerImpl.DELETED_DIR_TABLE, - String.class, StringCodec.get(), OmKeyInfo.class, + StringCodec.get(), OmKeyInfo.getCodec(true)); public static final DBColumnFamilyDefinition META_TABLE = new DBColumnFamilyDefinition<>( OmMetadataManagerImpl.META_TABLE, - String.class, StringCodec.get(), - String.class, StringCodec.get()); // Tables for multi-tenancy @@ -197,27 +167,26 @@ public final class OMDBDefinition extends DBDefinition.WithMap { TENANT_ACCESS_ID_TABLE = new DBColumnFamilyDefinition<>( OmMetadataManagerImpl.TENANT_ACCESS_ID_TABLE, - String.class, // accessId + // accessId StringCodec.get(), - OmDBAccessIdInfo.class, // tenantId, secret, principal + // tenantId, secret, principal OmDBAccessIdInfo.getCodec()); public static final DBColumnFamilyDefinition PRINCIPAL_TO_ACCESS_IDS_TABLE = new DBColumnFamilyDefinition<>( OmMetadataManagerImpl.PRINCIPAL_TO_ACCESS_IDS_TABLE, - String.class, // User principal + // User principal StringCodec.get(), - OmDBUserPrincipalInfo.class, // List of accessIds + // List of accessIds OmDBUserPrincipalInfo.getCodec()); public static final DBColumnFamilyDefinition TENANT_STATE_TABLE = new DBColumnFamilyDefinition<>( OmMetadataManagerImpl.TENANT_STATE_TABLE, - String.class, // tenantId (tenant name) + // tenantId (tenant name) StringCodec.get(), - OmDBTenantState.class, OmDBTenantState.getCodec()); // End tables for S3 multi-tenancy @@ -226,18 +195,15 @@ public final class OMDBDefinition extends DBDefinition.WithMap { SNAPSHOT_INFO_TABLE = new DBColumnFamilyDefinition<>( OmMetadataManagerImpl.SNAPSHOT_INFO_TABLE, - String.class, // snapshot path + // snapshot path StringCodec.get(), - SnapshotInfo.class, SnapshotInfo.getCodec()); public static final DBColumnFamilyDefinition COMPACTION_LOG_TABLE = new DBColumnFamilyDefinition<>( OmMetadataManagerImpl.COMPACTION_LOG_TABLE, - String.class, StringCodec.get(), - CompactionLogEntry.class, CompactionLogEntry.getCodec()); /** @@ -254,9 +220,9 @@ public final class OMDBDefinition extends DBDefinition.WithMap { SNAPSHOT_RENAMED_TABLE = new DBColumnFamilyDefinition<>( OmMetadataManagerImpl.SNAPSHOT_RENAMED_TABLE, - String.class, // /volumeName/bucketName/objectID + // /volumeName/bucketName/objectID StringCodec.get(), - String.class, // path to key in prev snapshot's key(file)/dir Table. + // path to key in prev snapshot's key(file)/dir Table. StringCodec.get()); private static final Map> diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconSCMDBDefinition.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconSCMDBDefinition.java index fa6d6f925dd..9c014c56b50 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconSCMDBDefinition.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconSCMDBDefinition.java @@ -43,9 +43,7 @@ public class ReconSCMDBDefinition extends SCMDBDefinition { NODES = new DBColumnFamilyDefinition( "nodes", - UUID.class, UUID_CODEC, - DatanodeDetails.class, DatanodeDetails.getCodec()); private static final Map> diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconDBDefinition.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconDBDefinition.java index 8cb3b4188ed..cde24d7570b 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconDBDefinition.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconDBDefinition.java @@ -46,44 +46,34 @@ public ReconDBDefinition(String dbName) { CONTAINER_KEY = new DBColumnFamilyDefinition<>( "containerKeyTable", - ContainerKeyPrefix.class, ContainerKeyPrefixCodec.get(), - Integer.class, IntegerCodec.get()); public static final DBColumnFamilyDefinition KEY_CONTAINER = new DBColumnFamilyDefinition<>( "keyContainerTable", - KeyPrefixContainer.class, KeyPrefixContainerCodec.get(), - Integer.class, IntegerCodec.get()); public static final DBColumnFamilyDefinition CONTAINER_KEY_COUNT = new DBColumnFamilyDefinition<>( "containerKeyCountTable", - Long.class, LongCodec.get(), - Long.class, LongCodec.get()); public static final DBColumnFamilyDefinition REPLICA_HISTORY = new DBColumnFamilyDefinition( "replica_history", - Long.class, LongCodec.get(), - ContainerReplicaHistoryList.class, ContainerReplicaHistoryList.getCodec()); public static final DBColumnFamilyDefinition NAMESPACE_SUMMARY = new DBColumnFamilyDefinition( "namespaceSummaryTable", - Long.class, LongCodec.get(), - NSSummary.class, NSSummaryCodec.get()); // Container Replica History with bcsId tracking. @@ -91,9 +81,7 @@ public ReconDBDefinition(String dbName) { REPLICA_HISTORY_V2 = new DBColumnFamilyDefinition( "replica_history_v2", - Long.class, LongCodec.get(), - ContainerReplicaHistoryList.class, ContainerReplicaHistoryList.getCodec()); private static final Map>