Skip to content

Commit

Permalink
HDDS-11555. SCMDBDefinition should be singleton. (apache#7296)
Browse files Browse the repository at this point in the history
szetszwo authored Oct 12, 2024
1 parent d473134 commit cb44d5e
Showing 26 changed files with 56 additions and 74 deletions.
Original file line number Diff line number Diff line change
@@ -18,8 +18,7 @@
package org.apache.hadoop.hdds.scm.ha;

import java.io.IOException;

import com.google.common.base.Preconditions;
import java.util.Objects;

import org.apache.hadoop.hdds.protocol.scm.proto.InterSCMProtocolProtos.CopyDBCheckpointRequestProto;
import org.apache.hadoop.hdds.protocol.scm.proto.InterSCMProtocolProtos.CopyDBCheckpointResponseProto;
@@ -52,12 +51,11 @@ public class InterSCMGrpcService extends
private final Table<String, TransactionInfo> transactionInfoTable;

InterSCMGrpcService(final StorageContainerManager scm) throws IOException {
Preconditions.checkNotNull(scm);
Objects.requireNonNull(scm, "scm");
this.scm = scm;
this.transactionInfoTable = HAUtils.getTransactionInfoTable(
scm.getScmMetadataStore().getStore(), new SCMDBDefinition());
provider =
new SCMDBCheckpointProvider(scm.getScmMetadataStore().getStore());
scm.getScmMetadataStore().getStore(), SCMDBDefinition.get());
this.provider = new SCMDBCheckpointProvider(scm.getScmMetadataStore().getStore());
}

@Override
@@ -67,7 +65,7 @@ public void download(CopyDBCheckpointRequestProto request,
scm.getScmHAManager().asSCMHADBTransactionBuffer().flush();
TransactionInfo transactionInfo =
transactionInfoTable.get(TRANSACTION_INFO_KEY);
Preconditions.checkNotNull(transactionInfo);
Objects.requireNonNull(transactionInfo, "transactionInfo");
SCMGrpcOutputStream outputStream =
new SCMGrpcOutputStream(responseObserver, scm.getClusterId(),
BUFFER_SIZE);
Original file line number Diff line number Diff line change
@@ -72,6 +72,7 @@ public class SCMHAManagerImpl implements SCMHAManager {

private final SCMRatisServer ratisServer;
private final ConfigurationSource conf;
private final OzoneConfiguration ozoneConf;
private final SecurityConfig securityConfig;
private final DBTransactionBuffer transactionBuffer;
private final SCMSnapshotProvider scmSnapshotProvider;
@@ -89,6 +90,7 @@ public SCMHAManagerImpl(final ConfigurationSource conf,
final SecurityConfig securityConfig,
final StorageContainerManager scm) throws IOException {
this.conf = conf;
this.ozoneConf = OzoneConfiguration.of(conf);
this.securityConfig = securityConfig;
this.scm = scm;
this.exitManager = new ExitManager();
@@ -128,7 +130,7 @@ public void start() throws IOException {
// It will first try to add itself to existing ring
final SCMNodeDetails nodeDetails =
scm.getSCMHANodeDetails().getLocalNodeDetails();
final boolean success = HAUtils.addSCM(OzoneConfiguration.of(conf),
final boolean success = HAUtils.addSCM(ozoneConf,
new AddSCMRequest.Builder().setClusterId(scm.getClusterId())
.setScmId(scm.getScmId())
.setRatisAddr(nodeDetails
@@ -221,17 +223,18 @@ public List<ManagedSecretKey> getSecretKeysFromLeader(String leaderID)
}
}

private TransactionInfo getTransactionInfoFromCheckpoint(Path checkpointLocation) throws IOException {
return HAUtils.getTrxnInfoFromCheckpoint(
ozoneConf, checkpointLocation, SCMDBDefinition.get());
}

@Override
public TermIndex verifyCheckpointFromLeader(String leaderId,
DBCheckpoint checkpoint) {
try {
Path checkpointLocation = checkpoint.getCheckpointLocation();
TransactionInfo checkpointTxnInfo = HAUtils
.getTrxnInfoFromCheckpoint(OzoneConfiguration.of(conf),
checkpointLocation, new SCMDBDefinition());

LOG.info("Installing checkpoint with SCMTransactionInfo {}",
checkpointTxnInfo);
final TransactionInfo checkpointTxnInfo = getTransactionInfoFromCheckpoint(checkpointLocation);
LOG.info("{}: Verify checkpoint {} from leader {}", scm.getScmId(), checkpointTxnInfo, leaderId);

TermIndex termIndex =
getRatisServer().getSCMStateMachine().getLastAppliedTermIndex();
@@ -281,12 +284,9 @@ public TermIndex installCheckpoint(DBCheckpoint dbCheckpoint)
throws Exception {

Path checkpointLocation = dbCheckpoint.getCheckpointLocation();
TransactionInfo checkpointTrxnInfo = HAUtils
.getTrxnInfoFromCheckpoint(OzoneConfiguration.of(conf),
checkpointLocation, new SCMDBDefinition());
final TransactionInfo checkpointTrxnInfo = getTransactionInfoFromCheckpoint(checkpointLocation);

LOG.info("Installing checkpoint with SCMTransactionInfo {}",
checkpointTrxnInfo);
LOG.info("{}: Install checkpoint {}", scm.getScmId(), checkpointTrxnInfo);

return installCheckpoint(checkpointLocation, checkpointTrxnInfo);
}
@@ -457,7 +457,7 @@ public void startServices() throws IOException {

// TODO: Fix the metrics ??
final SCMMetadataStore metadataStore = scm.getScmMetadataStore();
metadataStore.start(OzoneConfiguration.of(conf));
metadataStore.start(ozoneConf);
scm.getSequenceIdGen().reinitialize(metadataStore.getSequenceIdTable());
scm.getPipelineManager().reinitialize(metadataStore.getPipelineTable());
scm.getContainerManager().reinitialize(metadataStore.getContainerTable());
Original file line number Diff line number Diff line change
@@ -41,14 +41,6 @@
* Class defines the structure and types of the scm.db.
*/
public class SCMDBDefinition extends DBDefinition.WithMap {
public SCMDBDefinition() {
this(COLUMN_FAMILIES);
}

protected SCMDBDefinition(Map<String, DBColumnFamilyDefinition<?, ?>> map) {
super(map);
}

public static final DBColumnFamilyDefinition<Long, DeletedBlocksTransaction>
DELETED_BLOCKS =
new DBColumnFamilyDefinition<>(
@@ -156,6 +148,16 @@ protected SCMDBDefinition(Map<String, DBColumnFamilyDefinition<?, ?>> map) {
VALID_CERTS,
VALID_SCM_CERTS);

private static final SCMDBDefinition INSTANCE = new SCMDBDefinition(COLUMN_FAMILIES);

public static SCMDBDefinition get() {
return INSTANCE;
}

protected SCMDBDefinition(Map<String, DBColumnFamilyDefinition<?, ?>> map) {
super(map);
}

@Override
public String getName() {
return "scm.db";
Original file line number Diff line number Diff line change
@@ -104,7 +104,7 @@ public SCMMetadataStoreImpl(OzoneConfiguration config)
public void start(OzoneConfiguration config)
throws IOException {
if (this.store == null) {
SCMDBDefinition scmdbDefinition = new SCMDBDefinition();
final SCMDBDefinition scmdbDefinition = SCMDBDefinition.get();
File metaDir = HAUtils.getMetaDir(scmdbDefinition, configuration);
// Check if there is a DB Inconsistent Marker in the metaDir. This
// marker indicates that the DB is in an inconsistent state and hence
Original file line number Diff line number Diff line change
@@ -77,8 +77,7 @@ public class TestContainerManagerImpl {
@BeforeEach
void setUp() throws Exception {
final OzoneConfiguration conf = SCMTestUtils.getConf(testDir);
dbStore = DBStoreBuilder.createDBStore(
conf, new SCMDBDefinition());
dbStore = DBStoreBuilder.createDBStore(conf, SCMDBDefinition.get());
scmhaManager = SCMHAManagerStub.getInstance(true);
nodeManager = new MockNodeManager(true, 10);
sequenceIdGen = new SequenceIdGenerator(
Original file line number Diff line number Diff line change
@@ -98,8 +98,7 @@ void setup() throws IOException, InvalidStateTransitionException {
final OzoneConfiguration conf = SCMTestUtils.getConf(testDir);
nodeManager = new MockNodeManager(true, 10);
containerManager = mock(ContainerManager.class);
dbStore = DBStoreBuilder.createDBStore(
conf, new SCMDBDefinition());
dbStore = DBStoreBuilder.createDBStore(conf, SCMDBDefinition.get());
scmhaManager = SCMHAManagerStub.getInstance(true);
pipelineManager =
new MockPipelineManager(dbStore, scmhaManager, nodeManager);
Original file line number Diff line number Diff line change
@@ -75,8 +75,7 @@ public void init() throws IOException, TimeoutException {
OzoneConfiguration conf = new OzoneConfiguration();
scmhaManager = SCMHAManagerStub.getInstance(true);
conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, testDir.getAbsolutePath());
dbStore = DBStoreBuilder.createDBStore(
conf, new SCMDBDefinition());
dbStore = DBStoreBuilder.createDBStore(conf, SCMDBDefinition.get());
pipelineManager = mock(PipelineManager.class);
pipeline = Pipeline.newBuilder().setState(Pipeline.PipelineState.CLOSED)
.setId(PipelineID.randomId())
Original file line number Diff line number Diff line change
@@ -126,8 +126,7 @@ public void setup() throws IOException, InvalidStateTransitionException,
new SCMNodeManager(conf, storageConfig, eventQueue, clusterMap,
scmContext, versionManager);
scmhaManager = SCMHAManagerStub.getInstance(true);
dbStore = DBStoreBuilder.createDBStore(
conf, new SCMDBDefinition());
dbStore = DBStoreBuilder.createDBStore(conf, SCMDBDefinition.get());

pipelineManager =
new MockPipelineManager(dbStore, scmhaManager, nodeManager);
Original file line number Diff line number Diff line change
@@ -78,8 +78,7 @@ public void setup() throws IOException {
final OzoneConfiguration conf = SCMTestUtils.getConf(testDir);
this.nodeManager = new MockNodeManager(true, 10);
this.containerManager = mock(ContainerManager.class);
dbStore = DBStoreBuilder.createDBStore(
conf, new SCMDBDefinition());
dbStore = DBStoreBuilder.createDBStore(conf, SCMDBDefinition.get());
scmhaManager = SCMHAManagerStub.getInstance(true);
pipelineManager =
new MockPipelineManager(dbStore, scmhaManager, nodeManager);
Original file line number Diff line number Diff line change
@@ -179,8 +179,7 @@ void setup(@TempDir File testDir) throws IOException, InterruptedException,
nodeManager = new SimpleMockNodeManager();
eventQueue = new EventQueue();
SCMHAManager scmhaManager = SCMHAManagerStub.getInstance(true);
dbStore = DBStoreBuilder.createDBStore(
conf, new SCMDBDefinition());
dbStore = DBStoreBuilder.createDBStore(conf, SCMDBDefinition.get());
PipelineManager pipelineManager = mock(PipelineManager.class);
when(pipelineManager.containsPipeline(any(PipelineID.class)))
.thenReturn(true);
@@ -277,8 +276,7 @@ private void createReplicationManager(ReplicationManagerConfiguration rmConf,

SCMHAManager scmHAManager = SCMHAManagerStub
.getInstance(true, new SCMDBTransactionBufferImpl());
dbStore = DBStoreBuilder.createDBStore(
config, new SCMDBDefinition());
dbStore = DBStoreBuilder.createDBStore(config, SCMDBDefinition.get());

LegacyReplicationManager legacyRM = new LegacyReplicationManager(
config, containerManager, ratisContainerPlacementPolicy, eventQueue,
Original file line number Diff line number Diff line change
@@ -49,7 +49,7 @@ public class TestStatefulServiceStateManagerImpl {
void setup(@TempDir File testDir) throws IOException {
conf = SCMTestUtils.getConf(testDir);
conf.setBoolean(ScmConfigKeys.OZONE_SCM_HA_ENABLE_KEY, true);
dbStore = DBStoreBuilder.createDBStore(conf, new SCMDBDefinition());
dbStore = DBStoreBuilder.createDBStore(conf, SCMDBDefinition.get());
statefulServiceConfig =
SCMDBDefinition.STATEFUL_SERVICE_CONFIG.getTable(dbStore);
scmhaManager = SCMHAManagerStub.getInstance(true, dbStore);
Original file line number Diff line number Diff line change
@@ -100,8 +100,7 @@ public class TestContainerPlacement {
public void setUp() throws Exception {
conf = getConf();
conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, testDir.getAbsolutePath());
dbStore = DBStoreBuilder.createDBStore(
conf, new SCMDBDefinition());
dbStore = DBStoreBuilder.createDBStore(conf, SCMDBDefinition.get());
scmhaManager = SCMHAManagerStub.getInstance(true);
sequenceIdGen = new SequenceIdGenerator(
conf, scmhaManager, SCMDBDefinition.SEQUENCE_ID.getTable(dbStore));
Original file line number Diff line number Diff line change
@@ -66,8 +66,7 @@ public class TestPipelineDatanodesIntersection {
public void initialize() throws IOException {
conf = SCMTestUtils.getConf(testDir);
end = false;
dbStore = DBStoreBuilder.createDBStore(
conf, new SCMDBDefinition());
dbStore = DBStoreBuilder.createDBStore(conf, SCMDBDefinition.get());
}

@AfterEach
Original file line number Diff line number Diff line change
@@ -136,7 +136,7 @@ void init(@TempDir File testDir, @TempDir File dbDir) throws Exception {
// placement policy (Rack Scatter), so just use the random one.
conf.set(ScmConfigKeys.OZONE_SCM_CONTAINER_PLACEMENT_EC_IMPL_KEY,
SCMContainerPlacementRandom.class.getName());
dbStore = DBStoreBuilder.createDBStore(conf, new SCMDBDefinition());
dbStore = DBStoreBuilder.createDBStore(conf, SCMDBDefinition.get());
nodeManager = new MockNodeManager(true, 20);
maxPipelineCount = nodeManager.getNodeCount(
HddsProtos.NodeOperationalState.IN_SERVICE,
Original file line number Diff line number Diff line change
@@ -135,8 +135,7 @@ private void setupRacks(int datanodeCount, int nodesPerRack,
.thenReturn(dn);
}

dbStore = DBStoreBuilder.createDBStore(
conf, new SCMDBDefinition());
dbStore = DBStoreBuilder.createDBStore(conf, SCMDBDefinition.get());
scmhaManager = SCMHAManagerStub.getInstance(true);

stateManager = PipelineStateManagerImpl.newBuilder()
Original file line number Diff line number Diff line change
@@ -112,8 +112,7 @@ public void init() throws Exception {
conf.setStorageSize(OZONE_DATANODE_RATIS_VOLUME_FREE_SPACE_MIN,
10, StorageUnit.MB);
nodeManager.setNumPipelinePerDatanode(PIPELINE_LOAD_LIMIT);
dbStore = DBStoreBuilder.createDBStore(
conf, new SCMDBDefinition());
dbStore = DBStoreBuilder.createDBStore(conf, SCMDBDefinition.get());
scmhaManager = SCMHAManagerStub.getInstance(true);
stateManager = PipelineStateManagerImpl.newBuilder()
.setPipelineStore(SCMDBDefinition.PIPELINES.getTable(dbStore))
Original file line number Diff line number Diff line change
@@ -68,8 +68,7 @@ public class TestPipelineStateManagerImpl {
@BeforeEach
public void init() throws Exception {
final OzoneConfiguration conf = SCMTestUtils.getConf(testDir);
dbStore = DBStoreBuilder.createDBStore(
conf, new SCMDBDefinition());
dbStore = DBStoreBuilder.createDBStore(conf, SCMDBDefinition.get());

SCMHAManager scmhaManager = SCMHAManagerStub.getInstance(true);
NodeManager nodeManager = new MockNodeManager(true, 10);
Original file line number Diff line number Diff line change
@@ -93,8 +93,7 @@ public void init(int maxPipelinePerNode, OzoneConfiguration conf)

public void init(int maxPipelinePerNode, OzoneConfiguration conf, File dir) throws Exception {
conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, dir.getAbsolutePath());
dbStore = DBStoreBuilder.createDBStore(
conf, new SCMDBDefinition());
dbStore = DBStoreBuilder.createDBStore(conf, SCMDBDefinition.get());
nodeManager = new MockNodeManager(true, 10);
nodeManager.setNumPipelinePerDatanode(maxPipelinePerNode);
SCMHAManager scmhaManager = SCMHAManagerStub.getInstance(true);
Original file line number Diff line number Diff line change
@@ -60,8 +60,7 @@ public class TestSimplePipelineProvider {
public void init() throws Exception {
nodeManager = new MockNodeManager(true, 10);
final OzoneConfiguration conf = SCMTestUtils.getConf(testDir);
dbStore = DBStoreBuilder.createDBStore(
conf, new SCMDBDefinition());
dbStore = DBStoreBuilder.createDBStore(conf, SCMDBDefinition.get());
SCMHAManager scmhaManager = SCMHAManagerStub.getInstance(true);
stateManager = PipelineStateManagerImpl.newBuilder()
.setPipelineStore(SCMDBDefinition.PIPELINES.getTable(dbStore))
Original file line number Diff line number Diff line change
@@ -128,8 +128,7 @@ void setup(@TempDir File testDir) throws IOException {

containers = new HashMap<>();
conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, testDir.getAbsolutePath());
dbStore = DBStoreBuilder.createDBStore(
conf, new SCMDBDefinition());
dbStore = DBStoreBuilder.createDBStore(conf, SCMDBDefinition.get());
scmhaManager = SCMHAManagerStub.getInstance(true);
pipelineManager =
new MockPipelineManager(dbStore, scmhaManager, nodeManager);
Original file line number Diff line number Diff line change
@@ -128,15 +128,14 @@ private DBCheckpoint downloadSnapshot() throws Exception {
public void testInstallCheckPoint() throws Exception {
DBCheckpoint checkpoint = downloadSnapshot();
StorageContainerManager scm = cluster.getStorageContainerManager();
DBStore db = HAUtils
.loadDB(conf, checkpoint.getCheckpointLocation().getParent().toFile(),
checkpoint.getCheckpointLocation().getFileName().toString(),
new SCMDBDefinition());
final Path location = checkpoint.getCheckpointLocation();
final DBStore db = HAUtils.loadDB(conf, location.getParent().toFile(),
location.getFileName().toString(), SCMDBDefinition.get());
// Hack the transaction index in the checkpoint so as to ensure the
// checkpointed transaction index is higher than when it was downloaded
// from.
assertNotNull(db);
HAUtils.getTransactionInfoTable(db, new SCMDBDefinition())
HAUtils.getTransactionInfoTable(db, SCMDBDefinition.get())
.put(OzoneConsts.TRANSACTION_INFO_KEY, TransactionInfo.valueOf(10, 100));
db.close();
ContainerID cid =
Original file line number Diff line number Diff line change
@@ -224,9 +224,8 @@ public void testInstallCorruptedCheckpointFailure() throws Exception {
DBCheckpoint leaderDbCheckpoint = leaderSCM.getScmMetadataStore().getStore()
.getCheckpoint(false);
Path leaderCheckpointLocation = leaderDbCheckpoint.getCheckpointLocation();
TransactionInfo leaderCheckpointTrxnInfo = HAUtils
.getTrxnInfoFromCheckpoint(conf, leaderCheckpointLocation,
new SCMDBDefinition());
final TransactionInfo leaderCheckpointTrxnInfo = HAUtils.getTrxnInfoFromCheckpoint(
conf, leaderCheckpointLocation, SCMDBDefinition.get());

assertNotNull(leaderCheckpointLocation);
// Take a backup of the current DB
Original file line number Diff line number Diff line change
@@ -50,7 +50,7 @@ public class ReconSCMDBDefinition extends SCMDBDefinition {

private static final Map<String, DBColumnFamilyDefinition<?, ?>>
COLUMN_FAMILIES = DBColumnFamilyDefinition.newUnmodifiableMap(
new SCMDBDefinition().getMap(), NODES);
SCMDBDefinition.get().getMap(), NODES);

public ReconSCMDBDefinition() {
super(COLUMN_FAMILIES);
Original file line number Diff line number Diff line change
@@ -56,7 +56,7 @@ private DBDefinitionFactory() {

static {
final Map<String, DBDefinition> map = new HashMap<>();
Arrays.asList(new SCMDBDefinition(), OMDBDefinition.get(), new ReconSCMDBDefinition())
Arrays.asList(SCMDBDefinition.get(), OMDBDefinition.get(), new ReconSCMDBDefinition())
.forEach(dbDefinition -> map.put(dbDefinition.getName(), dbDefinition));
DB_MAP = Collections.unmodifiableMap(map);
}
Original file line number Diff line number Diff line change
@@ -59,7 +59,7 @@ public Void call() throws Exception {

ConfigurationSource config = createOzoneConfiguration();

scmDb = DBStoreBuilder.createDBStore(config, new SCMDBDefinition());
scmDb = DBStoreBuilder.createDBStore(config, SCMDBDefinition.get());

containerStore = CONTAINERS.getTable(scmDb);

Original file line number Diff line number Diff line change
@@ -47,8 +47,7 @@ public void testGetDefinition() {
DBDefinition definition = DBDefinitionFactory.getDefinition(OMDBDefinition.get().getName());
assertInstanceOf(OMDBDefinition.class, definition);

definition = DBDefinitionFactory.getDefinition(
new SCMDBDefinition().getName());
definition = DBDefinitionFactory.getDefinition(SCMDBDefinition.get().getName());
assertInstanceOf(SCMDBDefinition.class, definition);

definition = DBDefinitionFactory.getDefinition(

0 comments on commit cb44d5e

Please sign in to comment.