diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java index 4873a7db491..6698ece4a8d 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java @@ -108,6 +108,8 @@ import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_FS_SNAPSHOT_MAX_LIMIT_DEFAULT; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_DB_MAX_OPEN_FILES; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_DB_MAX_OPEN_FILES_DEFAULT; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_DB_MAX_OPEN_FILES; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_DB_MAX_OPEN_FILES_DEFAULT; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_SNAPSHOT_CHECKPOINT_DIR_CREATION_POLL_TIMEOUT; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_SNAPSHOT_CHECKPOINT_DIR_CREATION_POLL_TIMEOUT_DEFAULT; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.BUCKET_NOT_FOUND; @@ -401,8 +403,9 @@ private OmMetadataManagerImpl(OzoneConfiguration conf, File dir, String name) throws IOException { lock = new OmReadOnlyLock(); omEpoch = 0; - setStore(loadDB(conf, dir, name, true, - java.util.Optional.of(Boolean.TRUE), Optional.empty())); + int maxOpenFiles = conf.getInt(OZONE_OM_SNAPSHOT_DB_MAX_OPEN_FILES, OZONE_OM_SNAPSHOT_DB_MAX_OPEN_FILES_DEFAULT); + + setStore(loadDB(conf, dir, name, true, Optional.of(Boolean.TRUE), maxOpenFiles, false, false)); initializeOmTables(CacheType.PARTIAL_CACHE, false); perfMetrics = null; } @@ -435,8 +438,7 @@ private OmMetadataManagerImpl(OzoneConfiguration conf, File dir, String name) checkSnapshotDirExist(checkpoint); } setStore(loadDB(conf, metaDir, dbName, false, - java.util.Optional.of(Boolean.TRUE), - Optional.of(maxOpenFiles), false, false)); + java.util.Optional.of(Boolean.TRUE), maxOpenFiles, false, false)); initializeOmTables(CacheType.PARTIAL_CACHE, false); } catch (IOException e) { stop(); @@ -578,7 +580,7 @@ public void start(OzoneConfiguration configuration) throws IOException { int maxOpenFiles = configuration.getInt(OZONE_OM_DB_MAX_OPEN_FILES, OZONE_OM_DB_MAX_OPEN_FILES_DEFAULT); - this.store = loadDB(configuration, metaDir, Optional.of(maxOpenFiles)); + this.store = loadDB(configuration, metaDir, maxOpenFiles); initializeOmTables(CacheType.FULL_CACHE, true); } @@ -586,33 +588,15 @@ public void start(OzoneConfiguration configuration) throws IOException { snapshotChainManager = new SnapshotChainManager(this); } - public static DBStore loadDB(OzoneConfiguration configuration, File metaDir) - throws IOException { - return loadDB(configuration, metaDir, Optional.empty()); - } - - public static DBStore loadDB(OzoneConfiguration configuration, File metaDir, Optional maxOpenFiles) - throws IOException { - return loadDB(configuration, metaDir, OM_DB_NAME, false, - java.util.Optional.empty(), maxOpenFiles, true, true); - } - - public static DBStore loadDB(OzoneConfiguration configuration, File metaDir, - String dbName, boolean readOnly, - java.util.Optional - disableAutoCompaction, - java.util.Optional maxOpenFiles) - throws IOException { - return loadDB(configuration, metaDir, dbName, readOnly, - disableAutoCompaction, maxOpenFiles, true, true); + public static DBStore loadDB(OzoneConfiguration configuration, File metaDir, int maxOpenFiles) throws IOException { + return loadDB(configuration, metaDir, OM_DB_NAME, false, java.util.Optional.empty(), maxOpenFiles, true, true); } @SuppressWarnings("checkstyle:parameternumber") public static DBStore loadDB(OzoneConfiguration configuration, File metaDir, String dbName, boolean readOnly, - java.util.Optional - disableAutoCompaction, - java.util.Optional maxOpenFiles, + java.util.Optional disableAutoCompaction, + int maxOpenFiles, boolean enableCompactionDag, boolean createCheckpointDirs) throws IOException { @@ -626,10 +610,10 @@ public static DBStore loadDB(OzoneConfiguration configuration, File metaDir, .setPath(Paths.get(metaDir.getPath())) .setMaxFSSnapshots(maxFSSnapshots) .setEnableCompactionDag(enableCompactionDag) - .setCreateCheckpointDirs(createCheckpointDirs); + .setCreateCheckpointDirs(createCheckpointDirs) + .setMaxNumberOfOpenFiles(maxOpenFiles); disableAutoCompaction.ifPresent( dbStoreBuilder::disableDefaultCFAutoCompaction); - maxOpenFiles.ifPresent(dbStoreBuilder::setMaxNumberOfOpenFiles); return addOMTablesAndCodecs(dbStoreBuilder).build(); } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOMDBDefinition.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOMDBDefinition.java index 9ae85b0fcb6..680853cdc32 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOMDBDefinition.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOMDBDefinition.java @@ -27,13 +27,14 @@ import org.junit.jupiter.api.io.TempDir; import java.io.File; +import java.io.IOException; import java.nio.file.Path; import java.util.ArrayList; import java.util.Collection; +import java.util.List; /** - * Test that all the tables are covered both by OMDBDefinition - * as well as OmMetadataManagerImpl. + * Test that all the tables are covered both by OMDBDefinition and OmMetadataManagerImpl. */ public class TestOMDBDefinition { @@ -41,32 +42,33 @@ public class TestOMDBDefinition { private Path folder; @Test - public void testDBDefinition() throws Exception { + public void testDBDefinition() throws IOException { OzoneConfiguration configuration = new OzoneConfiguration(); File metaDir = folder.toFile(); - DBStore store = OmMetadataManagerImpl.loadDB(configuration, metaDir); + OMDBDefinition dbDef = OMDBDefinition.get(); // Get list of tables from DB Definitions - final Collection> columnFamilyDefinitions - = OMDBDefinition.get().getColumnFamilies(); + final Collection> columnFamilyDefinitions = dbDef.getColumnFamilies(); final int countOmDefTables = columnFamilyDefinitions.size(); - ArrayList missingDBDefTables = new ArrayList<>(); + List missingDBDefTables = new ArrayList<>(); - // Get list of tables from the RocksDB Store - final Collection missingOmDBTables = new ArrayList<>(store.getTableNames().values()); - missingOmDBTables.remove("default"); - int countOmDBTables = missingOmDBTables.size(); - // Remove the file if it is found in both the datastructures - for (DBColumnFamilyDefinition definition : columnFamilyDefinitions) { - if (!missingOmDBTables.remove(definition.getName())) { - missingDBDefTables.add(definition.getName()); + try (DBStore store = OmMetadataManagerImpl.loadDB(configuration, metaDir, -1)) { + // Get list of tables from the RocksDB Store + final Collection missingOmDBTables = new ArrayList<>(store.getTableNames().values()); + missingOmDBTables.remove("default"); + int countOmDBTables = missingOmDBTables.size(); + // Remove the file if it is found in both the datastructures + for (DBColumnFamilyDefinition definition : columnFamilyDefinitions) { + if (!missingOmDBTables.remove(definition.getName())) { + missingDBDefTables.add(definition.getName()); + } } - } - assertEquals(0, missingDBDefTables.size(), - "Tables in OmMetadataManagerImpl are:" + missingDBDefTables); - assertEquals(0, missingOmDBTables.size(), - "Tables missing in OMDBDefinition are:" + missingOmDBTables); - assertEquals(countOmDBTables, countOmDefTables); + assertEquals(0, missingDBDefTables.size(), + "Tables in OmMetadataManagerImpl are:" + missingDBDefTables); + assertEquals(0, missingOmDBTables.size(), + "Tables missing in OMDBDefinition are:" + missingOmDBTables); + assertEquals(countOmDBTables, countOmDefTables); + } } }