Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

HDDS-11914. Snapshot diff should not filter SST Files based by reading SST file reader #7563

Merged
merged 9 commits into from
Dec 13, 2024
Prev Previous commit
Next Next commit
HDDS-11914. Fix test cases
Change-Id: I189e26e59142ef4f6d1f035e6663bcc1639a59f4
swamirishi committed Dec 12, 2024
commit b0a00a47fc365fb20d5a905d6d24fb5b05441548
Original file line number Diff line number Diff line change
@@ -580,8 +580,8 @@ private void createLink(Path link, Path source) {
private long getSSTFileSummary(String filename)
throws RocksDBException, FileNotFoundException {

if (!filename.endsWith(ManagedRocksDB.SST_FILE_EXTENSION)) {
filename += ManagedRocksDB.SST_FILE_EXTENSION;
if (!filename.endsWith(SST_FILE_EXTENSION)) {
filename += SST_FILE_EXTENSION;
}

try (ManagedOptions option = new ManagedOptions();
@@ -599,8 +599,8 @@ private long getSSTFileSummary(String filename)

private String getAbsoluteSstFilePath(String filename)
throws FileNotFoundException {
if (!filename.endsWith(ManagedRocksDB.SST_FILE_EXTENSION)) {
filename += ManagedRocksDB.SST_FILE_EXTENSION;
if (!filename.endsWith(SST_FILE_EXTENSION)) {
filename += SST_FILE_EXTENSION;
}
File sstFile = new File(sstBackupDir + filename);
File sstFileInActiveDB = new File(activeDBLocationStr + filename);
@@ -623,7 +623,7 @@ private String trimSSTFilename(String filename) {
LOG.error(errorMsg);
throw new RuntimeException(errorMsg);
}
if (!filename.endsWith(ManagedRocksDB.SST_FILE_EXTENSION)) {
if (!filename.endsWith(SST_FILE_EXTENSION)) {
final String errorMsg = String.format(
"Invalid extension of file: '%s'. Expected '%s'",
filename, SST_FILE_EXTENSION_LENGTH);
@@ -793,7 +793,7 @@ private String getSSTFullPath(String sstFilenameWithoutExtension,

// Try to locate the SST in the backup dir first
final Path sstPathInBackupDir = Paths.get(sstBackupDir,
sstFilenameWithoutExtension + ManagedRocksDB.SST_FILE_EXTENSION);
sstFilenameWithoutExtension + SST_FILE_EXTENSION);
if (Files.exists(sstPathInBackupDir)) {
return sstPathInBackupDir.toString();
}
@@ -803,7 +803,7 @@ private String getSSTFullPath(String sstFilenameWithoutExtension,
// src DB directory or destDB directory
for (String dbPath : dbPaths) {
final Path sstPathInDBDir = Paths.get(dbPath,
sstFilenameWithoutExtension + ManagedRocksDB.SST_FILE_EXTENSION);
sstFilenameWithoutExtension + SST_FILE_EXTENSION);
if (Files.exists(sstPathInDBDir)) {
return sstPathInDBDir.toString();
}
@@ -838,7 +838,7 @@ public synchronized List<String> getSSTDiffListWithFullPath(DifferSnapshotInfo s
sst -> {
String sstFullPath = getSSTFullPath(sst, src.getDbPath(), dest.getDbPath());
Path link = Paths.get(sstFilesDirForSnapDiffJob,
sst + ManagedRocksDB.SST_FILE_EXTENSION);
sst + SST_FILE_EXTENSION);
Path srcFile = Paths.get(sstFullPath);
createLink(link, srcFile);
return link.toString();
@@ -1231,7 +1231,7 @@ private synchronized void removeKeyFromCompactionLogTable(
private void removeSstFiles(Set<String> sstFileNodes) {
for (String sstFileNode: sstFileNodes) {
File file =
new File(sstBackupDir + "/" + sstFileNode + ManagedRocksDB.SST_FILE_EXTENSION);
new File(sstBackupDir + "/" + sstFileNode + SST_FILE_EXTENSION);
try {
Files.deleteIfExists(file.toPath());
} catch (IOException exception) {
Original file line number Diff line number Diff line change
@@ -95,7 +95,7 @@
import static org.apache.ozone.rocksdiff.RocksDBCheckpointDiffer.COMPACTION_LOG_FILE_NAME_SUFFIX;
import static org.apache.ozone.rocksdiff.RocksDBCheckpointDiffer.DEBUG_DAG_LIVE_NODES;
import static org.apache.ozone.rocksdiff.RocksDBCheckpointDiffer.DEBUG_READ_ALL_DB_KEYS;
import static org.apache.hadoop.hdds.utils.db.managed.ManagedRocksDB.SST_FILE_EXTENSION;
import static org.apache.ozone.rocksdiff.RocksDBCheckpointDiffer.SST_FILE_EXTENSION;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
Original file line number Diff line number Diff line change
@@ -69,6 +69,7 @@
import org.apache.ratis.util.TimeDuration;
import jakarta.annotation.Nonnull;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Assertions;
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: please use static import for assertions and mocks (see HDDS-9961).

import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
@@ -82,6 +83,7 @@
import org.mockito.Mock;
import org.mockito.MockedConstruction;
import org.mockito.MockedStatic;
import org.mockito.Mockito;
import org.mockito.junit.jupiter.MockitoExtension;
import org.mockito.junit.jupiter.MockitoSettings;
import org.mockito.quality.Strictness;
@@ -1512,6 +1514,27 @@ private void setupMocksForRunningASnapDiff(
when(bucketInfoTable.get(bucketKey)).thenReturn(bucketInfo);
}

@Test
public void testGetDeltaFilesWithFullDiff() throws IOException {
SnapshotDiffManager spy = spy(snapshotDiffManager);
OmSnapshot fromSnapshot = getMockedOmSnapshot(UUID.randomUUID());
OmSnapshot toSnapshot = getMockedOmSnapshot(UUID.randomUUID());
Mockito.doAnswer(invocation -> {
OmSnapshot snapshot = invocation.getArgument(0);
if (snapshot == fromSnapshot) {
return Sets.newHashSet("1", "2", "3");
}
if (snapshot == toSnapshot) {
return Sets.newHashSet("3", "4", "5");
}
return Sets.newHashSet("6", "7", "8");
}).when(spy).getSSTFileListForSnapshot(Mockito.any(OmSnapshot.class),
Mockito.anyList());
Set<String> deltaFiles = spy.getDeltaFiles(fromSnapshot, toSnapshot, Collections.emptyList(), snapshotInfo,
snapshotInfo, true, Collections.emptyMap(), null);
Assertions.assertEquals(Sets.newHashSet("1", "2", "3", "4", "5"), deltaFiles);
}

@Test
public void testGetSnapshotDiffReportHappyCase() throws Exception {
SnapshotInfo fromSnapInfo = snapshotInfo;