HDDS-11705. Snapshot operations on linked buckets should work on actual underlying bucket

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestDataUtil.java

@@ -164,21 +164,57 @@ private static BucketLayout getDefaultBucketLayout(OzoneClient client) {
   public static OzoneBucket createBucket(OzoneClient client,
       String vol, BucketArgs bucketArgs, String bukName)
       throws IOException {
+    return createBucket(client, vol, bucketArgs, bukName, false);
+  }
+
+  public static OzoneBucket createBucket(OzoneClient client,
+                                         String vol, BucketArgs bucketArgs, String bukName,
+                                         boolean createLinkedBucket)
+      throws IOException {
     ObjectStore objectStore = client.getObjectStore();
     OzoneVolume volume = objectStore.getVolume(vol);
-    volume.createBucket(bukName, bucketArgs);
-    return volume.getBucket(bukName);
+    String sourceBucket = bukName;
+    if (createLinkedBucket) {
+      sourceBucket = bukName + RandomStringUtils.randomNumeric(5);
+    }
+    volume.createBucket(sourceBucket, bucketArgs);
+    OzoneBucket ozoneBucket = volume.getBucket(sourceBucket);
+    if (createLinkedBucket) {
+      ozoneBucket = createLinkedBucket(client, vol, sourceBucket, bukName);
+    }
+    return ozoneBucket;
+  }
+
+  public static OzoneBucket createLinkedBucket(OzoneClient client, String vol, String sourceBucketName,
+                                               String linkedBucketName) throws IOException {
+    BucketArgs.Builder bb = new BucketArgs.Builder()
+        .setStorageType(StorageType.DEFAULT)
+        .setVersioning(false)
+        .setSourceVolume(vol)
+        .setSourceBucket(sourceBucketName);
+    return createBucket(client, vol, bb.build(), linkedBucketName);
+  }
+
+  public static OzoneBucket createVolumeAndBucket(OzoneClient client,
+                                                  BucketLayout bucketLayout)
+      throws IOException {
+    return createVolumeAndBucket(client, bucketLayout, false);
   }
 
   public static OzoneBucket createVolumeAndBucket(OzoneClient client,
-      BucketLayout bucketLayout) throws IOException {
+      BucketLayout bucketLayout, boolean createLinkedBucket) throws IOException {
     final int attempts = 5;
     for (int i = 0; i < attempts; i++) {
       try {
         String volumeName = "volume" + RandomStringUtils.randomNumeric(5);
         String bucketName = "bucket" + RandomStringUtils.randomNumeric(5);
-        return createVolumeAndBucket(client, volumeName, bucketName,
+        OzoneBucket ozoneBucket = createVolumeAndBucket(client, volumeName, bucketName,
             bucketLayout);
+        if (createLinkedBucket) {
+          String targetBucketName = ozoneBucket.getName() + RandomStringUtils.randomNumeric(5);
+          ozoneBucket = createLinkedBucket(client, volumeName, bucketName, targetBucketName);
+        }
+        return ozoneBucket;
       } catch (OMException e) {
         if (e.getResult() != OMException.ResultCodes.VOLUME_ALREADY_EXISTS
             && e.getResult() != OMException.ResultCodes.BUCKET_ALREADY_EXISTS) {

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFileSystem.java

@@ -38,6 +38,7 @@
 import org.apache.hadoop.ozone.client.OzoneBucket;
 import org.apache.hadoop.ozone.client.OzoneClient;
 import org.apache.hadoop.ozone.client.OzoneKey;
+import org.apache.hadoop.ozone.client.OzoneSnapshot;
 import org.apache.hadoop.ozone.client.OzoneVolume;
 import org.apache.hadoop.ozone.client.io.OzoneInputStream;
 import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
@@ -54,9 +55,9 @@
 import org.apache.ozone.test.GenericTestUtils;
 import org.junit.jupiter.api.AfterAll;
 import org.junit.jupiter.api.AfterEach;
-import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestInstance;
 import org.junit.jupiter.api.Timeout;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -70,9 +71,11 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.Iterator;
 import java.util.LinkedList;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
 import java.util.TreeSet;
 import java.util.UUID;
@@ -99,6 +102,7 @@
  * Abstract class for OmSnapshot file system tests.
  */
 @Timeout(120)
+@TestInstance(TestInstance.Lifecycle.PER_CLASS)
 public abstract class TestOmSnapshotFileSystem {
   protected static final String VOLUME_NAME =
       "volume" + RandomStringUtils.randomNumeric(5);
@@ -107,26 +111,29 @@ public abstract class TestOmSnapshotFileSystem {
   protected static final String BUCKET_NAME_LEGACY =
       "bucket-legacy-" + RandomStringUtils.randomNumeric(5);
 
-  private static MiniOzoneCluster cluster = null;
-  private static OzoneClient client;
-  private static ObjectStore objectStore;
-  private static OzoneConfiguration conf;
-  private static OzoneManagerProtocol writeClient;
-  private static OzoneManager ozoneManager;
-  private static String keyPrefix;
+  private MiniOzoneCluster cluster = null;
+  private OzoneClient client;
+  private ObjectStore objectStore;
+  private OzoneConfiguration conf;
+  private OzoneManagerProtocol writeClient;
+  private OzoneManager ozoneManager;
+  private String keyPrefix;
   private final String bucketName;
+  private final boolean createLinkedBuckets;
   private FileSystem fs;
   private OzoneFileSystem o3fs;
+  private Map<String, String> linkedBucketMaps = new HashMap<>();
 
   private static final Logger LOG =
       LoggerFactory.getLogger(TestOmSnapshot.class);
 
-  public TestOmSnapshotFileSystem(String bucketName) {
+  public TestOmSnapshotFileSystem(String bucketName, boolean createLinkedBuckets) throws Exception {
     this.bucketName = bucketName;
+    this.createLinkedBuckets = createLinkedBuckets;
+    init();
   }
 
-  @BeforeAll
-  public static void init() throws Exception {
+  private void init() throws Exception {
     conf = new OzoneConfiguration();
     conf.setBoolean(OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM_PATHS, true);
     cluster = MiniOzoneCluster.newBuilder(conf).build();
@@ -138,12 +145,20 @@ public static void init() throws Exception {
     ozoneManager = cluster.getOzoneManager();
 
     TestDataUtil.createVolume(client, VOLUME_NAME);
-    TestDataUtil.createBucket(client, VOLUME_NAME,
+    OzoneBucket bucket = TestDataUtil.createBucket(client, VOLUME_NAME,
         new BucketArgs.Builder().setBucketLayout(FILE_SYSTEM_OPTIMIZED).build(),
-        BUCKET_NAME_FSO);
-    TestDataUtil.createBucket(client, VOLUME_NAME,
+        BUCKET_NAME_FSO, createLinkedBuckets);
+    if (createLinkedBuckets) {
+      linkedBucketMaps.put(bucket.getName(), bucket.getSourceBucket());
+    }
+    OzoneBucket ozoneBucket = TestDataUtil.createBucket(client, VOLUME_NAME,
         new BucketArgs.Builder().setBucketLayout(LEGACY).build(),
-        BUCKET_NAME_LEGACY);
+        BUCKET_NAME_LEGACY, createLinkedBuckets);
+    if (createLinkedBuckets) {
+      linkedBucketMaps.put(bucket.getName(), bucket.getSourceBucket());
+    }
+
+
 
     // stop the deletion services so that keys can still be read
     KeyManagerImpl keyManager = (KeyManagerImpl) ozoneManager.getKeyManager();
@@ -163,7 +178,7 @@ public void setupFsClient() throws IOException {
   }
 
   @AfterAll
-  public static void tearDown() throws Exception {
+  void tearDown() {
     IOUtils.closeQuietly(client);
     if (cluster != null) {
       cluster.shutdown();
@@ -273,7 +288,7 @@ public void testListKeysAtDifferentLevels() throws Exception {
     deleteSnapshot(snapshotName);
     String expectedMessage = String.format("Unable to load snapshot. " +
             "Snapshot with table key '/%s/%s/%s' is no longer active",
-        VOLUME_NAME, bucketName, snapshotName);
+        VOLUME_NAME, linkedBucketMaps.getOrDefault(bucketName, bucketName), snapshotName);
     OMException exception = assertThrows(OMException.class,
         () -> ozoneBucket.listKeys(keyPrefix + "a/", null));
     assertEquals(expectedMessage, exception.getMessage());
@@ -376,7 +391,7 @@ private void createKey(OzoneBucket ozoneBucket, String key, int length,
     assertEquals(inputString, new String(read, StandardCharsets.UTF_8));
   }
 
-  private static void setKeyPrefix(String s) {
+  private void setKeyPrefix(String s) {
     keyPrefix = s;
   }
 
@@ -493,21 +508,21 @@ public void testListStatus() throws Exception {
         () -> fs.listStatus(snapshotRoot1));
     assertEquals(String.format("Unable to load snapshot. " +
             "Snapshot with table key '/%s/%s/%s' is no longer active",
-        VOLUME_NAME, bucketName, snapshotName1), exception1.getMessage());
+        VOLUME_NAME, linkedBucketMaps.getOrDefault(bucketName, bucketName), snapshotName1), exception1.getMessage());
 
     deleteSnapshot(snapshotName2);
     FileNotFoundException exception2 = assertThrows(FileNotFoundException.class,
         () -> fs.listStatus(snapshotRoot2));
     assertEquals(String.format("Unable to load snapshot. " +
             "Snapshot with table key '/%s/%s/%s' is no longer active",
-        VOLUME_NAME, bucketName, snapshotName2), exception2.getMessage());
+        VOLUME_NAME, linkedBucketMaps.getOrDefault(bucketName, bucketName), snapshotName2), exception2.getMessage());
 
     deleteSnapshot(snapshotName3);
     FileNotFoundException exception3 = assertThrows(FileNotFoundException.class,
         () -> fs.listStatus(snapshotParent3));
     assertEquals(String.format("Unable to load snapshot. " +
             "Snapshot with table key '/%s/%s/%s' is no longer active",
-        VOLUME_NAME, bucketName, snapshotName3), exception3.getMessage());
+        VOLUME_NAME, linkedBucketMaps.getOrDefault(bucketName, bucketName), snapshotName3), exception3.getMessage());
   }
 
   @Test
@@ -542,7 +557,7 @@ public void testListStatusWithIntermediateDir() throws Exception {
         () -> fs.listStatus(snapshotParent));
     assertEquals(String.format("Unable to load snapshot. " +
             "Snapshot with table key '/%s/%s/%s' is no longer active",
-        VOLUME_NAME, bucketName, snapshotName), exception.getMessage());
+        VOLUME_NAME, linkedBucketMaps.getOrDefault(bucketName, bucketName), snapshotName), exception.getMessage());
   }
 
   @Test
@@ -578,7 +593,7 @@ public void testGetFileStatus() throws Exception {
         () -> fs.listStatus(snapshotParent));
     assertEquals(String.format("Unable to load snapshot. " +
             "Snapshot with table key '/%s/%s/%s' is no longer active",
-        VOLUME_NAME, bucketName, snapshotName), exception.getMessage());
+        VOLUME_NAME, linkedBucketMaps.getOrDefault(bucketName, bucketName), snapshotName), exception.getMessage());
   }
 
   @Test
@@ -619,7 +634,7 @@ void testReadFileFromSnapshot() throws Exception {
         () -> fs.open(fileInSnapshot));
     assertEquals(String.format("FILE_NOT_FOUND: Unable to load snapshot. " +
             "Snapshot with table key '/%s/%s/%s' is no longer active",
-        VOLUME_NAME, bucketName, snapshotName), exception.getMessage());
+        VOLUME_NAME, linkedBucketMaps.getOrDefault(bucketName, bucketName), snapshotName), exception.getMessage());
   }
 
   private void createAndCommitKey(String keyName) throws IOException {
@@ -669,7 +684,7 @@ public void testListStatusOnRoot() throws Exception {
         () -> fs.listStatus(snapshotRoot));
     assertEquals(String.format("Unable to load snapshot. " +
             "Snapshot with table key '/%s/%s/%s' is no longer active",
-        VOLUME_NAME, bucketName, snapshotName), exception.getMessage());
+        VOLUME_NAME, linkedBucketMaps.getOrDefault(bucketName, bucketName), snapshotName), exception.getMessage());
   }
 
   /**
@@ -726,7 +741,7 @@ public void testListStatusOnLargeDirectory() throws Exception {
         () -> fs.listStatus(snapshotRoot));
     assertEquals(String.format("Unable to load snapshot. " +
             "Snapshot with table key '/%s/%s/%s' is no longer active",
-        VOLUME_NAME, bucketName, snapshotName), exception.getMessage());
+        VOLUME_NAME, linkedBucketMaps.getOrDefault(bucketName, bucketName), snapshotName), exception.getMessage());
   }
 
   private String createSnapshot(String snapshotName)
@@ -736,9 +751,10 @@ private String createSnapshot(String snapshotName)
     writeClient.createSnapshot(VOLUME_NAME, bucketName, snapshotName);
 
     // wait till the snapshot directory exists
+    OzoneSnapshot snapshot = objectStore.getSnapshotInfo(VOLUME_NAME, bucketName, snapshotName);
     SnapshotInfo snapshotInfo = ozoneManager.getMetadataManager()
         .getSnapshotInfoTable()
-        .get(SnapshotInfo.getTableKey(VOLUME_NAME, bucketName, snapshotName));
+        .get(SnapshotInfo.getTableKey(snapshot.getVolumeName(), snapshot.getBucketName(), snapshotName));
     String snapshotDirName = getSnapshotPath(conf, snapshotInfo) +
         OM_KEY_PREFIX + "CURRENT";
     GenericTestUtils.waitFor(() -> new File(snapshotDirName).exists(),

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFileSystemFso.java → hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFileSystemFsoWithLinkedBuckets.java

@@ -24,8 +24,8 @@
  * OmSnapshot file system tests for FSO.
  */
 @Timeout(120)
-public class TestOmSnapshotFileSystemFso extends TestOmSnapshotFileSystem {
-  TestOmSnapshotFileSystemFso() {
-    super(BUCKET_NAME_FSO);
+public class TestOmSnapshotFileSystemFsoWithLinkedBuckets extends TestOmSnapshotFileSystem {
+  TestOmSnapshotFileSystemFsoWithLinkedBuckets() throws Exception {
+    super(BUCKET_NAME_FSO, true);
   }
 }

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFileSystemFsoWithoutLinkedBuckets.java

@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om.snapshot;
+
+import org.junit.jupiter.api.Timeout;
+
+/**
+ * OmSnapshot file system tests for FSO.
+ */
+@Timeout(120)
+public class TestOmSnapshotFileSystemFsoWithoutLinkedBuckets extends TestOmSnapshotFileSystem {
+  TestOmSnapshotFileSystemFsoWithoutLinkedBuckets() throws Exception {
+    super(BUCKET_NAME_FSO, false);
+  }
+}

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFileSystemLegacy.java → hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFileSystemLegacyWithLinkedBuckets.java

@@ -24,8 +24,8 @@
  * OmSnapshot file system tests for Legacy.
  */
 @Timeout(120)
-public class TestOmSnapshotFileSystemLegacy extends TestOmSnapshotFileSystem {
-  TestOmSnapshotFileSystemLegacy() {
-    super(BUCKET_NAME_LEGACY);
+public class TestOmSnapshotFileSystemLegacyWithLinkedBuckets extends TestOmSnapshotFileSystem {
+  TestOmSnapshotFileSystemLegacyWithLinkedBuckets() throws Exception {
+    super(BUCKET_NAME_LEGACY, true);
   }
 }

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFileSystemLegacyWithoutLinkedBuckets.java

@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om.snapshot;
+
+import org.junit.jupiter.api.Timeout;
+
+/**
+ * OmSnapshot file system tests for Legacy.
+ */
+@Timeout(120)
+public class TestOmSnapshotFileSystemLegacyWithoutLinkedBuckets extends TestOmSnapshotFileSystem {
+  TestOmSnapshotFileSystemLegacyWithoutLinkedBuckets() throws Exception {
+    super(BUCKET_NAME_LEGACY, false);
+  }
+}

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFsoWithNativeLib.java → hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFsoWithNativeLibWithLinkedBuckets.java

@@ -29,8 +29,8 @@
  */
 @Native(ROCKS_TOOLS_NATIVE_LIBRARY_NAME)
 @Timeout(300)
-class TestOmSnapshotFsoWithNativeLib extends TestOmSnapshot {
-  TestOmSnapshotFsoWithNativeLib() throws Exception {
-    super(FILE_SYSTEM_OPTIMIZED, false, false, false);
+class TestOmSnapshotFsoWithNativeLibWithLinkedBuckets extends TestOmSnapshot {
+  TestOmSnapshotFsoWithNativeLibWithLinkedBuckets() throws Exception {
+    super(FILE_SYSTEM_OPTIMIZED, false, false, false, true);
   }
 }