HDDS-11705. Snapshot operations on linked buckets should work on actual underlying bucket

dev-support/ci/selective_ci_checks.bats

@@ -190,14 +190,14 @@ load bats-assert/load.bash
 }
 
 @test "native test in other module" {
-  run dev-support/ci/selective_ci_checks.sh 822c0dee1a
+  run dev-support/ci/selective_ci_checks.sh 3115158830483c527fb978e1edc15473d81191d2
 
   assert_output -p 'basic-checks=["rat","author","checkstyle","findbugs","native"]'
   assert_output -p needs-build=true
   assert_output -p needs-compile=true
   assert_output -p needs-compose-tests=false
   assert_output -p needs-dependency-check=true
-  assert_output -p needs-integration-tests=false
+  assert_output -p needs-integration-tests=true
   assert_output -p needs-kubernetes-tests=false
 }
 

hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/SnapshotInfo.java

@@ -114,6 +114,9 @@ public static SnapshotStatus valueOf(SnapshotStatusProto status) {
   private UUID globalPreviousSnapshotId;
   private String snapshotPath; // snapshot mask
   private String checkpointDir;
+  private boolean isLinked;
+  private String linkedVolumeName;
+  private String linkedBucketName;
   /**
    * RocksDB's transaction sequence number at the time of checkpoint creation.
    */
@@ -148,6 +151,9 @@ private SnapshotInfo(Builder b) {
     this.exclusiveReplicatedSize = b.exclusiveReplicatedSize;
     this.deepCleanedDeletedDir = b.deepCleanedDeletedDir;
     this.lastTransactionInfo = b.lastTransactionInfo;
+    this.linkedVolumeName = b.linkedVolumeName;
+    this.linkedBucketName = b.linkedBucketName;
+    this.isLinked = b.isLinked;
   }
 
   public void setName(String name) {
@@ -246,6 +252,18 @@ public void setSstFiltered(boolean sstFiltered) {
     this.sstFiltered = sstFiltered;
   }
 
+  public void setLinked(boolean linked) {
+    isLinked = linked;
+  }
+
+  public void setLinkedBucketName(String linkedBucketName) {
+    this.linkedBucketName = linkedBucketName;
+  }
+
+  public void setLinkedVolumeName(String linkedVolumeName) {
+    this.linkedVolumeName = linkedVolumeName;
+  }
+
   public static org.apache.hadoop.ozone.om.helpers.SnapshotInfo.Builder
       newBuilder() {
     return new org.apache.hadoop.ozone.om.helpers.SnapshotInfo.Builder();
@@ -272,7 +290,10 @@ public SnapshotInfo.Builder toBuilder() {
         .setExclusiveSize(exclusiveSize)
         .setExclusiveReplicatedSize(exclusiveReplicatedSize)
         .setDeepCleanedDeletedDir(deepCleanedDeletedDir)
-        .setLastTransactionInfo(lastTransactionInfo);
+        .setLastTransactionInfo(lastTransactionInfo)
+        .setLinked(isLinked)
+        .setLinkedVolumeName(linkedVolumeName)
+        .setLinkedBucketName(linkedBucketName);
   }
 
   /**
@@ -299,6 +320,9 @@ public static class Builder {
     private long exclusiveReplicatedSize;
     private boolean deepCleanedDeletedDir;
     private ByteString lastTransactionInfo;
+    private boolean isLinked = false;
+    private String linkedVolumeName;
+    private String linkedBucketName;
 
     public Builder() {
       // default values
@@ -422,6 +446,21 @@ public Builder setLastTransactionInfo(ByteString lastTransactionInfo) {
       return this;
     }
 
+    public Builder setLinked(boolean linked) {
+      isLinked = linked;
+      return this;
+    }
+
+    public Builder setLinkedBucketName(String linkedBucketName) {
+      this.linkedBucketName = linkedBucketName;
+      return this;
+    }
+
+    public Builder setLinkedVolumeName(String linkedVolumeName) {
+      this.linkedVolumeName = linkedVolumeName;
+      return this;
+    }
+
     public SnapshotInfo build() {
       Preconditions.checkNotNull(name);
       return new SnapshotInfo(this);
@@ -459,6 +498,15 @@ public OzoneManagerProtocolProtos.SnapshotInfo getProtobuf() {
     if (lastTransactionInfo != null) {
       sib.setLastTransactionInfo(lastTransactionInfo);
     }
+    sib.setIsLinked(isLinked);
+
+    if (linkedVolumeName != null) {
+      sib.setLinkedVolumeName(linkedVolumeName);
+    }
+
+    if (linkedBucketName != null) {
+      sib.setLinkedBucketName(linkedBucketName);
+    }
 
     sib.setSnapshotPath(snapshotPath)
         .setCheckpointDir(checkpointDir)
@@ -532,6 +580,16 @@ public static SnapshotInfo getFromProtobuf(
       osib.setLastTransactionInfo(snapshotInfoProto.getLastTransactionInfo());
     }
 
+    if (snapshotInfoProto.hasLinkedVolumeName()) {
+      osib.setLinkedVolumeName(snapshotInfoProto.getLinkedVolumeName());
+    }
+
+    if (snapshotInfoProto.hasLinkedBucketName()) {
+      osib.setLinkedBucketName(snapshotInfoProto.getLinkedBucketName());
+    }
+
+    osib.setLinked(snapshotInfoProto.hasIsLinked() && snapshotInfoProto.getIsLinked());
+
     osib.setSnapshotPath(snapshotInfoProto.getSnapshotPath())
         .setCheckpointDir(snapshotInfoProto.getCheckpointDir())
         .setDbTxSequenceNumber(snapshotInfoProto.getDbTxSequenceNumber());
@@ -701,7 +759,10 @@ public boolean equals(Object o) {
         exclusiveSize == that.exclusiveSize &&
         exclusiveReplicatedSize == that.exclusiveReplicatedSize &&
         deepCleanedDeletedDir == that.deepCleanedDeletedDir &&
-        Objects.equals(lastTransactionInfo, that.lastTransactionInfo);
+        Objects.equals(lastTransactionInfo, that.lastTransactionInfo) &&
+        isLinked == that.isLinked &&
+        Objects.equals(linkedVolumeName, that.linkedVolumeName) &&
+        Objects.equals(linkedBucketName, that.linkedBucketName);
   }
 
   @Override
@@ -712,7 +773,8 @@ public int hashCode() {
         globalPreviousSnapshotId, snapshotPath, checkpointDir,
         deepClean, sstFiltered,
         referencedSize, referencedReplicatedSize,
-        exclusiveSize, exclusiveReplicatedSize, deepCleanedDeletedDir, lastTransactionInfo);
+        exclusiveSize, exclusiveReplicatedSize, deepCleanedDeletedDir, lastTransactionInfo,
+        linkedBucketName, linkedVolumeName, isLinked);
   }
 
   /**
@@ -746,6 +808,9 @@ public String toString() {
         ", exclusiveReplicatedSize: '" + exclusiveReplicatedSize + '\'' +
         ", deepCleanedDeletedDir: '" + deepCleanedDeletedDir + '\'' +
         ", lastTransactionInfo: '" + lastTransactionInfo + '\'' +
+        ", isLinked: '" + isLinked + '\'' +
+        ", linkedVolumeName: '" + linkedVolumeName + '\'' +
+        ", linkedBucketName: '" + linkedBucketName + '\'' +
         '}';
   }
 }

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestDataUtil.java

@@ -164,21 +164,57 @@ private static BucketLayout getDefaultBucketLayout(OzoneClient client) {
   public static OzoneBucket createBucket(OzoneClient client,
       String vol, BucketArgs bucketArgs, String bukName)
       throws IOException {
+    return createBucket(client, vol, bucketArgs, bukName, false);
+  }
+
+  public static OzoneBucket createBucket(OzoneClient client,
+                                         String vol, BucketArgs bucketArgs, String bukName,
+                                         boolean createLinkedBucket)
+      throws IOException {
     ObjectStore objectStore = client.getObjectStore();
     OzoneVolume volume = objectStore.getVolume(vol);
-    volume.createBucket(bukName, bucketArgs);
-    return volume.getBucket(bukName);
+    String sourceBucket = bukName;
+    if (createLinkedBucket) {
+      sourceBucket = bukName + RandomStringUtils.randomNumeric(5);
+    }
+    volume.createBucket(sourceBucket, bucketArgs);
+    OzoneBucket ozoneBucket = volume.getBucket(sourceBucket);
+    if (createLinkedBucket) {
+      ozoneBucket = createLinkedBucket(client, vol, sourceBucket, bukName);
+    }
+    return ozoneBucket;
+  }
+
+  public static OzoneBucket createLinkedBucket(OzoneClient client, String vol, String sourceBucketName,
+                                               String linkedBucketName) throws IOException {
+    BucketArgs.Builder bb = new BucketArgs.Builder()
+        .setStorageType(StorageType.DEFAULT)
+        .setVersioning(false)
+        .setSourceVolume(vol)
+        .setSourceBucket(sourceBucketName);
+    return createBucket(client, vol, bb.build(), linkedBucketName);
+  }
+
+  public static OzoneBucket createVolumeAndBucket(OzoneClient client,
+                                                  BucketLayout bucketLayout)
+      throws IOException {
+    return createVolumeAndBucket(client, bucketLayout, false);
   }
 
   public static OzoneBucket createVolumeAndBucket(OzoneClient client,
-      BucketLayout bucketLayout) throws IOException {
+      BucketLayout bucketLayout, boolean createLinkedBucket) throws IOException {
     final int attempts = 5;
     for (int i = 0; i < attempts; i++) {
       try {
         String volumeName = "volume" + RandomStringUtils.randomNumeric(5);
         String bucketName = "bucket" + RandomStringUtils.randomNumeric(5);
-        return createVolumeAndBucket(client, volumeName, bucketName,
+        OzoneBucket ozoneBucket = createVolumeAndBucket(client, volumeName, bucketName,
             bucketLayout);
+        if (createLinkedBucket) {
+          String targetBucketName = ozoneBucket.getName() + RandomStringUtils.randomNumeric(5);
+          ozoneBucket = createLinkedBucket(client, volumeName, bucketName, targetBucketName);
+        }
+        return ozoneBucket;
       } catch (OMException e) {
         if (e.getResult() != OMException.ResultCodes.VOLUME_ALREADY_EXISTS
             && e.getResult() != OMException.ResultCodes.BUCKET_ALREADY_EXISTS) {