HDDS-11697. Implement ListStatus Light API implement and integrate it…

… with Ozone Filesystem API Change-Id: I7695087b00fc9f16bfb68f1aaae4bc1e7e5323f8
apache · Nov 16, 2024 · 37ada7a · 37ada7a
1 parent dd22dbe
commit 37ada7a
Show file tree

Hide file tree

Showing 19 changed files with 260 additions and 48 deletions.
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java
@@ -985,8 +985,23 @@ public OzoneDataStreamOutput createStreamFile(String keyName, long size,
    */
   public List<OzoneFileStatus> listStatus(String keyName, boolean recursive,
       String startKey, long numEntries) throws IOException {
-    return proxy
-        .listStatus(volumeName, name, keyName, recursive, startKey, numEntries);
+    return proxy.listStatus(volumeName, name, keyName, recursive, startKey, numEntries);
+  }
+
+  /**
+   * List the lightweight status for a file or a directory and its contents.
+   *
+   * @param keyName    Absolute path of the entry to be listed
+   * @param recursive  For a directory if true all the descendants of a
+   *                   particular directory are listed
+   * @param startKey   Key from which listing needs to start. If startKey exists
+   *                   its status is included in the final list.
+   * @param numEntries Number of entries to list from the start key
+   * @return list of file status
+   */
+  public List<OzoneFileStatusLight> listStatusLight(String keyName, boolean recursive,
+      String startKey, long numEntries) throws IOException {
+    return proxy.listStatusLight(volumeName, name, keyName, recursive, startKey, numEntries);
   }
 
   /**

diff --git a/...op-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java b/...op-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
@@ -989,6 +989,22 @@ List<OzoneFileStatus> listStatus(String volumeName, String bucketName,
       String keyName, boolean recursive, String startKey,
       long numEntries, boolean allowPartialPrefixes) throws IOException;
 
+  /**
+   * Lightweight listStatus API.
+   *
+   * @param volumeName Volume name
+   * @param bucketName Bucket name
+   * @param keyName    Absolute path of the entry to be listed
+   * @param recursive  For a directory if true all the descendants of a
+   *                   particular directory are listed
+   * @param startKey   Key from which listing needs to start. If startKey exists
+   *                   its status is included in the final list.
+   * @param numEntries Number of entries to list from the start key
+   * @return list of file status
+   */
+  List<OzoneFileStatusLight> listStatusLight(String volumeName,
+      String bucketName, String keyName, boolean recursive, String startKey, long numEntries) throws IOException;
+
   /**
    * Lightweight listStatus API.
    *

diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
@@ -2311,6 +2311,22 @@ public List<OzoneFileStatus> listStatus(String volumeName, String bucketName,
             allowPartialPrefixes);
   }
 
+  @Override
+  public List<OzoneFileStatusLight> listStatusLight(String volumeName, String bucketName, String keyName,
+      boolean recursive, String startKey, long numEntries) throws IOException {
+    OmKeyArgs keyArgs = prepareOmKeyArgs(volumeName, bucketName, keyName);
+    if (omVersion.compareTo(OzoneManagerVersion.LIGHTWEIGHT_LIST_STATUS) >= 0) {
+      return ozoneManagerClient.listStatusLight(keyArgs, recursive, startKey,
+          numEntries);
+    } else {
+      return ozoneManagerClient.listStatus(keyArgs, recursive, startKey,
+              numEntries)
+          .stream()
+          .map(OzoneFileStatusLight::fromOzoneFileStatus)
+          .collect(Collectors.toList());
+    }
+  }
+
   @Override
   public List<OzoneFileStatusLight> listStatusLight(String volumeName,
       String bucketName, String keyName, boolean recursive, String startKey,

diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/BasicOmKeyInfo.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/BasicOmKeyInfo.java
@@ -110,6 +110,10 @@ public String getOwnerName() {
     return ownerName;
   }
 
+  public long getReplicatedSize() {
+    return QuotaUtil.getReplicatedSize(getDataSize(), replicationConfig);
+  }
+
   /**
    * Builder of BasicOmKeyInfo.
    */

diff --git a/...-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java b/...-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java
@@ -986,6 +986,20 @@ List<OzoneFileStatus> listStatus(OmKeyArgs keyArgs, boolean recursive,
                                    boolean allowPartialPrefixes)
       throws IOException;
 
+  /**
+   * Lightweight listStatus API.
+   *
+   * @param keyArgs    Key args
+   * @param recursive  For a directory if true all the descendants of a
+   *                   particular directory are listed
+   * @param startKey   Key from which listing needs to start. If startKey exists
+   *                   its status is included in the final list.
+   * @param numEntries Number of entries to list from the start key
+   * @return list of file status
+   */
+  List<OzoneFileStatusLight> listStatusLight(OmKeyArgs keyArgs, boolean recursive, String startKey,
+                                             long numEntries) throws IOException;
+
   /**
    * Lightweight listStatus API.
    *

diff --git a/...ava/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java b/...ava/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
@@ -2374,6 +2374,12 @@ public List<OzoneFileStatus> listStatus(OmKeyArgs args, boolean recursive,
     return statusList;
   }
 
+  @Override
+  public List<OzoneFileStatusLight> listStatusLight(OmKeyArgs args,
+      boolean recursive, String startKey, long numEntries) throws IOException {
+    return listStatusLight(args, recursive, startKey, numEntries, false);
+  }
+
   @Override
   public List<OzoneFileStatusLight> listStatusLight(OmKeyArgs args,
       boolean recursive, String startKey, long numEntries,

diff --git a/...ntegration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractOzoneFileSystemTest.java b/...ntegration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractOzoneFileSystemTest.java
@@ -2112,6 +2112,41 @@ void testListStatus2() throws IOException {
     }
   }
 
+  @Test
+  public void testOzoneManagerListLocatedStatusAndListStatus() throws IOException {
+    String data = RandomStringUtils.randomAlphanumeric(20);
+    String directory = RandomStringUtils.randomAlphanumeric(5);
+    String filePath = RandomStringUtils.randomAlphanumeric(5);
+    Path path = createPath("/" + directory + "/" + filePath);
+    try (FSDataOutputStream stream = fs.create(path)) {
+      stream.writeBytes(data);
+    }
+    RemoteIterator<LocatedFileStatus> listLocatedStatus = fs.listLocatedStatus(path);
+    int count = 0;
+    while (listLocatedStatus.hasNext()) {
+      LocatedFileStatus locatedFileStatus = listLocatedStatus.next();
+      assertTrue(locatedFileStatus.getBlockLocations().length >= 1);
+
+      for (BlockLocation blockLocation : locatedFileStatus.getBlockLocations()) {
+        assertTrue(blockLocation.getNames().length >= 1);
+        assertTrue(blockLocation.getHosts().length >= 1);
+      }
+      count++;
+    }
+    assertEquals(1, count);
+    count = 0;
+    RemoteIterator<FileStatus> listStatus = fs.listStatusIterator(path);
+    while (listStatus.hasNext()) {
+      FileStatus fileStatus = listStatus.next();
+      assertFalse(fileStatus instanceof LocatedFileStatus);
+      count++;
+    }
+    assertEquals(1, count);
+    FileStatus[] fileStatuses = fs.listStatus(path.getParent());
+    assertEquals(1, fileStatuses.length);
+    assertFalse(fileStatuses[0] instanceof LocatedFileStatus);
+  }
+
   @Test
   void testOzoneManagerFileSystemInterface() throws IOException {
     String dirPath = RandomStringUtils.randomAlphanumeric(5);

diff --git a/...tion-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractRootedOzoneFileSystemTest.java b/...tion-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractRootedOzoneFileSystemTest.java
@@ -1071,7 +1071,7 @@ private void listStatusRecursiveHelper(Path curPath, List<FileStatus> result)
   private List<FileStatus> callAdapterListStatus(String pathStr,
       boolean recursive, String startPath, long numEntries) throws IOException {
     return adapter.listStatus(pathStr, recursive, startPath, numEntries,
-        ofs.getUri(), ofs.getWorkingDirectory(), ofs.getUsername())
+        ofs.getUri(), ofs.getWorkingDirectory(), ofs.getUsername(), false)
         .stream().map(ofs::convertFileStatus).collect(Collectors.toList());
   }
 

diff --git a/.../test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientWithKeyLatestVersion.java b/.../test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientWithKeyLatestVersion.java
@@ -31,6 +31,7 @@
 import org.apache.hadoop.ozone.client.OzoneClientFactory;
 import org.apache.hadoop.ozone.client.OzoneVolume;
 import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus;
+import org.apache.hadoop.ozone.om.helpers.OzoneFileStatusLight;
 import org.junit.jupiter.api.AfterAll;
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.TestInstance;
@@ -151,5 +152,11 @@ private void assertListStatus(OzoneBucket bucket, String keyName,
 
     List<?> versions = files.get(0).getKeyInfo().getKeyLocationVersions();
     assertEquals(expectedVersionCount, versions.size());
+
+    List<OzoneFileStatusLight> lightFiles = bucket.listStatusLight(keyName, false, "", 1);
+
+    assertNotNull(lightFiles);
+    assertEquals(1, lightFiles.size());
+
   }
 }
diff --git a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
@@ -1254,6 +1254,7 @@ message ListStatusRequest {
     required string startKey = 3;
     required uint64 numEntries = 4;
     optional bool allowPartialPrefix = 5;
+    optional bool lite = 6;
 }
 
 message ListStatusResponse {
@@ -1473,6 +1474,7 @@ message ListKeysRequest {
     optional string startKey = 3;
     optional string prefix = 4;
     optional int32 count = 5;
+    optional bool lite = 6;
 }
 
 message ListKeysResponse {

diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataReader.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataReader.java
@@ -264,7 +264,7 @@ public List<OzoneFileStatusLight> listStatusLight(OmKeyArgs args,
         .map(OzoneFileStatusLight::fromOzoneFileStatus)
         .collect(Collectors.toList());
   }
-  
+
   @Override
   public OzoneFileStatus getFileStatus(OmKeyArgs args) throws IOException {
     ResolvedBucket bucket = ozoneManager.resolveBucketLink(args);
@@ -501,7 +501,7 @@ void checkAcls(ResourceType resType, StoreType store,
             ozoneManager.getOmRpcServerAddr().getHostName());
   }
 
-  
+
   /**
    * CheckAcls for the ozone object.
    *

diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
@@ -3802,6 +3802,12 @@ public List<OzoneFileStatus> listStatus(OmKeyArgs args, boolean recursive,
     }
   }
 
+  @Override
+  public List<OzoneFileStatusLight> listStatusLight(OmKeyArgs args,
+      boolean recursive, String startKey, long numEntries) throws IOException {
+    return listStatusLight(args, recursive, startKey, numEntries, false);
+  }
+
   @Override
   public List<OzoneFileStatusLight> listStatusLight(OmKeyArgs args,
       boolean recursive, String startKey, long numEntries,

diff --git a/.../ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneClientAdapterImpl.java b/.../ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneClientAdapterImpl.java
@@ -69,6 +69,7 @@
 import org.apache.hadoop.ozone.client.rpc.RpcClient;
 import org.apache.hadoop.ozone.container.common.helpers.BlockData;
 import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.ozone.om.helpers.BasicOmKeyInfo;
 import org.apache.hadoop.ozone.om.helpers.BucketLayout;
 import org.apache.hadoop.ozone.om.helpers.LeaseKeyInfo;
 import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
@@ -77,6 +78,7 @@
 import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup;
 import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils;
 import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus;
+import org.apache.hadoop.ozone.om.helpers.OzoneFileStatusLight;
 import org.apache.hadoop.ozone.security.OzoneTokenIdentifier;
 import org.apache.hadoop.ozone.snapshot.SnapshotDiffReportOzone;
 import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse;
@@ -436,15 +438,22 @@ public Iterator<BasicKeyInfo> listKeys(String pathKey) throws IOException {
   @Override
   public List<FileStatusAdapter> listStatus(String keyName, boolean recursive,
       String startKey, long numEntries, URI uri,
-      Path workingDir, String username) throws IOException {
+      Path workingDir, String username, boolean lite) throws IOException {
     try {
       incrementCounter(Statistic.OBJECTS_LIST, 1);
-      List<OzoneFileStatus> statuses = bucket
-          .listStatus(keyName, recursive, startKey, numEntries);
-
       List<FileStatusAdapter> result = new ArrayList<>();
-      for (OzoneFileStatus status : statuses) {
-        result.add(toFileStatusAdapter(status, username, uri, workingDir));
+      if (lite) {
+        List<OzoneFileStatusLight> statuses = bucket
+            .listStatusLight(keyName, recursive, startKey, numEntries);
+        for (OzoneFileStatusLight status : statuses) {
+          result.add(toFileStatusAdapter(status, username, uri, workingDir));
+        }
+      } else {
+        List<OzoneFileStatus> statuses = bucket
+            .listStatus(keyName, recursive, startKey, numEntries);
+        for (OzoneFileStatus status : statuses) {
+          result.add(toFileStatusAdapter(status, username, uri, workingDir));
+        }
       }
       return result;
     } catch (OMException e) {
@@ -549,6 +558,31 @@ private FileStatusAdapter toFileStatusAdapter(OzoneFileStatus status,
     );
   }
 
+  private FileStatusAdapter toFileStatusAdapter(OzoneFileStatusLight status,
+                                                String owner, URI defaultUri, Path workingDir) {
+    BasicOmKeyInfo keyInfo = status.getKeyInfo();
+    short replication = (short) keyInfo.getReplicationConfig()
+        .getRequiredNodes();
+    return new FileStatusAdapter(
+        keyInfo.getDataSize(),
+        keyInfo.getReplicatedSize(),
+        new Path(OZONE_URI_DELIMITER + keyInfo.getKeyName())
+            .makeQualified(defaultUri, workingDir),
+        status.isDirectory(),
+        replication,
+        status.getBlockSize(),
+        keyInfo.getModificationTime(),
+        keyInfo.getModificationTime(),
+        status.isDirectory() ? (short) 00777 : (short) 00666,
+        StringUtils.defaultIfEmpty(keyInfo.getOwnerName(), owner),
+        owner,
+        null,
+        getBlockLocations(null),
+        false,
+        OzoneClientUtils.isKeyErasureCode(keyInfo)
+    );
+  }
+
   /**
    * Helper method to get List of BlockLocation from OM Key info.
    * @param fileStatus Ozone key file status.

diff --git a/...p-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneFileSystem.java b/...p-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneFileSystem.java
@@ -688,7 +688,7 @@ public FileStatus[] listStatus(Path f) throws IOException {
     do {
       tmpStatusList =
           adapter.listStatus(pathToKey(f), false, startKey, numEntries, uri,
-              workingDir, getUsername())
+                  workingDir, getUsername(), true)
               .stream()
               .map(this::convertFileStatus)
               .collect(Collectors.toList());
@@ -947,13 +947,13 @@ public RemoteIterator<LocatedFileStatus> listFiles(Path f, boolean recursive)
   public RemoteIterator<LocatedFileStatus> listLocatedStatus(Path f)
       throws IOException {
     incrementCounter(Statistic.INVOCATION_LIST_LOCATED_STATUS);
-    return super.listLocatedStatus(f);
+    return new OzoneFileStatusIterator<>(f, false);
   }
 
   @Override
   public RemoteIterator<FileStatus> listStatusIterator(Path f)
       throws IOException {
-    return new OzoneFileStatusIterator<>(f);
+    return new OzoneFileStatusIterator<>(f, true);
   }
 
   @Override
@@ -999,6 +999,7 @@ private final class OzoneFileStatusIterator<T extends FileStatus>
     private Path p;
     private T curStat = null;
     private String startPath = "";
+    private boolean lite;
 
     /**
      * Constructor to initialize OzoneFileStatusIterator.
@@ -1007,10 +1008,11 @@ private final class OzoneFileStatusIterator<T extends FileStatus>
      * @param p path to file/directory.
      * @throws IOException
      */
-    private OzoneFileStatusIterator(Path p) throws IOException {
+    private OzoneFileStatusIterator(Path p, boolean lite) throws IOException {
       this.p = p;
+      this.lite = lite;
       // fetch the first batch of entries in the directory
-      thisListing = listFileStatus(p, startPath);
+      thisListing = listFileStatus(p, startPath, lite);
       if (thisListing != null && !thisListing.isEmpty()) {
         startPath = pathToKey(
             thisListing.get(thisListing.size() - 1).getPath());
@@ -1050,7 +1052,7 @@ private boolean hasNextNoFilter() throws IOException {
         if (startPath != null && (thisListing.size() == listingPageSize ||
             thisListing.size() == listingPageSize - 1)) {
           // current listing is exhausted & fetch a new listing
-          thisListing = listFileStatus(p, startPath);
+          thisListing = listFileStatus(p, startPath, lite);
           if (thisListing != null && !thisListing.isEmpty()) {
             startPath = pathToKey(
                 thisListing.get(thisListing.size() - 1).getPath());
@@ -1088,15 +1090,15 @@ public T next() throws IOException {
    * @return list of file status.
    * @throws IOException
    */
-  private List<FileStatus> listFileStatus(Path f, String startPath)
+  private List<FileStatus> listFileStatus(Path f, String startPath, boolean lite)
       throws IOException {
     incrementCounter(Statistic.INVOCATION_LIST_STATUS, 1);
     statistics.incrementReadOps(1);
     LOG.trace("listFileStatus() path:{}", f);
     List<FileStatus> statusList;
     statusList =
         adapter.listStatus(pathToKey(f), false, startPath,
-                listingPageSize, uri, workingDir, getUsername())
+                listingPageSize, uri, workingDir, getUsername(), lite)
             .stream()
             .map(this::convertFileStatus)
             .collect(Collectors.toList());