diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java index 405f3a42e9e..289e8f48c3e 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java @@ -985,8 +985,23 @@ public OzoneDataStreamOutput createStreamFile(String keyName, long size, */ public List listStatus(String keyName, boolean recursive, String startKey, long numEntries) throws IOException { - return proxy - .listStatus(volumeName, name, keyName, recursive, startKey, numEntries); + return proxy.listStatus(volumeName, name, keyName, recursive, startKey, numEntries); + } + + /** + * List the lightweight status for a file or a directory and its contents. + * + * @param keyName Absolute path of the entry to be listed + * @param recursive For a directory if true all the descendants of a + * particular directory are listed + * @param startKey Key from which listing needs to start. If startKey exists + * its status is included in the final list. + * @param numEntries Number of entries to list from the start key + * @return list of file status + */ + public List listStatusLight(String keyName, boolean recursive, + String startKey, long numEntries) throws IOException { + return proxy.listStatusLight(volumeName, name, keyName, recursive, startKey, numEntries, false); } /** diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/BasicOmKeyInfo.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/BasicOmKeyInfo.java index a9fa742a108..82b9d8cccfb 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/BasicOmKeyInfo.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/BasicOmKeyInfo.java @@ -110,6 +110,10 @@ public String getOwnerName() { return ownerName; } + public long getReplicatedSize() { + return QuotaUtil.getReplicatedSize(getDataSize(), replicationConfig); + } + /** * Builder of BasicOmKeyInfo. */ diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractOzoneFileSystemTest.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractOzoneFileSystemTest.java index e7c4cbee1d5..5dd8f7db862 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractOzoneFileSystemTest.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractOzoneFileSystemTest.java @@ -2112,6 +2112,41 @@ void testListStatus2() throws IOException { } } + @Test + public void testOzoneManagerListLocatedStatusAndListStatus() throws IOException { + String data = RandomStringUtils.randomAlphanumeric(20); + String directory = RandomStringUtils.randomAlphanumeric(5); + String filePath = RandomStringUtils.randomAlphanumeric(5); + Path path = createPath("/" + directory + "/" + filePath); + try (FSDataOutputStream stream = fs.create(path)) { + stream.writeBytes(data); + } + RemoteIterator listLocatedStatus = fs.listLocatedStatus(path); + int count = 0; + while (listLocatedStatus.hasNext()) { + LocatedFileStatus locatedFileStatus = listLocatedStatus.next(); + assertTrue(locatedFileStatus.getBlockLocations().length >= 1); + + for (BlockLocation blockLocation : locatedFileStatus.getBlockLocations()) { + assertTrue(blockLocation.getNames().length >= 1); + assertTrue(blockLocation.getHosts().length >= 1); + } + count++; + } + assertEquals(1, count); + count = 0; + RemoteIterator listStatus = fs.listStatusIterator(path); + while (listStatus.hasNext()) { + FileStatus fileStatus = listStatus.next(); + assertFalse(fileStatus instanceof LocatedFileStatus); + count++; + } + assertEquals(1, count); + FileStatus[] fileStatuses = fs.listStatus(path.getParent()); + assertEquals(1, fileStatuses.length); + assertFalse(fileStatuses[0] instanceof LocatedFileStatus); + } + @Test void testOzoneManagerFileSystemInterface() throws IOException { String dirPath = RandomStringUtils.randomAlphanumeric(5); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractRootedOzoneFileSystemTest.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractRootedOzoneFileSystemTest.java index cfc9029019a..8b71a216003 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractRootedOzoneFileSystemTest.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractRootedOzoneFileSystemTest.java @@ -1071,7 +1071,7 @@ private void listStatusRecursiveHelper(Path curPath, List result) private List callAdapterListStatus(String pathStr, boolean recursive, String startPath, long numEntries) throws IOException { return adapter.listStatus(pathStr, recursive, startPath, numEntries, - ofs.getUri(), ofs.getWorkingDirectory(), ofs.getUsername()) + ofs.getUri(), ofs.getWorkingDirectory(), ofs.getUsername(), false) .stream().map(ofs::convertFileStatus).collect(Collectors.toList()); } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientWithKeyLatestVersion.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientWithKeyLatestVersion.java index fd32698eec2..62429368690 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientWithKeyLatestVersion.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientWithKeyLatestVersion.java @@ -31,6 +31,7 @@ import org.apache.hadoop.ozone.client.OzoneClientFactory; import org.apache.hadoop.ozone.client.OzoneVolume; import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus; +import org.apache.hadoop.ozone.om.helpers.OzoneFileStatusLight; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.TestInstance; @@ -151,5 +152,11 @@ private void assertListStatus(OzoneBucket bucket, String keyName, List versions = files.get(0).getKeyInfo().getKeyLocationVersions(); assertEquals(expectedVersionCount, versions.size()); + + List lightFiles = bucket.listStatusLight(keyName, false, "", 1); + + assertNotNull(lightFiles); + assertEquals(1, lightFiles.size()); + } } diff --git a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneClientAdapterImpl.java b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneClientAdapterImpl.java index bc7c0120446..689e340ff5d 100644 --- a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneClientAdapterImpl.java +++ b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneClientAdapterImpl.java @@ -69,6 +69,7 @@ import org.apache.hadoop.ozone.client.rpc.RpcClient; import org.apache.hadoop.ozone.container.common.helpers.BlockData; import org.apache.hadoop.ozone.om.exceptions.OMException; +import org.apache.hadoop.ozone.om.helpers.BasicOmKeyInfo; import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.LeaseKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyArgs; @@ -77,6 +78,7 @@ import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup; import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils; import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus; +import org.apache.hadoop.ozone.om.helpers.OzoneFileStatusLight; import org.apache.hadoop.ozone.security.OzoneTokenIdentifier; import org.apache.hadoop.ozone.snapshot.SnapshotDiffReportOzone; import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse; @@ -436,15 +438,22 @@ public Iterator listKeys(String pathKey) throws IOException { @Override public List listStatus(String keyName, boolean recursive, String startKey, long numEntries, URI uri, - Path workingDir, String username) throws IOException { + Path workingDir, String username, boolean lite) throws IOException { try { incrementCounter(Statistic.OBJECTS_LIST, 1); - List statuses = bucket - .listStatus(keyName, recursive, startKey, numEntries); - List result = new ArrayList<>(); - for (OzoneFileStatus status : statuses) { - result.add(toFileStatusAdapter(status, username, uri, workingDir)); + if (lite) { + List statuses = bucket + .listStatusLight(keyName, recursive, startKey, numEntries); + for (OzoneFileStatusLight status : statuses) { + result.add(toFileStatusAdapter(status, username, uri, workingDir)); + } + } else { + List statuses = bucket + .listStatus(keyName, recursive, startKey, numEntries); + for (OzoneFileStatus status : statuses) { + result.add(toFileStatusAdapter(status, username, uri, workingDir)); + } } return result; } catch (OMException e) { @@ -549,6 +558,31 @@ private FileStatusAdapter toFileStatusAdapter(OzoneFileStatus status, ); } + private FileStatusAdapter toFileStatusAdapter(OzoneFileStatusLight status, + String owner, URI defaultUri, Path workingDir) { + BasicOmKeyInfo keyInfo = status.getKeyInfo(); + short replication = (short) keyInfo.getReplicationConfig() + .getRequiredNodes(); + return new FileStatusAdapter( + keyInfo.getDataSize(), + keyInfo.getReplicatedSize(), + new Path(OZONE_URI_DELIMITER + keyInfo.getKeyName()) + .makeQualified(defaultUri, workingDir), + status.isDirectory(), + replication, + status.getBlockSize(), + keyInfo.getModificationTime(), + keyInfo.getModificationTime(), + status.isDirectory() ? (short) 00777 : (short) 00666, + StringUtils.defaultIfEmpty(keyInfo.getOwnerName(), owner), + owner, + null, + getBlockLocations(null), + false, + OzoneClientUtils.isKeyErasureCode(keyInfo) + ); + } + /** * Helper method to get List of BlockLocation from OM Key info. * @param fileStatus Ozone key file status. diff --git a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneFileSystem.java b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneFileSystem.java index ed5574af32b..e03a2178226 100644 --- a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneFileSystem.java +++ b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneFileSystem.java @@ -18,6 +18,7 @@ package org.apache.hadoop.fs.ozone; +import com.google.common.base.Function; import com.google.common.base.Preconditions; import org.apache.hadoop.conf.Configuration; @@ -688,7 +689,7 @@ public FileStatus[] listStatus(Path f) throws IOException { do { tmpStatusList = adapter.listStatus(pathToKey(f), false, startKey, numEntries, uri, - workingDir, getUsername()) + workingDir, getUsername(), true) .stream() .map(this::convertFileStatus) .collect(Collectors.toList()); @@ -947,13 +948,15 @@ public RemoteIterator listFiles(Path f, boolean recursive) public RemoteIterator listLocatedStatus(Path f) throws IOException { incrementCounter(Statistic.INVOCATION_LIST_LOCATED_STATUS); - return super.listLocatedStatus(f); + return new OzoneFileStatusIterator<>(f, + (stat) -> stat instanceof LocatedFileStatus ? (LocatedFileStatus) stat : new LocatedFileStatus(stat, null), + false); } @Override public RemoteIterator listStatusIterator(Path f) throws IOException { - return new OzoneFileStatusIterator<>(f); + return new OzoneFileStatusIterator<>(f, stat -> stat, true); } @Override @@ -986,7 +989,6 @@ public void setTimes(Path f, long mtime, long atime) throws IOException { String key = pathToKey(qualifiedPath); adapter.setTimes(key, mtime, atime); } - /** * A private class implementation for iterating list of file status. * @@ -999,18 +1001,24 @@ private final class OzoneFileStatusIterator private Path p; private T curStat = null; private String startPath = ""; + private boolean lite; + private Function transformFunc; /** * Constructor to initialize OzoneFileStatusIterator. * Get the first batch of entry for iteration. * * @param p path to file/directory. + * @param transformFunc function to convert FileStatus into an expected type. + * @param lite if true it should look into fetching a lightweight keys from server. * @throws IOException */ - private OzoneFileStatusIterator(Path p) throws IOException { + private OzoneFileStatusIterator(Path p, Function transformFunc, boolean lite) throws IOException { this.p = p; + this.lite = lite; + this.transformFunc = transformFunc; // fetch the first batch of entries in the directory - thisListing = listFileStatus(p, startPath); + thisListing = listFileStatus(p, startPath, lite); if (thisListing != null && !thisListing.isEmpty()) { startPath = pathToKey( thisListing.get(thisListing.size() - 1).getPath()); @@ -1029,7 +1037,7 @@ public boolean hasNext() throws IOException { while (curStat == null && hasNextNoFilter()) { T next; FileStatus fileStat = thisListing.get(i++); - next = (T) (fileStat); + next = this.transformFunc.apply(fileStat); curStat = next; } return curStat != null; @@ -1050,7 +1058,7 @@ private boolean hasNextNoFilter() throws IOException { if (startPath != null && (thisListing.size() == listingPageSize || thisListing.size() == listingPageSize - 1)) { // current listing is exhausted & fetch a new listing - thisListing = listFileStatus(p, startPath); + thisListing = listFileStatus(p, startPath, lite); if (thisListing != null && !thisListing.isEmpty()) { startPath = pathToKey( thisListing.get(thisListing.size() - 1).getPath()); @@ -1085,10 +1093,11 @@ public T next() throws IOException { * * @param f * @param startPath + * @param lite if true return lightweight keys * @return list of file status. * @throws IOException */ - private List listFileStatus(Path f, String startPath) + private List listFileStatus(Path f, String startPath, boolean lite) throws IOException { incrementCounter(Statistic.INVOCATION_LIST_STATUS, 1); statistics.incrementReadOps(1); @@ -1096,7 +1105,7 @@ private List listFileStatus(Path f, String startPath) List statusList; statusList = adapter.listStatus(pathToKey(f), false, startPath, - listingPageSize, uri, workingDir, getUsername()) + listingPageSize, uri, workingDir, getUsername(), lite) .stream() .map(this::convertFileStatus) .collect(Collectors.toList()); diff --git a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java index 46602068ccd..039c4ad898f 100644 --- a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java +++ b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java @@ -79,6 +79,7 @@ import org.apache.hadoop.ozone.client.rpc.RpcClient; import org.apache.hadoop.ozone.container.common.helpers.BlockData; import org.apache.hadoop.ozone.om.exceptions.OMException; +import org.apache.hadoop.ozone.om.helpers.BasicOmKeyInfo; import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.LeaseKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyArgs; @@ -87,6 +88,7 @@ import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup; import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils; import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus; +import org.apache.hadoop.ozone.om.helpers.OzoneFileStatusLight; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; import org.apache.hadoop.ozone.security.OzoneTokenIdentifier; import org.apache.hadoop.ozone.snapshot.SnapshotDiffReportOzone; @@ -784,7 +786,7 @@ public Iterator listKeys(String pathStr) throws IOException { */ private List listStatusRoot( boolean recursive, String startPath, long numEntries, - URI uri, Path workingDir, String username) throws IOException { + URI uri, Path workingDir, String username, boolean lite) throws IOException { OFSPath ofsStartPath = new OFSPath(startPath, config); // list volumes @@ -797,7 +799,7 @@ private List listStatusRoot( if (recursive) { String pathStrNextVolume = volume.getName(); res.addAll(listStatus(pathStrNextVolume, recursive, startPath, - numEntries - res.size(), uri, workingDir, username)); + numEntries - res.size(), uri, workingDir, username, lite)); } } return res; @@ -806,9 +808,10 @@ private List listStatusRoot( /** * Helper for OFS listStatus on a volume. */ + @SuppressWarnings("checkstyle:ParameterNumber") private List listStatusVolume(String volumeStr, boolean recursive, String startPath, long numEntries, - URI uri, Path workingDir, String username) throws IOException { + URI uri, Path workingDir, String username, boolean lite) throws IOException { OFSPath ofsStartPath = new OFSPath(startPath, config); // list buckets in the volume @@ -822,7 +825,7 @@ private List listStatusVolume(String volumeStr, if (recursive) { String pathStrNext = volumeStr + OZONE_URI_DELIMITER + bucket.getName(); res.addAll(listStatus(pathStrNext, recursive, startPath, - numEntries - res.size(), uri, workingDir, username)); + numEntries - res.size(), uri, workingDir, username, lite)); } } return res; @@ -874,13 +877,15 @@ private List listStatusBucketSnapshot( * Used in making the return path qualified. * @param username User name. * Used in making the return path qualified. + * @param lite true if lightweight response needs to be returned otherwise false. * @return A list of FileStatusAdapter. * @throws IOException Bucket exception or FileNotFoundException. */ + @SuppressWarnings("checkstyle:ParameterNumber") @Override public List listStatus(String pathStr, boolean recursive, String startPath, long numEntries, URI uri, - Path workingDir, String username) throws IOException { + Path workingDir, String username, boolean lite) throws IOException { incrementCounter(Statistic.OBJECTS_LIST, 1); // Remove authority from startPath if it exists @@ -899,44 +904,53 @@ public List listStatus(String pathStr, boolean recursive, OFSPath ofsPath = new OFSPath(pathStr, config); if (ofsPath.isRoot()) { return listStatusRoot( - recursive, startPath, numEntries, uri, workingDir, username); + recursive, startPath, numEntries, uri, workingDir, username, lite); } OFSPath ofsStartPath = new OFSPath(startPath, config); if (ofsPath.isVolume()) { String startBucketPath = ofsStartPath.getNonKeyPath(); return listStatusVolume(ofsPath.getVolumeName(), - recursive, startBucketPath, numEntries, uri, workingDir, username); + recursive, startBucketPath, numEntries, uri, workingDir, username, lite); } if (ofsPath.isSnapshotPath()) { return listStatusBucketSnapshot(ofsPath.getVolumeName(), ofsPath.getBucketName(), uri); } - + List result = new ArrayList<>(); String keyName = ofsPath.getKeyName(); // Internally we need startKey to be passed into bucket.listStatus String startKey = ofsStartPath.getKeyName(); try { OzoneBucket bucket = getBucket(ofsPath, false); - List statuses; + List statuses = Collections.emptyList(); + List lightStatuses = Collections.emptyList(); if (bucket.isSourcePathExist()) { - statuses = bucket - .listStatus(keyName, recursive, startKey, numEntries); + if (lite) { + lightStatuses = bucket.listStatusLight(keyName, recursive, startKey, numEntries); + } else { + statuses = bucket.listStatus(keyName, recursive, startKey, numEntries); + } + } else { LOG.warn("Source Bucket does not exist, link bucket {} is orphan " + "and returning empty list of files inside it", bucket.getName()); - statuses = Collections.emptyList(); } // Note: result in statuses above doesn't have volume/bucket path since // they are from the server. String ofsPathPrefix = ofsPath.getNonKeyPath(); - List result = new ArrayList<>(); - for (OzoneFileStatus status : statuses) { - result.add(toFileStatusAdapter(status, username, uri, workingDir, - ofsPathPrefix)); + if (lite) { + for (OzoneFileStatusLight status : lightStatuses) { + result.add(toFileStatusAdapter(status, username, uri, workingDir, ofsPathPrefix)); + } + } else { + for (OzoneFileStatus status : statuses) { + result.add(toFileStatusAdapter(status, username, uri, workingDir, ofsPathPrefix)); + } } + return result; } catch (OMException e) { if (e.getResult() == OMException.ResultCodes.FILE_NOT_FOUND) { @@ -1039,6 +1053,31 @@ private FileStatusAdapter toFileStatusAdapter(OzoneFileStatus status, ); } + private FileStatusAdapter toFileStatusAdapter(OzoneFileStatusLight status, + String owner, URI defaultUri, Path workingDir, String ofsPathPrefix) { + BasicOmKeyInfo keyInfo = status.getKeyInfo(); + short replication = (short) keyInfo.getReplicationConfig() + .getRequiredNodes(); + return new FileStatusAdapter( + keyInfo.getDataSize(), + keyInfo.getReplicatedSize(), + new Path(ofsPathPrefix + OZONE_URI_DELIMITER + keyInfo.getKeyName()) + .makeQualified(defaultUri, workingDir), + status.isDirectory(), + replication, + status.getBlockSize(), + keyInfo.getModificationTime(), + keyInfo.getModificationTime(), + status.isDirectory() ? (short) 00777 : (short) 00666, + StringUtils.defaultIfEmpty(keyInfo.getOwnerName(), owner), + owner, + null, + getBlockLocations(null), + false, + OzoneClientUtils.isKeyErasureCode(keyInfo) + ); + } + /** * Helper method to get List of BlockLocation from OM Key info. * @param fileStatus Ozone key file status. diff --git a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java index 3e0a3730627..58b59766a28 100644 --- a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java +++ b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.fs.ozone; +import com.google.common.base.Function; import com.google.common.base.Preconditions; import io.opentracing.Span; import io.opentracing.util.GlobalTracer; @@ -915,7 +916,7 @@ private boolean o3Exists(final Path f) throws IOException { @Override public FileStatus[] listStatus(Path f) throws IOException { return TracingUtil.executeInNewSpan("ofs listStatus", - () -> convertFileStatusArr(listStatusAdapter(f))); + () -> convertFileStatusArr(listStatusAdapter(f, true))); } private FileStatus[] convertFileStatusArr( @@ -929,7 +930,7 @@ private FileStatus[] convertFileStatusArr( } - public List listStatusAdapter(Path f) throws IOException { + private List listStatusAdapter(Path f, boolean lite) throws IOException { incrementCounter(Statistic.INVOCATION_LIST_STATUS, 1); statistics.incrementReadOps(1); LOG.trace("listStatus() path:{}", f); @@ -941,7 +942,7 @@ public List listStatusAdapter(Path f) throws IOException { do { tmpStatusList = adapter.listStatus(pathToKey(f), false, startPath, - numEntries, uri, workingDir, getUsername()); + numEntries, uri, workingDir, getUsername(), lite); if (!tmpStatusList.isEmpty()) { if (startPath.isEmpty() || !statuses.getLast().getPath().toString() @@ -1178,7 +1179,9 @@ public RemoteIterator listFiles(Path f, boolean recursive) public RemoteIterator listLocatedStatus(Path f) throws IOException { incrementCounter(Statistic.INVOCATION_LIST_LOCATED_STATUS); - return super.listLocatedStatus(f); + return new OzoneFileStatusIterator<>(f, + (stat) -> stat instanceof LocatedFileStatus ? (LocatedFileStatus) stat : new LocatedFileStatus(stat, null), + false); } @Override @@ -1193,7 +1196,7 @@ public RemoteIterator listStatusIterator(Path f) "Instead use 'ozone sh key list " + "' command"); } - return new OzoneFileStatusIterator<>(f); + return new OzoneFileStatusIterator<>(f, stat -> stat, true); } /** @@ -1203,23 +1206,29 @@ public RemoteIterator listStatusIterator(Path f) */ private final class OzoneFileStatusIterator implements RemoteIterator { + private final Function transformFunc; private List thisListing; private int i; private Path p; private T curStat = null; private String startPath = ""; + private boolean lite; /** * Constructor to initialize OzoneFileStatusIterator. * Get the first batch of entry for iteration. * * @param p path to file/directory. + * @param transformFunc function to convert FileStatus into an expected type. + * @param lite if true it should look into fetching a lightweight keys from server. * @throws IOException */ - private OzoneFileStatusIterator(Path p) throws IOException { + private OzoneFileStatusIterator(Path p, Function transformFunc, boolean lite) throws IOException { this.p = p; + this.lite = lite; + this.transformFunc = transformFunc; // fetch the first batch of entries in the directory - thisListing = listFileStatus(p, startPath); + thisListing = listFileStatus(p, startPath, lite); if (thisListing != null && !thisListing.isEmpty()) { startPath = pathToKey( thisListing.get(thisListing.size() - 1).getPath()); @@ -1238,7 +1247,7 @@ public boolean hasNext() throws IOException { while (curStat == null && hasNextNoFilter()) { T next; FileStatus fileStat = thisListing.get(i++); - next = (T) (fileStat); + next = transformFunc.apply(fileStat); curStat = next; } return curStat != null; @@ -1259,7 +1268,7 @@ private boolean hasNextNoFilter() throws IOException { if (startPath != null && (thisListing.size() == listingPageSize || thisListing.size() == listingPageSize - 1)) { // current listing is exhausted & fetch a new listing - thisListing = listFileStatus(p, startPath); + thisListing = listFileStatus(p, startPath, lite); if (thisListing != null && !thisListing.isEmpty()) { startPath = pathToKey( thisListing.get(thisListing.size() - 1).getPath()); @@ -1294,10 +1303,11 @@ public T next() throws IOException { * * @param f * @param startPath + * @param lite if true return lightweight keys * @return list of file status. * @throws IOException */ - private List listFileStatus(Path f, String startPath) + private List listFileStatus(Path f, String startPath, boolean lite) throws IOException { incrementCounter(Statistic.INVOCATION_LIST_STATUS, 1); statistics.incrementReadOps(1); @@ -1305,7 +1315,7 @@ private List listFileStatus(Path f, String startPath) List statusList; statusList = adapter.listStatus(pathToKey(f), false, startPath, - listingPageSize, uri, workingDir, getUsername()) + listingPageSize, uri, workingDir, getUsername(), lite) .stream() .map(this::convertFileStatus) .collect(Collectors.toList()); @@ -1442,7 +1452,7 @@ boolean iterate() throws IOException { ofsPath.getNonKeyPathNoPrefixDelim() + OZONE_URI_DELIMITER; if (isFSO) { List fileStatuses; - fileStatuses = listStatusAdapter(path); + fileStatuses = listStatusAdapter(path, true); for (FileStatusAdapter fileStatus : fileStatuses) { String keyName = new OFSPath(fileStatus.getPath().toString(), @@ -1567,7 +1577,7 @@ private ContentSummary getContentSummaryInSpan(Path f) throws IOException { // f is a directory long[] summary = {0, 0, 0, 1}; int i = 0; - for (FileStatusAdapter s : listStatusAdapter(f)) { + for (FileStatusAdapter s : listStatusAdapter(f, true)) { long length = s.getLength(); long spaceConsumed = s.getDiskConsumed(); ContentSummary c = s.isDir() ? getContentSummary(s.getPath()) : diff --git a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/OzoneClientAdapter.java b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/OzoneClientAdapter.java index e468ac498c4..24ff692e1b4 100644 --- a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/OzoneClientAdapter.java +++ b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/OzoneClientAdapter.java @@ -69,9 +69,10 @@ OzoneFSDataStreamOutput createStreamFile(String key, short replication, Iterator listKeys(String pathKey) throws IOException; + @SuppressWarnings("checkstyle:ParameterNumber") List listStatus(String keyName, boolean recursive, String startKey, long numEntries, URI uri, - Path workingDir, String username) throws IOException; + Path workingDir, String username, boolean lite) throws IOException; Token getDelegationToken(String renewer) throws IOException; diff --git a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/OzoneClientUtils.java b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/OzoneClientUtils.java index 383ad6db495..6c9fb3ccc7b 100644 --- a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/OzoneClientUtils.java +++ b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/OzoneClientUtils.java @@ -32,6 +32,7 @@ import org.apache.hadoop.ozone.client.checksum.ChecksumHelperFactory; import org.apache.hadoop.ozone.client.protocol.ClientProtocol; import org.apache.hadoop.ozone.om.exceptions.OMException; +import org.apache.hadoop.ozone.om.helpers.BasicOmKeyInfo; import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OmKeyArgs; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; @@ -246,6 +247,11 @@ public static boolean isKeyErasureCode(OmKeyInfo keyInfo) { HddsProtos.ReplicationType.EC; } + public static boolean isKeyErasureCode(BasicOmKeyInfo keyInfo) { + return keyInfo.getReplicationConfig().getReplicationType() == + HddsProtos.ReplicationType.EC; + } + public static boolean isKeyEncrypted(OmKeyInfo keyInfo) { return !Objects.isNull(keyInfo.getFileEncryptionInfo()); }