Skip to content

Commit 419f16b

Browse files
anujmodi2021anujmodi2021
committed
Unused Imports
1 parent 8f27ac1 commit 419f16b

File tree

7 files changed

+72
-27
lines changed

7 files changed

+72
-27
lines changed

‎hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsBlobClient.java‎

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -401,7 +401,18 @@ public ListResponseData listPath(final String relativePath, final boolean recurs
401401
return listResponseData;
402402
}
403403

404-
404+
/**
405+
* Post-processing of the list operation on Blob endpoint.
406+
* There are two client hanlding to be done on list output.
407+
* 1. Empty List returned on server could potentially mean path is a file.
408+
* 2. There can be duplicates returned from the server for explicit non-empty directory.
409+
* @param relativePath relative path to be listed.
410+
* @param fileStatuses list of file statuses returned from the server.
411+
* @param tracingContext tracing context to trace server calls.
412+
* @param uri URI to be used for path conversion.
413+
* @return rectified list of file statuses.
414+
* @throws AzureBlobFileSystemException if any failure occurs.
415+
*/
405416
@Override
406417
public List<FileStatus> postListProcessing(String relativePath, List<FileStatus> fileStatuses,
407418
TracingContext tracingContext, URI uri) throws AzureBlobFileSystemException {
@@ -413,11 +424,12 @@ public List<FileStatus> postListProcessing(String relativePath, List<FileStatus>
413424
// If it is a non-existing path, we need to throw a FileNotFoundException.
414425
AbfsRestOperation pathStatus = this.getPathStatus(relativePath, tracingContext, null, false);
415426
BlobListResultSchema listResultSchema = getListResultSchemaFromPathStatus(relativePath, pathStatus);
416-
LOG.debug("ListStatus attempted on a file path. Returning file status.");
427+
LOG.debug("ListStatus attempted on a file path {}. Returning file status.", relativePath);
417428
for (BlobListResultEntrySchema entry : listResultSchema.paths()) {
418429
rectifiedFileStatuses.add(getVersionedFileStatusFromEntry(entry, uri));
419430
}
420431
} else {
432+
// Remove duplicates from the non-empty list output only.
421433
rectifiedFileStatuses.addAll(ListUtils.getUniqueListResult(fileStatuses));
422434
LOG.debug(
423435
"ListBlob API returned a total of {} elements including duplicates."

‎hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -525,6 +525,15 @@ public abstract AbfsRestOperation setFilesystemProperties(Hashtable<String, Stri
525525
public abstract ListResponseData listPath(String relativePath, boolean recursive,
526526
int listMaxResults, String continuation, TracingContext tracingContext, URI uri) throws IOException;
527527

528+
/**
529+
* Post-processing of the list operation.
530+
* @param relativePath which is used to list the blobs.
531+
* @param fileStatuses list of file statuses to be processed.
532+
* @param tracingContext for tracing the server calls.
533+
* @param uri to be used for the path conversion.
534+
* @return list of file statuses to be returned.
535+
* @throws AzureBlobFileSystemException if rest operation fails.
536+
*/
528537
public abstract List<FileStatus> postListProcessing(String relativePath,
529538
List<FileStatus> fileStatuses, TracingContext tracingContext, URI uri) throws AzureBlobFileSystemException;
530539

‎hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsDfsClient.java‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -349,9 +349,18 @@ public ListResponseData listPath(final String relativePath,
349349
return listResponseData;
350350
}
351351

352+
/**
353+
* Non-functional implementation.
354+
* Client side handling to remove duplicates not needed in DFSClient.
355+
* @param relativePath on which listing was attempted.
356+
* @param fileStatuses result of listing operation.
357+
* @param tracingContext for tracing the server calls.
358+
* @param uri to be used for path conversion.
359+
* @return fileStatuses as it is without any processing.
360+
*/
352361
@Override
353-
public List<FileStatus> postListProcessing(String relativePath, List<FileStatus> fileStatuses,
354-
TracingContext tracingContext, URI uri) throws AzureBlobFileSystemException {
362+
public List<FileStatus> postListProcessing(String relativePath,
363+
List<FileStatus> fileStatuses, TracingContext tracingContext, URI uri){
355364
return fileStatuses;
356365
}
357366

‎hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelete.java‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,6 @@
6868
import static org.apache.hadoop.fs.contract.ContractTestUtils.assertPathDoesNotExist;
6969
import static org.apache.hadoop.test.LambdaTestUtils.intercept;
7070
import static org.mockito.ArgumentMatchers.any;
71-
import static org.mockito.ArgumentMatchers.eq;
7271
import static org.mockito.Mockito.doCallRealMethod;
7372
import static org.mockito.Mockito.doReturn;
7473
import static org.mockito.Mockito.mock;

‎hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemListStatus.java‎

Lines changed: 38 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -378,28 +378,46 @@ public void testRenameTrailingPeriodFile() throws IOException {
378378

379379
@Test
380380
public void testEmptyListingInSubsequentCall() throws IOException {
381-
testEmptyListingInSubsequentCallInternal(EMPTY_STRING, true, EMPTY_STRING, true, 1, 0);
382-
testEmptyListingInSubsequentCallInternal(EMPTY_STRING, true, EMPTY_STRING, false, 1, 0);
383-
testEmptyListingInSubsequentCallInternal(EMPTY_STRING, true, TEST_CONTINUATION_TOKEN, true, 1, 0);
384-
testEmptyListingInSubsequentCallInternal(EMPTY_STRING, true, TEST_CONTINUATION_TOKEN, false, 1, 0);
385-
386-
testEmptyListingInSubsequentCallInternal(TEST_CONTINUATION_TOKEN, true, EMPTY_STRING, true, 2, 0);
387-
testEmptyListingInSubsequentCallInternal(TEST_CONTINUATION_TOKEN, true, EMPTY_STRING, false, 2, 1);
388-
testEmptyListingInSubsequentCallInternal(TEST_CONTINUATION_TOKEN, true, TEST_CONTINUATION_TOKEN, true, 3, 0);
389-
testEmptyListingInSubsequentCallInternal(TEST_CONTINUATION_TOKEN, true, TEST_CONTINUATION_TOKEN, false, 3, 1);
390-
391-
testEmptyListingInSubsequentCallInternal(EMPTY_STRING, false, EMPTY_STRING, true, 1, 1);
392-
testEmptyListingInSubsequentCallInternal(EMPTY_STRING, false, EMPTY_STRING, false, 1, 1);
393-
testEmptyListingInSubsequentCallInternal(EMPTY_STRING, false, TEST_CONTINUATION_TOKEN, true, 1, 1);
394-
testEmptyListingInSubsequentCallInternal(EMPTY_STRING, false, TEST_CONTINUATION_TOKEN, false, 1, 1);
395-
396-
testEmptyListingInSubsequentCallInternal(TEST_CONTINUATION_TOKEN, false, EMPTY_STRING, true, 2, 1);
397-
testEmptyListingInSubsequentCallInternal(TEST_CONTINUATION_TOKEN, false, EMPTY_STRING, false, 2, 2);
398-
testEmptyListingInSubsequentCallInternal(TEST_CONTINUATION_TOKEN, false, TEST_CONTINUATION_TOKEN, true, 3, 1);
399-
testEmptyListingInSubsequentCallInternal(TEST_CONTINUATION_TOKEN, false, TEST_CONTINUATION_TOKEN, false, 3, 2);
381+
testEmptyListingInSubsequentCallInternal(EMPTY_STRING, true, EMPTY_STRING,
382+
true, 1, 0);
383+
testEmptyListingInSubsequentCallInternal(EMPTY_STRING, true, EMPTY_STRING,
384+
false, 1, 0);
385+
testEmptyListingInSubsequentCallInternal(EMPTY_STRING, true, TEST_CONTINUATION_TOKEN,
386+
true, 1, 0);
387+
testEmptyListingInSubsequentCallInternal(EMPTY_STRING, true, TEST_CONTINUATION_TOKEN,
388+
false, 1, 0);
389+
390+
testEmptyListingInSubsequentCallInternal(TEST_CONTINUATION_TOKEN, true, EMPTY_STRING,
391+
true, 2, 0);
392+
testEmptyListingInSubsequentCallInternal(TEST_CONTINUATION_TOKEN, true, EMPTY_STRING,
393+
false, 2, 1);
394+
testEmptyListingInSubsequentCallInternal(TEST_CONTINUATION_TOKEN, true, TEST_CONTINUATION_TOKEN,
395+
true, 3, 0);
396+
testEmptyListingInSubsequentCallInternal(TEST_CONTINUATION_TOKEN, true, TEST_CONTINUATION_TOKEN,
397+
false, 3, 1);
398+
399+
testEmptyListingInSubsequentCallInternal(EMPTY_STRING, false, EMPTY_STRING,
400+
true, 1, 1);
401+
testEmptyListingInSubsequentCallInternal(EMPTY_STRING, false, EMPTY_STRING,
402+
false, 1, 1);
403+
testEmptyListingInSubsequentCallInternal(EMPTY_STRING, false, TEST_CONTINUATION_TOKEN,
404+
true, 1, 1);
405+
testEmptyListingInSubsequentCallInternal(EMPTY_STRING, false, TEST_CONTINUATION_TOKEN,
406+
false, 1, 1);
407+
408+
testEmptyListingInSubsequentCallInternal(TEST_CONTINUATION_TOKEN, false, EMPTY_STRING,
409+
true, 2, 1);
410+
testEmptyListingInSubsequentCallInternal(TEST_CONTINUATION_TOKEN, false, EMPTY_STRING,
411+
false, 2, 2);
412+
testEmptyListingInSubsequentCallInternal(TEST_CONTINUATION_TOKEN, false, TEST_CONTINUATION_TOKEN,
413+
true, 3, 1);
414+
testEmptyListingInSubsequentCallInternal(TEST_CONTINUATION_TOKEN, false, TEST_CONTINUATION_TOKEN,
415+
false, 3, 2);
400416
}
401417

402-
private void testEmptyListingInSubsequentCallInternal(String firstCT, boolean isfirstEmpty, String secondCT, boolean isSecondEmpty, int expectedInvocations, int expectedSize) throws IOException {
418+
private void testEmptyListingInSubsequentCallInternal(String firstCT,
419+
boolean isfirstEmpty, String secondCT, boolean isSecondEmpty,
420+
int expectedInvocations, int expectedSize) throws IOException {
403421
AzureBlobFileSystem spiedFs = Mockito.spy(getFileSystem());
404422
AzureBlobFileSystemStore spiedStore = Mockito.spy(spiedFs.getAbfsStore());
405423
spiedStore.getAbfsConfiguration().setListMaxResults(1);

‎hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRename.java‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -108,7 +108,6 @@
108108
import static org.apache.hadoop.fs.contract.ContractTestUtils.dataset;
109109
import static org.apache.hadoop.fs.contract.ContractTestUtils.writeDataset;
110110
import static org.apache.hadoop.test.LambdaTestUtils.intercept;
111-
import static org.mockito.ArgumentMatchers.eq;
112111

113112
/**
114113
* Test rename operation.

‎hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestListActionTaker.java‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,6 @@
3838
import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DEFAULT_AZURE_LIST_MAX_RESULTS;
3939
import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DEFAULT_FS_AZURE_LISTING_ACTION_THREADS;
4040
import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DEFAULT_FS_AZURE_PRODUCER_QUEUE_MAX_SIZE;
41-
import static org.mockito.ArgumentMatchers.eq;
4241

4342
public class TestListActionTaker extends AbstractAbfsTestWithTimeout {
4443

0 commit comments

Comments
 (0)