Skip to content

Commit cc96f4d

Browse files
anujmodi2021anujmodi2021
committed
Fixing Create isNotEmpty
1 parent 5e36e56 commit cc96f4d

File tree

8 files changed

+28
-38
lines changed

8 files changed

+28
-38
lines changed

‎hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsBlobClient.java‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -348,12 +348,6 @@ public AbfsRestOperation deleteFilesystem(TracingContext tracingContext)
348348
* @return {@link ListResponseData}. containing listing response.
349349
* @throws AzureBlobFileSystemException if rest operation or response parsing fails.
350350
*/
351-
@Override
352-
public ListResponseData listPath(final String relativePath, final boolean recursive,
353-
final int listMaxResults, final String continuation, TracingContext tracingContext, URI uri) throws IOException {
354-
return listPath(relativePath, recursive, listMaxResults, continuation, tracingContext, uri, true);
355-
}
356-
357351
@Override
358352
public ListResponseData listPath(final String relativePath, final boolean recursive,
359353
final int listMaxResults, final String continuation, TracingContext tracingContext, URI uri, boolean is404CheckRequired)
@@ -405,7 +399,7 @@ public ListResponseData listPath(final String relativePath, final boolean recurs
405399
}
406400
}
407401

408-
if (isEmptyListResults(listResponseData) && is404CheckRequired) {
402+
if (is404CheckRequired && isEmptyListResults(listResponseData)) {
409403
// If the list operation returns no paths, we need to check if the path is a file.
410404
// If it is a file, we need to return the file in the list.
411405
// If it is a non-existing path, we need to throw a FileNotFoundException.
@@ -2053,8 +2047,14 @@ public boolean isNonEmptyDirectory(String path,
20532047
TracingContext tracingContext) throws AzureBlobFileSystemException {
20542048
// This method is only called internally to determine state of a path
20552049
// and hence don't need identity transformation to happen.
2056-
ListResponseData listResponseData = listPath(path, false, 1, null, tracingContext, null, false);
2057-
return !isEmptyListResults(listResponseData);
2050+
String continuationToken = null;
2051+
List<FileStatus> fileStatusList = new ArrayList<>();
2052+
do {
2053+
ListResponseData listResponseData = listPath(path, false, 1, null, tracingContext, null, false);
2054+
fileStatusList.addAll(listResponseData.getFileStatusList());
2055+
continuationToken = listResponseData.getContinuationToken();
2056+
} while (StringUtils.isNotEmpty(continuationToken));
2057+
return !fileStatusList.isEmpty();
20582058
}
20592059

20602060
/**

‎hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -522,9 +522,6 @@ public abstract AbfsRestOperation setFilesystemProperties(Hashtable<String, Stri
522522
* @return {@link ListResponseData}. containing listing response.
523523
* @throws AzureBlobFileSystemException if rest operation or response parsing fails.
524524
*/
525-
public abstract ListResponseData listPath(String relativePath, boolean recursive,
526-
int listMaxResults, String continuation, TracingContext tracingContext, URI uri) throws IOException;
527-
528525
public abstract ListResponseData listPath(String relativePath, boolean recursive,
529526
int listMaxResults, String continuation, TracingContext tracingContext, URI uri, boolean is404CheckRequired) throws IOException;
530527

‎hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsDfsClient.java‎

Lines changed: 1 addition & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -323,7 +323,7 @@ public ListResponseData listPath(final String relativePath,
323323
final boolean recursive,
324324
final int listMaxResults,
325325
final String continuation,
326-
TracingContext tracingContext, URI uri) throws IOException {
326+
TracingContext tracingContext, URI uri, boolean is404CheckRequired) throws IOException {
327327
final List<AbfsHttpHeader> requestHeaders = createDefaultHeaders();
328328

329329
final AbfsUriQueryBuilder abfsUriQueryBuilder = createDefaultUriQueryBuilder();
@@ -349,14 +349,6 @@ public ListResponseData listPath(final String relativePath,
349349
return listResponseData;
350350
}
351351

352-
@Override
353-
public ListResponseData listPath(final String relativePath, final boolean recursive,
354-
final int listMaxResults, final String continuation, TracingContext tracingContext,
355-
URI uri, boolean is404CheckRequired) throws IOException {
356-
return listPath(relativePath, recursive,
357-
listMaxResults, continuation, tracingContext, uri);
358-
}
359-
360352
@Override
361353
public List<FileStatus> postListProcessing(String relativePath, List<FileStatus> fileStatuses,
362354
TracingContext tracingContext, URI uri) throws AzureBlobFileSystemException {

‎hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsClient.java‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -65,7 +65,7 @@ public void testContinuationTokenHavingEqualSign() throws Exception {
6565
try {
6666
AbfsRestOperation op = abfsClient
6767
.listPath("/", true, LIST_MAX_RESULTS, "===========",
68-
getTestTracingContext(fs, true), null).getOp();
68+
getTestTracingContext(fs, true), null, false).getOp();
6969
Assert.assertTrue(false);
7070
} catch (AbfsRestOperationException ex) {
7171
Assert.assertEquals("InvalidQueryParameterValue", ex.getErrorCode().getErrorCode());
@@ -106,7 +106,7 @@ public void testListPathWithValidListMaxResultsValues()
106106

107107
AbfsRestOperation op = getFileSystem().getAbfsClient().listPath(
108108
directory.toString(), false, getListMaxResults(), null,
109-
getTestTracingContext(getFileSystem(), true), null).getOp();
109+
getTestTracingContext(getFileSystem(), true), null, false).getOp();
110110

111111
List<? extends ListResultEntrySchema> list = op.getResult().getListResultSchema().paths();
112112
String continuationToken = op.getResult().getResponseHeader(HttpHeaderConfigurations.X_MS_CONTINUATION);
@@ -141,7 +141,7 @@ public void testListPathWithValueGreaterThanServerMaximum()
141141

142142
AbfsRestOperation op = getFileSystem().getAbfsClient().listPath(
143143
directory.toString(), false, getListMaxResults(), null,
144-
getTestTracingContext(getFileSystem(), true), null).getOp();
144+
getTestTracingContext(getFileSystem(), true), null, false).getOp();
145145

146146
List<? extends ListResultEntrySchema> list = op.getResult().getListResultSchema().paths();
147147
String continuationToken = op.getResult().getResponseHeader(HttpHeaderConfigurations.X_MS_CONTINUATION);
@@ -179,7 +179,7 @@ private List<? extends ListResultEntrySchema> listPath(String directory)
179179
throws IOException {
180180
return getFileSystem().getAbfsClient()
181181
.listPath(directory, false, getListMaxResults(), null,
182-
getTestTracingContext(getFileSystem(), true), null).getOp().getResult()
182+
getTestTracingContext(getFileSystem(), true), null, false).getOp().getResult()
183183
.getListResultSchema().paths();
184184
}
185185

‎hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsCustomEncryption.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -344,7 +344,7 @@ true, new BlobAppendRequestParameters(BLOCK_ID, null)),
344344
getTestTracingContext(fs, false));
345345
case LISTSTATUS:
346346
return client.listPath(path, false, 5, null,
347-
getTestTracingContext(fs, true), null).getOp();
347+
getTestTracingContext(fs, true), null, false).getOp();
348348
case RENAME:
349349
TracingContext tc = getTestTracingContext(fs, true);
350350
return client.renamePath(path, new Path(path + "_2").toString(),

‎hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelete.java‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -313,7 +313,7 @@ public void testDeleteIdempotencyTriggerHttp404() throws Exception {
313313
doCallRealMethod().when(mockClient)
314314
.listPath(Mockito.nullable(String.class), Mockito.anyBoolean(),
315315
Mockito.anyInt(), Mockito.nullable(String.class),
316-
Mockito.nullable(TracingContext.class), Mockito.nullable(URI.class));
316+
Mockito.nullable(TracingContext.class), Mockito.nullable(URI.class), eq(false));
317317
doCallRealMethod().when((AbfsBlobClient) mockClient)
318318
.listPath(Mockito.nullable(String.class), Mockito.anyBoolean(),
319319
Mockito.anyInt(), Mockito.nullable(String.class),
@@ -532,12 +532,12 @@ public void testDeleteImplicitDirWithSingleListResults() throws Exception {
532532
boolean recursive = answer.getArgument(1);
533533
String continuation = answer.getArgument(3);
534534
TracingContext context = answer.getArgument(4);
535-
return client.listPath(path, recursive, 1, continuation, context, null);
535+
return client.listPath(path, recursive, 1, continuation, context, null, false);
536536
})
537537
.when(spiedClient)
538538
.listPath(Mockito.anyString(), Mockito.anyBoolean(), Mockito.anyInt(),
539539
Mockito.nullable(String.class),
540-
Mockito.any(TracingContext.class), Mockito.nullable(URI.class));
540+
Mockito.any(TracingContext.class), Mockito.nullable(URI.class), eq(false));
541541
client.deleteBlobPath(new Path("/testDir/dir1"),
542542
null, getTestTracingContext(fs, true));
543543
fs.delete(new Path("/testDir/dir1"), true);
@@ -684,7 +684,7 @@ public void testProducerStopOnDeleteFailure() throws Exception {
684684
})
685685
.when(spiedClient)
686686
.listPath(Mockito.anyString(), Mockito.anyBoolean(), Mockito.anyInt(),
687-
Mockito.nullable(String.class), Mockito.any(TracingContext.class), Mockito.nullable(URI.class));
687+
Mockito.nullable(String.class), Mockito.any(TracingContext.class), Mockito.nullable(URI.class), eq(false));
688688
intercept(AccessDeniedException.class,
689689
() -> {
690690
fs.delete(new Path("/src"), true);

‎hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemListStatus.java‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -560,7 +560,7 @@ public void testContinuationTokenAcrossListStatus() throws Exception {
560560

561561
ListResponseData listResponseData = fs.getAbfsStore().getClient().listPath(
562562
"/testContinuationToken", false, 1, null, getTestTracingContext(fs, true),
563-
fs.getAbfsStore().getUri());
563+
fs.getAbfsStore().getUri(), false);
564564

565565
Assertions.assertThat(listResponseData.getContinuationToken())
566566
.describedAs("Continuation Token Should not be null").isNotNull();
@@ -569,7 +569,7 @@ public void testContinuationTokenAcrossListStatus() throws Exception {
569569

570570
ListResponseData listResponseData1 = fs.getAbfsStore().getClient().listPath(
571571
"/testContinuationToken", false, 1, listResponseData.getContinuationToken(), getTestTracingContext(fs, true),
572-
fs.getAbfsStore().getUri());
572+
fs.getAbfsStore().getUri(), false);
573573

574574
Assertions.assertThat(listResponseData1.getContinuationToken())
575575
.describedAs("Continuation Token Should be null").isNull();
@@ -589,7 +589,7 @@ public void testInvalidContinuationToken() throws Exception {
589589
intercept(AbfsRestOperationException.class,
590590
() -> fs.getAbfsStore().getClient().listPath(
591591
"/testInvalidContinuationToken", false, 1, "invalidToken",
592-
getTestTracingContext(fs, true), fs.getAbfsStore().getUri()));
592+
getTestTracingContext(fs, true), fs.getAbfsStore().getUri(), false));
593593
}
594594

595595
@Test
@@ -602,7 +602,7 @@ public void testEmptyContinuationToken() throws Exception {
602602

603603
ListResponseData listResponseData = fs.getAbfsStore().getClient().listPath(
604604
"/testInvalidContinuationToken", false, 1, "",
605-
getTestTracingContext(fs, true), fs.getAbfsStore().getUri());
605+
getTestTracingContext(fs, true), fs.getAbfsStore().getUri(), false);
606606

607607
Assertions.assertThat(listResponseData.getContinuationToken())
608608
.describedAs("Continuation Token Should Not be null").isNotNull();

‎hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRename.java‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -108,6 +108,7 @@
108108
import static org.apache.hadoop.fs.contract.ContractTestUtils.dataset;
109109
import static org.apache.hadoop.fs.contract.ContractTestUtils.writeDataset;
110110
import static org.apache.hadoop.test.LambdaTestUtils.intercept;
111+
import static org.mockito.ArgumentMatchers.eq;
111112

112113
/**
113114
* Test rename operation.
@@ -1144,12 +1145,12 @@ public void testBlobRenameWithListGivingPaginatedResultWithOneObjectPerList()
11441145
String continuation = answer.getArgument(3);
11451146
TracingContext context = answer.getArgument(4);
11461147
return getFileSystem().getAbfsClient()
1147-
.listPath(path, recursive, 1, continuation, context, null);
1148+
.listPath(path, recursive, 1, continuation, context, null, false);
11481149
})
11491150
.when(spiedClient)
11501151
.listPath(Mockito.anyString(), Mockito.anyBoolean(), Mockito.anyInt(),
11511152
Mockito.nullable(String.class),
1152-
Mockito.any(TracingContext.class), Mockito.nullable(URI.class));
1153+
Mockito.any(TracingContext.class), Mockito.nullable(URI.class), eq(false));
11531154
fs.rename(new Path("/testDir/dir1"), new Path("/testDir/dir2"));
11541155
for (int i = 0; i < 10; i++) {
11551156
Assertions.assertThat(fs.exists(new Path("/testDir/dir2/file" + i)))
@@ -1227,13 +1228,13 @@ public void testProducerStopOnRenameFailure() throws Exception {
12271228
listCallInvocation[0]++;
12281229
return getFileSystem().getAbfsClient().listPath(answer.getArgument(0),
12291230
answer.getArgument(1), 1,
1230-
answer.getArgument(3), answer.getArgument(4), answer.getArgument(5));
1231+
answer.getArgument(3), answer.getArgument(4), answer.getArgument(5), answer.getArgument(6));
12311232
}
12321233
return answer.callRealMethod();
12331234
})
12341235
.when(spiedClient)
12351236
.listPath(Mockito.anyString(), Mockito.anyBoolean(), Mockito.anyInt(),
1236-
Mockito.nullable(String.class), Mockito.any(TracingContext.class), Mockito.nullable(URI.class));
1237+
Mockito.nullable(String.class), Mockito.any(TracingContext.class), Mockito.nullable(URI.class), eq(false));
12371238
intercept(AccessDeniedException.class,
12381239
() -> {
12391240
fs.rename(new Path("/src"), new Path("/dst"));

0 commit comments

Comments
 (0)