Skip to content

Commit 16e1e58

Browse files
anujmodi2021anujmodi2021
committed
Added more tests
1 parent e18350c commit 16e1e58

File tree

3 files changed

+115
-38
lines changed

3 files changed

+115
-38
lines changed

‎hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java‎

Lines changed: 22 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -165,6 +165,8 @@ public abstract class AbfsClient implements Closeable {
165165
private String clientProvidedEncryptionKey = null;
166166
private String clientProvidedEncryptionKeySHA = null;
167167
private final IdentityTransformerInterface identityTransformer;
168+
private final String userName;
169+
private String primaryUserGroup;
168170

169171
private final String accountName;
170172
private final AuthType authType;
@@ -306,6 +308,22 @@ private AbfsClient(final URL baseUrl,
306308
throw new IOException(e);
307309
}
308310
LOG.trace("IdentityTransformer init complete");
311+
312+
UserGroupInformation userGroupInformation = UserGroupInformation.getCurrentUser();
313+
this.userName = userGroupInformation.getShortUserName();
314+
LOG.trace("UGI init complete");
315+
if (!abfsConfiguration.getSkipUserGroupMetadataDuringInitialization()) {
316+
try {
317+
this.primaryUserGroup = userGroupInformation.getPrimaryGroupName();
318+
} catch (IOException ex) {
319+
LOG.error("Failed to get primary group for {}, using user name as primary group name", userName);
320+
this.primaryUserGroup = userName;
321+
}
322+
} else {
323+
//Provide a default group name
324+
this.primaryUserGroup = userName;
325+
}
326+
LOG.trace("primaryUserGroup is {}", this.primaryUserGroup);
309327
}
310328

311329
public AbfsClient(final URL baseUrl, final SharedKeyCredentials sharedKeyCredentials,
@@ -1785,37 +1803,6 @@ protected AbfsRestOperation getSuccessOp(final AbfsRestOperationType operationTy
17851803
return successOp;
17861804
}
17871805

1788-
/**
1789-
* Get the primary user group name.
1790-
* @return primary user group name
1791-
* @throws AzureBlobFileSystemException if unable to get the primary user group
1792-
*/
1793-
private String getPrimaryUserGroup() throws AzureBlobFileSystemException {
1794-
if (!getAbfsConfiguration().getSkipUserGroupMetadataDuringInitialization()) {
1795-
try {
1796-
return UserGroupInformation.getCurrentUser().getPrimaryGroupName();
1797-
} catch (IOException ex) {
1798-
LOG.error("Failed to get primary group for {}, using user name as primary group name",
1799-
getPrimaryUser());
1800-
}
1801-
}
1802-
//Provide a default group name
1803-
return getPrimaryUser();
1804-
}
1805-
1806-
/**
1807-
* Get the primary username.
1808-
* @return primary username
1809-
* @throws AzureBlobFileSystemException if unable to get the primary user
1810-
*/
1811-
private String getPrimaryUser() throws AzureBlobFileSystemException {
1812-
try {
1813-
return UserGroupInformation.getCurrentUser().getUserName();
1814-
} catch (IOException ex) {
1815-
throw new AbfsDriverException(ex);
1816-
}
1817-
}
1818-
18191806
/**
18201807
* Creates a VersionedFileStatus object from the ListResultEntrySchema.
18211808
* @param entry ListResultEntrySchema object.
@@ -1829,10 +1816,10 @@ protected VersionedFileStatus getVersionedFileStatusFromEntry(
18291816
String owner = null, group = null;
18301817
try{
18311818
if (identityTransformer != null) {
1832-
owner = identityTransformer.transformIdentityForGetRequest(
1833-
entry.owner(), true, getPrimaryUser());
1834-
group = identityTransformer.transformIdentityForGetRequest(
1835-
entry.group(), false, getPrimaryUserGroup());
1819+
owner = identityTransformer.transformIdentityForGetRequest(entry.owner(),
1820+
true, userName);
1821+
group = identityTransformer.transformIdentityForGetRequest(entry.group(),
1822+
false, primaryUserGroup);
18361823
}
18371824
} catch (IOException ex) {
18381825
LOG.error("Failed to get owner/group for path {}", entry.name(), ex);

‎hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemListStatus.java‎

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@
4343
import org.apache.hadoop.fs.FileStatus;
4444
import org.apache.hadoop.fs.LocatedFileStatus;
4545
import org.apache.hadoop.fs.Path;
46+
import org.apache.hadoop.fs.azurebfs.constants.AbfsServiceType;
4647
import org.apache.hadoop.fs.azurebfs.constants.FSOperationType;
4748
import org.apache.hadoop.fs.azurebfs.constants.HttpHeaderConfigurations;
4849
import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AbfsDriverException;
@@ -418,10 +419,11 @@ public void testEmptyListingInSubsequentCall() throws IOException {
418419
private void testEmptyListingInSubsequentCallInternal(String firstCT,
419420
boolean isfirstEmpty, String secondCT, boolean isSecondEmpty,
420421
int expectedInvocations, int expectedSize) throws IOException {
422+
assumeBlobServiceType();
421423
AzureBlobFileSystem spiedFs = Mockito.spy(getFileSystem());
422424
AzureBlobFileSystemStore spiedStore = Mockito.spy(spiedFs.getAbfsStore());
423425
spiedStore.getAbfsConfiguration().setListMaxResults(1);
424-
AbfsClient spiedClient = Mockito.spy(spiedStore.getClient());
426+
AbfsBlobClient spiedClient = Mockito.spy(spiedStore.getClientHandler().getBlobClient());
425427
Mockito.doReturn(spiedStore).when(spiedFs).getAbfsStore();
426428
Mockito.doReturn(spiedClient).when(spiedStore).getClient();
427429
spiedFs.mkdirs(new Path("/testPath"));
@@ -458,12 +460,19 @@ private void testEmptyListingInSubsequentCallInternal(String firstCT,
458460

459461
FileStatus[] list = spiedFs.listStatus(new Path("/testPath"));
460462

461-
Mockito.verify(spiedClient, times(expectedInvocations)).listPath(eq("/testPath"), eq(false), eq(1),
463+
Mockito.verify(spiedClient, times(expectedInvocations))
464+
.listPath(eq("/testPath"), eq(false), eq(1),
462465
any(), any(TracingContext.class), any());
466+
Mockito.verify(spiedClient, times(1))
467+
.postListProcessing(eq("/testPath"), any(), any(), any());
463468
Assertions.assertThat(list).hasSize(expectedSize);
464469

465470
if (expectedSize == 0) {
466-
Mockito.verify(spiedClient, times(1)).postListProcessing(eq("/testPath"), any(), any(), any());
471+
Mockito.verify(spiedClient, times(1))
472+
.getPathStatus(eq("/testPath"), any(), eq(null), eq(false));
473+
} else {
474+
Mockito.verify(spiedClient, times(0))
475+
.getPathStatus(eq("/testPath"), any(), eq(null), eq(false));
467476
}
468477
}
469478

‎hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestAbfsClient.java‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
import java.net.URI;
2424
import java.net.URISyntaxException;
2525
import java.net.URL;
26+
import java.util.ArrayList;
2627
import java.util.Arrays;
2728
import java.util.List;
2829
import java.util.Random;
@@ -42,6 +43,7 @@
4243
import org.apache.hadoop.fs.azurebfs.AbfsCountersImpl;
4344
import org.apache.hadoop.fs.azurebfs.AbstractAbfsIntegrationTest;
4445
import org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem;
46+
import org.apache.hadoop.fs.azurebfs.AzureBlobFileSystemStore;
4547
import org.apache.hadoop.fs.azurebfs.TestAbfsConfigurationFieldsValidation;
4648
import org.apache.hadoop.fs.azurebfs.constants.AbfsServiceType;
4749
import org.apache.hadoop.fs.azurebfs.constants.FSOperationType;
@@ -58,6 +60,7 @@
5860
import org.apache.http.HttpResponse;
5961

6062
import static java.net.HttpURLConnection.HTTP_NOT_FOUND;
63+
import static org.apache.hadoop.fs.azurebfs.ITestAzureBlobFileSystemListStatus.TEST_CONTINUATION_TOKEN;
6164
import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.APPEND_ACTION;
6265
import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.EXPECT_100_JDK_ERROR;
6366
import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.HTTP_METHOD_PATCH;
@@ -75,6 +78,7 @@
7578
import static org.mockito.ArgumentMatchers.any;
7679
import static org.mockito.ArgumentMatchers.eq;
7780
import static org.mockito.Mockito.mock;
81+
import static org.mockito.Mockito.times;
7882
import static org.mockito.Mockito.when;
7983

8084
import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.APN_VERSION;
@@ -729,4 +733,81 @@ public void testExpectHundredContinue() throws Exception {
729733
.describedAs("The expect header is not false")
730734
.isFalse();
731735
}
736+
737+
@Test
738+
public void testIsNonEmptyDirectory() throws IOException {
739+
testIsNonEmptyDirectoryInternal(EMPTY_STRING, true, EMPTY_STRING,
740+
true, 1, false);
741+
testIsNonEmptyDirectoryInternal(EMPTY_STRING, false, EMPTY_STRING,
742+
false, 1, true);
743+
744+
testIsNonEmptyDirectoryInternal(TEST_CONTINUATION_TOKEN, true, EMPTY_STRING,
745+
true, 2, false);
746+
testIsNonEmptyDirectoryInternal(TEST_CONTINUATION_TOKEN, true, EMPTY_STRING,
747+
false, 2, true);
748+
testIsNonEmptyDirectoryInternal(TEST_CONTINUATION_TOKEN, true, TEST_CONTINUATION_TOKEN,
749+
true, 3, false);
750+
testIsNonEmptyDirectoryInternal(TEST_CONTINUATION_TOKEN, true, TEST_CONTINUATION_TOKEN,
751+
false, 2, true);
752+
753+
testIsNonEmptyDirectoryInternal(TEST_CONTINUATION_TOKEN, false, EMPTY_STRING,
754+
true, 1, true);
755+
testIsNonEmptyDirectoryInternal(TEST_CONTINUATION_TOKEN, false, EMPTY_STRING,
756+
false, 1, true);
757+
testIsNonEmptyDirectoryInternal(TEST_CONTINUATION_TOKEN, false, TEST_CONTINUATION_TOKEN,
758+
true, 1, true);
759+
testIsNonEmptyDirectoryInternal(TEST_CONTINUATION_TOKEN, false, TEST_CONTINUATION_TOKEN,
760+
false, 1, true);
761+
}
762+
763+
private void testIsNonEmptyDirectoryInternal(String firstCT,
764+
boolean isfirstEmpty, String secondCT, boolean isSecondEmpty,
765+
int expectedInvocations, boolean isNonEmpty) throws IOException {
766+
767+
assumeBlobServiceType();
768+
AzureBlobFileSystem spiedFs = Mockito.spy(getFileSystem());
769+
AzureBlobFileSystemStore spiedStore = Mockito.spy(spiedFs.getAbfsStore());
770+
AbfsBlobClient spiedClient = Mockito.spy(spiedStore.getClientHandler().getBlobClient());
771+
Mockito.doReturn(spiedStore).when(spiedFs).getAbfsStore();
772+
Mockito.doReturn(spiedClient).when(spiedStore).getClient();
773+
VersionedFileStatus status1 = new VersionedFileStatus(
774+
"owner", "group", null, false, 0, false, 0, 0, 0,
775+
new Path("/testPath/file1"), "version", "encryptionContext");
776+
VersionedFileStatus status2 = new VersionedFileStatus(
777+
"owner", "group", null, false, 0, false, 0, 0, 0,
778+
new Path("/testPath/file2"), "version", "encryptionContext");
779+
780+
List<VersionedFileStatus> mockedList1 = new ArrayList<>();
781+
mockedList1.add(status1);
782+
List<VersionedFileStatus> mockedList2 = new ArrayList<>();
783+
mockedList2.add(status2);
784+
785+
ListResponseData listResponseData1 = new ListResponseData();
786+
listResponseData1.setContinuationToken(firstCT);
787+
listResponseData1.setFileStatusList(isfirstEmpty ? new ArrayList<>() : mockedList1);
788+
listResponseData1.setOp(Mockito.mock(AbfsRestOperation.class));
789+
790+
ListResponseData listResponseData2 = new ListResponseData();
791+
listResponseData2.setContinuationToken(secondCT);
792+
listResponseData2.setFileStatusList(isSecondEmpty ? new ArrayList<>() : mockedList2);
793+
listResponseData2.setOp(Mockito.mock(AbfsRestOperation.class));
794+
795+
ListResponseData listResponseData3 = new ListResponseData();
796+
listResponseData3.setContinuationToken(EMPTY_STRING);
797+
listResponseData3.setFileStatusList(new ArrayList<>());
798+
listResponseData3.setOp(Mockito.mock(AbfsRestOperation.class));
799+
800+
Mockito.doReturn(listResponseData1).doReturn(listResponseData2).doReturn(listResponseData3)
801+
.when(spiedClient).listPath(eq("/testPath"), eq(false), eq(1),
802+
any(), any(), any());
803+
804+
Assertions.assertThat(spiedClient.isNonEmptyDirectory("/testPath",
805+
Mockito.mock(TracingContext.class)))
806+
.describedAs("isNonEmptyDirectory in client giving unexpected results")
807+
.isEqualTo(isNonEmpty);
808+
809+
Mockito.verify(spiedClient, times(expectedInvocations))
810+
.listPath(eq("/testPath"), eq(false), eq(1),
811+
any(), any(TracingContext.class), any());
812+
}
732813
}

0 commit comments

Comments
 (0)