Skip to content

Commit 1418d52

Browse files
bilaharithRogPodge
bilaharith
authored and
RogPodge
committed
HADOOP-16587. Make ABFS AAD endpoints configurable.
Contributed by Bilahari T H. This also addresses HADOOP-16498: AzureADAuthenticator cannot authenticate in China. Change-Id: I2441dd48b50b59b912b0242f7f5a4418cf94a87c
1 parent 19d7540 commit 1418d52

File tree

6 files changed

+118
-17
lines changed

6 files changed

+118
-17
lines changed

‎hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java‎

Lines changed: 33 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,9 +25,12 @@
2525

2626
import com.google.common.annotations.VisibleForTesting;
2727

28+
import org.apache.commons.lang3.StringUtils;
2829
import org.apache.hadoop.classification.InterfaceAudience;
2930
import org.apache.hadoop.classification.InterfaceStability;
3031
import org.apache.hadoop.conf.Configuration;
32+
import org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants;
33+
import org.apache.hadoop.fs.azurebfs.constants.AuthConfigurations;
3134
import org.apache.hadoop.fs.azurebfs.contracts.annotations.ConfigurationValidationAnnotations.IntegerConfigurationValidatorAnnotation;
3235
import org.apache.hadoop.fs.azurebfs.contracts.annotations.ConfigurationValidationAnnotations.LongConfigurationValidatorAnnotation;
3336
import org.apache.hadoop.fs.azurebfs.contracts.annotations.ConfigurationValidationAnnotations.StringConfigurationValidatorAnnotation;
@@ -69,6 +72,7 @@
6972
@InterfaceAudience.Private
7073
@InterfaceStability.Evolving
7174
public class AbfsConfiguration{
75+
7276
private final Configuration rawConfig;
7377
private final String accountName;
7478
private final boolean isSecure;
@@ -486,13 +490,25 @@ public AccessTokenProvider getTokenProvider() throws TokenAccessProviderExceptio
486490
String password = getPasswordString(FS_AZURE_ACCOUNT_OAUTH_USER_PASSWORD);
487491
tokenProvider = new UserPasswordTokenProvider(authEndpoint, username, password);
488492
} else if (tokenProviderClass == MsiTokenProvider.class) {
493+
String authEndpoint = getTrimmedPasswordString(
494+
FS_AZURE_ACCOUNT_OAUTH_MSI_ENDPOINT,
495+
AuthConfigurations.DEFAULT_FS_AZURE_ACCOUNT_OAUTH_MSI_ENDPOINT);
489496
String tenantGuid = getPasswordString(FS_AZURE_ACCOUNT_OAUTH_MSI_TENANT);
490497
String clientId = getPasswordString(FS_AZURE_ACCOUNT_OAUTH_CLIENT_ID);
491-
tokenProvider = new MsiTokenProvider(tenantGuid, clientId);
498+
String authority = getTrimmedPasswordString(
499+
FS_AZURE_ACCOUNT_OAUTH_MSI_AUTHORITY,
500+
AuthConfigurations.DEFAULT_FS_AZURE_ACCOUNT_OAUTH_MSI_AUTHORITY);
501+
authority = appendSlashIfNeeded(authority);
502+
tokenProvider = new MsiTokenProvider(authEndpoint, tenantGuid,
503+
clientId, authority);
492504
} else if (tokenProviderClass == RefreshTokenBasedTokenProvider.class) {
505+
String authEndpoint = getTrimmedPasswordString(
506+
FS_AZURE_ACCOUNT_OAUTH_REFRESH_TOKEN_ENDPOINT,
507+
AuthConfigurations.DEFAULT_FS_AZURE_ACCOUNT_OAUTH_REFRESH_TOKEN_ENDPOINT);
493508
String refreshToken = getPasswordString(FS_AZURE_ACCOUNT_OAUTH_REFRESH_TOKEN);
494509
String clientId = getPasswordString(FS_AZURE_ACCOUNT_OAUTH_CLIENT_ID);
495-
tokenProvider = new RefreshTokenBasedTokenProvider(clientId, refreshToken);
510+
tokenProvider = new RefreshTokenBasedTokenProvider(authEndpoint,
511+
clientId, refreshToken);
496512
} else {
497513
throw new IllegalArgumentException("Failed to initialize " + tokenProviderClass);
498514
}
@@ -649,4 +665,19 @@ void setDisableOutputStreamFlush(boolean disableOutputStreamFlush) {
649665
this.disableOutputStreamFlush = disableOutputStreamFlush;
650666
}
651667

668+
private String getTrimmedPasswordString(String key, String defaultValue) throws IOException {
669+
String value = getPasswordString(key);
670+
if (StringUtils.isBlank(value)) {
671+
value = defaultValue;
672+
}
673+
return value.trim();
674+
}
675+
676+
private String appendSlashIfNeeded(String authority) {
677+
if (!authority.endsWith(AbfsHttpConstants.FORWARD_SLASH)) {
678+
authority = authority + AbfsHttpConstants.FORWARD_SLASH;
679+
}
680+
return authority;
681+
}
682+
652683
}

‎hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/AuthConfigurations.java‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
/**
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
* <p>
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
* <p>
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
19+
package org.apache.hadoop.fs.azurebfs.constants;
20+
21+
import org.apache.hadoop.classification.InterfaceAudience;
22+
import org.apache.hadoop.classification.InterfaceStability;
23+
24+
/**
25+
* Responsible to keep all the Azure Blob File System auth related
26+
* configurations.
27+
*/
28+
@InterfaceAudience.Public
29+
@InterfaceStability.Evolving
30+
public final class AuthConfigurations {
31+
32+
/** Default OAuth token end point for the MSI flow. */
33+
public static final String DEFAULT_FS_AZURE_ACCOUNT_OAUTH_MSI_ENDPOINT =
34+
"http://169.254.169.254/metadata/identity/oauth2/token";
35+
/** Default value for authority for the MSI flow. */
36+
public static final String DEFAULT_FS_AZURE_ACCOUNT_OAUTH_MSI_AUTHORITY =
37+
"https://login.microsoftonline.com/";
38+
/** Default OAuth token end point for the refresh token flow. */
39+
public static final String
40+
DEFAULT_FS_AZURE_ACCOUNT_OAUTH_REFRESH_TOKEN_ENDPOINT =
41+
"https://login.microsoftonline.com/Common/oauth2/token";
42+
43+
private AuthConfigurations() {
44+
}
45+
}

‎hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -102,12 +102,18 @@ public final class ConfigurationKeys {
102102
public static final String FS_AZURE_ACCOUNT_OAUTH_CLIENT_ENDPOINT = "fs.azure.account.oauth2.client.endpoint";
103103
/** Key for oauth msi tenant id: {@value}. */
104104
public static final String FS_AZURE_ACCOUNT_OAUTH_MSI_TENANT = "fs.azure.account.oauth2.msi.tenant";
105+
/** Key for oauth msi endpoint: {@value}. */
106+
public static final String FS_AZURE_ACCOUNT_OAUTH_MSI_ENDPOINT = "fs.azure.account.oauth2.msi.endpoint";
107+
/** Key for oauth msi Authority: {@value}. */
108+
public static final String FS_AZURE_ACCOUNT_OAUTH_MSI_AUTHORITY = "fs.azure.account.oauth2.msi.authority";
105109
/** Key for oauth user name: {@value}. */
106110
public static final String FS_AZURE_ACCOUNT_OAUTH_USER_NAME = "fs.azure.account.oauth2.user.name";
107111
/** Key for oauth user password: {@value}. */
108112
public static final String FS_AZURE_ACCOUNT_OAUTH_USER_PASSWORD = "fs.azure.account.oauth2.user.password";
109113
/** Key for oauth refresh token: {@value}. */
110114
public static final String FS_AZURE_ACCOUNT_OAUTH_REFRESH_TOKEN = "fs.azure.account.oauth2.refresh.token";
115+
/** Key for oauth AAD refresh token endpoint: {@value}. */
116+
public static final String FS_AZURE_ACCOUNT_OAUTH_REFRESH_TOKEN_ENDPOINT = "fs.azure.account.oauth2.refresh.token.endpoint";
111117

112118
public static String accountProperty(String property, String account) {
113119
return property + "." + account;

‎hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/oauth2/AzureADAuthenticator.java‎

Lines changed: 14 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -101,6 +101,9 @@ public static AzureADToken getTokenUsingClientCreds(String authEndpoint,
101101
* an Azure VM with MSI extension
102102
* enabled.
103103
*
104+
* @param authEndpoint the OAuth 2.0 token endpoint associated
105+
* with the user's directory (obtain from
106+
* Active Directory configuration)
104107
* @param tenantGuid (optional) The guid of the AAD tenant. Can be {@code null}.
105108
* @param clientId (optional) The clientId guid of the MSI service
106109
* principal to use. Can be {@code null}.
@@ -109,17 +112,16 @@ public static AzureADToken getTokenUsingClientCreds(String authEndpoint,
109112
* @return {@link AzureADToken} obtained using the creds
110113
* @throws IOException throws IOException if there is a failure in obtaining the token
111114
*/
112-
public static AzureADToken getTokenFromMsi(String tenantGuid, String clientId,
113-
boolean bypassCache) throws IOException {
114-
String authEndpoint = "http://169.254.169.254/metadata/identity/oauth2/token";
115-
115+
public static AzureADToken getTokenFromMsi(final String authEndpoint,
116+
final String tenantGuid, final String clientId, String authority,
117+
boolean bypassCache) throws IOException {
116118
QueryParams qp = new QueryParams();
117119
qp.add("api-version", "2018-02-01");
118120
qp.add("resource", RESOURCE_NAME);
119121

120-
121122
if (tenantGuid != null && tenantGuid.length() > 0) {
122-
String authority = "https://login.microsoftonline.com/" + tenantGuid;
123+
authority = authority + tenantGuid;
124+
LOG.debug("MSI authority : {}", authority);
123125
qp.add("authority", authority);
124126
}
125127

@@ -141,14 +143,17 @@ public static AzureADToken getTokenFromMsi(String tenantGuid, String clientId,
141143
/**
142144
* Gets Azure Active Directory token using refresh token.
143145
*
146+
* @param authEndpoint the OAuth 2.0 token endpoint associated
147+
* with the user's directory (obtain from
148+
* Active Directory configuration)
144149
* @param clientId the client ID (GUID) of the client web app obtained from Azure Active Directory configuration
145150
* @param refreshToken the refresh token
146151
* @return {@link AzureADToken} obtained using the refresh token
147152
* @throws IOException throws IOException if there is a failure in connecting to Azure AD
148153
*/
149-
public static AzureADToken getTokenUsingRefreshToken(String clientId,
150-
String refreshToken) throws IOException {
151-
String authEndpoint = "https://login.microsoftonline.com/Common/oauth2/token";
154+
public static AzureADToken getTokenUsingRefreshToken(
155+
final String authEndpoint, final String clientId,
156+
final String refreshToken) throws IOException {
152157
QueryParams qp = new QueryParams();
153158
qp.add("grant_type", "refresh_token");
154159
qp.add("refresh_token", refreshToken);

‎hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/oauth2/MsiTokenProvider.java‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -28,21 +28,29 @@
2828
*/
2929
public class MsiTokenProvider extends AccessTokenProvider {
3030

31+
private final String authEndpoint;
32+
33+
private final String authority;
34+
3135
private final String tenantGuid;
3236

3337
private final String clientId;
3438

3539
private static final Logger LOG = LoggerFactory.getLogger(AccessTokenProvider.class);
3640

37-
public MsiTokenProvider(final String tenantGuid, final String clientId) {
41+
public MsiTokenProvider(final String authEndpoint, final String tenantGuid,
42+
final String clientId, final String authority) {
43+
this.authEndpoint = authEndpoint;
3844
this.tenantGuid = tenantGuid;
3945
this.clientId = clientId;
46+
this.authority = authority;
4047
}
4148

4249
@Override
4350
protected AzureADToken refreshToken() throws IOException {
4451
LOG.debug("AADToken: refreshing token from MSI");
45-
AzureADToken token = AzureADAuthenticator.getTokenFromMsi(tenantGuid, clientId, false);
52+
AzureADToken token = AzureADAuthenticator
53+
.getTokenFromMsi(authEndpoint, tenantGuid, clientId, authority, false);
4654
return token;
4755
}
48-
}
56+
}

‎hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/oauth2/RefreshTokenBasedTokenProvider.java‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,8 @@
3131
public class RefreshTokenBasedTokenProvider extends AccessTokenProvider {
3232
private static final Logger LOG = LoggerFactory.getLogger(AccessTokenProvider.class);
3333

34+
private final String authEndpoint;
35+
3436
private final String clientId;
3537

3638
private final String refreshToken;
@@ -41,9 +43,12 @@ public class RefreshTokenBasedTokenProvider extends AccessTokenProvider {
4143
* @param clientId the client ID (GUID) of the client web app obtained from Azure Active Directory configuration
4244
* @param refreshToken the refresh token
4345
*/
44-
public RefreshTokenBasedTokenProvider(String clientId, String refreshToken) {
46+
public RefreshTokenBasedTokenProvider(final String authEndpoint,
47+
String clientId, String refreshToken) {
48+
Preconditions.checkNotNull(authEndpoint, "authEndpoint");
4549
Preconditions.checkNotNull(clientId, "clientId");
4650
Preconditions.checkNotNull(refreshToken, "refreshToken");
51+
this.authEndpoint = authEndpoint;
4752
this.clientId = clientId;
4853
this.refreshToken = refreshToken;
4954
}
@@ -52,6 +57,7 @@ public RefreshTokenBasedTokenProvider(String clientId, String refreshToken) {
5257
@Override
5358
protected AzureADToken refreshToken() throws IOException {
5459
LOG.debug("AADToken: refreshing refresh-token based token");
55-
return AzureADAuthenticator.getTokenUsingRefreshToken(clientId, refreshToken);
60+
return AzureADAuthenticator
61+
.getTokenUsingRefreshToken(authEndpoint, clientId, refreshToken);
5662
}
57-
}
63+
}

0 commit comments

Comments
 (0)