diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index e8e04678400b2..ff9f4f3e464a7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -793,6 +793,10 @@ public static boolean isAclEnabled(Configuration conf) { RM_PREFIX + "delegation.token.max-lifetime"; public static final long RM_DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT = 7*24*60*60*1000; // 7 days + public static final String RM_DELEGATION_TOKEN_REMOVE_SCAN_INTERVAL_KEY = + RM_PREFIX + "delegation.token.remove-scan-interval"; + public static final long RM_DELEGATION_TOKEN_REMOVE_SCAN_INTERVAL_DEFAULT = + 60*60*1000; // 1 hour public static final String RM_DELEGATION_TOKEN_MAX_CONF_SIZE = RM_PREFIX + "delegation-token.max-conf-size-bytes"; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 313ac8b7142ee..8fd509d1e06f1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -1077,6 +1077,18 @@ 86400000 + + + This configuration is used for + how often the tokens are scanned for expired tokens in milliseconds. + the background thread(delegation token remover thread) + will delete expired tokens after the configured time. + the default value is 1h. + + yarn.resourcemanager.delegation.token.remove-scan-interval + 1h + + RM DelegationTokenRenewer thread timeout diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMSecretManagerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMSecretManagerService.java index 2933b40f4ac00..32369ba53ed5c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMSecretManagerService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMSecretManagerService.java @@ -30,6 +30,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.security.RMDelegationTokenSecretManager; import java.io.IOException; +import java.util.concurrent.TimeUnit; public class RMSecretManagerService extends AbstractService { @@ -135,9 +136,13 @@ protected RMDelegationTokenSecretManager createRMDelegationTokenSecretManager( long tokenRenewInterval = conf.getLong(YarnConfiguration.RM_DELEGATION_TOKEN_RENEW_INTERVAL_KEY, YarnConfiguration.RM_DELEGATION_TOKEN_RENEW_INTERVAL_DEFAULT); + long removeScanInterval = + conf.getTimeDuration(YarnConfiguration.RM_DELEGATION_TOKEN_REMOVE_SCAN_INTERVAL_KEY, + YarnConfiguration.RM_DELEGATION_TOKEN_REMOVE_SCAN_INTERVAL_DEFAULT, + TimeUnit.MILLISECONDS); return new RMDelegationTokenSecretManager(secretKeyInterval, - tokenMaxLifetime, tokenRenewInterval, 3600000, rmContext); + tokenMaxLifetime, tokenRenewInterval, removeScanInterval, rmContext); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMTokens.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMTokens.java index 556fd5bdf00d8..90ba812632854 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMTokens.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMTokens.java @@ -34,6 +34,7 @@ import java.net.InetSocketAddress; import java.security.PrivilegedAction; import java.security.PrivilegedExceptionAction; +import java.util.concurrent.TimeUnit; import org.apache.hadoop.test.LambdaTestUtils; import org.apache.hadoop.thirdparty.protobuf.InvalidProtocolBufferException; @@ -124,9 +125,13 @@ public void testDelegationToken() throws Exception { long initialInterval = 10000l; long maxLifetime= 20000l; long renewInterval = 10000l; + long delegationTokenRemoverScanInterval = + conf.getTimeDuration(YarnConfiguration.RM_DELEGATION_TOKEN_REMOVE_SCAN_INTERVAL_KEY, + YarnConfiguration.RM_DELEGATION_TOKEN_REMOVE_SCAN_INTERVAL_DEFAULT, + TimeUnit.MILLISECONDS); RMDelegationTokenSecretManager rmDtSecretManager = createRMDelegationTokenSecretManager( - initialInterval, maxLifetime, renewInterval); + initialInterval, maxLifetime, renewInterval, delegationTokenRemoverScanInterval); rmDtSecretManager.startThreads(); LOG.info("Creating DelegationTokenSecretManager with initialInterval: " + initialInterval + ", maxLifetime: " + maxLifetime @@ -574,7 +579,8 @@ private static ResourceScheduler createMockScheduler(Configuration conf) { private static RMDelegationTokenSecretManager createRMDelegationTokenSecretManager(long secretKeyInterval, - long tokenMaxLifetime, long tokenRenewInterval) { + long tokenMaxLifetime, long tokenRenewInterval, + long delegationTokenRemoverScanInterval) { ResourceManager rm = mock(ResourceManager.class); RMContext rmContext = mock(RMContext.class); when(rmContext.getStateStore()).thenReturn(new NullRMStateStore()); @@ -583,7 +589,7 @@ private static ResourceScheduler createMockScheduler(Configuration conf) { RMDelegationTokenSecretManager rmDtSecretManager = new RMDelegationTokenSecretManager(secretKeyInterval, tokenMaxLifetime, - tokenRenewInterval, 3600000, rmContext); + tokenRenewInterval, delegationTokenRemoverScanInterval, rmContext); return rmDtSecretManager; } }