From f2d59297a8c26d11307bcd46f6b0c7d46810185e Mon Sep 17 00:00:00 2001 From: jaymode Date: Tue, 5 Feb 2019 13:43:56 -0700 Subject: [PATCH] Add an authentication cache for API keys This commit adds an authentication cache for API keys that caches the hash of an API key with a faster hash. This will enable better performance when API keys are used for bulk or heavy searching. --- .../xpack/security/Security.java | 6 +- .../xpack/security/authc/ApiKeyService.java | 144 +++++++++++++++--- .../security/authc/ApiKeyServiceTests.java | 113 +++++++++++++- .../authc/AuthenticationServiceTests.java | 2 +- 4 files changed, 230 insertions(+), 35 deletions(-) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java index bf4a7080e6d71..b5b10b06622fe 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java @@ -438,7 +438,8 @@ Collection createComponents(Client client, ThreadPool threadPool, Cluste rolesProviders.addAll(extension.getRolesProviders(settings, resourceWatcherService)); } - final ApiKeyService apiKeyService = new ApiKeyService(settings, Clock.systemUTC(), client, securityIndex.get(), clusterService); + final ApiKeyService apiKeyService = new ApiKeyService(settings, Clock.systemUTC(), client, securityIndex.get(), clusterService, + threadPool); components.add(apiKeyService); final CompositeRolesStore allRolesStore = new CompositeRolesStore(settings, fileRolesStore, nativeRolesStore, reservedRolesStore, privilegeStore, rolesProviders, threadPool.getThreadContext(), getLicenseState(), fieldPermissionsCache, apiKeyService); @@ -631,6 +632,9 @@ public static List> getSettings(boolean transportClientMode, List PASSWORD_HASHING_ALGORITHM = new Setting<>( "xpack.security.authc.api_key_hashing.algorithm", "pbkdf2", Function.identity(), v -> { @@ -117,6 +124,12 @@ public class ApiKeyService { TimeValue.MINUS_ONE, Property.NodeScope); public static final Setting DELETE_INTERVAL = Setting.timeSetting("xpack.security.authc.api_key.delete.interval", TimeValue.timeValueHours(24L), Property.NodeScope); + public static final Setting CACHE_HASH_ALGO_SETTING = Setting.simpleString("xpack.security.authc.api_key.cache.hash_algo", + "ssha256", Setting.Property.NodeScope); + public static final Setting CACHE_TTL_SETTING = Setting.timeSetting("xpack.security.authc.api_key.cache.ttl", + TimeValue.timeValueHours(24L), Property.NodeScope); + public static final Setting CACHE_MAX_KEYS_SETTING = Setting.intSetting("xpack.security.authc.api_key.cache.max_keys", + 10000, Property.NodeScope); private final Clock clock; private final Client client; @@ -127,11 +140,14 @@ public class ApiKeyService { private final Settings settings; private final ExpiredApiKeysRemover expiredApiKeysRemover; private final TimeValue deleteInterval; + private final Cache> apiKeyAuthCache; + private final Hasher cacheHasher; + private final ThreadPool threadPool; private volatile long lastExpirationRunMs; - public ApiKeyService(Settings settings, Clock clock, Client client, SecurityIndexManager securityIndex, - ClusterService clusterService) { + public ApiKeyService(Settings settings, Clock clock, Client client, SecurityIndexManager securityIndex, ClusterService clusterService, + ThreadPool threadPool) { this.clock = clock; this.client = client; this.securityIndex = securityIndex; @@ -141,6 +157,17 @@ public ApiKeyService(Settings settings, Clock clock, Client client, SecurityInde this.settings = settings; this.deleteInterval = DELETE_INTERVAL.get(settings); this.expiredApiKeysRemover = new ExpiredApiKeysRemover(settings, client); + this.threadPool = threadPool; + this.cacheHasher = Hasher.resolve(CACHE_HASH_ALGO_SETTING.get(settings)); + final TimeValue ttl = CACHE_TTL_SETTING.get(settings); + if (ttl.getNanos() > 0) { + this.apiKeyAuthCache = CacheBuilder.>builder() + .setExpireAfterWrite(ttl) + .setMaximumWeight(CACHE_MAX_KEYS_SETTING.get(settings)) + .build(); + } else { + this.apiKeyAuthCache = null; + } } /** @@ -363,8 +390,8 @@ private List parseRoleDescriptors(final String apiKeyId, final M * @param credentials the credentials provided by the user * @param listener the listener to notify after verification */ - static void validateApiKeyCredentials(Map source, ApiKeyCredentials credentials, Clock clock, - ActionListener listener) { + void validateApiKeyCredentials(Map source, ApiKeyCredentials credentials, Clock clock, + ActionListener listener) { final Boolean invalidated = (Boolean) source.get("api_key_invalidated"); if (invalidated == null) { listener.onResponse(AuthenticationResult.terminate("api key document is missing invalidated field", null)); @@ -375,33 +402,87 @@ static void validateApiKeyCredentials(Map source, ApiKeyCredenti if (apiKeyHash == null) { throw new IllegalStateException("api key hash is missing"); } - final boolean verified = verifyKeyAgainstHash(apiKeyHash, credentials); - - if (verified) { - final Long expirationEpochMilli = (Long) source.get("expiration_time"); - if (expirationEpochMilli == null || Instant.ofEpochMilli(expirationEpochMilli).isAfter(clock.instant())) { - final Map creator = Objects.requireNonNull((Map) source.get("creator")); - final String principal = Objects.requireNonNull((String) creator.get("principal")); - final Map metadata = (Map) creator.get("metadata"); - final Map roleDescriptors = (Map) source.get("role_descriptors"); - final Map limitedByRoleDescriptors = (Map) source.get("limited_by_role_descriptors"); - final String[] roleNames = (roleDescriptors != null) ? roleDescriptors.keySet().toArray(Strings.EMPTY_ARRAY) - : limitedByRoleDescriptors.keySet().toArray(Strings.EMPTY_ARRAY); - final User apiKeyUser = new User(principal, roleNames, null, null, metadata, true); - final Map authResultMetadata = new HashMap<>(); - authResultMetadata.put(API_KEY_ROLE_DESCRIPTORS_KEY, roleDescriptors); - authResultMetadata.put(API_KEY_LIMITED_ROLE_DESCRIPTORS_KEY, limitedByRoleDescriptors); - authResultMetadata.put(API_KEY_ID_KEY, credentials.getId()); - listener.onResponse(AuthenticationResult.success(apiKeyUser, authResultMetadata)); + + if (apiKeyAuthCache != null) { + final AtomicBoolean valueAlreadyInCache = new AtomicBoolean(true); + final ListenableFuture listenableCacheEntry; + try { + listenableCacheEntry = apiKeyAuthCache.computeIfAbsent(credentials.getId(), + k -> { + valueAlreadyInCache.set(false); + return new ListenableFuture<>(); + }); + } catch (ExecutionException e) { + listener.onFailure(e); + return; + } + + if (valueAlreadyInCache.get()) { + listenableCacheEntry.addListener(ActionListener.wrap(result -> { + if (result.success) { + if (result.verify(credentials.getKey())) { + // move on + validateApiKeyExpiration(source, credentials, clock, listener); + } else { + listener.onResponse(AuthenticationResult.unsuccessful("invalid credentials", null)); + } + } else if (result.verify(credentials.getKey())) { // same key, pass the same result + listener.onResponse(AuthenticationResult.unsuccessful("invalid credentials", null)); + } else { + apiKeyAuthCache.invalidate(credentials.getId(), listenableCacheEntry); + validateApiKeyCredentials(source, credentials, clock, listener); + } + }, listener::onFailure), + threadPool.generic(), threadPool.getThreadContext()); } else { - listener.onResponse(AuthenticationResult.terminate("api key is expired", null)); + final boolean verified = verifyKeyAgainstHash(apiKeyHash, credentials); + listenableCacheEntry.onResponse(new CachedApiKeyHashResult(verified, credentials.getKey())); + if (verified) { + // move on + validateApiKeyExpiration(source, credentials, clock, listener); + } else { + listener.onResponse(AuthenticationResult.unsuccessful("invalid credentials", null)); + } } } else { - listener.onResponse(AuthenticationResult.unsuccessful("invalid credentials", null)); + final boolean verified = verifyKeyAgainstHash(apiKeyHash, credentials); + if (verified) { + // move on + validateApiKeyExpiration(source, credentials, clock, listener); + } else { + listener.onResponse(AuthenticationResult.unsuccessful("invalid credentials", null)); + } } } } + // pkg private for testing + CachedApiKeyHashResult getFromCache(String id) { + return apiKeyAuthCache == null ? null : FutureUtils.get(apiKeyAuthCache.get(id), 0L, TimeUnit.MILLISECONDS); + } + + private void validateApiKeyExpiration(Map source, ApiKeyCredentials credentials, Clock clock, + ActionListener listener) { + final Long expirationEpochMilli = (Long) source.get("expiration_time"); + if (expirationEpochMilli == null || Instant.ofEpochMilli(expirationEpochMilli).isAfter(clock.instant())) { + final Map creator = Objects.requireNonNull((Map) source.get("creator")); + final String principal = Objects.requireNonNull((String) creator.get("principal")); + final Map metadata = (Map) creator.get("metadata"); + final Map roleDescriptors = (Map) source.get("role_descriptors"); + final Map limitedByRoleDescriptors = (Map) source.get("limited_by_role_descriptors"); + final String[] roleNames = (roleDescriptors != null) ? roleDescriptors.keySet().toArray(Strings.EMPTY_ARRAY) + : limitedByRoleDescriptors.keySet().toArray(Strings.EMPTY_ARRAY); + final User apiKeyUser = new User(principal, roleNames, null, null, metadata, true); + final Map authResultMetadata = new HashMap<>(); + authResultMetadata.put(API_KEY_ROLE_DESCRIPTORS_KEY, roleDescriptors); + authResultMetadata.put(API_KEY_LIMITED_ROLE_DESCRIPTORS_KEY, limitedByRoleDescriptors); + authResultMetadata.put(API_KEY_ID_KEY, credentials.getId()); + listener.onResponse(AuthenticationResult.success(apiKeyUser, authResultMetadata)); + } else { + listener.onResponse(AuthenticationResult.terminate("api key is expired", null)); + } + } + /** * Gets the API Key from the Authorization header if the header begins with * ApiKey @@ -851,4 +932,17 @@ public void getApiKeyForApiKeyName(String apiKeyName, ActionListener future = new PlainActionFuture<>(); - ApiKeyService.validateApiKeyCredentials(sourceMap, creds, Clock.systemUTC(), future); + service.validateApiKeyCredentials(sourceMap, creds, Clock.systemUTC(), future); AuthenticationResult result = future.get(); assertNotNull(result); assertTrue(result.isAuthenticated()); @@ -134,7 +140,7 @@ public void testValidateApiKey() throws Exception { sourceMap.put("expiration_time", Clock.systemUTC().instant().plus(1L, ChronoUnit.HOURS).toEpochMilli()); future = new PlainActionFuture<>(); - ApiKeyService.validateApiKeyCredentials(sourceMap, creds, Clock.systemUTC(), future); + service.validateApiKeyCredentials(sourceMap, creds, Clock.systemUTC(), future); result = future.get(); assertNotNull(result); assertTrue(result.isAuthenticated()); @@ -147,7 +153,7 @@ public void testValidateApiKey() throws Exception { sourceMap.put("expiration_time", Clock.systemUTC().instant().minus(1L, ChronoUnit.HOURS).toEpochMilli()); future = new PlainActionFuture<>(); - ApiKeyService.validateApiKeyCredentials(sourceMap, creds, Clock.systemUTC(), future); + service.validateApiKeyCredentials(sourceMap, creds, Clock.systemUTC(), future); result = future.get(); assertNotNull(result); assertFalse(result.isAuthenticated()); @@ -155,7 +161,7 @@ public void testValidateApiKey() throws Exception { sourceMap.remove("expiration_time"); creds = new ApiKeyService.ApiKeyCredentials(randomAlphaOfLength(12), new SecureString(randomAlphaOfLength(15).toCharArray())); future = new PlainActionFuture<>(); - ApiKeyService.validateApiKeyCredentials(sourceMap, creds, Clock.systemUTC(), future); + service.validateApiKeyCredentials(sourceMap, creds, Clock.systemUTC(), future); result = future.get(); assertNotNull(result); assertFalse(result.isAuthenticated()); @@ -163,7 +169,7 @@ public void testValidateApiKey() throws Exception { sourceMap.put("api_key_invalidated", true); creds = new ApiKeyService.ApiKeyCredentials(randomAlphaOfLength(12), new SecureString(randomAlphaOfLength(15).toCharArray())); future = new PlainActionFuture<>(); - ApiKeyService.validateApiKeyCredentials(sourceMap, creds, Clock.systemUTC(), future); + service.validateApiKeyCredentials(sourceMap, creds, Clock.systemUTC(), future); result = future.get(); assertNotNull(result); assertFalse(result.isAuthenticated()); @@ -188,7 +194,7 @@ public void testGetRolesForApiKeyNotInContext() throws Exception { final Authentication authentication = new Authentication(new User("joe"), new RealmRef("apikey", "apikey", "node"), null, Version.CURRENT, AuthenticationType.API_KEY, authMetadata); ApiKeyService service = new ApiKeyService(Settings.EMPTY, Clock.systemUTC(), null, null, - ClusterServiceUtils.createClusterService(threadPool)); + ClusterServiceUtils.createClusterService(threadPool), threadPool); PlainActionFuture roleFuture = new PlainActionFuture<>(); service.getRoleForApiKey(authentication, roleFuture); @@ -242,7 +248,7 @@ public void testGetRolesForApiKey() throws Exception { } ).when(privilegesStore).getPrivileges(any(Collection.class), any(Collection.class), any(ActionListener.class)); ApiKeyService service = new ApiKeyService(Settings.EMPTY, Clock.systemUTC(), null, null, - ClusterServiceUtils.createClusterService(threadPool)); + ClusterServiceUtils.createClusterService(threadPool), threadPool); PlainActionFuture roleFuture = new PlainActionFuture<>(); service.getRoleForApiKey(authentication, roleFuture); @@ -258,4 +264,95 @@ public void testGetRolesForApiKey() throws Exception { assertThat(result.getLimitedByRoleDescriptors().get(0).getName(), is("limited role")); } } + + public void testApiKeyCache() { + final String apiKey = randomAlphaOfLength(16); + Hasher hasher = randomFrom(Hasher.PBKDF2, Hasher.BCRYPT4, Hasher.BCRYPT); + final char[] hash = hasher.hash(new SecureString(apiKey.toCharArray())); + + Map sourceMap = new HashMap<>(); + sourceMap.put("api_key_hash", new String(hash)); + sourceMap.put("role_descriptors", Collections.singletonMap("a role", Collections.singletonMap("cluster", "all"))); + sourceMap.put("limited_by_role_descriptors", Collections.singletonMap("limited role", Collections.singletonMap("cluster", "all"))); + Map creatorMap = new HashMap<>(); + creatorMap.put("principal", "test_user"); + creatorMap.put("metadata", Collections.emptyMap()); + sourceMap.put("creator", creatorMap); + sourceMap.put("api_key_invalidated", false); + + ApiKeyService service = new ApiKeyService(Settings.EMPTY, Clock.systemUTC(), null, null, + ClusterServiceUtils.createClusterService(threadPool), threadPool); + ApiKeyCredentials creds = new ApiKeyCredentials(randomAlphaOfLength(12), new SecureString(apiKey.toCharArray())); + PlainActionFuture future = new PlainActionFuture<>(); + service.validateApiKeyCredentials(sourceMap, creds, Clock.systemUTC(), future); + AuthenticationResult result = future.actionGet(); + assertThat(result.isAuthenticated(), is(true)); + CachedApiKeyHashResult cachedApiKeyHashResult = service.getFromCache(creds.getId()); + assertNotNull(cachedApiKeyHashResult); + assertThat(cachedApiKeyHashResult.success, is(true)); + + creds = new ApiKeyCredentials(creds.getId(), new SecureString("foobar".toCharArray())); + future = new PlainActionFuture<>(); + service.validateApiKeyCredentials(sourceMap, creds, Clock.systemUTC(), future); + result = future.actionGet(); + assertThat(result.isAuthenticated(), is(false)); + final CachedApiKeyHashResult shouldBeSame = service.getFromCache(creds.getId()); + assertNotNull(shouldBeSame); + assertThat(shouldBeSame, sameInstance(cachedApiKeyHashResult)); + + sourceMap.put("api_key_hash", new String(hasher.hash(new SecureString("foobar".toCharArray())))); + creds = new ApiKeyCredentials(randomAlphaOfLength(12), new SecureString("foobar1".toCharArray())); + future = new PlainActionFuture<>(); + service.validateApiKeyCredentials(sourceMap, creds, Clock.systemUTC(), future); + result = future.actionGet(); + assertThat(result.isAuthenticated(), is(false)); + cachedApiKeyHashResult = service.getFromCache(creds.getId()); + assertNotNull(cachedApiKeyHashResult); + assertThat(cachedApiKeyHashResult.success, is(false)); + + creds = new ApiKeyCredentials(creds.getId(), new SecureString("foobar2".toCharArray())); + future = new PlainActionFuture<>(); + service.validateApiKeyCredentials(sourceMap, creds, Clock.systemUTC(), future); + result = future.actionGet(); + assertThat(result.isAuthenticated(), is(false)); + assertThat(service.getFromCache(creds.getId()), not(sameInstance(cachedApiKeyHashResult))); + assertThat(service.getFromCache(creds.getId()).success, is(false)); + + creds = new ApiKeyCredentials(creds.getId(), new SecureString("foobar".toCharArray())); + future = new PlainActionFuture<>(); + service.validateApiKeyCredentials(sourceMap, creds, Clock.systemUTC(), future); + result = future.actionGet(); + assertThat(result.isAuthenticated(), is(true)); + assertThat(service.getFromCache(creds.getId()), not(sameInstance(cachedApiKeyHashResult))); + assertThat(service.getFromCache(creds.getId()).success, is(true)); + } + + public void testApiKeyCacheDisabled() { + final String apiKey = randomAlphaOfLength(16); + Hasher hasher = randomFrom(Hasher.PBKDF2, Hasher.BCRYPT4, Hasher.BCRYPT); + final char[] hash = hasher.hash(new SecureString(apiKey.toCharArray())); + final Settings settings = Settings.builder() + .put(ApiKeyService.CACHE_TTL_SETTING.getKey(), "0s") + .build(); + + Map sourceMap = new HashMap<>(); + sourceMap.put("api_key_hash", new String(hash)); + sourceMap.put("role_descriptors", Collections.singletonMap("a role", Collections.singletonMap("cluster", "all"))); + sourceMap.put("limited_by_role_descriptors", Collections.singletonMap("limited role", Collections.singletonMap("cluster", "all"))); + Map creatorMap = new HashMap<>(); + creatorMap.put("principal", "test_user"); + creatorMap.put("metadata", Collections.emptyMap()); + sourceMap.put("creator", creatorMap); + sourceMap.put("api_key_invalidated", false); + + ApiKeyService service = new ApiKeyService(settings, Clock.systemUTC(), null, null, + ClusterServiceUtils.createClusterService(threadPool), threadPool); + ApiKeyCredentials creds = new ApiKeyCredentials(randomAlphaOfLength(12), new SecureString(apiKey.toCharArray())); + PlainActionFuture future = new PlainActionFuture<>(); + service.validateApiKeyCredentials(sourceMap, creds, Clock.systemUTC(), future); + AuthenticationResult result = future.actionGet(); + assertThat(result.isAuthenticated(), is(true)); + CachedApiKeyHashResult cachedApiKeyHashResult = service.getFromCache(creds.getId()); + assertNull(cachedApiKeyHashResult); + } } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java index 7c6e9428f0759..40d9a71d023d4 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java @@ -210,7 +210,7 @@ licenseState, threadContext, mock(ReservedRealm.class), Arrays.asList(firstRealm return null; }).when(securityIndex).checkIndexVersionThenExecute(any(Consumer.class), any(Runnable.class)); ClusterService clusterService = ClusterServiceUtils.createClusterService(threadPool); - apiKeyService = new ApiKeyService(settings, Clock.systemUTC(), client, securityIndex, clusterService); + apiKeyService = new ApiKeyService(settings, Clock.systemUTC(), client, securityIndex, clusterService, threadPool); tokenService = new TokenService(settings, Clock.systemUTC(), client, securityIndex, clusterService); service = new AuthenticationService(settings, realms, auditTrail, new DefaultAuthenticationFailureHandler(Collections.emptyMap()), threadPool, new AnonymousUser(settings), tokenService, apiKeyService);