From 70a147258a6338d32fe97a59374b5dc68ca17f72 Mon Sep 17 00:00:00 2001 From: Rory Hunter Date: Tue, 2 Feb 2021 13:43:40 +0000 Subject: [PATCH] Move Security to use auto-managed system indices Backport of #67114. Part of #61656. Change the Security plugin so that its system indices are managed automatically by the system indices infrastructure. Also add an `origin` field to `CreateIndexRequest` and `UpdateSettingsRequest`. --- .../indices/TestSystemIndexDescriptor.java | 4 + .../indices/create/AutoCreateAction.java | 21 +- .../indices/create/CreateIndexRequest.java | 32 +- .../create/TransportCreateIndexAction.java | 26 +- .../put/TransportAutoPutMappingAction.java | 18 +- .../put/TransportPutMappingAction.java | 33 +- .../put/TransportUpdateSettingsAction.java | 39 +- .../settings/put/UpdateSettingsRequest.java | 17 + .../indices/SystemIndexDescriptor.java | 54 +- .../indices/SystemIndexManager.java | 9 +- .../snapshots/SnapshotResiliencyTests.java | 2 +- .../resources/security-index-template-7.json | 296 -------- .../security-tokens-index-template-7.json | 108 --- .../test/NativeRealmIntegTestCase.java | 10 - .../xpack/security/Security.java | 651 +++++++++++++++++- .../mapper/NativeRoleMappingStore.java | 5 +- .../support/SecurityIndexManager.java | 212 ++---- .../test/SecurityIntegTestCase.java | 9 - .../authc/AuthenticationServiceTests.java | 2 +- .../authc/esnative/NativeRealmTests.java | 2 +- .../authc/esnative/NativeUsersStoreTests.java | 1 - .../authc/ldap/ActiveDirectoryRealmTests.java | 1 - .../security/authc/ldap/LdapRealmTests.java | 1 - .../mapper/NativeRoleMappingStoreTests.java | 2 +- .../authz/store/CompositeRolesStoreTests.java | 2 +- .../store/NativePrivilegeStoreTests.java | 4 +- .../authz/store/NativeRolesStoreTests.java | 13 +- .../CacheInvalidatorRegistryTests.java | 2 +- .../support/SecurityIndexManagerTests.java | 189 +---- .../ldap/AbstractAdLdapRealmTestCase.java | 10 - 30 files changed, 926 insertions(+), 849 deletions(-) delete mode 100644 x-pack/plugin/core/src/main/resources/security-index-template-7.json delete mode 100644 x-pack/plugin/core/src/main/resources/security-tokens-index-template-7.json diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/TestSystemIndexDescriptor.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/TestSystemIndexDescriptor.java index e116cd7187334..2aae6d930ac9f 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/TestSystemIndexDescriptor.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/TestSystemIndexDescriptor.java @@ -51,6 +51,7 @@ public class TestSystemIndexDescriptor extends SystemIndexDescriptor { .build(); TestSystemIndexDescriptor() { +<<<<<<< HEAD super( INDEX_NAME + "*", PRIMARY_INDEX_NAME, @@ -63,6 +64,9 @@ public class TestSystemIndexDescriptor extends SystemIndexDescriptor { "stack", MapperService.SINGLE_MAPPING_NAME ); +======= + super(INDEX_NAME + "*", PRIMARY_INDEX_NAME, "Test system index", null, SETTINGS, INDEX_NAME, 0, "version", "stack", null); +>>>>>>> 6ffddf8f018... Move Security to use auto-managed system indices (#67114) } @Override diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/create/AutoCreateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/create/AutoCreateAction.java index ab9dc3f88992b..1ab3a67b2bb3c 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/create/AutoCreateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/create/AutoCreateAction.java @@ -151,9 +151,24 @@ public ClusterState execute(ClusterState currentState) throws Exception { } final SystemIndexDescriptor descriptor = systemIndices.findMatchingDescriptor(indexName); - CreateIndexClusterStateUpdateRequest updateRequest = descriptor != null && descriptor.isAutomaticallyManaged() - ? buildSystemIndexUpdateRequest(descriptor) - : buildUpdateRequest(indexName); + final boolean isSystemIndex = descriptor != null && descriptor.isAutomaticallyManaged(); + + final CreateIndexClusterStateUpdateRequest updateRequest; + + if (isSystemIndex) { + final String message = descriptor.checkMinimumNodeVersion( + "auto-create index", + state.nodes().getMinNodeVersion() + ); + if (message != null) { + logger.warn(message); + throw new IllegalStateException(message); + } + + updateRequest = buildSystemIndexUpdateRequest(descriptor); + } else { + updateRequest = buildUpdateRequest(indexName); + } return createIndexService.applyCreateIndexRequest(currentState, updateRequest, false); } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java index 766a6fa939eaf..34dbdff956011 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java @@ -87,6 +87,12 @@ public class CreateIndexRequest extends AcknowledgedRequest private ActiveShardCount waitForActiveShards = ActiveShardCount.DEFAULT; + private String origin = ""; + + /** + * Constructs a new request by deserializing an input + * @param in the input from which to deserialize + */ public CreateIndexRequest(StreamInput in) throws IOException { super(in); cause = in.readString(); @@ -118,20 +124,28 @@ public CreateIndexRequest(StreamInput in) throws IOException { in.readBoolean(); // updateAllTypes } waitForActiveShards = ActiveShardCount.readFrom(in); + if (in.getVersion().onOrAfter(Version.V_7_12_0)) { + origin = in.readString(); + } } public CreateIndexRequest() { } /** - * Constructs a new request to create an index with the specified name. + * Constructs a request to create an index. + * + * @param index the name of the index */ public CreateIndexRequest(String index) { this(index, EMPTY_SETTINGS); } /** - * Constructs a new request to create an index with the specified name and settings. + * Constructs a request to create an index. + * + * @param index the name of the index + * @param settings the settings to apply to the index */ public CreateIndexRequest(String index, Settings settings) { this.index = index; @@ -183,6 +197,15 @@ public String cause() { return cause; } + public String origin() { + return origin; + } + + public CreateIndexRequest origin(String origin) { + this.origin = Objects.requireNonNull(origin); + return this; + } + /** * The settings to create the index with. */ @@ -276,7 +299,7 @@ public CreateIndexRequest mapping(String type, Map source) { throw new IllegalStateException("mappings for type \"" + type + "\" were already defined"); } // wrap it in a type map if its not - if (source.size() != 1 || !source.containsKey(type)) { + if (source.size() != 1 || source.containsKey(type) == false) { source = MapBuilder.newMapBuilder().put(type, source).map(); } try { @@ -477,6 +500,9 @@ public void writeTo(StreamOutput out) throws IOException { out.writeBoolean(true); // updateAllTypes } waitForActiveShards.writeTo(out); + if (out.getVersion().onOrAfter(Version.V_7_12_0)) { + out.writeString(origin); + } } @Override diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/create/TransportCreateIndexAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/create/TransportCreateIndexAction.java index 24ee944000f25..c08fab9b017bc 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/create/TransportCreateIndexAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/create/TransportCreateIndexAction.java @@ -19,6 +19,8 @@ package org.elasticsearch.action.admin.indices.create; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.indices.alias.Alias; import org.elasticsearch.action.support.ActionFilters; @@ -30,6 +32,7 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.MetadataCreateIndexService; import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.indices.SystemIndexDescriptor; @@ -46,6 +49,7 @@ * Create index action. */ public class TransportCreateIndexAction extends TransportMasterNodeAction { + private static final Logger logger = LogManager.getLogger(TransportCreateIndexAction.class); private final MetadataCreateIndexService createIndexService; private final SystemIndices systemIndices; @@ -77,9 +81,25 @@ protected void masterOperation(final CreateIndexRequest request, final ClusterSt final String indexName = indexNameExpressionResolver.resolveDateMathExpression(request.index()); final SystemIndexDescriptor descriptor = systemIndices.findMatchingDescriptor(indexName); - final CreateIndexClusterStateUpdateRequest updateRequest = descriptor != null && descriptor.isAutomaticallyManaged() - ? buildSystemIndexUpdateRequest(request, cause, descriptor) - : buildUpdateRequest(request, cause, indexName); + final boolean isSystemIndex = descriptor != null && descriptor.isAutomaticallyManaged(); + + final CreateIndexClusterStateUpdateRequest updateRequest; + + // Requests that a cluster generates itself are permitted to create a system index with + // different mappings, settings etc. This is so that rolling upgrade scenarios still work. + // We check this via the request's origin. Eventually, `SystemIndexManager` will reconfigure + // the index to the latest settings. + if (isSystemIndex && Strings.isNullOrEmpty(request.origin())) { + final String message = descriptor.checkMinimumNodeVersion("create index", state.nodes().getMinNodeVersion()); + if (message != null) { + logger.warn(message); + listener.onFailure(new IllegalStateException(message)); + return; + } + updateRequest = buildSystemIndexUpdateRequest(request, cause, descriptor); + } else { + updateRequest = buildUpdateRequest(request, cause, indexName); + } createIndexService.createIndex(updateRequest, listener.map(response -> new CreateIndexResponse(response.isAcknowledged(), response.isShardsAcknowledged(), indexName))); diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/TransportAutoPutMappingAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/TransportAutoPutMappingAction.java index ecc97c94603a5..f182b35f16323 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/TransportAutoPutMappingAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/TransportAutoPutMappingAction.java @@ -18,6 +18,8 @@ */ package org.elasticsearch.action.admin.indices.mapping.put; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.master.AcknowledgedResponse; @@ -30,6 +32,7 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.index.Index; +import org.elasticsearch.indices.SystemIndices; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; @@ -38,7 +41,10 @@ public class TransportAutoPutMappingAction extends AcknowledgedTransportMasterNodeAction { + private static final Logger logger = LogManager.getLogger(TransportAutoPutMappingAction.class); + private final MetadataMappingService metadataMappingService; + private final SystemIndices systemIndices; @Inject public TransportAutoPutMappingAction( @@ -47,10 +53,12 @@ public TransportAutoPutMappingAction( final ThreadPool threadPool, final MetadataMappingService metadataMappingService, final ActionFilters actionFilters, - final IndexNameExpressionResolver indexNameExpressionResolver) { + final IndexNameExpressionResolver indexNameExpressionResolver, + final SystemIndices systemIndices) { super(AutoPutMappingAction.NAME, transportService, clusterService, threadPool, actionFilters, PutMappingRequest::new, indexNameExpressionResolver, ThreadPool.Names.SAME); this.metadataMappingService = metadataMappingService; + this.systemIndices = systemIndices; } @Override @@ -72,6 +80,14 @@ protected ClusterBlockException checkBlock(PutMappingRequest request, ClusterSta protected void masterOperation(final PutMappingRequest request, final ClusterState state, final ActionListener listener) { final Index[] concreteIndices = new Index[] {request.getConcreteIndex()}; + + final String message = TransportPutMappingAction.checkForSystemIndexViolations(systemIndices, concreteIndices, request); + if (message != null) { + logger.warn(message); + listener.onFailure(new IllegalStateException(message)); + return; + } + performMappingUpdate(concreteIndices, request, listener, metadataMappingService); } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/TransportPutMappingAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/TransportPutMappingAction.java index a526c8dd4dde4..0833c4c4d5746 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/TransportPutMappingAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/TransportPutMappingAction.java @@ -33,6 +33,7 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.MetadataMappingService; import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexNotFoundException; @@ -91,7 +92,6 @@ protected void masterOperation(final PutMappingRequest request, final ClusterSta final ActionListener listener) { try { final Index[] concreteIndices = resolveIndices(state, request, indexNameExpressionResolver); - final String mappingSource = request.source(); final Optional maybeValidationException = requestValidators.validateRequest(request, state, concreteIndices); if (maybeValidationException.isPresent()) { @@ -99,14 +99,10 @@ protected void masterOperation(final PutMappingRequest request, final ClusterSta return; } - final List violations = checkForSystemIndexViolations(concreteIndices, mappingSource); - if (violations.isEmpty() == false) { - final String message = "Cannot update mappings in " - + violations - + ": system indices can only use mappings from their descriptors, " - + "but the mappings in the request did not match those in the descriptors(s)"; + final String message = checkForSystemIndexViolations(systemIndices, concreteIndices, request); + if (message != null) { logger.warn(message); - listener.onFailure(new IllegalArgumentException(message)); + listener.onFailure(new IllegalStateException(message)); return; } @@ -160,14 +156,21 @@ public void onFailure(Exception t) { }); } - private List checkForSystemIndexViolations(Index[] concreteIndices, String requestMappings) { + static String checkForSystemIndexViolations(SystemIndices systemIndices, Index[] concreteIndices, PutMappingRequest request) { + // Requests that a cluster generates itself are permitted to have a difference in mappings + // so that rolling upgrade scenarios still work. We check this via the request's origin. + if (Strings.isNullOrEmpty(request.origin()) == false) { + return null; + } + List violations = new ArrayList<>(); + final String requestMappings = request.source(); + for (Index index : concreteIndices) { final SystemIndexDescriptor descriptor = systemIndices.findMatchingDescriptor(index.getName()); if (descriptor != null && descriptor.isAutomaticallyManaged()) { final String descriptorMappings = descriptor.getMappings(); - // Technically we could trip over a difference in whitespace here, but then again nobody should be trying to manually // update a descriptor's mappings. if (descriptorMappings.equals(requestMappings) == false) { @@ -175,6 +178,14 @@ private List checkForSystemIndexViolations(Index[] concreteIndices, Stri } } } - return violations; + + if (violations.isEmpty() == false) { + return "Cannot update mappings in " + + violations + + ": system indices can only use mappings from their descriptors, " + + "but the mappings in the request did not match those in the descriptors(s)"; + } + + return null; } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/settings/put/TransportUpdateSettingsAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/settings/put/TransportUpdateSettingsAction.java index b62d3f2254c2b..af929350c6239 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/settings/put/TransportUpdateSettingsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/settings/put/TransportUpdateSettingsAction.java @@ -19,13 +19,6 @@ package org.elasticsearch.action.admin.indices.settings.put; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.stream.Collectors; - import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; @@ -40,6 +33,7 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.MetadataUpdateSettingsService; import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.Index; @@ -48,6 +42,14 @@ import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; + public class TransportUpdateSettingsAction extends AcknowledgedTransportMasterNodeAction { private static final Logger logger = LogManager.getLogger(TransportUpdateSettingsAction.class); @@ -89,8 +91,7 @@ protected void masterOperation(final UpdateSettingsRequest request, final Cluste final Index[] concreteIndices = indexNameExpressionResolver.concreteIndices(state, request); final Settings requestSettings = request.settings(); - - final Map> systemIndexViolations = checkForSystemIndexViolations(concreteIndices, requestSettings); + final Map> systemIndexViolations = checkForSystemIndexViolations(concreteIndices, request); if (systemIndexViolations.isEmpty() == false) { final String message = "Cannot override settings on system indices: " + systemIndexViolations.entrySet() @@ -98,7 +99,7 @@ protected void masterOperation(final UpdateSettingsRequest request, final Cluste .map(entry -> "[" + entry.getKey() + "] -> " + entry.getValue()) .collect(Collectors.joining(", ")); logger.warn(message); - listener.onFailure(new IllegalArgumentException(message)); + listener.onFailure(new IllegalStateException(message)); return; } @@ -128,11 +129,18 @@ public void onFailure(Exception t) { * that the system index's descriptor expects. * * @param concreteIndices the indices being updated - * @param requestSettings the settings to be applied + * @param request the update request * @return a mapping from system index pattern to the settings whose values would be overridden. Empty if there are no violations. */ - private Map> checkForSystemIndexViolations(Index[] concreteIndices, Settings requestSettings) { - final Map> violations = new HashMap<>(); + private Map> checkForSystemIndexViolations(Index[] concreteIndices, UpdateSettingsRequest request) { + // Requests that a cluster generates itself are permitted to have a difference in settings + // so that rolling upgrade scenarios still work. We check this via the request's origin. + if (Strings.isNullOrEmpty(request.origin()) == false) { + return Collections.emptyMap(); + } + + final Map> violationsByIndex = new HashMap<>(); + final Settings requestSettings = request.settings(); for (Index index : concreteIndices) { final SystemIndexDescriptor descriptor = systemIndices.findMatchingDescriptor(index.getName()); @@ -149,10 +157,11 @@ private Map> checkForSystemIndexViolations(Index[] concrete } if (failedKeys.isEmpty() == false) { - violations.put(descriptor.getIndexPattern(), failedKeys); + violationsByIndex.put(descriptor.getIndexPattern(), failedKeys); } } } - return violations; + + return violationsByIndex; } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/settings/put/UpdateSettingsRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/settings/put/UpdateSettingsRequest.java index 7bb6d672b6337..60bb18b622684 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/settings/put/UpdateSettingsRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/settings/put/UpdateSettingsRequest.java @@ -19,6 +19,7 @@ package org.elasticsearch.action.admin.indices.settings.put; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.IndicesRequest; import org.elasticsearch.action.support.IndicesOptions; @@ -53,6 +54,7 @@ public class UpdateSettingsRequest extends AcknowledgedRequest source) { return this; } + public String origin() { + return origin; + } + + public UpdateSettingsRequest origin(String origin) { + this.origin = Objects.requireNonNull(origin, "origin cannot be null"); + return this; + } + @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); @@ -178,6 +192,9 @@ public void writeTo(StreamOutput out) throws IOException { indicesOptions.writeIndicesOptions(out); writeSettingsToStream(settings, out); out.writeBoolean(preserveExisting); + if (out.getVersion().onOrAfter(Version.V_7_12_0)) { + out.writeString(origin); + } } @Override diff --git a/server/src/main/java/org/elasticsearch/indices/SystemIndexDescriptor.java b/server/src/main/java/org/elasticsearch/indices/SystemIndexDescriptor.java index d03e57ccca582..ad0c6baa6c417 100644 --- a/server/src/main/java/org/elasticsearch/indices/SystemIndexDescriptor.java +++ b/server/src/main/java/org/elasticsearch/indices/SystemIndexDescriptor.java @@ -23,12 +23,14 @@ import org.apache.lucene.util.automaton.CharacterRunAutomaton; import org.apache.lucene.util.automaton.Operations; import org.apache.lucene.util.automaton.RegExp; +import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.index.mapper.MapperService; +import java.util.Locale; import java.util.Objects; /** @@ -77,6 +79,9 @@ public class SystemIndexDescriptor { /** The index type to use when creating an index. */ private final String indexType; + /** The minimum cluster node version required for this descriptor, or null if there is no restriction */ + private final Version minimumNodeVersion; + /** * Creates a descriptor for system indices matching the supplied pattern. These indices will not be managed * by Elasticsearch internally. @@ -84,7 +89,7 @@ public class SystemIndexDescriptor { * @param description The name of the plugin responsible for this system index. */ public SystemIndexDescriptor(String indexPattern, String description) { - this(indexPattern, null, description, null, null, null, 0, null, null, MapperService.SINGLE_MAPPING_NAME); + this(indexPattern, null, description, null, null, null, 0, null, null, MapperService.SINGLE_MAPPING_NAME, null); } /** @@ -99,9 +104,10 @@ public SystemIndexDescriptor(String indexPattern, String description) { * @param aliasName An alias for the index, or null * @param indexFormat A value for the `index.format` setting. Pass 0 or higher. * @param versionMetaKey a mapping key under _meta where a version can be found, which indicates the - * Elasticsearch version when the index was created. + * Elasticsearch version when the index was created. * @param origin the client origin to use when creating this index. * @param indexType the index type. Should be {@link MapperService#SINGLE_MAPPING_NAME} for any new system indices. + * @param minimumNodeVersion the minimum cluster node version required for this descriptor, or null if there is no restriction */ SystemIndexDescriptor( String indexPattern, @@ -113,7 +119,8 @@ public SystemIndexDescriptor(String indexPattern, String description) { int indexFormat, String versionMetaKey, String origin, - String indexType + String indexType, + Version minimumNodeVersion ) { Objects.requireNonNull(indexPattern, "system index pattern must not be null"); if (indexPattern.length() < 2) { @@ -175,6 +182,7 @@ public SystemIndexDescriptor(String indexPattern, String description) { this.versionMetaKey = versionMetaKey; this.origin = origin; this.indexType = indexType; + this.minimumNodeVersion = minimumNodeVersion; } /** @@ -245,6 +253,29 @@ public String getIndexType() { return indexType; } + /** + * Checks that this descriptor can be used within this cluster, by comparing the supplied minimum + * node version to this descriptor's minimum version. + * + * @param cause the action being attempted that triggered the check. Used in the error message. + * @param actualMinimumNodeVersion the lower node version in the cluster + * @return an error message if the lowest node version is lower that the version in this descriptor, + * or null if the supplied version is acceptable or this descriptor has no minimum version. + */ + public String checkMinimumNodeVersion(String cause, Version actualMinimumNodeVersion) { + Objects.requireNonNull(cause); + if (this.minimumNodeVersion != null && this.minimumNodeVersion.after(actualMinimumNodeVersion)) { + return String.format( + Locale.ROOT, + "[%s] failed - system index [%s] requires all cluster nodes to be at least version [%s]", + cause, + this.getPrimaryIndex(), + minimumNodeVersion + ); + } + return null; + } + // TODO: getThreadpool() // TODO: Upgrade handling (reindex script?) @@ -252,6 +283,9 @@ public static Builder builder() { return new Builder(); } + /** + * Provides a fluent API for building a {@link SystemIndexDescriptor}. Validation still happens in that class. + */ public static class Builder { private String indexPattern; private String primaryIndex; @@ -263,6 +297,7 @@ public static class Builder { private String versionMetaKey = null; private String origin = null; private String indexType = MapperService.SINGLE_MAPPING_NAME; + private Version minimumNodeVersion = null; private Builder() {} @@ -321,6 +356,15 @@ public Builder setIndexType(String indexType) { return this; } + public Builder setMinimumNodeVersion(Version version) { + this.minimumNodeVersion = version; + return this; + } + + /** + * Builds a {@link SystemIndexDescriptor} using the fields supplied to this builder. + * @return a populated descriptor. + */ public SystemIndexDescriptor build() { String mappings = mappingsBuilder == null ? null : Strings.toString(mappingsBuilder); @@ -334,7 +378,8 @@ public SystemIndexDescriptor build() { indexFormat, versionMetaKey, origin, - indexType + indexType, + minimumNodeVersion ); } } @@ -364,6 +409,7 @@ static Automaton buildAutomaton(String pattern, String alias) { * {@link RegExp} instance. This exists because although * {@link org.elasticsearch.common.regex.Regex#simpleMatchToAutomaton(String)} is useful * for simple patterns, it doesn't support character ranges. + * * @param input the string to translate * @return the translate string */ diff --git a/server/src/main/java/org/elasticsearch/indices/SystemIndexManager.java b/server/src/main/java/org/elasticsearch/indices/SystemIndexManager.java index dad83ad92bd27..d6fec8808a544 100644 --- a/server/src/main/java/org/elasticsearch/indices/SystemIndexManager.java +++ b/server/src/main/java/org/elasticsearch/indices/SystemIndexManager.java @@ -75,7 +75,7 @@ public void clusterChanged(ClusterChangedEvent event) { if (state.blocks().hasGlobalBlock(GatewayService.STATE_NOT_RECOVERED_BLOCK)) { // wait until the gateway has recovered from disk, otherwise we may think we don't have some // indices but they may not have been restored from the cluster state on disk - logger.debug("system indices manager waiting until state has been recovered"); + logger.debug("Waiting until state has been recovered"); return; } @@ -84,6 +84,11 @@ public void clusterChanged(ClusterChangedEvent event) { return; } + if (state.nodes().getMaxNodeVersion().after(state.nodes().getMinNodeVersion())) { + logger.debug("Skipping system indices up-to-date check as cluster has mixed versions"); + return; + } + if (isUpgradeInProgress.compareAndSet(false, true)) { final List descriptors = getEligibleDescriptors(state.getMetadata()).stream() .filter(descriptor -> getUpgradeStatus(state, descriptor) == UpgradeStatus.NEEDS_MAPPINGS_UPDATE) @@ -101,6 +106,8 @@ public void clusterChanged(ClusterChangedEvent event) { } else { isUpgradeInProgress.set(false); } + } else { + logger.trace("Update already in progress"); } } diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java index f17281dd757f8..09de2ac3f3a48 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java +++ b/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java @@ -1594,7 +1594,7 @@ clusterService, indicesService, threadPool, shardStateAction, mappingUpdatedActi systemIndices)); actions.put(AutoPutMappingAction.INSTANCE, new TransportAutoPutMappingAction(transportService, clusterService, threadPool, metadataMappingService, - actionFilters, indexNameExpressionResolver)); + actionFilters, indexNameExpressionResolver, systemIndices)); final ResponseCollectorService responseCollectorService = new ResponseCollectorService(clusterService); final SearchTransportService searchTransportService = new SearchTransportService(transportService, client, SearchExecutionStatsCollector.makeWrapper(responseCollectorService)); diff --git a/x-pack/plugin/core/src/main/resources/security-index-template-7.json b/x-pack/plugin/core/src/main/resources/security-index-template-7.json deleted file mode 100644 index bf587ce8825a1..0000000000000 --- a/x-pack/plugin/core/src/main/resources/security-index-template-7.json +++ /dev/null @@ -1,296 +0,0 @@ -{ - "index_patterns" : [ ".security-7" ], - "order" : 1000, - "settings" : { - "number_of_shards" : 1, - "number_of_replicas" : 0, - "auto_expand_replicas" : "0-1", - "index.priority": 1000, - "index.refresh_interval": "1s", - "index.format": 6, - "analysis" : { - "filter" : { - "email" : { - "type" : "pattern_capture", - "preserve_original" : true, - "patterns" : [ - "([^@]+)", - "(\\p{L}+)", - "(\\d+)", - "@(.+)" - ] - } - }, - "analyzer" : { - "email" : { - "tokenizer" : "uax_url_email", - "filter" : [ - "email", - "lowercase", - "unique" - ] - } - } - } - }, - "mappings" : { - "_doc" : { - "_meta": { - "security-version": "${security.template.version}" - }, - "dynamic" : "strict", - "properties" : { - "username" : { - "type" : "keyword" - }, - "roles" : { - "type" : "keyword" - }, - "role_templates" : { - "properties": { - "template" : { - "type": "text" - }, - "format" : { - "type": "keyword" - } - } - }, - "password" : { - "type" : "keyword", - "index" : false, - "doc_values": false - }, - "full_name" : { - "type" : "text" - }, - "email" : { - "type" : "text", - "analyzer" : "email" - }, - "metadata" : { - "type" : "object", - "dynamic" : false - }, - "enabled": { - "type": "boolean" - }, - "cluster" : { - "type" : "keyword" - }, - "indices" : { - "type" : "object", - "properties" : { - "field_security" : { - "properties" : { - "grant": { - "type": "keyword" - }, - "except": { - "type": "keyword" - } - } - }, - "names" : { - "type" : "keyword" - }, - "privileges" : { - "type" : "keyword" - }, - "query" : { - "type" : "keyword" - }, - "allow_restricted_indices" : { - "type" : "boolean" - } - } - }, - "applications": { - "type": "object", - "properties": { - "application": { - "type": "keyword" - }, - "privileges": { - "type": "keyword" - }, - "resources": { - "type": "keyword" - } - } - }, - "application" : { - "type" : "keyword" - }, - "global": { - "type": "object", - "properties": { - "application": { - "type": "object", - "properties": { - "manage": { - "type": "object", - "properties": { - "applications": { - "type": "keyword" - } - } - } - } - } - } - }, - "name" : { - "type" : "keyword" - }, - "run_as" : { - "type" : "keyword" - }, - "doc_type" : { - "type" : "keyword" - }, - "type" : { - "type" : "keyword" - }, - "actions" : { - "type" : "keyword" - }, - "expiration_time" : { - "type" : "date", - "format" : "epoch_millis" - }, - "creation_time" : { - "type" : "date", - "format" : "epoch_millis" - }, - "api_key_hash" : { - "type" : "keyword", - "index": false, - "doc_values": false - }, - "api_key_invalidated" : { - "type" : "boolean" - }, - "role_descriptors" : { - "type" : "object", - "enabled": false - }, - "limited_by_role_descriptors" : { - "type" : "object", - "enabled": false - }, - "version" : { - "type" : "integer" - }, - "creator" : { - "type" : "object", - "properties" : { - "principal" : { - "type": "keyword" - }, - "full_name" : { - "type" : "text" - }, - "email" : { - "type" : "text", - "analyzer" : "email" - }, - "metadata" : { - "type" : "object", - "dynamic" : false - }, - "realm" : { - "type" : "keyword" - }, - "realm_type" : { - "type" : "keyword" - } - } - }, - "rules" : { - "type" : "object", - "dynamic" : false - }, - "refresh_token" : { - "type" : "object", - "properties" : { - "token" : { - "type" : "keyword" - }, - "refreshed" : { - "type" : "boolean" - }, - "refresh_time": { - "type": "date", - "format": "epoch_millis" - }, - "superseding": { - "type": "object", - "properties": { - "encrypted_tokens": { - "type": "binary" - }, - "encryption_iv": { - "type": "binary" - }, - "encryption_salt": { - "type": "binary" - } - } - }, - "invalidated" : { - "type" : "boolean" - }, - "client" : { - "type" : "object", - "properties" : { - "type" : { - "type" : "keyword" - }, - "user" : { - "type" : "keyword" - }, - "realm" : { - "type" : "keyword" - } - } - } - } - }, - "access_token" : { - "type" : "object", - "properties" : { - "user_token" : { - "type" : "object", - "properties" : { - "id" : { - "type" : "keyword" - }, - "expiration_time" : { - "type" : "date", - "format" : "epoch_millis" - }, - "version" : { - "type" : "integer" - }, - "metadata" : { - "type" : "object", - "dynamic" : false - }, - "authentication" : { - "type" : "binary" - } - } - }, - "invalidated" : { - "type" : "boolean" - }, - "realm" : { - "type" : "keyword" - } - } - } - } - } - } -} diff --git a/x-pack/plugin/core/src/main/resources/security-tokens-index-template-7.json b/x-pack/plugin/core/src/main/resources/security-tokens-index-template-7.json deleted file mode 100644 index 502daae3f79bd..0000000000000 --- a/x-pack/plugin/core/src/main/resources/security-tokens-index-template-7.json +++ /dev/null @@ -1,108 +0,0 @@ -{ - "index_patterns" : [ ".security-tokens-7" ], - "order" : 1000, - "settings" : { - "number_of_shards" : 1, - "number_of_replicas" : 0, - "auto_expand_replicas" : "0-1", - "index.priority": 1000, - "index.refresh_interval": "1s", - "index.format": 7 - }, - "mappings" : { - "_doc" : { - "_meta": { - "security-version": "${security.template.version}" - }, - "dynamic" : "strict", - "properties" : { - "doc_type" : { - "type" : "keyword" - }, - "creation_time" : { - "type" : "date", - "format" : "epoch_millis" - }, - "refresh_token" : { - "type" : "object", - "properties" : { - "token" : { - "type" : "keyword" - }, - "refreshed" : { - "type" : "boolean" - }, - "refresh_time": { - "type": "date", - "format": "epoch_millis" - }, - "superseding": { - "type": "object", - "properties": { - "encrypted_tokens": { - "type": "binary" - }, - "encryption_iv": { - "type": "binary" - }, - "encryption_salt": { - "type": "binary" - } - } - }, - "invalidated" : { - "type" : "boolean" - }, - "client" : { - "type" : "object", - "properties" : { - "type" : { - "type" : "keyword" - }, - "user" : { - "type" : "keyword" - }, - "realm" : { - "type" : "keyword" - } - } - } - } - }, - "access_token" : { - "type" : "object", - "properties" : { - "user_token" : { - "type" : "object", - "properties" : { - "id" : { - "type" : "keyword" - }, - "expiration_time" : { - "type" : "date", - "format" : "epoch_millis" - }, - "version" : { - "type" : "integer" - }, - "metadata" : { - "type" : "object", - "dynamic" : false - }, - "authentication" : { - "type" : "binary" - } - } - }, - "invalidated" : { - "type" : "boolean" - }, - "realm" : { - "type" : "keyword" - } - } - } - } - } - } -} diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/test/NativeRealmIntegTestCase.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/test/NativeRealmIntegTestCase.java index 25e12ebb440ca..b1442bcaceb68 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/test/NativeRealmIntegTestCase.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/test/NativeRealmIntegTestCase.java @@ -10,7 +10,6 @@ import org.elasticsearch.client.RestClient; import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.transport.netty4.Netty4Transport; import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken; import org.elasticsearch.xpack.core.security.client.SecurityClient; @@ -21,14 +20,12 @@ import org.elasticsearch.xpack.core.security.user.KibanaUser; import org.elasticsearch.xpack.core.security.user.LogstashSystemUser; import org.elasticsearch.xpack.core.security.user.RemoteMonitoringUser; -import org.elasticsearch.xpack.security.support.SecurityIndexManager; import org.junit.After; import org.junit.Before; import java.io.IOException; import java.util.Arrays; import java.util.List; -import java.util.Set; /** * Test case with method to handle the starting and stopping the stores for native users and roles @@ -59,13 +56,6 @@ protected boolean addMockHttpTransport() { return false; // enable http } - @Override - public Set excludeTemplates() { - Set templates = Sets.newHashSet(super.excludeTemplates()); - templates.add(SecurityIndexManager.SECURITY_MAIN_TEMPLATE_7); // don't remove the security index template - return templates; - } - @Override protected Settings nodeSettings(int nodeOrdinal) { Settings.Builder builder = Settings.builder().put(super.nodeSettings(nodeOrdinal)); diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java index b8d21aff7f19f..a428df29b3469 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java @@ -41,6 +41,7 @@ import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.env.Environment; import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.http.HttpServerTransport; @@ -105,9 +106,9 @@ import org.elasticsearch.xpack.core.security.action.rolemapping.GetRoleMappingsAction; import org.elasticsearch.xpack.core.security.action.rolemapping.PutRoleMappingAction; import org.elasticsearch.xpack.core.security.action.saml.SamlAuthenticateAction; +import org.elasticsearch.xpack.core.security.action.saml.SamlCompleteLogoutAction; import org.elasticsearch.xpack.core.security.action.saml.SamlInvalidateSessionAction; import org.elasticsearch.xpack.core.security.action.saml.SamlLogoutAction; -import org.elasticsearch.xpack.core.security.action.saml.SamlCompleteLogoutAction; import org.elasticsearch.xpack.core.security.action.saml.SamlPrepareAuthenticationAction; import org.elasticsearch.xpack.core.security.action.saml.SamlSpMetadataAction; import org.elasticsearch.xpack.core.security.action.token.CreateTokenAction; @@ -172,9 +173,9 @@ import org.elasticsearch.xpack.security.action.rolemapping.TransportGetRoleMappingsAction; import org.elasticsearch.xpack.security.action.rolemapping.TransportPutRoleMappingAction; import org.elasticsearch.xpack.security.action.saml.TransportSamlAuthenticateAction; +import org.elasticsearch.xpack.security.action.saml.TransportSamlCompleteLogoutAction; import org.elasticsearch.xpack.security.action.saml.TransportSamlInvalidateSessionAction; import org.elasticsearch.xpack.security.action.saml.TransportSamlLogoutAction; -import org.elasticsearch.xpack.security.action.saml.TransportSamlCompleteLogoutAction; import org.elasticsearch.xpack.security.action.saml.TransportSamlPrepareAuthenticationAction; import org.elasticsearch.xpack.security.action.saml.TransportSamlSpMetadataAction; import org.elasticsearch.xpack.security.action.token.TransportCreateTokenAction; @@ -215,14 +216,14 @@ import org.elasticsearch.xpack.security.authz.store.NativePrivilegeStore; import org.elasticsearch.xpack.security.authz.store.NativeRolesStore; import org.elasticsearch.xpack.security.ingest.SetSecurityUserProcessor; +import org.elasticsearch.xpack.security.operator.FileOperatorUsersStore; import org.elasticsearch.xpack.security.operator.OperatorOnlyRegistry; import org.elasticsearch.xpack.security.operator.OperatorPrivileges; import org.elasticsearch.xpack.security.operator.OperatorPrivileges.OperatorPrivilegesService; -import org.elasticsearch.xpack.security.operator.FileOperatorUsersStore; import org.elasticsearch.xpack.security.rest.SecurityRestFilter; import org.elasticsearch.xpack.security.rest.action.RestAuthenticateAction; -import org.elasticsearch.xpack.security.rest.action.apikey.RestClearApiKeyCacheAction; import org.elasticsearch.xpack.security.rest.action.RestDelegatePkiAuthenticationAction; +import org.elasticsearch.xpack.security.rest.action.apikey.RestClearApiKeyCacheAction; import org.elasticsearch.xpack.security.rest.action.apikey.RestCreateApiKeyAction; import org.elasticsearch.xpack.security.rest.action.apikey.RestGetApiKeyAction; import org.elasticsearch.xpack.security.rest.action.apikey.RestGrantApiKeyAction; @@ -246,9 +247,9 @@ import org.elasticsearch.xpack.security.rest.action.rolemapping.RestGetRoleMappingsAction; import org.elasticsearch.xpack.security.rest.action.rolemapping.RestPutRoleMappingAction; import org.elasticsearch.xpack.security.rest.action.saml.RestSamlAuthenticateAction; +import org.elasticsearch.xpack.security.rest.action.saml.RestSamlCompleteLogoutAction; import org.elasticsearch.xpack.security.rest.action.saml.RestSamlInvalidateSessionAction; import org.elasticsearch.xpack.security.rest.action.saml.RestSamlLogoutAction; -import org.elasticsearch.xpack.security.rest.action.saml.RestSamlCompleteLogoutAction; import org.elasticsearch.xpack.security.rest.action.saml.RestSamlPrepareAuthenticationAction; import org.elasticsearch.xpack.security.rest.action.saml.RestSamlSpMetadataAction; import org.elasticsearch.xpack.security.rest.action.user.RestChangePasswordAction; @@ -258,8 +259,8 @@ import org.elasticsearch.xpack.security.rest.action.user.RestHasPrivilegesAction; import org.elasticsearch.xpack.security.rest.action.user.RestPutUserAction; import org.elasticsearch.xpack.security.rest.action.user.RestSetEnabledAction; -import org.elasticsearch.xpack.security.support.ExtensionComponents; import org.elasticsearch.xpack.security.support.CacheInvalidatorRegistry; +import org.elasticsearch.xpack.security.support.ExtensionComponents; import org.elasticsearch.xpack.security.support.SecurityIndexManager; import org.elasticsearch.xpack.security.support.SecurityStatusChangeListener; import org.elasticsearch.xpack.security.transport.SecurityHttpSettings; @@ -270,10 +271,11 @@ import org.elasticsearch.xpack.security.transport.nio.SecurityNioHttpServerTransport; import org.elasticsearch.xpack.security.transport.nio.SecurityNioTransport; +import java.io.IOException; +import java.io.UncheckedIOException; import java.nio.file.Path; import java.time.Clock; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.HashMap; @@ -289,16 +291,21 @@ import java.util.function.UnaryOperator; import java.util.stream.Collectors; +import static java.util.Arrays.asList; import static java.util.Collections.emptyList; import static java.util.Collections.emptyMap; import static java.util.Collections.singletonList; import static org.elasticsearch.cluster.metadata.IndexMetadata.INDEX_FORMAT_SETTING; +import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.elasticsearch.xpack.core.ClientHelper.SECURITY_ORIGIN; import static org.elasticsearch.xpack.core.XPackSettings.API_KEY_SERVICE_ENABLED_SETTING; import static org.elasticsearch.xpack.core.XPackSettings.HTTP_SSL_ENABLED; import static org.elasticsearch.xpack.core.security.index.RestrictedIndicesNames.SECURITY_MAIN_ALIAS; -import static org.elasticsearch.xpack.security.support.SecurityIndexManager.INTERNAL_MAIN_INDEX_FORMAT; +import static org.elasticsearch.xpack.core.security.index.RestrictedIndicesNames.SECURITY_TOKENS_ALIAS; import static org.elasticsearch.xpack.security.operator.OperatorPrivileges.OPERATOR_PRIVILEGES_ENABLED; -import static org.elasticsearch.xpack.security.support.SecurityIndexManager.SECURITY_MAIN_TEMPLATE_7; +import static org.elasticsearch.xpack.security.support.SecurityIndexManager.INTERNAL_MAIN_INDEX_FORMAT; +import static org.elasticsearch.xpack.security.support.SecurityIndexManager.INTERNAL_TOKENS_INDEX_FORMAT; +import static org.elasticsearch.xpack.security.support.SecurityIndexManager.SECURITY_VERSION_STRING; public class Security extends Plugin implements SystemIndexPlugin, IngestPlugin, NetworkPlugin, ClusterPlugin, DiscoveryPlugin, MapperPlugin, ExtensiblePlugin { @@ -307,6 +314,9 @@ public class Security extends Plugin implements SystemIndexPlugin, IngestPlugin, private static final Logger logger = LogManager.getLogger(Security.class); + public static final SystemIndexDescriptor SECURITY_MAIN_INDEX_DESCRIPTOR = getSecurityMainIndexDescriptor(); + public static final SystemIndexDescriptor SECURITY_TOKEN_INDEX_DESCRIPTOR = getSecurityTokenIndexDescriptor(); + private final Settings settings; private final boolean enabled; private final boolean transportClientMode; @@ -427,7 +437,7 @@ Collection createComponents(Client client, ThreadPool threadPool, Cluste // We need to construct the checks here while the secure settings are still available. // If we wait until #getBoostrapChecks the secure settings will have been cleared/closed. final List checks = new ArrayList<>(); - checks.addAll(Arrays.asList( + checks.addAll(asList( new ApiKeySSLBootstrapCheck(), new TokenSSLBootstrapCheck(), new PkiRealmBootstrapCheck(getSslService()), @@ -448,10 +458,18 @@ Collection createComponents(Client client, ThreadPool threadPool, Cluste components.add(auditTrailService); this.auditTrailService.set(auditTrailService); - securityIndex.set(SecurityIndexManager.buildSecurityMainIndexManager(client, clusterService)); - - final TokenService tokenService = new TokenService(settings, Clock.systemUTC(), client, getLicenseState(), securityContext.get(), - securityIndex.get(), SecurityIndexManager.buildSecurityTokensIndexManager(client, clusterService), clusterService); + securityIndex.set(SecurityIndexManager.buildSecurityIndexManager(client, clusterService, SECURITY_MAIN_INDEX_DESCRIPTOR)); + + final TokenService tokenService = new TokenService( + settings, + Clock.systemUTC(), + client, + getLicenseState(), + securityContext.get(), + securityIndex.get(), + SecurityIndexManager.buildSecurityIndexManager(client, clusterService, SECURITY_TOKEN_INDEX_DESCRIPTOR), + clusterService + ); this.tokenService.set(tokenService); components.add(tokenService); @@ -533,7 +551,7 @@ Collection createComponents(Client client, ThreadPool threadPool, Cluste new ResizeRequestInterceptor(threadPool, getLicenseState(), auditTrailService), new IndicesAliasesRequestInterceptor(threadPool.getThreadContext(), getLicenseState(), auditTrailService)); if (XPackSettings.DLS_FLS_ENABLED.get(settings)) { - requestInterceptors.addAll(Arrays.asList( + requestInterceptors.addAll(asList( new SearchRequestInterceptor(threadPool, getLicenseState()), new UpdateRequestInterceptor(threadPool, getLicenseState()), new BulkShardRequestInterceptor(threadPool, getLicenseState()) @@ -814,7 +832,7 @@ public void onIndexModule(IndexModule module) { if (enabled == false) { return emptyList(); } - return Arrays.asList( + return asList( new ActionHandler<>(ClearRealmCacheAction.INSTANCE, TransportClearRealmCacheAction.class), new ActionHandler<>(ClearRolesCacheAction.INSTANCE, TransportClearRolesCacheAction.class), new ActionHandler<>(ClearPrivilegesCacheAction.INSTANCE, TransportClearPrivilegesCacheAction.class), @@ -878,7 +896,7 @@ public List getRestHandlers(Settings settings, RestController restC if (enabled == false) { return emptyList(); } - return Arrays.asList( + return asList( new RestAuthenticateAction(settings, securityContext.get(), getLicenseState()), new RestClearRealmCacheAction(settings, getLicenseState()), new RestClearRolesCacheAction(settings, getLicenseState()), @@ -1088,7 +1106,6 @@ public List> getExecutorBuilders(final Settings settings) { public UnaryOperator> getIndexTemplateMetadataUpgrader() { return templates -> { // .security index is not managed by using templates anymore - templates.remove(SECURITY_MAIN_TEMPLATE_7); templates.remove("security_audit_log"); return templates; }; @@ -1206,15 +1223,599 @@ private synchronized SharedGroupFactory getNettySharedGroupFactory(Settings sett } } + private static SystemIndexDescriptor getSecurityMainIndexDescriptor() { + return SystemIndexDescriptor.builder() + // This can't just be `.security-*` because that would overlap with the tokens index pattern + .setIndexPattern(".security-[0-9]+") + .setPrimaryIndex(RestrictedIndicesNames.INTERNAL_SECURITY_MAIN_INDEX_7) + .setDescription("Contains Security configuration") + .setMappings(getIndexMappings()) + .setSettings(getIndexSettings()) + .setAliasName(SECURITY_MAIN_ALIAS) + .setIndexFormat(INTERNAL_MAIN_INDEX_FORMAT) + .setVersionMetaKey("security-version") + .setOrigin(SECURITY_ORIGIN) + .build(); + } + + private static SystemIndexDescriptor getSecurityTokenIndexDescriptor() { + return SystemIndexDescriptor.builder() + .setIndexPattern(".security-tokens-[0-9]+") + .setPrimaryIndex(RestrictedIndicesNames.INTERNAL_SECURITY_TOKENS_INDEX_7) + .setDescription("Contains auth token data") + .setMappings(getTokenIndexMappings()) + .setSettings(getTokenIndexSettings()) + .setAliasName(SECURITY_TOKENS_ALIAS) + .setIndexFormat(INTERNAL_TOKENS_INDEX_FORMAT) + .setVersionMetaKey(SECURITY_VERSION_STRING) + .setOrigin(SECURITY_ORIGIN) + .build(); + } + @Override public Collection getSystemIndexDescriptors(Settings settings) { - return Collections.unmodifiableList(Arrays.asList( - new SystemIndexDescriptor(SECURITY_MAIN_ALIAS, "Contains Security configuration"), - new SystemIndexDescriptor(RestrictedIndicesNames.INTERNAL_SECURITY_MAIN_INDEX_6, "Contains Security configuration"), - new SystemIndexDescriptor(RestrictedIndicesNames.INTERNAL_SECURITY_MAIN_INDEX_7, "Contains Security configuration"), + return Collections.unmodifiableList(asList(SECURITY_MAIN_INDEX_DESCRIPTOR, SECURITY_TOKEN_INDEX_DESCRIPTOR)); + } - new SystemIndexDescriptor(RestrictedIndicesNames.SECURITY_TOKENS_ALIAS, "Contains auth token data"), - new SystemIndexDescriptor(RestrictedIndicesNames.INTERNAL_SECURITY_TOKENS_INDEX_7, "Contains auth token data") - )); + private static Settings getIndexSettings() { + return Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .put(IndexMetadata.SETTING_AUTO_EXPAND_REPLICAS, "0-1") + .put(IndexMetadata.SETTING_PRIORITY, 1000) + .put("index.refresh_interval", "1s") + .put(IndexMetadata.INDEX_FORMAT_SETTING.getKey(), INTERNAL_MAIN_INDEX_FORMAT) + .put("analysis.filter.email.type", "pattern_capture") + .put("analysis.filter.email.preserve_original", true) + .putList("analysis.filter.email.patterns", asList("([^@]+)", "(\\p{L}+)", "(\\d+)", "@(.+)")) + .put("analysis.analyzer.email.tokenizer", "uax_url_email") + .putList("analysis.analyzer.email.filter", asList("email", "lowercase", "unique")) + .build(); } + + private static XContentBuilder getIndexMappings() { + try { + final XContentBuilder builder = jsonBuilder(); + builder.startObject(); + { + builder.startObject("_meta"); + builder.field(SECURITY_VERSION_STRING, Version.CURRENT.toString()); + builder.endObject(); + + builder.field("dynamic", "strict"); + builder.startObject("properties"); + { + builder.startObject("username"); + builder.field("type", "keyword"); + builder.endObject(); + + builder.startObject("roles"); + builder.field("type", "keyword"); + builder.endObject(); + + builder.startObject("role_templates"); + { + builder.startObject("properties"); + { + builder.startObject("template"); + builder.field("type", "text"); + builder.endObject(); + + builder.startObject("format"); + builder.field("type", "keyword"); + builder.endObject(); + } + builder.endObject(); + } + builder.endObject(); + + builder.startObject("password"); + builder.field("type", "keyword"); + builder.field("index", false); + builder.field("doc_values", false); + builder.endObject(); + + builder.startObject("full_name"); + builder.field("type", "text"); + builder.endObject(); + + builder.startObject("email"); + builder.field("type", "text"); + builder.field("analyzer", "email"); + builder.endObject(); + + builder.startObject("metadata"); + builder.field("type", "object"); + builder.field("dynamic", false); + builder.endObject(); + + builder.startObject("enabled"); + builder.field("type", "boolean"); + builder.endObject(); + + builder.startObject("cluster"); + builder.field("type", "keyword"); + builder.endObject(); + + builder.startObject("indices"); + { + builder.field("type", "object"); + builder.startObject("properties"); + { + builder.startObject("field_security"); + { + builder.startObject("properties"); + { + builder.startObject("grant"); + builder.field("type", "keyword"); + builder.endObject(); + + builder.startObject("except"); + builder.field("type", "keyword"); + builder.endObject(); + } + builder.endObject(); + } + builder.endObject(); + + builder.startObject("names"); + builder.field("type", "keyword"); + builder.endObject(); + + builder.startObject("privileges"); + builder.field("type", "keyword"); + builder.endObject(); + + builder.startObject("query"); + builder.field("type", "keyword"); + builder.endObject(); + + builder.startObject("allow_restricted_indices"); + builder.field("type", "boolean"); + builder.endObject(); + } + builder.endObject(); + } + builder.endObject(); + + builder.startObject("applications"); + { + builder.field("type", "object"); + builder.startObject("properties"); + { + builder.startObject("application"); + builder.field("type", "keyword"); + builder.endObject(); + + builder.startObject("privileges"); + builder.field("type", "keyword"); + builder.endObject(); + + builder.startObject("resources"); + builder.field("type", "keyword"); + builder.endObject(); + } + builder.endObject(); + } + builder.endObject(); + + builder.startObject("application"); + builder.field("type", "keyword"); + builder.endObject(); + + builder.startObject("global"); + { + builder.field("type", "object"); + builder.startObject("properties"); + { + builder.startObject("application"); + { + builder.field("type", "object"); + builder.startObject("properties"); + { + builder.startObject("manage"); + { + builder.field("type", "object"); + builder.startObject("properties"); + { + builder.startObject("applications"); + builder.field("type", "keyword"); + builder.endObject(); + } + builder.endObject(); + } + builder.endObject(); + } + builder.endObject(); + } + builder.endObject(); + } + builder.endObject(); + } + builder.endObject(); + + builder.startObject("name"); + builder.field("type", "keyword"); + builder.endObject(); + + builder.startObject("run_as"); + builder.field("type", "keyword"); + builder.endObject(); + + builder.startObject("doc_type"); + builder.field("type", "keyword"); + builder.endObject(); + + builder.startObject("type"); + builder.field("type", "keyword"); + builder.endObject(); + + builder.startObject("actions"); + builder.field("type", "keyword"); + builder.endObject(); + + builder.startObject("expiration_time"); + builder.field("type", "date"); + builder.field("format", "epoch_millis"); + builder.endObject(); + + builder.startObject("creation_time"); + builder.field("type", "date"); + builder.field("format", "epoch_millis"); + builder.endObject(); + + builder.startObject("api_key_hash"); + builder.field("type", "keyword"); + builder.field("index", false); + builder.field("doc_values", false); + builder.endObject(); + + builder.startObject("api_key_invalidated"); + builder.field("type", "boolean"); + builder.endObject(); + + builder.startObject("role_descriptors"); + builder.field("type", "object"); + builder.field("enabled", false); + builder.endObject(); + + builder.startObject("limited_by_role_descriptors"); + builder.field("type", "object"); + builder.field("enabled", false); + builder.endObject(); + + builder.startObject("version"); + builder.field("type", "integer"); + builder.endObject(); + + builder.startObject("creator"); + { + builder.field("type", "object"); + builder.startObject("properties"); + { + builder.startObject("principal"); + builder.field("type", "keyword"); + builder.endObject(); + + builder.startObject("full_name"); + builder.field("type", "text"); + builder.endObject(); + + builder.startObject("email"); + builder.field("type", "text"); + builder.field("analyzer", "email"); + builder.endObject(); + + builder.startObject("metadata"); + builder.field("type", "object"); + builder.field("dynamic", false); + builder.endObject(); + + builder.startObject("realm"); + builder.field("type", "keyword"); + builder.endObject(); + + builder.startObject("realm_type"); + builder.field("type", "keyword"); + builder.endObject(); + } + builder.endObject(); + } + builder.endObject(); + + builder.startObject("rules"); + builder.field("type", "object"); + builder.field("dynamic", false); + builder.endObject(); + + builder.startObject("refresh_token"); + { + builder.field("type", "object"); + builder.startObject("properties"); + { + builder.startObject("token"); + builder.field("type", "keyword"); + builder.endObject(); + + builder.startObject("refreshed"); + builder.field("type", "boolean"); + builder.endObject(); + + builder.startObject("refresh_time"); + builder.field("type", "date"); + builder.field("format", "epoch_millis"); + builder.endObject(); + + builder.startObject("superseding"); + { + builder.field("type", "object"); + builder.startObject("properties"); + { + builder.startObject("encrypted_tokens"); + builder.field("type", "binary"); + builder.endObject(); + + builder.startObject("encryption_iv"); + builder.field("type", "binary"); + builder.endObject(); + + builder.startObject("encryption_salt"); + builder.field("type", "binary"); + builder.endObject(); + } + builder.endObject(); + } + builder.endObject(); + + builder.startObject("invalidated"); + builder.field("type", "boolean"); + builder.endObject(); + + builder.startObject("client"); + { + builder.field("type", "object"); + builder.startObject("properties"); + { + builder.startObject("type"); + builder.field("type", "keyword"); + builder.endObject(); + + builder.startObject("user"); + builder.field("type", "keyword"); + builder.endObject(); + + builder.startObject("realm"); + builder.field("type", "keyword"); + builder.endObject(); + } + builder.endObject(); + } + builder.endObject(); + } + builder.endObject(); + } + builder.endObject(); + + builder.startObject("access_token"); + { + builder.field("type", "object"); + builder.startObject("properties"); + { + builder.startObject("user_token"); + { + builder.field("type", "object"); + builder.startObject("properties"); + { + builder.startObject("id"); + builder.field("type", "keyword"); + builder.endObject(); + + builder.startObject("expiration_time"); + builder.field("type", "date"); + builder.field("format", "epoch_millis"); + builder.endObject(); + + builder.startObject("version"); + builder.field("type", "integer"); + builder.endObject(); + + builder.startObject("metadata"); + builder.field("type", "object"); + builder.field("dynamic", false); + builder.endObject(); + + builder.startObject("authentication"); + builder.field("type", "binary"); + builder.endObject(); + } + builder.endObject(); + } + builder.endObject(); + + builder.startObject("invalidated"); + builder.field("type", "boolean"); + builder.endObject(); + + builder.startObject("realm"); + builder.field("type", "keyword"); + builder.endObject(); + } + builder.endObject(); + } + builder.endObject(); + } + builder.endObject(); + } + builder.endObject(); + + return builder; + } catch (IOException e) { + logger.fatal("Failed to build " + RestrictedIndicesNames.INTERNAL_SECURITY_MAIN_INDEX_7 + " index mappings", e); + throw new UncheckedIOException( + "Failed to build " + RestrictedIndicesNames.INTERNAL_SECURITY_MAIN_INDEX_7 + " index mappings", e); + } + } + + private static Settings getTokenIndexSettings() { + return Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .put(IndexMetadata.SETTING_AUTO_EXPAND_REPLICAS, "0-1") + .put(IndexMetadata.SETTING_PRIORITY, 1000) + .put("index.refresh_interval", "1s") + .put(IndexMetadata.INDEX_FORMAT_SETTING.getKey(), INTERNAL_TOKENS_INDEX_FORMAT) + .build(); + } + + + private static XContentBuilder getTokenIndexMappings() { + try { + final XContentBuilder builder = jsonBuilder(); + + builder.startObject(); + { + builder.startObject("_meta"); + builder.field(SECURITY_VERSION_STRING, Version.CURRENT); + builder.endObject(); + + builder.field("dynamic", "strict"); + builder.startObject("properties"); + { + builder.startObject("doc_type"); + builder.field("type", "keyword"); + builder.endObject(); + + builder.startObject("creation_time"); + builder.field("type", "date"); + builder.field("format", "epoch_millis"); + builder.endObject(); + + builder.startObject("refresh_token"); + { + builder.field("type", "object"); + builder.startObject("properties"); + { + builder.startObject("token"); + builder.field("type", "keyword"); + builder.endObject(); + + builder.startObject("refreshed"); + builder.field("type", "boolean"); + builder.endObject(); + + builder.startObject("refresh_time"); + builder.field("type", "date"); + builder.field("format", "epoch_millis"); + builder.endObject(); + + builder.startObject("superseding"); + { + builder.field("type", "object"); + builder.startObject("properties"); + { + builder.startObject("encrypted_tokens"); + builder.field("type", "binary"); + builder.endObject(); + + builder.startObject("encryption_iv"); + builder.field("type", "binary"); + builder.endObject(); + + builder.startObject("encryption_salt"); + builder.field("type", "binary"); + builder.endObject(); + } + builder.endObject(); + } + builder.endObject(); + + builder.startObject("invalidated"); + builder.field("type", "boolean"); + builder.endObject(); + + builder.startObject("client"); + { + builder.field("type", "object"); + builder.startObject("properties"); + { + builder.startObject("type"); + builder.field("type", "keyword"); + builder.endObject(); + + builder.startObject("user"); + builder.field("type", "keyword"); + builder.endObject(); + + builder.startObject("realm"); + builder.field("type", "keyword"); + builder.endObject(); + } + builder.endObject(); + } + builder.endObject(); + } + builder.endObject(); + } + builder.endObject(); + + builder.startObject("access_token"); + { + builder.field("type", "object"); + builder.startObject("properties"); + { + builder.startObject("user_token"); + { + builder.field("type", "object"); + builder.startObject("properties"); + { + builder.startObject("id"); + builder.field("type", "keyword"); + builder.endObject(); + + builder.startObject("expiration_time"); + builder.field("type", "date"); + builder.field("format", "epoch_millis"); + builder.endObject(); + + builder.startObject("version"); + builder.field("type", "integer"); + builder.endObject(); + + builder.startObject("metadata"); + builder.field("type", "object"); + builder.field("dynamic", false); + builder.endObject(); + + builder.startObject("authentication"); + builder.field("type", "binary"); + builder.endObject(); + } + builder.endObject(); + } + builder.endObject(); + + builder.startObject("invalidated"); + builder.field("type", "boolean"); + builder.endObject(); + + builder.startObject("realm"); + builder.field("type", "keyword"); + builder.endObject(); + } + builder.endObject(); + } + builder.endObject(); + } + builder.endObject(); + } + + builder.endObject(); + return builder; + } catch (IOException e) { + throw new UncheckedIOException( + "Failed to build " + RestrictedIndicesNames.INTERNAL_SECURITY_TOKENS_INDEX_7 + " index mappings", e); + } + } + } + diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStore.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStore.java index 9faa58d5a63a2..029b5d0d5f4b8 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStore.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStore.java @@ -292,11 +292,10 @@ private void getMappings(ActionListener> listener) { } else { logger.info("The security index is not yet available - no role mappings can be loaded"); if (logger.isDebugEnabled()) { - logger.debug("Security Index [{}] [exists: {}] [available: {}] [mapping up to date: {}]", + logger.debug("Security Index [{}] [exists: {}] [available: {}]", SECURITY_MAIN_ALIAS, securityIndex.indexExists(), - securityIndex.isAvailable(), - securityIndex.isMappingUpToDate() + securityIndex.isAvailable() ); } listener.onResponse(Collections.emptyList()); diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java index eb879a368f39d..fbca73bac8c1e 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java @@ -19,10 +19,7 @@ import org.elasticsearch.action.admin.indices.alias.Alias; import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; -import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest; -import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest; import org.elasticsearch.action.support.ActiveShardCount; -import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.client.Client; import org.elasticsearch.cluster.ClusterChangedEvent; import org.elasticsearch.cluster.ClusterState; @@ -35,26 +32,15 @@ import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.routing.IndexRoutingTable; import org.elasticsearch.cluster.service.ClusterService; -import org.elasticsearch.common.Strings; -import org.elasticsearch.common.collect.Tuple; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.xcontent.DeprecationHandler; -import org.elasticsearch.common.xcontent.NamedXContentRegistry; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; -import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.gateway.GatewayService; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.indices.IndexClosedException; +import org.elasticsearch.indices.SystemIndexDescriptor; import org.elasticsearch.rest.RestStatus; -import org.elasticsearch.xpack.core.security.index.RestrictedIndicesNames; -import org.elasticsearch.xpack.core.template.TemplateUtils; -import java.io.IOException; -import java.nio.charset.StandardCharsets; import java.time.Instant; import java.util.HashSet; import java.util.List; @@ -65,13 +51,9 @@ import java.util.function.BiConsumer; import java.util.function.Consumer; import java.util.function.Predicate; -import java.util.function.Supplier; import java.util.stream.Collectors; import static org.elasticsearch.cluster.metadata.IndexMetadata.INDEX_FORMAT_SETTING; -import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken; -import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureFieldName; -import static org.elasticsearch.xpack.core.ClientHelper.SECURITY_ORIGIN; import static org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin; /** @@ -81,54 +63,36 @@ public class SecurityIndexManager implements ClusterStateListener { public static final int INTERNAL_MAIN_INDEX_FORMAT = 6; public static final int INTERNAL_TOKENS_INDEX_FORMAT = 7; - public static final String SECURITY_MAIN_TEMPLATE_7 = "security-index-template-7"; - public static final String SECURITY_TOKENS_TEMPLATE_7 = "security-tokens-index-template-7"; public static final String SECURITY_VERSION_STRING = "security-version"; public static final String TEMPLATE_VERSION_VARIABLE = "security.template.version"; private static final Logger logger = LogManager.getLogger(SecurityIndexManager.class); - private final String aliasName; - private final String internalIndexName; - private final int internalIndexFormat; - private final Supplier mappingSourceSupplier; private final Client client; + private final SystemIndexDescriptor systemIndexDescriptor; private final List> stateChangeListeners = new CopyOnWriteArrayList<>(); private volatile State indexState; - public static SecurityIndexManager buildSecurityMainIndexManager(Client client, ClusterService clusterService) { - return new SecurityIndexManager(client, clusterService, RestrictedIndicesNames.SECURITY_MAIN_ALIAS, - RestrictedIndicesNames.INTERNAL_SECURITY_MAIN_INDEX_7, INTERNAL_MAIN_INDEX_FORMAT, - () -> SecurityIndexManager.readTemplateAsBytes(SECURITY_MAIN_TEMPLATE_7)); + public static SecurityIndexManager buildSecurityIndexManager( + Client client, + ClusterService clusterService, + SystemIndexDescriptor descriptor + ) { + final SecurityIndexManager securityIndexManager = new SecurityIndexManager(client, descriptor, State.UNRECOVERED_STATE); + clusterService.addListener(securityIndexManager); + return securityIndexManager; } - public static SecurityIndexManager buildSecurityTokensIndexManager(Client client, ClusterService clusterService) { - return new SecurityIndexManager(client, clusterService, RestrictedIndicesNames.SECURITY_TOKENS_ALIAS, - RestrictedIndicesNames.INTERNAL_SECURITY_TOKENS_INDEX_7, INTERNAL_TOKENS_INDEX_FORMAT, - () -> SecurityIndexManager.readTemplateAsBytes(SECURITY_TOKENS_TEMPLATE_7)); - } - - private SecurityIndexManager(Client client, ClusterService clusterService, String aliasName, String internalIndexName, - int internalIndexFormat, Supplier mappingSourceSupplier) { - this(client, aliasName, internalIndexName, internalIndexFormat, mappingSourceSupplier, State.UNRECOVERED_STATE); - clusterService.addListener(this); - } - - // protected for testing - protected SecurityIndexManager(Client client, String aliasName, String internalIndexName, int internalIndexFormat, - Supplier mappingSourceSupplier, State indexState) { - this.aliasName = aliasName; - this.internalIndexName = internalIndexName; - this.internalIndexFormat = internalIndexFormat; - this.mappingSourceSupplier = mappingSourceSupplier; - this.indexState = indexState; + private SecurityIndexManager(Client client, SystemIndexDescriptor descriptor, State indexState) { this.client = client; + this.indexState = indexState; + this.systemIndexDescriptor = descriptor; } public SecurityIndexManager freeze() { - return new SecurityIndexManager(null, aliasName, internalIndexName, internalIndexFormat, mappingSourceSupplier, indexState); + return new SecurityIndexManager(null, systemIndexDescriptor, indexState); } public boolean checkMappingVersion(Predicate requiredVersion) { @@ -138,7 +102,7 @@ public boolean checkMappingVersion(Predicate requiredVersion) { } public String aliasName() { - return aliasName; + return systemIndexDescriptor.getAliasName(); } public boolean indexExists() { @@ -161,10 +125,6 @@ public boolean isAvailable() { return this.indexState.indexAvailable; } - public boolean isMappingUpToDate() { - return this.indexState.mappingUpToDate; - } - public boolean isStateRecovered() { return this.indexState != State.UNRECOVERED_STATE; } @@ -203,14 +163,15 @@ public void clusterChanged(ClusterChangedEvent event) { return; } final State previousState = indexState; - final IndexMetadata indexMetadata = resolveConcreteIndex(aliasName, event.state().metadata()); + final IndexMetadata indexMetadata = resolveConcreteIndex(systemIndexDescriptor.getAliasName(), event.state().metadata()); final Instant creationTime = indexMetadata != null ? Instant.ofEpochMilli(indexMetadata.getCreationDate()) : null; final boolean isIndexUpToDate = indexMetadata == null || - INDEX_FORMAT_SETTING.get(indexMetadata.getSettings()).intValue() == internalIndexFormat; + INDEX_FORMAT_SETTING.get(indexMetadata.getSettings()) == systemIndexDescriptor.getIndexFormat(); final boolean indexAvailable = checkIndexAvailable(event.state()); - final boolean mappingIsUpToDate = indexMetadata == null || checkIndexMappingUpToDate(event.state()); final Version mappingVersion = oldestIndexMappingVersion(event.state()); - final String concreteIndexName = indexMetadata == null ? internalIndexName : indexMetadata.getIndex().getName(); + final String concreteIndexName = indexMetadata == null + ? systemIndexDescriptor.getPrimaryIndex() + : indexMetadata.getIndex().getName(); final ClusterHealthStatus indexHealth; final IndexMetadata.State indexState; if (indexMetadata == null) { @@ -226,7 +187,7 @@ public void clusterChanged(ClusterChangedEvent event) { final IndexRoutingTable routingTable = event.state().getRoutingTable().index(indexMetadata.getIndex()); indexHealth = new ClusterIndexHealth(indexMetadata, routingTable).getStatus(); } - final State newState = new State(creationTime, isIndexUpToDate, indexAvailable, mappingIsUpToDate, mappingVersion, + final State newState = new State(creationTime, isIndexUpToDate, indexAvailable, mappingVersion, concreteIndexName, indexHealth, indexState); this.indexState = newState; @@ -238,6 +199,7 @@ public void clusterChanged(ClusterChangedEvent event) { } private boolean checkIndexAvailable(ClusterState state) { + final String aliasName = systemIndexDescriptor.getAliasName(); IndexMetadata metadata = resolveConcreteIndex(aliasName, state.metadata()); if (metadata == null) { logger.debug("Index [{}] is not available - no metadata", aliasName); @@ -256,26 +218,8 @@ private boolean checkIndexAvailable(ClusterState state) { } } - public static boolean checkTemplateExistsAndVersionMatches(String templateName, ClusterState state, Logger logger, - Predicate predicate) { - return TemplateUtils.checkTemplateExistsAndVersionMatches(templateName, SECURITY_VERSION_STRING, state, logger, predicate); - } - - private boolean checkIndexMappingUpToDate(ClusterState clusterState) { - return checkIndexMappingVersionMatches(clusterState, Version.CURRENT::equals); - } - - private boolean checkIndexMappingVersionMatches(ClusterState clusterState, Predicate predicate) { - return checkIndexMappingVersionMatches(aliasName, clusterState, logger, predicate); - } - - public static boolean checkIndexMappingVersionMatches(String indexName, ClusterState clusterState, Logger logger, - Predicate predicate) { - return loadIndexMappingVersions(indexName, clusterState, logger).stream().allMatch(predicate); - } - private Version oldestIndexMappingVersion(ClusterState clusterState) { - final Set versions = loadIndexMappingVersions(aliasName, clusterState, logger); + final Set versions = loadIndexMappingVersions(systemIndexDescriptor.getAliasName(), clusterState, logger); return versions.stream().min(Version::compareTo).orElse(null); } @@ -346,8 +290,9 @@ public void checkIndexVersionThenExecute(final Consumer consumer, fin } /** - * Prepares the index by creating it if it doesn't exist or updating the mappings if the mappings are - * out of date. After any tasks have been executed, the runnable is then executed. + * Prepares the index by creating it if it doesn't exist, then executes the runnable. + * @param consumer a handler for any exceptions that are raised either during preparation or execution + * @param andThen executed if the index exists or after preparation is performed successfully */ public void prepareIndexIfNeededThenExecute(final Consumer consumer, final Runnable andThen) { final State indexState = this.indexState; // use a local copy so all checks execute against the same state! @@ -362,52 +307,44 @@ public void prepareIndexIfNeededThenExecute(final Consumer consumer, + "Security features relying on the index will not be available until the upgrade API is run on the index"); } else if (indexState.indexExists() == false) { assert indexState.concreteIndexName != null; - logger.info("security index does not exist. Creating [{}] with alias [{}]", indexState.concreteIndexName, this.aliasName); - final byte[] mappingSource = mappingSourceSupplier.get(); - final Tuple mappingAndSettings = parseMappingAndSettingsFromTemplateBytes(mappingSource); + logger.info( + "security index does not exist, creating [{}] with alias [{}]", + indexState.concreteIndexName, + systemIndexDescriptor.getAliasName() + ); + + // Although `TransportCreateIndexAction` is capable of automatically applying the right mappings, settings and aliases for + // system indices, we nonetheless specify them here so that the values from `this.systemIndexDescriptor` are used. CreateIndexRequest request = new CreateIndexRequest(indexState.concreteIndexName) - .alias(new Alias(this.aliasName)) - .mapping(MapperService.SINGLE_MAPPING_NAME, mappingAndSettings.v1(), XContentType.JSON) - .waitForActiveShards(ActiveShardCount.ALL) - .settings(mappingAndSettings.v2()); - executeAsyncWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN, request, - new ActionListener() { - @Override - public void onResponse(CreateIndexResponse createIndexResponse) { - if (createIndexResponse.isAcknowledged()) { - andThen.run(); - } else { - consumer.accept(new ElasticsearchException("Failed to create security index")); - } - } - - @Override - public void onFailure(Exception e) { - final Throwable cause = ExceptionsHelper.unwrapCause(e); - if (cause instanceof ResourceAlreadyExistsException) { - // the index already exists - it was probably just created so this - // node hasn't yet received the cluster state update with the index - andThen.run(); - } else { - consumer.accept(e); + .origin(systemIndexDescriptor.getOrigin()) + .mapping(MapperService.SINGLE_MAPPING_NAME, systemIndexDescriptor.getMappings(), XContentType.JSON) + .settings(systemIndexDescriptor.getSettings()) + .alias(new Alias(systemIndexDescriptor.getAliasName())) + .waitForActiveShards(ActiveShardCount.ALL); + + executeAsyncWithOrigin(client.threadPool().getThreadContext(), systemIndexDescriptor.getOrigin(), request, + new ActionListener() { + @Override + public void onResponse(CreateIndexResponse createIndexResponse) { + if (createIndexResponse.isAcknowledged()) { + andThen.run(); + } else { + consumer.accept(new ElasticsearchException("Failed to create security index")); + } } - } - }, client.admin().indices()::create); - } else if (indexState.mappingUpToDate == false) { - logger.info("Index [{}] (alias [{}]) is not up to date. Updating mapping", indexState.concreteIndexName, this.aliasName); - final byte[] mappingSource = mappingSourceSupplier.get(); - final Tuple mappingAndSettings = parseMappingAndSettingsFromTemplateBytes(mappingSource); - PutMappingRequest request = new PutMappingRequest(indexState.concreteIndexName) - .source(mappingAndSettings.v1(), XContentType.JSON) - .type(MapperService.SINGLE_MAPPING_NAME); - executeAsyncWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN, request, - ActionListener.wrap(putMappingResponse -> { - if (putMappingResponse.isAcknowledged()) { + + @Override + public void onFailure(Exception e) { + final Throwable cause = ExceptionsHelper.unwrapCause(e); + if (cause instanceof ResourceAlreadyExistsException) { + // the index already exists - it was probably just created so this + // node hasn't yet received the cluster state update with the index andThen.run(); } else { - consumer.accept(new IllegalStateException("put mapping request was not acknowledged")); + consumer.accept(e); } - }, consumer), client.admin().indices()::putMapping); + } + }, client.admin().indices()::create); } else { andThen.run(); } @@ -431,48 +368,24 @@ public static boolean isIndexDeleted(State previousState, State currentState) { return previousState.indexHealth != null && currentState.indexHealth == null; } - private static byte[] readTemplateAsBytes(String templateName) { - return TemplateUtils.loadTemplate("/" + templateName + ".json", Version.CURRENT.toString(), - SecurityIndexManager.TEMPLATE_VERSION_VARIABLE).getBytes(StandardCharsets.UTF_8); - } - - private static Tuple parseMappingAndSettingsFromTemplateBytes(byte[] template) throws IOException { - final PutIndexTemplateRequest request = new PutIndexTemplateRequest("name_is_not_important").source(template, XContentType.JSON); - final String mappingSource = request.mappings().get(MapperService.SINGLE_MAPPING_NAME); - try (XContentParser parser = XContentType.JSON.xContent().createParser(NamedXContentRegistry.EMPTY, - DeprecationHandler.THROW_UNSUPPORTED_OPERATION, mappingSource)) { - // remove the type wrapping to get the mapping - ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser); // { - ensureFieldName(parser, parser.nextToken(), MapperService.SINGLE_MAPPING_NAME); // _doc - ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser); // { - - XContentBuilder builder = JsonXContent.contentBuilder(); - builder.generator().copyCurrentStructure(parser); - return new Tuple<>(Strings.toString(builder), request.settings()); - } - } - /** * State of the security index. */ public static class State { - public static final State UNRECOVERED_STATE = new State(null, false, false, false, null, null, null, null); + public static final State UNRECOVERED_STATE = new State(null, false, false, null, null, null, null); public final Instant creationTime; public final boolean isIndexUpToDate; public final boolean indexAvailable; - public final boolean mappingUpToDate; public final Version mappingVersion; public final String concreteIndexName; public final ClusterHealthStatus indexHealth; public final IndexMetadata.State indexState; public State(Instant creationTime, boolean isIndexUpToDate, boolean indexAvailable, - boolean mappingUpToDate, Version mappingVersion, String concreteIndexName, ClusterHealthStatus indexHealth, - IndexMetadata.State indexState) { + Version mappingVersion, String concreteIndexName, ClusterHealthStatus indexHealth, IndexMetadata.State indexState) { this.creationTime = creationTime; this.isIndexUpToDate = isIndexUpToDate; this.indexAvailable = indexAvailable; - this.mappingUpToDate = mappingUpToDate; this.mappingVersion = mappingVersion; this.concreteIndexName = concreteIndexName; this.indexHealth = indexHealth; @@ -487,7 +400,6 @@ public boolean equals(Object o) { return Objects.equals(creationTime, state.creationTime) && isIndexUpToDate == state.isIndexUpToDate && indexAvailable == state.indexAvailable && - mappingUpToDate == state.mappingUpToDate && Objects.equals(mappingVersion, state.mappingVersion) && Objects.equals(concreteIndexName, state.concreteIndexName) && indexHealth == state.indexHealth && @@ -500,7 +412,7 @@ public boolean indexExists() { @Override public int hashCode() { - return Objects.hash(creationTime, isIndexUpToDate, indexAvailable, mappingUpToDate, mappingVersion, concreteIndexName, + return Objects.hash(creationTime, isIndexUpToDate, indexAvailable, mappingVersion, concreteIndexName, indexHealth); } } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SecurityIntegTestCase.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SecurityIntegTestCase.java index 901716bc6a45d..0243bee6be3b1 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SecurityIntegTestCase.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SecurityIntegTestCase.java @@ -7,8 +7,6 @@ import io.netty.util.ThreadDeathWatcher; import io.netty.util.concurrent.GlobalEventExecutor; - -import org.elasticsearch.Version; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; import org.elasticsearch.action.admin.cluster.node.info.NodeInfo; import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; @@ -26,14 +24,12 @@ import org.elasticsearch.cluster.metadata.IndexAbstraction; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.routing.IndexRoutingTable; -import org.elasticsearch.common.Strings; import org.elasticsearch.common.network.NetworkAddress; import org.elasticsearch.common.network.NetworkModule; import org.elasticsearch.common.settings.MockSecureSettings; import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.TransportAddress; -import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.gateway.GatewayService; @@ -48,7 +44,6 @@ import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken; import org.elasticsearch.xpack.core.security.client.SecurityClient; import org.elasticsearch.xpack.security.LocalStateSecurity; -import org.elasticsearch.xpack.security.support.SecurityIndexManager; import org.junit.AfterClass; import org.junit.Before; import org.junit.BeforeClass; @@ -496,10 +491,6 @@ public void assertSecurityIndexActive(TestCluster testCluster) throws Exception ClusterState clusterState = client.admin().cluster().prepareState().setLocal(true).get().getState(); assertFalse(clusterState.blocks().hasGlobalBlock(GatewayService.STATE_NOT_RECOVERED_BLOCK)); XContentBuilder builder = JsonXContent.contentBuilder().prettyPrint().startObject(); - assertTrue("security index mapping not sufficient to read:\n" + - Strings.toString(clusterState.toXContent(builder, ToXContent.EMPTY_PARAMS).endObject()), - SecurityIndexManager.checkIndexMappingVersionMatches(SECURITY_MAIN_ALIAS, clusterState, logger, - Version.CURRENT.minimumIndexCompatibilityVersion()::onOrBefore)); Index securityIndex = resolveSecurityIndex(clusterState.metadata()); if (securityIndex != null) { IndexRoutingTable indexRoutingTable = clusterState.routingTable().index(securityIndex); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java index 7cc587ffc2270..697f909658a65 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java @@ -2029,6 +2029,6 @@ private void setCompletedToTrue(AtomicBoolean completed) { private SecurityIndexManager.State dummyState(ClusterHealthStatus indexStatus) { return new SecurityIndexManager.State( - Instant.now(), true, true, true, null, concreteSecurityIndexName, indexStatus, IndexMetadata.State.OPEN); + Instant.now(), true, true, null, concreteSecurityIndexName, indexStatus, IndexMetadata.State.OPEN); } } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealmTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealmTests.java index 5d7b3fd0bb54c..ea2c423306b55 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealmTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealmTests.java @@ -29,7 +29,7 @@ public class NativeRealmTests extends ESTestCase { private SecurityIndexManager.State dummyState(ClusterHealthStatus indexStatus) { return new SecurityIndexManager.State( - Instant.now(), true, true, true, null, concreteSecurityIndexName, indexStatus, IndexMetadata.State.OPEN); + Instant.now(), true, true, null, concreteSecurityIndexName, indexStatus, IndexMetadata.State.OPEN); } public void testCacheClearOnIndexHealthChange() { diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeUsersStoreTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeUsersStoreTests.java index 7cebb6328c3d0..b7fd84659c7fc 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeUsersStoreTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeUsersStoreTests.java @@ -253,7 +253,6 @@ private NativeUsersStore startNativeUsersStore() { SecurityIndexManager securityIndex = mock(SecurityIndexManager.class); when(securityIndex.isAvailable()).thenReturn(true); when(securityIndex.indexExists()).thenReturn(true); - when(securityIndex.isMappingUpToDate()).thenReturn(true); when(securityIndex.isIndexUpToDate()).thenReturn(true); when(securityIndex.freeze()).thenReturn(securityIndex); doAnswer((i) -> { diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/ActiveDirectoryRealmTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/ActiveDirectoryRealmTests.java index 3fffe25f53493..e1f980f411969 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/ActiveDirectoryRealmTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/ActiveDirectoryRealmTests.java @@ -381,7 +381,6 @@ public void testRealmWithTemplatedRoleMapping() throws Exception { SecurityIndexManager mockSecurityIndex = mock(SecurityIndexManager.class); when(mockSecurityIndex.isAvailable()).thenReturn(true); when(mockSecurityIndex.isIndexUpToDate()).thenReturn(true); - when(mockSecurityIndex.isMappingUpToDate()).thenReturn(true); Client mockClient = mock(Client.class); when(mockClient.threadPool()).thenReturn(threadPool); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/LdapRealmTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/LdapRealmTests.java index a57a22e83f0d7..fd1de61d4d221 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/LdapRealmTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/LdapRealmTests.java @@ -431,7 +431,6 @@ public void testLdapRealmWithTemplatedRoleMapping() throws Exception { SecurityIndexManager mockSecurityIndex = mock(SecurityIndexManager.class); when(mockSecurityIndex.isAvailable()).thenReturn(true); when(mockSecurityIndex.isIndexUpToDate()).thenReturn(true); - when(mockSecurityIndex.isMappingUpToDate()).thenReturn(true); Client mockClient = mock(Client.class); when(mockClient.threadPool()).thenReturn(threadPool); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStoreTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStoreTests.java index 7dbd48596faf9..320554c93c4cf 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStoreTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStoreTests.java @@ -145,7 +145,7 @@ private SecurityIndexManager.State dummyState(ClusterHealthStatus indexStatus) { private SecurityIndexManager.State indexState(boolean isUpToDate, ClusterHealthStatus healthStatus) { return new SecurityIndexManager.State( - Instant.now(), isUpToDate, true, true, null, concreteSecurityIndexName, healthStatus, IndexMetadata.State.OPEN); + Instant.now(), isUpToDate, true, null, concreteSecurityIndexName, healthStatus, IndexMetadata.State.OPEN); } public void testCacheClearOnIndexHealthChange() { diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStoreTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStoreTests.java index 05dfaa4587304..b9762ce69e5f6 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStoreTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStoreTests.java @@ -809,7 +809,7 @@ private SecurityIndexManager.State dummyState(ClusterHealthStatus indexStatus) { public SecurityIndexManager.State dummyIndexState(boolean isIndexUpToDate, ClusterHealthStatus healthStatus) { return new SecurityIndexManager.State( - Instant.now(), isIndexUpToDate, true, true, null, concreteSecurityIndexName, healthStatus, IndexMetadata.State.OPEN); + Instant.now(), isIndexUpToDate, true, null, concreteSecurityIndexName, healthStatus, IndexMetadata.State.OPEN); } public void testCacheClearOnIndexHealthChange() { diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStoreTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStoreTests.java index ff560d536aa7c..bc24dd72694f3 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStoreTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStoreTests.java @@ -582,7 +582,7 @@ public void testCacheClearOnIndexHealthChange() { CollectionUtils.appendToCopy(Arrays.asList(ClusterHealthStatus.values()), null); store.onSecurityIndexStateChange( dummyState(securityIndexName, isIndexUpToDate, randomFrom(allPossibleHealthStatus)), - dummyState(securityIndexName, !isIndexUpToDate, randomFrom(allPossibleHealthStatus))); + dummyState(securityIndexName, isIndexUpToDate == false, randomFrom(allPossibleHealthStatus))); assertEquals(++count, store.getNumInvalidation().get()); } @@ -615,7 +615,7 @@ public void testGetPrivilegesWorkWithoutCache() throws Exception { private SecurityIndexManager.State dummyState( String concreteSecurityIndexName, boolean isIndexUpToDate, ClusterHealthStatus healthStatus) { return new SecurityIndexManager.State( - Instant.now(), isIndexUpToDate, true, true, null, + Instant.now(), isIndexUpToDate, true, null, concreteSecurityIndexName, healthStatus, IndexMetadata.State.OPEN ); } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStoreTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStoreTests.java index 4c282f23f18f1..12d07b3da288c 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStoreTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStoreTests.java @@ -14,7 +14,6 @@ import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexMetadata; -import org.elasticsearch.cluster.metadata.IndexTemplateMetadata; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.routing.IndexRoutingTable; import org.elasticsearch.cluster.routing.IndexShardRoutingTable; @@ -26,7 +25,6 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -44,6 +42,7 @@ import org.elasticsearch.xpack.core.security.action.role.PutRoleRequest; import org.elasticsearch.xpack.core.security.authz.RoleDescriptor; import org.elasticsearch.xpack.core.security.authz.RoleDescriptor.IndicesPrivileges; +import org.elasticsearch.xpack.security.Security; import org.elasticsearch.xpack.security.support.SecurityIndexManager; import org.elasticsearch.xpack.security.test.SecurityTestUtils; import org.junit.After; @@ -53,7 +52,6 @@ import java.nio.charset.Charset; import java.nio.file.Files; import java.nio.file.Path; -import java.util.Collections; import java.util.List; import java.util.UUID; import java.util.concurrent.atomic.AtomicBoolean; @@ -190,7 +188,11 @@ public void testPutOfRoleWithFlsDlsUnlicensed() throws IOException { final ClusterService clusterService = mock(ClusterService.class); final XPackLicenseState licenseState = mock(XPackLicenseState.class); final AtomicBoolean methodCalled = new AtomicBoolean(false); - final SecurityIndexManager securityIndex = SecurityIndexManager.buildSecurityMainIndexManager(client, clusterService); + final SecurityIndexManager securityIndex = SecurityIndexManager.buildSecurityIndexManager( + client, + clusterService, + Security.SECURITY_MAIN_INDEX_DESCRIPTOR + ); final NativeRolesStore rolesStore = new NativeRolesStore(Settings.EMPTY, client, licenseState, securityIndex) { @Override void innerPutRole(final PutRoleRequest request, final RoleDescriptor role, final ActionListener listener) { @@ -259,9 +261,6 @@ private ClusterState getClusterStateWithSecurityIndex() { .build(); Metadata metadata = Metadata.builder() .put(IndexMetadata.builder(securityIndexName).settings(settings)) - .put(new IndexTemplateMetadata(SecurityIndexManager.SECURITY_MAIN_TEMPLATE_7, 0, 0, - Collections.singletonList(securityIndexName), Settings.EMPTY, ImmutableOpenMap.of(), - ImmutableOpenMap.of())) .build(); if (withAlias) { diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/CacheInvalidatorRegistryTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/CacheInvalidatorRegistryTests.java index 6ba336c740163..02bad9c42a95f 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/CacheInvalidatorRegistryTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/CacheInvalidatorRegistryTests.java @@ -46,7 +46,7 @@ public void testSecurityIndexStateChangeWillInvalidateAllRegisteredInvalidators( final SecurityIndexManager.State previousState = SecurityIndexManager.State.UNRECOVERED_STATE; final SecurityIndexManager.State currentState = new SecurityIndexManager.State( - Instant.now(), true, true, true, Version.CURRENT, + Instant.now(), true, true, Version.CURRENT, ".security", ClusterHealthStatus.GREEN, IndexMetadata.State.OPEN); cacheInvalidatorRegistry.onSecurityIndexStageChange(previousState, currentState); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/SecurityIndexManagerTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/SecurityIndexManagerTests.java index 6af2c67e8be04..a5289d9a71c10 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/SecurityIndexManagerTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/SecurityIndexManagerTests.java @@ -19,13 +19,8 @@ import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.Version; -import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.ActionRequest; -import org.elasticsearch.action.ActionResponse; -import org.elasticsearch.action.ActionType; import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest; import org.elasticsearch.client.Client; -import org.elasticsearch.client.FilterClient; import org.elasticsearch.cluster.ClusterChangedEvent; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; @@ -57,19 +52,24 @@ import org.elasticsearch.xpack.security.test.SecurityTestUtils; import org.elasticsearch.xpack.core.security.index.RestrictedIndicesNames; import org.elasticsearch.xpack.core.template.TemplateUtils; +import org.elasticsearch.xpack.security.Security; +import org.elasticsearch.xpack.security.test.SecurityTestUtils; import org.hamcrest.Matchers; import org.junit.Before; -import static org.elasticsearch.xpack.security.support.SecurityIndexManager.SECURITY_MAIN_TEMPLATE_7; +import java.io.IOException; +import java.util.Arrays; +import java.util.UUID; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.BiConsumer; + import static org.elasticsearch.xpack.security.support.SecurityIndexManager.TEMPLATE_VERSION_VARIABLE; -import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; public class SecurityIndexManagerTests extends ESTestCase { @@ -78,7 +78,6 @@ public class SecurityIndexManagerTests extends ESTestCase { private static final ClusterState EMPTY_CLUSTER_STATE = new ClusterState.Builder(CLUSTER_NAME).build(); private static final String TEMPLATE_NAME = "SecurityIndexManagerTests-template"; private SecurityIndexManager manager; - private Map, Map>> actions; @Before public void setUpManager() { @@ -90,34 +89,7 @@ public void setUpManager() { when(mockClient.settings()).thenReturn(Settings.EMPTY); final ClusterService clusterService = mock(ClusterService.class); - actions = new LinkedHashMap<>(); - final Client client = new FilterClient(mockClient) { - @Override - protected - void doExecute(ActionType action, Request request, ActionListener listener) { - final Map> map = actions.getOrDefault(action, new HashMap<>()); - map.put(request, listener); - actions.put(action, map); - } - }; - manager = SecurityIndexManager.buildSecurityMainIndexManager(client, clusterService); - - } - - public void testIndexWithFaultyMappingOnDisk() { - SecurityIndexManager.State state = new SecurityIndexManager.State(randomBoolean() ? Instant.now() : null, true, randomBoolean(), - false, null, "not_important", null, null); - Supplier mappingSourceSupplier = () -> { - throw new RuntimeException(); - }; - Runnable runnable = mock(Runnable.class); - manager = new SecurityIndexManager(mock(Client.class), RestrictedIndicesNames.SECURITY_MAIN_ALIAS, - RestrictedIndicesNames.INTERNAL_SECURITY_MAIN_INDEX_7, SecurityIndexManager.INTERNAL_MAIN_INDEX_FORMAT, - mappingSourceSupplier, state); - AtomicReference exceptionConsumer = new AtomicReference<>(); - manager.prepareIndexIfNeededThenExecute(e -> exceptionConsumer.set(e), runnable); - verify(runnable, never()).run(); - assertThat(exceptionConsumer.get(), is(notNullValue())); + manager = SecurityIndexManager.buildSecurityIndexManager(mockClient, clusterService, Security.SECURITY_MAIN_INDEX_DESCRIPTOR); } public void testIndexWithUpToDateMappingAndTemplate() throws IOException { @@ -130,7 +102,6 @@ public void testIndexWithUpToDateMappingAndTemplate() throws IOException { assertThat(manager.indexExists(), Matchers.equalTo(true)); assertThat(manager.isAvailable(), Matchers.equalTo(true)); - assertThat(manager.isMappingUpToDate(), Matchers.equalTo(true)); } public void testIndexWithoutPrimaryShards() throws IOException { @@ -354,14 +325,12 @@ public void testProcessClosedIndexState() throws Exception { private void assertInitialState() { assertThat(manager.indexExists(), Matchers.equalTo(false)); assertThat(manager.isAvailable(), Matchers.equalTo(false)); - assertThat(manager.isMappingUpToDate(), Matchers.equalTo(false)); assertThat(manager.isStateRecovered(), Matchers.equalTo(false)); } private void assertIndexUpToDateButNotAvailable() { assertThat(manager.indexExists(), Matchers.equalTo(true)); assertThat(manager.isAvailable(), Matchers.equalTo(false)); - assertThat(manager.isMappingUpToDate(), Matchers.equalTo(true)); assertThat(manager.isStateRecovered(), Matchers.equalTo(true)); } @@ -444,142 +413,4 @@ private static String loadTemplate(String templateName) { final String resource = "/" + templateName + ".json"; return TemplateUtils.loadTemplate(resource, Version.CURRENT.toString(), TEMPLATE_VERSION_VARIABLE); } - - public void testMappingVersionMatching() throws IOException { - String templateString = "/" + SECURITY_MAIN_TEMPLATE_7 + ".json"; - ClusterState.Builder clusterStateBuilder = createClusterStateWithMappingAndTemplate(templateString); - manager.clusterChanged(new ClusterChangedEvent("test-event", clusterStateBuilder.build(), EMPTY_CLUSTER_STATE)); - assertTrue(manager.checkMappingVersion(Version.CURRENT.minimumIndexCompatibilityVersion()::before)); - assertFalse(manager.checkMappingVersion(Version.CURRENT.minimumIndexCompatibilityVersion()::after)); - } - - public void testMissingVersionMappingThrowsError() throws IOException { - String templateString = "/missing-version-security-index-template.json"; - ClusterState.Builder clusterStateBuilder = createClusterStateWithMappingAndTemplate(templateString); - final ClusterState clusterState = clusterStateBuilder.build(); - IllegalStateException exception = expectThrows(IllegalStateException.class, - () -> SecurityIndexManager.checkIndexMappingVersionMatches(RestrictedIndicesNames.SECURITY_MAIN_ALIAS, - clusterState, logger, Version.CURRENT::equals)); - assertEquals("Cannot read security-version string in index " + RestrictedIndicesNames.SECURITY_MAIN_ALIAS, - exception.getMessage()); - } - - public void testIndexTemplateIsIdentifiedAsUpToDate() throws IOException { - ClusterState.Builder clusterStateBuilder = createClusterStateWithTemplate( - "/" + SECURITY_MAIN_TEMPLATE_7 + ".json" - ); - manager.clusterChanged(new ClusterChangedEvent("test-event", clusterStateBuilder.build(), EMPTY_CLUSTER_STATE)); - // No upgrade actions run - assertThat(actions.size(), equalTo(0)); - } - - public void testIndexTemplateVersionMatching() throws Exception { - String templateString = "/" + SECURITY_MAIN_TEMPLATE_7 + ".json"; - ClusterState.Builder clusterStateBuilder = createClusterStateWithTemplate(templateString); - final ClusterState clusterState = clusterStateBuilder.build(); - - assertTrue(SecurityIndexManager.checkTemplateExistsAndVersionMatches( - SecurityIndexManager.SECURITY_MAIN_TEMPLATE_7, clusterState, logger, - Version.V_6_0_0::before)); - assertFalse(SecurityIndexManager.checkTemplateExistsAndVersionMatches( - SecurityIndexManager.SECURITY_MAIN_TEMPLATE_7, clusterState, logger, - Version.V_6_0_0::after)); - } - - public void testUpToDateMappingsAreIdentifiedAsUpToDate() throws IOException { - String securityTemplateString = "/" + SECURITY_MAIN_TEMPLATE_7 + ".json"; - ClusterState.Builder clusterStateBuilder = createClusterStateWithMappingAndTemplate(securityTemplateString); - manager.clusterChanged(new ClusterChangedEvent("test-event", - clusterStateBuilder.build(), EMPTY_CLUSTER_STATE)); - assertThat(actions.size(), equalTo(0)); - } - - public void testMissingIndexIsIdentifiedAsUpToDate() throws IOException { - final ClusterName clusterName = new ClusterName("test-cluster"); - final ClusterState.Builder clusterStateBuilder = ClusterState.builder(clusterName); - String mappingString = "/" + SECURITY_MAIN_TEMPLATE_7 + ".json"; - IndexTemplateMetadata.Builder templateMeta = getIndexTemplateMetadata(SECURITY_MAIN_TEMPLATE_7, mappingString); - final ClusterState clusterState = clusterStateBuilder.build(); - Metadata.Builder builder = new Metadata.Builder(clusterState.getMetadata()); - builder.put(templateMeta); - manager.clusterChanged(new ClusterChangedEvent("test-event", ClusterState.builder(clusterState).metadata(builder).build() - , EMPTY_CLUSTER_STATE)); - assertThat(actions.size(), equalTo(0)); - } - - private ClusterState.Builder createClusterStateWithTemplate(String securityTemplateString) throws IOException { - // add the correct mapping no matter what the template - ClusterState clusterState = createClusterStateWithIndex("/" + SECURITY_MAIN_TEMPLATE_7 + ".json").build(); - final Metadata.Builder metadataBuilder = new Metadata.Builder(clusterState.metadata()); - metadataBuilder.put(getIndexTemplateMetadata(SECURITY_MAIN_TEMPLATE_7, securityTemplateString)); - return ClusterState.builder(clusterState).metadata(metadataBuilder); - } - - private ClusterState.Builder createClusterStateWithMapping(String securityTemplateString) throws IOException { - final ClusterState clusterState = createClusterStateWithIndex(securityTemplateString).build(); - final String indexName = clusterState.metadata().getIndicesLookup() - .get(RestrictedIndicesNames.SECURITY_MAIN_ALIAS).getIndices().get(0).getIndex().getName(); - return ClusterState.builder(clusterState).routingTable(SecurityTestUtils.buildIndexRoutingTable(indexName)); - } - - private ClusterState.Builder createClusterStateWithMappingAndTemplate(String securityTemplateString) throws IOException { - final ClusterState state = createClusterStateWithMapping(securityTemplateString).build(); - Metadata.Builder metadataBuilder = new Metadata.Builder(state.metadata()); - String securityMappingString = "/" + SECURITY_MAIN_TEMPLATE_7 + ".json"; - IndexTemplateMetadata.Builder securityTemplateMeta = getIndexTemplateMetadata(SECURITY_MAIN_TEMPLATE_7, securityMappingString); - metadataBuilder.put(securityTemplateMeta); - return ClusterState.builder(state).metadata(metadataBuilder); - } - - private static IndexMetadata.Builder createIndexMetadata(String indexName, String templateString) throws IOException { - String template = TemplateUtils.loadTemplate(templateString, Version.CURRENT.toString(), - SecurityIndexManager.TEMPLATE_VERSION_VARIABLE); - PutIndexTemplateRequest request = new PutIndexTemplateRequest(); - request.source(template, XContentType.JSON); - IndexMetadata.Builder indexMetadata = IndexMetadata.builder(indexName); - indexMetadata.settings(Settings.builder() - .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) - .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) - .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) - .build()); - - for (Map.Entry entry : request.mappings().entrySet()) { - indexMetadata.putMapping(entry.getKey(), entry.getValue()); - } - return indexMetadata; - } - - private ClusterState.Builder createClusterStateWithIndex(String securityTemplate) throws IOException { - final Metadata.Builder metadataBuilder = new Metadata.Builder(); - final boolean withAlias = randomBoolean(); - final String securityIndexName = RestrictedIndicesNames.SECURITY_MAIN_ALIAS - + (withAlias ? "-" + randomAlphaOfLength(5) : ""); - metadataBuilder.put(createIndexMetadata(securityIndexName, securityTemplate)); - - ClusterState.Builder clusterStateBuilder = ClusterState.builder(state()); - if (withAlias) { - // try with .security index as an alias - clusterStateBuilder.metadata(SecurityTestUtils.addAliasToMetadata(metadataBuilder.build(), securityIndexName)); - } else { - // try with .security index as a concrete index - clusterStateBuilder.metadata(metadataBuilder); - } - - clusterStateBuilder.routingTable(SecurityTestUtils.buildIndexRoutingTable(securityIndexName)); - return clusterStateBuilder; - } - - private static IndexTemplateMetadata.Builder getIndexTemplateMetadata(String templateName, String templateString) throws IOException { - - String template = TemplateUtils.loadTemplate(templateString, Version.CURRENT.toString(), - SecurityIndexManager.TEMPLATE_VERSION_VARIABLE); - PutIndexTemplateRequest request = new PutIndexTemplateRequest(); - request.source(template, XContentType.JSON); - IndexTemplateMetadata.Builder templateBuilder = IndexTemplateMetadata.builder(templateName) - .patterns(Arrays.asList(generateRandomStringArray(10, 100, false, false))); - for (Map.Entry entry : request.mappings().entrySet()) { - templateBuilder.putMapping(entry.getKey(), entry.getValue()); - } - return templateBuilder; - } } diff --git a/x-pack/qa/third-party/active-directory/src/test/java/org/elasticsearch/xpack/security/authc/ldap/AbstractAdLdapRealmTestCase.java b/x-pack/qa/third-party/active-directory/src/test/java/org/elasticsearch/xpack/security/authc/ldap/AbstractAdLdapRealmTestCase.java index e879c227aaa5d..4d83d54e89d8d 100644 --- a/x-pack/qa/third-party/active-directory/src/test/java/org/elasticsearch/xpack/security/authc/ldap/AbstractAdLdapRealmTestCase.java +++ b/x-pack/qa/third-party/active-directory/src/test/java/org/elasticsearch/xpack/security/authc/ldap/AbstractAdLdapRealmTestCase.java @@ -16,7 +16,6 @@ import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.SecurityIntegTestCase; @@ -26,7 +25,6 @@ import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken; import org.elasticsearch.xpack.core.security.client.SecurityClient; import org.elasticsearch.xpack.core.ssl.VerificationMode; -import org.elasticsearch.xpack.security.support.SecurityIndexManager; import org.junit.After; import org.junit.AfterClass; import org.junit.Before; @@ -40,7 +38,6 @@ import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.Set; import java.util.function.Function; import java.util.function.Supplier; import java.util.stream.Collectors; @@ -158,13 +155,6 @@ public void cleanupSecurityIndex() throws Exception { super.deleteSecurityIndex(); } - @Override - public Set excludeTemplates() { - Set templates = Sets.newHashSet(super.excludeTemplates()); - templates.add(SecurityIndexManager.SECURITY_MAIN_TEMPLATE_7); // don't remove the security index template - return templates; - } - private List getRoleMappingContent(Function contentFunction) { return getRoleMappingContent(contentFunction, AbstractAdLdapRealmTestCase.roleMappings); }