diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/TestSystemIndexDescriptor.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/TestSystemIndexDescriptor.java index 30537004ff8a5..6df2d6123d2a5 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/TestSystemIndexDescriptor.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/TestSystemIndexDescriptor.java @@ -50,7 +50,8 @@ public class TestSystemIndexDescriptor extends SystemIndexDescriptor { 0, "version", "stack", - MapperService.SINGLE_MAPPING_NAME + MapperService.SINGLE_MAPPING_NAME, + null ); } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/create/AutoCreateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/create/AutoCreateAction.java index c13c235cc1903..af91b36711efa 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/create/AutoCreateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/create/AutoCreateAction.java @@ -140,9 +140,24 @@ public ClusterState execute(ClusterState currentState) throws Exception { } final SystemIndexDescriptor descriptor = systemIndices.findMatchingDescriptor(indexName); - CreateIndexClusterStateUpdateRequest updateRequest = descriptor != null && descriptor.isAutomaticallyManaged() - ? buildSystemIndexUpdateRequest(descriptor) - : buildUpdateRequest(indexName); + final boolean isSystemIndex = descriptor != null && descriptor.isAutomaticallyManaged(); + + final CreateIndexClusterStateUpdateRequest updateRequest; + + if (isSystemIndex) { + final String message = descriptor.checkMinimumNodeVersion( + "auto-create index", + state.nodes().getMinNodeVersion() + ); + if (message != null) { + logger.warn(message); + throw new IllegalStateException(message); + } + + updateRequest = buildSystemIndexUpdateRequest(descriptor); + } else { + updateRequest = buildUpdateRequest(indexName); + } return createIndexService.applyCreateIndexRequest(currentState, updateRequest, false); } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java index 9b60048d193a5..7a44a769556d7 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java @@ -76,6 +76,12 @@ public class CreateIndexRequest extends AcknowledgedRequest private ActiveShardCount waitForActiveShards = ActiveShardCount.DEFAULT; + private String origin = ""; + + /** + * Constructs a new request by deserializing an input + * @param in the input from which to deserialize + */ public CreateIndexRequest(StreamInput in) throws IOException { super(in); cause = in.readString(); @@ -107,20 +113,28 @@ public CreateIndexRequest(StreamInput in) throws IOException { in.readBoolean(); // updateAllTypes } waitForActiveShards = ActiveShardCount.readFrom(in); + if (in.getVersion().onOrAfter(Version.V_7_12_0)) { + origin = in.readString(); + } } public CreateIndexRequest() { } /** - * Constructs a new request to create an index with the specified name. + * Constructs a request to create an index. + * + * @param index the name of the index */ public CreateIndexRequest(String index) { this(index, EMPTY_SETTINGS); } /** - * Constructs a new request to create an index with the specified name and settings. + * Constructs a request to create an index. + * + * @param index the name of the index + * @param settings the settings to apply to the index */ public CreateIndexRequest(String index, Settings settings) { this.index = index; @@ -172,6 +186,15 @@ public String cause() { return cause; } + public String origin() { + return origin; + } + + public CreateIndexRequest origin(String origin) { + this.origin = Objects.requireNonNull(origin); + return this; + } + /** * The settings to create the index with. */ @@ -265,7 +288,7 @@ public CreateIndexRequest mapping(String type, Map source) { throw new IllegalStateException("mappings for type \"" + type + "\" were already defined"); } // wrap it in a type map if its not - if (source.size() != 1 || !source.containsKey(type)) { + if (source.size() != 1 || source.containsKey(type) == false) { source = MapBuilder.newMapBuilder().put(type, source).map(); } try { @@ -466,6 +489,9 @@ public void writeTo(StreamOutput out) throws IOException { out.writeBoolean(true); // updateAllTypes } waitForActiveShards.writeTo(out); + if (out.getVersion().onOrAfter(Version.V_7_12_0)) { + out.writeString(origin); + } } @Override diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/create/TransportCreateIndexAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/create/TransportCreateIndexAction.java index 4ca400759e352..38d40d9b9b2fa 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/create/TransportCreateIndexAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/create/TransportCreateIndexAction.java @@ -8,6 +8,8 @@ package org.elasticsearch.action.admin.indices.create; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.indices.alias.Alias; import org.elasticsearch.action.support.ActionFilters; @@ -19,6 +21,7 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.MetadataCreateIndexService; import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.indices.SystemIndexDescriptor; @@ -35,6 +38,7 @@ * Create index action. */ public class TransportCreateIndexAction extends TransportMasterNodeAction { + private static final Logger logger = LogManager.getLogger(TransportCreateIndexAction.class); private final MetadataCreateIndexService createIndexService; private final SystemIndices systemIndices; @@ -66,9 +70,25 @@ protected void masterOperation(final CreateIndexRequest request, final ClusterSt final String indexName = indexNameExpressionResolver.resolveDateMathExpression(request.index()); final SystemIndexDescriptor descriptor = systemIndices.findMatchingDescriptor(indexName); - final CreateIndexClusterStateUpdateRequest updateRequest = descriptor != null && descriptor.isAutomaticallyManaged() - ? buildSystemIndexUpdateRequest(request, cause, descriptor) - : buildUpdateRequest(request, cause, indexName); + final boolean isSystemIndex = descriptor != null && descriptor.isAutomaticallyManaged(); + + final CreateIndexClusterStateUpdateRequest updateRequest; + + // Requests that a cluster generates itself are permitted to create a system index with + // different mappings, settings etc. This is so that rolling upgrade scenarios still work. + // We check this via the request's origin. Eventually, `SystemIndexManager` will reconfigure + // the index to the latest settings. + if (isSystemIndex && Strings.isNullOrEmpty(request.origin())) { + final String message = descriptor.checkMinimumNodeVersion("create index", state.nodes().getMinNodeVersion()); + if (message != null) { + logger.warn(message); + listener.onFailure(new IllegalStateException(message)); + return; + } + updateRequest = buildSystemIndexUpdateRequest(request, cause, descriptor); + } else { + updateRequest = buildUpdateRequest(request, cause, indexName); + } createIndexService.createIndex(updateRequest, listener.map(response -> new CreateIndexResponse(response.isAcknowledged(), response.isShardsAcknowledged(), indexName))); diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/TransportAutoPutMappingAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/TransportAutoPutMappingAction.java index 135834084f9cc..d1c8df0b0e57a 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/TransportAutoPutMappingAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/TransportAutoPutMappingAction.java @@ -7,6 +7,8 @@ */ package org.elasticsearch.action.admin.indices.mapping.put; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.master.AcknowledgedResponse; @@ -19,6 +21,7 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.index.Index; +import org.elasticsearch.indices.SystemIndices; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; @@ -27,7 +30,10 @@ public class TransportAutoPutMappingAction extends AcknowledgedTransportMasterNodeAction { + private static final Logger logger = LogManager.getLogger(TransportAutoPutMappingAction.class); + private final MetadataMappingService metadataMappingService; + private final SystemIndices systemIndices; @Inject public TransportAutoPutMappingAction( @@ -36,10 +42,12 @@ public TransportAutoPutMappingAction( final ThreadPool threadPool, final MetadataMappingService metadataMappingService, final ActionFilters actionFilters, - final IndexNameExpressionResolver indexNameExpressionResolver) { + final IndexNameExpressionResolver indexNameExpressionResolver, + final SystemIndices systemIndices) { super(AutoPutMappingAction.NAME, transportService, clusterService, threadPool, actionFilters, PutMappingRequest::new, indexNameExpressionResolver, ThreadPool.Names.SAME); this.metadataMappingService = metadataMappingService; + this.systemIndices = systemIndices; } @Override @@ -61,6 +69,14 @@ protected ClusterBlockException checkBlock(PutMappingRequest request, ClusterSta protected void masterOperation(final PutMappingRequest request, final ClusterState state, final ActionListener listener) { final Index[] concreteIndices = new Index[] {request.getConcreteIndex()}; + + final String message = TransportPutMappingAction.checkForSystemIndexViolations(systemIndices, concreteIndices, request); + if (message != null) { + logger.warn(message); + listener.onFailure(new IllegalStateException(message)); + return; + } + performMappingUpdate(concreteIndices, request, listener, metadataMappingService); } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/TransportPutMappingAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/TransportPutMappingAction.java index e9fa5af73820e..537c54e480627 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/TransportPutMappingAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/TransportPutMappingAction.java @@ -22,6 +22,7 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.MetadataMappingService; import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexNotFoundException; @@ -80,7 +81,6 @@ protected void masterOperation(final PutMappingRequest request, final ClusterSta final ActionListener listener) { try { final Index[] concreteIndices = resolveIndices(state, request, indexNameExpressionResolver); - final String mappingSource = request.source(); final Optional maybeValidationException = requestValidators.validateRequest(request, state, concreteIndices); if (maybeValidationException.isPresent()) { @@ -88,14 +88,10 @@ protected void masterOperation(final PutMappingRequest request, final ClusterSta return; } - final List violations = checkForSystemIndexViolations(concreteIndices, mappingSource); - if (violations.isEmpty() == false) { - final String message = "Cannot update mappings in " - + violations - + ": system indices can only use mappings from their descriptors, " - + "but the mappings in the request did not match those in the descriptors(s)"; + final String message = checkForSystemIndexViolations(systemIndices, concreteIndices, request); + if (message != null) { logger.warn(message); - listener.onFailure(new IllegalArgumentException(message)); + listener.onFailure(new IllegalStateException(message)); return; } @@ -149,14 +145,21 @@ public void onFailure(Exception t) { }); } - private List checkForSystemIndexViolations(Index[] concreteIndices, String requestMappings) { + static String checkForSystemIndexViolations(SystemIndices systemIndices, Index[] concreteIndices, PutMappingRequest request) { + // Requests that a cluster generates itself are permitted to have a difference in mappings + // so that rolling upgrade scenarios still work. We check this via the request's origin. + if (Strings.isNullOrEmpty(request.origin()) == false) { + return null; + } + List violations = new ArrayList<>(); + final String requestMappings = request.source(); + for (Index index : concreteIndices) { final SystemIndexDescriptor descriptor = systemIndices.findMatchingDescriptor(index.getName()); if (descriptor != null && descriptor.isAutomaticallyManaged()) { final String descriptorMappings = descriptor.getMappings(); - // Technically we could trip over a difference in whitespace here, but then again nobody should be trying to manually // update a descriptor's mappings. if (descriptorMappings.equals(requestMappings) == false) { @@ -164,6 +167,14 @@ private List checkForSystemIndexViolations(Index[] concreteIndices, Stri } } } - return violations; + + if (violations.isEmpty() == false) { + return "Cannot update mappings in " + + violations + + ": system indices can only use mappings from their descriptors, " + + "but the mappings in the request did not match those in the descriptors(s)"; + } + + return null; } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/settings/put/TransportUpdateSettingsAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/settings/put/TransportUpdateSettingsAction.java index 237b2f8d26695..30515b8988bc2 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/settings/put/TransportUpdateSettingsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/settings/put/TransportUpdateSettingsAction.java @@ -8,13 +8,6 @@ package org.elasticsearch.action.admin.indices.settings.put; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.stream.Collectors; - import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; @@ -29,6 +22,7 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.MetadataUpdateSettingsService; import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.Index; @@ -37,6 +31,14 @@ import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; + public class TransportUpdateSettingsAction extends AcknowledgedTransportMasterNodeAction { private static final Logger logger = LogManager.getLogger(TransportUpdateSettingsAction.class); @@ -78,8 +80,7 @@ protected void masterOperation(final UpdateSettingsRequest request, final Cluste final Index[] concreteIndices = indexNameExpressionResolver.concreteIndices(state, request); final Settings requestSettings = request.settings(); - - final Map> systemIndexViolations = checkForSystemIndexViolations(concreteIndices, requestSettings); + final Map> systemIndexViolations = checkForSystemIndexViolations(concreteIndices, request); if (systemIndexViolations.isEmpty() == false) { final String message = "Cannot override settings on system indices: " + systemIndexViolations.entrySet() @@ -87,7 +88,7 @@ protected void masterOperation(final UpdateSettingsRequest request, final Cluste .map(entry -> "[" + entry.getKey() + "] -> " + entry.getValue()) .collect(Collectors.joining(", ")); logger.warn(message); - listener.onFailure(new IllegalArgumentException(message)); + listener.onFailure(new IllegalStateException(message)); return; } @@ -117,11 +118,18 @@ public void onFailure(Exception t) { * that the system index's descriptor expects. * * @param concreteIndices the indices being updated - * @param requestSettings the settings to be applied + * @param request the update request * @return a mapping from system index pattern to the settings whose values would be overridden. Empty if there are no violations. */ - private Map> checkForSystemIndexViolations(Index[] concreteIndices, Settings requestSettings) { - final Map> violations = new HashMap<>(); + private Map> checkForSystemIndexViolations(Index[] concreteIndices, UpdateSettingsRequest request) { + // Requests that a cluster generates itself are permitted to have a difference in settings + // so that rolling upgrade scenarios still work. We check this via the request's origin. + if (Strings.isNullOrEmpty(request.origin()) == false) { + return Collections.emptyMap(); + } + + final Map> violationsByIndex = new HashMap<>(); + final Settings requestSettings = request.settings(); for (Index index : concreteIndices) { final SystemIndexDescriptor descriptor = systemIndices.findMatchingDescriptor(index.getName()); @@ -138,10 +146,11 @@ private Map> checkForSystemIndexViolations(Index[] concrete } if (failedKeys.isEmpty() == false) { - violations.put(descriptor.getIndexPattern(), failedKeys); + violationsByIndex.put(descriptor.getIndexPattern(), failedKeys); } } } - return violations; + + return violationsByIndex; } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/settings/put/UpdateSettingsRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/settings/put/UpdateSettingsRequest.java index 5249a082c4b23..ec41573a86759 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/settings/put/UpdateSettingsRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/settings/put/UpdateSettingsRequest.java @@ -8,6 +8,7 @@ package org.elasticsearch.action.admin.indices.settings.put; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.IndicesRequest; import org.elasticsearch.action.support.IndicesOptions; @@ -42,6 +43,7 @@ public class UpdateSettingsRequest extends AcknowledgedRequest source) { return this; } + public String origin() { + return origin; + } + + public UpdateSettingsRequest origin(String origin) { + this.origin = Objects.requireNonNull(origin, "origin cannot be null"); + return this; + } + @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); @@ -167,6 +181,9 @@ public void writeTo(StreamOutput out) throws IOException { indicesOptions.writeIndicesOptions(out); writeSettingsToStream(settings, out); out.writeBoolean(preserveExisting); + if (out.getVersion().onOrAfter(Version.V_7_12_0)) { + out.writeString(origin); + } } @Override diff --git a/server/src/main/java/org/elasticsearch/indices/SystemIndexDescriptor.java b/server/src/main/java/org/elasticsearch/indices/SystemIndexDescriptor.java index 16627119bcf75..7a019c602e578 100644 --- a/server/src/main/java/org/elasticsearch/indices/SystemIndexDescriptor.java +++ b/server/src/main/java/org/elasticsearch/indices/SystemIndexDescriptor.java @@ -12,12 +12,14 @@ import org.apache.lucene.util.automaton.CharacterRunAutomaton; import org.apache.lucene.util.automaton.Operations; import org.apache.lucene.util.automaton.RegExp; +import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.index.mapper.MapperService; +import java.util.Locale; import java.util.Objects; /** @@ -66,6 +68,9 @@ public class SystemIndexDescriptor { /** The index type to use when creating an index. */ private final String indexType; + /** The minimum cluster node version required for this descriptor, or null if there is no restriction */ + private final Version minimumNodeVersion; + /** * Creates a descriptor for system indices matching the supplied pattern. These indices will not be managed * by Elasticsearch internally. @@ -73,7 +78,7 @@ public class SystemIndexDescriptor { * @param description The name of the plugin responsible for this system index. */ public SystemIndexDescriptor(String indexPattern, String description) { - this(indexPattern, null, description, null, null, null, 0, null, null, MapperService.SINGLE_MAPPING_NAME); + this(indexPattern, null, description, null, null, null, 0, null, null, MapperService.SINGLE_MAPPING_NAME, null); } /** @@ -88,9 +93,10 @@ public SystemIndexDescriptor(String indexPattern, String description) { * @param aliasName An alias for the index, or null * @param indexFormat A value for the `index.format` setting. Pass 0 or higher. * @param versionMetaKey a mapping key under _meta where a version can be found, which indicates the - * Elasticsearch version when the index was created. + * Elasticsearch version when the index was created. * @param origin the client origin to use when creating this index. * @param indexType the index type. Should be {@link MapperService#SINGLE_MAPPING_NAME} for any new system indices. + * @param minimumNodeVersion the minimum cluster node version required for this descriptor, or null if there is no restriction */ SystemIndexDescriptor( String indexPattern, @@ -102,7 +108,8 @@ public SystemIndexDescriptor(String indexPattern, String description) { int indexFormat, String versionMetaKey, String origin, - String indexType + String indexType, + Version minimumNodeVersion ) { Objects.requireNonNull(indexPattern, "system index pattern must not be null"); if (indexPattern.length() < 2) { @@ -164,6 +171,7 @@ public SystemIndexDescriptor(String indexPattern, String description) { this.versionMetaKey = versionMetaKey; this.origin = origin; this.indexType = indexType; + this.minimumNodeVersion = minimumNodeVersion; } /** @@ -234,6 +242,29 @@ public String getIndexType() { return indexType; } + /** + * Checks that this descriptor can be used within this cluster e.g. the cluster supports all required + * features, by comparing the supplied minimum node version to this descriptor's minimum version. + * + * @param cause the action being attempted that triggered the check. Used in the error message. + * @param actualMinimumNodeVersion the lower node version in the cluster + * @return an error message if the lowest node version is lower that the version in this descriptor, + * or null if the supplied version is acceptable or this descriptor has no minimum version. + */ + public String checkMinimumNodeVersion(String cause, Version actualMinimumNodeVersion) { + Objects.requireNonNull(cause); + if (this.minimumNodeVersion != null && this.minimumNodeVersion.after(actualMinimumNodeVersion)) { + return String.format( + Locale.ROOT, + "[%s] failed - system index [%s] requires all cluster nodes to be at least version [%s]", + cause, + this.getPrimaryIndex(), + minimumNodeVersion + ); + } + return null; + } + // TODO: getThreadpool() // TODO: Upgrade handling (reindex script?) @@ -241,6 +272,9 @@ public static Builder builder() { return new Builder(); } + /** + * Provides a fluent API for building a {@link SystemIndexDescriptor}. Validation still happens in that class. + */ public static class Builder { private String indexPattern; private String primaryIndex; @@ -252,6 +286,7 @@ public static class Builder { private String versionMetaKey = null; private String origin = null; private String indexType = MapperService.SINGLE_MAPPING_NAME; + private Version minimumNodeVersion = null; private Builder() {} @@ -310,6 +345,15 @@ public Builder setIndexType(String indexType) { return this; } + public Builder setMinimumNodeVersion(Version version) { + this.minimumNodeVersion = version; + return this; + } + + /** + * Builds a {@link SystemIndexDescriptor} using the fields supplied to this builder. + * @return a populated descriptor. + */ public SystemIndexDescriptor build() { String mappings = mappingsBuilder == null ? null : Strings.toString(mappingsBuilder); @@ -323,7 +367,8 @@ public SystemIndexDescriptor build() { indexFormat, versionMetaKey, origin, - indexType + indexType, + minimumNodeVersion ); } } @@ -353,6 +398,7 @@ static Automaton buildAutomaton(String pattern, String alias) { * {@link RegExp} instance. This exists because although * {@link org.elasticsearch.common.regex.Regex#simpleMatchToAutomaton(String)} is useful * for simple patterns, it doesn't support character ranges. + * * @param input the string to translate * @return the translate string */ diff --git a/server/src/main/java/org/elasticsearch/indices/SystemIndexManager.java b/server/src/main/java/org/elasticsearch/indices/SystemIndexManager.java index 545fea78f5ca7..f54e69a6f8aac 100644 --- a/server/src/main/java/org/elasticsearch/indices/SystemIndexManager.java +++ b/server/src/main/java/org/elasticsearch/indices/SystemIndexManager.java @@ -64,7 +64,7 @@ public void clusterChanged(ClusterChangedEvent event) { if (state.blocks().hasGlobalBlock(GatewayService.STATE_NOT_RECOVERED_BLOCK)) { // wait until the gateway has recovered from disk, otherwise we may think we don't have some // indices but they may not have been restored from the cluster state on disk - logger.debug("system indices manager waiting until state has been recovered"); + logger.debug("Waiting until state has been recovered"); return; } @@ -73,6 +73,11 @@ public void clusterChanged(ClusterChangedEvent event) { return; } + if (state.nodes().getMaxNodeVersion().after(state.nodes().getMinNodeVersion())) { + logger.debug("Skipping system indices up-to-date check as cluster has mixed versions"); + return; + } + if (isUpgradeInProgress.compareAndSet(false, true)) { final List descriptors = getEligibleDescriptors(state.getMetadata()).stream() .filter(descriptor -> getUpgradeStatus(state, descriptor) == UpgradeStatus.NEEDS_MAPPINGS_UPDATE) @@ -90,6 +95,8 @@ public void clusterChanged(ClusterChangedEvent event) { } else { isUpgradeInProgress.set(false); } + } else { + logger.trace("Update already in progress"); } } diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java index aecc7b1aa0fab..1f6a0b83a505f 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java +++ b/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java @@ -1582,7 +1582,7 @@ clusterService, indicesService, threadPool, shardStateAction, mappingUpdatedActi systemIndices)); actions.put(AutoPutMappingAction.INSTANCE, new TransportAutoPutMappingAction(transportService, clusterService, threadPool, metadataMappingService, - actionFilters, indexNameExpressionResolver)); + actionFilters, indexNameExpressionResolver, systemIndices)); final ResponseCollectorService responseCollectorService = new ResponseCollectorService(clusterService); final SearchTransportService searchTransportService = new SearchTransportService(transportService, client, SearchExecutionStatsCollector.makeWrapper(responseCollectorService)); diff --git a/x-pack/plugin/core/src/main/resources/security-index-template-7.json b/x-pack/plugin/core/src/main/resources/security-index-template-7.json deleted file mode 100644 index bf587ce8825a1..0000000000000 --- a/x-pack/plugin/core/src/main/resources/security-index-template-7.json +++ /dev/null @@ -1,296 +0,0 @@ -{ - "index_patterns" : [ ".security-7" ], - "order" : 1000, - "settings" : { - "number_of_shards" : 1, - "number_of_replicas" : 0, - "auto_expand_replicas" : "0-1", - "index.priority": 1000, - "index.refresh_interval": "1s", - "index.format": 6, - "analysis" : { - "filter" : { - "email" : { - "type" : "pattern_capture", - "preserve_original" : true, - "patterns" : [ - "([^@]+)", - "(\\p{L}+)", - "(\\d+)", - "@(.+)" - ] - } - }, - "analyzer" : { - "email" : { - "tokenizer" : "uax_url_email", - "filter" : [ - "email", - "lowercase", - "unique" - ] - } - } - } - }, - "mappings" : { - "_doc" : { - "_meta": { - "security-version": "${security.template.version}" - }, - "dynamic" : "strict", - "properties" : { - "username" : { - "type" : "keyword" - }, - "roles" : { - "type" : "keyword" - }, - "role_templates" : { - "properties": { - "template" : { - "type": "text" - }, - "format" : { - "type": "keyword" - } - } - }, - "password" : { - "type" : "keyword", - "index" : false, - "doc_values": false - }, - "full_name" : { - "type" : "text" - }, - "email" : { - "type" : "text", - "analyzer" : "email" - }, - "metadata" : { - "type" : "object", - "dynamic" : false - }, - "enabled": { - "type": "boolean" - }, - "cluster" : { - "type" : "keyword" - }, - "indices" : { - "type" : "object", - "properties" : { - "field_security" : { - "properties" : { - "grant": { - "type": "keyword" - }, - "except": { - "type": "keyword" - } - } - }, - "names" : { - "type" : "keyword" - }, - "privileges" : { - "type" : "keyword" - }, - "query" : { - "type" : "keyword" - }, - "allow_restricted_indices" : { - "type" : "boolean" - } - } - }, - "applications": { - "type": "object", - "properties": { - "application": { - "type": "keyword" - }, - "privileges": { - "type": "keyword" - }, - "resources": { - "type": "keyword" - } - } - }, - "application" : { - "type" : "keyword" - }, - "global": { - "type": "object", - "properties": { - "application": { - "type": "object", - "properties": { - "manage": { - "type": "object", - "properties": { - "applications": { - "type": "keyword" - } - } - } - } - } - } - }, - "name" : { - "type" : "keyword" - }, - "run_as" : { - "type" : "keyword" - }, - "doc_type" : { - "type" : "keyword" - }, - "type" : { - "type" : "keyword" - }, - "actions" : { - "type" : "keyword" - }, - "expiration_time" : { - "type" : "date", - "format" : "epoch_millis" - }, - "creation_time" : { - "type" : "date", - "format" : "epoch_millis" - }, - "api_key_hash" : { - "type" : "keyword", - "index": false, - "doc_values": false - }, - "api_key_invalidated" : { - "type" : "boolean" - }, - "role_descriptors" : { - "type" : "object", - "enabled": false - }, - "limited_by_role_descriptors" : { - "type" : "object", - "enabled": false - }, - "version" : { - "type" : "integer" - }, - "creator" : { - "type" : "object", - "properties" : { - "principal" : { - "type": "keyword" - }, - "full_name" : { - "type" : "text" - }, - "email" : { - "type" : "text", - "analyzer" : "email" - }, - "metadata" : { - "type" : "object", - "dynamic" : false - }, - "realm" : { - "type" : "keyword" - }, - "realm_type" : { - "type" : "keyword" - } - } - }, - "rules" : { - "type" : "object", - "dynamic" : false - }, - "refresh_token" : { - "type" : "object", - "properties" : { - "token" : { - "type" : "keyword" - }, - "refreshed" : { - "type" : "boolean" - }, - "refresh_time": { - "type": "date", - "format": "epoch_millis" - }, - "superseding": { - "type": "object", - "properties": { - "encrypted_tokens": { - "type": "binary" - }, - "encryption_iv": { - "type": "binary" - }, - "encryption_salt": { - "type": "binary" - } - } - }, - "invalidated" : { - "type" : "boolean" - }, - "client" : { - "type" : "object", - "properties" : { - "type" : { - "type" : "keyword" - }, - "user" : { - "type" : "keyword" - }, - "realm" : { - "type" : "keyword" - } - } - } - } - }, - "access_token" : { - "type" : "object", - "properties" : { - "user_token" : { - "type" : "object", - "properties" : { - "id" : { - "type" : "keyword" - }, - "expiration_time" : { - "type" : "date", - "format" : "epoch_millis" - }, - "version" : { - "type" : "integer" - }, - "metadata" : { - "type" : "object", - "dynamic" : false - }, - "authentication" : { - "type" : "binary" - } - } - }, - "invalidated" : { - "type" : "boolean" - }, - "realm" : { - "type" : "keyword" - } - } - } - } - } - } -} diff --git a/x-pack/plugin/core/src/main/resources/security-tokens-index-template-7.json b/x-pack/plugin/core/src/main/resources/security-tokens-index-template-7.json deleted file mode 100644 index 502daae3f79bd..0000000000000 --- a/x-pack/plugin/core/src/main/resources/security-tokens-index-template-7.json +++ /dev/null @@ -1,108 +0,0 @@ -{ - "index_patterns" : [ ".security-tokens-7" ], - "order" : 1000, - "settings" : { - "number_of_shards" : 1, - "number_of_replicas" : 0, - "auto_expand_replicas" : "0-1", - "index.priority": 1000, - "index.refresh_interval": "1s", - "index.format": 7 - }, - "mappings" : { - "_doc" : { - "_meta": { - "security-version": "${security.template.version}" - }, - "dynamic" : "strict", - "properties" : { - "doc_type" : { - "type" : "keyword" - }, - "creation_time" : { - "type" : "date", - "format" : "epoch_millis" - }, - "refresh_token" : { - "type" : "object", - "properties" : { - "token" : { - "type" : "keyword" - }, - "refreshed" : { - "type" : "boolean" - }, - "refresh_time": { - "type": "date", - "format": "epoch_millis" - }, - "superseding": { - "type": "object", - "properties": { - "encrypted_tokens": { - "type": "binary" - }, - "encryption_iv": { - "type": "binary" - }, - "encryption_salt": { - "type": "binary" - } - } - }, - "invalidated" : { - "type" : "boolean" - }, - "client" : { - "type" : "object", - "properties" : { - "type" : { - "type" : "keyword" - }, - "user" : { - "type" : "keyword" - }, - "realm" : { - "type" : "keyword" - } - } - } - } - }, - "access_token" : { - "type" : "object", - "properties" : { - "user_token" : { - "type" : "object", - "properties" : { - "id" : { - "type" : "keyword" - }, - "expiration_time" : { - "type" : "date", - "format" : "epoch_millis" - }, - "version" : { - "type" : "integer" - }, - "metadata" : { - "type" : "object", - "dynamic" : false - }, - "authentication" : { - "type" : "binary" - } - } - }, - "invalidated" : { - "type" : "boolean" - }, - "realm" : { - "type" : "keyword" - } - } - } - } - } - } -} diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/test/NativeRealmIntegTestCase.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/test/NativeRealmIntegTestCase.java index f61f67a6d040e..76204d0949433 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/test/NativeRealmIntegTestCase.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/test/NativeRealmIntegTestCase.java @@ -11,7 +11,6 @@ import org.elasticsearch.client.RestClient; import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.transport.netty4.Netty4Transport; import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken; import org.elasticsearch.xpack.core.security.client.SecurityClient; @@ -22,14 +21,12 @@ import org.elasticsearch.xpack.core.security.user.KibanaUser; import org.elasticsearch.xpack.core.security.user.LogstashSystemUser; import org.elasticsearch.xpack.core.security.user.RemoteMonitoringUser; -import org.elasticsearch.xpack.security.support.SecurityIndexManager; import org.junit.After; import org.junit.Before; import java.io.IOException; import java.util.Arrays; import java.util.List; -import java.util.Set; /** * Test case with method to handle the starting and stopping the stores for native users and roles @@ -60,13 +57,6 @@ protected boolean addMockHttpTransport() { return false; // enable http } - @Override - public Set excludeTemplates() { - Set templates = Sets.newHashSet(super.excludeTemplates()); - templates.add(SecurityIndexManager.SECURITY_MAIN_TEMPLATE_7); // don't remove the security index template - return templates; - } - @Override protected Settings nodeSettings(int nodeOrdinal) { Settings.Builder builder = Settings.builder().put(super.nodeSettings(nodeOrdinal)); diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java index 87c8177d4ff90..d75e90e4ea355 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java @@ -42,6 +42,7 @@ import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.env.Environment; import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.http.HttpServerTransport; @@ -106,9 +107,9 @@ import org.elasticsearch.xpack.core.security.action.rolemapping.GetRoleMappingsAction; import org.elasticsearch.xpack.core.security.action.rolemapping.PutRoleMappingAction; import org.elasticsearch.xpack.core.security.action.saml.SamlAuthenticateAction; +import org.elasticsearch.xpack.core.security.action.saml.SamlCompleteLogoutAction; import org.elasticsearch.xpack.core.security.action.saml.SamlInvalidateSessionAction; import org.elasticsearch.xpack.core.security.action.saml.SamlLogoutAction; -import org.elasticsearch.xpack.core.security.action.saml.SamlCompleteLogoutAction; import org.elasticsearch.xpack.core.security.action.saml.SamlPrepareAuthenticationAction; import org.elasticsearch.xpack.core.security.action.saml.SamlSpMetadataAction; import org.elasticsearch.xpack.core.security.action.token.CreateTokenAction; @@ -173,9 +174,9 @@ import org.elasticsearch.xpack.security.action.rolemapping.TransportGetRoleMappingsAction; import org.elasticsearch.xpack.security.action.rolemapping.TransportPutRoleMappingAction; import org.elasticsearch.xpack.security.action.saml.TransportSamlAuthenticateAction; +import org.elasticsearch.xpack.security.action.saml.TransportSamlCompleteLogoutAction; import org.elasticsearch.xpack.security.action.saml.TransportSamlInvalidateSessionAction; import org.elasticsearch.xpack.security.action.saml.TransportSamlLogoutAction; -import org.elasticsearch.xpack.security.action.saml.TransportSamlCompleteLogoutAction; import org.elasticsearch.xpack.security.action.saml.TransportSamlPrepareAuthenticationAction; import org.elasticsearch.xpack.security.action.saml.TransportSamlSpMetadataAction; import org.elasticsearch.xpack.security.action.token.TransportCreateTokenAction; @@ -216,14 +217,14 @@ import org.elasticsearch.xpack.security.authz.store.NativePrivilegeStore; import org.elasticsearch.xpack.security.authz.store.NativeRolesStore; import org.elasticsearch.xpack.security.ingest.SetSecurityUserProcessor; +import org.elasticsearch.xpack.security.operator.FileOperatorUsersStore; import org.elasticsearch.xpack.security.operator.OperatorOnlyRegistry; import org.elasticsearch.xpack.security.operator.OperatorPrivileges; import org.elasticsearch.xpack.security.operator.OperatorPrivileges.OperatorPrivilegesService; -import org.elasticsearch.xpack.security.operator.FileOperatorUsersStore; import org.elasticsearch.xpack.security.rest.SecurityRestFilter; import org.elasticsearch.xpack.security.rest.action.RestAuthenticateAction; -import org.elasticsearch.xpack.security.rest.action.apikey.RestClearApiKeyCacheAction; import org.elasticsearch.xpack.security.rest.action.RestDelegatePkiAuthenticationAction; +import org.elasticsearch.xpack.security.rest.action.apikey.RestClearApiKeyCacheAction; import org.elasticsearch.xpack.security.rest.action.apikey.RestCreateApiKeyAction; import org.elasticsearch.xpack.security.rest.action.apikey.RestGetApiKeyAction; import org.elasticsearch.xpack.security.rest.action.apikey.RestGrantApiKeyAction; @@ -247,9 +248,9 @@ import org.elasticsearch.xpack.security.rest.action.rolemapping.RestGetRoleMappingsAction; import org.elasticsearch.xpack.security.rest.action.rolemapping.RestPutRoleMappingAction; import org.elasticsearch.xpack.security.rest.action.saml.RestSamlAuthenticateAction; +import org.elasticsearch.xpack.security.rest.action.saml.RestSamlCompleteLogoutAction; import org.elasticsearch.xpack.security.rest.action.saml.RestSamlInvalidateSessionAction; import org.elasticsearch.xpack.security.rest.action.saml.RestSamlLogoutAction; -import org.elasticsearch.xpack.security.rest.action.saml.RestSamlCompleteLogoutAction; import org.elasticsearch.xpack.security.rest.action.saml.RestSamlPrepareAuthenticationAction; import org.elasticsearch.xpack.security.rest.action.saml.RestSamlSpMetadataAction; import org.elasticsearch.xpack.security.rest.action.user.RestChangePasswordAction; @@ -259,8 +260,8 @@ import org.elasticsearch.xpack.security.rest.action.user.RestHasPrivilegesAction; import org.elasticsearch.xpack.security.rest.action.user.RestPutUserAction; import org.elasticsearch.xpack.security.rest.action.user.RestSetEnabledAction; -import org.elasticsearch.xpack.security.support.ExtensionComponents; import org.elasticsearch.xpack.security.support.CacheInvalidatorRegistry; +import org.elasticsearch.xpack.security.support.ExtensionComponents; import org.elasticsearch.xpack.security.support.SecurityIndexManager; import org.elasticsearch.xpack.security.support.SecurityStatusChangeListener; import org.elasticsearch.xpack.security.transport.SecurityHttpSettings; @@ -271,10 +272,11 @@ import org.elasticsearch.xpack.security.transport.nio.SecurityNioHttpServerTransport; import org.elasticsearch.xpack.security.transport.nio.SecurityNioTransport; +import java.io.IOException; +import java.io.UncheckedIOException; import java.nio.file.Path; import java.time.Clock; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.HashMap; @@ -290,16 +292,21 @@ import java.util.function.UnaryOperator; import java.util.stream.Collectors; +import static java.util.Arrays.asList; import static java.util.Collections.emptyList; import static java.util.Collections.emptyMap; import static java.util.Collections.singletonList; import static org.elasticsearch.cluster.metadata.IndexMetadata.INDEX_FORMAT_SETTING; +import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.elasticsearch.xpack.core.ClientHelper.SECURITY_ORIGIN; import static org.elasticsearch.xpack.core.XPackSettings.API_KEY_SERVICE_ENABLED_SETTING; import static org.elasticsearch.xpack.core.XPackSettings.HTTP_SSL_ENABLED; import static org.elasticsearch.xpack.core.security.index.RestrictedIndicesNames.SECURITY_MAIN_ALIAS; -import static org.elasticsearch.xpack.security.support.SecurityIndexManager.INTERNAL_MAIN_INDEX_FORMAT; +import static org.elasticsearch.xpack.core.security.index.RestrictedIndicesNames.SECURITY_TOKENS_ALIAS; import static org.elasticsearch.xpack.security.operator.OperatorPrivileges.OPERATOR_PRIVILEGES_ENABLED; -import static org.elasticsearch.xpack.security.support.SecurityIndexManager.SECURITY_MAIN_TEMPLATE_7; +import static org.elasticsearch.xpack.security.support.SecurityIndexManager.INTERNAL_MAIN_INDEX_FORMAT; +import static org.elasticsearch.xpack.security.support.SecurityIndexManager.INTERNAL_TOKENS_INDEX_FORMAT; +import static org.elasticsearch.xpack.security.support.SecurityIndexManager.SECURITY_VERSION_STRING; public class Security extends Plugin implements SystemIndexPlugin, IngestPlugin, NetworkPlugin, ClusterPlugin, DiscoveryPlugin, MapperPlugin, ExtensiblePlugin { @@ -308,6 +315,9 @@ public class Security extends Plugin implements SystemIndexPlugin, IngestPlugin, private static final Logger logger = LogManager.getLogger(Security.class); + public static final SystemIndexDescriptor SECURITY_MAIN_INDEX_DESCRIPTOR = getSecurityMainIndexDescriptor(); + public static final SystemIndexDescriptor SECURITY_TOKEN_INDEX_DESCRIPTOR = getSecurityTokenIndexDescriptor(); + private final Settings settings; private final boolean enabled; private final boolean transportClientMode; @@ -428,7 +438,7 @@ Collection createComponents(Client client, ThreadPool threadPool, Cluste // We need to construct the checks here while the secure settings are still available. // If we wait until #getBoostrapChecks the secure settings will have been cleared/closed. final List checks = new ArrayList<>(); - checks.addAll(Arrays.asList( + checks.addAll(asList( new ApiKeySSLBootstrapCheck(), new TokenSSLBootstrapCheck(), new PkiRealmBootstrapCheck(getSslService()), @@ -449,10 +459,18 @@ Collection createComponents(Client client, ThreadPool threadPool, Cluste components.add(auditTrailService); this.auditTrailService.set(auditTrailService); - securityIndex.set(SecurityIndexManager.buildSecurityMainIndexManager(client, clusterService)); - - final TokenService tokenService = new TokenService(settings, Clock.systemUTC(), client, getLicenseState(), securityContext.get(), - securityIndex.get(), SecurityIndexManager.buildSecurityTokensIndexManager(client, clusterService), clusterService); + securityIndex.set(SecurityIndexManager.buildSecurityIndexManager(client, clusterService, SECURITY_MAIN_INDEX_DESCRIPTOR)); + + final TokenService tokenService = new TokenService( + settings, + Clock.systemUTC(), + client, + getLicenseState(), + securityContext.get(), + securityIndex.get(), + SecurityIndexManager.buildSecurityIndexManager(client, clusterService, SECURITY_TOKEN_INDEX_DESCRIPTOR), + clusterService + ); this.tokenService.set(tokenService); components.add(tokenService); @@ -536,7 +554,7 @@ Collection createComponents(Client client, ThreadPool threadPool, Cluste new ResizeRequestInterceptor(threadPool, getLicenseState(), auditTrailService), new IndicesAliasesRequestInterceptor(threadPool.getThreadContext(), getLicenseState(), auditTrailService)); if (XPackSettings.DLS_FLS_ENABLED.get(settings)) { - requestInterceptors.addAll(Arrays.asList( + requestInterceptors.addAll(asList( new SearchRequestInterceptor(threadPool, getLicenseState()), new UpdateRequestInterceptor(threadPool, getLicenseState()), new BulkShardRequestInterceptor(threadPool, getLicenseState()) @@ -817,7 +835,7 @@ public void onIndexModule(IndexModule module) { if (enabled == false) { return emptyList(); } - return Arrays.asList( + return asList( new ActionHandler<>(ClearRealmCacheAction.INSTANCE, TransportClearRealmCacheAction.class), new ActionHandler<>(ClearRolesCacheAction.INSTANCE, TransportClearRolesCacheAction.class), new ActionHandler<>(ClearPrivilegesCacheAction.INSTANCE, TransportClearPrivilegesCacheAction.class), @@ -881,7 +899,7 @@ public List getRestHandlers(Settings settings, RestController restC if (enabled == false) { return emptyList(); } - return Arrays.asList( + return asList( new RestAuthenticateAction(settings, securityContext.get(), getLicenseState()), new RestClearRealmCacheAction(settings, getLicenseState()), new RestClearRolesCacheAction(settings, getLicenseState()), @@ -1091,7 +1109,6 @@ public List> getExecutorBuilders(final Settings settings) { public UnaryOperator> getIndexTemplateMetadataUpgrader() { return templates -> { // .security index is not managed by using templates anymore - templates.remove(SECURITY_MAIN_TEMPLATE_7); templates.remove("security_audit_log"); return templates; }; @@ -1209,15 +1226,599 @@ private synchronized SharedGroupFactory getNettySharedGroupFactory(Settings sett } } + private static SystemIndexDescriptor getSecurityMainIndexDescriptor() { + return SystemIndexDescriptor.builder() + // This can't just be `.security-*` because that would overlap with the tokens index pattern + .setIndexPattern(".security-[0-9]+") + .setPrimaryIndex(RestrictedIndicesNames.INTERNAL_SECURITY_MAIN_INDEX_7) + .setDescription("Contains Security configuration") + .setMappings(getIndexMappings()) + .setSettings(getIndexSettings()) + .setAliasName(SECURITY_MAIN_ALIAS) + .setIndexFormat(INTERNAL_MAIN_INDEX_FORMAT) + .setVersionMetaKey("security-version") + .setOrigin(SECURITY_ORIGIN) + .build(); + } + + private static SystemIndexDescriptor getSecurityTokenIndexDescriptor() { + return SystemIndexDescriptor.builder() + .setIndexPattern(".security-tokens-[0-9]+") + .setPrimaryIndex(RestrictedIndicesNames.INTERNAL_SECURITY_TOKENS_INDEX_7) + .setDescription("Contains auth token data") + .setMappings(getTokenIndexMappings()) + .setSettings(getTokenIndexSettings()) + .setAliasName(SECURITY_TOKENS_ALIAS) + .setIndexFormat(INTERNAL_TOKENS_INDEX_FORMAT) + .setVersionMetaKey(SECURITY_VERSION_STRING) + .setOrigin(SECURITY_ORIGIN) + .build(); + } + @Override public Collection getSystemIndexDescriptors(Settings settings) { - return Collections.unmodifiableList(Arrays.asList( - new SystemIndexDescriptor(SECURITY_MAIN_ALIAS, "Contains Security configuration"), - new SystemIndexDescriptor(RestrictedIndicesNames.INTERNAL_SECURITY_MAIN_INDEX_6, "Contains Security configuration"), - new SystemIndexDescriptor(RestrictedIndicesNames.INTERNAL_SECURITY_MAIN_INDEX_7, "Contains Security configuration"), + return Collections.unmodifiableList(asList(SECURITY_MAIN_INDEX_DESCRIPTOR, SECURITY_TOKEN_INDEX_DESCRIPTOR)); + } - new SystemIndexDescriptor(RestrictedIndicesNames.SECURITY_TOKENS_ALIAS, "Contains auth token data"), - new SystemIndexDescriptor(RestrictedIndicesNames.INTERNAL_SECURITY_TOKENS_INDEX_7, "Contains auth token data") - )); + private static Settings getIndexSettings() { + return Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .put(IndexMetadata.SETTING_AUTO_EXPAND_REPLICAS, "0-1") + .put(IndexMetadata.SETTING_PRIORITY, 1000) + .put("index.refresh_interval", "1s") + .put(IndexMetadata.INDEX_FORMAT_SETTING.getKey(), INTERNAL_MAIN_INDEX_FORMAT) + .put("analysis.filter.email.type", "pattern_capture") + .put("analysis.filter.email.preserve_original", true) + .putList("analysis.filter.email.patterns", asList("([^@]+)", "(\\p{L}+)", "(\\d+)", "@(.+)")) + .put("analysis.analyzer.email.tokenizer", "uax_url_email") + .putList("analysis.analyzer.email.filter", asList("email", "lowercase", "unique")) + .build(); } + + private static XContentBuilder getIndexMappings() { + try { + final XContentBuilder builder = jsonBuilder(); + builder.startObject(); + { + builder.startObject("_meta"); + builder.field(SECURITY_VERSION_STRING, Version.CURRENT.toString()); + builder.endObject(); + + builder.field("dynamic", "strict"); + builder.startObject("properties"); + { + builder.startObject("username"); + builder.field("type", "keyword"); + builder.endObject(); + + builder.startObject("roles"); + builder.field("type", "keyword"); + builder.endObject(); + + builder.startObject("role_templates"); + { + builder.startObject("properties"); + { + builder.startObject("template"); + builder.field("type", "text"); + builder.endObject(); + + builder.startObject("format"); + builder.field("type", "keyword"); + builder.endObject(); + } + builder.endObject(); + } + builder.endObject(); + + builder.startObject("password"); + builder.field("type", "keyword"); + builder.field("index", false); + builder.field("doc_values", false); + builder.endObject(); + + builder.startObject("full_name"); + builder.field("type", "text"); + builder.endObject(); + + builder.startObject("email"); + builder.field("type", "text"); + builder.field("analyzer", "email"); + builder.endObject(); + + builder.startObject("metadata"); + builder.field("type", "object"); + builder.field("dynamic", false); + builder.endObject(); + + builder.startObject("enabled"); + builder.field("type", "boolean"); + builder.endObject(); + + builder.startObject("cluster"); + builder.field("type", "keyword"); + builder.endObject(); + + builder.startObject("indices"); + { + builder.field("type", "object"); + builder.startObject("properties"); + { + builder.startObject("field_security"); + { + builder.startObject("properties"); + { + builder.startObject("grant"); + builder.field("type", "keyword"); + builder.endObject(); + + builder.startObject("except"); + builder.field("type", "keyword"); + builder.endObject(); + } + builder.endObject(); + } + builder.endObject(); + + builder.startObject("names"); + builder.field("type", "keyword"); + builder.endObject(); + + builder.startObject("privileges"); + builder.field("type", "keyword"); + builder.endObject(); + + builder.startObject("query"); + builder.field("type", "keyword"); + builder.endObject(); + + builder.startObject("allow_restricted_indices"); + builder.field("type", "boolean"); + builder.endObject(); + } + builder.endObject(); + } + builder.endObject(); + + builder.startObject("applications"); + { + builder.field("type", "object"); + builder.startObject("properties"); + { + builder.startObject("application"); + builder.field("type", "keyword"); + builder.endObject(); + + builder.startObject("privileges"); + builder.field("type", "keyword"); + builder.endObject(); + + builder.startObject("resources"); + builder.field("type", "keyword"); + builder.endObject(); + } + builder.endObject(); + } + builder.endObject(); + + builder.startObject("application"); + builder.field("type", "keyword"); + builder.endObject(); + + builder.startObject("global"); + { + builder.field("type", "object"); + builder.startObject("properties"); + { + builder.startObject("application"); + { + builder.field("type", "object"); + builder.startObject("properties"); + { + builder.startObject("manage"); + { + builder.field("type", "object"); + builder.startObject("properties"); + { + builder.startObject("applications"); + builder.field("type", "keyword"); + builder.endObject(); + } + builder.endObject(); + } + builder.endObject(); + } + builder.endObject(); + } + builder.endObject(); + } + builder.endObject(); + } + builder.endObject(); + + builder.startObject("name"); + builder.field("type", "keyword"); + builder.endObject(); + + builder.startObject("run_as"); + builder.field("type", "keyword"); + builder.endObject(); + + builder.startObject("doc_type"); + builder.field("type", "keyword"); + builder.endObject(); + + builder.startObject("type"); + builder.field("type", "keyword"); + builder.endObject(); + + builder.startObject("actions"); + builder.field("type", "keyword"); + builder.endObject(); + + builder.startObject("expiration_time"); + builder.field("type", "date"); + builder.field("format", "epoch_millis"); + builder.endObject(); + + builder.startObject("creation_time"); + builder.field("type", "date"); + builder.field("format", "epoch_millis"); + builder.endObject(); + + builder.startObject("api_key_hash"); + builder.field("type", "keyword"); + builder.field("index", false); + builder.field("doc_values", false); + builder.endObject(); + + builder.startObject("api_key_invalidated"); + builder.field("type", "boolean"); + builder.endObject(); + + builder.startObject("role_descriptors"); + builder.field("type", "object"); + builder.field("enabled", false); + builder.endObject(); + + builder.startObject("limited_by_role_descriptors"); + builder.field("type", "object"); + builder.field("enabled", false); + builder.endObject(); + + builder.startObject("version"); + builder.field("type", "integer"); + builder.endObject(); + + builder.startObject("creator"); + { + builder.field("type", "object"); + builder.startObject("properties"); + { + builder.startObject("principal"); + builder.field("type", "keyword"); + builder.endObject(); + + builder.startObject("full_name"); + builder.field("type", "text"); + builder.endObject(); + + builder.startObject("email"); + builder.field("type", "text"); + builder.field("analyzer", "email"); + builder.endObject(); + + builder.startObject("metadata"); + builder.field("type", "object"); + builder.field("dynamic", false); + builder.endObject(); + + builder.startObject("realm"); + builder.field("type", "keyword"); + builder.endObject(); + + builder.startObject("realm_type"); + builder.field("type", "keyword"); + builder.endObject(); + } + builder.endObject(); + } + builder.endObject(); + + builder.startObject("rules"); + builder.field("type", "object"); + builder.field("dynamic", false); + builder.endObject(); + + builder.startObject("refresh_token"); + { + builder.field("type", "object"); + builder.startObject("properties"); + { + builder.startObject("token"); + builder.field("type", "keyword"); + builder.endObject(); + + builder.startObject("refreshed"); + builder.field("type", "boolean"); + builder.endObject(); + + builder.startObject("refresh_time"); + builder.field("type", "date"); + builder.field("format", "epoch_millis"); + builder.endObject(); + + builder.startObject("superseding"); + { + builder.field("type", "object"); + builder.startObject("properties"); + { + builder.startObject("encrypted_tokens"); + builder.field("type", "binary"); + builder.endObject(); + + builder.startObject("encryption_iv"); + builder.field("type", "binary"); + builder.endObject(); + + builder.startObject("encryption_salt"); + builder.field("type", "binary"); + builder.endObject(); + } + builder.endObject(); + } + builder.endObject(); + + builder.startObject("invalidated"); + builder.field("type", "boolean"); + builder.endObject(); + + builder.startObject("client"); + { + builder.field("type", "object"); + builder.startObject("properties"); + { + builder.startObject("type"); + builder.field("type", "keyword"); + builder.endObject(); + + builder.startObject("user"); + builder.field("type", "keyword"); + builder.endObject(); + + builder.startObject("realm"); + builder.field("type", "keyword"); + builder.endObject(); + } + builder.endObject(); + } + builder.endObject(); + } + builder.endObject(); + } + builder.endObject(); + + builder.startObject("access_token"); + { + builder.field("type", "object"); + builder.startObject("properties"); + { + builder.startObject("user_token"); + { + builder.field("type", "object"); + builder.startObject("properties"); + { + builder.startObject("id"); + builder.field("type", "keyword"); + builder.endObject(); + + builder.startObject("expiration_time"); + builder.field("type", "date"); + builder.field("format", "epoch_millis"); + builder.endObject(); + + builder.startObject("version"); + builder.field("type", "integer"); + builder.endObject(); + + builder.startObject("metadata"); + builder.field("type", "object"); + builder.field("dynamic", false); + builder.endObject(); + + builder.startObject("authentication"); + builder.field("type", "binary"); + builder.endObject(); + } + builder.endObject(); + } + builder.endObject(); + + builder.startObject("invalidated"); + builder.field("type", "boolean"); + builder.endObject(); + + builder.startObject("realm"); + builder.field("type", "keyword"); + builder.endObject(); + } + builder.endObject(); + } + builder.endObject(); + } + builder.endObject(); + } + builder.endObject(); + + return builder; + } catch (IOException e) { + logger.fatal("Failed to build " + RestrictedIndicesNames.INTERNAL_SECURITY_MAIN_INDEX_7 + " index mappings", e); + throw new UncheckedIOException( + "Failed to build " + RestrictedIndicesNames.INTERNAL_SECURITY_MAIN_INDEX_7 + " index mappings", e); + } + } + + private static Settings getTokenIndexSettings() { + return Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .put(IndexMetadata.SETTING_AUTO_EXPAND_REPLICAS, "0-1") + .put(IndexMetadata.SETTING_PRIORITY, 1000) + .put("index.refresh_interval", "1s") + .put(IndexMetadata.INDEX_FORMAT_SETTING.getKey(), INTERNAL_TOKENS_INDEX_FORMAT) + .build(); + } + + + private static XContentBuilder getTokenIndexMappings() { + try { + final XContentBuilder builder = jsonBuilder(); + + builder.startObject(); + { + builder.startObject("_meta"); + builder.field(SECURITY_VERSION_STRING, Version.CURRENT); + builder.endObject(); + + builder.field("dynamic", "strict"); + builder.startObject("properties"); + { + builder.startObject("doc_type"); + builder.field("type", "keyword"); + builder.endObject(); + + builder.startObject("creation_time"); + builder.field("type", "date"); + builder.field("format", "epoch_millis"); + builder.endObject(); + + builder.startObject("refresh_token"); + { + builder.field("type", "object"); + builder.startObject("properties"); + { + builder.startObject("token"); + builder.field("type", "keyword"); + builder.endObject(); + + builder.startObject("refreshed"); + builder.field("type", "boolean"); + builder.endObject(); + + builder.startObject("refresh_time"); + builder.field("type", "date"); + builder.field("format", "epoch_millis"); + builder.endObject(); + + builder.startObject("superseding"); + { + builder.field("type", "object"); + builder.startObject("properties"); + { + builder.startObject("encrypted_tokens"); + builder.field("type", "binary"); + builder.endObject(); + + builder.startObject("encryption_iv"); + builder.field("type", "binary"); + builder.endObject(); + + builder.startObject("encryption_salt"); + builder.field("type", "binary"); + builder.endObject(); + } + builder.endObject(); + } + builder.endObject(); + + builder.startObject("invalidated"); + builder.field("type", "boolean"); + builder.endObject(); + + builder.startObject("client"); + { + builder.field("type", "object"); + builder.startObject("properties"); + { + builder.startObject("type"); + builder.field("type", "keyword"); + builder.endObject(); + + builder.startObject("user"); + builder.field("type", "keyword"); + builder.endObject(); + + builder.startObject("realm"); + builder.field("type", "keyword"); + builder.endObject(); + } + builder.endObject(); + } + builder.endObject(); + } + builder.endObject(); + } + builder.endObject(); + + builder.startObject("access_token"); + { + builder.field("type", "object"); + builder.startObject("properties"); + { + builder.startObject("user_token"); + { + builder.field("type", "object"); + builder.startObject("properties"); + { + builder.startObject("id"); + builder.field("type", "keyword"); + builder.endObject(); + + builder.startObject("expiration_time"); + builder.field("type", "date"); + builder.field("format", "epoch_millis"); + builder.endObject(); + + builder.startObject("version"); + builder.field("type", "integer"); + builder.endObject(); + + builder.startObject("metadata"); + builder.field("type", "object"); + builder.field("dynamic", false); + builder.endObject(); + + builder.startObject("authentication"); + builder.field("type", "binary"); + builder.endObject(); + } + builder.endObject(); + } + builder.endObject(); + + builder.startObject("invalidated"); + builder.field("type", "boolean"); + builder.endObject(); + + builder.startObject("realm"); + builder.field("type", "keyword"); + builder.endObject(); + } + builder.endObject(); + } + builder.endObject(); + } + builder.endObject(); + } + + builder.endObject(); + return builder; + } catch (IOException e) { + throw new UncheckedIOException( + "Failed to build " + RestrictedIndicesNames.INTERNAL_SECURITY_TOKENS_INDEX_7 + " index mappings", e); + } + } + } + diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java index d35afb76c2410..6a2fdf1f82b3e 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java @@ -21,7 +21,6 @@ import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest; -import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest; import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.client.Client; @@ -36,26 +35,15 @@ import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.routing.IndexRoutingTable; import org.elasticsearch.cluster.service.ClusterService; -import org.elasticsearch.common.Strings; -import org.elasticsearch.common.collect.Tuple; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.xcontent.DeprecationHandler; -import org.elasticsearch.common.xcontent.NamedXContentRegistry; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; -import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.gateway.GatewayService; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.indices.IndexClosedException; +import org.elasticsearch.indices.SystemIndexDescriptor; import org.elasticsearch.rest.RestStatus; -import org.elasticsearch.xpack.core.security.index.RestrictedIndicesNames; -import org.elasticsearch.xpack.core.template.TemplateUtils; -import java.io.IOException; -import java.nio.charset.StandardCharsets; import java.time.Instant; import java.util.HashSet; import java.util.List; @@ -66,13 +54,9 @@ import java.util.function.BiConsumer; import java.util.function.Consumer; import java.util.function.Predicate; -import java.util.function.Supplier; import java.util.stream.Collectors; import static org.elasticsearch.cluster.metadata.IndexMetadata.INDEX_FORMAT_SETTING; -import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken; -import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureFieldName; -import static org.elasticsearch.xpack.core.ClientHelper.SECURITY_ORIGIN; import static org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin; /** @@ -82,54 +66,36 @@ public class SecurityIndexManager implements ClusterStateListener { public static final int INTERNAL_MAIN_INDEX_FORMAT = 6; public static final int INTERNAL_TOKENS_INDEX_FORMAT = 7; - public static final String SECURITY_MAIN_TEMPLATE_7 = "security-index-template-7"; - public static final String SECURITY_TOKENS_TEMPLATE_7 = "security-tokens-index-template-7"; public static final String SECURITY_VERSION_STRING = "security-version"; public static final String TEMPLATE_VERSION_VARIABLE = "security.template.version"; private static final Logger logger = LogManager.getLogger(SecurityIndexManager.class); - private final String aliasName; - private final String internalIndexName; - private final int internalIndexFormat; - private final Supplier mappingSourceSupplier; private final Client client; + private final SystemIndexDescriptor systemIndexDescriptor; private final List> stateChangeListeners = new CopyOnWriteArrayList<>(); private volatile State indexState; - public static SecurityIndexManager buildSecurityMainIndexManager(Client client, ClusterService clusterService) { - return new SecurityIndexManager(client, clusterService, RestrictedIndicesNames.SECURITY_MAIN_ALIAS, - RestrictedIndicesNames.INTERNAL_SECURITY_MAIN_INDEX_7, INTERNAL_MAIN_INDEX_FORMAT, - () -> SecurityIndexManager.readTemplateAsBytes(SECURITY_MAIN_TEMPLATE_7)); + public static SecurityIndexManager buildSecurityIndexManager( + Client client, + ClusterService clusterService, + SystemIndexDescriptor descriptor + ) { + final SecurityIndexManager securityIndexManager = new SecurityIndexManager(client, descriptor, State.UNRECOVERED_STATE); + clusterService.addListener(securityIndexManager); + return securityIndexManager; } - public static SecurityIndexManager buildSecurityTokensIndexManager(Client client, ClusterService clusterService) { - return new SecurityIndexManager(client, clusterService, RestrictedIndicesNames.SECURITY_TOKENS_ALIAS, - RestrictedIndicesNames.INTERNAL_SECURITY_TOKENS_INDEX_7, INTERNAL_TOKENS_INDEX_FORMAT, - () -> SecurityIndexManager.readTemplateAsBytes(SECURITY_TOKENS_TEMPLATE_7)); - } - - private SecurityIndexManager(Client client, ClusterService clusterService, String aliasName, String internalIndexName, - int internalIndexFormat, Supplier mappingSourceSupplier) { - this(client, aliasName, internalIndexName, internalIndexFormat, mappingSourceSupplier, State.UNRECOVERED_STATE); - clusterService.addListener(this); - } - - // protected for testing - protected SecurityIndexManager(Client client, String aliasName, String internalIndexName, int internalIndexFormat, - Supplier mappingSourceSupplier, State indexState) { - this.aliasName = aliasName; - this.internalIndexName = internalIndexName; - this.internalIndexFormat = internalIndexFormat; - this.mappingSourceSupplier = mappingSourceSupplier; - this.indexState = indexState; + private SecurityIndexManager(Client client, SystemIndexDescriptor descriptor, State indexState) { this.client = client; + this.indexState = indexState; + this.systemIndexDescriptor = descriptor; } public SecurityIndexManager freeze() { - return new SecurityIndexManager(null, aliasName, internalIndexName, internalIndexFormat, mappingSourceSupplier, indexState); + return new SecurityIndexManager(null, systemIndexDescriptor, indexState); } public boolean checkMappingVersion(Predicate requiredVersion) { @@ -139,7 +105,7 @@ public boolean checkMappingVersion(Predicate requiredVersion) { } public String aliasName() { - return aliasName; + return systemIndexDescriptor.getAliasName(); } public boolean indexExists() { @@ -204,14 +170,16 @@ public void clusterChanged(ClusterChangedEvent event) { return; } final State previousState = indexState; - final IndexMetadata indexMetadata = resolveConcreteIndex(aliasName, event.state().metadata()); + final IndexMetadata indexMetadata = resolveConcreteIndex(systemIndexDescriptor.getAliasName(), event.state().metadata()); final Instant creationTime = indexMetadata != null ? Instant.ofEpochMilli(indexMetadata.getCreationDate()) : null; final boolean isIndexUpToDate = indexMetadata == null || - INDEX_FORMAT_SETTING.get(indexMetadata.getSettings()).intValue() == internalIndexFormat; + INDEX_FORMAT_SETTING.get(indexMetadata.getSettings()) == systemIndexDescriptor.getIndexFormat(); final boolean indexAvailable = checkIndexAvailable(event.state()); final boolean mappingIsUpToDate = indexMetadata == null || checkIndexMappingUpToDate(event.state()); final Version mappingVersion = oldestIndexMappingVersion(event.state()); - final String concreteIndexName = indexMetadata == null ? internalIndexName : indexMetadata.getIndex().getName(); + final String concreteIndexName = indexMetadata == null + ? systemIndexDescriptor.getPrimaryIndex() + : indexMetadata.getIndex().getName(); final ClusterHealthStatus indexHealth; final IndexMetadata.State indexState; if (indexMetadata == null) { @@ -228,7 +196,7 @@ public void clusterChanged(ClusterChangedEvent event) { indexHealth = new ClusterIndexHealth(indexMetadata, routingTable).getStatus(); } final State newState = new State(creationTime, isIndexUpToDate, indexAvailable, mappingIsUpToDate, mappingVersion, - concreteIndexName, indexHealth, indexState); + concreteIndexName, indexHealth, indexState, event.state().nodes().getMinNodeVersion()); this.indexState = newState; if (newState.equals(previousState) == false) { @@ -239,6 +207,7 @@ public void clusterChanged(ClusterChangedEvent event) { } private boolean checkIndexAvailable(ClusterState state) { + final String aliasName = systemIndexDescriptor.getAliasName(); IndexMetadata metadata = resolveConcreteIndex(aliasName, state.metadata()); if (metadata == null) { logger.debug("Index [{}] is not available - no metadata", aliasName); @@ -257,17 +226,19 @@ private boolean checkIndexAvailable(ClusterState state) { } } - public static boolean checkTemplateExistsAndVersionMatches(String templateName, ClusterState state, Logger logger, - Predicate predicate) { - return TemplateUtils.checkTemplateExistsAndVersionMatches(templateName, SECURITY_VERSION_STRING, state, logger, predicate); - } - private boolean checkIndexMappingUpToDate(ClusterState clusterState) { - return checkIndexMappingVersionMatches(clusterState, Version.CURRENT::equals); + /* + * The method reference looks wrong here, but it's just counter-intuitive. It expands to: + * + * mappingVersion -> Version.CURRENT.onOrBefore(mappingVersion) + * + * ...which is true if the mappings have been updated. + */ + return checkIndexMappingVersionMatches(clusterState, Version.CURRENT::onOrBefore); } private boolean checkIndexMappingVersionMatches(ClusterState clusterState, Predicate predicate) { - return checkIndexMappingVersionMatches(aliasName, clusterState, logger, predicate); + return checkIndexMappingVersionMatches(this.systemIndexDescriptor.getAliasName(), clusterState, logger, predicate); } public static boolean checkIndexMappingVersionMatches(String indexName, ClusterState clusterState, Logger logger, @@ -276,7 +247,7 @@ public static boolean checkIndexMappingVersionMatches(String indexName, ClusterS } private Version oldestIndexMappingVersion(ClusterState clusterState) { - final Set versions = loadIndexMappingVersions(aliasName, clusterState, logger); + final Set versions = loadIndexMappingVersions(systemIndexDescriptor.getAliasName(), clusterState, logger); return versions.stream().min(Version::compareTo).orElse(null); } @@ -347,8 +318,9 @@ public void checkIndexVersionThenExecute(final Consumer consumer, fin } /** - * Prepares the index by creating it if it doesn't exist or updating the mappings if the mappings are - * out of date. After any tasks have been executed, the runnable is then executed. + * Prepares the index by creating it if it doesn't exist, then executes the runnable. + * @param consumer a handler for any exceptions that are raised either during preparation or execution + * @param andThen executed if the index exists or after preparation is performed successfully */ public void prepareIndexIfNeededThenExecute(final Consumer consumer, final Runnable andThen) { final State indexState = this.indexState; // use a local copy so all checks execute against the same state! @@ -363,45 +335,59 @@ public void prepareIndexIfNeededThenExecute(final Consumer consumer, + "Security features relying on the index will not be available until the upgrade API is run on the index"); } else if (indexState.indexExists() == false) { assert indexState.concreteIndexName != null; - logger.info("security index does not exist. Creating [{}] with alias [{}]", indexState.concreteIndexName, this.aliasName); - final byte[] mappingSource = mappingSourceSupplier.get(); - final Tuple mappingAndSettings = parseMappingAndSettingsFromTemplateBytes(mappingSource); + logger.info( + "security index does not exist, creating [{}] with alias [{}]", + indexState.concreteIndexName, + systemIndexDescriptor.getAliasName() + ); + + // Although `TransportCreateIndexAction` is capable of automatically applying the right mappings, settings and aliases for + // system indices, we nonetheless specify them here so that the values from `this.systemIndexDescriptor` are used. CreateIndexRequest request = new CreateIndexRequest(indexState.concreteIndexName) - .alias(new Alias(this.aliasName)) - .mapping(MapperService.SINGLE_MAPPING_NAME, mappingAndSettings.v1(), XContentType.JSON) - .waitForActiveShards(ActiveShardCount.ALL) - .settings(mappingAndSettings.v2()); - executeAsyncWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN, request, - new ActionListener() { - @Override - public void onResponse(CreateIndexResponse createIndexResponse) { - if (createIndexResponse.isAcknowledged()) { - andThen.run(); - } else { - consumer.accept(new ElasticsearchException("Failed to create security index")); + .origin(systemIndexDescriptor.getOrigin()) + .mapping(MapperService.SINGLE_MAPPING_NAME, systemIndexDescriptor.getMappings(), XContentType.JSON) + .settings(systemIndexDescriptor.getSettings()) + .alias(new Alias(systemIndexDescriptor.getAliasName())) + .waitForActiveShards(ActiveShardCount.ALL); + + executeAsyncWithOrigin(client.threadPool().getThreadContext(), systemIndexDescriptor.getOrigin(), request, + new ActionListener() { + @Override + public void onResponse(CreateIndexResponse createIndexResponse) { + if (createIndexResponse.isAcknowledged()) { + andThen.run(); + } else { + consumer.accept(new ElasticsearchException("Failed to create security index")); + } } - } - - @Override - public void onFailure(Exception e) { - final Throwable cause = ExceptionsHelper.unwrapCause(e); - if (cause instanceof ResourceAlreadyExistsException) { - // the index already exists - it was probably just created so this - // node hasn't yet received the cluster state update with the index - andThen.run(); - } else { - consumer.accept(e); + + @Override + public void onFailure(Exception e) { + final Throwable cause = ExceptionsHelper.unwrapCause(e); + if (cause instanceof ResourceAlreadyExistsException) { + // the index already exists - it was probably just created so this + // node hasn't yet received the cluster state update with the index + andThen.run(); + } else { + consumer.accept(e); + } } - } - }, client.admin().indices()::create); + }, client.admin().indices()::create); } else if (indexState.mappingUpToDate == false) { - logger.info("Index [{}] (alias [{}]) is not up to date. Updating mapping", indexState.concreteIndexName, this.aliasName); - final byte[] mappingSource = mappingSourceSupplier.get(); - final Tuple mappingAndSettings = parseMappingAndSettingsFromTemplateBytes(mappingSource); - PutMappingRequest request = new PutMappingRequest(indexState.concreteIndexName) - .source(mappingAndSettings.v1(), XContentType.JSON) - .type(MapperService.SINGLE_MAPPING_NAME); - executeAsyncWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN, request, + final String error = systemIndexDescriptor.checkMinimumNodeVersion("create index", indexState.minimumNodeVersion); + if (error != null) { + consumer.accept(new IllegalStateException(error)); + } else { + logger.info( + "Index [{}] (alias [{}]) is not up to date. Updating mapping", + indexState.concreteIndexName, + systemIndexDescriptor.getAliasName() + ); + PutMappingRequest request = new PutMappingRequest(indexState.concreteIndexName).source( + systemIndexDescriptor.getMappings(), + XContentType.JSON + ).type(MapperService.SINGLE_MAPPING_NAME).origin(systemIndexDescriptor.getOrigin()); + executeAsyncWithOrigin(client.threadPool().getThreadContext(), systemIndexDescriptor.getOrigin(), request, ActionListener.wrap(putMappingResponse -> { if (putMappingResponse.isAcknowledged()) { andThen.run(); @@ -409,6 +395,7 @@ public void onFailure(Exception e) { consumer.accept(new IllegalStateException("put mapping request was not acknowledged")); } }, consumer), client.admin().indices()::putMapping); + } } else { andThen.run(); } @@ -432,32 +419,11 @@ public static boolean isIndexDeleted(State previousState, State currentState) { return previousState.indexHealth != null && currentState.indexHealth == null; } - private static byte[] readTemplateAsBytes(String templateName) { - return TemplateUtils.loadTemplate("/" + templateName + ".json", Version.CURRENT.toString(), - SecurityIndexManager.TEMPLATE_VERSION_VARIABLE).getBytes(StandardCharsets.UTF_8); - } - - private static Tuple parseMappingAndSettingsFromTemplateBytes(byte[] template) throws IOException { - final PutIndexTemplateRequest request = new PutIndexTemplateRequest("name_is_not_important").source(template, XContentType.JSON); - final String mappingSource = request.mappings().get(MapperService.SINGLE_MAPPING_NAME); - try (XContentParser parser = XContentType.JSON.xContent().createParser(NamedXContentRegistry.EMPTY, - DeprecationHandler.THROW_UNSUPPORTED_OPERATION, mappingSource)) { - // remove the type wrapping to get the mapping - ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser); // { - ensureFieldName(parser, parser.nextToken(), MapperService.SINGLE_MAPPING_NAME); // _doc - ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser); // { - - XContentBuilder builder = JsonXContent.contentBuilder(); - builder.generator().copyCurrentStructure(parser); - return new Tuple<>(Strings.toString(builder), request.settings()); - } - } - /** * State of the security index. */ public static class State { - public static final State UNRECOVERED_STATE = new State(null, false, false, false, null, null, null, null); + public static final State UNRECOVERED_STATE = new State(null, false, false, false, null, null, null, null, null); public final Instant creationTime; public final boolean isIndexUpToDate; public final boolean indexAvailable; @@ -466,10 +432,11 @@ public static class State { public final String concreteIndexName; public final ClusterHealthStatus indexHealth; public final IndexMetadata.State indexState; + public final Version minimumNodeVersion; public State(Instant creationTime, boolean isIndexUpToDate, boolean indexAvailable, boolean mappingUpToDate, Version mappingVersion, String concreteIndexName, ClusterHealthStatus indexHealth, - IndexMetadata.State indexState) { + IndexMetadata.State indexState, Version minimumNodeVersion) { this.creationTime = creationTime; this.isIndexUpToDate = isIndexUpToDate; this.indexAvailable = indexAvailable; @@ -478,6 +445,7 @@ public State(Instant creationTime, boolean isIndexUpToDate, boolean indexAvailab this.concreteIndexName = concreteIndexName; this.indexHealth = indexHealth; this.indexState = indexState; + this.minimumNodeVersion = minimumNodeVersion; } @Override @@ -492,7 +460,8 @@ public boolean equals(Object o) { Objects.equals(mappingVersion, state.mappingVersion) && Objects.equals(concreteIndexName, state.concreteIndexName) && indexHealth == state.indexHealth && - indexState == state.indexState; + indexState == state.indexState && + Objects.equals(minimumNodeVersion, state.minimumNodeVersion); } public boolean indexExists() { @@ -502,7 +471,7 @@ public boolean indexExists() { @Override public int hashCode() { return Objects.hash(creationTime, isIndexUpToDate, indexAvailable, mappingUpToDate, mappingVersion, concreteIndexName, - indexHealth); + indexHealth, minimumNodeVersion); } } } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SecurityIntegTestCase.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SecurityIntegTestCase.java index 152597fff92c3..70d5998c9ce7c 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SecurityIntegTestCase.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SecurityIntegTestCase.java @@ -8,8 +8,6 @@ import io.netty.util.ThreadDeathWatcher; import io.netty.util.concurrent.GlobalEventExecutor; - -import org.elasticsearch.Version; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; import org.elasticsearch.action.admin.cluster.node.info.NodeInfo; import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; @@ -27,14 +25,12 @@ import org.elasticsearch.cluster.metadata.IndexAbstraction; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.routing.IndexRoutingTable; -import org.elasticsearch.common.Strings; import org.elasticsearch.common.network.NetworkAddress; import org.elasticsearch.common.network.NetworkModule; import org.elasticsearch.common.settings.MockSecureSettings; import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.TransportAddress; -import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.gateway.GatewayService; @@ -49,7 +45,6 @@ import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken; import org.elasticsearch.xpack.core.security.client.SecurityClient; import org.elasticsearch.xpack.security.LocalStateSecurity; -import org.elasticsearch.xpack.security.support.SecurityIndexManager; import org.junit.AfterClass; import org.junit.Before; import org.junit.BeforeClass; @@ -497,10 +492,6 @@ public void assertSecurityIndexActive(TestCluster testCluster) throws Exception ClusterState clusterState = client.admin().cluster().prepareState().setLocal(true).get().getState(); assertFalse(clusterState.blocks().hasGlobalBlock(GatewayService.STATE_NOT_RECOVERED_BLOCK)); XContentBuilder builder = JsonXContent.contentBuilder().prettyPrint().startObject(); - assertTrue("security index mapping not sufficient to read:\n" + - Strings.toString(clusterState.toXContent(builder, ToXContent.EMPTY_PARAMS).endObject()), - SecurityIndexManager.checkIndexMappingVersionMatches(SECURITY_MAIN_ALIAS, clusterState, logger, - Version.CURRENT.minimumIndexCompatibilityVersion()::onOrBefore)); Index securityIndex = resolveSecurityIndex(clusterState.metadata()); if (securityIndex != null) { IndexRoutingTable indexRoutingTable = clusterState.routingTable().index(securityIndex); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java index 4ab07ccf16fde..4cc07b8640570 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java @@ -2030,6 +2030,6 @@ private void setCompletedToTrue(AtomicBoolean completed) { private SecurityIndexManager.State dummyState(ClusterHealthStatus indexStatus) { return new SecurityIndexManager.State( - Instant.now(), true, true, true, null, concreteSecurityIndexName, indexStatus, IndexMetadata.State.OPEN); + Instant.now(), true, true, true, null, concreteSecurityIndexName, indexStatus, IndexMetadata.State.OPEN, null); } } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealmTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealmTests.java index 87a3a3c3b057d..054159a0b0cd8 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealmTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealmTests.java @@ -30,7 +30,7 @@ public class NativeRealmTests extends ESTestCase { private SecurityIndexManager.State dummyState(ClusterHealthStatus indexStatus) { return new SecurityIndexManager.State( - Instant.now(), true, true, true, null, concreteSecurityIndexName, indexStatus, IndexMetadata.State.OPEN); + Instant.now(), true, true, true, null, concreteSecurityIndexName, indexStatus, IndexMetadata.State.OPEN, null); } public void testCacheClearOnIndexHealthChange() { diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStoreTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStoreTests.java index 59f6078458cf7..752b17726e19f 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStoreTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStoreTests.java @@ -146,7 +146,7 @@ private SecurityIndexManager.State dummyState(ClusterHealthStatus indexStatus) { private SecurityIndexManager.State indexState(boolean isUpToDate, ClusterHealthStatus healthStatus) { return new SecurityIndexManager.State( - Instant.now(), isUpToDate, true, true, null, concreteSecurityIndexName, healthStatus, IndexMetadata.State.OPEN); + Instant.now(), isUpToDate, true, true, null, concreteSecurityIndexName, healthStatus, IndexMetadata.State.OPEN, null); } public void testCacheClearOnIndexHealthChange() { diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStoreTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStoreTests.java index 19eabf4fd3a68..1d506b82b333c 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStoreTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStoreTests.java @@ -810,7 +810,7 @@ private SecurityIndexManager.State dummyState(ClusterHealthStatus indexStatus) { public SecurityIndexManager.State dummyIndexState(boolean isIndexUpToDate, ClusterHealthStatus healthStatus) { return new SecurityIndexManager.State( - Instant.now(), isIndexUpToDate, true, true, null, concreteSecurityIndexName, healthStatus, IndexMetadata.State.OPEN); + Instant.now(), isIndexUpToDate, true, true, null, concreteSecurityIndexName, healthStatus, IndexMetadata.State.OPEN, null); } public void testCacheClearOnIndexHealthChange() { diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStoreTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStoreTests.java index 1052e0f05566e..ad8d0838d44b6 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStoreTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStoreTests.java @@ -583,7 +583,7 @@ public void testCacheClearOnIndexHealthChange() { CollectionUtils.appendToCopy(Arrays.asList(ClusterHealthStatus.values()), null); store.onSecurityIndexStateChange( dummyState(securityIndexName, isIndexUpToDate, randomFrom(allPossibleHealthStatus)), - dummyState(securityIndexName, !isIndexUpToDate, randomFrom(allPossibleHealthStatus))); + dummyState(securityIndexName, isIndexUpToDate == false, randomFrom(allPossibleHealthStatus))); assertEquals(++count, store.getNumInvalidation().get()); } @@ -617,7 +617,7 @@ private SecurityIndexManager.State dummyState( String concreteSecurityIndexName, boolean isIndexUpToDate, ClusterHealthStatus healthStatus) { return new SecurityIndexManager.State( Instant.now(), isIndexUpToDate, true, true, null, - concreteSecurityIndexName, healthStatus, IndexMetadata.State.OPEN + concreteSecurityIndexName, healthStatus, IndexMetadata.State.OPEN, null ); } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStoreTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStoreTests.java index b88b4f359a6de..8513867afb899 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStoreTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStoreTests.java @@ -15,7 +15,6 @@ import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexMetadata; -import org.elasticsearch.cluster.metadata.IndexTemplateMetadata; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.routing.IndexRoutingTable; import org.elasticsearch.cluster.routing.IndexShardRoutingTable; @@ -27,7 +26,6 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -45,6 +43,7 @@ import org.elasticsearch.xpack.core.security.action.role.PutRoleRequest; import org.elasticsearch.xpack.core.security.authz.RoleDescriptor; import org.elasticsearch.xpack.core.security.authz.RoleDescriptor.IndicesPrivileges; +import org.elasticsearch.xpack.security.Security; import org.elasticsearch.xpack.security.support.SecurityIndexManager; import org.elasticsearch.xpack.security.test.SecurityTestUtils; import org.junit.After; @@ -54,7 +53,6 @@ import java.nio.charset.Charset; import java.nio.file.Files; import java.nio.file.Path; -import java.util.Collections; import java.util.List; import java.util.UUID; import java.util.concurrent.atomic.AtomicBoolean; @@ -191,7 +189,11 @@ public void testPutOfRoleWithFlsDlsUnlicensed() throws IOException { final ClusterService clusterService = mock(ClusterService.class); final XPackLicenseState licenseState = mock(XPackLicenseState.class); final AtomicBoolean methodCalled = new AtomicBoolean(false); - final SecurityIndexManager securityIndex = SecurityIndexManager.buildSecurityMainIndexManager(client, clusterService); + final SecurityIndexManager securityIndex = SecurityIndexManager.buildSecurityIndexManager( + client, + clusterService, + Security.SECURITY_MAIN_INDEX_DESCRIPTOR + ); final NativeRolesStore rolesStore = new NativeRolesStore(Settings.EMPTY, client, licenseState, securityIndex) { @Override void innerPutRole(final PutRoleRequest request, final RoleDescriptor role, final ActionListener listener) { @@ -260,9 +262,6 @@ private ClusterState getClusterStateWithSecurityIndex() { .build(); Metadata metadata = Metadata.builder() .put(IndexMetadata.builder(securityIndexName).settings(settings)) - .put(new IndexTemplateMetadata(SecurityIndexManager.SECURITY_MAIN_TEMPLATE_7, 0, 0, - Collections.singletonList(securityIndexName), Settings.EMPTY, ImmutableOpenMap.of(), - ImmutableOpenMap.of())) .build(); if (withAlias) { diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/CacheInvalidatorRegistryTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/CacheInvalidatorRegistryTests.java index c0fc721200227..6267f61bb57da 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/CacheInvalidatorRegistryTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/CacheInvalidatorRegistryTests.java @@ -48,7 +48,7 @@ public void testSecurityIndexStateChangeWillInvalidateAllRegisteredInvalidators( final SecurityIndexManager.State previousState = SecurityIndexManager.State.UNRECOVERED_STATE; final SecurityIndexManager.State currentState = new SecurityIndexManager.State( Instant.now(), true, true, true, Version.CURRENT, - ".security", ClusterHealthStatus.GREEN, IndexMetadata.State.OPEN); + ".security", ClusterHealthStatus.GREEN, IndexMetadata.State.OPEN, null); cacheInvalidatorRegistry.onSecurityIndexStageChange(previousState, currentState); verify(invalidator1).invalidateAll(); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/SecurityIndexManagerTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/SecurityIndexManagerTests.java index 9abfcdce3828e..aa714011e381c 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/SecurityIndexManagerTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/SecurityIndexManagerTests.java @@ -6,27 +6,15 @@ */ package org.elasticsearch.xpack.security.support; -import java.io.IOException; -import java.time.Instant; -import java.util.Arrays; -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.Map; -import java.util.UUID; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicReference; -import java.util.function.BiConsumer; -import java.util.function.Supplier; - import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.ActionType; -import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest; +import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest; +import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.client.Client; -import org.elasticsearch.client.FilterClient; import org.elasticsearch.cluster.ClusterChangedEvent; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; @@ -34,7 +22,7 @@ import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.cluster.metadata.AliasMetadata; import org.elasticsearch.cluster.metadata.IndexMetadata; -import org.elasticsearch.cluster.metadata.IndexTemplateMetadata; +import org.elasticsearch.cluster.metadata.MappingMetadata; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.routing.IndexRoutingTable; @@ -44,88 +32,87 @@ import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.routing.UnassignedInfo; import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.common.util.concurrent.ThreadContext; -import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.gateway.GatewayService; import org.elasticsearch.index.Index; +import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.indices.SystemIndexDescriptor; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.client.NoOpClient; import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.xpack.security.test.SecurityTestUtils; import org.elasticsearch.xpack.core.security.index.RestrictedIndicesNames; -import org.elasticsearch.xpack.core.template.TemplateUtils; +import org.elasticsearch.xpack.security.Security; +import org.elasticsearch.xpack.security.test.SecurityTestUtils; import org.hamcrest.Matchers; import org.junit.Before; -import static org.elasticsearch.xpack.security.support.SecurityIndexManager.SECURITY_MAIN_TEMPLATE_7; -import static org.elasticsearch.xpack.security.support.SecurityIndexManager.TEMPLATE_VERSION_VARIABLE; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.UUID; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.BiConsumer; + +import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyString; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.spy; import static org.mockito.Mockito.when; public class SecurityIndexManagerTests extends ESTestCase { private static final ClusterName CLUSTER_NAME = new ClusterName("security-index-manager-tests"); private static final ClusterState EMPTY_CLUSTER_STATE = new ClusterState.Builder(CLUSTER_NAME).build(); - private static final String TEMPLATE_NAME = "SecurityIndexManagerTests-template"; private SecurityIndexManager manager; - private Map, Map>> actions; + private SystemIndexDescriptor descriptorSpy; @Before public void setUpManager() { - final Client mockClient = mock(Client.class); final ThreadPool threadPool = mock(ThreadPool.class); when(threadPool.getThreadContext()).thenReturn(new ThreadContext(Settings.EMPTY)); when(threadPool.generic()).thenReturn(EsExecutors.newDirectExecutorService()); - when(mockClient.threadPool()).thenReturn(threadPool); - when(mockClient.settings()).thenReturn(Settings.EMPTY); - final ClusterService clusterService = mock(ClusterService.class); - actions = new LinkedHashMap<>(); - final Client client = new FilterClient(mockClient) { + // Build a mock client that always accepts put mappings requests + final Client client = new NoOpClient(threadPool) { @Override - protected - void doExecute(ActionType action, Request request, ActionListener listener) { - final Map> map = actions.getOrDefault(action, new HashMap<>()); - map.put(request, listener); - actions.put(action, map); + @SuppressWarnings("unchecked") + protected void doExecute( + ActionType action, + Request request, + ActionListener listener + ) { + if (request instanceof PutMappingRequest) { + listener.onResponse((Response) AcknowledgedResponse.of(true)); + } } }; - manager = SecurityIndexManager.buildSecurityMainIndexManager(client, clusterService); - - } - public void testIndexWithFaultyMappingOnDisk() { - SecurityIndexManager.State state = new SecurityIndexManager.State(randomBoolean() ? Instant.now() : null, true, randomBoolean(), - false, null, "not_important", null, null); - Supplier mappingSourceSupplier = () -> { - throw new RuntimeException(); - }; - Runnable runnable = mock(Runnable.class); - manager = new SecurityIndexManager(mock(Client.class), RestrictedIndicesNames.SECURITY_MAIN_ALIAS, - RestrictedIndicesNames.INTERNAL_SECURITY_MAIN_INDEX_7, SecurityIndexManager.INTERNAL_MAIN_INDEX_FORMAT, - mappingSourceSupplier, state); - AtomicReference exceptionConsumer = new AtomicReference<>(); - manager.prepareIndexIfNeededThenExecute(e -> exceptionConsumer.set(e), runnable); - verify(runnable, never()).run(); - assertThat(exceptionConsumer.get(), is(notNullValue())); + final ClusterService clusterService = mock(ClusterService.class); + descriptorSpy = spy(Security.SECURITY_MAIN_INDEX_DESCRIPTOR); + manager = SecurityIndexManager.buildSecurityIndexManager(client, clusterService, descriptorSpy); } - public void testIndexWithUpToDateMappingAndTemplate() throws IOException { + public void testIndexWithUpToDateMappingAndTemplate() { assertInitialState(); final ClusterState.Builder clusterStateBuilder = createClusterState(RestrictedIndicesNames.INTERNAL_SECURITY_MAIN_INDEX_7, - RestrictedIndicesNames.SECURITY_MAIN_ALIAS, TEMPLATE_NAME); + RestrictedIndicesNames.SECURITY_MAIN_ALIAS); markShardsAvailable(clusterStateBuilder); manager.clusterChanged(event(clusterStateBuilder)); @@ -134,11 +121,11 @@ public void testIndexWithUpToDateMappingAndTemplate() throws IOException { assertThat(manager.isMappingUpToDate(), Matchers.equalTo(true)); } - public void testIndexWithoutPrimaryShards() throws IOException { + public void testIndexWithoutPrimaryShards() { assertInitialState(); final ClusterState.Builder clusterStateBuilder = createClusterState(RestrictedIndicesNames.INTERNAL_SECURITY_MAIN_INDEX_7, - RestrictedIndicesNames.SECURITY_MAIN_ALIAS, TEMPLATE_NAME); + RestrictedIndicesNames.SECURITY_MAIN_ALIAS); Index index = new Index(RestrictedIndicesNames.INTERNAL_SECURITY_MAIN_INDEX_7, UUID.randomUUID().toString()); ShardRouting shardRouting = ShardRouting.newUnassigned(new ShardId(index, 0), true, RecoverySource.ExistingStoreRecoverySource.INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "")); @@ -159,7 +146,7 @@ private ClusterChangedEvent event(ClusterState.Builder clusterStateBuilder) { return new ClusterChangedEvent("test-event", clusterStateBuilder.build(), EMPTY_CLUSTER_STATE); } - public void testIndexHealthChangeListeners() throws Exception { + public void testIndexHealthChangeListeners() { final AtomicBoolean listenerCalled = new AtomicBoolean(false); final AtomicReference previousState = new AtomicReference<>(); final AtomicReference currentState = new AtomicReference<>(); @@ -172,7 +159,7 @@ public void testIndexHealthChangeListeners() throws Exception { // index doesn't exist and now exists final ClusterState.Builder clusterStateBuilder = createClusterState(RestrictedIndicesNames.INTERNAL_SECURITY_MAIN_INDEX_7, - RestrictedIndicesNames.SECURITY_MAIN_ALIAS, TEMPLATE_NAME); + RestrictedIndicesNames.SECURITY_MAIN_ALIAS); markShardsAvailable(clusterStateBuilder); final ClusterState clusterState = clusterStateBuilder.build(); manager.clusterChanged(event(ClusterState.builder(clusterState))); @@ -225,53 +212,107 @@ public void testIndexHealthChangeListeners() throws Exception { assertEquals(ClusterHealthStatus.GREEN, currentState.get().indexHealth); } - public void testWriteBeforeStateNotRecovered() throws Exception { + public void testWriteBeforeStateNotRecovered() { final AtomicBoolean prepareRunnableCalled = new AtomicBoolean(false); final AtomicReference prepareException = new AtomicReference<>(null); - manager.prepareIndexIfNeededThenExecute(ex -> { - prepareException.set(ex); - }, () -> { - prepareRunnableCalled.set(true); - }); + manager.prepareIndexIfNeededThenExecute(prepareException::set, () -> prepareRunnableCalled.set(true)); assertThat(prepareException.get(), is(notNullValue())); assertThat(prepareException.get(), instanceOf(ElasticsearchStatusException.class)); assertThat(((ElasticsearchStatusException)prepareException.get()).status(), is(RestStatus.SERVICE_UNAVAILABLE)); assertThat(prepareRunnableCalled.get(), is(false)); + prepareException.set(null); prepareRunnableCalled.set(false); // state not recovered final ClusterBlocks.Builder blocks = ClusterBlocks.builder().addGlobalBlock(GatewayService.STATE_NOT_RECOVERED_BLOCK); manager.clusterChanged(event(new ClusterState.Builder(CLUSTER_NAME).blocks(blocks))); - manager.prepareIndexIfNeededThenExecute(ex -> { - prepareException.set(ex); - }, () -> { - prepareRunnableCalled.set(true); - }); + manager.prepareIndexIfNeededThenExecute(prepareException::set, () -> prepareRunnableCalled.set(true)); assertThat(prepareException.get(), is(notNullValue())); assertThat(prepareException.get(), instanceOf(ElasticsearchStatusException.class)); assertThat(((ElasticsearchStatusException)prepareException.get()).status(), is(RestStatus.SERVICE_UNAVAILABLE)); assertThat(prepareRunnableCalled.get(), is(false)); + prepareException.set(null); prepareRunnableCalled.set(false); // state recovered with index ClusterState.Builder clusterStateBuilder = createClusterState(RestrictedIndicesNames.INTERNAL_SECURITY_MAIN_INDEX_7, - RestrictedIndicesNames.SECURITY_MAIN_ALIAS, TEMPLATE_NAME, SecurityIndexManager.INTERNAL_MAIN_INDEX_FORMAT); + RestrictedIndicesNames.SECURITY_MAIN_ALIAS, SecurityIndexManager.INTERNAL_MAIN_INDEX_FORMAT); markShardsAvailable(clusterStateBuilder); manager.clusterChanged(event(clusterStateBuilder)); - manager.prepareIndexIfNeededThenExecute(ex -> { - prepareException.set(ex); - }, () -> { - prepareRunnableCalled.set(true); - }); + manager.prepareIndexIfNeededThenExecute(prepareException::set, () -> prepareRunnableCalled.set(true)); assertThat(prepareException.get(), is(nullValue())); assertThat(prepareRunnableCalled.get(), is(true)); } - public void testListenerNotCalledBeforeStateNotRecovered() throws Exception { + /** + * Check that the security index manager will update an index's mappings if they are out-of-date. + * Although the {@index SystemIndexManager} normally handles this, the {@link SecurityIndexManager} + * expects to be able to handle this also. + */ + public void testCanUpdateIndexMappings() { + final AtomicBoolean prepareRunnableCalled = new AtomicBoolean(false); + final AtomicReference prepareException = new AtomicReference<>(null); + + // Ensure that the mappings for the index are out-of-date, so that the security index manager will + // attempt to update them. + String previousVersion = getPreviousVersion(Version.CURRENT); + + // State recovered with index, with mappings with a prior version + ClusterState.Builder clusterStateBuilder = createClusterState( + RestrictedIndicesNames.INTERNAL_SECURITY_MAIN_INDEX_7, + RestrictedIndicesNames.SECURITY_MAIN_ALIAS, + SecurityIndexManager.INTERNAL_MAIN_INDEX_FORMAT, + IndexMetadata.State.OPEN, + getMappings(previousVersion) + ); + markShardsAvailable(clusterStateBuilder); + manager.clusterChanged(event(clusterStateBuilder)); + + manager.prepareIndexIfNeededThenExecute(prepareException::set, () -> prepareRunnableCalled.set(true)); + + assertThat(prepareRunnableCalled.get(), is(true)); + assertThat(prepareException.get(), nullValue()); + } + + /** + * Check that the security index manager will refuse to update mappings on an index + * if the corresponding {@link SystemIndexDescriptor} requires a higher node version + * that the cluster's current minimum version. + */ + public void testCannotUpdateIndexMappingsWhenMinNodeVersionTooLow() { + final AtomicBoolean prepareRunnableCalled = new AtomicBoolean(false); + final AtomicReference prepareException = new AtomicReference<>(null); + + // Hard-code a failure here. + when(descriptorSpy.checkMinimumNodeVersion(anyString(), any(Version.class))).thenReturn("Nope"); + + // Ensure that the mappings for the index are out-of-date, so that the security index manager will + // attempt to update them. + String previousVersion = getPreviousVersion(Version.CURRENT); + + // State recovered with index, with mappings with a prior version + ClusterState.Builder clusterStateBuilder = createClusterState( + RestrictedIndicesNames.INTERNAL_SECURITY_MAIN_INDEX_7, + RestrictedIndicesNames.SECURITY_MAIN_ALIAS, + SecurityIndexManager.INTERNAL_MAIN_INDEX_FORMAT, + IndexMetadata.State.OPEN, + getMappings(previousVersion) + ); + markShardsAvailable(clusterStateBuilder); + manager.clusterChanged(event(clusterStateBuilder)); + manager.prepareIndexIfNeededThenExecute(prepareException::set, () -> prepareRunnableCalled.set(true)); + + assertThat(prepareRunnableCalled.get(), is(false)); + + final Exception exception = prepareException.get(); + assertThat(exception, not(nullValue())); + assertThat(exception, instanceOf(IllegalStateException.class)); + assertThat(exception.getMessage(), equalTo("Nope")); + } + + public void testListenerNotCalledBeforeStateNotRecovered() { final AtomicBoolean listenerCalled = new AtomicBoolean(false); - manager.addIndexStateListener((prev, current) -> { - listenerCalled.set(true); - }); + manager.addIndexStateListener((prev, current) -> listenerCalled.set(true)); final ClusterBlocks.Builder blocks = ClusterBlocks.builder().addGlobalBlock(GatewayService.STATE_NOT_RECOVERED_BLOCK); // state not recovered manager.clusterChanged(event(new ClusterState.Builder(CLUSTER_NAME).blocks(blocks))); @@ -279,14 +320,14 @@ public void testListenerNotCalledBeforeStateNotRecovered() throws Exception { assertThat(listenerCalled.get(), is(false)); // state recovered with index ClusterState.Builder clusterStateBuilder = createClusterState(RestrictedIndicesNames.INTERNAL_SECURITY_MAIN_INDEX_7, - RestrictedIndicesNames.SECURITY_MAIN_ALIAS, TEMPLATE_NAME, SecurityIndexManager.INTERNAL_MAIN_INDEX_FORMAT); + RestrictedIndicesNames.SECURITY_MAIN_ALIAS, SecurityIndexManager.INTERNAL_MAIN_INDEX_FORMAT); markShardsAvailable(clusterStateBuilder); manager.clusterChanged(event(clusterStateBuilder)); assertThat(manager.isStateRecovered(), is(true)); assertThat(listenerCalled.get(), is(true)); } - public void testIndexOutOfDateListeners() throws Exception { + public void testIndexOutOfDateListeners() { final AtomicBoolean listenerCalled = new AtomicBoolean(false); manager.clusterChanged(event(new ClusterState.Builder(CLUSTER_NAME))); AtomicBoolean upToDateChanged = new AtomicBoolean(); @@ -302,7 +343,7 @@ public void testIndexOutOfDateListeners() throws Exception { // index doesn't exist and now exists with wrong format ClusterState.Builder clusterStateBuilder = createClusterState(RestrictedIndicesNames.INTERNAL_SECURITY_MAIN_INDEX_7, - RestrictedIndicesNames.SECURITY_MAIN_ALIAS, TEMPLATE_NAME, SecurityIndexManager.INTERNAL_MAIN_INDEX_FORMAT - 1); + RestrictedIndicesNames.SECURITY_MAIN_ALIAS, SecurityIndexManager.INTERNAL_MAIN_INDEX_FORMAT - 1); markShardsAvailable(clusterStateBuilder); manager.clusterChanged(event(clusterStateBuilder)); assertTrue(listenerCalled.get()); @@ -319,7 +360,7 @@ public void testIndexOutOfDateListeners() throws Exception { listenerCalled.set(false); // index doesn't exist and now exists with correct format clusterStateBuilder = createClusterState(RestrictedIndicesNames.INTERNAL_SECURITY_MAIN_INDEX_7, - RestrictedIndicesNames.SECURITY_MAIN_ALIAS, TEMPLATE_NAME, SecurityIndexManager.INTERNAL_MAIN_INDEX_FORMAT); + RestrictedIndicesNames.SECURITY_MAIN_ALIAS, SecurityIndexManager.INTERNAL_MAIN_INDEX_FORMAT); markShardsAvailable(clusterStateBuilder); manager.clusterChanged(event(clusterStateBuilder)); assertTrue(listenerCalled.get()); @@ -327,10 +368,10 @@ public void testIndexOutOfDateListeners() throws Exception { assertTrue(manager.isIndexUpToDate()); } - public void testProcessClosedIndexState() throws Exception { + public void testProcessClosedIndexState() { // Index initially exists final ClusterState.Builder indexAvailable = createClusterState(RestrictedIndicesNames.INTERNAL_SECURITY_MAIN_INDEX_7, - RestrictedIndicesNames.SECURITY_MAIN_ALIAS, TEMPLATE_NAME, IndexMetadata.State.OPEN); + RestrictedIndicesNames.SECURITY_MAIN_ALIAS, IndexMetadata.State.OPEN); markShardsAvailable(indexAvailable); manager.clusterChanged(event(indexAvailable)); @@ -339,7 +380,7 @@ public void testProcessClosedIndexState() throws Exception { // Now close it final ClusterState.Builder indexClosed = createClusterState(RestrictedIndicesNames.INTERNAL_SECURITY_MAIN_INDEX_7, - RestrictedIndicesNames.SECURITY_MAIN_ALIAS, TEMPLATE_NAME, IndexMetadata.State.CLOSE); + RestrictedIndicesNames.SECURITY_MAIN_ALIAS, IndexMetadata.State.CLOSE); if (randomBoolean()) { // In old/mixed cluster versions closed indices have no routing table indexClosed.routingTable(RoutingTable.EMPTY_ROUTING_TABLE); @@ -366,27 +407,23 @@ private void assertIndexUpToDateButNotAvailable() { assertThat(manager.isStateRecovered(), Matchers.equalTo(true)); } - public static ClusterState.Builder createClusterState(String indexName, String aliasName, String templateName) throws IOException { - return createClusterState(indexName, aliasName, templateName, IndexMetadata.State.OPEN); + public static ClusterState.Builder createClusterState(String indexName, String aliasName) { + return createClusterState(indexName, aliasName, IndexMetadata.State.OPEN); } - public static ClusterState.Builder createClusterState(String indexName, String aliasName, String templateName, - IndexMetadata.State state) throws IOException { - return createClusterState(indexName, aliasName, templateName, templateName, SecurityIndexManager.INTERNAL_MAIN_INDEX_FORMAT, state); + public static ClusterState.Builder createClusterState(String indexName, String aliasName, IndexMetadata.State state) { + return createClusterState(indexName, aliasName, SecurityIndexManager.INTERNAL_MAIN_INDEX_FORMAT, state, getMappings()); } - public static ClusterState.Builder createClusterState(String indexName, String aliasName, String templateName, int format) - throws IOException { - return createClusterState(indexName, aliasName, templateName, templateName, format, IndexMetadata.State.OPEN); + public static ClusterState.Builder createClusterState(String indexName, String aliasName, int format) { + return createClusterState(indexName, aliasName, format, IndexMetadata.State.OPEN, getMappings()); } - private static ClusterState.Builder createClusterState(String indexName, String aliasName, String templateName, String buildMappingFrom, - int format, IndexMetadata.State state) throws IOException { - IndexTemplateMetadata.Builder templateBuilder = getIndexTemplateMetadata(templateName); - IndexMetadata.Builder indexMeta = getIndexMetadata(indexName, aliasName, buildMappingFrom, format, state); + private static ClusterState.Builder createClusterState(String indexName, String aliasName, int format, IndexMetadata.State state, + String mappings) { + IndexMetadata.Builder indexMeta = getIndexMetadata(indexName, aliasName, format, state, mappings); Metadata.Builder metadataBuilder = new Metadata.Builder(); - metadataBuilder.put(templateBuilder); metadataBuilder.put(indexMeta); return ClusterState.builder(state()).metadata(metadataBuilder.build()); @@ -404,9 +441,8 @@ private static ClusterState state() { .build(); } - private static IndexMetadata.Builder getIndexMetadata(String indexName, String aliasName, String templateName, int format, - IndexMetadata.State state) - throws IOException { + private static IndexMetadata.Builder getIndexMetadata(String indexName, String aliasName, int format, IndexMetadata.State state, + String mappings) { IndexMetadata.Builder indexMetadata = IndexMetadata.builder(indexName); indexMetadata.settings(Settings.builder() .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) @@ -416,171 +452,56 @@ private static IndexMetadata.Builder getIndexMetadata(String indexName, String a .build()); indexMetadata.putAlias(AliasMetadata.builder(aliasName).build()); indexMetadata.state(state); - final Map mappings = getTemplateMappings(templateName); - for (Map.Entry entry : mappings.entrySet()) { - indexMetadata.putMapping(entry.getKey(), entry.getValue()); - } - - return indexMetadata; - } - private static IndexTemplateMetadata.Builder getIndexTemplateMetadata(String templateName) throws IOException { - final Map mappings = getTemplateMappings(templateName); - IndexTemplateMetadata.Builder templateBuilder = IndexTemplateMetadata.builder(TEMPLATE_NAME) - .patterns(Arrays.asList(generateRandomStringArray(10, 100, false, false))); - for (Map.Entry entry : mappings.entrySet()) { - templateBuilder.putMapping(entry.getKey(), entry.getValue()); + try { + MappingMetadata mappingMetadata = new MappingMetadata( + MapperService.SINGLE_MAPPING_NAME, + XContentHelper.convertToMap(JsonXContent.jsonXContent, mappings, true) + ); + indexMetadata.putMapping(mappingMetadata); + } catch (IOException e) { + throw new UncheckedIOException(e); } - return templateBuilder; - } - - private static Map getTemplateMappings(String templateName) { - String template = loadTemplate(templateName); - PutIndexTemplateRequest request = new PutIndexTemplateRequest(); - request.source(template, XContentType.JSON); - return request.mappings(); - } - private static String loadTemplate(String templateName) { - final String resource = "/" + templateName + ".json"; - return TemplateUtils.loadTemplate(resource, Version.CURRENT.toString(), TEMPLATE_VERSION_VARIABLE); - } - - public void testMappingVersionMatching() throws IOException { - String templateString = "/" + SECURITY_MAIN_TEMPLATE_7 + ".json"; - ClusterState.Builder clusterStateBuilder = createClusterStateWithMappingAndTemplate(templateString); - manager.clusterChanged(new ClusterChangedEvent("test-event", clusterStateBuilder.build(), EMPTY_CLUSTER_STATE)); - assertTrue(manager.checkMappingVersion(Version.CURRENT.minimumIndexCompatibilityVersion()::before)); - assertFalse(manager.checkMappingVersion(Version.CURRENT.minimumIndexCompatibilityVersion()::after)); - } - - public void testMissingVersionMappingThrowsError() throws IOException { - String templateString = "/missing-version-security-index-template.json"; - ClusterState.Builder clusterStateBuilder = createClusterStateWithMappingAndTemplate(templateString); - final ClusterState clusterState = clusterStateBuilder.build(); - IllegalStateException exception = expectThrows(IllegalStateException.class, - () -> SecurityIndexManager.checkIndexMappingVersionMatches(RestrictedIndicesNames.SECURITY_MAIN_ALIAS, - clusterState, logger, Version.CURRENT::equals)); - assertEquals("Cannot read security-version string in index " + RestrictedIndicesNames.SECURITY_MAIN_ALIAS, - exception.getMessage()); - } - - public void testIndexTemplateIsIdentifiedAsUpToDate() throws IOException { - ClusterState.Builder clusterStateBuilder = createClusterStateWithTemplate( - "/" + SECURITY_MAIN_TEMPLATE_7 + ".json" - ); - manager.clusterChanged(new ClusterChangedEvent("test-event", clusterStateBuilder.build(), EMPTY_CLUSTER_STATE)); - // No upgrade actions run - assertThat(actions.size(), equalTo(0)); - } - - public void testIndexTemplateVersionMatching() throws Exception { - String templateString = "/" + SECURITY_MAIN_TEMPLATE_7 + ".json"; - ClusterState.Builder clusterStateBuilder = createClusterStateWithTemplate(templateString); - final ClusterState clusterState = clusterStateBuilder.build(); - - assertTrue(SecurityIndexManager.checkTemplateExistsAndVersionMatches( - SecurityIndexManager.SECURITY_MAIN_TEMPLATE_7, clusterState, logger, - Version.V_6_0_0::before)); - assertFalse(SecurityIndexManager.checkTemplateExistsAndVersionMatches( - SecurityIndexManager.SECURITY_MAIN_TEMPLATE_7, clusterState, logger, - Version.V_6_0_0::after)); - } - - public void testUpToDateMappingsAreIdentifiedAsUpToDate() throws IOException { - String securityTemplateString = "/" + SECURITY_MAIN_TEMPLATE_7 + ".json"; - ClusterState.Builder clusterStateBuilder = createClusterStateWithMappingAndTemplate(securityTemplateString); - manager.clusterChanged(new ClusterChangedEvent("test-event", - clusterStateBuilder.build(), EMPTY_CLUSTER_STATE)); - assertThat(actions.size(), equalTo(0)); + return indexMetadata; } - public void testMissingIndexIsIdentifiedAsUpToDate() throws IOException { - final ClusterName clusterName = new ClusterName("test-cluster"); - final ClusterState.Builder clusterStateBuilder = ClusterState.builder(clusterName); - String mappingString = "/" + SECURITY_MAIN_TEMPLATE_7 + ".json"; - IndexTemplateMetadata.Builder templateMeta = getIndexTemplateMetadata(SECURITY_MAIN_TEMPLATE_7, mappingString); - final ClusterState clusterState = clusterStateBuilder.build(); - Metadata.Builder builder = new Metadata.Builder(clusterState.getMetadata()); - builder.put(templateMeta); - manager.clusterChanged(new ClusterChangedEvent("test-event", ClusterState.builder(clusterState).metadata(builder).build() - , EMPTY_CLUSTER_STATE)); - assertThat(actions.size(), equalTo(0)); + private static String getMappings() { + return getMappings(Version.CURRENT.toString()); } - private ClusterState.Builder createClusterStateWithTemplate(String securityTemplateString) throws IOException { - // add the correct mapping no matter what the template - ClusterState clusterState = createClusterStateWithIndex("/" + SECURITY_MAIN_TEMPLATE_7 + ".json").build(); - final Metadata.Builder metadataBuilder = new Metadata.Builder(clusterState.metadata()); - metadataBuilder.put(getIndexTemplateMetadata(SECURITY_MAIN_TEMPLATE_7, securityTemplateString)); - return ClusterState.builder(clusterState).metadata(metadataBuilder); - } + private static String getMappings(String version) { + try { + final XContentBuilder builder = jsonBuilder(); - private ClusterState.Builder createClusterStateWithMapping(String securityTemplateString) throws IOException { - final ClusterState clusterState = createClusterStateWithIndex(securityTemplateString).build(); - final String indexName = clusterState.metadata().getIndicesLookup() - .get(RestrictedIndicesNames.SECURITY_MAIN_ALIAS).getIndices().get(0).getIndex().getName(); - return ClusterState.builder(clusterState).routingTable(SecurityTestUtils.buildIndexRoutingTable(indexName)); - } + builder.startObject(); + { + builder.startObject("_meta"); + builder.field("security-version", version); + builder.endObject(); - private ClusterState.Builder createClusterStateWithMappingAndTemplate(String securityTemplateString) throws IOException { - final ClusterState state = createClusterStateWithMapping(securityTemplateString).build(); - Metadata.Builder metadataBuilder = new Metadata.Builder(state.metadata()); - String securityMappingString = "/" + SECURITY_MAIN_TEMPLATE_7 + ".json"; - IndexTemplateMetadata.Builder securityTemplateMeta = getIndexTemplateMetadata(SECURITY_MAIN_TEMPLATE_7, securityMappingString); - metadataBuilder.put(securityTemplateMeta); - return ClusterState.builder(state).metadata(metadataBuilder); - } - - private static IndexMetadata.Builder createIndexMetadata(String indexName, String templateString) throws IOException { - String template = TemplateUtils.loadTemplate(templateString, Version.CURRENT.toString(), - SecurityIndexManager.TEMPLATE_VERSION_VARIABLE); - PutIndexTemplateRequest request = new PutIndexTemplateRequest(); - request.source(template, XContentType.JSON); - IndexMetadata.Builder indexMetadata = IndexMetadata.builder(indexName); - indexMetadata.settings(Settings.builder() - .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) - .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) - .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) - .build()); + builder.field("dynamic", "strict"); + builder.startObject("properties"); + { + builder.startObject("completed"); + builder.field("type", "boolean"); + builder.endObject(); + } + builder.endObject(); + } - for (Map.Entry entry : request.mappings().entrySet()) { - indexMetadata.putMapping(entry.getKey(), entry.getValue()); + builder.endObject(); + return Strings.toString(builder); + } catch (IOException e) { + throw new UncheckedIOException("Failed to build index mappings", e); } - return indexMetadata; } - private ClusterState.Builder createClusterStateWithIndex(String securityTemplate) throws IOException { - final Metadata.Builder metadataBuilder = new Metadata.Builder(); - final boolean withAlias = randomBoolean(); - final String securityIndexName = RestrictedIndicesNames.SECURITY_MAIN_ALIAS - + (withAlias ? "-" + randomAlphaOfLength(5) : ""); - metadataBuilder.put(createIndexMetadata(securityIndexName, securityTemplate)); - - ClusterState.Builder clusterStateBuilder = ClusterState.builder(state()); - if (withAlias) { - // try with .security index as an alias - clusterStateBuilder.metadata(SecurityTestUtils.addAliasToMetadata(metadataBuilder.build(), securityIndexName)); - } else { - // try with .security index as a concrete index - clusterStateBuilder.metadata(metadataBuilder); + private String getPreviousVersion(Version version) { + if (version.minor == 0) { + return version.major - 1 + ".99.0"; } - clusterStateBuilder.routingTable(SecurityTestUtils.buildIndexRoutingTable(securityIndexName)); - return clusterStateBuilder; - } - - private static IndexTemplateMetadata.Builder getIndexTemplateMetadata(String templateName, String templateString) throws IOException { - - String template = TemplateUtils.loadTemplate(templateString, Version.CURRENT.toString(), - SecurityIndexManager.TEMPLATE_VERSION_VARIABLE); - PutIndexTemplateRequest request = new PutIndexTemplateRequest(); - request.source(template, XContentType.JSON); - IndexTemplateMetadata.Builder templateBuilder = IndexTemplateMetadata.builder(templateName) - .patterns(Arrays.asList(generateRandomStringArray(10, 100, false, false))); - for (Map.Entry entry : request.mappings().entrySet()) { - templateBuilder.putMapping(entry.getKey(), entry.getValue()); - } - return templateBuilder; + return version.major + "." + (version.minor - 1) + ".0"; } } diff --git a/x-pack/qa/third-party/active-directory/src/test/java/org/elasticsearch/xpack/security/authc/ldap/AbstractAdLdapRealmTestCase.java b/x-pack/qa/third-party/active-directory/src/test/java/org/elasticsearch/xpack/security/authc/ldap/AbstractAdLdapRealmTestCase.java index b91b4a03f6268..18705c4f3c5c3 100644 --- a/x-pack/qa/third-party/active-directory/src/test/java/org/elasticsearch/xpack/security/authc/ldap/AbstractAdLdapRealmTestCase.java +++ b/x-pack/qa/third-party/active-directory/src/test/java/org/elasticsearch/xpack/security/authc/ldap/AbstractAdLdapRealmTestCase.java @@ -17,7 +17,6 @@ import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.SecurityIntegTestCase; @@ -27,7 +26,6 @@ import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken; import org.elasticsearch.xpack.core.security.client.SecurityClient; import org.elasticsearch.xpack.core.ssl.VerificationMode; -import org.elasticsearch.xpack.security.support.SecurityIndexManager; import org.junit.After; import org.junit.AfterClass; import org.junit.Before; @@ -41,7 +39,6 @@ import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.Set; import java.util.function.Function; import java.util.function.Supplier; import java.util.stream.Collectors; @@ -159,13 +156,6 @@ public void cleanupSecurityIndex() throws Exception { super.deleteSecurityIndex(); } - @Override - public Set excludeTemplates() { - Set templates = Sets.newHashSet(super.excludeTemplates()); - templates.add(SecurityIndexManager.SECURITY_MAIN_TEMPLATE_7); // don't remove the security index template - return templates; - } - private List getRoleMappingContent(Function contentFunction) { return getRoleMappingContent(contentFunction, AbstractAdLdapRealmTestCase.roleMappings); }