Restore a noop _all metadata field for 6x indices (#37808)

This commit restores a noop version of the AllFieldMapper that is instanciated only
for indices created in 6x. We need this metadata field mapper to be present in this version
in order to allow the upgrade of indices that explicitly disable _all (enabled: false).
The mapping of these indices contains a reference to the _all field that we cannot remove
in 7 so we'll need to keep this metadata mapper in 7x. Since indices created in 6x will not
be compatible with 8, we'll remove this noop mapper in the next major version.

Closes #37429

Author: jimczi

qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/10_basic.yml

@@ -67,3 +67,9 @@
                 field3: value
   - match: { hits.total: 1 }
   - match: { hits.hits.0._id: q3 }
+
+---
+"Index with _all is available":
+  - do:
+      indices.get:
+        index: all-index

qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/10_basic.yml

@@ -203,3 +203,21 @@
       tasks.get:
         wait_for_completion: true
         task_id: $task
+
+---
+"Create an index with _all explicitly disabled":
+  - skip:
+      features: warnings
+  - do:
+      warnings:
+        - "[_all] is deprecated in 6.0+ and will be removed in 7.0. As a replacement, you can use [copy_to] on mapping fields to create your own catch all field."
+      indices.create:
+        index: all-index
+        body:
+          mappings:
+            type:
+              _all:
+                enabled: false
+              properties:
+                field:
+                  type: text

qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/10_basic.yml

@@ -125,3 +125,17 @@
         wait_for_completion: true
         task_id: $task_id
   - match: { task.headers.X-Opaque-Id: "Reindexing Again" }
+
+---
+"Index with _all is available":
+  - do:
+      indices.get:
+        index: all-index
+
+  - do:
+      indices.get_mapping:
+        index: all-index
+
+  - is_true: all-index.mappings._all
+  - match: { all-index.mappings._all.enabled: false}
+

server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetFieldMappingsIndexAction.java

@@ -20,6 +20,7 @@
 package org.elasticsearch.action.admin.indices.mapping.get;
 
 import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.Version;
 import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse.FieldMappingMetaData;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.single.shard.TransportSingleShardAction;
@@ -91,7 +92,8 @@ protected ShardsIterator shards(ClusterState state, InternalRequest request) {
     protected GetFieldMappingsResponse shardOperation(final GetFieldMappingsIndexRequest request, ShardId shardId) {
         assert shardId != null;
         IndexService indexService = indicesService.indexServiceSafe(shardId.getIndex());
-        Predicate<String> metadataFieldPredicate = indicesService::isMetaDataField;
+        Version indexCreatedVersion = indexService.mapperService().getIndexSettings().getIndexVersionCreated();
+        Predicate<String> metadataFieldPredicate = (f) -> indicesService.isMetaDataField(indexCreatedVersion, f);
         Predicate<String> fieldPredicate = metadataFieldPredicate.or(indicesService.getFieldFilter().apply(shardId.getIndexName()));
 
         DocumentMapper mapper = indexService.mapperService().documentMapper();

server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesIndexAction.java

@@ -83,7 +83,8 @@ protected FieldCapabilitiesIndexResponse shardOperation(final FieldCapabilitiesI
         for (String field : fieldNames) {
             MappedFieldType ft = mapperService.fullName(field);
             if (ft != null) {
-                if (indicesService.isMetaDataField(field) || fieldPredicate.test(ft.name())) {
+                if (indicesService.isMetaDataField(mapperService.getIndexSettings().getIndexVersionCreated(), field)
+                        || fieldPredicate.test(ft.name())) {
                     FieldCapabilities fieldCap = new FieldCapabilities(field, ft.typeName(), ft.isSearchable(), ft.isAggregatable());
                     responseMap.put(field, fieldCap);
                 } else {

server/src/main/java/org/elasticsearch/index/mapper/AllFieldMapper.java

@@ -0,0 +1,175 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.mapper;
+
+import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.index.IndexableField;
+import org.apache.lucene.search.MatchNoDocsQuery;
+import org.apache.lucene.search.Query;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.support.XContentMapValues;
+import org.elasticsearch.index.query.QueryShardContext;
+
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Noop mapper that ensures that mappings created in 6x that explicitly disable the _all field
+ * can be restored in this version.
+ *
+ * TODO: Remove in 8
+ */
+public class AllFieldMapper extends MetadataFieldMapper {
+    public static final String NAME = "_all";
+    public static final String CONTENT_TYPE = "_all";
+
+    public static class Defaults {
+        public static final MappedFieldType FIELD_TYPE = new AllFieldType();
+
+        static {
+            FIELD_TYPE.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
+            FIELD_TYPE.setTokenized(true);
+            FIELD_TYPE.setName(NAME);
+            FIELD_TYPE.freeze();
+        }
+    }
+
+    public static class Builder extends MetadataFieldMapper.Builder<Builder, AllFieldMapper> {
+        private boolean disableExplicit = false;
+
+        public Builder(MappedFieldType existing) {
+            super(NAME, existing == null ? Defaults.FIELD_TYPE : existing, Defaults.FIELD_TYPE);
+            builder = this;
+        }
+
+        private Builder setDisableExplicit() {
+            this.disableExplicit = true;
+            return this;
+        }
+
+        @Override
+        public AllFieldMapper build(BuilderContext context) {
+            return new AllFieldMapper(fieldType, context.indexSettings(), disableExplicit);
+        }
+    }
+
+    public static class TypeParser implements MetadataFieldMapper.TypeParser {
+        @Override
+        public MetadataFieldMapper.Builder<?,?> parse(String name, Map<String, Object> node,
+                                                 ParserContext parserContext) throws MapperParsingException {
+            Builder builder = new Builder(parserContext.mapperService().fullName(NAME));
+            for (Iterator<Map.Entry<String, Object>> iterator = node.entrySet().iterator(); iterator.hasNext();) {
+                Map.Entry<String, Object> entry = iterator.next();
+                String fieldName = entry.getKey();
+                if (fieldName.equals("enabled")) {
+                    boolean enabled = XContentMapValues.nodeBooleanValue(entry.getValue(), "enabled");
+                    if (enabled) {
+                        throw new IllegalArgumentException("[_all] is disabled in this version.");
+                    }
+                    builder.setDisableExplicit();
+                    iterator.remove();
+                }
+            }
+            return builder;
+        }
+
+        @Override
+        public MetadataFieldMapper getDefault(MappedFieldType fieldType, ParserContext context) {
+            final Settings indexSettings = context.mapperService().getIndexSettings().getSettings();
+            return new AllFieldMapper(indexSettings, Defaults.FIELD_TYPE, false);
+        }
+    }
+
+    static final class AllFieldType extends StringFieldType {
+        AllFieldType() {
+        }
+
+        protected AllFieldType(AllFieldType ref) {
+            super(ref);
+        }
+
+        @Override
+        public MappedFieldType clone() {
+            return new AllFieldType(this);
+        }
+
+        @Override
+        public String typeName() {
+            return CONTENT_TYPE;
+        }
+
+        @Override
+        public Query existsQuery(QueryShardContext context) {
+            return new MatchNoDocsQuery();
+        }
+    }
+
+    private final boolean disableExplicit;
+
+    private AllFieldMapper(Settings indexSettings, MappedFieldType existing, boolean disableExplicit) {
+        this(existing.clone(), indexSettings, disableExplicit);
+    }
+
+    private AllFieldMapper(MappedFieldType fieldType, Settings indexSettings, boolean disableExplicit) {
+        super(NAME, fieldType, Defaults.FIELD_TYPE, indexSettings);
+        this.disableExplicit = disableExplicit;
+    }
+
+    @Override
+    public void preParse(ParseContext context) throws IOException {
+    }
+
+    @Override
+    public void postParse(ParseContext context) throws IOException {
+        super.parse(context);
+    }
+
+    @Override
+    public void parse(ParseContext context) throws IOException {
+        // we parse in post parse
+    }
+
+    @Override
+    protected void parseCreateField(ParseContext context, List<IndexableField> fields) throws IOException {
+        // noop mapper
+        return;
+    }
+
+    @Override
+    protected String contentType() {
+        return CONTENT_TYPE;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        boolean includeDefaults = params.paramAsBoolean("include_defaults", false);
+        if (includeDefaults || disableExplicit) {
+            builder.startObject(CONTENT_TYPE);
+            if (disableExplicit) {
+                builder.field("enabled", false);
+            }
+            builder.endObject();
+        }
+        return builder;
+    }
+}

server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java

@@ -27,6 +27,7 @@
 import org.apache.lucene.search.Weight;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.ElasticsearchGenerationException;
+import org.elasticsearch.Version;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.settings.Settings;
@@ -73,7 +74,9 @@ public Builder(RootObjectMapper.Builder builder, MapperService mapperService) {
 
             final String type = rootObjectMapper.name();
             final DocumentMapper existingMapper = mapperService.documentMapper(type);
-            final Map<String, TypeParser> metadataMapperParsers = mapperService.mapperRegistry.getMetadataMapperParsers();
+            final Version indexCreatedVersion = mapperService.getIndexSettings().getIndexVersionCreated();
+            final Map<String, TypeParser> metadataMapperParsers =
+                mapperService.mapperRegistry.getMetadataMapperParsers(indexCreatedVersion);
             for (Map.Entry<String, MetadataFieldMapper.TypeParser> entry : metadataMapperParsers.entrySet()) {
                 final String name = entry.getKey();
                 final MetadataFieldMapper existingMetadataMapper = existingMapper == null

server/src/main/java/org/elasticsearch/index/mapper/DocumentMapperParser.java

@@ -65,8 +65,8 @@ public DocumentMapperParser(IndexSettings indexSettings, MapperService mapperSer
         this.similarityService = similarityService;
         this.queryShardContextSupplier = queryShardContextSupplier;
         this.typeParsers = mapperRegistry.getMapperParsers();
-        this.rootTypeParsers = mapperRegistry.getMetadataMapperParsers();
-        indexVersionCreated = indexSettings.getIndexVersionCreated();
+        this.indexVersionCreated = indexSettings.getIndexVersionCreated();
+        this.rootTypeParsers = mapperRegistry.getMetadataMapperParsers(indexVersionCreated);
     }
 
     public Mapper.TypeParser.ParserContext parserContext(String type) {

server/src/main/java/org/elasticsearch/indices/IndicesService.java

@@ -29,6 +29,7 @@
 import org.apache.lucene.util.RamUsageEstimator;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ResourceAlreadyExistsException;
+import org.elasticsearch.Version;
 import org.elasticsearch.action.admin.indices.stats.CommonStats;
 import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags;
 import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags.Flag;
@@ -1382,8 +1383,8 @@ public Function<String, Predicate<String>> getFieldFilter() {
     /**
      * Returns true if the provided field is a registered metadata field (including ones registered via plugins), false otherwise.
      */
-    public boolean isMetaDataField(String field) {
-        return mapperRegistry.isMetaDataField(field);
+    public boolean isMetaDataField(Version indexCreatedVersion, String field) {
+        return mapperRegistry.isMetaDataField(indexCreatedVersion, field);
     }
 
     /**

server/src/main/java/org/elasticsearch/indices/mapper/MapperRegistry.java

@@ -19,6 +19,8 @@
 
 package org.elasticsearch.indices.mapper;
 
+import org.elasticsearch.Version;
+import org.elasticsearch.index.mapper.AllFieldMapper;
 import org.elasticsearch.index.mapper.Mapper;
 import org.elasticsearch.index.mapper.MetadataFieldMapper;
 import org.elasticsearch.plugins.MapperPlugin;
@@ -36,13 +38,19 @@ public final class MapperRegistry {
 
     private final Map<String, Mapper.TypeParser> mapperParsers;
     private final Map<String, MetadataFieldMapper.TypeParser> metadataMapperParsers;
+    private final Map<String, MetadataFieldMapper.TypeParser> metadataMapperParsers6x;
     private final Function<String, Predicate<String>> fieldFilter;
 
 
     public MapperRegistry(Map<String, Mapper.TypeParser> mapperParsers,
             Map<String, MetadataFieldMapper.TypeParser> metadataMapperParsers, Function<String, Predicate<String>> fieldFilter) {
         this.mapperParsers = Collections.unmodifiableMap(new LinkedHashMap<>(mapperParsers));
         this.metadataMapperParsers = Collections.unmodifiableMap(new LinkedHashMap<>(metadataMapperParsers));
+        // add the _all field mapper for indices created in 6x
+        Map<String, MetadataFieldMapper.TypeParser> metadata6x = new LinkedHashMap<>();
+        metadata6x.put(AllFieldMapper.NAME, new AllFieldMapper.TypeParser());
+        metadata6x.putAll(metadataMapperParsers);
+        this.metadataMapperParsers6x = Collections.unmodifiableMap(metadata6x);
         this.fieldFilter = fieldFilter;
     }
 
@@ -58,15 +66,15 @@ public Map<String, Mapper.TypeParser> getMapperParsers() {
      * Return a map of the meta mappers that have been registered. The
      * returned map uses the name of the field as a key.
      */
-    public Map<String, MetadataFieldMapper.TypeParser> getMetadataMapperParsers() {
-        return metadataMapperParsers;
+    public Map<String, MetadataFieldMapper.TypeParser> getMetadataMapperParsers(Version indexCreatedVersion) {
+        return indexCreatedVersion.onOrAfter(Version.V_7_0_0) ? metadataMapperParsers : metadataMapperParsers6x;
     }
 
     /**
-     * Returns true if the provide field is a registered metadata field, false otherwise
+     * Returns true if the provided field is a registered metadata field, false otherwise
      */
-    public boolean isMetaDataField(String field) {
-        return getMetadataMapperParsers().containsKey(field);
+    public boolean isMetaDataField(Version indexCreatedVersion, String field) {
+        return getMetadataMapperParsers(indexCreatedVersion).containsKey(field);
     }
 
     /**