Add analysis components and mapping types to the usage API.

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java

@@ -236,6 +236,7 @@
 import org.elasticsearch.xpack.core.watcher.transport.actions.put.PutWatchAction;
 import org.elasticsearch.xpack.core.watcher.transport.actions.service.WatcherServiceAction;
 import org.elasticsearch.xpack.core.watcher.transport.actions.stats.WatcherStatsAction;
+import org.elasticsearch.xpack.oss.IndexFeatureSetUsage;
 
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -561,7 +562,8 @@ public List<NamedWriteableRegistry.Entry> getNamedWriteables() {
                 // Spatial
                 new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.SPATIAL, SpatialFeatureSetUsage::new),
                 // data science
-                new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.ANALYTICS, AnalyticsFeatureSetUsage::new)
+                new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.ANALYTICS, AnalyticsFeatureSetUsage::new),
+                new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.INDEX, IndexFeatureSetUsage::new)
             ).stream(),
             MlEvaluationNamedXContentProvider.getNamedWriteables().stream()
         ).collect(toList());

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackField.java

@@ -53,6 +53,8 @@ public final class XPackField {
     public static final String ANALYTICS = "analytics";
     /** Name constant for the enrich plugin. */
     public static final String ENRICH = "enrich";
+    /** Name constant for indices. */
+    public static final String INDEX = "index";
 
     private XPackField() {}
 

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java

@@ -59,6 +59,7 @@
 import org.elasticsearch.xpack.core.action.TransportXPackUsageAction;
 import org.elasticsearch.xpack.core.action.XPackInfoAction;
 import org.elasticsearch.xpack.core.action.XPackUsageAction;
+import org.elasticsearch.xpack.core.action.XPackUsageFeatureAction;
 import org.elasticsearch.xpack.core.action.XPackUsageResponse;
 import org.elasticsearch.xpack.core.ml.MlMetadata;
 import org.elasticsearch.xpack.core.rest.action.RestReloadAnalyzersAction;
@@ -68,6 +69,7 @@
 import org.elasticsearch.xpack.core.ssl.SSLConfigurationReloader;
 import org.elasticsearch.xpack.core.ssl.SSLService;
 import org.elasticsearch.xpack.core.watcher.WatcherMetaData;
+import org.elasticsearch.xpack.oss.IndexUsageTransportAction;
 
 import java.nio.file.Files;
 import java.nio.file.Path;
@@ -255,6 +257,7 @@ public Collection<Object> createComponents(Client client, ClusterService cluster
         actions.add(new ActionHandler<>(XPackUsageAction.INSTANCE, getUsageAction()));
         actions.addAll(licensing.getActions());
         actions.add(new ActionHandler<>(ReloadAnalyzerAction.INSTANCE, TransportReloadAnalyzersAction.class));
+        actions.add(new ActionHandler<>(XPackUsageFeatureAction.INDEX, IndexUsageTransportAction.class));
         return actions;
     }
 

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackUsageFeatureAction.java

@@ -40,10 +40,11 @@ public class XPackUsageFeatureAction extends ActionType<XPackUsageFeatureRespons
     public static final XPackUsageFeatureAction FROZEN_INDICES = new XPackUsageFeatureAction(XPackField.FROZEN_INDICES);
     public static final XPackUsageFeatureAction SPATIAL = new XPackUsageFeatureAction(XPackField.SPATIAL);
     public static final XPackUsageFeatureAction ANALYTICS = new XPackUsageFeatureAction(XPackField.ANALYTICS);
+    public static final XPackUsageFeatureAction INDEX = new XPackUsageFeatureAction(XPackField.INDEX);
 
     public static final List<XPackUsageFeatureAction> ALL = Arrays.asList(
         SECURITY, MONITORING, WATCHER, GRAPH, MACHINE_LEARNING, LOGSTASH, SQL, ROLLUP, INDEX_LIFECYCLE, SNAPSHOT_LIFECYCLE, CCR,
-        TRANSFORM, FLATTENED, VECTORS, VOTING_ONLY, FROZEN_INDICES, SPATIAL, ANALYTICS
+        TRANSFORM, FLATTENED, VECTORS, VOTING_ONLY, FROZEN_INDICES, SPATIAL, ANALYTICS, INDEX
     );
 
     private XPackUsageFeatureAction(String name) {

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/oss/IndexFeatureSetUsage.java

@@ -0,0 +1,185 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.oss;
+
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.xpack.core.XPackFeatureSet;
+import org.elasticsearch.xpack.core.XPackField;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Objects;
+import java.util.Set;
+import java.util.TreeSet;
+
+public class IndexFeatureSetUsage extends XPackFeatureSet.Usage {
+
+    private static Set<String> sort(Set<String> set) {
+        return Collections.unmodifiableSet(new TreeSet<>(set));
+    }
+
+    private final Set<String> usedFieldTypes;
+    private final Set<String> usedCharFilters, usedTokenizers, usedTokenFilters, usedAnalyzers;
+    private final Set<String> usedBuiltInCharFilters, usedBuiltInTokenizers, usedBuiltInTokenFilters, usedBuiltInAnalyzers;
+
+    public IndexFeatureSetUsage(Set<String> usedFieldTypes,
+            Set<String> usedCharFilters, Set<String> usedTokenizers, Set<String> usedTokenFilters, Set<String> usedAnalyzers,
+            Set<String> usedBuiltInCharFilters, Set<String> usedBuiltInTokenizers, Set<String> usedBuiltInTokenFilters,
+            Set<String> usedBuiltInAnalyzers) {
+        super(XPackField.INDEX, true, true);
+        this.usedFieldTypes = sort(usedFieldTypes);
+        this.usedCharFilters = sort(usedCharFilters);
+        this.usedTokenizers = sort(usedTokenizers);
+        this.usedTokenFilters = sort(usedTokenFilters);
+        this.usedAnalyzers = sort(usedAnalyzers);
+        this.usedBuiltInCharFilters = sort(usedBuiltInCharFilters);
+        this.usedBuiltInTokenizers = sort(usedBuiltInTokenizers);
+        this.usedBuiltInTokenFilters = sort(usedBuiltInTokenFilters);
+        this.usedBuiltInAnalyzers = sort(usedBuiltInAnalyzers);
+    }
+
+    public IndexFeatureSetUsage(StreamInput input) throws IOException {
+        super(input);
+        usedFieldTypes = input.readSet(StreamInput::readString);
+        usedCharFilters = input.readSet(StreamInput::readString);
+        usedTokenizers = input.readSet(StreamInput::readString);
+        usedTokenFilters = input.readSet(StreamInput::readString);
+        usedAnalyzers = input.readSet(StreamInput::readString);
+        usedBuiltInCharFilters = input.readSet(StreamInput::readString);
+        usedBuiltInTokenizers = input.readSet(StreamInput::readString);
+        usedBuiltInTokenFilters = input.readSet(StreamInput::readString);
+        usedBuiltInAnalyzers = input.readSet(StreamInput::readString);
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        super.writeTo(out);
+        out.writeCollection(usedFieldTypes, StreamOutput::writeString);
+        out.writeCollection(usedCharFilters, StreamOutput::writeString);
+        out.writeCollection(usedTokenizers, StreamOutput::writeString);
+        out.writeCollection(usedTokenFilters, StreamOutput::writeString);
+        out.writeCollection(usedAnalyzers, StreamOutput::writeString);
+        out.writeCollection(usedBuiltInCharFilters, StreamOutput::writeString);
+        out.writeCollection(usedBuiltInTokenizers, StreamOutput::writeString);
+        out.writeCollection(usedBuiltInTokenFilters, StreamOutput::writeString);
+        out.writeCollection(usedBuiltInAnalyzers, StreamOutput::writeString);
+    }
+
+    /**
+     * Return the set of used field types in the cluster.
+     */
+    public Set<String> getUsedFieldTypes() {
+        return usedFieldTypes;
+    }
+
+    /**
+     * Return the set of used char filters in the cluster.
+     */
+    public Set<String> getUsedCharFilterTypes() {
+        return usedCharFilters;
+    }
+
+    /**
+     * Return the set of used tokenizers in the cluster.
+     */
+    public Set<String> getUsedTokenizerTypes() {
+        return usedTokenizers;
+    }
+
+    /**
+     * Return the set of used token filters in the cluster.
+     */
+    public Set<String> getUsedTokenFilterTypes() {
+        return usedTokenFilters;
+    }
+
+    /**
+     * Return the set of used analyzers in the cluster.
+     */
+    public Set<String> getUsedAnalyzerTypes() {
+        return usedAnalyzers;
+    }
+
+    /**
+     * Return the set of used built-in char filters in the cluster.
+     */
+    public Set<String> getUsedBuiltInCharFilters() {
+        return usedCharFilters;
+    }
+
+    /**
+     * Return the set of used built-in tokenizers in the cluster.
+     */
+    public Set<String> getUsedBuiltInTokenizers() {
+        return usedTokenizers;
+    }
+
+    /**
+     * Return the set of used built-in token filters in the cluster.
+     */
+    public Set<String> getUsedBuiltInTokenFilters() {
+        return usedTokenFilters;
+    }
+
+    /**
+     * Return the set of used built-in analyzers in the cluster.
+     */
+    public Set<String> getUsedBuiltInAnalyzers() {
+        return usedAnalyzers;
+    }
+
+    @Override
+    protected void innerXContent(XContentBuilder builder, Params params) throws IOException {
+        super.innerXContent(builder, params);
+
+        builder.startObject("analysis");
+        {
+            builder.field("char_filter_types", usedCharFilters);
+            builder.field("tokenizer_types", usedTokenizers);
+            builder.field("filter_types", usedTokenFilters);
+            builder.field("analyzer_types", usedAnalyzers);
+
+            builder.field("built_in_char_filters", usedBuiltInCharFilters);
+            builder.field("built_in_tokenizers", usedBuiltInTokenizers);
+            builder.field("built_in_filters", usedBuiltInTokenFilters);
+            builder.field("built_in_analyzers", usedBuiltInAnalyzers);
+        }
+        builder.endObject();
+
+        builder.startObject("mappings");
+        {
+            builder.field("field_types", usedFieldTypes);
+        }
+        builder.endObject();
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        IndexFeatureSetUsage that = (IndexFeatureSetUsage) o;
+        return available == that.available && enabled == that.enabled &&
+                Objects.equals(usedFieldTypes, that.usedFieldTypes) &&
+                Objects.equals(usedCharFilters, that.usedCharFilters) &&
+                Objects.equals(usedTokenizers, that.usedTokenizers) &&
+                Objects.equals(usedTokenFilters, that.usedTokenFilters) &&
+                Objects.equals(usedAnalyzers, that.usedAnalyzers) &&
+                Objects.equals(usedBuiltInCharFilters, that.usedBuiltInCharFilters) &&
+                Objects.equals(usedBuiltInTokenizers, that.usedBuiltInTokenizers) &&
+                Objects.equals(usedBuiltInTokenFilters, that.usedBuiltInTokenFilters) &&
+                Objects.equals(usedBuiltInAnalyzers, that.usedBuiltInAnalyzers);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(available, enabled, usedFieldTypes, usedCharFilters, usedTokenizers, usedTokenFilters,
+                usedAnalyzers, usedBuiltInCharFilters, usedBuiltInTokenizers, usedBuiltInTokenFilters,
+                usedBuiltInAnalyzers);
+    }
+}

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/oss/IndexUsageTransportAction.java

@@ -0,0 +1,147 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.oss;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.metadata.MappingMetaData;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.protocol.xpack.XPackUsageRequest;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.xpack.core.action.XPackUsageFeatureAction;
+import org.elasticsearch.xpack.core.action.XPackUsageFeatureResponse;
+import org.elasticsearch.xpack.core.action.XPackUsageFeatureTransportAction;
+
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.Consumer;
+
+public class IndexUsageTransportAction extends XPackUsageFeatureTransportAction {
+
+    @Inject
+    public IndexUsageTransportAction(TransportService transportService, ClusterService clusterService, ThreadPool threadPool,
+                                     ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver) {
+        super(XPackUsageFeatureAction.INDEX.name(), transportService, clusterService, threadPool, actionFilters,
+            indexNameExpressionResolver);
+    }
+
+    @Override
+    protected void masterOperation(Task task, XPackUsageRequest request, ClusterState state,
+                                   ActionListener<XPackUsageFeatureResponse> listener) {
+
+        final Set<String> usedFieldTypes = new HashSet<>();
+        final Set<String> usedCharFilters = new HashSet<>();
+        final Set<String> usedTokenizers = new HashSet<>();
+        final Set<String> usedTokenFilters = new HashSet<>();
+        final Set<String> usedAnalyzers = new HashSet<>();
+        final Set<String> usedBuiltInCharFilters = new HashSet<>();
+        final Set<String> usedBuiltInTokenizers = new HashSet<>();
+        final Set<String> usedBuiltInTokenFilters = new HashSet<>();
+        final Set<String> usedBuiltInAnalyzers = new HashSet<>();
+
+        for (IndexMetaData indexMetaData : state.metaData()) {
+            MappingMetaData mappingMetaData = indexMetaData.mapping();
+            if (mappingMetaData != null) {
+                visitMapping(mappingMetaData.getSourceAsMap(), fieldMapping -> {
+                    Object type = fieldMapping.get("type");
+                    if (type != null) {
+                        usedFieldTypes.add(type.toString());
+                    } else if (fieldMapping.containsKey("properties")) {
+                        usedFieldTypes.add("object");
+                    }
+
+                    for (String key : new String[] { "analyzer", "search_analyzer", "search_quote_analyzer" }) {
+                        Object analyzer = fieldMapping.get(key);
+                        if (analyzer != null) {
+                            usedBuiltInAnalyzers.add(analyzer.toString());
+                        }
+                    }
+                });
+            }
+
+            Settings indexSettings = indexMetaData.getSettings();
+
+            Map<String, Settings> analyzerSettings = indexSettings.getGroups("index.analysis.analyzer");
+            usedBuiltInAnalyzers.removeAll(analyzerSettings.keySet());
+            for (Settings analyzerSetting : analyzerSettings.values()) {
+                usedAnalyzers.add(analyzerSetting.get("type", "custom"));
+                usedBuiltInCharFilters.addAll(analyzerSetting.getAsList("char_filter"));
+                String tokenizer = analyzerSetting.get("tokenizer");
+                if (tokenizer != null) {
+                    usedBuiltInTokenizers.add(tokenizer);
+                }
+                usedBuiltInTokenFilters.addAll(analyzerSetting.getAsList("filter"));
+            }
+
+            Map<String, Settings> charFilterSettings = indexSettings.getGroups("index.analysis.char_filter");
+            usedBuiltInCharFilters.removeAll(charFilterSettings.keySet());
+            aggregateAnalysisTypes(charFilterSettings.values(), usedCharFilters);
+
+            Map<String, Settings> tokenizerSettings = indexSettings.getGroups("index.analysis.tokenizer");
+            usedBuiltInTokenizers.removeAll(tokenizerSettings.keySet());
+            aggregateAnalysisTypes(tokenizerSettings.values(), usedTokenizers);
+
+            Map<String, Settings> tokenFilterSettings = indexSettings.getGroups("index.analysis.filter");
+            usedBuiltInTokenFilters.removeAll(tokenFilterSettings.keySet());
+            aggregateAnalysisTypes(tokenFilterSettings.values(), usedTokenFilters);
+        }
+
+        listener.onResponse(new XPackUsageFeatureResponse(
+                new IndexFeatureSetUsage(usedFieldTypes,
+                        usedCharFilters, usedTokenizers, usedTokenFilters, usedAnalyzers,
+                        usedBuiltInCharFilters, usedBuiltInTokenizers, usedBuiltInTokenFilters, usedBuiltInAnalyzers)));
+    }
+
+    static void visitMapping(Map<String, ?> mapping, Consumer<Map<String, ?>> fieldMappingConsumer) {
+        Object properties = mapping.get("properties");
+        if (properties != null && properties instanceof Map) {
+            @SuppressWarnings("unchecked")
+            Map<String, ?> propertiesAsMap = (Map<String, ?>) properties;
+            for (Object v : propertiesAsMap.values()) {
+                if (v != null && v instanceof Map) {
+
+                    @SuppressWarnings("unchecked")
+                    Map<String, ?> fieldMapping = (Map<String, ?>) v;
+                    fieldMappingConsumer.accept(fieldMapping);
+                    visitMapping(fieldMapping, fieldMappingConsumer);
+
+                    // Multi fields
+                    Object fieldsO = fieldMapping.get("fields");
+                    if (fieldsO != null && fieldsO instanceof Map) {
+                        @SuppressWarnings("unchecked")
+                        Map<String, ?> fields = (Map<String, ?>) fieldsO;
+                        for (Object v2 : fields.values()) {
+                            if (v2 instanceof Map) {
+                                @SuppressWarnings("unchecked")
+                                Map<String, ?> fieldMapping2 = (Map<String, ?>) v2;
+                                fieldMappingConsumer.accept(fieldMapping2);
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+    static void aggregateAnalysisTypes(Collection<Settings> analysisComponents, Set<String> usedTypes) {
+        for (Settings settings : analysisComponents) {
+            String type = settings.get("type");
+            if (type != null) {
+                usedTypes.add(type);
+            }
+        }
+    }
+}