x-pack/plugin/apm: introduce x-pack-apm plugin

.github/CODEOWNERS

@@ -22,3 +22,6 @@ x-pack/plugin/core/src/main/resources/fleet-* @elastic/fleet
 
 # Kibana Security
 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/KibanaOwnedReservedRoleDescriptors.java @elastic/kibana-security
+
+# APM
+x-pack/plugin/apm-data @elastic/apm-server

test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java

@@ -1882,6 +1882,13 @@ protected static boolean isXPackTemplate(String name) {
             // We have a naming convention that internal component templates contain `@`. See also index-templates.asciidoc.
             return true;
         }
+        if (name.startsWith("apm@")
+            || name.startsWith("apm-")
+            || name.startsWith("traces-apm")
+            || name.startsWith("metrics-apm")
+            || name.startsWith("logs-apm")) {
+            return true;
+        }
         switch (name) {
             case ".watches":
             case "security_audit_log":

x-pack/plugin/apm-data/README.md

@@ -0,0 +1,33 @@
+## APM Data plugin
+
+The APM data plugin installs index templates, component templates, and ingest pipelines for Elastic APM.
+
+All resources are defined as YAML under [src/main/resources](src/main/resources).
+
+The APM index templates rely on mappings from `x-pack-core`.
+See [x-pack/plugin/core/src/main/resources](../core/src/main/resources).
+
+This plugin is intended to work with data produced by https://github.com/elastic/apm-data.
+
+## Testing
+
+## Unit testing
+
+Java unit tests cover basic, low-level details of the plugin, such as the parsing and loading of resources.
+These can be run with:
+
+```
+./gradlew x-pack:plugin:apm-data:test
+```
+
+## Integration testing
+
+The index templates and ingest pipeline functionality is tested using YAML REST tests.
+These can be run with:
+
+```
+./gradlew x-pack:plugin:apm-data:yamlRestTest
+```
+
+Refer to the [rest-api-spec documentation](../../../rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/README.asciidoc)
+for information about writing YAML REST tests.

x-pack/plugin/apm-data/build.gradle

@@ -0,0 +1,34 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+apply plugin: 'elasticsearch.internal-es-plugin'
+apply plugin: 'elasticsearch.internal-yaml-rest-test'
+apply plugin: 'elasticsearch.internal-cluster-test'
+
+esplugin {
+  name 'x-pack-apm-data'
+  description 'The APM plugin defines APM data streams and ingest pipelines.'
+  classname 'org.elasticsearch.xpack.apmdata.APMPlugin'
+  extendedPlugins = ['x-pack-core']
+}
+
+dependencies {
+  compileOnly project(path: xpackModule('core'))
+  testImplementation project(path: ':x-pack:plugin:stack')
+  testImplementation(testArtifact(project(xpackModule('core'))))
+  clusterModules project(':modules:data-streams')
+  clusterModules project(':modules:ingest-common')
+  clusterModules project(':modules:ingest-geoip')
+  clusterModules project(':modules:ingest-user-agent')
+  clusterModules project(':modules:mapper-extras')
+  clusterModules project(xpackModule('analytics'))
+  clusterModules project(xpackModule('ilm'))
+  clusterModules project(xpackModule('mapper-aggregate-metric'))
+  clusterModules project(xpackModule('mapper-constant-keyword'))
+  clusterModules project(xpackModule('stack'))
+  clusterModules project(xpackModule('wildcard'))
+}

x-pack/plugin/apm-data/src/main/java/org/elasticsearch/xpack/apmdata/APMIndexTemplateRegistry.java

@@ -0,0 +1,169 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.apmdata;
+
+import org.elasticsearch.client.internal.Client;
+import org.elasticsearch.cluster.metadata.ComponentTemplate;
+import org.elasticsearch.cluster.metadata.ComposableIndexTemplate;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.core.Nullable;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.xcontent.NamedXContentRegistry;
+import org.elasticsearch.xcontent.XContentParserConfiguration;
+import org.elasticsearch.xcontent.yaml.YamlXContent;
+import org.elasticsearch.xpack.core.ClientHelper;
+import org.elasticsearch.xpack.core.XPackSettings;
+import org.elasticsearch.xpack.core.template.IndexTemplateRegistry;
+import org.elasticsearch.xpack.core.template.IngestPipelineConfig;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+import static org.elasticsearch.xpack.apmdata.ResourceUtils.APM_TEMPLATE_VERSION_VARIABLE;
+import static org.elasticsearch.xpack.apmdata.ResourceUtils.loadResource;
+import static org.elasticsearch.xpack.apmdata.ResourceUtils.loadVersionedResourceUTF8;
+
+/**
+ * Creates all index templates and ingest pipelines that are required for using Elastic APM.
+ */
+public class APMIndexTemplateRegistry extends IndexTemplateRegistry {
+    private final int version;
+
+    private final Map<String, ComponentTemplate> componentTemplates;
+    private final Map<String, ComposableIndexTemplate> composableIndexTemplates;
+    private final List<IngestPipelineConfig> ingestPipelines;
+    private final boolean enabled;
+
+    @SuppressWarnings("unchecked")
+    public APMIndexTemplateRegistry(
+        Settings nodeSettings,
+        ClusterService clusterService,
+        ThreadPool threadPool,
+        Client client,
+        NamedXContentRegistry xContentRegistry
+    ) {
+        super(nodeSettings, clusterService, threadPool, client, xContentRegistry);
+
+        try {
+            final Map<String, Object> apmResources = XContentHelper.convertToMap(
+                YamlXContent.yamlXContent,
+                loadResource("/resources.yaml"),
+                false
+            );
+            version = (((Number) apmResources.get("version")).intValue());
+            final List<Object> componentTemplateNames = (List<Object>) apmResources.get("component-templates");
+            final List<Object> indexTemplateNames = (List<Object>) apmResources.get("index-templates");
+            final List<Object> ingestPipelineConfigs = (List<Object>) apmResources.get("ingest-pipelines");
+
+            componentTemplates = componentTemplateNames.stream()
+                .map(o -> (String) o)
+                .collect(Collectors.toMap(name -> name, name -> loadComponentTemplate(name, version)));
+            composableIndexTemplates = indexTemplateNames.stream()
+                .map(o -> (String) o)
+                .collect(Collectors.toMap(name -> name, name -> loadIndexTemplate(name, version)));
+            ingestPipelines = ingestPipelineConfigs.stream().map(o -> (Map<String, Map<String, Object>>) o).map(map -> {
+                Map.Entry<String, Map<String, Object>> pipelineConfig = map.entrySet().iterator().next();
+                return loadIngestPipeline(pipelineConfig.getKey(), version, (List<String>) pipelineConfig.getValue().get("dependencies"));
+            }).collect(Collectors.toList());
+
+            enabled = XPackSettings.APM_DATA_ENABLED.get(nodeSettings);
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    public int getVersion() {
+        return version;
+    }
+
+    public boolean isEnabled() {
+        return enabled;
+    }
+
+    public void close() {
+        clusterService.removeListener(this);
+    }
+
+    @Override
+    protected String getOrigin() {
+        return ClientHelper.APM_ORIGIN;
+    }
+
+    @Override
+    protected boolean requiresMasterNode() {
+        return true;
+    }
+
+    @Override
+    protected Map<String, ComponentTemplate> getComponentTemplateConfigs() {
+        if (enabled) {
+            return componentTemplates;
+        } else {
+            return Map.of();
+        }
+    }
+
+    @Override
+    protected Map<String, ComposableIndexTemplate> getComposableTemplateConfigs() {
+        if (enabled) {
+            return composableIndexTemplates;
+        } else {
+            return Map.of();
+        }
+    }
+
+    @Override
+    protected List<IngestPipelineConfig> getIngestPipelines() {
+        if (enabled) {
+            return ingestPipelines;
+        } else {
+            return Collections.emptyList();
+        }
+    }
+
+    private static ComponentTemplate loadComponentTemplate(String name, int version) {
+        try {
+            final byte[] content = loadVersionedResourceUTF8("/component-templates/" + name + ".yaml", version);
+            return ComponentTemplate.parse(YamlXContent.yamlXContent.createParser(XContentParserConfiguration.EMPTY, content));
+        } catch (Exception e) {
+            throw new RuntimeException("failed to load APM component template: " + name, e);
+        }
+    }
+
+    private static ComposableIndexTemplate loadIndexTemplate(String name, int version) {
+        try {
+            final byte[] content = loadVersionedResourceUTF8("/index-templates/" + name + ".yaml", version);
+            return ComposableIndexTemplate.parse(YamlXContent.yamlXContent.createParser(XContentParserConfiguration.EMPTY, content));
+        } catch (Exception e) {
+            throw new RuntimeException("failed to load APM index template: " + name, e);
+        }
+    }
+
+    private static IngestPipelineConfig loadIngestPipeline(String name, int version, @Nullable List<String> dependencies) {
+        if (dependencies == null) {
+            dependencies = Collections.emptyList();
+        }
+        return new YamlIngestPipelineConfig(
+            name,
+            "/ingest-pipelines/" + name + ".yaml",
+            version,
+            APM_TEMPLATE_VERSION_VARIABLE,
+            dependencies
+        );
+    }
+
+    @Override
+    protected boolean applyRolloverAfterTemplateV2Upgrade() {
+        return true;
+    }
+}

x-pack/plugin/apm-data/src/main/java/org/elasticsearch/xpack/apmdata/APMPlugin.java

@@ -0,0 +1,73 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.apmdata;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.apache.lucene.util.SetOnce;
+import org.elasticsearch.client.internal.Client;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.routing.allocation.AllocationService;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.env.NodeEnvironment;
+import org.elasticsearch.indices.IndicesService;
+import org.elasticsearch.plugins.ActionPlugin;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.repositories.RepositoriesService;
+import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.telemetry.TelemetryProvider;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.watcher.ResourceWatcherService;
+import org.elasticsearch.xcontent.NamedXContentRegistry;
+
+import java.util.Collection;
+import java.util.List;
+import java.util.function.Supplier;
+
+public class APMPlugin extends Plugin implements ActionPlugin {
+    private static final Logger logger = LogManager.getLogger(APMPlugin.class);
+    private final Settings settings;
+
+    private final SetOnce<APMIndexTemplateRegistry> registry = new SetOnce<>();
+
+    public APMPlugin(Settings settings) {
+        this.settings = settings;
+    }
+
+    @Override
+    public Collection<Object> createComponents(
+        Client client,
+        ClusterService clusterService,
+        ThreadPool threadPool,
+        ResourceWatcherService resourceWatcherService,
+        ScriptService scriptService,
+        NamedXContentRegistry xContentRegistry,
+        Environment environment,
+        NodeEnvironment nodeEnvironment,
+        NamedWriteableRegistry namedWriteableRegistry,
+        IndexNameExpressionResolver indexNameExpressionResolver,
+        Supplier<RepositoriesService> repositoriesServiceSupplier,
+        TelemetryProvider telemetryProvider,
+        AllocationService allocationService,
+        IndicesService indicesService
+    ) {
+        registry.set(new APMIndexTemplateRegistry(settings, clusterService, threadPool, client, xContentRegistry));
+        APMIndexTemplateRegistry registryInstance = registry.get();
+        logger.info("APM is {}", registryInstance.isEnabled() ? "enabled" : "disabled");
+        registryInstance.initialize();
+        return List.of(registryInstance);
+    }
+
+    @Override
+    public void close() {
+        registry.get().close();
+    }
+}

x-pack/plugin/apm-data/src/main/java/org/elasticsearch/xpack/apmdata/ResourceUtils.java

@@ -0,0 +1,38 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.apmdata;
+
+import org.elasticsearch.common.io.Streams;
+import org.elasticsearch.xpack.core.template.TemplateUtils;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+
+public class ResourceUtils {
+
+    public static final String APM_TEMPLATE_VERSION_VARIABLE = "xpack.apmdata.template.version";
+
+    static byte[] loadVersionedResourceUTF8(String name, int version) {
+        try {
+            String content = loadResource(name);
+            content = TemplateUtils.replaceVariable(content, APM_TEMPLATE_VERSION_VARIABLE, String.valueOf(version));
+            return content.getBytes(StandardCharsets.UTF_8);
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    static String loadResource(String name) throws IOException {
+        InputStream is = APMIndexTemplateRegistry.class.getResourceAsStream(name);
+        if (is == null) {
+            throw new IOException("Resource [" + name + "] not found in classpath.");
+        }
+        return Streams.readFully(is).utf8ToString();
+    }
+}