x-pack/plugin/apm: introduce x-pack-apm plugin

.github/CODEOWNERS

@@ -19,3 +19,6 @@ x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/Monito
 # Fleet
 x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet @elastic/fleet
 x-pack/plugin/core/src/main/resources/fleet-* @elastic/fleet
+
+# APM
+x-pack/plugin/apm @elastic/apm-server

test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java

@@ -1828,6 +1828,13 @@ protected static boolean isXPackTemplate(String name) {
         if (name.startsWith("profiling-")) {
             return true;
         }
+        if (name.equals("apm")
+            || name.startsWith("apm-")
+            || name.startsWith("traces-apm")
+            || name.startsWith("metrics-apm")
+            || name.startsWith("logs-apm")) {
+            return true;
+        }
         switch (name) {
             case ".watches":
             case "security_audit_log":

x-pack/plugin/apm/README.md

@@ -0,0 +1,31 @@
+## APM plugin
+
+The APM plugin installs index templates, component templates, and ingest pipelines for Elastic APM.
+
+All resources are defined as YAML under [src/main/resources](src/main/resources).
+
+The APM index templates rely on mappings from `x-pack-core`.
+See [x-pack/plugin/core/src/main/resources](../core/src/main/resources).
+
+## Testing
+
+## Unit testing
+
+Java unit tests cover basic, low-level details of the plugin, such as the parsing and loading of resources.
+These can be run with:
+
+```
+./gradlew x-pack:plugin:apm:test
+```
+
+## Integration testing
+
+The index templates and ingest pipeline functionality is tested using YAML REST tests.
+These can be run with:
+
+```
+./gradlew x-pack:plugin:apm:yamlRestTest
+```
+
+Refer to the [rest-api-spec documentation](../../../rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/README.asciidoc)
+for information about writing YAML REST tests.

x-pack/plugin/apm/build.gradle

@@ -0,0 +1,32 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+apply plugin: 'elasticsearch.internal-es-plugin'
+apply plugin: 'elasticsearch.internal-yaml-rest-test'
+
+esplugin {
+  name 'x-pack-apm'
+  description 'The APM plugin defines APM data streams and ingest pipelines.'
+  classname 'org.elasticsearch.xpack.apm.APMPlugin'
+  extendedPlugins = ['x-pack-core']
+}
+
+dependencies {
+  compileOnly project(path: xpackModule('core'))
+  testImplementation(testArtifact(project(xpackModule('core'))))
+  clusterModules project(':modules:data-streams')
+  clusterModules project(':modules:ingest-common')
+  clusterModules project(':modules:ingest-geoip')
+  clusterModules project(':modules:ingest-user-agent')
+  clusterModules project(':modules:mapper-extras')
+  clusterModules project(xpackModule('analytics'))
+  clusterModules project(xpackModule('ilm'))
+  clusterModules project(xpackModule('mapper-aggregate-metric'))
+  clusterModules project(xpackModule('mapper-constant-keyword'))
+  clusterModules project(xpackModule('stack'))
+  clusterModules project(xpackModule('wildcard'))
+}

x-pack/plugin/apm/src/main/java/org/elasticsearch/xpack/apm/APMIndexTemplateRegistry.java

@@ -0,0 +1,168 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.apm;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.action.ingest.PutPipelineRequest;
+import org.elasticsearch.client.internal.Client;
+import org.elasticsearch.cluster.metadata.ComponentTemplate;
+import org.elasticsearch.cluster.metadata.ComposableIndexTemplate;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.io.Streams;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.xcontent.NamedXContentRegistry;
+import org.elasticsearch.xcontent.XContentParserConfiguration;
+import org.elasticsearch.xcontent.XContentType;
+import org.elasticsearch.xcontent.yaml.YamlXContent;
+import org.elasticsearch.xpack.core.ClientHelper;
+import org.elasticsearch.xpack.core.template.IndexTemplateRegistry;
+import org.elasticsearch.xpack.core.template.IngestPipelineConfig;
+import org.elasticsearch.xpack.core.template.TemplateUtils;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+/**
+ * Creates all index templates and ingest pipelines that are required for using Elastic APM.
+ */
+public class APMIndexTemplateRegistry extends IndexTemplateRegistry {
+    private static final Logger logger = LogManager.getLogger(APMIndexTemplateRegistry.class);
+
+    // TODO(axw) when the template version is updated, data streams using the old
+    // version should be rolled over after upgrading. Ideally this would be done
+    // automatically: https://github.com/elastic/elasticsearch/issues/96521
+
+    public static final String APM_TEMPLATE_VERSION_VARIABLE = "xpack.apm.template.version";
+
+    private final Map<String, ComponentTemplate> componentTemplates;
+    private final Map<String, ComposableIndexTemplate> composableIndexTemplates;
+    private final List<IngestPipelineConfig> ingestPipelines;
+
+    @SuppressWarnings("unchecked")
+    public APMIndexTemplateRegistry(
+        Settings nodeSettings,
+        ClusterService clusterService,
+        ThreadPool threadPool,
+        Client client,
+        NamedXContentRegistry xContentRegistry
+    ) {
+        super(nodeSettings, clusterService, threadPool, client, xContentRegistry);
+
+        try {
+            final Map<String, Object> apmResources = XContentHelper.convertToMap(
+                YamlXContent.yamlXContent,
+                loadResource("/resources.yaml"),
+                false
+            );
+            final int version = ((Number) apmResources.get("version")).intValue();
+            final List<Object> componentTemplateNames = (List<Object>) apmResources.get("component-templates");
+            final List<Object> indexTemplateNames = (List<Object>) apmResources.get("index-templates");
+            final List<Object> ingestPipelineNames = (List<Object>) apmResources.get("ingest-pipelines");
+
+            componentTemplates = componentTemplateNames.stream()
+                .map(o -> (String) o)
+                .collect(Collectors.toMap(name -> name, name -> loadComponentTemplate(name, version)));
+            composableIndexTemplates = indexTemplateNames.stream()
+                .map(o -> (String) o)
+                .collect(Collectors.toMap(name -> name, name -> loadIndexTemplate(name, version)));
+            ingestPipelines = ingestPipelineNames.stream()
+                .map(o -> (String) o)
+                .map(name -> loadIngestPipeline(name, version))
+                .collect(Collectors.toList());
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    public void close() {
+        clusterService.removeListener(this);
+    }
+
+    @Override
+    protected String getOrigin() {
+        return ClientHelper.APM_ORIGIN;
+    }
+
+    @Override
+    protected boolean requiresMasterNode() {
+        return true;
+    }
+
+    @Override
+    protected Map<String, ComponentTemplate> getComponentTemplateConfigs() {
+        return componentTemplates;
+    }
+
+    @Override
+    protected Map<String, ComposableIndexTemplate> getComposableTemplateConfigs() {
+        return composableIndexTemplates;
+    }
+
+    @Override
+    protected List<IngestPipelineConfig> getIngestPipelines() {
+        return ingestPipelines;
+    }
+
+    private static ComponentTemplate loadComponentTemplate(String name, int version) {
+        try {
+            final byte[] content = loadVersionedResourceUTF8("/component-templates/" + name + ".yaml", version);
+            return ComponentTemplate.parse(YamlXContent.yamlXContent.createParser(XContentParserConfiguration.EMPTY, content));
+        } catch (Exception e) {
+            throw new RuntimeException("failed to load APM component template: " + name, e);
+        }
+    }
+
+    private static ComposableIndexTemplate loadIndexTemplate(String name, int version) {
+        try {
+            final byte[] content = loadVersionedResourceUTF8("/index-templates/" + name + ".yaml", version);
+            return ComposableIndexTemplate.parse(YamlXContent.yamlXContent.createParser(XContentParserConfiguration.EMPTY, content));
+        } catch (Exception e) {
+            throw new RuntimeException("failed to load APM index template: " + name, e);
+        }
+    }
+
+    private static IngestPipelineConfig loadIngestPipeline(String name, int version) {
+        return new IngestPipelineConfig(
+            name,
+            version,
+            Collections.emptyList(),
+            () -> new PutPipelineRequest(
+                name,
+                new BytesArray(loadVersionedResourceUTF8("/ingest-pipelines/" + name + ".yaml", version)),
+                XContentType.YAML
+            )
+        );
+    }
+
+    private static byte[] loadVersionedResourceUTF8(String name, int version) {
+        try {
+            String content = loadResource(name);
+            content = TemplateUtils.replaceVariable(content, APM_TEMPLATE_VERSION_VARIABLE, String.valueOf(version));
+            return content.getBytes(StandardCharsets.UTF_8);
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    private static String loadResource(String name) throws IOException {
+        InputStream is = APMIndexTemplateRegistry.class.getResourceAsStream(name);
+        if (is == null) {
+            throw new IOException("Resource [" + name + "] not found in classpath.");
+        }
+        return Streams.readFully(is).utf8ToString();
+    }
+}

x-pack/plugin/apm/src/main/java/org/elasticsearch/xpack/apm/APMPlugin.java

@@ -0,0 +1,80 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.apm;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.apache.lucene.util.SetOnce;
+import org.elasticsearch.client.internal.Client;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.routing.allocation.AllocationService;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.env.NodeEnvironment;
+import org.elasticsearch.indices.IndicesService;
+import org.elasticsearch.plugins.ActionPlugin;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.repositories.RepositoriesService;
+import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.tracing.Tracer;
+import org.elasticsearch.watcher.ResourceWatcherService;
+import org.elasticsearch.xcontent.NamedXContentRegistry;
+import org.elasticsearch.xpack.core.XPackSettings;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.function.Supplier;
+
+public class APMPlugin extends Plugin implements ActionPlugin {
+    private static final Logger logger = LogManager.getLogger(APMPlugin.class);
+    private final Settings settings;
+    private final boolean enabled;
+
+    private final SetOnce<APMIndexTemplateRegistry> registry = new SetOnce<>();
+
+    public APMPlugin(Settings settings) {
+        this.settings = settings;
+        this.enabled = XPackSettings.PROFILING_ENABLED.get(settings);
+    }
+
+    @Override
+    public Collection<Object> createComponents(
+        Client client,
+        ClusterService clusterService,
+        ThreadPool threadPool,
+        ResourceWatcherService resourceWatcherService,
+        ScriptService scriptService,
+        NamedXContentRegistry xContentRegistry,
+        Environment environment,
+        NodeEnvironment nodeEnvironment,
+        NamedWriteableRegistry namedWriteableRegistry,
+        IndexNameExpressionResolver indexNameExpressionResolver,
+        Supplier<RepositoriesService> repositoriesServiceSupplier,
+        Tracer tracer,
+        AllocationService allocationService,
+        IndicesService indicesService
+    ) {
+        logger.info("APM is {}", enabled ? "enabled" : "disabled");
+        registry.set(new APMIndexTemplateRegistry(settings, clusterService, threadPool, client, xContentRegistry));
+        if (enabled) {
+            registry.get().initialize();
+            return List.of(registry.get());
+        } else {
+            return Collections.emptyList();
+        }
+    }
+
+    @Override
+    public void close() {
+        registry.get().close();
+    }
+}

x-pack/plugin/apm/src/main/resources/component-templates/apm-retention-10d.yaml

@@ -0,0 +1,5 @@
+---
+version: ${xpack.apm.template.version}
+template:
+  lifecycle:
+    data_retention: "10d"

x-pack/plugin/apm/src/main/resources/component-templates/apm-retention-180d.yaml

@@ -0,0 +1,5 @@
+---
+version: ${xpack.apm.template.version}
+template:
+  lifecycle:
+    data_retention: "180d"

x-pack/plugin/apm/src/main/resources/component-templates/apm-retention-390d.yaml

@@ -0,0 +1,5 @@
+---
+version: ${xpack.apm.template.version}
+template:
+  lifecycle:
+    data_retention: "390d"

x-pack/plugin/apm/src/main/resources/component-templates/apm-retention-90d.yaml

@@ -0,0 +1,5 @@
+---
+version: ${xpack.apm.template.version}
+template:
+  lifecycle:
+    data_retention: "90d"

x-pack/plugin/apm/src/main/resources/component-templates/apm.yaml

@@ -0,0 +1,23 @@
+version: ${xpack.apm.template.version}
+template:
+  settings:
+    index:
+      default_pipeline: apm
+      sort:
+        field: "@timestamp"
+        order: desc
+  mappings:
+    dynamic: true
+    dynamic_templates:
+    - numeric_labels:
+        path_match: numeric_labels.*
+        mapping:
+          type: scaled_float
+          scaling_factor: 1000000
+    properties:
+      data_stream.type:
+        type: constant_keyword
+      data_stream.dataset:
+        type: constant_keyword
+      data_stream.namespace:
+        type: constant_keyword