Add example plugins for mapping transformer and system ingest processor.

plugins/examples/mapping-transformer/build.gradle

@@ -0,0 +1,20 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ *
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+apply plugin: 'opensearch.opensearchplugin'
+apply plugin: 'opensearch.yaml-rest-test'
+
+opensearchplugin {
+  name = 'example-mapping-transformer'
+  description = 'An example plugin implementing a mapping transformer.'
+  classname = 'org.opensearch.example.mappingtransformer.ExampleMappingTransformerPlugin'
+  licenseFile = rootProject.file('licenses/APACHE-LICENSE-2.0.txt')
+  noticeFile = rootProject.file('NOTICE.txt')
+}

plugins/examples/mapping-transformer/src/main/java/org/opensearch/example/mappingtransformer/ExampleMappingTransformer.java

@@ -0,0 +1,77 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.example.mappingtransformer;
+
+import org.opensearch.core.action.ActionListener;
+import org.opensearch.index.mapper.MappingTransformer;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * This example mapping transformer will automatically add a text field to the mapping if it finds a
+ * mapping_transform_trigger_field.
+ */
+public class ExampleMappingTransformer implements MappingTransformer {
+    /**
+     * Constructs a new ExampleMappingTransformer
+     */
+    public ExampleMappingTransformer() {}
+
+    /**
+     * Name for doc. Actions like create index and legacy create/update index template will have the
+     * mapping properties under a _doc key.
+     */
+    public static final String DOC = "_doc";
+    /**
+     * Name for properties. An object field will define subfields as properties.
+     */
+    public static final String PROPERTIES = "properties";
+
+    /**
+     * The name of the trigger field. The trigger field will trigger the logic to transform the mapping.
+     */
+    public static final String TRIGGER_FIELD_NAME = "mapping_transform_trigger_field";
+    /**
+     * The name of the auto added field.
+     */
+    public static final String AUTO_ADDED_FIELD_NAME = "field_auto_added_by_example_mapping_transformer";
+
+    @Override
+    public void transform(Map<String, Object> mapping, TransformContext context, ActionListener<Void> listener) {
+        Map<String, Object> properties = getProperties(mapping);
+        if (properties.containsKey(TRIGGER_FIELD_NAME)) {
+            properties.put(AUTO_ADDED_FIELD_NAME, Map.of("type", "text"));
+        }
+        listener.onResponse(null);
+    }
+
+    /**
+     * Help extract the properties from a mapping
+     * @param mapping index mapping
+     * @return properties of the mapping
+     */
+    @SuppressWarnings("unchecked")
+    private Map<String, Object> getProperties(Map<String, Object> mapping) {
+        if (mapping == null) {
+            return new HashMap<>();
+        }
+
+        if (mapping.containsKey(DOC) && mapping.get(DOC) instanceof Map) {
+            Map<String, Object> doc = (Map<String, Object>) mapping.get(DOC);
+            if (doc.containsKey(PROPERTIES) && doc.get(PROPERTIES) instanceof Map) {
+                return (Map<String, Object>) doc.get(PROPERTIES);
+            }
+        } else if (mapping.containsKey(PROPERTIES) && mapping.get(PROPERTIES) instanceof Map) {
+            return (Map<String, Object>) mapping.get(PROPERTIES);
+        }
+
+        return new HashMap<>();
+    }
+}

plugins/examples/mapping-transformer/src/main/java/org/opensearch/example/mappingtransformer/ExampleMappingTransformerPlugin.java

@@ -0,0 +1,34 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.example.mappingtransformer;
+
+import org.opensearch.index.mapper.MappingTransformer;
+import org.opensearch.plugins.MapperPlugin;
+import org.opensearch.plugins.Plugin;
+
+import java.util.List;
+
+/**
+ * Example plugin that implements a custom mapping transformer.
+ */
+public class ExampleMappingTransformerPlugin extends Plugin implements MapperPlugin {
+
+    /**
+     * Constructs a new ExampleMappingTransformerPlugin
+     */
+    public ExampleMappingTransformerPlugin() {}
+
+    /**
+     * @return Available mapping transformer
+     */
+    @Override
+    public List<MappingTransformer> getMappingTransformers() {
+        return List.of(new ExampleMappingTransformer());
+    }
+}

plugins/examples/mapping-transformer/src/main/java/org/opensearch/example/mappingtransformer/package-info.java

@@ -0,0 +1,12 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/**
+ * Example classes demonstrating the use of a custom mapping transformer in a plugin.
+ */
+package org.opensearch.example.mappingtransformer;

plugins/examples/mapping-transformer/src/test/java/org/opensearch/example/mappingtransformer/ExampleMappingTransformerPluginTests.java

@@ -0,0 +1,24 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.example.mappingtransformer;
+
+import org.opensearch.index.mapper.MappingTransformer;
+import org.opensearch.test.OpenSearchTestCase;
+
+import java.util.List;
+
+public class ExampleMappingTransformerPluginTests extends OpenSearchTestCase {
+    private final ExampleMappingTransformerPlugin plugin = new ExampleMappingTransformerPlugin();
+
+    public void testGetMappingTransformers() {
+        List<MappingTransformer> mappingTransformerList = plugin.getMappingTransformers();
+
+        assertTrue("Should return an example mapping transformer.", mappingTransformerList.getFirst() instanceof ExampleMappingTransformer);
+    }
+}

plugins/examples/mapping-transformer/src/test/java/org/opensearch/example/mappingtransformer/ExampleMappingTransformerTests.java

@@ -0,0 +1,84 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.example.mappingtransformer;
+
+import org.opensearch.action.support.PlainActionFuture;
+import org.opensearch.test.OpenSearchTestCase;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.opensearch.example.mappingtransformer.ExampleMappingTransformer.AUTO_ADDED_FIELD_NAME;
+import static org.opensearch.example.mappingtransformer.ExampleMappingTransformer.DOC;
+import static org.opensearch.example.mappingtransformer.ExampleMappingTransformer.PROPERTIES;
+import static org.opensearch.example.mappingtransformer.ExampleMappingTransformer.TRIGGER_FIELD_NAME;
+
+public class ExampleMappingTransformerTests extends OpenSearchTestCase {
+    private final ExampleMappingTransformer transformer = new ExampleMappingTransformer();
+
+    public void testExampleMappingTransformer_whenMappingWithoutDocLayer() {
+        Map<String, Object> mapping = new HashMap<>();
+        Map<String, Object> properties = new HashMap<>();
+        mapping.put(PROPERTIES, properties);
+        properties.put("dummyField", Map.of("type", "text"));
+
+        transformer.transform(mapping, null, new PlainActionFuture<>());
+
+        // verify no field is auto-injected
+        assertFalse(
+            "No trigger field in the mapping so should not transform the mapping to inject the field.",
+            properties.containsKey(AUTO_ADDED_FIELD_NAME)
+        );
+
+        properties.put(TRIGGER_FIELD_NAME, Map.of("type", "text"));
+        transformer.transform(mapping, null, new PlainActionFuture<>());
+
+        // verify the field should be auto added to the mapping
+        assertTrue(
+            "The mapping has the trigger field so should transform the mapping to auto inject a field.",
+            properties.containsKey(AUTO_ADDED_FIELD_NAME)
+        );
+    }
+
+    public void testExampleMappingTransformer_whenMappingWithDocLayer() {
+        Map<String, Object> doc = new HashMap<>();
+        Map<String, Object> mapping = new HashMap<>();
+        Map<String, Object> properties = new HashMap<>();
+        doc.put(DOC, mapping);
+        mapping.put(PROPERTIES, properties);
+        properties.put("dummyField", Map.of("type", "text"));
+
+        transformer.transform(doc, null, new PlainActionFuture<>());
+
+        // verify no field is auto-injected
+        assertFalse(
+            "No trigger field in the mapping so should not transform the mapping to inject the field.",
+            properties.containsKey(AUTO_ADDED_FIELD_NAME)
+        );
+
+        properties.put(TRIGGER_FIELD_NAME, Map.of("type", "text"));
+        transformer.transform(mapping, null, new PlainActionFuture<>());
+
+        // verify the field should be auto added to the mapping
+        assertTrue(
+            "The mapping has the trigger field so should transform the mapping to auto inject a field.",
+            properties.containsKey(AUTO_ADDED_FIELD_NAME)
+        );
+    }
+
+    public void testExampleMappingTransformer_whenNoMapping_thenDoNothing() {
+        transformer.transform(null, null, new PlainActionFuture<>());
+    }
+
+    public void testExampleMappingTransformer_whenEmptyMapping_thenDoNothing() {
+        Map<String, Object> doc = new HashMap<>();
+        transformer.transform(doc, null, new PlainActionFuture<>());
+        assertEquals(new HashMap<>(), doc);
+    }
+}

plugins/examples/mapping-transformer/src/yamlRestTest/java/org/opensearch/example/mappingtransformer/ExampleMappingTransformerClientYamlTestSuiteIT.java

@@ -0,0 +1,27 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.example.mappingtransformer;
+
+import com.carrotsearch.randomizedtesting.annotations.Name;
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+
+import org.opensearch.test.rest.yaml.ClientYamlTestCandidate;
+import org.opensearch.test.rest.yaml.OpenSearchClientYamlSuiteTestCase;
+
+public class ExampleMappingTransformerClientYamlTestSuiteIT extends OpenSearchClientYamlSuiteTestCase {
+
+    public ExampleMappingTransformerClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
+        super(testCandidate);
+    }
+
+    @ParametersFactory
+    public static Iterable<Object[]> parameters() throws Exception {
+        return OpenSearchClientYamlSuiteTestCase.createParameters();
+    }
+}

plugins/examples/mapping-transformer/src/yamlRestTest/resources/rest-api-spec/test/example-mapping-transformer/10_basic.yml

@@ -0,0 +1,16 @@
+# Integration tests for the mapping transformer example plugin
+#
+"Plugin loaded":
+    - skip:
+        reason: "contains is a newly added assertion"
+        features: contains
+    - do:
+        cluster.state: {}
+
+    # Get cluster-manager node id
+    - set: { cluster_manager_node: cluster_manager }
+
+    - do:
+        nodes.info: {}
+
+    - contains:  { nodes.$cluster_manager.plugins: { name: example-mapping-transformer } }

plugins/examples/mapping-transformer/src/yamlRestTest/resources/rest-api-spec/test/example-mapping-transformer/20_mapping_transformer.yml

@@ -0,0 +1,83 @@
+---
+"Test auto-injected field on index creation":
+  - do:
+      indices.create:
+        index: test-index
+        body:
+          mappings:
+            properties:
+              mapping_transform_trigger_field:
+                type: keyword
+  - do:
+      indices.get_mapping:
+        index: test-index
+  - match:
+      test-index.mappings.properties.field_auto_added_by_example_mapping_transformer.type: text
+
+---
+"Test auto-injected field on mapping update":
+  - do:
+      indices.create:
+        index: test-index-update
+        body:
+          mappings:
+            properties:
+              dummy_field:
+                type: keyword
+  - do:
+      indices.put_mapping:
+        index: test-index-update
+        body:
+          properties:
+            mapping_transform_trigger_field:
+              type: keyword
+  - do:
+      indices.get_mapping:
+        index: test-index-update
+  - match:
+      test-index-update.mappings.properties.field_auto_added_by_example_mapping_transformer.type: text
+
+---
+"Test auto-injected field via index template":
+  - do:
+      indices.put_index_template:
+        name: example_template
+        body:
+          index_patterns: ["auto-template-*"]
+          template:
+            mappings:
+              properties:
+                mapping_transform_trigger_field:
+                  type: keyword
+  - do:
+      indices.create:
+        index: auto-template-1
+  - do:
+      indices.get_mapping:
+        index: auto-template-1
+  - match:
+      auto-template-1.mappings.properties.mapping_transform_trigger_field.type: keyword
+  - match:
+      auto-template-1.mappings.properties.field_auto_added_by_example_mapping_transformer.type: text
+
+---
+"Test auto-injected field via legacy create template API":
+  - do:
+      indices.put_template:
+        name: legacy_template
+        body:
+          index_patterns: ["legacy-*"]
+          mappings:
+            properties:
+              mapping_transform_trigger_field:
+                type: keyword
+  - do:
+      indices.create:
+        index: legacy-1
+  - do:
+      indices.get_mapping:
+        index: legacy-1
+  - match:
+      legacy-1.mappings.properties.mapping_transform_trigger_field.type: keyword
+  - match:
+      legacy-1.mappings.properties.field_auto_added_by_example_mapping_transformer.type: text