[Backport 2.x] Index template cleanup

src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java

@@ -44,6 +44,7 @@
 import org.opensearch.securityanalytics.action.UpdateIndexMappingsAction;
 import org.opensearch.securityanalytics.indexmanagment.DetectorIndexManagementService;
 import org.opensearch.securityanalytics.action.ValidateRulesAction;
+import org.opensearch.securityanalytics.mapper.IndexTemplateManager;
 import org.opensearch.securityanalytics.mapper.MapperService;
 import org.opensearch.securityanalytics.resthandler.RestAcknowledgeAlertsAction;
 import org.opensearch.securityanalytics.resthandler.RestGetAllRuleCategoriesAction;
@@ -110,6 +111,8 @@ public class SecurityAnalyticsPlugin extends Plugin implements ActionPlugin {
 
     private DetectorIndexManagementService detectorIndexManagementService;
 
+    private IndexTemplateManager indexTemplateManager;
+
     @Override
     public Collection<Object> createComponents(Client client,
                                                ClusterService clusterService,
@@ -124,9 +127,11 @@ public Collection<Object> createComponents(Client client,
                                                Supplier<RepositoriesService> repositoriesServiceSupplier) {
         detectorIndices = new DetectorIndices(client.admin(), clusterService, threadPool);
         ruleTopicIndices = new RuleTopicIndices(client, clusterService);
-        mapperService = new MapperService(client.admin().indices(), clusterService, indexNameExpressionResolver);
+        indexTemplateManager = new IndexTemplateManager(client, clusterService, indexNameExpressionResolver, xContentRegistry);
+        mapperService = new MapperService(client, clusterService, indexNameExpressionResolver, indexTemplateManager);
         ruleIndices = new RuleIndices(client, clusterService, threadPool);
-        return List.of(detectorIndices, ruleTopicIndices, ruleIndices, mapperService);
+
+        return List.of(detectorIndices, ruleTopicIndices, ruleIndices, mapperService, indexTemplateManager);
     }
 
     @Override

src/main/java/org/opensearch/securityanalytics/mapper/IndexTemplateUtils.java

@@ -0,0 +1,115 @@
+/*
+Copyright OpenSearch Contributors
+SPDX-License-Identifier: Apache-2.0
+ */
+
+package org.opensearch.securityanalytics.mapper;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import org.opensearch.cluster.ClusterState;
+import org.opensearch.cluster.metadata.ComposableIndexTemplate;
+import org.opensearch.cluster.metadata.Template;
+import org.opensearch.common.bytes.BytesReference;
+import org.opensearch.common.compress.CompressedXContent;
+import org.opensearch.common.xcontent.XContentBuilder;
+import org.opensearch.common.xcontent.XContentFactory;
+import org.opensearch.common.xcontent.XContentHelper;
+import org.opensearch.common.xcontent.XContentType;
+
+
+import static org.opensearch.index.mapper.MapperService.SINGLE_MAPPING_NAME;
+import static org.opensearch.securityanalytics.mapper.IndexTemplateManager.OPENSEARCH_SAP_COMPONENT_TEMPLATE_PREFIX;
+import static org.opensearch.securityanalytics.mapper.IndexTemplateManager.OPENSEARCH_SAP_INDEX_TEMPLATE_PREFIX;
+
+public class IndexTemplateUtils {
+
+
+    public static Set<String> getAllSapComponentTemplates(ClusterState state) {
+        Set<String> componentTemplates = new HashSet<>();
+
+        state.metadata().componentTemplates().forEach( (name, instance) -> {
+            if (name.startsWith(OPENSEARCH_SAP_COMPONENT_TEMPLATE_PREFIX)) {
+                componentTemplates.add(name);
+            }
+        });
+        return componentTemplates;
+    }
+
+    public static boolean isSapComposableIndexTemplate(String templateName, ComposableIndexTemplate template) {
+        // We don't ever set template field inside ComposableIndexTemplate and our template always starts with OPENSEARCH_SAP_INDEX_TEMPLATE_PREFIX
+        // If any of these is true, that means that user touched template and we're not owner of it anymore
+        if (templateName.startsWith(OPENSEARCH_SAP_INDEX_TEMPLATE_PREFIX) == false || template.template() != null) {
+            return false;
+        }
+        // If user added ComponentTemplate then this ComposableIndexTemplate is owned by user
+        for (String componentTemplate : template.composedOf()) {
+            if (componentTemplate.startsWith(OPENSEARCH_SAP_COMPONENT_TEMPLATE_PREFIX) == false) {
+                return false;
+            }
+        }
+        return true;
+    }
+
+    public static Map<String, ComposableIndexTemplate> getAllSapComposableIndexTemplates(ClusterState state) {
+        Map<String, ComposableIndexTemplate> sapTemplates = new HashMap<>();
+
+        state.metadata().templatesV2().forEach( (name, instance) -> {
+            if (isSapComposableIndexTemplate(name, instance)) {
+                sapTemplates.put(name, instance);
+            }
+        });
+        return sapTemplates;
+    }
+
+    public static String computeIndexTemplateName(String indexName) {
+        return OPENSEARCH_SAP_INDEX_TEMPLATE_PREFIX + normalizeIndexName(indexName);
+    }
+
+    public static String computeComponentTemplateName(String indexName) {
+        if (indexName.endsWith("*")) {
+            indexName = indexName.substring(0, indexName.length() - 1);
+        }
+        return OPENSEARCH_SAP_COMPONENT_TEMPLATE_PREFIX + normalizeIndexName(indexName);
+    }
+
+    public static String normalizeIndexName(String indexName) {
+        if (indexName.endsWith("*")) {
+            return indexName.substring(0, indexName.length() - 1);
+        } else {
+            return indexName;
+        }
+    }
+
+    public static boolean isUserCreatedComposableTemplate(String templateName) {
+        return templateName.startsWith(OPENSEARCH_SAP_INDEX_TEMPLATE_PREFIX) == false;
+    }
+
+    public static Template copyTemplate(Template template) throws IOException {
+
+        if (template == null) {
+            return null;
+        }
+
+        CompressedXContent outMappings = null;
+        CompressedXContent mappings = template.mappings();
+        if (mappings != null) {
+            Map<String, Object> mappingsAsMap = XContentHelper.convertToMap(mappings.compressedReference(), true, XContentType.JSON).v2();
+            if (mappingsAsMap.containsKey(SINGLE_MAPPING_NAME)) {
+                mappingsAsMap = (Map<String, Object>)mappingsAsMap.get(SINGLE_MAPPING_NAME);
+            }
+            XContentBuilder builder = XContentFactory.jsonBuilder();
+            builder.map(mappingsAsMap);
+
+            outMappings = new CompressedXContent(BytesReference.bytes(builder));
+        }
+        return new Template(
+                template.settings(),
+                outMappings,
+                template.aliases()
+        );
+    }
+}

src/main/java/org/opensearch/securityanalytics/mapper/MapperService.java

@@ -27,6 +27,7 @@
 import org.opensearch.action.admin.indices.mapping.put.PutMappingRequest;
 import org.opensearch.action.support.GroupedActionListener;
 import org.opensearch.action.support.master.AcknowledgedResponse;
+import org.opensearch.client.Client;
 import org.opensearch.client.IndicesAdminClient;
 import org.opensearch.cluster.metadata.IndexNameExpressionResolver;
 import org.opensearch.cluster.metadata.MappingMetadata;
@@ -54,11 +55,11 @@ public class MapperService {
 
     public MapperService() {}
 
-    public MapperService(IndicesAdminClient indicesClient, ClusterService clusterService, IndexNameExpressionResolver indexNameExpressionResolver) {
-        this.indicesClient = indicesClient;
+    public MapperService(Client client, ClusterService clusterService, IndexNameExpressionResolver indexNameExpressionResolver, IndexTemplateManager indexTemplateManager) {
+        this.indicesClient = client.admin().indices();
         this.clusterService = clusterService;
         this.indexNameExpressionResolver = indexNameExpressionResolver;
-        indexTemplateManager = new IndexTemplateManager(indicesClient, clusterService, indexNameExpressionResolver);
+        this.indexTemplateManager = indexTemplateManager;
     }
 
     public void createMappingAction(String indexName, String ruleTopic, boolean partial, ActionListener<AcknowledgedResponse> actionListener) {
@@ -207,6 +208,7 @@ private void doCreateMapping(
             indicesClient.putMapping(request, new ActionListener<>() {
                 @Override
                 public void onResponse(AcknowledgedResponse acknowledgedResponse) {
+                    //((Map<String, Object>)mappingsRoot.get(PROPERTIES)).putAll(presentPathsMappings);
                     CreateMappingResult result = new CreateMappingResult(
                             acknowledgedResponse,
                             indexName,

src/main/java/org/opensearch/securityanalytics/transport/TransportDeleteDetectorAction.java

@@ -4,6 +4,11 @@
  */
 package org.opensearch.securityanalytics.transport;
 
+import java.util.Collection;
+import java.util.List;
+import java.util.Locale;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicReference;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.opensearch.common.SetOnce;
@@ -14,12 +19,10 @@
 import org.opensearch.action.delete.DeleteResponse;
 import org.opensearch.action.get.GetRequest;
 import org.opensearch.action.get.GetResponse;
-import org.opensearch.action.search.SearchResponse;
 import org.opensearch.action.support.ActionFilters;
 import org.opensearch.action.support.GroupedActionListener;
 import org.opensearch.action.support.HandledTransportAction;
 import org.opensearch.action.support.WriteRequest;
-import org.opensearch.action.support.master.AcknowledgedResponse;
 import org.opensearch.client.Client;
 import org.opensearch.client.node.NodeClient;
 import org.opensearch.common.inject.Inject;
@@ -35,19 +38,14 @@
 import org.opensearch.securityanalytics.action.DeleteDetectorAction;
 import org.opensearch.securityanalytics.action.DeleteDetectorRequest;
 import org.opensearch.securityanalytics.action.DeleteDetectorResponse;
+import org.opensearch.securityanalytics.mapper.IndexTemplateManager;
 import org.opensearch.securityanalytics.model.Detector;
 import org.opensearch.securityanalytics.util.RuleTopicIndices;
 import org.opensearch.securityanalytics.util.SecurityAnalyticsException;
 import org.opensearch.tasks.Task;
 import org.opensearch.threadpool.ThreadPool;
 import org.opensearch.transport.TransportService;
 
-import java.io.IOException;
-import java.util.Collection;
-import java.util.List;
-import java.util.Locale;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicReference;
 
 import static org.opensearch.securityanalytics.model.Detector.NO_VERSION;
 
@@ -63,13 +61,16 @@ public class TransportDeleteDetectorAction extends HandledTransportAction<Delete
 
     private final ThreadPool threadPool;
 
+    private final IndexTemplateManager indexTemplateManager;
+
     @Inject
-    public TransportDeleteDetectorAction(TransportService transportService, Client client, ActionFilters actionFilters, NamedXContentRegistry xContentRegistry, RuleTopicIndices ruleTopicIndices) {
+    public TransportDeleteDetectorAction(TransportService transportService, IndexTemplateManager indexTemplateManager, Client client, ActionFilters actionFilters, NamedXContentRegistry xContentRegistry, RuleTopicIndices ruleTopicIndices) {
         super(DeleteDetectorAction.NAME, transportService, actionFilters, DeleteDetectorRequest::new);
         this.client = client;
         this.ruleTopicIndices = ruleTopicIndices;
         this.xContentRegistry = xContentRegistry;
         this.threadPool = client.threadPool();
+        this.indexTemplateManager = indexTemplateManager;
     }
 
     @Override
@@ -173,9 +174,21 @@ private void deleteDetectorFromConfig(String detectorId, WriteRequest.RefreshPol
                     new ActionListener<>() {
                         @Override
                         public void onResponse(DeleteResponse response) {
-                            onOperation(response);
-                        }
 
+                            indexTemplateManager.deleteAllUnusedTemplates(new ActionListener<Void>() {
+                                @Override
+                                public void onResponse(Void unused) {
+                                    onOperation(response);
+                                }
+
+                                @Override
+                                public void onFailure(Exception e) {
+                                    log.error("Error deleting unused templates: " + e.getMessage());
+                                    onOperation(response);
+                                }
+                            });
+
+                        }
                         @Override
                         public void onFailure(Exception t) {
                             onFailures(t);

src/main/java/org/opensearch/securityanalytics/util/DetectorUtils.java

@@ -4,21 +4,24 @@
  */
 package org.opensearch.securityanalytics.util;
 
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Set;
+import org.opensearch.action.ActionListener;
+import org.opensearch.action.search.SearchRequest;
 import org.opensearch.action.search.SearchResponse;
-import org.opensearch.common.bytes.BytesReference;
+import org.opensearch.client.Client;
 import org.opensearch.common.xcontent.LoggingDeprecationHandler;
 import org.opensearch.common.xcontent.NamedXContentRegistry;
-import org.opensearch.common.xcontent.ToXContent;
-import org.opensearch.common.xcontent.XContentBuilder;
-import org.opensearch.common.xcontent.XContentFactory;
 import org.opensearch.common.xcontent.XContentParser;
 import org.opensearch.common.xcontent.XContentType;
 import org.opensearch.search.SearchHit;
+import org.opensearch.search.builder.SearchSourceBuilder;
+import org.opensearch.search.fetch.subphase.FetchSourceContext;
 import org.opensearch.securityanalytics.model.Detector;
-
-import java.io.IOException;
-import java.util.LinkedList;
-import java.util.List;
+import org.opensearch.securityanalytics.model.DetectorInput;
 
 public class DetectorUtils {
 
@@ -36,4 +39,39 @@ public static List<Detector> getDetectors(SearchResponse response, NamedXContent
         }
         return detectors;
     }
+
+    public static void getAllDetectorInputs(Client client, NamedXContentRegistry xContentRegistry, ActionListener<Set<String>> actionListener) {
+
+        SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
+        searchSourceBuilder.fetchSource(FetchSourceContext.FETCH_SOURCE);
+        searchSourceBuilder.seqNoAndPrimaryTerm(true);
+        searchSourceBuilder.version(true);
+
+        SearchRequest searchRequest = new SearchRequest();
+        searchRequest.source(searchSourceBuilder);
+        searchRequest.indices(Detector.DETECTORS_INDEX);
+
+        client.search(searchRequest, new ActionListener<>() {
+            @Override
+            public void onResponse(SearchResponse response) {
+                Set<String> allDetectorIndices = new HashSet<>();
+                try {
+                    List<Detector> detectors = DetectorUtils.getDetectors(response, xContentRegistry);
+                    for (Detector detector : detectors) {
+                        for (DetectorInput input : detector.getInputs()) {
+                            allDetectorIndices.addAll(input.getIndices());
+                        }
+                    }
+                } catch (IOException e) {
+                    actionListener.onFailure(e);
+                }
+                actionListener.onResponse(allDetectorIndices);
+            }
+
+            @Override
+            public void onFailure(Exception e) {
+                actionListener.onFailure(e);
+            }
+        });
+    }
 }

src/main/java/org/opensearch/securityanalytics/util/RuleTopicIndices.java

@@ -12,10 +12,12 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.opensearch.action.ActionListener;
-import org.opensearch.action.admin.indices.template.put.PutIndexTemplateRequest;
+import org.opensearch.action.admin.indices.template.put.PutComposableIndexTemplateAction;
 import org.opensearch.action.support.master.AcknowledgedResponse;
 import org.opensearch.client.Client;
 import org.opensearch.cluster.ClusterState;
+import org.opensearch.cluster.metadata.ComposableIndexTemplate;
+import org.opensearch.cluster.metadata.Template;
 import org.opensearch.cluster.service.ClusterService;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.xcontent.XContentType;
@@ -44,19 +46,36 @@ public void initRuleTopicIndexTemplate(ActionListener<AcknowledgedResponse> acti
             for(String ruleIndex : DetectorMonitorConfig.getAllRuleIndices()) {
                 indexPatterns.add(ruleIndex + "*");
             }
-            PutIndexTemplateRequest indexRequest =
-                    new PutIndexTemplateRequest(DetectorMonitorConfig.OPENSEARCH_SAP_RULE_INDEX_TEMPLATE)
-                    .patterns(indexPatterns)
-                    .settings(Settings.builder().loadFromSource(ruleTopicIndexSettings(), XContentType.JSON).build());
-            client.admin().indices().putTemplate(indexRequest, actionListener);
+
+            ComposableIndexTemplate template = new ComposableIndexTemplate(
+                    indexPatterns,
+                    new Template(
+                            Settings.builder().loadFromSource(ruleTopicIndexSettings(), XContentType.JSON).build(),
+                            null,
+                            null
+                    ),
+                    null,
+                    500L,
+                    null,
+                    null
+            );
+
+            client.execute(
+                    PutComposableIndexTemplateAction.INSTANCE,
+                    new PutComposableIndexTemplateAction.Request(DetectorMonitorConfig.OPENSEARCH_SAP_RULE_INDEX_TEMPLATE)
+                    .indexTemplate(template)
+                    .create(true),
+                    actionListener
+            );
+
         } else {
             actionListener.onResponse(new AcknowledgedResponse(true));
         }
     }
 
     public boolean ruleTopicIndexTemplateExists() {
         ClusterState clusterState = clusterService.state();
-        return clusterState.metadata().templates()
+        return clusterState.metadata().templatesV2()
                 .get(DetectorMonitorConfig.OPENSEARCH_SAP_RULE_INDEX_TEMPLATE) != null;
     }
 }