mapper topics fix; Index Rule category validation (opensearch-project…

…#123) * conflict resolve Signed-off-by: Petar Dzepina <petar.dzepina@vroom.com> * IT fixes Signed-off-by: Petar Dzepina <petar.dzepina@vroom.com> Signed-off-by: Petar Dzepina <petar.dzepina@vroom.com> Co-authored-by: Petar Dzepina <petar.dzepina@vroom.com>
jowg-amazon · Nov 10, 2022 · 4fba8fc · 4fba8fc
1 parent 098ad7b
commit 4fba8fc
Show file tree

Hide file tree

Showing 14 changed files with 275 additions and 4 deletions.
diff --git a/src/main/java/org/opensearch/securityanalytics/action/IndexRuleRequest.java b/src/main/java/org/opensearch/securityanalytics/action/IndexRuleRequest.java
@@ -4,6 +4,9 @@
  */
 package org.opensearch.securityanalytics.action;
 
+import java.util.Arrays;
+import java.util.Locale;
+import java.util.Optional;
 import org.opensearch.action.ActionRequest;
 import org.opensearch.action.ActionRequestValidationException;
 import org.opensearch.action.support.WriteRequest;
@@ -12,6 +15,10 @@
 import org.opensearch.rest.RestRequest;
 
 import java.io.IOException;
+import org.opensearch.securityanalytics.model.Detector;
+
+
+import static org.opensearch.action.ValidateActions.addValidationError;
 
 public class IndexRuleRequest extends ActionRequest {
 
@@ -74,7 +81,20 @@ public IndexRuleRequest(StreamInput sin) throws IOException {
 
     @Override
     public ActionRequestValidationException validate() {
-        return null;
+        ActionRequestValidationException validationException = null;
+
+        if (logType == null || logType.length() == 0) {
+            validationException = addValidationError("rule categoty is missing", validationException);
+        } else {
+            Optional<Detector.DetectorType> found =
+                    Arrays.stream(Detector.DetectorType.values())
+                            .filter(e -> e.getDetectorType().equals(logType))
+                            .findFirst();
+            if (found.isPresent() == false) {
+                validationException = addValidationError("Invalid rule category", validationException);
+            }
+        }
+        return validationException;
     }
 
     @Override

diff --git a/src/main/java/org/opensearch/securityanalytics/mapper/MapperService.java b/src/main/java/org/opensearch/securityanalytics/mapper/MapperService.java
@@ -5,9 +5,12 @@
 
 package org.opensearch.securityanalytics.mapper;
 
+import java.util.Locale;
 import org.apache.commons.lang3.tuple.Pair;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.opensearch.OpenSearchStatusException;
+import org.opensearch.ResourceNotFoundException;
 import org.opensearch.action.ActionListener;
 import org.opensearch.action.admin.indices.mapping.get.GetMappingsRequest;
 import org.opensearch.action.admin.indices.mapping.get.GetMappingsResponse;
@@ -17,6 +20,7 @@
 import org.opensearch.cluster.metadata.MappingMetadata;
 import org.opensearch.common.collect.ImmutableOpenMap;
 import org.opensearch.common.xcontent.XContentType;
+import org.opensearch.rest.RestStatus;
 import org.opensearch.securityanalytics.action.GetIndexMappingsResponse;
 
 import java.io.IOException;
@@ -26,6 +30,7 @@
 import java.util.Map;
 import java.util.stream.Collectors;
 import org.opensearch.securityanalytics.action.GetMappingsViewResponse;
+import org.opensearch.securityanalytics.util.SecurityAnalyticsException;
 
 
 import static org.opensearch.securityanalytics.mapper.MapperUtils.PATH;
@@ -173,6 +178,14 @@ public void onResponse(GetMappingsResponse getMappingsResponse) {
                             break;
                         }
                     }
+
+                    if (appliedAliases.size() == 0) {
+                        actionListener.onFailure(SecurityAnalyticsException.wrap(
+                                new OpenSearchStatusException("No applied aliases not found", RestStatus.NOT_FOUND))
+                        );
+                        return;
+                    }
+
                     // Traverse mappings and do copy with excluded type=alias properties
                     MappingsTraverser mappingsTraverser = new MappingsTraverser(mappingMetadata);
                     // Resulting properties after filtering

diff --git a/src/main/java/org/opensearch/securityanalytics/util/SecurityAnalyticsException.java b/src/main/java/org/opensearch/securityanalytics/util/SecurityAnalyticsException.java
@@ -51,6 +51,19 @@ public static OpenSearchException wrap(Exception ex) {
         return new SecurityAnalyticsException(friendlyMsg, status, new Exception(String.format(Locale.getDefault(), "%s: %s", ex.getClass().getName(), ex.getMessage())));
     }
 
+    public static OpenSearchException wrap(OpenSearchException ex) {
+        log.error(String.format(Locale.getDefault(), "Security Analytics error: %s", ex.getMessage()));
+
+        String friendlyMsg = "Unknown error";
+        RestStatus status = ex.status();
+
+        if (!Strings.isNullOrEmpty(ex.getMessage())) {
+            friendlyMsg = ex.getMessage();
+        }
+
+        return new SecurityAnalyticsException(friendlyMsg, status, new Exception(String.format(Locale.getDefault(), "%s: %s", ex.getClass().getName(), ex.getMessage())));
+    }
+
     public static OpenSearchException wrap(List<Exception> ex) {
         try {
             RestStatus status = RestStatus.BAD_REQUEST;

diff --git a/src/main/resources/OSMapping/mapper_topics.json b/src/main/resources/OSMapping/mapper_topics.json
@@ -1,7 +1,18 @@
 {
   "netflow": "OSMapping/network/NetFlowMapping.json",
-  "macos": "OSMapping/macos/mappings.json",
+  "ad_ldap": "OSMapping/ad_ldap/mappings.json",
+  "apache_access": "OSMapping/apache_access/mappings.json",
+  "cloudtrail": "OSMapping/cloudtrail/mappings.json",
+  "dns": "OSMapping/dns/mappings.json",
   "network": "OSMapping/network/mappings.json",
+  "others_application": "OSMapping/others_application/mappings.json",
+  "others_apt": "OSMapping/others_apt/mappings.json",
+  "others_cloud": "OSMapping/others_cloud/mappings.json",
+  "others_compliance": "OSMapping/others_compliance/mappings.json",
+  "others_macos": "OSMapping/others_macos/mappings.json",
+  "others_proxy": "OSMapping/others_proxy/mappings.json",
+  "others_web": "OSMapping/others_web/mappings.json",
+  "s3": "OSMapping/s3/mappings.json",
   "windows": "OSMapping/windows/mappings.json",
   "test_windows": "OSMapping/test_windows/mappings.json"
 }
diff --git a/src/main/resources/OSMapping/others_application/mappings.json b/src/main/resources/OSMapping/others_application/mappings.json
@@ -0,0 +1,28 @@
+{
+  "properties": {
+    "windows-event_data-CommandLine": {
+      "type": "alias",
+      "path": "CommandLine"
+    },
+    "event_uid": {
+      "type": "alias",
+      "path": "EventID"
+    },
+    "windows-hostname": {
+      "type": "alias",
+      "path": "HostName"
+    },
+    "windows-message": {
+      "type": "alias",
+      "path": "Message"
+    },
+    "windows-provider-name": {
+      "type": "alias",
+      "path": "Provider_Name"
+    },
+    "windows-servicename": {
+      "type": "alias",
+      "path": "ServiceName"
+    }
+  }
+}
diff --git a/src/main/resources/OSMapping/others_apt/mappings.json b/src/main/resources/OSMapping/others_apt/mappings.json
@@ -0,0 +1,28 @@
+{
+  "properties": {
+    "windows-event_data-CommandLine": {
+      "type": "alias",
+      "path": "CommandLine"
+    },
+    "event_uid": {
+      "type": "alias",
+      "path": "EventID"
+    },
+    "windows-hostname": {
+      "type": "alias",
+      "path": "HostName"
+    },
+    "windows-message": {
+      "type": "alias",
+      "path": "Message"
+    },
+    "windows-provider-name": {
+      "type": "alias",
+      "path": "Provider_Name"
+    },
+    "windows-servicename": {
+      "type": "alias",
+      "path": "ServiceName"
+    }
+  }
+}
diff --git a/src/main/resources/OSMapping/others_cloud/mappings.json b/src/main/resources/OSMapping/others_cloud/mappings.json
@@ -0,0 +1,28 @@
+{
+  "properties": {
+    "windows-event_data-CommandLine": {
+      "type": "alias",
+      "path": "CommandLine"
+    },
+    "event_uid": {
+      "type": "alias",
+      "path": "EventID"
+    },
+    "windows-hostname": {
+      "type": "alias",
+      "path": "HostName"
+    },
+    "windows-message": {
+      "type": "alias",
+      "path": "Message"
+    },
+    "windows-provider-name": {
+      "type": "alias",
+      "path": "Provider_Name"
+    },
+    "windows-servicename": {
+      "type": "alias",
+      "path": "ServiceName"
+    }
+  }
+}
diff --git a/src/main/resources/OSMapping/others_compliance/mappings.json b/src/main/resources/OSMapping/others_compliance/mappings.json
@@ -0,0 +1,28 @@
+{
+  "properties": {
+    "windows-event_data-CommandLine": {
+      "type": "alias",
+      "path": "CommandLine"
+    },
+    "event_uid": {
+      "type": "alias",
+      "path": "EventID"
+    },
+    "windows-hostname": {
+      "type": "alias",
+      "path": "HostName"
+    },
+    "windows-message": {
+      "type": "alias",
+      "path": "Message"
+    },
+    "windows-provider-name": {
+      "type": "alias",
+      "path": "Provider_Name"
+    },
+    "windows-servicename": {
+      "type": "alias",
+      "path": "ServiceName"
+    }
+  }
+}
diff --git a/src/main/resources/OSMapping/others_proxy/mappings.json b/src/main/resources/OSMapping/others_proxy/mappings.json
@@ -0,0 +1,28 @@
+{
+  "properties": {
+    "windows-event_data-CommandLine": {
+      "type": "alias",
+      "path": "CommandLine"
+    },
+    "event_uid": {
+      "type": "alias",
+      "path": "EventID"
+    },
+    "windows-hostname": {
+      "type": "alias",
+      "path": "HostName"
+    },
+    "windows-message": {
+      "type": "alias",
+      "path": "Message"
+    },
+    "windows-provider-name": {
+      "type": "alias",
+      "path": "Provider_Name"
+    },
+    "windows-servicename": {
+      "type": "alias",
+      "path": "ServiceName"
+    }
+  }
+}
diff --git a/src/main/resources/OSMapping/others_web/mappings.json b/src/main/resources/OSMapping/others_web/mappings.json
@@ -0,0 +1,28 @@
+{
+  "properties": {
+    "windows-event_data-CommandLine": {
+      "type": "alias",
+      "path": "CommandLine"
+    },
+    "event_uid": {
+      "type": "alias",
+      "path": "EventID"
+    },
+    "windows-hostname": {
+      "type": "alias",
+      "path": "HostName"
+    },
+    "windows-message": {
+      "type": "alias",
+      "path": "Message"
+    },
+    "windows-provider-name": {
+      "type": "alias",
+      "path": "Provider_Name"
+    },
+    "windows-servicename": {
+      "type": "alias",
+      "path": "ServiceName"
+    }
+  }
+}
diff --git a/src/test/java/org/opensearch/securityanalytics/alerts/AlertsIT.java b/src/test/java/org/opensearch/securityanalytics/alerts/AlertsIT.java
@@ -534,7 +534,7 @@ public void testAlertHistoryRollover_maxAge_low_retention() throws IOException,
         // both req params and req body are supported
         createMappingRequest.setJsonEntity(
                 "{ \"index_name\":\"" + index + "\"," +
-                        "  \"rule_topic\":\"windows\", " +
+                        "  \"rule_topic\":\"" + randomDetectorType() + "\", " +
                         "  \"partial\":true" +
                         "}"
         );

diff --git a/src/test/java/org/opensearch/securityanalytics/findings/FindingIT.java b/src/test/java/org/opensearch/securityanalytics/findings/FindingIT.java
@@ -385,7 +385,7 @@ public void testGetFindings_rolloverByMaxDoc_short_retention_success() throws IO
         // both req params and req body are supported
         createMappingRequest.setJsonEntity(
                 "{ \"index_name\":\"" + index + "\"," +
-                        "  \"rule_topic\":\"windows\", " +
+                        "  \"rule_topic\":\"" + randomDetectorType() + "\", " +
                         "  \"partial\":true" +
                         "}"
         );

diff --git a/src/test/java/org/opensearch/securityanalytics/mapper/MapperRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/mapper/MapperRestApiIT.java
@@ -169,6 +169,37 @@ public void testUpdateAndGetMappingSuccess() throws IOException {
         assertEquals(1L, searchResponse.getHits().getTotalHits().value);
     }
 
+    public void testUpdateAndGetMapping_notFound_Success() throws IOException {
+
+        String testIndexName = "my_index";
+
+        createSampleIndex(testIndexName);
+
+        // Execute UpdateMappingsAction to add alias mapping for index
+        Request updateRequest = new Request("PUT", SecurityAnalyticsPlugin.MAPPER_BASE_URI);
+        // both req params and req body are supported
+        updateRequest.setJsonEntity(
+                "{ \"index_name\":\"" + testIndexName + "\"," +
+                        "  \"field\":\"netflow.source_transport_port\","+
+                        "  \"alias\":\"\\u0000\" }"
+        );
+        // request.addParameter("indexName", testIndexName);
+        // request.addParameter("ruleTopic", "netflow");
+        Response response = client().performRequest(updateRequest);
+        assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode());
+
+        // Execute GetIndexMappingsAction and verify mappings
+        Request getRequest = new Request("GET", SecurityAnalyticsPlugin.MAPPER_BASE_URI);
+        getRequest.addParameter("index_name", testIndexName);
+        try {
+            client().performRequest(getRequest);
+            fail();
+        } catch (ResponseException e) {
+            assertEquals(HttpStatus.SC_NOT_FOUND, e.getResponse().getStatusLine().getStatusCode());
+            assertTrue(e.getMessage().contains("No applied aliases not found"));
+        }
+    }
+
     public void testExistingMappingsAreUntouched() throws IOException {
         String testIndexName = "existing_mappings_ok";
 

diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/RuleRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/RuleRestApiIT.java
@@ -102,6 +102,21 @@ public void testCreatingARule() throws IOException {
         Assert.assertEquals(0, hits.size());
     }
 
+    public void testCreatingARule_incorrect_category() throws IOException {
+        String rule = randomRule();
+
+        try {
+            makeRequest(client(), "POST", SecurityAnalyticsPlugin.RULE_BASE_URI, Collections.singletonMap("category", "unknown_category"),
+                    new StringEntity(rule), new BasicHeader("Content-Type", "application/json"));
+            fail("expected exception due to invalid category");
+        } catch (ResponseException e) {
+            assertEquals(HttpStatus.SC_BAD_REQUEST, e.getResponse().getStatusLine().getStatusCode());
+            Assert.assertTrue(
+                e.getMessage().contains("Invalid rule category")
+            );
+        }
+    }
+
     public void testCreatingAggregationRule() throws SigmaError, IOException {
         Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.RULE_BASE_URI, Collections.singletonMap("category", "windows"),
             new StringEntity(countAggregationTestRule()), new BasicHeader("Content-Type", "application/json"));