Rebased code changes in EI-2.X to EI-3.X and added TTL indexes automa…

…tic updation. (#507) * Logger info add for the received events * Enhancements on the below items. - Reduced the duplicate calls on subscription checks - Introduced ttl for event object map collection - Fixed Null pointer exceptions - Cache implemented for repeat handler collection. - Added log statements to print response time of the mongo and ER queries * Configurable paramter is introduced to restrict the default timeouts for the REST calls while triggering the notifications. Updated the code to handle the index creation only once. * Added loggers with additional info * Fixed test cases * Fixed the failure test cases * Dummu push to test the travis build * Travis build testing * Dummy push to test the travis build * Testing failed Fts * Testing failed test cases * Failure tests * Fix for waitlist queue issue * checking for testcase failures * dummy push * checking for itegration test failures * Checking for Integration test failures * dummy push * dummy push2 * checking for integration testcases * dummy push to check for IT failures * dummy push * Checking Integration tests * check for ft failures * Fixed EventToObjectMap handling and removed the configurable paramter "notification.httpRequest.timeout" * final check * fixed the comments * checking for testcase failures * dummy push * checking for itegration test failures * Checking for Integration test failures * dummy push * dummy push2 * checking for integration testcases * dummy push to check for IT failures * dummy push * Checking Integration tests * check for ft failures * final check * fixed the comments * final check * final push * Fixed all the comments * Updated version in pom file from 3.1.2 to 3.1.3 * Fixed the comments and modified documentation. Co-authored-by: Raju Beemreddy <raju.x.beemreddy@ericsson.com> Co-authored-by: Suryakumari Kolli <suryakumari.kolli@ericsson.com>
eiffel-community · Oct 21, 2021 · a1bd6ad · a1bd6ad
1 parent 37177d6
commit a1bd6ad
Show file tree

Hide file tree

Showing 31 changed files with 449 additions and 137 deletions.
diff --git a/pom.xml b/pom.xml
@@ -6,7 +6,7 @@
     <modelVersion>4.0.0</modelVersion>
     <groupId>com.github.ericsson</groupId>
     <artifactId>eiffel-intelligence</artifactId>
-    <version>3.1.2</version>
+    <version>3.1.3</version>
     <packaging>war</packaging>
 
     <parent>

diff --git a/src/functionaltests/java/com/ericsson/ei/encryption/EncryptionSteps.java b/src/functionaltests/java/com/ericsson/ei/encryption/EncryptionSteps.java
@@ -94,7 +94,7 @@ public void eiffelEventsAreSent() throws IOException, InterruptedException {
         LOGGER.debug("Sending Eiffel events.");
         List<String> eventNamesToSend = getEventNamesToSend();
         eventManager.sendEiffelEvents(EIFFEL_EVENTS_JSON_PATH, eventNamesToSend);
-        List<String> eventsIdList = eventManager.getEventsIdList(EIFFEL_EVENTS_JSON_PATH,
+        List<String> eventsIdList = eventManager.getEventIdsList(EIFFEL_EVENTS_JSON_PATH,
                 eventNamesToSend);
         List<String> missingEventIds = dbManager.verifyEventsInDB(eventsIdList, 0);
         String errorMessage = "The following events are missing in mongoDB: "

diff --git a/...tionaltests/java/com/ericsson/ei/notifications/trigger/SubscriptionNotificationSteps.java b/...tionaltests/java/com/ericsson/ei/notifications/trigger/SubscriptionNotificationSteps.java
@@ -13,7 +13,10 @@
 import java.util.Arrays;
 import java.util.List;
 import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
+import org.bson.Document;
 import org.json.JSONArray;
 import org.json.JSONException;
 import org.junit.Ignore;
@@ -45,13 +48,15 @@
 import cucumber.api.java.en.Given;
 import cucumber.api.java.en.Then;
 import cucumber.api.java.en.When;
+import util.IntegrationTestBase;
 
 @Ignore
 @TestPropertySource(properties = {
         "spring.data.mongodb.database: SubscriptionNotificationSteps",
         "failed.notifications.collection.name: SubscriptionNotificationSteps-failedNotifications",
         "rabbitmq.exchange.name: SubscriptionNotificationSteps-exchange",
-        "rabbitmq.queue.suffix: SubscriptionNotificationSteps" })
+        "rabbitmq.queue.suffix: SubscriptionNotificationSteps",
+        "aggregations.collection.ttl: 0"})
 public class SubscriptionNotificationSteps extends FunctionalTestBase {
 
     private static final Logger LOGGER = getLogger(SubscriptionNotificationSteps.class);
@@ -95,11 +100,12 @@ public class SubscriptionNotificationSteps extends FunctionalTestBase {
 
     @Autowired
     private EmailSender emailSender;
-
+    
     private SimpleSmtpServer smtpServer;
     private ClientAndServer restServer;
     private MockServerClient mockClient;
     private ResponseEntity response;
+    private String aggregatedEventId;
 
     @Before()
     public void beforeScenario() {
@@ -165,10 +171,16 @@ public void send_eiffel_events() throws Throwable {
         LOGGER.debug("About to send Eiffel events.");
         List<String> eventNamesToSend = getEventNamesToSend();
         eventManager.sendEiffelEvents(EIFFEL_EVENTS_JSON_PATH, eventNamesToSend);
-        List<String> missingEventIds = dbManager
-                                                .verifyEventsInDB(eventManager.getEventsIdList(
-                                                        EIFFEL_EVENTS_JSON_PATH, eventNamesToSend),
-                                                        0);
+        JsonNode parsedJSON = IntegrationTestBase.getJSONFromFile(EIFFEL_EVENTS_JSON_PATH);
+        eventNamesToSend.forEach((String eventName) -> {
+            JsonNode eventJson = parsedJSON.get(eventName);
+            if (eventName.contains("EiffelArtifactCreatedEvent")) {
+                aggregatedEventId = eventJson.get("meta").get("id").toString();
+            }
+        });
+        aggregatedEventId = aggregatedEventId.substring(1,aggregatedEventId.length()-1);
+        List<String> list = Stream.of(aggregatedEventId).collect(Collectors.toList());
+        List<String> missingEventIds = dbManager.verifyEventsInDB(list,0);
         assertEquals("The following events are missing in mongoDB: " + missingEventIds.toString(),
                 0,
                 missingEventIds.size());
@@ -180,7 +192,7 @@ public void wait_for_ei_to_aggregate_objects() throws Throwable {
         List<String> eventNamesToSend = getEventNamesToSend();
         LOGGER.debug("Checking Aggregated Objects.");
         List<String> arguments = new ArrayList<>(
-                eventManager.getEventsIdList(EIFFEL_EVENTS_JSON_PATH, eventNamesToSend));
+                eventManager.getEventIdsList(EIFFEL_EVENTS_JSON_PATH, eventNamesToSend));
         arguments.add("id=TC5");
         arguments.add("conclusion=SUCCESSFUL");
         List<String> missingArguments = dbManager.verifyAggregatedObjectInDB(arguments);
@@ -487,4 +499,5 @@ private int getDbSizeForCondition(int minWaitTime, int maxWaitTime, int expected
 
         return queryResult.size();
     }
+
 }
diff --git a/src/functionaltests/java/com/ericsson/ei/notifications/ttl/TestTTLSteps.java b/src/functionaltests/java/com/ericsson/ei/notifications/ttl/TestTTLSteps.java
@@ -28,8 +28,10 @@
 import com.ericsson.ei.exception.AuthenticationException;
 import com.ericsson.ei.exception.MongoDBConnectionException;
 import com.ericsson.ei.mongo.MongoCondition;
+import com.ericsson.ei.mongo.MongoConstants;
 import com.ericsson.ei.mongo.MongoDBHandler;
 import com.ericsson.ei.notifications.InformSubscriber;
+import com.ericsson.ei.subscription.SubscriptionHandler;
 import com.ericsson.ei.utils.FunctionalTestBase;
 import com.ericsson.ei.utils.HttpRequest;
 import com.ericsson.ei.utils.HttpRequest.HttpMethod;
@@ -91,6 +93,9 @@ public class TestTTLSteps extends FunctionalTestBase {
     @Autowired
     private InformSubscriber informSubscriber;
 
+    @Autowired
+    private SubscriptionHandler subscriptionHandler;
+
     @Before("@TestNotificationRetries")
     public void beforeScenario() {
         setUpMockServer();
@@ -104,7 +109,7 @@ public void afterScenario() throws IOException {
     }
 
     @Given("^Subscription is created$")
-    public void create_subscription_object() throws IOException, JSONException {
+    public void create_subscription_object() throws IOException, JSONException, MongoDBConnectionException {
 
         LOGGER.debug("Starting scenario @TestNotificationRetries.");
         mongoDBHandler.dropCollection(database, failedNotificationCollection);
@@ -115,6 +120,7 @@ public void create_subscription_object() throws IOException, JSONException {
         subscriptionStr = subscriptionStr.replaceAll("\\{port\\}", String.valueOf(clientAndServer.getPort()));
 
         subscriptionObject = new ObjectMapper().readTree(subscriptionStr);
+        mongoDBHandler.createTTLIndex(database, failedNotificationCollection, MongoConstants.TIME, 1);
         assertEquals(false, subscriptionObject.get("notificationMeta").toString().contains("{port}"));
     }
 
@@ -166,7 +172,7 @@ public void eiffel_events_are_sent() throws Throwable {
         List<String> eventNamesToSend = getEventNamesToSend();
         eventManager.sendEiffelEvents(EIFFEL_EVENTS_JSON_PATH, eventNamesToSend);
         List<String> missingEventIds = dbManager.verifyEventsInDB(
-                eventManager.getEventsIdList(EIFFEL_EVENTS_JSON_PATH, eventNamesToSend), 0);
+                eventManager.getEventIdsList(EIFFEL_EVENTS_JSON_PATH, eventNamesToSend), 0);
         assertEquals("The following events are missing in mongoDB: " + missingEventIds.toString(), 0,
                 missingEventIds.size());
         LOGGER.debug("Eiffel event is sent");
@@ -177,6 +183,8 @@ public void aggregated_object_is_created() throws Throwable {
         // verify that aggregated object is created and present in db
         LOGGER.debug("Checking presence of aggregated Object");
         List<String> allObjects = mongoDBHandler.getAllDocuments(database, collection);
+        String id = eventManager.getEventIdsList(EIFFEL_EVENTS_JSON_PATH, getEventNamesToSend()).get(0);
+        subscriptionHandler.checkSubscriptionForObject(allObjects.get(0), id);
         assertEquals(1, allObjects.size());
     }
 

diff --git a/src/functionaltests/java/com/ericsson/ei/rules/RuleTestSteps.java b/src/functionaltests/java/com/ericsson/ei/rules/RuleTestSteps.java
@@ -107,6 +107,10 @@ public void get_content(String contentFileName) throws Throwable {
         if (expectedResponse instanceof JSONArray) {
             JSONArray expectedArray = new JSONArray(responseBody);
             JSONArray responseArray = new JSONArray(response.getBody().toString());
+            for (int i = 0; i < responseArray.length(); i++) {
+                JSONObject jsonobject = responseArray.getJSONObject(i);
+                jsonobject.remove("Time");
+            }
             assertEquals(expectedArray, responseArray, true);
         } else {
             JSONObject expectedObject = new JSONObject(responseBody);

diff --git a/...ests/java/com/ericsson/ei/threadingAndWaitlistRepeat/ThreadingAndWaitlistRepeatSteps.java b/...ests/java/com/ericsson/ei/threadingAndWaitlistRepeat/ThreadingAndWaitlistRepeatSteps.java
@@ -80,7 +80,7 @@ public void event_to_object_map_is_manipulated_to_include_the_sent_events() thro
         List<String> eventNames = getEventNamesToSend();
         for (String eventName : eventNames) {
             JsonNode eventJson = parsedJSON.get(eventName);
-            eventToObjectMapHanler.updateEventToObjectMapInMemoryDB(rulesObject, eventJson.toString(), dummyObjectID);
+            eventToObjectMapHanler.updateEventToObjectMapInMemoryDB(rulesObject, eventJson.toString(), dummyObjectID, 0);
         }
     }
 

diff --git a/src/functionaltests/java/com/ericsson/ei/utils/EventManager.java b/src/functionaltests/java/com/ericsson/ei/utils/EventManager.java
@@ -60,7 +60,7 @@ public void sendEiffelEvent(String eiffelEventJson) {
      * @return list of event IDs
      * @throws IOException
      */
-    public List<String> getEventsIdList(String eiffelEventsJsonPath, List<String> eventNames) throws IOException {
+    public List<String> getEventIdsList(String eiffelEventsJsonPath, List<String> eventNames) throws IOException {
         List<String> eventsIdList = new ArrayList<>();
         JsonNode parsedJSON = getJSONFromFile(eiffelEventsJsonPath);
         for (String eventName : eventNames) {

diff --git a/src/integrationtests/java/com/ericsson/ei/integrationtests/flow/FlowStepsIT.java b/src/integrationtests/java/com/ericsson/ei/integrationtests/flow/FlowStepsIT.java
@@ -88,10 +88,12 @@ public class FlowStepsIT extends IntegrationTestBase {
     private JenkinsXmlData jenkinsXmlData;
     private SubscriptionObject subscriptionObject;
     private JSONObject jobStatusData;
+    public String aggregatedEvent;
 
     @Given("^the rules \"([^\"]*)\"$")
     public void rules(String rulesFilePath) throws Throwable {
         this.rulesFilePath = rulesFilePath;
+        aggregatedEvent = getStartEvent(this.rulesFilePath);
     }
 
     @Given("^the events \"([^\"]*)\"$")
@@ -167,7 +169,7 @@ public void conditionOrRequirementIsAddedInSubscription(String condition,
 
     @When("^the eiffel events are sent$")
     public void eiffelEventsAreSent() throws Throwable {
-        super.sendEventsAndConfirm();
+        super.sendEventsAndConfirm(aggregatedEvent);
     }
 
     @When("^the upstream input events are sent")

diff --git a/...ontests/java/com/ericsson/ei/integrationtests/notification/FailedNotificationStepsIT.java b/...ontests/java/com/ericsson/ei/integrationtests/notification/FailedNotificationStepsIT.java
@@ -55,6 +55,7 @@ public class FailedNotificationStepsIT extends IntegrationTestBase {
     private String eventsFilePath;
     private ObjectMapper objectMapper = new ObjectMapper();
     private int extraEventsCount = 0;
+    public String aggregatedEvent;
 
     private SubscriptionObject subscriptionObject;
 
@@ -64,6 +65,7 @@ public class FailedNotificationStepsIT extends IntegrationTestBase {
     @Given("^the rules \"([^\"]*)\"$")
     public void rules(String rulesFilePath) throws Throwable {
         this.rulesFilePath = rulesFilePath;
+        aggregatedEvent = getStartEvent(this.rulesFilePath);
     }
 
     @Given("^the events \"([^\"]*)\"$")
@@ -100,7 +102,7 @@ public void conditionOrRequirementIsAddedInSubscription(String condition,
 
     @When("^the eiffel events are sent$")
     public void eiffelEventsAreSent() throws Throwable {
-        super.sendEventsAndConfirm();
+        super.sendEventsAndConfirm(aggregatedEvent);
     }
 
     @When("^rest post body media type is set to \"([^\"]*)\" is set in subscription$")

diff --git a/src/integrationtests/java/util/IntegrationTestBase.java b/src/integrationtests/java/util/IntegrationTestBase.java
@@ -13,19 +13,28 @@
 */
 package util;
 
+import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.fail;
 
 import java.io.File;
 import java.io.IOException;
+import java.io.InputStream;
+import java.net.URI;
 import java.net.URISyntaxException;
+import java.nio.charset.Charset;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 import javax.annotation.PostConstruct;
 
 import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.IOUtils;
+import org.json.JSONArray;
+import org.json.JSONObject;
 import org.skyscreamer.jsonassert.JSONAssert;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -38,14 +47,11 @@
 import org.springframework.test.context.support.AbstractTestExecutionListener;
 
 import com.ericsson.ei.mongo.MongoDBHandler;
+import com.ericsson.ei.mongo.MongoStringQuery;
 import com.ericsson.ei.utils.HttpRequest;
 import com.ericsson.ei.utils.HttpRequest.HttpMethod;
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
-import com.mongodb.MongoClient;
-import com.mongodb.client.MongoCollection;
-import com.mongodb.client.MongoDatabase;
-
 import lombok.Setter;
 
 public abstract class IntegrationTestBase extends AbstractTestExecutionListener {
@@ -95,7 +101,8 @@ public abstract class IntegrationTestBase extends AbstractTestExecutionListener
     private String failedNotificationCollectionName;
     @Value("${sessions.collection.name}")
     private String sessionsCollectionName;
-
+
+    public String aggregatedEventId;
     /*
      * setFirstEventWaitTime: variable to set the wait time after publishing the first event. So any
      * thread looking for the events don't do it before actually populating events in the database
@@ -141,15 +148,17 @@ protected int extraEventsCount() {
      * @return
      * @throws Exception
      */
-    protected void sendEventsAndConfirm() throws Exception {
+    protected void sendEventsAndConfirm(String aggregatedEvent) throws Exception {
         List<String> eventNames = getEventNamesToSend();
         int eventsCount = eventNames.size() + extraEventsCount();
 
         JsonNode parsedJSON = getJSONFromFile(getEventsFilePath());
-
         boolean alreadyExecuted = false;
         for (String eventName : eventNames) {
             JsonNode eventJson = parsedJSON.get(eventName);
+            if (eventName.contains(aggregatedEvent)) {
+                aggregatedEventId = eventJson.get("meta").get("id").toString();
+            }
             String event = eventJson.toString();
 
             rabbitTemplate.convertAndSend(event);
@@ -163,7 +172,6 @@ protected void sendEventsAndConfirm() throws Exception {
              */
             TimeUnit.MILLISECONDS.sleep(DEFAULT_DELAY_BETWEEN_SENDING_EVENTS);
         }
-
         waitForEventsToBeProcessed(eventsCount);
         checkResult(getCheckData());
     }
@@ -191,7 +199,7 @@ protected void sendEventsAndConfirm() throws Exception {
      */
     protected abstract Map<String, JsonNode> getCheckData() throws IOException, Exception;
 
-    protected JsonNode getJSONFromFile(String filePath) throws Exception {
+    public static JsonNode getJSONFromFile(String filePath) throws Exception {
         try {
             String expectedDocument = FileUtils.readFileToString(new File(filePath), "UTF-8");
             return objectMapper.readTree(expectedDocument);
@@ -206,7 +214,6 @@ protected JsonNode getJSONFromFile(String filePath) throws Exception {
 
     /**
      * Wait for certain amount of events to be processed.
-     *
      * @param eventsCount - An int which indicated how many events that should be processed.
      * @return
      * @throws InterruptedException
@@ -236,12 +243,65 @@ protected void waitForEventsToBeProcessed(int eventsCount) throws InterruptedExc
      * @return amount of processed events
      */
     private long countProcessedEvents(String database, String collectionName) {
-        MongoClient mongoClient = mongoDBHandler.getMongoClient();
-        MongoDatabase db = mongoClient.getDatabase(database);
-        MongoCollection collection = db.getCollection(collectionName);
-        return collection.count();
+        int count = 0;
+        List<String> documents = null;
+        String queryString = "{\"_id\": " + aggregatedEventId + "}";
+        MongoStringQuery query = new MongoStringQuery(queryString);
+        documents = mongoDBHandler.find(database, collectionName, query);
+        JSONObject json = new JSONObject(documents.get(0));
+        JSONArray jsonArray = new JSONArray();
+        jsonArray.put(json.get("objects"));
+        String objectId = json.get("objects").toString();
+        for (int i = 0; i < objectId.length(); i++) {
+            if (objectId.charAt(i) == ',')
+                count++;
+        }
+        count++;
+        return count;
+    }
+
+    /**
+     * StartEvent is Fetched from the given rules
+     * @param rules file
+     * @return StartEvent type
+     */
+    public String getStartEvent(String rules) throws Exception {
+        String content, type = null;
+        content = readRulesFileFromPath(rules);
+        JSONArray json = new JSONArray(content);
+        if (content.isEmpty()) {
+            throw new Exception("Rules content cannot be empty");
+        } else {
+            for (int i = 0; i < json.length(); i++) {
+                JSONObject jsonObj = json.getJSONObject(i);
+                if (jsonObj.get("StartEvent").equals("YES")) {
+                    type = jsonObj.get("Type").toString();
+                    break;
+                }
+            }
+        }
+        return type;
     }
 
+    /**
+     * Reads the rules from given file path
+     * @param rules
+     * @return rules content
+     * @throws IOException
+     */
+    private String readRulesFileFromPath(String rules) throws IOException {
+        String rulesJsonFileContent = null;
+        try (InputStream inputStream = this.getClass().getResourceAsStream(rules)) {
+            if (inputStream == null) {
+                rulesJsonFileContent = FileUtils.readFileToString(new File(rules),
+                        Charset.defaultCharset());
+            } else {
+                rulesJsonFileContent = IOUtils.toString(inputStream, "UTF-8");
+            }
+        }
+        return rulesJsonFileContent;
+    }
+
     /**
      * Retrieves the result from EI and checks if it equals the expected data
      *