Merge pull request #301 from dcanar9/master

adding endpoint to cleanup old collector Items
hygieia · Oct 4, 2022 · b0f8bd4 · b0f8bd4
2 parents 35feda8 + 194c3da
commit b0f8bd4
Show file tree

Hide file tree

Showing 9 changed files with 206 additions and 2 deletions.
diff --git a/pom.xml b/pom.xml
@@ -59,7 +59,7 @@
 
     <properties>
         <!-- Dependencies -->
-        <com.capitalone.dashboard.core.version>3.15.44</com.capitalone.dashboard.core.version>
+        <com.capitalone.dashboard.core.version>3.15.46</com.capitalone.dashboard.core.version>
         <spring-security.version>4.2.18.RELEASE</spring-security.version>
         <tomcat.version>8.5.70</tomcat.version>
         <commons-beanutils.version>1.9.4</commons-beanutils.version>

diff --git a/src/main/java/com/capitalone/dashboard/rest/CollectorController.java b/src/main/java/com/capitalone/dashboard/rest/CollectorController.java
@@ -169,4 +169,5 @@ public ResponseEntity<Void> deletePropertiesCase(@PathVariable String id) {
         collectorService.deletePropertiesInCollectorById(id);
         return ResponseEntity.<Void>noContent().build();
     }
-}
+
+}
diff --git a/src/main/java/com/capitalone/dashboard/rest/CollectorItemController.java b/src/main/java/com/capitalone/dashboard/rest/CollectorItemController.java
@@ -0,0 +1,33 @@
+package com.capitalone.dashboard.rest;
+
+import com.capitalone.dashboard.service.CollectorItemService;
+import java.util.Objects;
+import org.apache.commons.lang3.StringUtils;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.http.HttpStatus;
+import org.springframework.http.ResponseEntity;
+import org.springframework.web.bind.annotation.RequestMethod;
+import org.springframework.web.bind.annotation.RequestParam;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RestController;
+
+
+@RestController
+public class CollectorItemController {
+
+    private CollectorItemService collectorItemService;
+
+    @Autowired
+    public CollectorItemController(CollectorItemService collectorItemService){
+        this.collectorItemService = collectorItemService;
+    }
+
+    @RequestMapping(path="/collector-items/cleanup", method = RequestMethod.DELETE)
+    public ResponseEntity<String> cleanup(@RequestParam(value = "collectorType", required = true, defaultValue = "") String collectorType, @RequestParam(value = "collectorName", required = true, defaultValue = "") String collectorName) {
+        if (StringUtils.isEmpty(collectorName) || Objects.isNull(collectorType)) {
+            return ResponseEntity.status(HttpStatus.BAD_REQUEST).body("Collector type and name are required parameters");
+        }
+            return  collectorItemService.cleanup(collectorType, collectorName);
+    }
+
+}
diff --git a/src/main/java/com/capitalone/dashboard/service/CollectorItemService.java b/src/main/java/com/capitalone/dashboard/service/CollectorItemService.java
@@ -0,0 +1,11 @@
+package com.capitalone.dashboard.service;
+
+import org.springframework.http.ResponseEntity;
+
+public interface CollectorItemService {
+    /**
+     *  Removing CollectorItems that are not connected
+     *  to a Dashboard
+     * */
+    ResponseEntity<String> cleanup(String collectorType, String collectorName);
+}
diff --git a/src/main/java/com/capitalone/dashboard/service/CollectorItemServiceImpl.java b/src/main/java/com/capitalone/dashboard/service/CollectorItemServiceImpl.java
@@ -0,0 +1,135 @@
+package com.capitalone.dashboard.service;
+
+import com.capitalone.dashboard.model.Collector;
+import com.capitalone.dashboard.model.CollectorItem;
+import com.capitalone.dashboard.model.CollectorType;
+import com.capitalone.dashboard.settings.ApiSettings;
+import org.bson.types.ObjectId;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.http.HttpStatus;
+import org.springframework.http.ResponseEntity;
+import org.springframework.stereotype.Service;
+import com.capitalone.dashboard.repository.ComponentRepository;
+import com.capitalone.dashboard.repository.CollectorRepository;
+import com.capitalone.dashboard.repository.CollectorItemRepository;
+
+import java.util.List;
+import java.util.Objects;
+import java.util.Optional;
+
+import static com.capitalone.dashboard.model.CollectorType.*;
+import static com.capitalone.dashboard.model.CollectorType.LibraryPolicy;
+
+@Service
+public class CollectorItemServiceImpl implements CollectorItemService {
+
+    private ApiSettings apiSettings;
+    private ComponentRepository componentRepository;
+    private CollectorRepository collectorRepository;
+    private CollectorItemRepository collectorItemRepository;
+    private final Logger LOG = LoggerFactory.getLogger(CollectorServiceImpl.class);
+    private final Long DAY_IN_MILLIS = 86400000l;
+
+    @Autowired
+    public CollectorItemServiceImpl(ComponentRepository componentRepository, CollectorRepository collectorRepository,
+                                    CollectorItemRepository collectorItemRepository, ApiSettings apiSettings){
+        this.componentRepository = componentRepository;
+        this.collectorRepository = collectorRepository;
+        this.collectorItemRepository = collectorItemRepository;
+        this.apiSettings = apiSettings;
+    }
+
+    public ResponseEntity<String> cleanup(String collectorTypeString, String collectorName) {
+
+        // to time duration, get max age of items, count of items deleted (vars respectively)
+        long startTime = System.currentTimeMillis();
+        long endDate = startTime - (apiSettings.getCollectorItemGracePeriod() * DAY_IN_MILLIS);
+        int count = 0;
+
+        CollectorType collectorType;
+        try{
+            collectorType = CollectorType.fromString(collectorTypeString);
+        }
+        catch (IllegalArgumentException exception){
+            return ResponseEntity.status(HttpStatus.BAD_REQUEST).body("Error :: No such collector type " + collectorTypeString);
+        }
+
+        // prevent cleaning the wrong collector (returns content only if there is an error)
+        ResponseEntity<String> isCleanable = isCleanableCollector(collectorType, collectorName);
+        if(Objects.nonNull(isCleanable)){return isCleanable;}
+
+        Optional<Collector> collector = collectorRepository.findByCollectorTypeAndName(collectorType, collectorName).stream().findFirst();
+        if (!collector.isPresent()) {
+            return ResponseEntity.status(HttpStatus.BAD_REQUEST).body("Error :: Could not find collector: " + collectorName);
+        }
+
+        // get collectorItems that have not been updated since the endDate given
+        List<CollectorItem> collectorItems = collectorItemRepository.findByCollectorIdAndLastUpdatedBefore(collector.get().getId(), endDate);
+        LOG.info(String.format("deleteDisconnectedItems :: Found %d collectorItems to verify", collectorItems.size()));
+
+        // iterate through enabled items and check if they are connected to a dashboard
+        for (CollectorItem collectorItem : collectorItems) {
+            if (!hasComponent(collectorType, collectorItem.getId())) {
+
+                String loggingPrefix = String.format("findOldCollectorItems :: Removing (#%d of %d):: could not find a dashboard for the collectorItem with the following options:"
+                        ,collectorItems.indexOf(collectorItem)+1, collectorItems.size());
+
+                logDeletedCollectorItem(collectorType, collectorItem, loggingPrefix);
+//                collectorItemRepository.delete(collectorItem.getId());
+                count++;
+            }
+        }
+
+        LOG.info(String.format("deleteDisconnectedItems :: Finished (duration=%s) :: collectorType=%s :: Found %d items with no corresponding dashboard.", System.currentTimeMillis() - startTime, collectorType.toString(), count));
+        return ResponseEntity.status(HttpStatus.OK).body(String.format("Successfully removed %d  %s type collectorItems out of %d", count, collectorType.toString(), collectorItems.size()));
+    }
+
+
+    /**
+     * ******************************************************************************
+     * Bottom three methods are helper functions for the deleteDisconnectedItems()
+     * ******************************************************************************
+     */
+    private ResponseEntity<String> isCleanableCollector(CollectorType collectorType, String collectorName){
+        if (!(collectorType.equals(LibraryPolicy) || collectorType.equals(StaticSecurityScan) || collectorType.equals(SCM))){
+            return ResponseEntity.status(HttpStatus.BAD_REQUEST)
+                    .body("Warning :: collectorItems of type " + collectorType.toString() + " are not cleanable");
+        }
+        if (!(collectorName.equalsIgnoreCase(apiSettings.getLibraryPolicyCollectorName()) || collectorName.equalsIgnoreCase(apiSettings.getSecurityScanCollectorName())
+        || collectorName.equalsIgnoreCase(apiSettings.getDataSyncSettings().getScm()))){
+            return ResponseEntity.status(HttpStatus.BAD_REQUEST).body("Error :: Could not find collector: " + collectorName);
+        }
+        return null;
+    }
+
+
+    private Boolean hasComponent(CollectorType collectorType, ObjectId collectorItemId){
+        switch (collectorType){
+            case LibraryPolicy:
+                return componentRepository.findByLibraryPolicyCollectorItems(collectorItemId).stream().findFirst().isPresent();
+            case SCM:
+                return componentRepository.findBySCMCollectorItemId(collectorItemId).stream().findFirst().isPresent();
+            case StaticSecurityScan:
+                return componentRepository.findByStaticSecurityScanCollectorItems(collectorItemId).stream().findFirst().isPresent();
+            // don't expect this case to be hit, but will skip deletion of item
+            default:
+                return true;
+        }
+    }
+
+    private void logDeletedCollectorItem(CollectorType collectorType, CollectorItem collectorItem, String logPrefix){
+        switch (collectorType){
+            case LibraryPolicy:
+                LOG.info(String.format("%s {projectToken: %s, projectName: %s}", logPrefix, collectorItem.getOptions().get("projectToken"), collectorItem.getOptions().get("projectName")));
+                break;
+            case SCM:
+                LOG.info(String.format("%s {url: %s, branch: %s}", logPrefix, collectorItem.getOptions().get("url"), collectorItem.getOptions().get("branch")));
+                break;
+            case StaticSecurityScan:
+                LOG.info(String.format("%s {team: %s, project: %s}", logPrefix, collectorItem.getOptions().get("team"), collectorItem.getOptions().get("project")));
+                break;
+        }
+    }
+}
diff --git a/src/main/java/com/capitalone/dashboard/settings/ApiSettings.java b/src/main/java/com/capitalone/dashboard/settings/ApiSettings.java
@@ -79,10 +79,14 @@ public class ApiSettings {
 
     private String hygieia_ui_url="";
 
+    @Value("${collectorItemGracePeriod:60}")
+    private long collectorItemGracePeriod;
+
     public String getSecurityScanCollectorName() {
         return securityScanCollectorName;
     }
 
+
     public void setSecurityScanCollectorName(String securityScanCollectorName) {
         this.securityScanCollectorName = securityScanCollectorName;
     }
@@ -298,4 +302,12 @@ public int getBatchSize() {
     public void setBatchSize(int batchSize) {
         this.batchSize = batchSize;
     }
+
+    public long getCollectorItemGracePeriod() {
+        return collectorItemGracePeriod;
+    }
+
+    public void setCollectorItemGracePeriod(long collectorItemGracePeriod) {
+        this.collectorItemGracePeriod = collectorItemGracePeriod;
+    }
 }
diff --git a/src/test/java/com/capitalone/dashboard/config/TestAuthConfig.java b/src/test/java/com/capitalone/dashboard/config/TestAuthConfig.java
@@ -55,6 +55,7 @@
 import com.capitalone.dashboard.service.UserInfoService;
 import com.capitalone.dashboard.service.UserInfoServiceImpl;
 import com.capitalone.dashboard.service.InfraStructureService;
+import com.capitalone.dashboard.service.CollectorItemService;
 import com.capitalone.dashboard.settings.ApiSettings;
 import com.capitalone.dashboard.util.PaginationHeaderUtility;
 import org.mockito.Mock;
@@ -134,6 +135,9 @@ public CollectorService collectorService() {
         return Mockito.mock(CollectorService.class);
     }
 
+    @Bean
+    public CollectorItemService collectorItemService() {return Mockito.mock(CollectorItemService.class);}
+
     @Bean
     public ServiceService serviceService() {
         return Mockito.mock(ServiceService.class);

diff --git a/src/test/java/com/capitalone/dashboard/config/TestConfig.java b/src/test/java/com/capitalone/dashboard/config/TestConfig.java
@@ -47,6 +47,7 @@
 import com.capitalone.dashboard.service.TestResultService;
 import com.capitalone.dashboard.service.UserInfoService;
 import com.capitalone.dashboard.service.InfraStructureService;
+import com.capitalone.dashboard.service.CollectorItemService;
 import com.capitalone.dashboard.util.PaginationHeaderUtility;
 import org.mockito.Mockito;
 import org.springframework.context.annotation.Bean;
@@ -292,4 +293,7 @@ public FeatureFlagService featureFlagService(){
 
     @Bean
     public InfraStructureService infraStructureService() { return Mockito.mock(InfraStructureService.class); }
+
+    @Bean
+    public CollectorItemService collectorItemService() {return Mockito.mock(CollectorItemService.class);}
 }
diff --git a/src/test/java/com/capitalone/dashboard/config/TestDefaultAuthConfig.java b/src/test/java/com/capitalone/dashboard/config/TestDefaultAuthConfig.java
@@ -53,6 +53,7 @@
 import com.capitalone.dashboard.service.UserInfoService;
 import com.capitalone.dashboard.service.UserInfoServiceImpl;
 import com.capitalone.dashboard.service.InfraStructureService;
+import com.capitalone.dashboard.service.CollectorItemService;
 import com.capitalone.dashboard.settings.ApiSettings;
 import com.capitalone.dashboard.util.PaginationHeaderUtility;
 import org.mockito.Mock;
@@ -309,5 +310,8 @@ public MetadataService metadataService() {
 	 @Bean
 	 public InfraStructureService infraStructureService() { return Mockito.mock(InfraStructureService.class); }
 
+	 @Bean
+	 public CollectorItemService collectorItemService() {return Mockito.mock(CollectorItemService.class);}
+
  }
-Original file line number
+Diff line change
@@ Expand Up @@
             collectorService.deletePropertiesInCollectorById(id);
             return ResponseEntity.<Void>noContent().build();
         }
-    }
+    }