Add criteria of file limit for test file set entities; (#421)

Add fileLimitDay attribute in yml for local storage;
microsoft · Apr 10, 2023 · f7611ea · f7611ea
1 parent 865e980
commit f7611ea
Show file tree

Hide file tree

Showing 6 changed files with 36 additions and 10 deletions.
diff --git a/center/src/main/java/com/microsoft/hydralab/center/service/TestFileSetService.java b/center/src/main/java/com/microsoft/hydralab/center/service/TestFileSetService.java
@@ -6,16 +6,20 @@
 import com.microsoft.hydralab.common.entity.common.CriteriaType;
 import com.microsoft.hydralab.common.entity.common.EntityType;
 import com.microsoft.hydralab.common.entity.common.TestFileSet;
+import com.microsoft.hydralab.common.file.StorageServiceClientProxy;
 import com.microsoft.hydralab.common.repository.TestFileSetRepository;
 import com.microsoft.hydralab.common.util.AttachmentService;
 import com.microsoft.hydralab.common.util.CriteriaTypeUtil;
+import org.apache.commons.lang3.time.DateUtils;
 import org.springframework.data.domain.Page;
 import org.springframework.data.domain.PageRequest;
 import org.springframework.data.domain.Sort;
 import org.springframework.data.jpa.domain.Specification;
 import org.springframework.stereotype.Service;
 
 import javax.annotation.Resource;
+import java.text.SimpleDateFormat;
+import java.util.Date;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -26,6 +30,8 @@ public class TestFileSetService {
     TestFileSetRepository testFileSetRepository;
     @Resource
     AttachmentService attachmentService;
+    @Resource
+    StorageServiceClientProxy storageServiceClientProxy;
 
     private Map<String, TestFileSet> testFileSetMap = new HashMap<>();
 
@@ -54,14 +60,33 @@ public synchronized TestFileSet getFileSetInfo(String fileSetId) {
 
     public Page<TestFileSet> queryFileSets(int page, int pageSize, List<CriteriaType> criteriaTypes) {
         Specification<TestFileSet> spec = null;
-        if (criteriaTypes != null && criteriaTypes.size() > 0) {
-            spec = new CriteriaTypeUtil<TestFileSet>().transferToSpecification(criteriaTypes, false);
+
+        if (storageServiceClientProxy.fileLimitEnabled()) {
+            CriteriaType fileLimitCriteria = getFileLimitCriteria();
+            if (criteriaTypes != null) {
+                criteriaTypes.add(fileLimitCriteria);
+                spec = new CriteriaTypeUtil<TestFileSet>().transferToSpecification(criteriaTypes, false);
+            } else {
+                spec = new CriteriaTypeUtil<TestFileSet>().transferToSpecification(List.of(fileLimitCriteria), false);
+            }
         }
         Sort sortByDate = Sort.by(Sort.Direction.DESC, "ingestTime");
         Page<TestFileSet> pageObj = testFileSetRepository.findAll(spec, PageRequest.of(page, pageSize, sortByDate));
         return pageObj;
     }
 
+    private CriteriaType getFileLimitCriteria() {
+        CriteriaType fileLimitCriteria = new CriteriaType();
+        fileLimitCriteria.setKey("ingestTime");
+        fileLimitCriteria.setOp(CriteriaType.OpType.GreaterThan);
+        String dateFormat = "yyyy-MM-dd HH:mm:ss";
+        SimpleDateFormat sdf = new SimpleDateFormat(dateFormat);
+        fileLimitCriteria.setValue(sdf.format(DateUtils.addDays(new Date(), -1 * storageServiceClientProxy.getStorageFileLimitDay())));
+        fileLimitCriteria.setDateFormatString(dateFormat);
+
+        return fileLimitCriteria;
+    }
+
     public void updateFileSetTeam(String teamId, String teamName) {
         List<TestFileSet> testFileSets = testFileSetRepository.findAllByTeamId(teamId);
         testFileSets.forEach(testFileSet -> {

diff --git a/center/src/main/resources/application.yml b/center/src/main/resources/application.yml
@@ -64,7 +64,7 @@ app:
     local:
       token: ${CENTER_TOKEN:token=CENTER_LOCAL_STORAGE_TOKEN}
       endpoint: ${LOCAL_STORAGE_ENDPOINT:http://localhost:9886/}
-
+      fileLimitDay: 6
   location: ${user.dir}
 management:
   endpoints:

diff --git a/common/src/main/java/com/microsoft/hydralab/common/file/StorageServiceClientProxy.java b/common/src/main/java/com/microsoft/hydralab/common/file/StorageServiceClientProxy.java
@@ -19,7 +19,7 @@
  * @date 3/1/2023
  */
 
-public class StorageServiceClientProxy extends StorageServiceClient {
+public class StorageServiceClientProxy {
     private String storageType;
     private StorageServiceClient storageServiceClient;
     private ApplicationContext applicationContext;
@@ -59,31 +59,30 @@ public void initAgentStorageClient(String storageType) {
         EntityType.setInstanceContainer(storageProperties);
     }
 
+    public boolean fileLimitEnabled() {
+        return storageServiceClient.getFileLimitDay() > 0;
+    }
+
     public int getStorageFileLimitDay() {
         return storageServiceClient.getFileLimitDay();
     }
 
-    @Override
     public void updateAccessToken(AccessToken token) {
         storageServiceClient.updateAccessToken(token);
     }
 
-    @Override
     public AccessToken generateAccessToken(String permissionType) {
         return storageServiceClient.generateAccessToken(permissionType);
     }
 
-    @Override
     public boolean isAccessTokenExpired(AccessToken token) {
         return storageServiceClient.isAccessTokenExpired(token);
     }
 
-    @Override
     public StorageFileInfo upload(File file, StorageFileInfo storageFileInfo) {
         return storageServiceClient.upload(file, storageFileInfo);
     }
 
-    @Override
     public StorageFileInfo download(File file, StorageFileInfo storageFileInfo) {
         return storageServiceClient.download(file, storageFileInfo);
     }

diff --git a/...rc/main/java/com/microsoft/hydralab/common/file/impl/local/LocalStorageClientAdapter.java b/...rc/main/java/com/microsoft/hydralab/common/file/impl/local/LocalStorageClientAdapter.java
@@ -26,6 +26,7 @@ public LocalStorageClientAdapter() {
     public LocalStorageClientAdapter(StorageProperties storageProperties) {
         LocalStorageProperty localStorageProperty = (LocalStorageProperty) storageProperties;
         this.localStorageClient = new LocalStorageClient(localStorageProperty);
+        fileLimitDay = localStorageProperty.getFileLimitDay();
         classLogger.info("Init Center local storage client successfully!");
     }
 

diff --git a/common/src/main/java/com/microsoft/hydralab/common/file/impl/local/LocalStorageProperty.java b/common/src/main/java/com/microsoft/hydralab/common/file/impl/local/LocalStorageProperty.java
@@ -21,4 +21,5 @@
 public class LocalStorageProperty extends StorageProperties {
     private String endpoint;
     private String token;
+    private int fileLimitDay;
 }
diff --git a/common/src/main/java/com/microsoft/hydralab/common/util/AttachmentService.java b/common/src/main/java/com/microsoft/hydralab/common/util/AttachmentService.java
@@ -81,7 +81,7 @@ public StorageFileInfo saveFileInStorageAndDB(StorageFileInfo storageFileInfo, F
 
     public StorageFileInfo updateFileInStorageAndDB(StorageFileInfo oldFileInfo, File file, EntityType entityType, Logger logger) {
         int days = (int) ((new Date().getTime() - oldFileInfo.getUpdateTime().getTime()) / 1000 / 60 / 60 / 24);
-        if (days >= storageServiceClientProxy.getStorageFileLimitDay()) {
+        if (storageServiceClientProxy.fileLimitEnabled() && days >= storageServiceClientProxy.getStorageFileLimitDay()) {
             oldFileInfo.setUpdateTime(new Date());
             oldFileInfo.setBlobContainer(entityType.storageContainer);
             oldFileInfo.setBlobUrl(saveFileInStorage(file, oldFileInfo, logger));