Optionally exclude archived files from GUI S3 storage listing (#3076)

* Issue #3029: Optionally exclude archived files from GUI S3 storage listing * Issue #3029: Optionally exclude archived files from GUI S3 storage listing - role model * Issue #3029: Optionally exclude archived files from GUI S3 storage listing - fix for deleted versions and paging * Issue #3029: Optionally exclude archived files from GUI S3 storage listing - fix for default roles test * Issue #3029: Optionally exclude archived files from GUI S3 storage listing - cleanups * Issue #3029: Optionally exclude archived files from GUI S3 storage listing - cleanups * Issue #3029: Optionally exclude archived files from GUI S3 storage listing - add owner to permissions * Issue #3029 move sql migration to current date --------- Co-authored-by: mzueva <mariia_zueva@epam.com>
epam · Feb 16, 2023 · 9655025 · 9655025
1 parent 580f96f
commit 9655025
Show file tree

Hide file tree

Showing 22 changed files with 265 additions and 59 deletions.
diff --git a/api/src/main/java/com/epam/pipeline/acl/datastorage/DataStorageApiService.java b/api/src/main/java/com/epam/pipeline/acl/datastorage/DataStorageApiService.java
@@ -131,16 +131,20 @@ public List<AbstractDataStorage> loadAllByPath(final String identifier) {
         return dataStorageManager.loadAllByPath(identifier);
     }
 
-    @PreAuthorize(AclExpressions.STORAGE_ID_READ)
-    public DataStorageListing getDataStorageItems(final Long id, final String path,
-                                                  Boolean showVersion, Integer pageSize, String marker) {
-        return dataStorageManager.getDataStorageItems(id, path, showVersion, pageSize, marker);
-    }
-
-    @PreAuthorize(AclExpressions.STORAGE_ID_OWNER)
-    public DataStorageListing getDataStorageItemsOwner(Long id, String path,
-                                                       Boolean showVersion, Integer pageSize, String marker) {
-        return dataStorageManager.getDataStorageItems(id, path, showVersion, pageSize, marker);
+    @PreAuthorize(AclExpressions.STORAGE_ID_READ + AclExpressions.AND
+            + AclExpressions.STORAGE_SHOW_ARCHIVED_PERMISSIONS)
+    public DataStorageListing getDataStorageItems(final Long id, final String path, final Boolean showVersion,
+                                                  final Integer pageSize, final String marker,
+                                                  final boolean showArchived) {
+        return dataStorageManager.getDataStorageItems(id, path, showVersion, pageSize, marker, showArchived);
+    }
+
+    @PreAuthorize(AclExpressions.STORAGE_ID_OWNER + AclExpressions.AND
+            + AclExpressions.STORAGE_SHOW_ARCHIVED_PERMISSIONS)
+    public DataStorageListing getDataStorageItemsOwner(final Long id, final String path,
+                                                       final Boolean showVersion, final Integer pageSize,
+                                                       final String marker, final boolean showArchived) {
+        return dataStorageManager.getDataStorageItems(id, path, showVersion, pageSize, marker, showArchived);
     }
 
     @PreAuthorize(AclExpressions.STORAGE_ID_WRITE)

diff --git a/api/src/main/java/com/epam/pipeline/controller/datastorage/DataStorageController.java b/api/src/main/java/com/epam/pipeline/controller/datastorage/DataStorageController.java
@@ -208,13 +208,16 @@ public Result<List<AbstractDataStorage>> findAllDataStorageByPath(@RequestParam(
     public Result<List<AbstractDataStorageItem>> getDataStorageItems(
             @PathVariable(value = ID) final Long id,
             @RequestParam(value = PATH, required = false) final String path,
-            @RequestParam(defaultValue = FALSE) final Boolean showVersion) {
+            @RequestParam(defaultValue = FALSE) final Boolean showVersion,
+            @RequestParam(defaultValue = FALSE) final boolean showArchived) {
         if (showVersion) {
             return Result.success(dataStorageApiService
-                    .getDataStorageItemsOwner(id, path, showVersion, null, null).getResults());
+                    .getDataStorageItemsOwner(id, path, showVersion, null, null, showArchived)
+                    .getResults());
         } else {
             return Result.success(dataStorageApiService
-                    .getDataStorageItems(id, path, showVersion, null, null).getResults());
+                    .getDataStorageItems(id, path, showVersion, null, null, showArchived)
+                    .getResults());
         }
     }
 
@@ -232,13 +235,14 @@ public Result<DataStorageListing> getDataStorageItems(
             @RequestParam(value = PATH, required = false) final String path,
             @RequestParam(defaultValue = FALSE) final Boolean showVersion,
             @RequestParam(required = false) final Integer pageSize,
-            @RequestParam(required = false) final String marker) {
+            @RequestParam(required = false) final String marker,
+            @RequestParam(defaultValue = FALSE) final boolean showArchived) {
         if (showVersion) {
             return Result.success(dataStorageApiService
-                    .getDataStorageItemsOwner(id, path, showVersion, pageSize, marker));
+                    .getDataStorageItemsOwner(id, path, showVersion, pageSize, marker, showArchived));
         } else {
             return Result.success(dataStorageApiService
-                    .getDataStorageItems(id, path, showVersion, pageSize, marker));
+                    .getDataStorageItems(id, path, showVersion, pageSize, marker, showArchived));
         }
     }
 

diff --git a/api/src/main/java/com/epam/pipeline/entity/user/DefaultRoles.java b/api/src/main/java/com/epam/pipeline/entity/user/DefaultRoles.java
@@ -33,7 +33,9 @@ public enum DefaultRoles {
     ROLE_ADVANCED_USER(new Role(null, "ROLE_ADVANCED_USER", true, false, null, null, null, null)),
     ROLE_DTS_MANAGER(new Role(null, "ROLE_DTS_MANAGER", true, false, null, null, null, null)),
     ROLE_SERVICE_ACCOUNT(new Role(null, "ROLE_SERVICE_ACCOUNT", true, false, null, null, null, null)),
-    ROLE_ALLOW_ALL_POLICY(new Role(null, "ROLE_ALLOW_ALL_POLICY", true, false, null, null, null, null));
+    ROLE_ALLOW_ALL_POLICY(new Role(null, "ROLE_ALLOW_ALL_POLICY", true, false, null, null, null, null)),
+    ROLE_STORAGE_ARCHIVE_MANAGER(new Role(null, "ROLE_STORAGE_ARCHIVE_MANAGER", true, false, null, null, null, null)),
+    ROLE_STORAGE_ARCHIVE_READER(new Role(null, "ROLE_STORAGE_ARCHIVE_READER", true, false, null, null, null, null));
 
     private Role role;
 

diff --git a/api/src/main/java/com/epam/pipeline/manager/datastorage/DataStorageManager.java b/api/src/main/java/com/epam/pipeline/manager/datastorage/DataStorageManager.java
@@ -24,6 +24,8 @@
 import com.epam.pipeline.controller.vo.MetadataVO;
 import com.epam.pipeline.controller.vo.data.storage.UpdateDataStorageItemVO;
 import com.epam.pipeline.dao.datastorage.DataStorageDao;
+import com.epam.pipeline.dto.datastorage.lifecycle.restore.StorageRestoreAction;
+import com.epam.pipeline.dto.datastorage.lifecycle.restore.StorageRestorePathType;
 import com.epam.pipeline.entity.AbstractSecuredEntity;
 import com.epam.pipeline.entity.SecuredEntityWithAction;
 import com.epam.pipeline.entity.datastorage.AbstractDataStorage;
@@ -72,6 +74,7 @@
 import com.epam.pipeline.entity.user.StorageContainer;
 import com.epam.pipeline.entity.utils.DateUtils;
 import com.epam.pipeline.exception.ObjectNotFoundException;
+import com.epam.pipeline.manager.datastorage.lifecycle.DataStorageLifecycleRestoredListingContainer;
 import com.epam.pipeline.manager.datastorage.lifecycle.DataStorageLifecycleManager;
 import com.epam.pipeline.manager.datastorage.lifecycle.DataStorageLifecycleRestoreManager;
 import com.epam.pipeline.manager.datastorage.providers.ProviderUtils;
@@ -210,7 +213,6 @@ public class DataStorageManager implements SecuredEntityManager {
     @Autowired
     private DataStorageLifecycleRestoreManager storageLifecycleRestoreManager;
 
-
     private AbstractDataStorageFactory dataStorageFactory =
             AbstractDataStorageFactory.getDefaultDataStorageFactory();
 
@@ -517,15 +519,21 @@ public List<AbstractDataStorage> loadRootDataStorages() {
         return dataStorageDao.loadRootDataStorages();
     }
 
-    public DataStorageListing getDataStorageItems(final Long dataStorageId,
-            final String path, Boolean showVersion, Integer pageSize, String marker) {
+    public DataStorageListing getDataStorageItems(final Long dataStorageId, final String path,
+                                                  final Boolean showVersion, final Integer pageSize,
+                                                  final String marker, final boolean showArchived) {
         AbstractDataStorage dataStorage = load(dataStorageId);
         if (showVersion) {
             Assert.isTrue(dataStorage.isVersioningEnabled(), messageHelper
                     .getMessage(MessageConstants.ERROR_DATASTORAGE_VERSIONING_REQUIRED, dataStorage.getName()));
         }
         Assert.isTrue(pageSize == null || pageSize > 0,
                 messageHelper.getMessage(MessageConstants.ERROR_PAGE_SIZE));
+        if (!showArchived && DataStorageType.S3.equals(dataStorage.getType())) {
+            final DataStorageLifecycleRestoredListingContainer restoredListing = loadRestoredPaths(dataStorage, path);
+            return storageProviderManager.getRestoredItems(dataStorage, path, showVersion, pageSize, marker,
+                    restoredListing);
+        }
         return storageProviderManager.getItems(dataStorage, path, showVersion, pageSize, marker);
     }
 
@@ -744,7 +752,7 @@ public AbstractDataStorageItem getDataStorageItemWithTags(final Long dataStorage
                                                               final String path,
                                                               final Boolean showVersion) {
         final List<AbstractDataStorageItem> dataStorageItems = getDataStorageItems(dataStorageId, path, showVersion,
-                null, null).getResults();
+                null, null, false).getResults();
         if (CollectionUtils.isEmpty(dataStorageItems)) {
             return null;
         }
@@ -1392,4 +1400,41 @@ private String createStorageNameIfRequired(final DataStorageVO dataStorageVO) {
                                sharedPathNamePrefix, SHARED_STORAGE_SUFFIX, Long.toString(latestMirrorNumber + 1));
         }
     }
+
+    private DataStorageLifecycleRestoredListingContainer loadRestoredPaths(final AbstractDataStorage storage,
+                                                                           final String path) {
+        final String normalizedPath = ProviderUtils.delimiterIfEmpty(ProviderUtils.withLeadingDelimiter(path));
+        final List<StorageRestoreAction> restoredItems = storageLifecycleRestoreManager
+                .loadSucceededRestoreActions(storage, path);
+
+        if (CollectionUtils.isEmpty(restoredItems)) {
+            return DataStorageLifecycleRestoredListingContainer.builder()
+                    .folderRestored(false)
+                    .restoredFiles(Collections.emptyList())
+                    .build();
+        }
+        final boolean parentFolderRestored = restoredItems.stream()
+                .filter(action -> StorageRestorePathType.FOLDER.equals(action.getType()))
+                .map(StorageRestoreAction::getPath)
+                .map(restoredPath -> ProviderUtils.DELIMITER.equals(restoredPath)
+                        ? restoredPath : ProviderUtils.withoutTrailingDelimiter(restoredPath))
+                .anyMatch(normalizedPath::startsWith);
+        if (parentFolderRestored) {
+            return DataStorageLifecycleRestoredListingContainer.builder()
+                    .folderRestored(true)
+                    .restoredFiles(Collections.emptyList())
+                    .build();
+        }
+        return DataStorageLifecycleRestoredListingContainer.builder()
+                .folderRestored(false)
+                .restoredFiles(getRestoredFilePaths(restoredItems))
+                .build();
+    }
+
+    private List<String> getRestoredFilePaths(final List<StorageRestoreAction> restoredItems) {
+        return ListUtils.emptyIfNull(restoredItems).stream()
+                .filter(action -> StorageRestorePathType.FILE.equals(action.getType()))
+                .map(StorageRestoreAction::getPath)
+                .collect(Collectors.toList());
+    }
 }
diff --git a/api/src/main/java/com/epam/pipeline/manager/datastorage/StorageProviderManager.java b/api/src/main/java/com/epam/pipeline/manager/datastorage/StorageProviderManager.java
@@ -36,6 +36,7 @@
 import com.epam.pipeline.entity.datastorage.PathDescription;
 import com.epam.pipeline.manager.datastorage.leakagepolicy.SensitiveStorageOperation;
 import com.epam.pipeline.manager.datastorage.leakagepolicy.StorageWriteOperation;
+import com.epam.pipeline.manager.datastorage.lifecycle.DataStorageLifecycleRestoredListingContainer;
 import com.epam.pipeline.manager.datastorage.providers.StorageProvider;
 import com.epam.pipeline.manager.preference.PreferenceManager;
 import com.epam.pipeline.manager.preference.SystemPreferences;
@@ -256,4 +257,11 @@ public DataStorageItemType getItemType(final AbstractDataStorage dataStorage,
                                            final String version) {
         return getStorageProvider(dataStorage).getItemType(dataStorage, path, version);
     }
+
+    public DataStorageListing getRestoredItems(final AbstractDataStorage dataStorage, final String path,
+                                               final Boolean showVersion, final Integer pageSize, final String marker,
+                                               final DataStorageLifecycleRestoredListingContainer restoredListing) {
+        return getStorageProvider(dataStorage)
+                .getItems(dataStorage, path, showVersion, pageSize, marker, restoredListing);
+    }
 }
diff --git a/...a/com/epam/pipeline/manager/datastorage/lifecycle/DataStorageLifecycleRestoreManager.java b/...a/com/epam/pipeline/manager/datastorage/lifecycle/DataStorageLifecycleRestoreManager.java
@@ -198,6 +198,24 @@ public void deleteRestoreActions(final Long datastorageId) {
         dataStoragePathRestoreActionRepository.flush();
     }
 
+    public List<StorageRestoreAction> loadSucceededRestoreActions(final AbstractDataStorage storage,
+                                                                  final String path) {
+        final List<StorageRestoreAction> restoredItems = ListUtils.emptyIfNull(
+                loadEffectiveRestoreStorageActionHierarchy(storage, StorageRestorePath.builder()
+                        .path(path)
+                        .type(StorageRestorePathType.FOLDER)
+                        .build(), false)).stream()
+                .filter(action -> StorageRestoreStatus.SUCCEEDED.equals(action.getStatus()))
+                .collect(Collectors.toList());
+        ListUtils.emptyIfNull(loadEffectiveRestoreStorageActionHierarchy(storage, StorageRestorePath.builder()
+                        .path(path)
+                        .type(StorageRestorePathType.FILE)
+                        .build(), false)).stream()
+                .filter(action -> StorageRestoreStatus.SUCCEEDED.equals(action.getStatus()))
+                .forEach(restoredItems::add);
+        return restoredItems;
+    }
+
     protected StorageRestoreActionEntity buildStoragePathRestoreAction(
             final AbstractDataStorage storage, final StorageRestorePath path,
             final String restoreMode, final Long days, final boolean restoreVersions, final boolean force,

diff --git a/.../pipeline/manager/datastorage/lifecycle/DataStorageLifecycleRestoredListingContainer.java b/.../pipeline/manager/datastorage/lifecycle/DataStorageLifecycleRestoredListingContainer.java
@@ -0,0 +1,36 @@
+/*
+ * Copyright 2023 EPAM Systems, Inc. (https://www.epam.com/)
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.epam.pipeline.manager.datastorage.lifecycle;
+
+import com.epam.pipeline.manager.datastorage.providers.ProviderUtils;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+
+import java.util.List;
+
+@Data
+@AllArgsConstructor
+@Builder
+public class DataStorageLifecycleRestoredListingContainer {
+
+    private List<String> restoredFiles;
+    private boolean folderRestored;
+
+    public boolean containsPath(final String path) {
+        return folderRestored || restoredFiles.contains(ProviderUtils.withLeadingDelimiter(path));
+    }
+}
diff --git a/api/src/main/java/com/epam/pipeline/manager/datastorage/providers/ProviderUtils.java b/api/src/main/java/com/epam/pipeline/manager/datastorage/providers/ProviderUtils.java
@@ -48,6 +48,14 @@ public static String withoutLeadingDelimiter(final String path) {
         return StringUtils.isNotBlank(path) && path.startsWith(DELIMITER) ? path.substring(1) : path;
     }
 
+    public static String withLeadingDelimiter(final String path) {
+        return StringUtils.isNotBlank(path) && path.startsWith(DELIMITER) ? path : DELIMITER + path;
+    }
+
+    public static String delimiterIfEmpty(final String path) {
+        return StringUtils.isBlank(path) ? DELIMITER : path;
+    }
+
     public static DatastoragePath parsePath(final String fullPath) {
         final String[] chunks = fullPath.split(DELIMITER);
         final String root = chunks[0];

diff --git a/api/src/main/java/com/epam/pipeline/manager/datastorage/providers/StorageProvider.java b/api/src/main/java/com/epam/pipeline/manager/datastorage/providers/StorageProvider.java
@@ -42,6 +42,7 @@
 import com.epam.pipeline.entity.datastorage.PathDescription;
 import com.epam.pipeline.entity.datastorage.StoragePolicy;
 import com.epam.pipeline.entity.region.VersioningAwareRegion;
+import com.epam.pipeline.manager.datastorage.lifecycle.DataStorageLifecycleRestoredListingContainer;
 
 public interface StorageProvider<T extends AbstractDataStorage> {
     DataStorageType getStorageType();
@@ -62,6 +63,10 @@ void restoreFileVersion(T dataStorage, String path, String version)
     DataStorageListing getItems(T dataStorage, String path,
             Boolean showVersion, Integer pageSize, String marker);
 
+    DataStorageListing getItems(T dataStorage, String path,
+                                Boolean showVersion, Integer pageSize, String marker,
+                                DataStorageLifecycleRestoredListingContainer restoredListing);
+
     Optional<DataStorageFile> findFile(T dataStorage, String path, String version);
 
     DataStorageDownloadFileUrl generateDownloadURL(T dataStorage, String path, String version,