From e3541dfe084e5595efcb3a79bf6a82b72bc64f1c Mon Sep 17 00:00:00 2001 From: Ekaterina_Kazachkova Date: Fri, 10 Feb 2023 17:28:45 +0100 Subject: [PATCH 1/8] Issue #3029: Optionally exclude archived files from GUI S3 storage listing --- .../datastorage/DataStorageApiService.java | 14 ++-- .../datastorage/DataStorageController.java | 16 ++-- .../datastorage/DataStorageManager.java | 73 ++++++++++++++++++- .../datastorage/StorageProviderManager.java | 8 ++ ...rageLifecycleRestoredListingContainer.java | 36 +++++++++ .../datastorage/providers/ProviderUtils.java | 8 ++ .../providers/StorageProvider.java | 5 ++ .../providers/aws/s3/S3Helper.java | 37 ++++++++-- .../providers/aws/s3/S3StorageProvider.java | 17 ++++- .../azure/AzureBlobStorageProvider.java | 12 +++ .../gcp/GSBucketStorageProvider.java | 12 +++ .../providers/nfs/NFSStorageProvider.java | 12 +++ .../NgsPreprocessingManager.java | 3 +- .../resource/StaticResourcesService.java | 3 +- .../DataStorageApiServiceFileTest.java | 32 ++++---- .../DataStorageItemControllerTest.java | 12 +-- 16 files changed, 248 insertions(+), 52 deletions(-) create mode 100644 api/src/main/java/com/epam/pipeline/manager/datastorage/lifecycle/DataStorageLifecycleRestoredListingContainer.java diff --git a/api/src/main/java/com/epam/pipeline/acl/datastorage/DataStorageApiService.java b/api/src/main/java/com/epam/pipeline/acl/datastorage/DataStorageApiService.java index 351a8695f8..367a171c55 100644 --- a/api/src/main/java/com/epam/pipeline/acl/datastorage/DataStorageApiService.java +++ b/api/src/main/java/com/epam/pipeline/acl/datastorage/DataStorageApiService.java @@ -132,15 +132,17 @@ public List loadAllByPath(final String identifier) { } @PreAuthorize(AclExpressions.STORAGE_ID_READ) - public DataStorageListing getDataStorageItems(final Long id, final String path, - Boolean showVersion, Integer pageSize, String marker) { - return dataStorageManager.getDataStorageItems(id, path, showVersion, pageSize, marker); + public DataStorageListing getDataStorageItems(final Long id, final String path, final Boolean showVersion, + final Integer pageSize, final String marker, + final boolean showArchived) { + return dataStorageManager.getDataStorageItems(id, path, showVersion, pageSize, marker, showArchived); } @PreAuthorize(AclExpressions.STORAGE_ID_OWNER) - public DataStorageListing getDataStorageItemsOwner(Long id, String path, - Boolean showVersion, Integer pageSize, String marker) { - return dataStorageManager.getDataStorageItems(id, path, showVersion, pageSize, marker); + public DataStorageListing getDataStorageItemsOwner(final Long id, final String path, + final Boolean showVersion, final Integer pageSize, + final String marker, final boolean showArchived) { + return dataStorageManager.getDataStorageItems(id, path, showVersion, pageSize, marker, showArchived); } @PreAuthorize(AclExpressions.STORAGE_ID_WRITE) diff --git a/api/src/main/java/com/epam/pipeline/controller/datastorage/DataStorageController.java b/api/src/main/java/com/epam/pipeline/controller/datastorage/DataStorageController.java index c0720592c3..2b1a4b9f20 100644 --- a/api/src/main/java/com/epam/pipeline/controller/datastorage/DataStorageController.java +++ b/api/src/main/java/com/epam/pipeline/controller/datastorage/DataStorageController.java @@ -208,13 +208,16 @@ public Result> findAllDataStorageByPath(@RequestParam( public Result> getDataStorageItems( @PathVariable(value = ID) final Long id, @RequestParam(value = PATH, required = false) final String path, - @RequestParam(defaultValue = FALSE) final Boolean showVersion) { + @RequestParam(defaultValue = FALSE) final Boolean showVersion, + @RequestParam(defaultValue = FALSE) final boolean showArchived) { if (showVersion) { return Result.success(dataStorageApiService - .getDataStorageItemsOwner(id, path, showVersion, null, null).getResults()); + .getDataStorageItemsOwner(id, path, showVersion, null, null, showArchived) + .getResults()); } else { return Result.success(dataStorageApiService - .getDataStorageItems(id, path, showVersion, null, null).getResults()); + .getDataStorageItems(id, path, showVersion, null, null, showArchived) + .getResults()); } } @@ -232,13 +235,14 @@ public Result getDataStorageItems( @RequestParam(value = PATH, required = false) final String path, @RequestParam(defaultValue = FALSE) final Boolean showVersion, @RequestParam(required = false) final Integer pageSize, - @RequestParam(required = false) final String marker) { + @RequestParam(required = false) final String marker, + @RequestParam(defaultValue = FALSE) final boolean showArchived) { if (showVersion) { return Result.success(dataStorageApiService - .getDataStorageItemsOwner(id, path, showVersion, pageSize, marker)); + .getDataStorageItemsOwner(id, path, showVersion, pageSize, marker, showArchived)); } else { return Result.success(dataStorageApiService - .getDataStorageItems(id, path, showVersion, pageSize, marker)); + .getDataStorageItems(id, path, showVersion, pageSize, marker, showArchived)); } } diff --git a/api/src/main/java/com/epam/pipeline/manager/datastorage/DataStorageManager.java b/api/src/main/java/com/epam/pipeline/manager/datastorage/DataStorageManager.java index 2cf8939212..e358529211 100644 --- a/api/src/main/java/com/epam/pipeline/manager/datastorage/DataStorageManager.java +++ b/api/src/main/java/com/epam/pipeline/manager/datastorage/DataStorageManager.java @@ -24,6 +24,10 @@ import com.epam.pipeline.controller.vo.MetadataVO; import com.epam.pipeline.controller.vo.data.storage.UpdateDataStorageItemVO; import com.epam.pipeline.dao.datastorage.DataStorageDao; +import com.epam.pipeline.dto.datastorage.lifecycle.restore.StorageRestoreAction; +import com.epam.pipeline.dto.datastorage.lifecycle.restore.StorageRestorePath; +import com.epam.pipeline.dto.datastorage.lifecycle.restore.StorageRestorePathType; +import com.epam.pipeline.dto.datastorage.lifecycle.restore.StorageRestoreStatus; import com.epam.pipeline.entity.AbstractSecuredEntity; import com.epam.pipeline.entity.SecuredEntityWithAction; import com.epam.pipeline.entity.datastorage.AbstractDataStorage; @@ -72,6 +76,7 @@ import com.epam.pipeline.entity.user.StorageContainer; import com.epam.pipeline.entity.utils.DateUtils; import com.epam.pipeline.exception.ObjectNotFoundException; +import com.epam.pipeline.manager.datastorage.lifecycle.DataStorageLifecycleRestoredListingContainer; import com.epam.pipeline.manager.datastorage.lifecycle.DataStorageLifecycleManager; import com.epam.pipeline.manager.datastorage.lifecycle.DataStorageLifecycleRestoreManager; import com.epam.pipeline.manager.datastorage.providers.ProviderUtils; @@ -210,7 +215,6 @@ public class DataStorageManager implements SecuredEntityManager { @Autowired private DataStorageLifecycleRestoreManager storageLifecycleRestoreManager; - private AbstractDataStorageFactory dataStorageFactory = AbstractDataStorageFactory.getDefaultDataStorageFactory(); @@ -517,8 +521,9 @@ public List loadRootDataStorages() { return dataStorageDao.loadRootDataStorages(); } - public DataStorageListing getDataStorageItems(final Long dataStorageId, - final String path, Boolean showVersion, Integer pageSize, String marker) { + public DataStorageListing getDataStorageItems(final Long dataStorageId, final String path, + final Boolean showVersion, final Integer pageSize, + final String marker, final boolean showArchived) { AbstractDataStorage dataStorage = load(dataStorageId); if (showVersion) { Assert.isTrue(dataStorage.isVersioningEnabled(), messageHelper @@ -526,6 +531,11 @@ public DataStorageListing getDataStorageItems(final Long dataStorageId, } Assert.isTrue(pageSize == null || pageSize > 0, messageHelper.getMessage(MessageConstants.ERROR_PAGE_SIZE)); + if (!showArchived && DataStorageType.S3.equals(dataStorage.getType())) { + final DataStorageLifecycleRestoredListingContainer restoredListing = loadRestoredPaths(dataStorage, path); + return storageProviderManager.getRestoredItems(dataStorage, path, showVersion, pageSize, marker, + restoredListing); + } return storageProviderManager.getItems(dataStorage, path, showVersion, pageSize, marker); } @@ -744,7 +754,7 @@ public AbstractDataStorageItem getDataStorageItemWithTags(final Long dataStorage final String path, final Boolean showVersion) { final List dataStorageItems = getDataStorageItems(dataStorageId, path, showVersion, - null, null).getResults(); + null, null, false).getResults(); if (CollectionUtils.isEmpty(dataStorageItems)) { return null; } @@ -1392,4 +1402,59 @@ private String createStorageNameIfRequired(final DataStorageVO dataStorageVO) { sharedPathNamePrefix, SHARED_STORAGE_SUFFIX, Long.toString(latestMirrorNumber + 1)); } } + + private DataStorageLifecycleRestoredListingContainer loadRestoredPaths(final AbstractDataStorage storage, + final String path) { + final String normalizedPath = ProviderUtils.delimiterIfEmpty(ProviderUtils.withLeadingDelimiter(path)); + final List restoredItems = loadSucceededRestoreActions(storage, path); + + if (CollectionUtils.isEmpty(restoredItems)) { + return DataStorageLifecycleRestoredListingContainer.builder() + .folderRestored(false) + .restoredFiles(Collections.emptyList()) + .build(); + } + final boolean parentFolderRestored = restoredItems.stream() + .filter(action -> StorageRestorePathType.FOLDER.equals(action.getType())) + .map(StorageRestoreAction::getPath) + .map(restoredPath -> ProviderUtils.DELIMITER.equals(restoredPath) + ? restoredPath : ProviderUtils.withoutTrailingDelimiter(restoredPath)) + .anyMatch(normalizedPath::startsWith); + if (parentFolderRestored) { + return DataStorageLifecycleRestoredListingContainer.builder() + .folderRestored(true) + .restoredFiles(Collections.emptyList()) + .build(); + } + return DataStorageLifecycleRestoredListingContainer.builder() + .folderRestored(false) + .restoredFiles(getRestoredFilePaths(restoredItems)) + .build(); + } + + private List getRestoredFilePaths(final List restoredItems) { + return ListUtils.emptyIfNull(restoredItems).stream() + .filter(action -> StorageRestorePathType.FILE.equals(action.getType())) + .map(StorageRestoreAction::getPath) + .collect(Collectors.toList()); + } + + private List loadSucceededRestoreActions(final AbstractDataStorage storage, + final String path) { + final List restoredItems = ListUtils.emptyIfNull(storageLifecycleRestoreManager + .loadEffectiveRestoreStorageActionHierarchy(storage, StorageRestorePath.builder() + .path(path) + .type(StorageRestorePathType.FOLDER) + .build(), false)).stream() + .filter(action -> StorageRestoreStatus.SUCCEEDED.equals(action.getStatus())) + .collect(Collectors.toList()); + ListUtils.emptyIfNull(storageLifecycleRestoreManager + .loadEffectiveRestoreStorageActionHierarchy(storage, StorageRestorePath.builder() + .path(path) + .type(StorageRestorePathType.FILE) + .build(), false)).stream() + .filter(action -> StorageRestoreStatus.SUCCEEDED.equals(action.getStatus())) + .forEach(restoredItems::add); + return restoredItems; + } } diff --git a/api/src/main/java/com/epam/pipeline/manager/datastorage/StorageProviderManager.java b/api/src/main/java/com/epam/pipeline/manager/datastorage/StorageProviderManager.java index 4c877e2f7e..11b6677187 100644 --- a/api/src/main/java/com/epam/pipeline/manager/datastorage/StorageProviderManager.java +++ b/api/src/main/java/com/epam/pipeline/manager/datastorage/StorageProviderManager.java @@ -36,6 +36,7 @@ import com.epam.pipeline.entity.datastorage.PathDescription; import com.epam.pipeline.manager.datastorage.leakagepolicy.SensitiveStorageOperation; import com.epam.pipeline.manager.datastorage.leakagepolicy.StorageWriteOperation; +import com.epam.pipeline.manager.datastorage.lifecycle.DataStorageLifecycleRestoredListingContainer; import com.epam.pipeline.manager.datastorage.providers.StorageProvider; import com.epam.pipeline.manager.preference.PreferenceManager; import com.epam.pipeline.manager.preference.SystemPreferences; @@ -256,4 +257,11 @@ public DataStorageItemType getItemType(final AbstractDataStorage dataStorage, final String version) { return getStorageProvider(dataStorage).getItemType(dataStorage, path, version); } + + public DataStorageListing getRestoredItems(final AbstractDataStorage dataStorage, final String path, + final Boolean showVersion, final Integer pageSize, final String marker, + final DataStorageLifecycleRestoredListingContainer restoredListing) { + return getStorageProvider(dataStorage) + .getItems(dataStorage, path, showVersion, pageSize, marker, restoredListing); + } } diff --git a/api/src/main/java/com/epam/pipeline/manager/datastorage/lifecycle/DataStorageLifecycleRestoredListingContainer.java b/api/src/main/java/com/epam/pipeline/manager/datastorage/lifecycle/DataStorageLifecycleRestoredListingContainer.java new file mode 100644 index 0000000000..1f754c9d50 --- /dev/null +++ b/api/src/main/java/com/epam/pipeline/manager/datastorage/lifecycle/DataStorageLifecycleRestoredListingContainer.java @@ -0,0 +1,36 @@ +/* + * Copyright 2023 EPAM Systems, Inc. (https://www.epam.com/) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.epam.pipeline.manager.datastorage.lifecycle; + +import com.epam.pipeline.manager.datastorage.providers.ProviderUtils; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; + +import java.util.List; + +@Data +@AllArgsConstructor +@Builder +public class DataStorageLifecycleRestoredListingContainer { + + private List restoredFiles; + private boolean folderRestored; + + public boolean containsPath(final String path) { + return folderRestored || restoredFiles.contains(ProviderUtils.withLeadingDelimiter(path)); + } +} diff --git a/api/src/main/java/com/epam/pipeline/manager/datastorage/providers/ProviderUtils.java b/api/src/main/java/com/epam/pipeline/manager/datastorage/providers/ProviderUtils.java index d1cf8c6052..dd843b51b0 100644 --- a/api/src/main/java/com/epam/pipeline/manager/datastorage/providers/ProviderUtils.java +++ b/api/src/main/java/com/epam/pipeline/manager/datastorage/providers/ProviderUtils.java @@ -48,6 +48,14 @@ public static String withoutLeadingDelimiter(final String path) { return StringUtils.isNotBlank(path) && path.startsWith(DELIMITER) ? path.substring(1) : path; } + public static String withLeadingDelimiter(final String path) { + return StringUtils.isNotBlank(path) && path.startsWith(DELIMITER) ? path : DELIMITER + path; + } + + public static String delimiterIfEmpty(final String path) { + return StringUtils.isBlank(path) ? DELIMITER : path; + } + public static DatastoragePath parsePath(final String fullPath) { final String[] chunks = fullPath.split(DELIMITER); final String root = chunks[0]; diff --git a/api/src/main/java/com/epam/pipeline/manager/datastorage/providers/StorageProvider.java b/api/src/main/java/com/epam/pipeline/manager/datastorage/providers/StorageProvider.java index 7457c47fee..1bf00d6f84 100644 --- a/api/src/main/java/com/epam/pipeline/manager/datastorage/providers/StorageProvider.java +++ b/api/src/main/java/com/epam/pipeline/manager/datastorage/providers/StorageProvider.java @@ -42,6 +42,7 @@ import com.epam.pipeline.entity.datastorage.PathDescription; import com.epam.pipeline.entity.datastorage.StoragePolicy; import com.epam.pipeline.entity.region.VersioningAwareRegion; +import com.epam.pipeline.manager.datastorage.lifecycle.DataStorageLifecycleRestoredListingContainer; public interface StorageProvider { DataStorageType getStorageType(); @@ -62,6 +63,10 @@ void restoreFileVersion(T dataStorage, String path, String version) DataStorageListing getItems(T dataStorage, String path, Boolean showVersion, Integer pageSize, String marker); + DataStorageListing getItems(T dataStorage, String path, + Boolean showVersion, Integer pageSize, String marker, + DataStorageLifecycleRestoredListingContainer restoredListing); + Optional findFile(T dataStorage, String path, String version); DataStorageDownloadFileUrl generateDownloadURL(T dataStorage, String path, String version, diff --git a/api/src/main/java/com/epam/pipeline/manager/datastorage/providers/aws/s3/S3Helper.java b/api/src/main/java/com/epam/pipeline/manager/datastorage/providers/aws/s3/S3Helper.java index 6ccad26a5f..6244101744 100644 --- a/api/src/main/java/com/epam/pipeline/manager/datastorage/providers/aws/s3/S3Helper.java +++ b/api/src/main/java/com/epam/pipeline/manager/datastorage/providers/aws/s3/S3Helper.java @@ -84,6 +84,7 @@ import com.epam.pipeline.entity.datastorage.aws.S3bucketDataStorage; import com.epam.pipeline.entity.region.AwsRegion; import com.epam.pipeline.exception.ObjectNotFoundException; +import com.epam.pipeline.manager.datastorage.lifecycle.DataStorageLifecycleRestoredListingContainer; import com.epam.pipeline.manager.datastorage.providers.ProviderUtils; import com.epam.pipeline.utils.FileContentUtils; import com.google.common.primitives.SignedBytes; @@ -111,6 +112,7 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.TimeZone; @@ -139,6 +141,7 @@ public class S3Helper { private static final String FOLDER_GLOB_SUFFIX = "/**"; private static final String EMPTY_STRING = ""; public static final String STANDARD_STORAGE_CLASS = "STANDARD"; + public static final String STORAGE_CLASS = "StorageClass"; private final MessageHelper messageHelper; @@ -341,7 +344,8 @@ public Stream listDataStorageFiles(final String bucket, final S public DataStorageListing getItems(final String bucket, final String path, final Boolean showVersion, final Integer pageSize, final String marker, final String prefix, - final Set masks) { + final Set masks, + final DataStorageLifecycleRestoredListingContainer restoredListing) { String requestPath = Optional.ofNullable(path).orElse(EMPTY_STRING); AmazonS3 client = getDefaultS3Client(); if (!StringUtils.isNullOrEmpty(requestPath)) { @@ -350,9 +354,9 @@ public DataStorageListing getItems(final String bucket, final String path, final requestPath += ProviderUtils.DELIMITER; } } - DataStorageListing result = showVersion ? - listVersions(client, bucket, requestPath, pageSize, marker, prefix, masks) : - listFiles(client, bucket, requestPath, pageSize, marker, prefix, masks); + DataStorageListing result = showVersion + ? listVersions(client, bucket, requestPath, pageSize, marker, prefix, masks, restoredListing) + : listFiles(client, bucket, requestPath, pageSize, marker, prefix, masks, restoredListing); result.getResults().sort(AbstractDataStorageItem.getStorageItemComparator()); return result; } @@ -386,7 +390,7 @@ private Optional findFile(final AmazonS3 client, file.setVersion(metadata.getVersionId()); final Map labels = new HashMap<>(); if (metadata.getStorageClass() != null) { - labels.put("StorageClass", metadata.getStorageClass()); + labels.put(STORAGE_CLASS, metadata.getStorageClass()); } file.setLabels(labels); return Optional.of(file); @@ -762,7 +766,8 @@ private void applyVersioningConfig(String bucketName, AmazonS3 s3client, private DataStorageListing listFiles(final AmazonS3 client, final String bucket, final String requestPath, final Integer pageSize, final String marker, final String prefix, - final Set masks) { + final Set masks, + final DataStorageLifecycleRestoredListingContainer restoredListing) { ListObjectsV2Request req = new ListObjectsV2Request(); req.setBucketName(bucket); req.setPrefix(requestPath); @@ -812,8 +817,8 @@ private DataStorageListing listFiles(final AmazonS3 client, final String bucket, AbstractS3ObjectWrapper.getWrapper(s3ObjectSummary) .convertToStorageFile(requestPath, prefix); if (file != null) { + final String fileName = requestPath + file.getName(); if (maskingEnabled) { - final String fileName = requestPath + file.getName(); if (compareStrings(fileName, latestMarker) > 0) { listing.setTruncated(false); break; @@ -822,6 +827,9 @@ private DataStorageListing listFiles(final AmazonS3 client, final String bucket, continue; } } + if (filterNotRestored(file, fileName, restoredListing)) { + continue; + } previous = getPreviousKey(previous, s3ObjectSummary.getKey()); items.add(file); } @@ -853,7 +861,8 @@ private DataStorageFolder parseFolder(String requestPath, String name, String pr private DataStorageListing listVersions(final AmazonS3 client, final String bucket, final String requestPath, final Integer pageSize, final String marker, final String prefix, - final Set masks) { + final Set masks, + final DataStorageLifecycleRestoredListingContainer restoredListing) { ListVersionsRequest request = new ListVersionsRequest() .withBucketName(bucket).withPrefix(requestPath).withDelimiter(ProviderUtils.DELIMITER); if (StringUtils.hasValue(marker)) { @@ -911,6 +920,9 @@ private DataStorageListing listVersions(final AmazonS3 client, final String buck continue; } final String fileName = file.getName(); + if (filterNotRestored(file, file.getPath(), restoredListing)) { + continue; + } if (maskingEnabled) { final String fileNameWithFolderPrefix = requestPath + fileName; if (compareStrings(fileNameWithFolderPrefix, latestMarker) > 0) { @@ -1293,4 +1305,13 @@ private int compareStrings(final String s1, final String s2) { return SignedBytes.lexicographicalComparator() .compare(s1.getBytes(Charsets.UTF_8), s2.getBytes(Charsets.UTF_8)); } + + private boolean filterNotRestored(final DataStorageFile file, final String fileName, + final DataStorageLifecycleRestoredListingContainer restoredListing) { + return Objects.nonNull(restoredListing) && isArchived(file) && !restoredListing.containsPath(fileName); + } + + private boolean isArchived(final DataStorageFile item) { + return !STANDARD_STORAGE_CLASS.equals(MapUtils.emptyIfNull(item.getLabels()).get(STORAGE_CLASS)); + } } diff --git a/api/src/main/java/com/epam/pipeline/manager/datastorage/providers/aws/s3/S3StorageProvider.java b/api/src/main/java/com/epam/pipeline/manager/datastorage/providers/aws/s3/S3StorageProvider.java index d9b5618e18..b23a9dc777 100644 --- a/api/src/main/java/com/epam/pipeline/manager/datastorage/providers/aws/s3/S3StorageProvider.java +++ b/api/src/main/java/com/epam/pipeline/manager/datastorage/providers/aws/s3/S3StorageProvider.java @@ -49,6 +49,7 @@ import com.epam.pipeline.entity.region.VersioningAwareRegion; import com.epam.pipeline.manager.cloud.aws.AWSUtils; import com.epam.pipeline.manager.cloud.aws.S3TemporaryCredentialsGenerator; +import com.epam.pipeline.manager.datastorage.lifecycle.DataStorageLifecycleRestoredListingContainer; import com.epam.pipeline.manager.datastorage.providers.ProviderUtils; import com.epam.pipeline.manager.datastorage.providers.StorageProvider; import com.epam.pipeline.manager.preference.PreferenceManager; @@ -199,13 +200,21 @@ public DataStorageStreamingContent getStream(S3bucketDataStorage dataStorage, St @Override public DataStorageListing getItems(S3bucketDataStorage dataStorage, String path, Boolean showVersion, Integer pageSize, String marker) { + return getItems(dataStorage, path, showVersion, pageSize, marker, null); + } + + @Override + public DataStorageListing getItems(final S3bucketDataStorage dataStorage, final String path, + final Boolean showVersion, final Integer pageSize, final String marker, + final DataStorageLifecycleRestoredListingContainer restoredListing) { final DatastoragePath datastoragePath = ProviderUtils.parsePath(dataStorage.getPath()); final Set activeLinkingMasks = resolveFolderPathListingMasks(dataStorage, path); return getS3Helper(dataStorage) - .getItems(datastoragePath.getRoot(), - ProviderUtils.buildPath(dataStorage, path), showVersion, pageSize, marker, - ProviderUtils.withTrailingDelimiter(datastoragePath.getPath()), - Optional.of(activeLinkingMasks).filter(CollectionUtils::isNotEmpty).orElse(null)); + .getItems(datastoragePath.getRoot(), + ProviderUtils.buildPath(dataStorage, path), showVersion, pageSize, marker, + ProviderUtils.withTrailingDelimiter(datastoragePath.getPath()), + Optional.of(activeLinkingMasks).filter(CollectionUtils::isNotEmpty).orElse(null), + restoredListing); } @Override diff --git a/api/src/main/java/com/epam/pipeline/manager/datastorage/providers/azure/AzureBlobStorageProvider.java b/api/src/main/java/com/epam/pipeline/manager/datastorage/providers/azure/AzureBlobStorageProvider.java index 250a68d631..b2b4d094ad 100644 --- a/api/src/main/java/com/epam/pipeline/manager/datastorage/providers/azure/AzureBlobStorageProvider.java +++ b/api/src/main/java/com/epam/pipeline/manager/datastorage/providers/azure/AzureBlobStorageProvider.java @@ -34,6 +34,7 @@ import com.epam.pipeline.entity.datastorage.azure.AzureBlobStorage; import com.epam.pipeline.entity.region.AzureRegion; import com.epam.pipeline.entity.region.AzureRegionCredentials; +import com.epam.pipeline.manager.datastorage.lifecycle.DataStorageLifecycleRestoredListingContainer; import com.epam.pipeline.manager.datastorage.providers.ProviderUtils; import com.epam.pipeline.manager.datastorage.providers.StorageProvider; import com.epam.pipeline.manager.region.CloudRegionManager; @@ -49,6 +50,7 @@ import java.time.Duration; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.stream.Stream; @@ -103,6 +105,16 @@ public DataStorageListing getItems(final AzureBlobStorage dataStorage, final Str return getAzureStorageHelper(dataStorage).getItems(dataStorage, path, pageSize, marker); } + @Override + public DataStorageListing getItems(final AzureBlobStorage dataStorage, final String path, final Boolean showVersion, + final Integer pageSize, final String marker, + final DataStorageLifecycleRestoredListingContainer restoredListing) { + if (Objects.nonNull(restoredListing)) { + throw new UnsupportedOperationException("Restore mechanism isn't supported for this provider."); + } + return getItems(dataStorage, path, showVersion, pageSize, marker); + } + @Override public Optional findFile(final AzureBlobStorage dataStorage, final String path, diff --git a/api/src/main/java/com/epam/pipeline/manager/datastorage/providers/gcp/GSBucketStorageProvider.java b/api/src/main/java/com/epam/pipeline/manager/datastorage/providers/gcp/GSBucketStorageProvider.java index 4ea8e8ffec..d5e6e78071 100644 --- a/api/src/main/java/com/epam/pipeline/manager/datastorage/providers/gcp/GSBucketStorageProvider.java +++ b/api/src/main/java/com/epam/pipeline/manager/datastorage/providers/gcp/GSBucketStorageProvider.java @@ -36,6 +36,7 @@ import com.epam.pipeline.entity.datastorage.gcp.GSBucketStorage; import com.epam.pipeline.entity.region.GCPRegion; import com.epam.pipeline.manager.cloud.gcp.GCPClient; +import com.epam.pipeline.manager.datastorage.lifecycle.DataStorageLifecycleRestoredListingContainer; import com.epam.pipeline.manager.datastorage.providers.StorageProvider; import com.epam.pipeline.manager.region.CloudRegionManager; import com.epam.pipeline.manager.security.AuthManager; @@ -47,6 +48,7 @@ import java.time.Duration; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.stream.Stream; @@ -105,6 +107,16 @@ public DataStorageListing getItems(final GSBucketStorage dataStorage, final Stri return getHelper(dataStorage).listItems(dataStorage, path, showVersion, pageSize, marker); } + @Override + public DataStorageListing getItems(final GSBucketStorage dataStorage, final String path, final Boolean showVersion, + final Integer pageSize, final String marker, + final DataStorageLifecycleRestoredListingContainer restoredListing) { + if (Objects.nonNull(restoredListing)) { + throw new UnsupportedOperationException("Restore mechanism isn't supported for this provider."); + } + return getItems(dataStorage, path, showVersion, pageSize, marker); + } + @Override public Optional findFile(final GSBucketStorage dataStorage, final String path, final String version) { diff --git a/api/src/main/java/com/epam/pipeline/manager/datastorage/providers/nfs/NFSStorageProvider.java b/api/src/main/java/com/epam/pipeline/manager/datastorage/providers/nfs/NFSStorageProvider.java index 02f02a9601..bce1e6c2d3 100644 --- a/api/src/main/java/com/epam/pipeline/manager/datastorage/providers/nfs/NFSStorageProvider.java +++ b/api/src/main/java/com/epam/pipeline/manager/datastorage/providers/nfs/NFSStorageProvider.java @@ -36,6 +36,7 @@ import com.epam.pipeline.entity.datastorage.PathDescription; import com.epam.pipeline.entity.datastorage.nfs.NFSDataStorage; import com.epam.pipeline.manager.datastorage.FileShareMountManager; +import com.epam.pipeline.manager.datastorage.lifecycle.DataStorageLifecycleRestoredListingContainer; import com.epam.pipeline.manager.datastorage.providers.StorageProvider; import com.epam.pipeline.manager.datastorage.providers.aws.s3.S3Constants; import com.epam.pipeline.manager.preference.PreferenceManager; @@ -70,6 +71,7 @@ import java.util.Date; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; @@ -255,6 +257,16 @@ public DataStorageListing getItems(final NFSDataStorage dataStorage, final Strin } } + @Override + public DataStorageListing getItems(final NFSDataStorage dataStorage, final String path, + final Boolean showVersion, final Integer pageSize, final String marker, + final DataStorageLifecycleRestoredListingContainer restoredListing) { + if (Objects.nonNull(restoredListing)) { + throw new UnsupportedOperationException("Restore mechanism isn't supported for this provider."); + } + return getItems(dataStorage, path, showVersion, pageSize, marker); + } + @Override public Optional findFile(final NFSDataStorage dataStorage, final String path, diff --git a/api/src/main/java/com/epam/pipeline/manager/preprocessing/NgsPreprocessingManager.java b/api/src/main/java/com/epam/pipeline/manager/preprocessing/NgsPreprocessingManager.java index 71713b1e51..9f8f25d11d 100644 --- a/api/src/main/java/com/epam/pipeline/manager/preprocessing/NgsPreprocessingManager.java +++ b/api/src/main/java/com/epam/pipeline/manager/preprocessing/NgsPreprocessingManager.java @@ -372,7 +372,8 @@ private List mapSampleSheetToMetadataEntities(final Long folde private boolean checkPathExistence(final Long dataStorageId, final String path) { try { // if we can list it, it should exist - storageManager.getDataStorageItems(dataStorageId, path, false, 1, null); + // TODO: do we need to check archived? + storageManager.getDataStorageItems(dataStorageId, path, false, 1, null, false); return true; } catch (RuntimeException e) { log.debug("Fail to list storage", e); diff --git a/api/src/main/java/com/epam/pipeline/manager/resource/StaticResourcesService.java b/api/src/main/java/com/epam/pipeline/manager/resource/StaticResourcesService.java index 6185ed97c0..26853e9ceb 100644 --- a/api/src/main/java/com/epam/pipeline/manager/resource/StaticResourcesService.java +++ b/api/src/main/java/com/epam/pipeline/manager/resource/StaticResourcesService.java @@ -69,8 +69,9 @@ public DataStorageStreamingContent getContent(final String requestPath) { throw new InvalidPathException( messageHelper.getMessage(MessageConstants.ERROR_STATIC_RESOURCES_FOLDER_PATH)); } + // TODO: Special error on archived content? final List items = dataStorageManager.getDataStorageItems(storage.getId(), - ProviderUtils.withTrailingDelimiter(filePath), false, null, null).getResults(); + ProviderUtils.withTrailingDelimiter(filePath), false, null, null, false).getResults(); final String templatePath = preferenceManager.getPreference( SystemPreferences.STATIC_RESOURCES_FOLDER_TEMPLATE_PATH); final String html = buildHtml(items, templatePath, filePath); diff --git a/api/src/test/java/com/epam/pipeline/acl/datastorage/DataStorageApiServiceFileTest.java b/api/src/test/java/com/epam/pipeline/acl/datastorage/DataStorageApiServiceFileTest.java index f69e125d69..c81c6c52f8 100644 --- a/api/src/test/java/com/epam/pipeline/acl/datastorage/DataStorageApiServiceFileTest.java +++ b/api/src/test/java/com/epam/pipeline/acl/datastorage/DataStorageApiServiceFileTest.java @@ -77,11 +77,11 @@ public class DataStorageApiServiceFileTest extends AbstractDataStorageAclTest { @WithMockUser(roles = ADMIN_ROLE) public void shouldGetDataStorageItemsForAdmin() { doReturn(dataStorageListing).when(mockDataStorageManager) - .getDataStorageItems(ID, TEST_STRING, true, TEST_INT, TEST_STRING); + .getDataStorageItems(ID, TEST_STRING, true, TEST_INT, TEST_STRING, false); mockUserContext(context); assertThat(dataStorageApiService.getDataStorageItems( - ID, TEST_STRING, true, TEST_INT, TEST_STRING)).isEqualTo(dataStorageListing); + ID, TEST_STRING, true, TEST_INT, TEST_STRING, false)).isEqualTo(dataStorageListing); } @Test @@ -89,11 +89,11 @@ public void shouldGetDataStorageItemsForAdmin() { public void shouldGetDataStorageItemsWhenPermissionIsGranted() { initAclEntity(s3bucket, AclPermission.READ); doReturn(dataStorageListing).when(mockDataStorageManager) - .getDataStorageItems(ID, TEST_STRING, true, TEST_INT, TEST_STRING); + .getDataStorageItems(ID, TEST_STRING, true, TEST_INT, TEST_STRING, false); initUserAndEntityMocks(SIMPLE_USER, s3bucket, context); assertThat(dataStorageApiService.getDataStorageItems( - ID, TEST_STRING, true, TEST_INT, TEST_STRING)).isEqualTo(dataStorageListing); + ID, TEST_STRING, true, TEST_INT, TEST_STRING, false)).isEqualTo(dataStorageListing); } @Test @@ -101,11 +101,11 @@ public void shouldGetDataStorageItemsWhenPermissionIsGranted() { public void shouldDenyGetDataStorageItemsWhenStoragePermissionIsNotGranted() { initAclEntity(s3bucket); doReturn(dataStorageListing).when(mockDataStorageManager) - .getDataStorageItems(ID, TEST_STRING, true, TEST_INT, TEST_STRING); + .getDataStorageItems(ID, TEST_STRING, true, TEST_INT, TEST_STRING, false); initUserAndEntityMocks(ANOTHER_SIMPLE_USER, s3bucket, context); assertThrows(AccessDeniedException.class, () -> - dataStorageApiService.getDataStorageItems(ID, TEST_STRING, true, TEST_INT, TEST_STRING)); + dataStorageApiService.getDataStorageItems(ID, TEST_STRING, true, TEST_INT, TEST_STRING, false)); } @Test @@ -113,22 +113,22 @@ public void shouldDenyGetDataStorageItemsWhenStoragePermissionIsNotGranted() { public void shouldDenyGetDataStorageItemsWhenCheckStorageSharedPermissionIsNotGranted() { initAclEntity(notSharedS3bucket, AclPermission.READ); doReturn(dataStorageListing).when(mockDataStorageManager) - .getDataStorageItems(ID, TEST_STRING, true, TEST_INT, TEST_STRING); + .getDataStorageItems(ID, TEST_STRING, true, TEST_INT, TEST_STRING, false); initUserAndEntityMocks(ANOTHER_SIMPLE_USER, notSharedS3bucket, externalContext); assertThrows(AccessDeniedException.class, () -> - dataStorageApiService.getDataStorageItems(ID, TEST_STRING, true, TEST_INT, TEST_STRING)); + dataStorageApiService.getDataStorageItems(ID, TEST_STRING, true, TEST_INT, TEST_STRING, false)); } @Test @WithMockUser(roles = ADMIN_ROLE) public void shouldGetDataStorageItemsOwnerForAdmin() { doReturn(dataStorageListing).when(mockDataStorageManager) - .getDataStorageItems(ID, TEST_STRING, true, TEST_INT, TEST_STRING); + .getDataStorageItems(ID, TEST_STRING, true, TEST_INT, TEST_STRING, false); mockUserContext(context); assertThat(dataStorageApiService.getDataStorageItemsOwner( - ID, TEST_STRING, true, TEST_INT, TEST_STRING)).isEqualTo(dataStorageListing); + ID, TEST_STRING, true, TEST_INT, TEST_STRING, false)).isEqualTo(dataStorageListing); } @Test @@ -136,11 +136,11 @@ public void shouldGetDataStorageItemsOwnerForAdmin() { public void shouldGetDataStorageItemsOwnerWhenPermissionIsGranted() { initAclEntity(notSharedS3bucket, AclPermission.OWNER); doReturn(dataStorageListing).when(mockDataStorageManager) - .getDataStorageItems(ID, TEST_STRING, true, TEST_INT, TEST_STRING); + .getDataStorageItems(ID, TEST_STRING, true, TEST_INT, TEST_STRING, false); initUserAndEntityMocks(SIMPLE_USER, notSharedS3bucket, context); assertThat(dataStorageApiService.getDataStorageItemsOwner( - ID, TEST_STRING, true, TEST_INT, TEST_STRING)).isEqualTo(dataStorageListing); + ID, TEST_STRING, true, TEST_INT, TEST_STRING, false)).isEqualTo(dataStorageListing); } @Test @@ -148,11 +148,11 @@ public void shouldGetDataStorageItemsOwnerWhenPermissionIsGranted() { public void shouldDenyGetDataStorageItemsOwnerWhenStoragePermissionIsNotGranted() { initAclEntity(s3bucket); doReturn(dataStorageListing).when(mockDataStorageManager) - .getDataStorageItems(ID, TEST_STRING, true, TEST_INT, TEST_STRING); + .getDataStorageItems(ID, TEST_STRING, true, TEST_INT, TEST_STRING, false); initUserAndEntityMocks(ANOTHER_SIMPLE_USER, s3bucket, context); assertThrows(AccessDeniedException.class, () -> dataStorageApiService.getDataStorageItemsOwner( - ID, TEST_STRING, true, TEST_INT, TEST_STRING)); + ID, TEST_STRING, true, TEST_INT, TEST_STRING, false)); } @Test @@ -160,11 +160,11 @@ public void shouldDenyGetDataStorageItemsOwnerWhenStoragePermissionIsNotGranted( public void shouldDenyGetDataStorageItemsOwnerWhenCheckStorageSharedPermissionIsNotGranted() { initAclEntity(notSharedS3bucket, AclPermission.OWNER); doReturn(dataStorageListing).when(mockDataStorageManager) - .getDataStorageItems(ID, TEST_STRING, true, TEST_INT, TEST_STRING); + .getDataStorageItems(ID, TEST_STRING, true, TEST_INT, TEST_STRING, false); initUserAndEntityMocks(ANOTHER_SIMPLE_USER, notSharedS3bucket, externalContext); assertThrows(AccessDeniedException.class, () -> dataStorageApiService.getDataStorageItemsOwner( - ID, TEST_STRING, true, TEST_INT, TEST_STRING)); + ID, TEST_STRING, true, TEST_INT, TEST_STRING, false)); } @Test diff --git a/api/src/test/java/com/epam/pipeline/controller/datastorage/DataStorageItemControllerTest.java b/api/src/test/java/com/epam/pipeline/controller/datastorage/DataStorageItemControllerTest.java index ad84f54d1c..dfc2eb028d 100644 --- a/api/src/test/java/com/epam/pipeline/controller/datastorage/DataStorageItemControllerTest.java +++ b/api/src/test/java/com/epam/pipeline/controller/datastorage/DataStorageItemControllerTest.java @@ -68,11 +68,11 @@ public void shouldGetDataStorageFolder() { params.add(PATH, TEST); params.add(SHOW_VERSION, FALSE_AS_STRING); Mockito.doReturn(dataStorageListing) - .when(mockStorageApiService).getDataStorageItems(ID, TEST, false, null, null); + .when(mockStorageApiService).getDataStorageItems(ID, TEST, false, null, null, false); final MvcResult mvcResult = performRequest(get(String.format(DATASTORAGE_ITEMS_URL, ID)).params(params)); - Mockito.verify(mockStorageApiService).getDataStorageItems(ID, TEST, false, null, null); + Mockito.verify(mockStorageApiService).getDataStorageItems(ID, TEST, false, null, null, false); assertResponse(mvcResult, folders, DatastorageCreatorUtils.DATA_STORAGE_FOLDER_LIST_TYPE); } @@ -86,11 +86,11 @@ public void shouldGetDataStorageOwnerFile() { params.add(PATH, TEST); params.add(SHOW_VERSION, TRUE_AS_STRING); Mockito.doReturn(dataStorageListing) - .when(mockStorageApiService).getDataStorageItemsOwner(ID, TEST, true, null, null); + .when(mockStorageApiService).getDataStorageItemsOwner(ID, TEST, true, null, null, false); final MvcResult mvcResult = performRequest(get(String.format(DATASTORAGE_ITEMS_URL, ID)).params(params)); - Mockito.verify(mockStorageApiService).getDataStorageItemsOwner(ID, TEST, true, null, null); + Mockito.verify(mockStorageApiService).getDataStorageItemsOwner(ID, TEST, true, null, null, false); assertResponse(mvcResult, files, DatastorageCreatorUtils.DATA_STORAGE_FILE_LIST_TYPE); } @@ -109,11 +109,11 @@ public void shouldGetDataStorageItems() { params.add(PAGE_SIZE, ID_AS_STRING); params.add(MARKER, TEST); Mockito.doReturn(dataStorageListing) - .when(mockStorageApiService).getDataStorageItems(ID, TEST, false, TEST_INT, TEST); + .when(mockStorageApiService).getDataStorageItems(ID, TEST, false, TEST_INT, TEST, false); final MvcResult mvcResult = performRequest(get(String.format(DATASTORAGE_LISTING_URL, ID)).params(params)); - Mockito.verify(mockStorageApiService).getDataStorageItems(ID, TEST, false, TEST_INT, TEST); + Mockito.verify(mockStorageApiService).getDataStorageItems(ID, TEST, false, TEST_INT, TEST, false); assertResponse(mvcResult, dataStorageListing, DatastorageCreatorUtils.DATA_STORAGE_LISTING_TYPE); } From 1c8befdb11c5136c24fae53fbb1ae0c7ff5aea10 Mon Sep 17 00:00:00 2001 From: Ekaterina_Kazachkova Date: Tue, 14 Feb 2023 10:57:03 +0100 Subject: [PATCH 2/8] Issue #3029: Optionally exclude archived files from GUI S3 storage listing - role model --- .../pipeline/acl/datastorage/DataStorageApiService.java | 6 ++++-- .../java/com/epam/pipeline/entity/user/DefaultRoles.java | 4 +++- .../java/com/epam/pipeline/security/acl/AclExpressions.java | 3 +++ ...v2023.02.14_14.00__issue_3029_archived_default_roles.sql | 2 ++ 4 files changed, 12 insertions(+), 3 deletions(-) create mode 100644 api/src/main/resources/db/migration/v2023.02.14_14.00__issue_3029_archived_default_roles.sql diff --git a/api/src/main/java/com/epam/pipeline/acl/datastorage/DataStorageApiService.java b/api/src/main/java/com/epam/pipeline/acl/datastorage/DataStorageApiService.java index 367a171c55..9d3e649ecf 100644 --- a/api/src/main/java/com/epam/pipeline/acl/datastorage/DataStorageApiService.java +++ b/api/src/main/java/com/epam/pipeline/acl/datastorage/DataStorageApiService.java @@ -131,14 +131,16 @@ public List loadAllByPath(final String identifier) { return dataStorageManager.loadAllByPath(identifier); } - @PreAuthorize(AclExpressions.STORAGE_ID_READ) + @PreAuthorize(AclExpressions.STORAGE_ID_READ + AclExpressions.AND + + AclExpressions.STORAGE_SHOW_ARCHIVED_PERMISSIONS) public DataStorageListing getDataStorageItems(final Long id, final String path, final Boolean showVersion, final Integer pageSize, final String marker, final boolean showArchived) { return dataStorageManager.getDataStorageItems(id, path, showVersion, pageSize, marker, showArchived); } - @PreAuthorize(AclExpressions.STORAGE_ID_OWNER) + @PreAuthorize(AclExpressions.STORAGE_ID_OWNER + AclExpressions.AND + + AclExpressions.STORAGE_SHOW_ARCHIVED_PERMISSIONS) public DataStorageListing getDataStorageItemsOwner(final Long id, final String path, final Boolean showVersion, final Integer pageSize, final String marker, final boolean showArchived) { diff --git a/api/src/main/java/com/epam/pipeline/entity/user/DefaultRoles.java b/api/src/main/java/com/epam/pipeline/entity/user/DefaultRoles.java index 6256ef5add..435f93b358 100644 --- a/api/src/main/java/com/epam/pipeline/entity/user/DefaultRoles.java +++ b/api/src/main/java/com/epam/pipeline/entity/user/DefaultRoles.java @@ -33,7 +33,9 @@ public enum DefaultRoles { ROLE_ADVANCED_USER(new Role(null, "ROLE_ADVANCED_USER", true, false, null, null, null, null)), ROLE_DTS_MANAGER(new Role(null, "ROLE_DTS_MANAGER", true, false, null, null, null, null)), ROLE_SERVICE_ACCOUNT(new Role(null, "ROLE_SERVICE_ACCOUNT", true, false, null, null, null, null)), - ROLE_ALLOW_ALL_POLICY(new Role(null, "ROLE_ALLOW_ALL_POLICY", true, false, null, null, null, null)); + ROLE_ALLOW_ALL_POLICY(new Role(null, "ROLE_ALLOW_ALL_POLICY", true, false, null, null, null, null)), + ROLE_STORAGE_ARCHIVE_MANAGER(new Role(null, "ROLE_STORAGE_ARCHIVE_MANAGER", true, false, null, null, null, null)), + ROLE_STORAGE_ARCHIVE_READER(new Role(null, "ROLE_STORAGE_ARCHIVE_READER", true, false, null, null, null, null)); private Role role; diff --git a/api/src/main/java/com/epam/pipeline/security/acl/AclExpressions.java b/api/src/main/java/com/epam/pipeline/security/acl/AclExpressions.java index b1b07d6cca..e6355e43fc 100644 --- a/api/src/main/java/com/epam/pipeline/security/acl/AclExpressions.java +++ b/api/src/main/java/com/epam/pipeline/security/acl/AclExpressions.java @@ -70,6 +70,9 @@ public final class AclExpressions { "(hasRole('ADMIN') OR @storagePermissionManager.storagePermissionById(#id, 'OWNER')) " + AND + STORAGE_SHARED; + public static final String STORAGE_SHOW_ARCHIVED_PERMISSIONS = "(#showArchived == false OR " + + "hasRole('ADMIN') OR hasRole('ROLE_STORAGE_ARCHIVE_MANAGER') OR hasRole('ROLE_STORAGE_ARCHIVE_READER'))"; + public static final String STORAGE_ID_PERMISSIONS = "(" + "hasRole('ADMIN') " diff --git a/api/src/main/resources/db/migration/v2023.02.14_14.00__issue_3029_archived_default_roles.sql b/api/src/main/resources/db/migration/v2023.02.14_14.00__issue_3029_archived_default_roles.sql new file mode 100644 index 0000000000..7d70c411b4 --- /dev/null +++ b/api/src/main/resources/db/migration/v2023.02.14_14.00__issue_3029_archived_default_roles.sql @@ -0,0 +1,2 @@ +INSERT INTO pipeline.role (id, name, predefined, user_default) VALUES (nextval('pipeline.s_role'), 'ROLE_STORAGE_ARCHIVE_MANAGER', TRUE, FALSE); +INSERT INTO pipeline.role (id, name, predefined, user_default) VALUES (nextval('pipeline.s_role'), 'ROLE_STORAGE_ARCHIVE_READER', TRUE, FALSE); From efa4dd277663a7cebb78c82dcfab9b112f572423 Mon Sep 17 00:00:00 2001 From: Ekaterina_Kazachkova Date: Tue, 14 Feb 2023 15:30:57 +0100 Subject: [PATCH 3/8] Issue #3029: Optionally exclude archived files from GUI S3 storage listing - fix for deleted versions and paging --- .../manager/datastorage/providers/aws/s3/S3Helper.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/api/src/main/java/com/epam/pipeline/manager/datastorage/providers/aws/s3/S3Helper.java b/api/src/main/java/com/epam/pipeline/manager/datastorage/providers/aws/s3/S3Helper.java index 6244101744..1f66be70e5 100644 --- a/api/src/main/java/com/epam/pipeline/manager/datastorage/providers/aws/s3/S3Helper.java +++ b/api/src/main/java/com/epam/pipeline/manager/datastorage/providers/aws/s3/S3Helper.java @@ -835,6 +835,9 @@ private DataStorageListing listFiles(final AmazonS3 client, final String bucket, } } req.setContinuationToken(listing.getNextContinuationToken()); + if (pageSize != null) { + req.setMaxKeys(pageSize - items.size()); + } } while(listing.isTruncated() && (pageSize == null || items.size() < pageSize)); String returnToken = listing.isTruncated() ? previous : null; return new DataStorageListing(returnToken, items); @@ -919,10 +922,10 @@ private DataStorageListing listVersions(final AmazonS3 client, final String buck if (file == null) { continue; } - final String fileName = file.getName(); if (filterNotRestored(file, file.getPath(), restoredListing)) { continue; } + final String fileName = file.getName(); if (maskingEnabled) { final String fileNameWithFolderPrefix = requestPath + fileName; if (compareStrings(fileNameWithFolderPrefix, latestMarker) > 0) { @@ -1312,6 +1315,7 @@ private boolean filterNotRestored(final DataStorageFile file, final String fileN } private boolean isArchived(final DataStorageFile item) { - return !STANDARD_STORAGE_CLASS.equals(MapUtils.emptyIfNull(item.getLabels()).get(STORAGE_CLASS)); + final String storageClass = MapUtils.emptyIfNull(item.getLabels()).get(STORAGE_CLASS); + return !StringUtils.isNullOrEmpty(storageClass) && !STANDARD_STORAGE_CLASS.equals(storageClass); } } From 245e8cf157c24ba491581d04561e1d6b86e25e37 Mon Sep 17 00:00:00 2001 From: Ekaterina_Kazachkova Date: Tue, 14 Feb 2023 15:57:52 +0100 Subject: [PATCH 4/8] Issue #3029: Optionally exclude archived files from GUI S3 storage listing - fix for default roles test --- api/src/test/java/com/epam/pipeline/dao/user/RoleDaoTest.java | 2 +- api/src/test/java/com/epam/pipeline/dao/user/UserDaoTest.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/api/src/test/java/com/epam/pipeline/dao/user/RoleDaoTest.java b/api/src/test/java/com/epam/pipeline/dao/user/RoleDaoTest.java index 8b19e3d424..c39977490e 100644 --- a/api/src/test/java/com/epam/pipeline/dao/user/RoleDaoTest.java +++ b/api/src/test/java/com/epam/pipeline/dao/user/RoleDaoTest.java @@ -45,7 +45,7 @@ @Transactional public class RoleDaoTest extends AbstractJdbcTest { - private static final int EXPECTED_DEFAULT_ROLES_NUMBER = 15; + private static final int EXPECTED_DEFAULT_ROLES_NUMBER = 17; private static final String TEST_USER1 = "test_user1"; private static final String TEST_ROLE = "ROLE_TEST"; private static final String TEST_ROLE_UPDATED = "NEW_ROLE"; diff --git a/api/src/test/java/com/epam/pipeline/dao/user/UserDaoTest.java b/api/src/test/java/com/epam/pipeline/dao/user/UserDaoTest.java index 4518fd01ad..19be30e032 100644 --- a/api/src/test/java/com/epam/pipeline/dao/user/UserDaoTest.java +++ b/api/src/test/java/com/epam/pipeline/dao/user/UserDaoTest.java @@ -56,7 +56,7 @@ public class UserDaoTest extends AbstractJdbcTest { private static final String ATTRIBUTES_KEY = "email"; private static final String ATTRIBUTES_VALUE = "test_email"; private static final String ATTRIBUTES_VALUE2 = "Mail@epam.com"; - private static final int EXPECTED_DEFAULT_ROLES_NUMBER = 15; + private static final int EXPECTED_DEFAULT_ROLES_NUMBER = 17; private static final String TEST_ROLE = "ROLE_TEST"; @Autowired From 9cb92987d7941020432f14943ad96ebf9b85e122 Mon Sep 17 00:00:00 2001 From: Ekaterina_Kazachkova Date: Wed, 15 Feb 2023 15:41:26 +0100 Subject: [PATCH 5/8] Issue #3029: Optionally exclude archived files from GUI S3 storage listing - cleanups --- .../datastorage/DataStorageManager.java | 22 ++----------------- .../DataStorageLifecycleRestoreManager.java | 18 +++++++++++++++ .../NgsPreprocessingManager.java | 1 - .../resource/StaticResourcesService.java | 1 - 4 files changed, 20 insertions(+), 22 deletions(-) diff --git a/api/src/main/java/com/epam/pipeline/manager/datastorage/DataStorageManager.java b/api/src/main/java/com/epam/pipeline/manager/datastorage/DataStorageManager.java index e358529211..a3e4832fcc 100644 --- a/api/src/main/java/com/epam/pipeline/manager/datastorage/DataStorageManager.java +++ b/api/src/main/java/com/epam/pipeline/manager/datastorage/DataStorageManager.java @@ -1406,7 +1406,8 @@ private String createStorageNameIfRequired(final DataStorageVO dataStorageVO) { private DataStorageLifecycleRestoredListingContainer loadRestoredPaths(final AbstractDataStorage storage, final String path) { final String normalizedPath = ProviderUtils.delimiterIfEmpty(ProviderUtils.withLeadingDelimiter(path)); - final List restoredItems = loadSucceededRestoreActions(storage, path); + final List restoredItems = storageLifecycleRestoreManager + .loadSucceededRestoreActions(storage, path); if (CollectionUtils.isEmpty(restoredItems)) { return DataStorageLifecycleRestoredListingContainer.builder() @@ -1438,23 +1439,4 @@ private List getRestoredFilePaths(final List resto .map(StorageRestoreAction::getPath) .collect(Collectors.toList()); } - - private List loadSucceededRestoreActions(final AbstractDataStorage storage, - final String path) { - final List restoredItems = ListUtils.emptyIfNull(storageLifecycleRestoreManager - .loadEffectiveRestoreStorageActionHierarchy(storage, StorageRestorePath.builder() - .path(path) - .type(StorageRestorePathType.FOLDER) - .build(), false)).stream() - .filter(action -> StorageRestoreStatus.SUCCEEDED.equals(action.getStatus())) - .collect(Collectors.toList()); - ListUtils.emptyIfNull(storageLifecycleRestoreManager - .loadEffectiveRestoreStorageActionHierarchy(storage, StorageRestorePath.builder() - .path(path) - .type(StorageRestorePathType.FILE) - .build(), false)).stream() - .filter(action -> StorageRestoreStatus.SUCCEEDED.equals(action.getStatus())) - .forEach(restoredItems::add); - return restoredItems; - } } diff --git a/api/src/main/java/com/epam/pipeline/manager/datastorage/lifecycle/DataStorageLifecycleRestoreManager.java b/api/src/main/java/com/epam/pipeline/manager/datastorage/lifecycle/DataStorageLifecycleRestoreManager.java index 5d9e33d866..43b1ab2c3f 100644 --- a/api/src/main/java/com/epam/pipeline/manager/datastorage/lifecycle/DataStorageLifecycleRestoreManager.java +++ b/api/src/main/java/com/epam/pipeline/manager/datastorage/lifecycle/DataStorageLifecycleRestoreManager.java @@ -198,6 +198,24 @@ public void deleteRestoreActions(final Long datastorageId) { dataStoragePathRestoreActionRepository.flush(); } + public List loadSucceededRestoreActions(final AbstractDataStorage storage, + final String path) { + final List restoredItems = ListUtils.emptyIfNull( + loadEffectiveRestoreStorageActionHierarchy(storage, StorageRestorePath.builder() + .path(path) + .type(StorageRestorePathType.FOLDER) + .build(), false)).stream() + .filter(action -> StorageRestoreStatus.SUCCEEDED.equals(action.getStatus())) + .collect(Collectors.toList()); + ListUtils.emptyIfNull(loadEffectiveRestoreStorageActionHierarchy(storage, StorageRestorePath.builder() + .path(path) + .type(StorageRestorePathType.FILE) + .build(), false)).stream() + .filter(action -> StorageRestoreStatus.SUCCEEDED.equals(action.getStatus())) + .forEach(restoredItems::add); + return restoredItems; + } + protected StorageRestoreActionEntity buildStoragePathRestoreAction( final AbstractDataStorage storage, final StorageRestorePath path, final String restoreMode, final Long days, final boolean restoreVersions, final boolean force, diff --git a/api/src/main/java/com/epam/pipeline/manager/preprocessing/NgsPreprocessingManager.java b/api/src/main/java/com/epam/pipeline/manager/preprocessing/NgsPreprocessingManager.java index 9f8f25d11d..6953ac7592 100644 --- a/api/src/main/java/com/epam/pipeline/manager/preprocessing/NgsPreprocessingManager.java +++ b/api/src/main/java/com/epam/pipeline/manager/preprocessing/NgsPreprocessingManager.java @@ -372,7 +372,6 @@ private List mapSampleSheetToMetadataEntities(final Long folde private boolean checkPathExistence(final Long dataStorageId, final String path) { try { // if we can list it, it should exist - // TODO: do we need to check archived? storageManager.getDataStorageItems(dataStorageId, path, false, 1, null, false); return true; } catch (RuntimeException e) { diff --git a/api/src/main/java/com/epam/pipeline/manager/resource/StaticResourcesService.java b/api/src/main/java/com/epam/pipeline/manager/resource/StaticResourcesService.java index 26853e9ceb..3911676a1c 100644 --- a/api/src/main/java/com/epam/pipeline/manager/resource/StaticResourcesService.java +++ b/api/src/main/java/com/epam/pipeline/manager/resource/StaticResourcesService.java @@ -69,7 +69,6 @@ public DataStorageStreamingContent getContent(final String requestPath) { throw new InvalidPathException( messageHelper.getMessage(MessageConstants.ERROR_STATIC_RESOURCES_FOLDER_PATH)); } - // TODO: Special error on archived content? final List items = dataStorageManager.getDataStorageItems(storage.getId(), ProviderUtils.withTrailingDelimiter(filePath), false, null, null, false).getResults(); final String templatePath = preferenceManager.getPreference( From 659676080fc95c7fd07751d108fa40912a95d967 Mon Sep 17 00:00:00 2001 From: Ekaterina_Kazachkova Date: Wed, 15 Feb 2023 16:01:27 +0100 Subject: [PATCH 6/8] Issue #3029: Optionally exclude archived files from GUI S3 storage listing - cleanups --- .../epam/pipeline/manager/datastorage/DataStorageManager.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/api/src/main/java/com/epam/pipeline/manager/datastorage/DataStorageManager.java b/api/src/main/java/com/epam/pipeline/manager/datastorage/DataStorageManager.java index a3e4832fcc..54753b6ed1 100644 --- a/api/src/main/java/com/epam/pipeline/manager/datastorage/DataStorageManager.java +++ b/api/src/main/java/com/epam/pipeline/manager/datastorage/DataStorageManager.java @@ -25,9 +25,7 @@ import com.epam.pipeline.controller.vo.data.storage.UpdateDataStorageItemVO; import com.epam.pipeline.dao.datastorage.DataStorageDao; import com.epam.pipeline.dto.datastorage.lifecycle.restore.StorageRestoreAction; -import com.epam.pipeline.dto.datastorage.lifecycle.restore.StorageRestorePath; import com.epam.pipeline.dto.datastorage.lifecycle.restore.StorageRestorePathType; -import com.epam.pipeline.dto.datastorage.lifecycle.restore.StorageRestoreStatus; import com.epam.pipeline.entity.AbstractSecuredEntity; import com.epam.pipeline.entity.SecuredEntityWithAction; import com.epam.pipeline.entity.datastorage.AbstractDataStorage; From 2c9cb407ada98c8f9de21c3bc9bf776cc0a22025 Mon Sep 17 00:00:00 2001 From: Ekaterina_Kazachkova Date: Thu, 16 Feb 2023 09:18:10 +0100 Subject: [PATCH 7/8] Issue #3029: Optionally exclude archived files from GUI S3 storage listing - add owner to permissions --- .../java/com/epam/pipeline/security/acl/AclExpressions.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/api/src/main/java/com/epam/pipeline/security/acl/AclExpressions.java b/api/src/main/java/com/epam/pipeline/security/acl/AclExpressions.java index e6355e43fc..4c302aabc1 100644 --- a/api/src/main/java/com/epam/pipeline/security/acl/AclExpressions.java +++ b/api/src/main/java/com/epam/pipeline/security/acl/AclExpressions.java @@ -71,7 +71,8 @@ public final class AclExpressions { + AND + STORAGE_SHARED; public static final String STORAGE_SHOW_ARCHIVED_PERMISSIONS = "(#showArchived == false OR " - + "hasRole('ADMIN') OR hasRole('ROLE_STORAGE_ARCHIVE_MANAGER') OR hasRole('ROLE_STORAGE_ARCHIVE_READER'))"; + + "hasRole('ADMIN') OR @storagePermissionManager.storagePermissionById(#id, 'OWNER') OR " + + "hasRole('ROLE_STORAGE_ARCHIVE_MANAGER') OR hasRole('ROLE_STORAGE_ARCHIVE_READER'))"; public static final String STORAGE_ID_PERMISSIONS = "(" From 74d5a111cbadffaffc4b6a79d3873dcda7f1b990 Mon Sep 17 00:00:00 2001 From: mzueva Date: Thu, 16 Feb 2023 17:27:09 +0100 Subject: [PATCH 8/8] Issue #3029 move sql migration to current date --- ...l => v2023.02.16_14.00__issue_3029_archived_default_roles.sql} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename api/src/main/resources/db/migration/{v2023.02.14_14.00__issue_3029_archived_default_roles.sql => v2023.02.16_14.00__issue_3029_archived_default_roles.sql} (100%) diff --git a/api/src/main/resources/db/migration/v2023.02.14_14.00__issue_3029_archived_default_roles.sql b/api/src/main/resources/db/migration/v2023.02.16_14.00__issue_3029_archived_default_roles.sql similarity index 100% rename from api/src/main/resources/db/migration/v2023.02.14_14.00__issue_3029_archived_default_roles.sql rename to api/src/main/resources/db/migration/v2023.02.16_14.00__issue_3029_archived_default_roles.sql