diff --git a/api/src/main/java/com/epam/pipeline/acl/datastorage/DataStorageApiService.java b/api/src/main/java/com/epam/pipeline/acl/datastorage/DataStorageApiService.java index 351a8695f8..9d3e649ecf 100644 --- a/api/src/main/java/com/epam/pipeline/acl/datastorage/DataStorageApiService.java +++ b/api/src/main/java/com/epam/pipeline/acl/datastorage/DataStorageApiService.java @@ -131,16 +131,20 @@ public List loadAllByPath(final String identifier) { return dataStorageManager.loadAllByPath(identifier); } - @PreAuthorize(AclExpressions.STORAGE_ID_READ) - public DataStorageListing getDataStorageItems(final Long id, final String path, - Boolean showVersion, Integer pageSize, String marker) { - return dataStorageManager.getDataStorageItems(id, path, showVersion, pageSize, marker); - } - - @PreAuthorize(AclExpressions.STORAGE_ID_OWNER) - public DataStorageListing getDataStorageItemsOwner(Long id, String path, - Boolean showVersion, Integer pageSize, String marker) { - return dataStorageManager.getDataStorageItems(id, path, showVersion, pageSize, marker); + @PreAuthorize(AclExpressions.STORAGE_ID_READ + AclExpressions.AND + + AclExpressions.STORAGE_SHOW_ARCHIVED_PERMISSIONS) + public DataStorageListing getDataStorageItems(final Long id, final String path, final Boolean showVersion, + final Integer pageSize, final String marker, + final boolean showArchived) { + return dataStorageManager.getDataStorageItems(id, path, showVersion, pageSize, marker, showArchived); + } + + @PreAuthorize(AclExpressions.STORAGE_ID_OWNER + AclExpressions.AND + + AclExpressions.STORAGE_SHOW_ARCHIVED_PERMISSIONS) + public DataStorageListing getDataStorageItemsOwner(final Long id, final String path, + final Boolean showVersion, final Integer pageSize, + final String marker, final boolean showArchived) { + return dataStorageManager.getDataStorageItems(id, path, showVersion, pageSize, marker, showArchived); } @PreAuthorize(AclExpressions.STORAGE_ID_WRITE) diff --git a/api/src/main/java/com/epam/pipeline/controller/datastorage/DataStorageController.java b/api/src/main/java/com/epam/pipeline/controller/datastorage/DataStorageController.java index c0720592c3..2b1a4b9f20 100644 --- a/api/src/main/java/com/epam/pipeline/controller/datastorage/DataStorageController.java +++ b/api/src/main/java/com/epam/pipeline/controller/datastorage/DataStorageController.java @@ -208,13 +208,16 @@ public Result> findAllDataStorageByPath(@RequestParam( public Result> getDataStorageItems( @PathVariable(value = ID) final Long id, @RequestParam(value = PATH, required = false) final String path, - @RequestParam(defaultValue = FALSE) final Boolean showVersion) { + @RequestParam(defaultValue = FALSE) final Boolean showVersion, + @RequestParam(defaultValue = FALSE) final boolean showArchived) { if (showVersion) { return Result.success(dataStorageApiService - .getDataStorageItemsOwner(id, path, showVersion, null, null).getResults()); + .getDataStorageItemsOwner(id, path, showVersion, null, null, showArchived) + .getResults()); } else { return Result.success(dataStorageApiService - .getDataStorageItems(id, path, showVersion, null, null).getResults()); + .getDataStorageItems(id, path, showVersion, null, null, showArchived) + .getResults()); } } @@ -232,13 +235,14 @@ public Result getDataStorageItems( @RequestParam(value = PATH, required = false) final String path, @RequestParam(defaultValue = FALSE) final Boolean showVersion, @RequestParam(required = false) final Integer pageSize, - @RequestParam(required = false) final String marker) { + @RequestParam(required = false) final String marker, + @RequestParam(defaultValue = FALSE) final boolean showArchived) { if (showVersion) { return Result.success(dataStorageApiService - .getDataStorageItemsOwner(id, path, showVersion, pageSize, marker)); + .getDataStorageItemsOwner(id, path, showVersion, pageSize, marker, showArchived)); } else { return Result.success(dataStorageApiService - .getDataStorageItems(id, path, showVersion, pageSize, marker)); + .getDataStorageItems(id, path, showVersion, pageSize, marker, showArchived)); } } diff --git a/api/src/main/java/com/epam/pipeline/entity/user/DefaultRoles.java b/api/src/main/java/com/epam/pipeline/entity/user/DefaultRoles.java index 6256ef5add..435f93b358 100644 --- a/api/src/main/java/com/epam/pipeline/entity/user/DefaultRoles.java +++ b/api/src/main/java/com/epam/pipeline/entity/user/DefaultRoles.java @@ -33,7 +33,9 @@ public enum DefaultRoles { ROLE_ADVANCED_USER(new Role(null, "ROLE_ADVANCED_USER", true, false, null, null, null, null)), ROLE_DTS_MANAGER(new Role(null, "ROLE_DTS_MANAGER", true, false, null, null, null, null)), ROLE_SERVICE_ACCOUNT(new Role(null, "ROLE_SERVICE_ACCOUNT", true, false, null, null, null, null)), - ROLE_ALLOW_ALL_POLICY(new Role(null, "ROLE_ALLOW_ALL_POLICY", true, false, null, null, null, null)); + ROLE_ALLOW_ALL_POLICY(new Role(null, "ROLE_ALLOW_ALL_POLICY", true, false, null, null, null, null)), + ROLE_STORAGE_ARCHIVE_MANAGER(new Role(null, "ROLE_STORAGE_ARCHIVE_MANAGER", true, false, null, null, null, null)), + ROLE_STORAGE_ARCHIVE_READER(new Role(null, "ROLE_STORAGE_ARCHIVE_READER", true, false, null, null, null, null)); private Role role; diff --git a/api/src/main/java/com/epam/pipeline/manager/datastorage/DataStorageManager.java b/api/src/main/java/com/epam/pipeline/manager/datastorage/DataStorageManager.java index 2cf8939212..54753b6ed1 100644 --- a/api/src/main/java/com/epam/pipeline/manager/datastorage/DataStorageManager.java +++ b/api/src/main/java/com/epam/pipeline/manager/datastorage/DataStorageManager.java @@ -24,6 +24,8 @@ import com.epam.pipeline.controller.vo.MetadataVO; import com.epam.pipeline.controller.vo.data.storage.UpdateDataStorageItemVO; import com.epam.pipeline.dao.datastorage.DataStorageDao; +import com.epam.pipeline.dto.datastorage.lifecycle.restore.StorageRestoreAction; +import com.epam.pipeline.dto.datastorage.lifecycle.restore.StorageRestorePathType; import com.epam.pipeline.entity.AbstractSecuredEntity; import com.epam.pipeline.entity.SecuredEntityWithAction; import com.epam.pipeline.entity.datastorage.AbstractDataStorage; @@ -72,6 +74,7 @@ import com.epam.pipeline.entity.user.StorageContainer; import com.epam.pipeline.entity.utils.DateUtils; import com.epam.pipeline.exception.ObjectNotFoundException; +import com.epam.pipeline.manager.datastorage.lifecycle.DataStorageLifecycleRestoredListingContainer; import com.epam.pipeline.manager.datastorage.lifecycle.DataStorageLifecycleManager; import com.epam.pipeline.manager.datastorage.lifecycle.DataStorageLifecycleRestoreManager; import com.epam.pipeline.manager.datastorage.providers.ProviderUtils; @@ -210,7 +213,6 @@ public class DataStorageManager implements SecuredEntityManager { @Autowired private DataStorageLifecycleRestoreManager storageLifecycleRestoreManager; - private AbstractDataStorageFactory dataStorageFactory = AbstractDataStorageFactory.getDefaultDataStorageFactory(); @@ -517,8 +519,9 @@ public List loadRootDataStorages() { return dataStorageDao.loadRootDataStorages(); } - public DataStorageListing getDataStorageItems(final Long dataStorageId, - final String path, Boolean showVersion, Integer pageSize, String marker) { + public DataStorageListing getDataStorageItems(final Long dataStorageId, final String path, + final Boolean showVersion, final Integer pageSize, + final String marker, final boolean showArchived) { AbstractDataStorage dataStorage = load(dataStorageId); if (showVersion) { Assert.isTrue(dataStorage.isVersioningEnabled(), messageHelper @@ -526,6 +529,11 @@ public DataStorageListing getDataStorageItems(final Long dataStorageId, } Assert.isTrue(pageSize == null || pageSize > 0, messageHelper.getMessage(MessageConstants.ERROR_PAGE_SIZE)); + if (!showArchived && DataStorageType.S3.equals(dataStorage.getType())) { + final DataStorageLifecycleRestoredListingContainer restoredListing = loadRestoredPaths(dataStorage, path); + return storageProviderManager.getRestoredItems(dataStorage, path, showVersion, pageSize, marker, + restoredListing); + } return storageProviderManager.getItems(dataStorage, path, showVersion, pageSize, marker); } @@ -744,7 +752,7 @@ public AbstractDataStorageItem getDataStorageItemWithTags(final Long dataStorage final String path, final Boolean showVersion) { final List dataStorageItems = getDataStorageItems(dataStorageId, path, showVersion, - null, null).getResults(); + null, null, false).getResults(); if (CollectionUtils.isEmpty(dataStorageItems)) { return null; } @@ -1392,4 +1400,41 @@ private String createStorageNameIfRequired(final DataStorageVO dataStorageVO) { sharedPathNamePrefix, SHARED_STORAGE_SUFFIX, Long.toString(latestMirrorNumber + 1)); } } + + private DataStorageLifecycleRestoredListingContainer loadRestoredPaths(final AbstractDataStorage storage, + final String path) { + final String normalizedPath = ProviderUtils.delimiterIfEmpty(ProviderUtils.withLeadingDelimiter(path)); + final List restoredItems = storageLifecycleRestoreManager + .loadSucceededRestoreActions(storage, path); + + if (CollectionUtils.isEmpty(restoredItems)) { + return DataStorageLifecycleRestoredListingContainer.builder() + .folderRestored(false) + .restoredFiles(Collections.emptyList()) + .build(); + } + final boolean parentFolderRestored = restoredItems.stream() + .filter(action -> StorageRestorePathType.FOLDER.equals(action.getType())) + .map(StorageRestoreAction::getPath) + .map(restoredPath -> ProviderUtils.DELIMITER.equals(restoredPath) + ? restoredPath : ProviderUtils.withoutTrailingDelimiter(restoredPath)) + .anyMatch(normalizedPath::startsWith); + if (parentFolderRestored) { + return DataStorageLifecycleRestoredListingContainer.builder() + .folderRestored(true) + .restoredFiles(Collections.emptyList()) + .build(); + } + return DataStorageLifecycleRestoredListingContainer.builder() + .folderRestored(false) + .restoredFiles(getRestoredFilePaths(restoredItems)) + .build(); + } + + private List getRestoredFilePaths(final List restoredItems) { + return ListUtils.emptyIfNull(restoredItems).stream() + .filter(action -> StorageRestorePathType.FILE.equals(action.getType())) + .map(StorageRestoreAction::getPath) + .collect(Collectors.toList()); + } } diff --git a/api/src/main/java/com/epam/pipeline/manager/datastorage/StorageProviderManager.java b/api/src/main/java/com/epam/pipeline/manager/datastorage/StorageProviderManager.java index 4c877e2f7e..11b6677187 100644 --- a/api/src/main/java/com/epam/pipeline/manager/datastorage/StorageProviderManager.java +++ b/api/src/main/java/com/epam/pipeline/manager/datastorage/StorageProviderManager.java @@ -36,6 +36,7 @@ import com.epam.pipeline.entity.datastorage.PathDescription; import com.epam.pipeline.manager.datastorage.leakagepolicy.SensitiveStorageOperation; import com.epam.pipeline.manager.datastorage.leakagepolicy.StorageWriteOperation; +import com.epam.pipeline.manager.datastorage.lifecycle.DataStorageLifecycleRestoredListingContainer; import com.epam.pipeline.manager.datastorage.providers.StorageProvider; import com.epam.pipeline.manager.preference.PreferenceManager; import com.epam.pipeline.manager.preference.SystemPreferences; @@ -256,4 +257,11 @@ public DataStorageItemType getItemType(final AbstractDataStorage dataStorage, final String version) { return getStorageProvider(dataStorage).getItemType(dataStorage, path, version); } + + public DataStorageListing getRestoredItems(final AbstractDataStorage dataStorage, final String path, + final Boolean showVersion, final Integer pageSize, final String marker, + final DataStorageLifecycleRestoredListingContainer restoredListing) { + return getStorageProvider(dataStorage) + .getItems(dataStorage, path, showVersion, pageSize, marker, restoredListing); + } } diff --git a/api/src/main/java/com/epam/pipeline/manager/datastorage/lifecycle/DataStorageLifecycleRestoreManager.java b/api/src/main/java/com/epam/pipeline/manager/datastorage/lifecycle/DataStorageLifecycleRestoreManager.java index 5d9e33d866..43b1ab2c3f 100644 --- a/api/src/main/java/com/epam/pipeline/manager/datastorage/lifecycle/DataStorageLifecycleRestoreManager.java +++ b/api/src/main/java/com/epam/pipeline/manager/datastorage/lifecycle/DataStorageLifecycleRestoreManager.java @@ -198,6 +198,24 @@ public void deleteRestoreActions(final Long datastorageId) { dataStoragePathRestoreActionRepository.flush(); } + public List loadSucceededRestoreActions(final AbstractDataStorage storage, + final String path) { + final List restoredItems = ListUtils.emptyIfNull( + loadEffectiveRestoreStorageActionHierarchy(storage, StorageRestorePath.builder() + .path(path) + .type(StorageRestorePathType.FOLDER) + .build(), false)).stream() + .filter(action -> StorageRestoreStatus.SUCCEEDED.equals(action.getStatus())) + .collect(Collectors.toList()); + ListUtils.emptyIfNull(loadEffectiveRestoreStorageActionHierarchy(storage, StorageRestorePath.builder() + .path(path) + .type(StorageRestorePathType.FILE) + .build(), false)).stream() + .filter(action -> StorageRestoreStatus.SUCCEEDED.equals(action.getStatus())) + .forEach(restoredItems::add); + return restoredItems; + } + protected StorageRestoreActionEntity buildStoragePathRestoreAction( final AbstractDataStorage storage, final StorageRestorePath path, final String restoreMode, final Long days, final boolean restoreVersions, final boolean force, diff --git a/api/src/main/java/com/epam/pipeline/manager/datastorage/lifecycle/DataStorageLifecycleRestoredListingContainer.java b/api/src/main/java/com/epam/pipeline/manager/datastorage/lifecycle/DataStorageLifecycleRestoredListingContainer.java new file mode 100644 index 0000000000..1f754c9d50 --- /dev/null +++ b/api/src/main/java/com/epam/pipeline/manager/datastorage/lifecycle/DataStorageLifecycleRestoredListingContainer.java @@ -0,0 +1,36 @@ +/* + * Copyright 2023 EPAM Systems, Inc. (https://www.epam.com/) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.epam.pipeline.manager.datastorage.lifecycle; + +import com.epam.pipeline.manager.datastorage.providers.ProviderUtils; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; + +import java.util.List; + +@Data +@AllArgsConstructor +@Builder +public class DataStorageLifecycleRestoredListingContainer { + + private List restoredFiles; + private boolean folderRestored; + + public boolean containsPath(final String path) { + return folderRestored || restoredFiles.contains(ProviderUtils.withLeadingDelimiter(path)); + } +} diff --git a/api/src/main/java/com/epam/pipeline/manager/datastorage/providers/ProviderUtils.java b/api/src/main/java/com/epam/pipeline/manager/datastorage/providers/ProviderUtils.java index d1cf8c6052..dd843b51b0 100644 --- a/api/src/main/java/com/epam/pipeline/manager/datastorage/providers/ProviderUtils.java +++ b/api/src/main/java/com/epam/pipeline/manager/datastorage/providers/ProviderUtils.java @@ -48,6 +48,14 @@ public static String withoutLeadingDelimiter(final String path) { return StringUtils.isNotBlank(path) && path.startsWith(DELIMITER) ? path.substring(1) : path; } + public static String withLeadingDelimiter(final String path) { + return StringUtils.isNotBlank(path) && path.startsWith(DELIMITER) ? path : DELIMITER + path; + } + + public static String delimiterIfEmpty(final String path) { + return StringUtils.isBlank(path) ? DELIMITER : path; + } + public static DatastoragePath parsePath(final String fullPath) { final String[] chunks = fullPath.split(DELIMITER); final String root = chunks[0]; diff --git a/api/src/main/java/com/epam/pipeline/manager/datastorage/providers/StorageProvider.java b/api/src/main/java/com/epam/pipeline/manager/datastorage/providers/StorageProvider.java index 7457c47fee..1bf00d6f84 100644 --- a/api/src/main/java/com/epam/pipeline/manager/datastorage/providers/StorageProvider.java +++ b/api/src/main/java/com/epam/pipeline/manager/datastorage/providers/StorageProvider.java @@ -42,6 +42,7 @@ import com.epam.pipeline.entity.datastorage.PathDescription; import com.epam.pipeline.entity.datastorage.StoragePolicy; import com.epam.pipeline.entity.region.VersioningAwareRegion; +import com.epam.pipeline.manager.datastorage.lifecycle.DataStorageLifecycleRestoredListingContainer; public interface StorageProvider { DataStorageType getStorageType(); @@ -62,6 +63,10 @@ void restoreFileVersion(T dataStorage, String path, String version) DataStorageListing getItems(T dataStorage, String path, Boolean showVersion, Integer pageSize, String marker); + DataStorageListing getItems(T dataStorage, String path, + Boolean showVersion, Integer pageSize, String marker, + DataStorageLifecycleRestoredListingContainer restoredListing); + Optional findFile(T dataStorage, String path, String version); DataStorageDownloadFileUrl generateDownloadURL(T dataStorage, String path, String version, diff --git a/api/src/main/java/com/epam/pipeline/manager/datastorage/providers/aws/s3/S3Helper.java b/api/src/main/java/com/epam/pipeline/manager/datastorage/providers/aws/s3/S3Helper.java index 6ccad26a5f..1f66be70e5 100644 --- a/api/src/main/java/com/epam/pipeline/manager/datastorage/providers/aws/s3/S3Helper.java +++ b/api/src/main/java/com/epam/pipeline/manager/datastorage/providers/aws/s3/S3Helper.java @@ -84,6 +84,7 @@ import com.epam.pipeline.entity.datastorage.aws.S3bucketDataStorage; import com.epam.pipeline.entity.region.AwsRegion; import com.epam.pipeline.exception.ObjectNotFoundException; +import com.epam.pipeline.manager.datastorage.lifecycle.DataStorageLifecycleRestoredListingContainer; import com.epam.pipeline.manager.datastorage.providers.ProviderUtils; import com.epam.pipeline.utils.FileContentUtils; import com.google.common.primitives.SignedBytes; @@ -111,6 +112,7 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.TimeZone; @@ -139,6 +141,7 @@ public class S3Helper { private static final String FOLDER_GLOB_SUFFIX = "/**"; private static final String EMPTY_STRING = ""; public static final String STANDARD_STORAGE_CLASS = "STANDARD"; + public static final String STORAGE_CLASS = "StorageClass"; private final MessageHelper messageHelper; @@ -341,7 +344,8 @@ public Stream listDataStorageFiles(final String bucket, final S public DataStorageListing getItems(final String bucket, final String path, final Boolean showVersion, final Integer pageSize, final String marker, final String prefix, - final Set masks) { + final Set masks, + final DataStorageLifecycleRestoredListingContainer restoredListing) { String requestPath = Optional.ofNullable(path).orElse(EMPTY_STRING); AmazonS3 client = getDefaultS3Client(); if (!StringUtils.isNullOrEmpty(requestPath)) { @@ -350,9 +354,9 @@ public DataStorageListing getItems(final String bucket, final String path, final requestPath += ProviderUtils.DELIMITER; } } - DataStorageListing result = showVersion ? - listVersions(client, bucket, requestPath, pageSize, marker, prefix, masks) : - listFiles(client, bucket, requestPath, pageSize, marker, prefix, masks); + DataStorageListing result = showVersion + ? listVersions(client, bucket, requestPath, pageSize, marker, prefix, masks, restoredListing) + : listFiles(client, bucket, requestPath, pageSize, marker, prefix, masks, restoredListing); result.getResults().sort(AbstractDataStorageItem.getStorageItemComparator()); return result; } @@ -386,7 +390,7 @@ private Optional findFile(final AmazonS3 client, file.setVersion(metadata.getVersionId()); final Map labels = new HashMap<>(); if (metadata.getStorageClass() != null) { - labels.put("StorageClass", metadata.getStorageClass()); + labels.put(STORAGE_CLASS, metadata.getStorageClass()); } file.setLabels(labels); return Optional.of(file); @@ -762,7 +766,8 @@ private void applyVersioningConfig(String bucketName, AmazonS3 s3client, private DataStorageListing listFiles(final AmazonS3 client, final String bucket, final String requestPath, final Integer pageSize, final String marker, final String prefix, - final Set masks) { + final Set masks, + final DataStorageLifecycleRestoredListingContainer restoredListing) { ListObjectsV2Request req = new ListObjectsV2Request(); req.setBucketName(bucket); req.setPrefix(requestPath); @@ -812,8 +817,8 @@ private DataStorageListing listFiles(final AmazonS3 client, final String bucket, AbstractS3ObjectWrapper.getWrapper(s3ObjectSummary) .convertToStorageFile(requestPath, prefix); if (file != null) { + final String fileName = requestPath + file.getName(); if (maskingEnabled) { - final String fileName = requestPath + file.getName(); if (compareStrings(fileName, latestMarker) > 0) { listing.setTruncated(false); break; @@ -822,11 +827,17 @@ private DataStorageListing listFiles(final AmazonS3 client, final String bucket, continue; } } + if (filterNotRestored(file, fileName, restoredListing)) { + continue; + } previous = getPreviousKey(previous, s3ObjectSummary.getKey()); items.add(file); } } req.setContinuationToken(listing.getNextContinuationToken()); + if (pageSize != null) { + req.setMaxKeys(pageSize - items.size()); + } } while(listing.isTruncated() && (pageSize == null || items.size() < pageSize)); String returnToken = listing.isTruncated() ? previous : null; return new DataStorageListing(returnToken, items); @@ -853,7 +864,8 @@ private DataStorageFolder parseFolder(String requestPath, String name, String pr private DataStorageListing listVersions(final AmazonS3 client, final String bucket, final String requestPath, final Integer pageSize, final String marker, final String prefix, - final Set masks) { + final Set masks, + final DataStorageLifecycleRestoredListingContainer restoredListing) { ListVersionsRequest request = new ListVersionsRequest() .withBucketName(bucket).withPrefix(requestPath).withDelimiter(ProviderUtils.DELIMITER); if (StringUtils.hasValue(marker)) { @@ -910,6 +922,9 @@ private DataStorageListing listVersions(final AmazonS3 client, final String buck if (file == null) { continue; } + if (filterNotRestored(file, file.getPath(), restoredListing)) { + continue; + } final String fileName = file.getName(); if (maskingEnabled) { final String fileNameWithFolderPrefix = requestPath + fileName; @@ -1293,4 +1308,14 @@ private int compareStrings(final String s1, final String s2) { return SignedBytes.lexicographicalComparator() .compare(s1.getBytes(Charsets.UTF_8), s2.getBytes(Charsets.UTF_8)); } + + private boolean filterNotRestored(final DataStorageFile file, final String fileName, + final DataStorageLifecycleRestoredListingContainer restoredListing) { + return Objects.nonNull(restoredListing) && isArchived(file) && !restoredListing.containsPath(fileName); + } + + private boolean isArchived(final DataStorageFile item) { + final String storageClass = MapUtils.emptyIfNull(item.getLabels()).get(STORAGE_CLASS); + return !StringUtils.isNullOrEmpty(storageClass) && !STANDARD_STORAGE_CLASS.equals(storageClass); + } } diff --git a/api/src/main/java/com/epam/pipeline/manager/datastorage/providers/aws/s3/S3StorageProvider.java b/api/src/main/java/com/epam/pipeline/manager/datastorage/providers/aws/s3/S3StorageProvider.java index d9b5618e18..b23a9dc777 100644 --- a/api/src/main/java/com/epam/pipeline/manager/datastorage/providers/aws/s3/S3StorageProvider.java +++ b/api/src/main/java/com/epam/pipeline/manager/datastorage/providers/aws/s3/S3StorageProvider.java @@ -49,6 +49,7 @@ import com.epam.pipeline.entity.region.VersioningAwareRegion; import com.epam.pipeline.manager.cloud.aws.AWSUtils; import com.epam.pipeline.manager.cloud.aws.S3TemporaryCredentialsGenerator; +import com.epam.pipeline.manager.datastorage.lifecycle.DataStorageLifecycleRestoredListingContainer; import com.epam.pipeline.manager.datastorage.providers.ProviderUtils; import com.epam.pipeline.manager.datastorage.providers.StorageProvider; import com.epam.pipeline.manager.preference.PreferenceManager; @@ -199,13 +200,21 @@ public DataStorageStreamingContent getStream(S3bucketDataStorage dataStorage, St @Override public DataStorageListing getItems(S3bucketDataStorage dataStorage, String path, Boolean showVersion, Integer pageSize, String marker) { + return getItems(dataStorage, path, showVersion, pageSize, marker, null); + } + + @Override + public DataStorageListing getItems(final S3bucketDataStorage dataStorage, final String path, + final Boolean showVersion, final Integer pageSize, final String marker, + final DataStorageLifecycleRestoredListingContainer restoredListing) { final DatastoragePath datastoragePath = ProviderUtils.parsePath(dataStorage.getPath()); final Set activeLinkingMasks = resolveFolderPathListingMasks(dataStorage, path); return getS3Helper(dataStorage) - .getItems(datastoragePath.getRoot(), - ProviderUtils.buildPath(dataStorage, path), showVersion, pageSize, marker, - ProviderUtils.withTrailingDelimiter(datastoragePath.getPath()), - Optional.of(activeLinkingMasks).filter(CollectionUtils::isNotEmpty).orElse(null)); + .getItems(datastoragePath.getRoot(), + ProviderUtils.buildPath(dataStorage, path), showVersion, pageSize, marker, + ProviderUtils.withTrailingDelimiter(datastoragePath.getPath()), + Optional.of(activeLinkingMasks).filter(CollectionUtils::isNotEmpty).orElse(null), + restoredListing); } @Override diff --git a/api/src/main/java/com/epam/pipeline/manager/datastorage/providers/azure/AzureBlobStorageProvider.java b/api/src/main/java/com/epam/pipeline/manager/datastorage/providers/azure/AzureBlobStorageProvider.java index 250a68d631..b2b4d094ad 100644 --- a/api/src/main/java/com/epam/pipeline/manager/datastorage/providers/azure/AzureBlobStorageProvider.java +++ b/api/src/main/java/com/epam/pipeline/manager/datastorage/providers/azure/AzureBlobStorageProvider.java @@ -34,6 +34,7 @@ import com.epam.pipeline.entity.datastorage.azure.AzureBlobStorage; import com.epam.pipeline.entity.region.AzureRegion; import com.epam.pipeline.entity.region.AzureRegionCredentials; +import com.epam.pipeline.manager.datastorage.lifecycle.DataStorageLifecycleRestoredListingContainer; import com.epam.pipeline.manager.datastorage.providers.ProviderUtils; import com.epam.pipeline.manager.datastorage.providers.StorageProvider; import com.epam.pipeline.manager.region.CloudRegionManager; @@ -49,6 +50,7 @@ import java.time.Duration; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.stream.Stream; @@ -103,6 +105,16 @@ public DataStorageListing getItems(final AzureBlobStorage dataStorage, final Str return getAzureStorageHelper(dataStorage).getItems(dataStorage, path, pageSize, marker); } + @Override + public DataStorageListing getItems(final AzureBlobStorage dataStorage, final String path, final Boolean showVersion, + final Integer pageSize, final String marker, + final DataStorageLifecycleRestoredListingContainer restoredListing) { + if (Objects.nonNull(restoredListing)) { + throw new UnsupportedOperationException("Restore mechanism isn't supported for this provider."); + } + return getItems(dataStorage, path, showVersion, pageSize, marker); + } + @Override public Optional findFile(final AzureBlobStorage dataStorage, final String path, diff --git a/api/src/main/java/com/epam/pipeline/manager/datastorage/providers/gcp/GSBucketStorageProvider.java b/api/src/main/java/com/epam/pipeline/manager/datastorage/providers/gcp/GSBucketStorageProvider.java index 4ea8e8ffec..d5e6e78071 100644 --- a/api/src/main/java/com/epam/pipeline/manager/datastorage/providers/gcp/GSBucketStorageProvider.java +++ b/api/src/main/java/com/epam/pipeline/manager/datastorage/providers/gcp/GSBucketStorageProvider.java @@ -36,6 +36,7 @@ import com.epam.pipeline.entity.datastorage.gcp.GSBucketStorage; import com.epam.pipeline.entity.region.GCPRegion; import com.epam.pipeline.manager.cloud.gcp.GCPClient; +import com.epam.pipeline.manager.datastorage.lifecycle.DataStorageLifecycleRestoredListingContainer; import com.epam.pipeline.manager.datastorage.providers.StorageProvider; import com.epam.pipeline.manager.region.CloudRegionManager; import com.epam.pipeline.manager.security.AuthManager; @@ -47,6 +48,7 @@ import java.time.Duration; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.stream.Stream; @@ -105,6 +107,16 @@ public DataStorageListing getItems(final GSBucketStorage dataStorage, final Stri return getHelper(dataStorage).listItems(dataStorage, path, showVersion, pageSize, marker); } + @Override + public DataStorageListing getItems(final GSBucketStorage dataStorage, final String path, final Boolean showVersion, + final Integer pageSize, final String marker, + final DataStorageLifecycleRestoredListingContainer restoredListing) { + if (Objects.nonNull(restoredListing)) { + throw new UnsupportedOperationException("Restore mechanism isn't supported for this provider."); + } + return getItems(dataStorage, path, showVersion, pageSize, marker); + } + @Override public Optional findFile(final GSBucketStorage dataStorage, final String path, final String version) { diff --git a/api/src/main/java/com/epam/pipeline/manager/datastorage/providers/nfs/NFSStorageProvider.java b/api/src/main/java/com/epam/pipeline/manager/datastorage/providers/nfs/NFSStorageProvider.java index 02f02a9601..bce1e6c2d3 100644 --- a/api/src/main/java/com/epam/pipeline/manager/datastorage/providers/nfs/NFSStorageProvider.java +++ b/api/src/main/java/com/epam/pipeline/manager/datastorage/providers/nfs/NFSStorageProvider.java @@ -36,6 +36,7 @@ import com.epam.pipeline.entity.datastorage.PathDescription; import com.epam.pipeline.entity.datastorage.nfs.NFSDataStorage; import com.epam.pipeline.manager.datastorage.FileShareMountManager; +import com.epam.pipeline.manager.datastorage.lifecycle.DataStorageLifecycleRestoredListingContainer; import com.epam.pipeline.manager.datastorage.providers.StorageProvider; import com.epam.pipeline.manager.datastorage.providers.aws.s3.S3Constants; import com.epam.pipeline.manager.preference.PreferenceManager; @@ -70,6 +71,7 @@ import java.util.Date; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; @@ -255,6 +257,16 @@ public DataStorageListing getItems(final NFSDataStorage dataStorage, final Strin } } + @Override + public DataStorageListing getItems(final NFSDataStorage dataStorage, final String path, + final Boolean showVersion, final Integer pageSize, final String marker, + final DataStorageLifecycleRestoredListingContainer restoredListing) { + if (Objects.nonNull(restoredListing)) { + throw new UnsupportedOperationException("Restore mechanism isn't supported for this provider."); + } + return getItems(dataStorage, path, showVersion, pageSize, marker); + } + @Override public Optional findFile(final NFSDataStorage dataStorage, final String path, diff --git a/api/src/main/java/com/epam/pipeline/manager/preprocessing/NgsPreprocessingManager.java b/api/src/main/java/com/epam/pipeline/manager/preprocessing/NgsPreprocessingManager.java index 71713b1e51..6953ac7592 100644 --- a/api/src/main/java/com/epam/pipeline/manager/preprocessing/NgsPreprocessingManager.java +++ b/api/src/main/java/com/epam/pipeline/manager/preprocessing/NgsPreprocessingManager.java @@ -372,7 +372,7 @@ private List mapSampleSheetToMetadataEntities(final Long folde private boolean checkPathExistence(final Long dataStorageId, final String path) { try { // if we can list it, it should exist - storageManager.getDataStorageItems(dataStorageId, path, false, 1, null); + storageManager.getDataStorageItems(dataStorageId, path, false, 1, null, false); return true; } catch (RuntimeException e) { log.debug("Fail to list storage", e); diff --git a/api/src/main/java/com/epam/pipeline/manager/resource/StaticResourcesService.java b/api/src/main/java/com/epam/pipeline/manager/resource/StaticResourcesService.java index 6185ed97c0..3911676a1c 100644 --- a/api/src/main/java/com/epam/pipeline/manager/resource/StaticResourcesService.java +++ b/api/src/main/java/com/epam/pipeline/manager/resource/StaticResourcesService.java @@ -70,7 +70,7 @@ public DataStorageStreamingContent getContent(final String requestPath) { messageHelper.getMessage(MessageConstants.ERROR_STATIC_RESOURCES_FOLDER_PATH)); } final List items = dataStorageManager.getDataStorageItems(storage.getId(), - ProviderUtils.withTrailingDelimiter(filePath), false, null, null).getResults(); + ProviderUtils.withTrailingDelimiter(filePath), false, null, null, false).getResults(); final String templatePath = preferenceManager.getPreference( SystemPreferences.STATIC_RESOURCES_FOLDER_TEMPLATE_PATH); final String html = buildHtml(items, templatePath, filePath); diff --git a/api/src/main/java/com/epam/pipeline/security/acl/AclExpressions.java b/api/src/main/java/com/epam/pipeline/security/acl/AclExpressions.java index b1b07d6cca..4c302aabc1 100644 --- a/api/src/main/java/com/epam/pipeline/security/acl/AclExpressions.java +++ b/api/src/main/java/com/epam/pipeline/security/acl/AclExpressions.java @@ -70,6 +70,10 @@ public final class AclExpressions { "(hasRole('ADMIN') OR @storagePermissionManager.storagePermissionById(#id, 'OWNER')) " + AND + STORAGE_SHARED; + public static final String STORAGE_SHOW_ARCHIVED_PERMISSIONS = "(#showArchived == false OR " + + "hasRole('ADMIN') OR @storagePermissionManager.storagePermissionById(#id, 'OWNER') OR " + + "hasRole('ROLE_STORAGE_ARCHIVE_MANAGER') OR hasRole('ROLE_STORAGE_ARCHIVE_READER'))"; + public static final String STORAGE_ID_PERMISSIONS = "(" + "hasRole('ADMIN') " diff --git a/api/src/main/resources/db/migration/v2023.02.16_14.00__issue_3029_archived_default_roles.sql b/api/src/main/resources/db/migration/v2023.02.16_14.00__issue_3029_archived_default_roles.sql new file mode 100644 index 0000000000..7d70c411b4 --- /dev/null +++ b/api/src/main/resources/db/migration/v2023.02.16_14.00__issue_3029_archived_default_roles.sql @@ -0,0 +1,2 @@ +INSERT INTO pipeline.role (id, name, predefined, user_default) VALUES (nextval('pipeline.s_role'), 'ROLE_STORAGE_ARCHIVE_MANAGER', TRUE, FALSE); +INSERT INTO pipeline.role (id, name, predefined, user_default) VALUES (nextval('pipeline.s_role'), 'ROLE_STORAGE_ARCHIVE_READER', TRUE, FALSE); diff --git a/api/src/test/java/com/epam/pipeline/acl/datastorage/DataStorageApiServiceFileTest.java b/api/src/test/java/com/epam/pipeline/acl/datastorage/DataStorageApiServiceFileTest.java index f69e125d69..c81c6c52f8 100644 --- a/api/src/test/java/com/epam/pipeline/acl/datastorage/DataStorageApiServiceFileTest.java +++ b/api/src/test/java/com/epam/pipeline/acl/datastorage/DataStorageApiServiceFileTest.java @@ -77,11 +77,11 @@ public class DataStorageApiServiceFileTest extends AbstractDataStorageAclTest { @WithMockUser(roles = ADMIN_ROLE) public void shouldGetDataStorageItemsForAdmin() { doReturn(dataStorageListing).when(mockDataStorageManager) - .getDataStorageItems(ID, TEST_STRING, true, TEST_INT, TEST_STRING); + .getDataStorageItems(ID, TEST_STRING, true, TEST_INT, TEST_STRING, false); mockUserContext(context); assertThat(dataStorageApiService.getDataStorageItems( - ID, TEST_STRING, true, TEST_INT, TEST_STRING)).isEqualTo(dataStorageListing); + ID, TEST_STRING, true, TEST_INT, TEST_STRING, false)).isEqualTo(dataStorageListing); } @Test @@ -89,11 +89,11 @@ public void shouldGetDataStorageItemsForAdmin() { public void shouldGetDataStorageItemsWhenPermissionIsGranted() { initAclEntity(s3bucket, AclPermission.READ); doReturn(dataStorageListing).when(mockDataStorageManager) - .getDataStorageItems(ID, TEST_STRING, true, TEST_INT, TEST_STRING); + .getDataStorageItems(ID, TEST_STRING, true, TEST_INT, TEST_STRING, false); initUserAndEntityMocks(SIMPLE_USER, s3bucket, context); assertThat(dataStorageApiService.getDataStorageItems( - ID, TEST_STRING, true, TEST_INT, TEST_STRING)).isEqualTo(dataStorageListing); + ID, TEST_STRING, true, TEST_INT, TEST_STRING, false)).isEqualTo(dataStorageListing); } @Test @@ -101,11 +101,11 @@ public void shouldGetDataStorageItemsWhenPermissionIsGranted() { public void shouldDenyGetDataStorageItemsWhenStoragePermissionIsNotGranted() { initAclEntity(s3bucket); doReturn(dataStorageListing).when(mockDataStorageManager) - .getDataStorageItems(ID, TEST_STRING, true, TEST_INT, TEST_STRING); + .getDataStorageItems(ID, TEST_STRING, true, TEST_INT, TEST_STRING, false); initUserAndEntityMocks(ANOTHER_SIMPLE_USER, s3bucket, context); assertThrows(AccessDeniedException.class, () -> - dataStorageApiService.getDataStorageItems(ID, TEST_STRING, true, TEST_INT, TEST_STRING)); + dataStorageApiService.getDataStorageItems(ID, TEST_STRING, true, TEST_INT, TEST_STRING, false)); } @Test @@ -113,22 +113,22 @@ public void shouldDenyGetDataStorageItemsWhenStoragePermissionIsNotGranted() { public void shouldDenyGetDataStorageItemsWhenCheckStorageSharedPermissionIsNotGranted() { initAclEntity(notSharedS3bucket, AclPermission.READ); doReturn(dataStorageListing).when(mockDataStorageManager) - .getDataStorageItems(ID, TEST_STRING, true, TEST_INT, TEST_STRING); + .getDataStorageItems(ID, TEST_STRING, true, TEST_INT, TEST_STRING, false); initUserAndEntityMocks(ANOTHER_SIMPLE_USER, notSharedS3bucket, externalContext); assertThrows(AccessDeniedException.class, () -> - dataStorageApiService.getDataStorageItems(ID, TEST_STRING, true, TEST_INT, TEST_STRING)); + dataStorageApiService.getDataStorageItems(ID, TEST_STRING, true, TEST_INT, TEST_STRING, false)); } @Test @WithMockUser(roles = ADMIN_ROLE) public void shouldGetDataStorageItemsOwnerForAdmin() { doReturn(dataStorageListing).when(mockDataStorageManager) - .getDataStorageItems(ID, TEST_STRING, true, TEST_INT, TEST_STRING); + .getDataStorageItems(ID, TEST_STRING, true, TEST_INT, TEST_STRING, false); mockUserContext(context); assertThat(dataStorageApiService.getDataStorageItemsOwner( - ID, TEST_STRING, true, TEST_INT, TEST_STRING)).isEqualTo(dataStorageListing); + ID, TEST_STRING, true, TEST_INT, TEST_STRING, false)).isEqualTo(dataStorageListing); } @Test @@ -136,11 +136,11 @@ public void shouldGetDataStorageItemsOwnerForAdmin() { public void shouldGetDataStorageItemsOwnerWhenPermissionIsGranted() { initAclEntity(notSharedS3bucket, AclPermission.OWNER); doReturn(dataStorageListing).when(mockDataStorageManager) - .getDataStorageItems(ID, TEST_STRING, true, TEST_INT, TEST_STRING); + .getDataStorageItems(ID, TEST_STRING, true, TEST_INT, TEST_STRING, false); initUserAndEntityMocks(SIMPLE_USER, notSharedS3bucket, context); assertThat(dataStorageApiService.getDataStorageItemsOwner( - ID, TEST_STRING, true, TEST_INT, TEST_STRING)).isEqualTo(dataStorageListing); + ID, TEST_STRING, true, TEST_INT, TEST_STRING, false)).isEqualTo(dataStorageListing); } @Test @@ -148,11 +148,11 @@ public void shouldGetDataStorageItemsOwnerWhenPermissionIsGranted() { public void shouldDenyGetDataStorageItemsOwnerWhenStoragePermissionIsNotGranted() { initAclEntity(s3bucket); doReturn(dataStorageListing).when(mockDataStorageManager) - .getDataStorageItems(ID, TEST_STRING, true, TEST_INT, TEST_STRING); + .getDataStorageItems(ID, TEST_STRING, true, TEST_INT, TEST_STRING, false); initUserAndEntityMocks(ANOTHER_SIMPLE_USER, s3bucket, context); assertThrows(AccessDeniedException.class, () -> dataStorageApiService.getDataStorageItemsOwner( - ID, TEST_STRING, true, TEST_INT, TEST_STRING)); + ID, TEST_STRING, true, TEST_INT, TEST_STRING, false)); } @Test @@ -160,11 +160,11 @@ public void shouldDenyGetDataStorageItemsOwnerWhenStoragePermissionIsNotGranted( public void shouldDenyGetDataStorageItemsOwnerWhenCheckStorageSharedPermissionIsNotGranted() { initAclEntity(notSharedS3bucket, AclPermission.OWNER); doReturn(dataStorageListing).when(mockDataStorageManager) - .getDataStorageItems(ID, TEST_STRING, true, TEST_INT, TEST_STRING); + .getDataStorageItems(ID, TEST_STRING, true, TEST_INT, TEST_STRING, false); initUserAndEntityMocks(ANOTHER_SIMPLE_USER, notSharedS3bucket, externalContext); assertThrows(AccessDeniedException.class, () -> dataStorageApiService.getDataStorageItemsOwner( - ID, TEST_STRING, true, TEST_INT, TEST_STRING)); + ID, TEST_STRING, true, TEST_INT, TEST_STRING, false)); } @Test diff --git a/api/src/test/java/com/epam/pipeline/controller/datastorage/DataStorageItemControllerTest.java b/api/src/test/java/com/epam/pipeline/controller/datastorage/DataStorageItemControllerTest.java index ad84f54d1c..dfc2eb028d 100644 --- a/api/src/test/java/com/epam/pipeline/controller/datastorage/DataStorageItemControllerTest.java +++ b/api/src/test/java/com/epam/pipeline/controller/datastorage/DataStorageItemControllerTest.java @@ -68,11 +68,11 @@ public void shouldGetDataStorageFolder() { params.add(PATH, TEST); params.add(SHOW_VERSION, FALSE_AS_STRING); Mockito.doReturn(dataStorageListing) - .when(mockStorageApiService).getDataStorageItems(ID, TEST, false, null, null); + .when(mockStorageApiService).getDataStorageItems(ID, TEST, false, null, null, false); final MvcResult mvcResult = performRequest(get(String.format(DATASTORAGE_ITEMS_URL, ID)).params(params)); - Mockito.verify(mockStorageApiService).getDataStorageItems(ID, TEST, false, null, null); + Mockito.verify(mockStorageApiService).getDataStorageItems(ID, TEST, false, null, null, false); assertResponse(mvcResult, folders, DatastorageCreatorUtils.DATA_STORAGE_FOLDER_LIST_TYPE); } @@ -86,11 +86,11 @@ public void shouldGetDataStorageOwnerFile() { params.add(PATH, TEST); params.add(SHOW_VERSION, TRUE_AS_STRING); Mockito.doReturn(dataStorageListing) - .when(mockStorageApiService).getDataStorageItemsOwner(ID, TEST, true, null, null); + .when(mockStorageApiService).getDataStorageItemsOwner(ID, TEST, true, null, null, false); final MvcResult mvcResult = performRequest(get(String.format(DATASTORAGE_ITEMS_URL, ID)).params(params)); - Mockito.verify(mockStorageApiService).getDataStorageItemsOwner(ID, TEST, true, null, null); + Mockito.verify(mockStorageApiService).getDataStorageItemsOwner(ID, TEST, true, null, null, false); assertResponse(mvcResult, files, DatastorageCreatorUtils.DATA_STORAGE_FILE_LIST_TYPE); } @@ -109,11 +109,11 @@ public void shouldGetDataStorageItems() { params.add(PAGE_SIZE, ID_AS_STRING); params.add(MARKER, TEST); Mockito.doReturn(dataStorageListing) - .when(mockStorageApiService).getDataStorageItems(ID, TEST, false, TEST_INT, TEST); + .when(mockStorageApiService).getDataStorageItems(ID, TEST, false, TEST_INT, TEST, false); final MvcResult mvcResult = performRequest(get(String.format(DATASTORAGE_LISTING_URL, ID)).params(params)); - Mockito.verify(mockStorageApiService).getDataStorageItems(ID, TEST, false, TEST_INT, TEST); + Mockito.verify(mockStorageApiService).getDataStorageItems(ID, TEST, false, TEST_INT, TEST, false); assertResponse(mvcResult, dataStorageListing, DatastorageCreatorUtils.DATA_STORAGE_LISTING_TYPE); } diff --git a/api/src/test/java/com/epam/pipeline/dao/user/RoleDaoTest.java b/api/src/test/java/com/epam/pipeline/dao/user/RoleDaoTest.java index 8b19e3d424..c39977490e 100644 --- a/api/src/test/java/com/epam/pipeline/dao/user/RoleDaoTest.java +++ b/api/src/test/java/com/epam/pipeline/dao/user/RoleDaoTest.java @@ -45,7 +45,7 @@ @Transactional public class RoleDaoTest extends AbstractJdbcTest { - private static final int EXPECTED_DEFAULT_ROLES_NUMBER = 15; + private static final int EXPECTED_DEFAULT_ROLES_NUMBER = 17; private static final String TEST_USER1 = "test_user1"; private static final String TEST_ROLE = "ROLE_TEST"; private static final String TEST_ROLE_UPDATED = "NEW_ROLE"; diff --git a/api/src/test/java/com/epam/pipeline/dao/user/UserDaoTest.java b/api/src/test/java/com/epam/pipeline/dao/user/UserDaoTest.java index 4518fd01ad..19be30e032 100644 --- a/api/src/test/java/com/epam/pipeline/dao/user/UserDaoTest.java +++ b/api/src/test/java/com/epam/pipeline/dao/user/UserDaoTest.java @@ -56,7 +56,7 @@ public class UserDaoTest extends AbstractJdbcTest { private static final String ATTRIBUTES_KEY = "email"; private static final String ATTRIBUTES_VALUE = "test_email"; private static final String ATTRIBUTES_VALUE2 = "Mail@epam.com"; - private static final int EXPECTED_DEFAULT_ROLES_NUMBER = 15; + private static final int EXPECTED_DEFAULT_ROLES_NUMBER = 17; private static final String TEST_ROLE = "ROLE_TEST"; @Autowired