From ed4c357f0c00fdcc16632a0c3d2799054bf8d18d Mon Sep 17 00:00:00 2001 From: Jon Harper Date: Tue, 12 Nov 2024 10:24:45 +0100 Subject: [PATCH] S3 based case store (#11) Signed-off-by: Etienne Homer Co-authored-by: Rehili Ghazwa Co-authored-by: HARPER Jon --- pom.xml | 6 + .../powsybl/caseserver/CaseController.java | 41 +- .../com/powsybl/caseserver/CaseException.java | 46 +- .../caseserver/ScheduledCaseCleaner.java | 1 + .../caseserver/SupervisionController.java | 3 +- .../{util => }/CaseDataSourceController.java | 8 +- .../datasource/CaseDataSourceService.java | 28 + ...vice.java => FsCaseDataSourceService.java} | 34 +- .../datasource/S3CaseDataSourceService.java | 77 +++ .../com/powsybl/caseserver/dto/CaseInfos.java | 3 + .../elasticsearch/CaseInfosService.java | 5 + .../repository/CaseMetadataEntity.java | 9 + .../caseserver/repository/StorageConfig.java | 55 ++ .../caseserver/service/CaseService.java | 171 ++++++ .../FsCaseService.java} | 224 ++----- .../caseserver/service/MetadataService.java | 37 ++ .../service/NotificationService.java | 31 + .../caseserver/service/S3CaseService.java | 569 ++++++++++++++++++ .../SupervisionService.java | 2 +- src/main/resources/application-local.yml | 7 +- src/main/resources/config/application.yaml | 14 + .../changesets/changelog_20241017T125247Z.xml | 18 + .../db/changelog/db.changelog-master.yaml | 3 + ...=> AbstractSupervisionControllerTest.java} | 35 +- .../caseserver/CaseFileNameParserTests.java | 95 --- .../CaseInfosELRepositoryTests.java | 27 +- .../ContextConfigurationWithTestChannel.java | 8 +- .../FsSupervisionControllerTest.java | 39 ++ .../S3SupervisionControllerTest.java | 39 ++ .../caseserver/ScheduledCaseCleanerTest.java | 7 +- .../AbstractCaseDataSourceControllerTest.java | 218 +++++++ .../FsCaseDataSourceControllerTest.java | 60 ++ .../S3CaseDataSourceControllerTest.java | 43 ++ .../util/CaseDataSourceControllerTest.java | 195 ------ .../AbstractCaseControllerTest.java} | 74 +-- .../caseserver/service/CaseServiceTest.java | 140 +++++ .../service/FsCaseControllerTest.java | 47 ++ .../service/MinioContainerConfig.java | 53 ++ .../service/S3CaseControllerTest.java | 34 ++ src/test/resources/LF.xml | 42 ++ src/test/resources/LF.xml.gz | Bin 0 -> 804 bytes src/test/resources/LF.zip | Bin 0 -> 941 bytes 42 files changed, 1951 insertions(+), 597 deletions(-) rename src/main/java/com/powsybl/caseserver/datasource/{util => }/CaseDataSourceController.java (92%) create mode 100644 src/main/java/com/powsybl/caseserver/datasource/CaseDataSourceService.java rename src/main/java/com/powsybl/caseserver/datasource/{util/CaseDataSourceService.java => FsCaseDataSourceService.java} (71%) create mode 100644 src/main/java/com/powsybl/caseserver/datasource/S3CaseDataSourceService.java create mode 100644 src/main/java/com/powsybl/caseserver/repository/StorageConfig.java create mode 100644 src/main/java/com/powsybl/caseserver/service/CaseService.java rename src/main/java/com/powsybl/caseserver/{CaseService.java => service/FsCaseService.java} (58%) create mode 100644 src/main/java/com/powsybl/caseserver/service/MetadataService.java create mode 100644 src/main/java/com/powsybl/caseserver/service/NotificationService.java create mode 100644 src/main/java/com/powsybl/caseserver/service/S3CaseService.java rename src/main/java/com/powsybl/caseserver/{services => service}/SupervisionService.java (97%) create mode 100644 src/main/resources/db/changelog/changesets/changelog_20241017T125247Z.xml rename src/test/java/com/powsybl/caseserver/{SupervisionControllerTest.java => AbstractSupervisionControllerTest.java} (78%) create mode 100644 src/test/java/com/powsybl/caseserver/FsSupervisionControllerTest.java create mode 100644 src/test/java/com/powsybl/caseserver/S3SupervisionControllerTest.java create mode 100644 src/test/java/com/powsybl/caseserver/datasource/AbstractCaseDataSourceControllerTest.java create mode 100644 src/test/java/com/powsybl/caseserver/datasource/FsCaseDataSourceControllerTest.java create mode 100644 src/test/java/com/powsybl/caseserver/datasource/S3CaseDataSourceControllerTest.java delete mode 100644 src/test/java/com/powsybl/caseserver/datasource/util/CaseDataSourceControllerTest.java rename src/test/java/com/powsybl/caseserver/{CaseControllerTest.java => service/AbstractCaseControllerTest.java} (94%) create mode 100644 src/test/java/com/powsybl/caseserver/service/CaseServiceTest.java create mode 100644 src/test/java/com/powsybl/caseserver/service/FsCaseControllerTest.java create mode 100644 src/test/java/com/powsybl/caseserver/service/MinioContainerConfig.java create mode 100644 src/test/java/com/powsybl/caseserver/service/S3CaseControllerTest.java create mode 100644 src/test/resources/LF.xml create mode 100644 src/test/resources/LF.xml.gz create mode 100644 src/test/resources/LF.zip diff --git a/pom.xml b/pom.xml index b039ad1..57560e8 100644 --- a/pom.xml +++ b/pom.xml @@ -46,6 +46,7 @@ com.powsybl.caseserver.repository 2.15.0 + 3.2.0 @@ -147,6 +148,11 @@ org.springframework.cloud spring-cloud-stream-binder-rabbit + + io.awspring.cloud + spring-cloud-aws-starter-s3 + ${spring-cloud-aws-starter-s3} + org.springframework.data spring-data-elasticsearch diff --git a/src/main/java/com/powsybl/caseserver/CaseController.java b/src/main/java/com/powsybl/caseserver/CaseController.java index 0f480f9..0e1d6e3 100644 --- a/src/main/java/com/powsybl/caseserver/CaseController.java +++ b/src/main/java/com/powsybl/caseserver/CaseController.java @@ -7,6 +7,9 @@ package com.powsybl.caseserver; import com.powsybl.caseserver.dto.CaseInfos; +import com.powsybl.caseserver.elasticsearch.CaseInfosService; +import com.powsybl.caseserver.service.CaseService; +import com.powsybl.caseserver.service.MetadataService; import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.media.Content; import io.swagger.v3.oas.annotations.media.Schema; @@ -16,11 +19,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.ComponentScan; -import org.springframework.http.ContentDisposition; -import org.springframework.http.HttpHeaders; -import org.springframework.http.MediaType; -import org.springframework.http.ResponseEntity; +import org.springframework.http.*; import org.springframework.web.bind.annotation.DeleteMapping; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PathVariable; @@ -33,7 +34,6 @@ import org.springframework.web.multipart.MultipartFile; import java.io.IOException; -import java.nio.file.Path; import java.util.List; import java.util.Map; import java.util.Properties; @@ -54,14 +54,21 @@ public class CaseController { private static final Logger LOGGER = LoggerFactory.getLogger(CaseController.class); @Autowired + @Qualifier("storageService") private CaseService caseService; + @Autowired + private CaseInfosService caseInfosService; + + @Autowired + private MetadataService metadataService; + @GetMapping(value = "/cases") @Operation(summary = "Get all cases") //For maintenance purpose public ResponseEntity> getCases() { LOGGER.debug("getCases request received"); - List cases = caseService.getCases(caseService.getStorageRootDir()); + List cases = caseService.getCases(); if (cases == null) { return ResponseEntity.noContent().build(); } @@ -72,11 +79,10 @@ public ResponseEntity> getCases() { @Operation(summary = "Get a case infos") public ResponseEntity getCaseInfos(@PathVariable("caseUuid") UUID caseUuid) { LOGGER.debug("getCaseInfos request received"); - Path file = caseService.getCaseFile(caseUuid); - if (file == null) { + if (!caseService.caseExists(caseUuid)) { return ResponseEntity.noContent().build(); } - CaseInfos caseInfos = caseService.getCase(file); + CaseInfos caseInfos = caseService.getCaseInfos(caseUuid); return ResponseEntity.ok().body(caseInfos); } @@ -84,11 +90,10 @@ public ResponseEntity getCaseInfos(@PathVariable("caseUuid") UUID cas @Operation(summary = "Get case Format") public ResponseEntity getCaseFormat(@PathVariable("caseUuid") UUID caseUuid) { LOGGER.debug("getCaseFormat request received"); - Path file = caseService.getCaseFile(caseUuid); - if (file == null) { + if (!caseService.caseExists(caseUuid)) { throw createDirectoryNotFound(caseUuid); } - String caseFormat = caseService.getFormat(file); + String caseFormat = caseService.getFormat(caseUuid); return ResponseEntity.ok().body(caseFormat); } @@ -96,6 +101,9 @@ public ResponseEntity getCaseFormat(@PathVariable("caseUuid") UUID caseU @Operation(summary = "Get case name") public ResponseEntity getCaseName(@PathVariable("caseUuid") UUID caseUuid) { LOGGER.debug("getCaseName request received"); + if (!caseService.caseExists(caseUuid)) { + throw createDirectoryNotFound(caseUuid); + } String caseName = caseService.getCaseName(caseUuid); return ResponseEntity.ok().body(caseName); } @@ -179,7 +187,7 @@ public ResponseEntity duplicateCase( @ApiResponse(responseCode = "404", description = "Source case not found")}) public ResponseEntity disableCaseExpiration(@PathVariable("caseUuid") UUID caseUuid) { LOGGER.debug("disableCaseExpiration request received for caseUuid = {}", caseUuid); - caseService.disableCaseExpiration(caseUuid); + metadataService.disableCaseExpiration(caseUuid); return ResponseEntity.ok().build(); } @@ -187,6 +195,9 @@ public ResponseEntity disableCaseExpiration(@PathVariable("caseUuid") UUID @Operation(summary = "delete a case") public ResponseEntity deleteCase(@PathVariable("caseUuid") UUID caseUuid) { LOGGER.debug("deleteCase request received with parameter caseUuid = {}", caseUuid); + if (!caseService.caseExists(caseUuid)) { + throw createDirectoryNotFound(caseUuid); + } caseService.deleteCase(caseUuid); return ResponseEntity.ok().build(); } @@ -203,14 +214,14 @@ public ResponseEntity deleteCases() { @Operation(summary = "Search cases by metadata") public ResponseEntity> searchCases(@RequestParam(value = "q") String query) { LOGGER.debug("search cases request received"); - List cases = caseService.searchCases(query); + List cases = caseInfosService.searchCaseInfos(query); return ResponseEntity.ok().body(cases); } @GetMapping(value = "/cases/metadata") @Operation(summary = "Get cases Metadata") public ResponseEntity> getMetadata(@RequestParam("ids") List ids) { - LOGGER.debug("get Case metadata"); + LOGGER.debug("get Cases metadata"); return ResponseEntity.ok().body(caseService.getMetadata(ids)); } } diff --git a/src/main/java/com/powsybl/caseserver/CaseException.java b/src/main/java/com/powsybl/caseserver/CaseException.java index 65308c7..8959bbb 100644 --- a/src/main/java/com/powsybl/caseserver/CaseException.java +++ b/src/main/java/com/powsybl/caseserver/CaseException.java @@ -20,11 +20,17 @@ public final class CaseException extends RuntimeException { public enum Type { FILE_NOT_IMPORTABLE, + FILE_NOT_FOUND, STORAGE_DIR_NOT_CREATED, ILLEGAL_FILE_NAME, DIRECTORY_ALREADY_EXISTS, DIRECTORY_EMPTY, DIRECTORY_NOT_FOUND, + ORIGINAL_FILE_NOT_FOUND, + TEMP_FILE_INIT, + TEMP_FILE_PROCESS, + TEMP_DIRECTORY_CREATION, + ZIP_FILE_PROCESS, UNSUPPORTED_FORMAT } @@ -35,7 +41,16 @@ private CaseException(Type type, String msg) { this.type = Objects.requireNonNull(type); } - public static CaseException createDirectoryAreadyExists(Path directory) { + public CaseException(Type type, String message, Throwable e) { + super(message, e); + this.type = type; + } + + public Type getType() { + return type; + } + + public static CaseException createDirectoryAreadyExists(String directory) { Objects.requireNonNull(directory); return new CaseException(Type.DIRECTORY_ALREADY_EXISTS, "A directory with the same name already exists: " + directory); } @@ -50,11 +65,26 @@ public static CaseException createDirectoryNotFound(UUID uuid) { return new CaseException(Type.DIRECTORY_NOT_FOUND, "The directory with the following uuid doesn't exist: " + uuid); } + public static CaseException createOriginalFileNotFound(UUID uuid) { + Objects.requireNonNull(uuid); + return new CaseException(Type.ORIGINAL_FILE_NOT_FOUND, "The original file were not retrieved in the directory with the following uuid: " + uuid); + } + public static CaseException createFileNotImportable(Path file) { Objects.requireNonNull(file); return new CaseException(Type.FILE_NOT_IMPORTABLE, "This file cannot be imported: " + file); } + public static CaseException createFileNotImportable(String file, Exception e) { + Objects.requireNonNull(file); + return new CaseException(Type.FILE_NOT_IMPORTABLE, "This file cannot be imported: " + file, e); + } + + public static CaseException createFileNameNotFound(UUID uuid) { + Objects.requireNonNull(uuid); + return new CaseException(Type.FILE_NOT_FOUND, "The file name with the following uuid doesn't exist: " + uuid); + } + public static CaseException createStorageNotInitialized(Path storageRootDir) { Objects.requireNonNull(storageRootDir); return new CaseException(Type.STORAGE_DIR_NOT_CREATED, "The storage is not initialized: " + storageRootDir); @@ -65,6 +95,20 @@ public static CaseException createIllegalCaseName(String caseName) { return new CaseException(Type.ILLEGAL_FILE_NAME, "This is not an acceptable case name: " + caseName); } + public static CaseException createTempDirectory(UUID uuid, Exception e) { + Objects.requireNonNull(uuid); + return new CaseException(Type.TEMP_DIRECTORY_CREATION, "Error creating temporary directory: " + uuid, e); + } + + public static CaseException createUInitTempFileError(UUID uuid, Throwable e) { + Objects.requireNonNull(uuid); + return new CaseException(Type.TEMP_FILE_INIT, "Error initializing temporary case file: " + uuid, e); + } + + public static CaseException createCopyZipContentError(UUID uuid, Exception e) { + return new CaseException(Type.ZIP_FILE_PROCESS, "Error copying zip content file: " + uuid, e); + } + public static CaseException createUnsupportedFormat(String format) { return new CaseException(Type.UNSUPPORTED_FORMAT, "The format: " + format + " is unsupported"); } diff --git a/src/main/java/com/powsybl/caseserver/ScheduledCaseCleaner.java b/src/main/java/com/powsybl/caseserver/ScheduledCaseCleaner.java index 011bdd3..8ee91eb 100644 --- a/src/main/java/com/powsybl/caseserver/ScheduledCaseCleaner.java +++ b/src/main/java/com/powsybl/caseserver/ScheduledCaseCleaner.java @@ -6,6 +6,7 @@ */ package com.powsybl.caseserver; +import com.powsybl.caseserver.service.CaseService; import com.powsybl.caseserver.repository.CaseMetadataRepository; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/src/main/java/com/powsybl/caseserver/SupervisionController.java b/src/main/java/com/powsybl/caseserver/SupervisionController.java index cf826a9..2f6cc95 100644 --- a/src/main/java/com/powsybl/caseserver/SupervisionController.java +++ b/src/main/java/com/powsybl/caseserver/SupervisionController.java @@ -7,7 +7,8 @@ package com.powsybl.caseserver; import com.powsybl.caseserver.elasticsearch.CaseInfosService; -import com.powsybl.caseserver.services.SupervisionService; +import com.powsybl.caseserver.service.CaseService; +import com.powsybl.caseserver.service.SupervisionService; import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.responses.ApiResponse; import io.swagger.v3.oas.annotations.responses.ApiResponses; diff --git a/src/main/java/com/powsybl/caseserver/datasource/util/CaseDataSourceController.java b/src/main/java/com/powsybl/caseserver/datasource/CaseDataSourceController.java similarity index 92% rename from src/main/java/com/powsybl/caseserver/datasource/util/CaseDataSourceController.java rename to src/main/java/com/powsybl/caseserver/datasource/CaseDataSourceController.java index 2d6e982..f46cda8 100644 --- a/src/main/java/com/powsybl/caseserver/datasource/util/CaseDataSourceController.java +++ b/src/main/java/com/powsybl/caseserver/datasource/CaseDataSourceController.java @@ -1,15 +1,16 @@ /** - * Copyright (c) 2020, RTE (http://www.rte-france.com) + * Copyright (c) 2024, RTE (http://www.rte-france.com) * This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -package com.powsybl.caseserver.datasource.util; +package com.powsybl.caseserver.datasource; import com.powsybl.caseserver.CaseConstants; import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.tags.Tag; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.ComponentScan; import org.springframework.http.MediaType; import org.springframework.http.ResponseEntity; @@ -21,6 +22,8 @@ /** * @author Abdelsalem Hedhili + * @author Ghazwa Rehili + * @author Etienne Homer */ @RestController @RequestMapping(value = "/" + CaseConstants.API_VERSION) @@ -29,6 +32,7 @@ public class CaseDataSourceController { @Autowired + @Qualifier("caseDataSourceService") private CaseDataSourceService caseDataSourceService; @GetMapping(value = "/cases/{caseUuid}/datasource/baseName") diff --git a/src/main/java/com/powsybl/caseserver/datasource/CaseDataSourceService.java b/src/main/java/com/powsybl/caseserver/datasource/CaseDataSourceService.java new file mode 100644 index 0000000..a7d05a4 --- /dev/null +++ b/src/main/java/com/powsybl/caseserver/datasource/CaseDataSourceService.java @@ -0,0 +1,28 @@ +/** + * Copyright (c) 2023, RTE (http://www.rte-france.com) + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ +package com.powsybl.caseserver.datasource; + +import java.util.Set; +import java.util.UUID; + +/** + * @author Abdelsalem Hedhili + */ +public interface CaseDataSourceService { + String getBaseName(UUID caseUuid); + + Boolean datasourceExists(UUID caseUuid, String suffix, String ext); + + Boolean datasourceExists(UUID caseUuid, String fileName); + + byte[] getInputStream(UUID caseUuid, String suffix, String ext); + + byte[] getInputStream(UUID caseUuid, String fileName); + + Set listName(UUID caseUuid, String regex); + +} diff --git a/src/main/java/com/powsybl/caseserver/datasource/util/CaseDataSourceService.java b/src/main/java/com/powsybl/caseserver/datasource/FsCaseDataSourceService.java similarity index 71% rename from src/main/java/com/powsybl/caseserver/datasource/util/CaseDataSourceService.java rename to src/main/java/com/powsybl/caseserver/datasource/FsCaseDataSourceService.java index 7ccc61b..1813812 100644 --- a/src/main/java/com/powsybl/caseserver/datasource/util/CaseDataSourceService.java +++ b/src/main/java/com/powsybl/caseserver/datasource/FsCaseDataSourceService.java @@ -1,12 +1,13 @@ /** - * Copyright (c) 2020, RTE (http://www.rte-france.com) + * Copyright (c) 2023, RTE (http://www.rte-france.com) * This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -package com.powsybl.caseserver.datasource.util; +package com.powsybl.caseserver.datasource; -import com.powsybl.caseserver.CaseService; +import com.powsybl.caseserver.service.CaseService; +import com.powsybl.caseserver.service.FsCaseService; import com.powsybl.commons.datasource.DataSource; import org.apache.commons.io.IOUtils; import org.springframework.beans.factory.annotation.Autowired; @@ -24,20 +25,23 @@ /** * @author Abdelsalem Hedhili + * @author Ghazwa Rehili */ @Service @ComponentScan(basePackageClasses = CaseService.class) -public class CaseDataSourceService { +public class FsCaseDataSourceService implements CaseDataSourceService { @Autowired - private CaseService caseService; + private FsCaseService fsCaseService; - String getBaseName(UUID caseUuid) { + @Override + public String getBaseName(UUID caseUuid) { DataSource dataSource = getDatasource(caseUuid); return dataSource.getBaseName(); } - Boolean datasourceExists(UUID caseUuid, String suffix, String ext) { + @Override + public Boolean datasourceExists(UUID caseUuid, String suffix, String ext) { DataSource dataSource = getDatasource(caseUuid); try { return dataSource.exists(suffix, ext); @@ -46,7 +50,8 @@ Boolean datasourceExists(UUID caseUuid, String suffix, String ext) { } } - Boolean datasourceExists(UUID caseUuid, String fileName) { + @Override + public Boolean datasourceExists(UUID caseUuid, String fileName) { DataSource dataSource = getDatasource(caseUuid); try { return dataSource.exists(fileName); @@ -55,7 +60,8 @@ Boolean datasourceExists(UUID caseUuid, String fileName) { } } - byte[] getInputStream(UUID caseUuid, String fileName) { + @Override + public byte[] getInputStream(UUID caseUuid, String fileName) { DataSource dataSource = getDatasource(caseUuid); try (InputStream inputStream = dataSource.newInputStream(fileName)) { return IOUtils.toByteArray(inputStream); @@ -64,7 +70,8 @@ byte[] getInputStream(UUID caseUuid, String fileName) { } } - byte[] getInputStream(UUID caseUuid, String suffix, String ext) { + @Override + public byte[] getInputStream(UUID caseUuid, String suffix, String ext) { DataSource dataSource = getDatasource(caseUuid); try (InputStream inputStream = dataSource.newInputStream(suffix, ext)) { return IOUtils.toByteArray(inputStream); @@ -73,7 +80,8 @@ byte[] getInputStream(UUID caseUuid, String suffix, String ext) { } } - Set listName(UUID caseUuid, String regex) { + @Override + public Set listName(UUID caseUuid, String regex) { DataSource dataSource = getDatasource(caseUuid); String decodedRegex = URLDecoder.decode(regex, StandardCharsets.UTF_8); try { @@ -84,12 +92,12 @@ Set listName(UUID caseUuid, String regex) { } private DataSource initDatasource(UUID caseUuid) { - Path file = caseService.getCaseFile(caseUuid); + Path file = fsCaseService.getCaseFile(caseUuid); return DataSource.fromPath(file); } private DataSource getDatasource(UUID caseUuid) { - caseService.checkStorageInitialization(); + fsCaseService.checkStorageInitialization(); return initDatasource(caseUuid); } diff --git a/src/main/java/com/powsybl/caseserver/datasource/S3CaseDataSourceService.java b/src/main/java/com/powsybl/caseserver/datasource/S3CaseDataSourceService.java new file mode 100644 index 0000000..87030ad --- /dev/null +++ b/src/main/java/com/powsybl/caseserver/datasource/S3CaseDataSourceService.java @@ -0,0 +1,77 @@ +/** + * Copyright (c) 2024, RTE (http://www.rte-france.com) + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ +package com.powsybl.caseserver.datasource; + +import com.powsybl.caseserver.service.CaseService; +import com.powsybl.caseserver.service.S3CaseService; +import com.powsybl.commons.datasource.DataSource; +import com.powsybl.commons.datasource.DataSourceUtil; +import org.apache.commons.io.IOUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.ComponentScan; +import org.springframework.stereotype.Service; + +import java.nio.file.Files; +import java.util.Set; +import java.util.UUID; + +import static com.powsybl.caseserver.service.S3CaseService.*; + +/** + * @author Ghazwa Rehili + * @author Etienne Homer + */ +@Service +@ComponentScan(basePackageClasses = CaseService.class) +public class S3CaseDataSourceService implements CaseDataSourceService { + + @Autowired + private S3CaseService s3CaseService; + + @Override + public String getBaseName(UUID caseUuid) { + return DataSourceUtil.getBaseName(s3CaseService.getCaseName(caseUuid)); + } + + @Override + public Boolean datasourceExists(UUID caseUuid, String suffix, String ext) { + return s3CaseService.datasourceExists(caseUuid, DataSourceUtil.getFileName(getBaseName(caseUuid), suffix, ext)); + } + + @Override + public Boolean datasourceExists(UUID caseUuid, String fileName) { + return s3CaseService.datasourceExists(caseUuid, fileName); + } + + @Override + public byte[] getInputStream(UUID caseUuid, String fileName) { + String caseName = s3CaseService.getCaseName(caseUuid); + String caseFileKey; + // For archived cases (.zip, .tar, ...), individual files are gzipped in S3 server. + // Here the requested file is decompressed and simply returned. + if (S3CaseService.isArchivedCaseFile(caseName)) { + caseFileKey = uuidToKeyWithFileName(caseUuid, fileName + GZIP_EXTENSION); + return s3CaseService.withS3DownloadedTempPath(caseUuid, caseFileKey, + file -> S3CaseService.decompress(Files.readAllBytes(file))); + } else { + caseFileKey = uuidToKeyWithFileName(caseUuid, caseName); + return s3CaseService.withS3DownloadedTempPath(caseUuid, caseFileKey, + casePath -> IOUtils.toByteArray(DataSource.fromPath(casePath).newInputStream(fileName))); + } + } + + @Override + public byte[] getInputStream(UUID caseUuid, String suffix, String ext) { + return getInputStream(caseUuid, DataSourceUtil.getFileName(getBaseName(caseUuid), suffix, ext)); + } + + @Override + public Set listName(UUID caseUuid, String regex) { + return s3CaseService.listName(caseUuid, regex); + } +} + diff --git a/src/main/java/com/powsybl/caseserver/dto/CaseInfos.java b/src/main/java/com/powsybl/caseserver/dto/CaseInfos.java index 69551f9..def32a5 100644 --- a/src/main/java/com/powsybl/caseserver/dto/CaseInfos.java +++ b/src/main/java/com/powsybl/caseserver/dto/CaseInfos.java @@ -15,6 +15,7 @@ import java.util.UUID; import io.swagger.v3.oas.annotations.media.Schema; +import lombok.AllArgsConstructor; import lombok.Getter; import lombok.NoArgsConstructor; import lombok.NonNull; @@ -30,6 +31,7 @@ */ @SuperBuilder @NoArgsConstructor +@AllArgsConstructor @Getter @Schema(description = "Case infos") @Document(indexName = "#{@environment.getProperty('powsybl-ws.elasticsearch.index.prefix')}cases") @@ -39,6 +41,7 @@ public class CaseInfos { public static final String NAME_HEADER_KEY = "name"; public static final String UUID_HEADER_KEY = "uuid"; public static final String FORMAT_HEADER_KEY = "format"; + public static final String CASE_NAME_HEADER_KEY = "casename"; @Id @NonNull protected UUID uuid; diff --git a/src/main/java/com/powsybl/caseserver/elasticsearch/CaseInfosService.java b/src/main/java/com/powsybl/caseserver/elasticsearch/CaseInfosService.java index 3286410..c6db4a5 100644 --- a/src/main/java/com/powsybl/caseserver/elasticsearch/CaseInfosService.java +++ b/src/main/java/com/powsybl/caseserver/elasticsearch/CaseInfosService.java @@ -62,6 +62,11 @@ public List getAllCaseInfos() { return Lists.newArrayList(caseInfosRepository.findAll()); } + /* + The query is an elasticsearch (Lucene) form query, so here it will be : + date:XXX AND geographicalCode:(X) + date:XXX AND geographicalCode:(X OR Y OR Z) +*/ public List searchCaseInfos(@NonNull final String query) { NativeQuery searchQuery = new NativeQueryBuilder().withQuery(QueryStringQuery.of(qs -> qs.query(query))._toQuery()).build(); return Lists.newArrayList(operations.search(searchQuery, CaseInfos.class) diff --git a/src/main/java/com/powsybl/caseserver/repository/CaseMetadataEntity.java b/src/main/java/com/powsybl/caseserver/repository/CaseMetadataEntity.java index 2bca67f..b84835a 100644 --- a/src/main/java/com/powsybl/caseserver/repository/CaseMetadataEntity.java +++ b/src/main/java/com/powsybl/caseserver/repository/CaseMetadataEntity.java @@ -36,4 +36,13 @@ public class CaseMetadataEntity { @Column(name = "indexed", columnDefinition = "boolean default false", nullable = false) private boolean indexed = false; + + @Column(name = "originalFilename", columnDefinition = "Original case file name") + private String originalFilename; + + @Column(name = "compressionFormat", columnDefinition = "Case compression format") + private String compressionFormat; + + @Column(name = "format", columnDefinition = "Case format") + private String format; } diff --git a/src/main/java/com/powsybl/caseserver/repository/StorageConfig.java b/src/main/java/com/powsybl/caseserver/repository/StorageConfig.java new file mode 100644 index 0000000..5e95737 --- /dev/null +++ b/src/main/java/com/powsybl/caseserver/repository/StorageConfig.java @@ -0,0 +1,55 @@ +/** + * Copyright (c) 2023, RTE (http://www.rte-france.com) + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ +package com.powsybl.caseserver.repository; + +import com.powsybl.caseserver.service.CaseService; +import com.powsybl.caseserver.service.FsCaseService; +import com.powsybl.caseserver.service.S3CaseService; +import com.powsybl.caseserver.datasource.CaseDataSourceService; +import com.powsybl.caseserver.datasource.FsCaseDataSourceService; +import com.powsybl.caseserver.datasource.S3CaseDataSourceService; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Primary; + +/** + * @author Ghazwa Rehili + */ +@Configuration +public class StorageConfig { + + private final CaseMetadataRepository caseMetadataRepository; + private final String storageType; + + public StorageConfig(@Value("${storage.type}")String storageType, CaseMetadataRepository caseMetadataRepository) { + this.storageType = storageType; + this.caseMetadataRepository = caseMetadataRepository; + } + + @Primary + @Bean + public CaseService storageService() { + if ("FS".equals(storageType)) { + return new FsCaseService(caseMetadataRepository); + } else if ("S3".equals(storageType)) { + return new S3CaseService(caseMetadataRepository); + } + return null; + } + + @Primary + @Bean + public CaseDataSourceService caseDataSourceService() { + if ("FS".equals(storageType)) { + return new FsCaseDataSourceService(); + } else if ("S3".equals(storageType)) { + return new S3CaseDataSourceService(); + } + return null; + } +} diff --git a/src/main/java/com/powsybl/caseserver/service/CaseService.java b/src/main/java/com/powsybl/caseserver/service/CaseService.java new file mode 100644 index 0000000..1cd3ca6 --- /dev/null +++ b/src/main/java/com/powsybl/caseserver/service/CaseService.java @@ -0,0 +1,171 @@ +/** + * Copyright (c) 2024, RTE (http://www.rte-france.com) + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ +package com.powsybl.caseserver.service; + +import com.powsybl.caseserver.CaseException; +import com.powsybl.caseserver.dto.CaseInfos; +import com.powsybl.caseserver.dto.ExportCaseInfos; +import com.powsybl.caseserver.parsers.FileNameInfos; +import com.powsybl.caseserver.parsers.FileNameParser; +import com.powsybl.caseserver.parsers.FileNameParsers; +import com.powsybl.caseserver.repository.CaseMetadataEntity; +import com.powsybl.caseserver.repository.CaseMetadataRepository; +import com.powsybl.commons.datasource.DataSource; +import com.powsybl.commons.datasource.DataSourceUtil; +import com.powsybl.commons.datasource.MemDataSource; +import com.powsybl.computation.ComputationManager; +import com.powsybl.iidm.network.Exporter; +import com.powsybl.iidm.network.Importer; +import com.powsybl.iidm.network.Network; +import org.springframework.web.multipart.MultipartFile; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.file.Path; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.*; +import java.util.stream.Collectors; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; + +/** + * @author Ghazwa Rehili + */ +public interface CaseService { + default void validateCaseName(String caseName) { + Objects.requireNonNull(caseName); + if (!caseName.matches("[^<>:\"/|?*]+(\\.[\\w]+)")) { + throw CaseException.createIllegalCaseName(caseName); + } + } + + default CaseInfos createInfos(String fileBaseName, UUID caseUuid, String format) { + FileNameParser parser = FileNameParsers.findParser(fileBaseName); + if (parser != null) { + Optional fileNameInfos = parser.parse(fileBaseName); + if (fileNameInfos.isPresent()) { + return CaseInfos.create(fileBaseName, caseUuid, format, fileNameInfos.get()); + } + } + return CaseInfos.builder().name(fileBaseName).uuid(caseUuid).format(format).build(); + } + + default void createCaseMetadataEntity(UUID newCaseUuid, boolean withExpiration, boolean withIndexation, String originalFilename, String compressionFormat, String format) { + Instant expirationTime = null; + if (withExpiration) { + expirationTime = Instant.now().plus(1, ChronoUnit.HOURS); + } + getCaseMetadataRepository().save(new CaseMetadataEntity(newCaseUuid, expirationTime, withIndexation, originalFilename, compressionFormat, format)); + } + + default void createCaseMetadataEntity(UUID newCaseUuid, boolean withExpiration, boolean withIndexation) { + createCaseMetadataEntity(newCaseUuid, withExpiration, withIndexation, null, null, null); + } + + default List getMetadata(List ids) { + List cases = new ArrayList<>(); + ids.forEach(caseUuid -> { + CaseInfos caseInfos = getCaseInfos(caseUuid); + if (Objects.nonNull(caseInfos)) { + cases.add(caseInfos); + } + }); + return cases; + } + + default Importer getImporterOrThrowsException(Path caseFile) { + DataSource dataSource = DataSource.fromPath(caseFile); + Importer importer = Importer.find(dataSource, getComputationManager()); + if (importer == null) { + throw CaseException.createFileNotImportable(caseFile); + } + return importer; + } + + default byte[] createZipFile(Collection names, MemDataSource dataSource) throws IOException { + try (var outputStream = new ByteArrayOutputStream(); + var zipOutputStream = new ZipOutputStream(outputStream)) { + for (String name : names) { + zipOutputStream.putNextEntry(new ZipEntry(name)); + zipOutputStream.write(dataSource.getData(name)); + zipOutputStream.closeEntry(); + } + return outputStream.toByteArray(); + } + } + + default Optional exportCase(UUID caseUuid, String format, String fileName, Map formatParameters) throws IOException { + if (!Exporter.getFormats().contains(format)) { + throw CaseException.createUnsupportedFormat(format); + } + + var optionalNetwork = loadNetwork(caseUuid); + if (optionalNetwork.isPresent()) { + var network = optionalNetwork.get(); + var memDataSource = new MemDataSource(); + Properties exportProperties = null; + if (formatParameters != null) { + exportProperties = new Properties(); + exportProperties.putAll(formatParameters); + } + + network.write(format, exportProperties, memDataSource); + + var listNames = memDataSource.listNames(".*"); + String fileOrNetworkName = fileName != null ? fileName : DataSourceUtil.getBaseName(getCaseName(caseUuid)); + byte[] networkData; + if (listNames.size() == 1) { + String extension = listNames.iterator().next(); + fileOrNetworkName += extension; + networkData = memDataSource.getData(extension); + } else { + fileOrNetworkName += ".zip"; + networkData = createZipFile(listNames, memDataSource); + } + return Optional.of(new ExportCaseInfos(fileOrNetworkName, networkData)); + } else { + return Optional.empty(); + } + } + + default List getCasesToReindex() { + Set casesToReindex = getCaseMetadataRepository().findAllByIndexedTrue() + .stream() + .map(CaseMetadataEntity::getId) + .collect(Collectors.toSet()); + return getCases().stream().filter(c -> casesToReindex.contains(c.getUuid())).toList(); + } + + List getCases(); + + boolean caseExists(UUID caseUuid); + + CaseInfos getCaseInfos(UUID caseUuid); + + String getFormat(UUID caseUuid); + + String getCaseName(UUID caseUuid); + + Optional loadNetwork(UUID caseUuid); + + Optional getCaseBytes(UUID caseUuid); + + UUID importCase(MultipartFile file, boolean withExpiration, boolean withIndexation); + + UUID duplicateCase(UUID sourceCaseUuid, boolean withExpiration); + + void deleteCase(UUID caseUuid); + + void deleteAllCases(); + + void setComputationManager(ComputationManager computationManager); + + CaseMetadataRepository getCaseMetadataRepository(); + + ComputationManager getComputationManager(); +} diff --git a/src/main/java/com/powsybl/caseserver/CaseService.java b/src/main/java/com/powsybl/caseserver/service/FsCaseService.java similarity index 58% rename from src/main/java/com/powsybl/caseserver/CaseService.java rename to src/main/java/com/powsybl/caseserver/service/FsCaseService.java index cf2ddf5..1a3af7d 100644 --- a/src/main/java/com/powsybl/caseserver/CaseService.java +++ b/src/main/java/com/powsybl/caseserver/service/FsCaseService.java @@ -1,73 +1,51 @@ /** - * Copyright (c) 2019, RTE (http://www.rte-france.com) + * Copyright (c) 2024, RTE (http://www.rte-france.com) * This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -package com.powsybl.caseserver; +package com.powsybl.caseserver.service; +import com.powsybl.caseserver.CaseException; import com.powsybl.caseserver.dto.CaseInfos; -import com.powsybl.caseserver.dto.ExportCaseInfos; import com.powsybl.caseserver.elasticsearch.CaseInfosService; -import com.powsybl.caseserver.parsers.FileNameInfos; -import com.powsybl.caseserver.parsers.FileNameParser; -import com.powsybl.caseserver.parsers.FileNameParsers; import com.powsybl.caseserver.repository.CaseMetadataEntity; import com.powsybl.caseserver.repository.CaseMetadataRepository; -import com.powsybl.commons.datasource.DataSource; -import com.powsybl.commons.datasource.DataSourceUtil; -import com.powsybl.commons.datasource.MemDataSource; import com.powsybl.computation.ComputationManager; import com.powsybl.computation.local.LocalComputationManager; -import com.powsybl.iidm.network.Exporter; import com.powsybl.iidm.network.Importer; import com.powsybl.iidm.network.Network; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; -import org.springframework.cloud.stream.function.StreamBridge; import org.springframework.context.annotation.ComponentScan; import org.springframework.http.HttpStatus; -import org.springframework.messaging.Message; import org.springframework.stereotype.Service; -import org.springframework.transaction.annotation.Transactional; import org.springframework.web.multipart.MultipartFile; import org.springframework.web.server.ResponseStatusException; -import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.UncheckedIOException; -import java.nio.file.DirectoryStream; -import java.nio.file.FileSystem; -import java.nio.file.FileSystems; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.StandardCopyOption; -import java.time.Instant; -import java.time.temporal.ChronoUnit; -import java.util.*; -import java.util.stream.Collectors; +import java.nio.file.*; +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.UUID; import java.util.stream.Stream; -import java.util.zip.ZipEntry; -import java.util.zip.ZipOutputStream; import static com.powsybl.caseserver.CaseException.createDirectoryNotFound; -import static com.powsybl.caseserver.dto.CaseInfos.*; /** * @author Abdelsalem Hedhili * @author Franck Lecuyer + * @author Ghazwa Rehili */ @Service @ComponentScan(basePackageClasses = {CaseInfosService.class}) -public class CaseService { +public class FsCaseService implements CaseService { - private static final Logger LOGGER = LoggerFactory.getLogger(CaseService.class); - - private static final String CATEGORY_BROKER_OUTPUT = CaseService.class.getName() + ".output-broker-messages"; - - private static final Logger OUTPUT_MESSAGE_LOGGER = LoggerFactory.getLogger(CATEGORY_BROKER_OUTPUT); + private static final Logger LOGGER = LoggerFactory.getLogger(FsCaseService.class); private FileSystem fileSystem = FileSystems.getDefault(); @@ -76,7 +54,7 @@ public class CaseService { private final CaseMetadataRepository caseMetadataRepository; @Autowired - private StreamBridge caseInfosPublisher; + private NotificationService notificationService; @Autowired private CaseInfosService caseInfosService; @@ -84,17 +62,14 @@ public class CaseService { @Value("${case-store-directory:#{systemProperties['user.home'].concat(\"/cases\")}}") private String rootDirectory; - public CaseService(CaseMetadataRepository caseMetadataRepository) { + public FsCaseService(CaseMetadataRepository caseMetadataRepository) { this.caseMetadataRepository = caseMetadataRepository; } - Importer getImporterOrThrowsException(Path caseFile) { - DataSource dataSource = DataSource.fromPath(caseFile); - Importer importer = Importer.find(dataSource, computationManager); - if (importer == null) { - throw CaseException.createFileNotImportable(caseFile); - } - return importer; + @Override + public String getFormat(UUID caseUuid) { + Path file = getCaseFile(caseUuid); + return getFormat(file); } String getFormat(Path caseFile) { @@ -102,12 +77,13 @@ String getFormat(Path caseFile) { return importer.getFormat(); } - public List getCases(Path directory) { - try (Stream walk = Files.walk(directory)) { + @Override + public List getCases() { + try (Stream walk = Files.walk(getStorageRootDir())) { return walk.filter(Files::isRegularFile) .map(this::getCaseInfos) .filter(Objects::nonNull) - .collect(Collectors.toList()); + .toList(); } catch (IOException e) { throw new UncheckedIOException(e); } @@ -122,19 +98,23 @@ private CaseInfos getCaseInfos(Path file) { } } + @Override public String getCaseName(UUID caseUuid) { Path file = getCaseFile(caseUuid); if (file == null) { throw createDirectoryNotFound(caseUuid); } - CaseInfos caseInfos = getCase(file); + CaseInfos caseInfos = getCaseInfos(file); + if (caseInfos == null) { + throw CaseException.createFileNameNotFound(caseUuid); + } return caseInfos.getName(); } - public CaseInfos getCase(Path casePath) { - checkStorageInitialization(); - Optional caseInfo = getCases(casePath).stream().findFirst(); - return caseInfo.orElseThrow(); + @Override + public CaseInfos getCaseInfos(UUID caseUuid) { + Path file = getCaseFile(caseUuid); + return getCaseInfos(file); } public Path getCaseFile(UUID caseUuid) { @@ -164,7 +144,8 @@ public Path walkCaseDirectory(Path caseDirectory) { return null; } - boolean caseExists(UUID caseName) { + @Override + public boolean caseExists(UUID caseName) { checkStorageInitialization(); Path caseFile = getCaseFile(caseName); if (caseFile == null) { @@ -173,7 +154,8 @@ boolean caseExists(UUID caseName) { return Files.exists(caseFile) && Files.isRegularFile(caseFile); } - UUID importCase(MultipartFile mpf, boolean withExpiration, boolean withIndexation) { + @Override + public UUID importCase(MultipartFile mpf, boolean withExpiration, boolean withIndexation) { checkStorageInitialization(); UUID caseUuid = UUID.randomUUID(); @@ -183,7 +165,7 @@ UUID importCase(MultipartFile mpf, boolean withExpiration, boolean withIndexatio validateCaseName(caseName); if (Files.exists(uuidDirectory)) { - throw CaseException.createDirectoryAreadyExists(uuidDirectory); + throw CaseException.createDirectoryAreadyExists(uuidDirectory.toString()); } Path caseFile; @@ -213,11 +195,12 @@ UUID importCase(MultipartFile mpf, boolean withExpiration, boolean withIndexatio if (withIndexation) { caseInfosService.addCaseInfos(caseInfos); } - sendImportMessage(caseInfos.createMessage()); + notificationService.sendImportMessage(caseInfos.createMessage()); return caseUuid; } - UUID duplicateCase(UUID sourceCaseUuid, boolean withExpiration) { + @Override + public UUID duplicateCase(UUID sourceCaseUuid, boolean withExpiration) { try { Path existingCaseFile = getCaseFile(sourceCaseUuid); if (existingCaseFile == null || existingCaseFile.getParent() == null) { @@ -236,10 +219,9 @@ UUID duplicateCase(UUID sourceCaseUuid, boolean withExpiration) { if (existingCase.isIndexed()) { caseInfosService.addCaseInfos(caseInfos); } - createCaseMetadataEntity(newCaseUuid, withExpiration, existingCase.isIndexed()); - sendImportMessage(caseInfos.createMessage()); + notificationService.sendImportMessage(caseInfos.createMessage()); return newCaseUuid; } catch (IOException e) { @@ -255,40 +237,8 @@ private CaseMetadataEntity getCaseMetaDataEntity(UUID caseUuid) { return caseMetadataRepository.findById(caseUuid).orElseThrow(() -> new ResponseStatusException(HttpStatus.NOT_FOUND, "case " + caseUuid + " not found")); } - private void createCaseMetadataEntity(UUID newCaseUuid, boolean withExpiration, boolean withIndexation) { - Instant expirationTime = null; - if (withExpiration) { - expirationTime = Instant.now().plus(1, ChronoUnit.HOURS); - } - caseMetadataRepository.save(new CaseMetadataEntity(newCaseUuid, expirationTime, withIndexation)); - } - - public List getCasesToReindex() { - Set casesToReindex = caseMetadataRepository.findAllByIndexedTrue() - .stream() - .map(CaseMetadataEntity::getId) - .collect(Collectors.toSet()); - return getCases(getStorageRootDir()).stream().filter(c -> casesToReindex.contains(c.getUuid())).toList(); - } - - CaseInfos createInfos(String fileBaseName, UUID caseUuid, String format) { - FileNameParser parser = FileNameParsers.findParser(fileBaseName); - if (parser != null) { - Optional fileNameInfos = parser.parse(fileBaseName); - if (fileNameInfos.isPresent()) { - return CaseInfos.create(fileBaseName, caseUuid, format, fileNameInfos.get()); - } - } - return CaseInfos.builder().name(fileBaseName).uuid(caseUuid).format(format).build(); - } - - @Transactional - public void disableCaseExpiration(UUID caseUuid) { - CaseMetadataEntity caseMetadataEntity = getCaseMetaDataEntity(caseUuid); - caseMetadataEntity.setExpirationDate(null); - } - - Optional loadNetwork(UUID caseUuid) { + @Override + public Optional loadNetwork(UUID caseUuid) { checkStorageInitialization(); Path caseFile = getCaseFile(caseUuid); @@ -321,7 +271,8 @@ void deleteDirectoryRecursively(Path caseDirectory) { } } - void deleteCase(UUID caseUuid) { + @Override + public void deleteCase(UUID caseUuid) { checkStorageInitialization(); Path caseDirectory = getCaseDirectory(caseUuid); deleteDirectoryRecursively(caseDirectory); @@ -329,7 +280,8 @@ void deleteCase(UUID caseUuid) { caseMetadataRepository.deleteById(caseUuid); } - void deleteAllCases() { + @Override + public void deleteAllCases() { checkStorageInitialization(); Path rootDirectoryPath = getStorageRootDir(); @@ -361,46 +313,18 @@ public void setFileSystem(FileSystem fileSystem) { this.fileSystem = Objects.requireNonNull(fileSystem); } + @Override public void setComputationManager(ComputationManager computationManager) { this.computationManager = Objects.requireNonNull(computationManager); } - static void validateCaseName(String caseName) { - Objects.requireNonNull(caseName); - if (!caseName.matches("[^<>:\"/|?*]+(\\.[\\w]+)")) { - throw CaseException.createIllegalCaseName(caseName); - } + @Override + public ComputationManager getComputationManager() { + return computationManager; } - /* - The query is an elasticsearch (Lucene) form query, so here it will be : - date:XXX AND geographicalCode:(X) - date:XXX AND geographicalCode:(X OR Y OR Z) - */ - List searchCases(String query) { - checkStorageInitialization(); - - return caseInfosService.searchCaseInfos(query); - } - - private void sendImportMessage(Message message) { - OUTPUT_MESSAGE_LOGGER.debug("Sending message : {}", message); - caseInfosPublisher.send("publishCaseImport-out-0", message); - } - - public List getMetadata(List ids) { - List cases = new ArrayList<>(); - ids.forEach(caseUuid -> { - Path file = getCaseFile(caseUuid); - if (file != null) { - CaseInfos caseInfos = getCase(file); - cases.add(caseInfos); - } - }); - return cases; - } - - Optional getCaseBytes(UUID caseUuid) { + @Override + public Optional getCaseBytes(UUID caseUuid) { checkStorageInitialization(); Path caseFile = getCaseFile(caseUuid); @@ -419,49 +343,9 @@ Optional getCaseBytes(UUID caseUuid) { return Optional.empty(); } - public Optional exportCase(UUID caseUuid, String format, String fileName, Map formatParameters) throws IOException { - if (!Exporter.getFormats().contains(format)) { - throw CaseException.createUnsupportedFormat(format); - } - - var optionalNetwork = loadNetwork(caseUuid); - if (optionalNetwork.isPresent()) { - var network = optionalNetwork.get(); - var memDataSource = new MemDataSource(); - Properties exportProperties = null; - if (formatParameters != null) { - exportProperties = new Properties(); - exportProperties.putAll(formatParameters); - } - - network.write(format, exportProperties, memDataSource); - - var listNames = memDataSource.listNames(".*"); - String fileOrNetworkName = fileName != null ? fileName : DataSourceUtil.getBaseName(getCaseName(caseUuid)); - byte[] networkData; - if (listNames.size() == 1) { - String extension = listNames.iterator().next(); - fileOrNetworkName += extension; - networkData = memDataSource.getData(extension); - } else { - fileOrNetworkName += ".zip"; - networkData = createZipFile(listNames, memDataSource); - } - return Optional.of(new ExportCaseInfos(fileOrNetworkName, networkData)); - } else { - return Optional.empty(); - } + @Override + public CaseMetadataRepository getCaseMetadataRepository() { + return caseMetadataRepository; } - private byte[] createZipFile(Collection names, MemDataSource dataSource) throws IOException { - try (var outputStream = new ByteArrayOutputStream(); - var zipOutputStream = new ZipOutputStream(outputStream)) { - for (String name : names) { - zipOutputStream.putNextEntry(new ZipEntry(name)); - zipOutputStream.write(dataSource.getData(name)); - zipOutputStream.closeEntry(); - } - return outputStream.toByteArray(); - } - } } diff --git a/src/main/java/com/powsybl/caseserver/service/MetadataService.java b/src/main/java/com/powsybl/caseserver/service/MetadataService.java new file mode 100644 index 0000000..d05e6fc --- /dev/null +++ b/src/main/java/com/powsybl/caseserver/service/MetadataService.java @@ -0,0 +1,37 @@ +/** + * Copyright (c) 2024, RTE (http://www.rte-france.com) + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ +package com.powsybl.caseserver.service; + +import com.powsybl.caseserver.repository.CaseMetadataEntity; +import com.powsybl.caseserver.repository.CaseMetadataRepository; +import org.springframework.http.HttpStatus; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; +import org.springframework.web.server.ResponseStatusException; + +import java.util.UUID; + +import static com.powsybl.caseserver.service.S3CaseService.NOT_FOUND; + +/** + * @author Etienne Homer + */ +@Service +public class MetadataService { + + private final CaseMetadataRepository caseMetadataRepository; + + public MetadataService(CaseMetadataRepository caseMetadataRepository) { + this.caseMetadataRepository = caseMetadataRepository; + } + + @Transactional + public void disableCaseExpiration(UUID caseUuid) { + CaseMetadataEntity caseMetadataEntity = caseMetadataRepository.findById(caseUuid).orElseThrow(() -> new ResponseStatusException(HttpStatus.NOT_FOUND, "case " + caseUuid + NOT_FOUND)); + caseMetadataEntity.setExpirationDate(null); + } +} diff --git a/src/main/java/com/powsybl/caseserver/service/NotificationService.java b/src/main/java/com/powsybl/caseserver/service/NotificationService.java new file mode 100644 index 0000000..7739709 --- /dev/null +++ b/src/main/java/com/powsybl/caseserver/service/NotificationService.java @@ -0,0 +1,31 @@ +/** + * Copyright (c) 2024, RTE (http://www.rte-france.com) + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ +package com.powsybl.caseserver.service; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.cloud.stream.function.StreamBridge; +import org.springframework.messaging.Message; +import org.springframework.stereotype.Service; + +/** + * @author Ghazwa Rehili + */ +@Service +public class NotificationService { + private static final String CATEGORY_BROKER_OUTPUT = CaseService.class.getName() + ".output-broker-messages"; + private static final Logger OUTPUT_MESSAGE_LOGGER = LoggerFactory.getLogger(CATEGORY_BROKER_OUTPUT); + + @Autowired + private StreamBridge caseInfosPublisher; + + public void sendImportMessage(Message message) { + OUTPUT_MESSAGE_LOGGER.debug("Sending message : {}", message); + caseInfosPublisher.send("publishCaseImport-out-0", message); + } +} diff --git a/src/main/java/com/powsybl/caseserver/service/S3CaseService.java b/src/main/java/com/powsybl/caseserver/service/S3CaseService.java new file mode 100644 index 0000000..48c523f --- /dev/null +++ b/src/main/java/com/powsybl/caseserver/service/S3CaseService.java @@ -0,0 +1,569 @@ +/** + * Copyright (c) 2024, RTE (http://www.rte-france.com) + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ +package com.powsybl.caseserver.service; + +import com.google.common.io.ByteStreams; +import com.powsybl.caseserver.CaseException; +import com.powsybl.caseserver.dto.CaseInfos; +import com.powsybl.caseserver.elasticsearch.CaseInfosService; +import com.powsybl.caseserver.repository.CaseMetadataEntity; +import com.powsybl.caseserver.repository.CaseMetadataRepository; +import com.powsybl.computation.ComputationManager; +import com.powsybl.computation.local.LocalComputationManager; +import com.powsybl.iidm.network.Importer; +import com.powsybl.iidm.network.Network; +import com.powsybl.ws.commons.SecuredZipInputStream; +import org.apache.commons.compress.utils.FileNameUtils; +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang3.function.FailableConsumer; +import org.apache.commons.lang3.function.FailableFunction; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.ComponentScan; +import org.springframework.http.HttpStatus; +import org.springframework.stereotype.Service; +import org.springframework.web.multipart.MultipartFile; + +import org.springframework.web.server.ResponseStatusException; +import software.amazon.awssdk.core.ResponseBytes; +import software.amazon.awssdk.core.sync.RequestBody; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.*; +import software.amazon.awssdk.services.s3.paginators.ListObjectsV2Iterable; + +import java.io.*; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.attribute.FileAttribute; +import java.nio.file.attribute.PosixFilePermission; +import java.nio.file.attribute.PosixFilePermissions; +import java.util.*; +import java.util.stream.Collectors; +import java.util.zip.GZIPInputStream; +import java.util.zip.GZIPOutputStream; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; + +/** + * @author Ghazwa Rehili + * @author Etienne Homer + */ +@Service +@ComponentScan(basePackageClasses = {CaseInfosService.class}) +public class S3CaseService implements CaseService { + + private static final Logger LOGGER = LoggerFactory.getLogger(S3CaseService.class); + public static final int MAX_SIZE = 500000000; + public static final List COMPRESSION_FORMATS = List.of("bz2", "gz", "xz", "zst"); + public static final List ARCHIVE_FORMATS = List.of("zip"); + public static final String DELIMITER = "/"; + public static final String GZIP_EXTENSION = ".gz"; + + private ComputationManager computationManager = LocalComputationManager.getDefault(); + + private final CaseMetadataRepository caseMetadataRepository; + + @Autowired + private CaseInfosService caseInfosService; + + @Autowired + NotificationService notificationService; + + @Value("${spring.cloud.aws.bucket}") + private String bucketName; + + private static final String CASES_PREFIX = "gsi-cases/"; + + public static final String NOT_FOUND = " not found"; + + @Autowired + private S3Client s3Client; + + public S3CaseService(CaseMetadataRepository caseMetadataRepository) { + this.caseMetadataRepository = caseMetadataRepository; + } + + String getFormat(Path caseFile) { + Importer importer = getImporterOrThrowsException(caseFile); + return importer.getFormat(); + } + + // creates a directory, and then in this directory, initializes a file with content. + // After applying f to the file, deletes the file and the directory. + private R withTempCopy(UUID caseUuid, String filename, + FailableConsumer contentInitializer, FailableFunction f) { + Path tempdirPath; + Path tempCasePath; + try { + FileAttribute> attr = PosixFilePermissions.asFileAttribute(PosixFilePermissions.fromString("rwx------")); + tempdirPath = Files.createTempDirectory(caseUuid.toString(), attr); + + // Create parent directory if necessary + Path parentPath = Paths.get(filename).getParent(); + if (parentPath != null) { + Files.createDirectory(tempdirPath.resolve(parentPath), attr); + } + // after this line, need to cleanup the dir + } catch (IOException e) { + throw CaseException.createTempDirectory(caseUuid, e); + } + try { + tempCasePath = tempdirPath.resolve(filename); + try { + contentInitializer.accept(tempCasePath); + } catch (Exception e) { + throw CaseException.createUInitTempFileError(caseUuid, e); + } + // after this line, need to cleanup the file + try { + try { + return f.apply(tempCasePath); + } catch (Exception e) { + throw CaseException.createFileNotImportable(tempdirPath); + } + } finally { + try { + Files.delete(tempCasePath); + } catch (IOException e) { + LOGGER.error("Error cleaning up temporary case file", e); + } + } + } finally { + try { + if (Files.exists(tempdirPath)) { + FileUtils.deleteDirectory(tempdirPath.toFile()); + } + } catch (IOException e) { + LOGGER.error("Error cleaning up temporary case directory: " + tempdirPath, e); + } + } + } + + // downloads from s3 and cleanup + public R withS3DownloadedTempPath(UUID caseUuid, FailableFunction f) { + return withS3DownloadedTempPath(caseUuid, null, f); + } + + public R withS3DownloadedTempPath(UUID caseUuid, String caseFileKey, FailableFunction f) { + String nonNullCaseFileKey = Objects.requireNonNullElse(caseFileKey, uuidToKeyWithOriginalFileName(caseUuid)); + String filename = parseFilenameFromKey(nonNullCaseFileKey); + return withTempCopy(caseUuid, filename, path -> + s3Client.getObject(GetObjectRequest.builder().bucket(bucketName).key(nonNullCaseFileKey).build(), path), f); + } + + @Override + public String getFormat(UUID caseUuid) { + return getCaseMetaDataEntity(caseUuid).getFormat(); + } + + public String getCompressionFormat(UUID caseUuid) { + return getCaseMetaDataEntity(caseUuid).getCompressionFormat(); + } + + public String getOriginalFilename(UUID caseUuid) { + return getCaseMetaDataEntity(caseUuid).getOriginalFilename(); + } + + // key format is "gsi-cases/UUID/path/to/file" + private UUID parseUuidFromKey(String key) { + int firstSlash = key.indexOf(DELIMITER); + int secondSlash = key.indexOf(DELIMITER, firstSlash + 1); + return UUID.fromString(key.substring(firstSlash + 1, secondSlash)); + } + + private String parseFilenameFromKey(String key) { + int firstSlash = key.indexOf(DELIMITER); + int secondSlash = key.indexOf(DELIMITER, firstSlash + 1); + return key.substring(secondSlash + 1); + } + + public static String uuidToKeyPrefix(UUID uuid) { + return CASES_PREFIX + uuid.toString() + DELIMITER; + } + + public static String uuidToKeyWithFileName(UUID uuid, String filename) { + return uuidToKeyPrefix(uuid) + filename; + } + + public String uuidToKeyWithOriginalFileName(UUID caseUuid) { + return uuidToKeyWithFileName(caseUuid, getOriginalFilename(caseUuid)); + } + + private List getCaseS3Objects(String keyPrefix) { + List s3Objects = new ArrayList<>(); + ListObjectsV2Iterable listObjectsV2Iterable = s3Client.listObjectsV2Paginator(getListObjectsV2Request(keyPrefix)); + listObjectsV2Iterable.iterator().forEachRemaining(listObjectsChunk -> + s3Objects.addAll(listObjectsChunk.contents()) + ); + return s3Objects; + } + + private ListObjectsV2Request getListObjectsV2Request(String prefix) { + return ListObjectsV2Request.builder().bucket(bucketName).prefix(prefix).build(); + } + + private List getCaseS3Objects(UUID caseUuid) { + return getCaseS3Objects(uuidToKeyPrefix(caseUuid)); + } + + @Override + public CaseInfos getCaseInfos(UUID caseUuid) { + return new CaseInfos(caseUuid, getCaseName(caseUuid), getFormat(caseUuid)); + } + + @Override + public String getCaseName(UUID caseUuid) { + String originalFilename = getOriginalFilename(caseUuid); + if (originalFilename == null) { + throw CaseException.createOriginalFileNotFound(caseUuid); + } + return originalFilename; + } + + @Override + public Optional getCaseBytes(UUID caseUuid) { + String caseFileKey = null; + try { + caseFileKey = uuidToKeyWithOriginalFileName(caseUuid); + GetObjectRequest getObjectRequest = GetObjectRequest.builder() + .bucket(bucketName) + .key(caseFileKey) + .build(); + + ResponseBytes objectBytes = s3Client.getObjectAsBytes(getObjectRequest); + return Optional.of(objectBytes.asByteArray()); + } catch (NoSuchKeyException e) { + LOGGER.error("The expected key does not exist in the bucket s3 : {}", caseFileKey); + return Optional.empty(); + } catch (CaseException | ResponseStatusException e) { + LOGGER.error(e.getMessage()); + return Optional.empty(); + } + } + + @Override + public List getCases() { + List caseInfosList = new ArrayList<>(); + CaseInfos caseInfos; + for (S3Object o : getCaseS3Objects(CASES_PREFIX)) { + caseInfos = getCaseInfos(parseUuidFromKey(o.key())); + if (Objects.nonNull(caseInfos)) { + caseInfosList.add(caseInfos); + } + } + return caseInfosList; + } + + @Override + public boolean caseExists(UUID uuid) { + return !getCaseS3Objects(uuid).isEmpty(); + } + + public Boolean datasourceExists(UUID caseUuid, String fileName) { + if (getCaseS3Objects(caseUuid).size() > 1 && fileName.equals(getCaseName(caseUuid))) { + return Boolean.FALSE; + } + + String key = uuidToKeyWithFileName(caseUuid, fileName); + String caseName = getCaseName(caseUuid); + // For compressed cases, we append the compression extension to the case name as only the compressed file is stored in S3. + // i.e. : Assuming test.xml.gz is stored in S3. When you request datasourceExists(randomUUID, "test.xml"), you ask to S3 API ("test.xml" + ".gz") exists ? => true + if (isCompressedCaseFile(caseName)) { + key = key + "." + getCompressionFormat(caseUuid); + } else if (isArchivedCaseFile(caseName)) { + key = key + GZIP_EXTENSION; + } + + HeadObjectRequest headObjectRequest = HeadObjectRequest.builder() + .bucket(bucketName) + .key(key) + .build(); + try { + s3Client.headObject(headObjectRequest); + return Boolean.TRUE; + } catch (NoSuchKeyException e) { + return Boolean.FALSE; + } + } + + public static boolean isCompressedCaseFile(String caseName) { + return COMPRESSION_FORMATS.stream().anyMatch(cf -> caseName.endsWith("." + cf)); + } + + public static boolean isArchivedCaseFile(String caseName) { + return ARCHIVE_FORMATS.stream().anyMatch(cf -> caseName.endsWith("." + cf)); + } + + private static String removeExtension(String filename, String extension) { + int index = filename.lastIndexOf(extension); + if (index == -1 || index < filename.length() - extension.length() /*extension to remove is not at the end*/) { + return filename; + } + return filename.substring(0, index); + } + + public Set listName(UUID caseUuid, String regex) { + List filenames; + String originalFilename = getOriginalFilename(caseUuid); + if (isCompressedCaseFile(originalFilename)) { + // For a compressed file basename.xml.gz, listName() should return ['basename.xml']. That's why we remove the compression extension to the filename. + filenames = List.of(removeExtension(originalFilename, "." + getCompressionFormat(caseUuid))); + } else { + List s3Objects = getCaseS3Objects(caseUuid); + filenames = s3Objects.stream().map(obj -> Paths.get(obj.key()).toString().replace(CASES_PREFIX + caseUuid.toString() + DELIMITER, "")).toList(); + // For archived cases : + if (isArchivedCaseFile(originalFilename)) { + filenames = filenames.stream() + // the original archive name has to be filtered. + .filter(name -> !name.equals(originalFilename)) + // each subfile hase been gzipped -> we have to remove the gz extension (only one, the one we added). + .map(name -> removeExtension(name, GZIP_EXTENSION)) + .collect(Collectors.toList()); + } + } + return filenames.stream().filter(n -> n.matches(regex)).collect(Collectors.toSet()); + } + + @Override + public UUID importCase(MultipartFile mpf, boolean withExpiration, boolean withIndexation) { + UUID caseUuid = UUID.randomUUID(); + + String caseName = Objects.requireNonNull(mpf.getOriginalFilename()); + validateCaseName(caseName); + + String format = withTempCopy(caseUuid, caseName, mpf::transferTo, this::getFormat); + String compressionFormat = FileNameUtils.getExtension(Paths.get(caseName)); + + try (InputStream inputStream = mpf.getInputStream()) { + String key = uuidToKeyWithFileName(caseUuid, caseName); + + // We store archived cases in S3 in a specific way : in the caseUuid directory, we store : + // - the original archive + // - the extracted files are exploded in the caseUuid directory. This allows to use HeadObjectRequest for datasource/exists, + // to download subfiles separately, or to anwser to datasource/list with ListObjectV2. + // But this unarchived storage could increase tenfold used disk space: so each extracted file is gzipped to avoid increasing it. + // Compression of subfiles is done in a simple way: no matter if the subfile is compressed or not, it will be gzipped in the storage. + // example : archive.zip containing [file1.xml, file2.xml.gz] + // will be stored as : + // - archive.zip + // - file1.xml.gz + // - file2.xml.gz.gz + if (isArchivedCaseFile(caseName)) { + importZipContent(mpf.getInputStream(), caseUuid); + } + + PutObjectRequest putObjectRequest = PutObjectRequest.builder() + .bucket(bucketName) + .key(key) + .contentType(mpf.getContentType()) + .build(); + // Use putObject to upload the file + s3Client.putObject(putObjectRequest, RequestBody.fromInputStream(inputStream, mpf.getSize())); + + } catch (IOException e) { + throw CaseException.createFileNotImportable(caseName, e); + } + + createCaseMetadataEntity(caseUuid, withExpiration, withIndexation, caseName, compressionFormat, format); + CaseInfos caseInfos = createInfos(caseName, caseUuid, format); + if (withIndexation) { + caseInfosService.addCaseInfos(caseInfos); + } + notificationService.sendImportMessage(caseInfos.createMessage()); + + return caseUuid; + } + + private void importZipContent(InputStream inputStream, UUID caseUuid) throws IOException { + try (ZipInputStream zipInputStream = new SecuredZipInputStream(inputStream, 1000, MAX_SIZE)) { + ZipEntry entry; + while ((entry = zipInputStream.getNextEntry()) != null) { + if (!entry.isDirectory()) { + processEntry(caseUuid, zipInputStream, entry); + } + zipInputStream.closeEntry(); + } + } + } + + private void copyZipContent(InputStream inputStream, UUID sourcecaseUuid, UUID caseUuid) throws IOException { + try (ZipInputStream zipInputStream = new SecuredZipInputStream(inputStream, 1000, MAX_SIZE)) { + ZipEntry entry; + while ((entry = zipInputStream.getNextEntry()) != null) { + if (!entry.isDirectory()) { + copyEntry(sourcecaseUuid, caseUuid, entry.getName() + GZIP_EXTENSION); + } + zipInputStream.closeEntry(); + } + } + } + + private void copyEntry(UUID sourcecaseUuid, UUID caseUuid, String fileName) { + // To optimize copy, files to copy are not downloaded on the case-server. They are directly copied on the S3 server. + CopyObjectRequest copyObjectRequest = CopyObjectRequest.builder() + .sourceBucket(bucketName) + .sourceKey(CASES_PREFIX + sourcecaseUuid + DELIMITER + fileName) + .destinationBucket(bucketName) + .destinationKey(CASES_PREFIX + caseUuid + DELIMITER + fileName) + .build(); + try { + s3Client.copyObject(copyObjectRequest); + } catch (S3Exception e) { + throw new ResponseStatusException(HttpStatus.NOT_FOUND, "Source file " + caseUuid + DELIMITER + fileName + NOT_FOUND); + } + } + + private void processEntry(UUID caseUuid, ZipInputStream zipInputStream, ZipEntry entry) throws IOException { + String fileName = entry.getName(); + String extractedKey = uuidToKeyWithFileName(caseUuid, fileName); + byte[] fileBytes = compress(ByteStreams.toByteArray(zipInputStream)); + + PutObjectRequest extractedFileRequest = PutObjectRequest.builder() + .bucket(bucketName) + .key(extractedKey + GZIP_EXTENSION) + .contentType(Files.probeContentType(Paths.get(fileName))) // Detect the MIME type + .build(); + s3Client.putObject(extractedFileRequest, RequestBody.fromBytes(fileBytes)); + } + + private static byte[] compress(byte[] data) throws IOException { + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + GZIPOutputStream gzipOutputStream = new GZIPOutputStream(outputStream); + gzipOutputStream.write(data, 0, data.length); + gzipOutputStream.close(); + return outputStream.toByteArray(); + } + + public static byte[] decompress(byte[] data) throws IOException { + GZIPInputStream gzipInputStream = new GZIPInputStream(new ByteArrayInputStream(data)); + return IOUtils.toByteArray(gzipInputStream); + } + + @Override + public UUID duplicateCase(UUID sourceCaseUuid, boolean withExpiration) { + if (!caseExists(sourceCaseUuid)) { + throw new ResponseStatusException(HttpStatus.NOT_FOUND, "Source case " + sourceCaseUuid + NOT_FOUND); + } + + String sourceKey = uuidToKeyWithOriginalFileName(sourceCaseUuid); + UUID newCaseUuid = UUID.randomUUID(); + String targetKey = uuidToKeyWithFileName(newCaseUuid, parseFilenameFromKey(sourceKey)); + // To optimize copy, cases to copy are not downloaded on the case-server. They are directly copied on the S3 server. + CopyObjectRequest copyObjectRequest = CopyObjectRequest.builder() + .sourceBucket(bucketName) + .sourceKey(sourceKey) + .destinationBucket(bucketName) + .destinationKey(targetKey) + .build(); + try { + s3Client.copyObject(copyObjectRequest); + } catch (S3Exception e) { + throw new ResponseStatusException(HttpStatus.NOT_FOUND, "source case " + sourceCaseUuid + NOT_FOUND); + } + CaseMetadataEntity existingCase = getCaseMetaDataEntity(sourceCaseUuid); + createCaseMetadataEntity(newCaseUuid, withExpiration, existingCase.isIndexed(), existingCase.getOriginalFilename(), existingCase.getCompressionFormat(), existingCase.getFormat()); + Optional caseBytes = getCaseBytes(newCaseUuid); + if (caseBytes.isPresent() && isArchivedCaseFile(existingCase.getOriginalFilename())) { + try { + copyZipContent(new ByteArrayInputStream(caseBytes.get()), sourceCaseUuid, newCaseUuid); + } catch (Exception e) { + throw CaseException.createCopyZipContentError(sourceCaseUuid, e); + } + } + CaseInfos existingCaseInfos = getCaseInfos(sourceCaseUuid); + CaseInfos caseInfos = createInfos(existingCaseInfos.getName(), newCaseUuid, existingCaseInfos.getFormat()); + if (existingCase.isIndexed()) { + caseInfosService.addCaseInfos(caseInfos); + } + notificationService.sendImportMessage(caseInfos.createMessage()); + return newCaseUuid; + } + + @Override + public Optional loadNetwork(UUID caseUuid) { + if (!caseExists(caseUuid)) { + return Optional.empty(); + } + return Optional.of(withS3DownloadedTempPath(caseUuid, path -> { + Network network = Network.read(path); + if (network == null) { + throw CaseException.createFileNotImportable(path); + } + return network; + })); + } + + @Override + public void deleteCase(UUID caseUuid) { + String prefixKey = uuidToKeyPrefix(caseUuid); + List objectsToDelete = s3Client.listObjectsV2(builder -> builder.bucket(bucketName).prefix(prefixKey)) + .contents() + .stream() + .map(s3Object -> ObjectIdentifier.builder().key(s3Object.key()).build()) + .toList(); + + if (!objectsToDelete.isEmpty()) { + DeleteObjectsRequest deleteObjectsRequest = DeleteObjectsRequest.builder() + .bucket(bucketName) + .delete(delete -> delete.objects(objectsToDelete)) + .build(); + s3Client.deleteObjects(deleteObjectsRequest); + caseInfosService.deleteCaseInfosByUuid(caseUuid.toString()); + caseMetadataRepository.deleteById(caseUuid); + } + } + + @Override + public void deleteAllCases() { + ListObjectsV2Request listObjectsRequest = ListObjectsV2Request.builder() + .bucket(bucketName) + .prefix(CASES_PREFIX) + .build(); + + ListObjectsV2Response listObjectsResponse = s3Client.listObjectsV2(listObjectsRequest); + List objectsToDelete = listObjectsResponse.contents().stream() + .map(s3Object -> ObjectIdentifier.builder().key(s3Object.key()).build()) + .toList(); + + if (!objectsToDelete.isEmpty()) { + DeleteObjectsRequest deleteObjectsRequest = DeleteObjectsRequest.builder() + .bucket(bucketName) + .delete(delete -> delete.objects(objectsToDelete)) + .build(); + + s3Client.deleteObjects(deleteObjectsRequest); + } + + caseInfosService.deleteAllCaseInfos(); + caseMetadataRepository.deleteAll(); + + } + + @Override + public void setComputationManager(ComputationManager computationManager) { + this.computationManager = Objects.requireNonNull(computationManager); + } + + @Override + public ComputationManager getComputationManager() { + return computationManager; + } + + @Override + public CaseMetadataRepository getCaseMetadataRepository() { + return caseMetadataRepository; + } + + private CaseMetadataEntity getCaseMetaDataEntity(UUID caseUuid) { + return caseMetadataRepository.findById(caseUuid).orElseThrow(() -> new ResponseStatusException(HttpStatus.NOT_FOUND, "case " + caseUuid + NOT_FOUND)); + } + +} diff --git a/src/main/java/com/powsybl/caseserver/services/SupervisionService.java b/src/main/java/com/powsybl/caseserver/service/SupervisionService.java similarity index 97% rename from src/main/java/com/powsybl/caseserver/services/SupervisionService.java rename to src/main/java/com/powsybl/caseserver/service/SupervisionService.java index 5a11ab6..5456678 100644 --- a/src/main/java/com/powsybl/caseserver/services/SupervisionService.java +++ b/src/main/java/com/powsybl/caseserver/service/SupervisionService.java @@ -4,7 +4,7 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -package com.powsybl.caseserver.services; +package com.powsybl.caseserver.service; import com.powsybl.caseserver.elasticsearch.CaseInfosRepository; import org.slf4j.Logger; diff --git a/src/main/resources/application-local.yml b/src/main/resources/application-local.yml index 3b4423e..c73be09 100644 --- a/src/main/resources/application-local.yml +++ b/src/main/resources/application-local.yml @@ -16,4 +16,9 @@ powsybl-ws: database: host: localhost -cleaning-cases-cron: 0 * 2 * * ? \ No newline at end of file +cleaning-cases-cron: 0 * 2 * * ? + +cloud: + aws: + s3: + endpoint: 172.17.0.1 diff --git a/src/main/resources/config/application.yaml b/src/main/resources/config/application.yaml index 83dec1f..2a27af7 100644 --- a/src/main/resources/config/application.yaml +++ b/src/main/resources/config/application.yaml @@ -8,6 +8,18 @@ spring: publishCaseImport-out-0: destination: ${powsybl-ws.rabbitmq.destination.prefix:}case.import output-bindings: publishCaseImport-out-0 + aws: + s3: + path-style-access-enabled: true + endpoint: http://172.17.0.1:19000 + region: + profile: + name: default + static: test + bucket: bucket-gridsuite + credentials: + access-key: minioadmin + secret-key: minioadmin powsybl-ws: database: @@ -15,3 +27,5 @@ powsybl-ws: cleaning-cases-cron: 0 * 2 * * ? +storage: + type: FS # FS or S3 \ No newline at end of file diff --git a/src/main/resources/db/changelog/changesets/changelog_20241017T125247Z.xml b/src/main/resources/db/changelog/changesets/changelog_20241017T125247Z.xml new file mode 100644 index 0000000..6b0a28e --- /dev/null +++ b/src/main/resources/db/changelog/changesets/changelog_20241017T125247Z.xml @@ -0,0 +1,18 @@ + + + + + + + + + + + + + + + + + + diff --git a/src/main/resources/db/changelog/db.changelog-master.yaml b/src/main/resources/db/changelog/db.changelog-master.yaml index 5ad8263..75ce859 100644 --- a/src/main/resources/db/changelog/db.changelog-master.yaml +++ b/src/main/resources/db/changelog/db.changelog-master.yaml @@ -11,3 +11,6 @@ databaseChangeLog: file: changesets/changelog_20240726T144717Z.xml relativeToChangelogFile: true + - include: + file: changesets/changelog_20241017T125247Z.xml + relativeToChangelogFile: true diff --git a/src/test/java/com/powsybl/caseserver/SupervisionControllerTest.java b/src/test/java/com/powsybl/caseserver/AbstractSupervisionControllerTest.java similarity index 78% rename from src/test/java/com/powsybl/caseserver/SupervisionControllerTest.java rename to src/test/java/com/powsybl/caseserver/AbstractSupervisionControllerTest.java index 2a43eda..5adb997 100644 --- a/src/test/java/com/powsybl/caseserver/SupervisionControllerTest.java +++ b/src/test/java/com/powsybl/caseserver/AbstractSupervisionControllerTest.java @@ -6,15 +6,11 @@ */ package com.powsybl.caseserver; -import com.google.common.jimfs.Configuration; -import com.google.common.jimfs.Jimfs; import com.powsybl.caseserver.repository.CaseMetadataRepository; -import com.powsybl.caseserver.services.SupervisionService; -import com.powsybl.computation.ComputationManager; +import com.powsybl.caseserver.service.CaseService; +import com.powsybl.caseserver.service.SupervisionService; import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.mockito.Mockito; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc; @@ -38,22 +34,20 @@ */ @AutoConfigureMockMvc @SpringBootTest(classes = {CaseApplication.class}, properties = {"case-store-directory=/cases"}) -class SupervisionControllerTest { +abstract class AbstractSupervisionControllerTest { @Autowired private SupervisionService supervisionService; @Autowired - private CaseMetadataRepository caseMetadataRepository; + CaseMetadataRepository caseMetadataRepository; + CaseService caseService; @Autowired - private CaseService caseService; + protected MockMvc mockMvc; - @Autowired - private MockMvc mockMvc; - - @Value("${case-store-directory}") - private String rootDirectory; + @Value("${case-store-directory:#{systemProperties['user.home'].concat(\"/cases\")}}") + String rootDirectory; private static final String TEST_CASE = "testCase.xiidm"; - private FileSystem fileSystem; + FileSystem fileSystem; @Test void testGetCaseInfosCount() throws Exception { @@ -106,18 +100,11 @@ private void importCase(Boolean indexed) throws Exception { } private static MockMultipartFile createMockMultipartFile() throws IOException { - try (InputStream inputStream = CaseControllerTest.class.getResourceAsStream("/" + SupervisionControllerTest.TEST_CASE)) { - return new MockMultipartFile("file", SupervisionControllerTest.TEST_CASE, MediaType.TEXT_PLAIN_VALUE, inputStream); + try (InputStream inputStream = AbstractSupervisionControllerTest.class.getResourceAsStream("/" + AbstractSupervisionControllerTest.TEST_CASE)) { + return new MockMultipartFile("file", AbstractSupervisionControllerTest.TEST_CASE, MediaType.TEXT_PLAIN_VALUE, inputStream); } } - @BeforeEach - void setUp() { - fileSystem = Jimfs.newFileSystem(Configuration.unix()); - caseService.setFileSystem(fileSystem); - caseService.setComputationManager(Mockito.mock(ComputationManager.class)); - } - @AfterEach void tearDown() throws Exception { fileSystem.close(); diff --git a/src/test/java/com/powsybl/caseserver/CaseFileNameParserTests.java b/src/test/java/com/powsybl/caseserver/CaseFileNameParserTests.java index f284dcc..f6591f9 100644 --- a/src/test/java/com/powsybl/caseserver/CaseFileNameParserTests.java +++ b/src/test/java/com/powsybl/caseserver/CaseFileNameParserTests.java @@ -6,23 +6,15 @@ */ package com.powsybl.caseserver; -import com.powsybl.caseserver.dto.CaseInfos; -import com.powsybl.caseserver.dto.cgmes.CgmesCaseInfos; -import com.powsybl.caseserver.dto.entsoe.EntsoeCaseInfos; import com.powsybl.caseserver.elasticsearch.DisableElasticsearch; import com.powsybl.caseserver.parsers.FileNameInfos; import com.powsybl.caseserver.parsers.FileNameParser; -import com.powsybl.caseserver.parsers.cgmes.CgmesFileNameParser; import com.powsybl.caseserver.parsers.entsoe.EntsoeFileNameParser; -import com.powsybl.entsoe.util.EntsoeGeographicalCode; -import com.powsybl.iidm.network.Country; import org.junit.jupiter.api.Test; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; import java.nio.file.Path; import java.util.Optional; -import java.util.UUID; import static org.junit.jupiter.api.Assertions.*; @@ -32,96 +24,9 @@ @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.NONE) @DisableElasticsearch class CaseFileNameParserTests { - private static final String SN_UCTE_CASE_FILE_NAME = "20200103_0915_SN5_D80.UCT"; - private static final String ID_UCTE_CASE_FILE_NAME = "20200424_1330_135_CH2.UCT"; - private static final String D1_UCTE_CASE_FILE_NAME = "20200110_0430_FO5_FR0.uct"; - private static final String D2_UCTE_CASE_FILE_NAME = "20200430_1530_2D4_D41.uct"; - private static final String TEST_CGMES_CASE_FILE_NAME = "20200424T1330Z_2D_RTEFRANCE_001.zip"; private static final String CASE_FILE_NAME_INCORRECT = "20200103_0915_SN5.UCT"; private static final String TEST_OTHER_CASE_FILE_NAME = "testCase.xiidm"; - @Autowired - private CaseService caseService; - - @Test - void testValidNameUcteSN() { - EntsoeCaseInfos caseInfos = (EntsoeCaseInfos) createInfos(SN_UCTE_CASE_FILE_NAME); - assertEquals(SN_UCTE_CASE_FILE_NAME, caseInfos.getName()); - assertEquals("UCTE", caseInfos.getFormat()); - assertTrue(caseInfos.getDate().isEqual(EntsoeFileNameParser.parseDateTime(SN_UCTE_CASE_FILE_NAME.substring(0, 13)))); - assertEquals(Integer.valueOf(0), caseInfos.getForecastDistance()); - assertSame(EntsoeGeographicalCode.D8, caseInfos.getGeographicalCode()); - assertSame(Country.DE, caseInfos.getCountry()); - assertEquals(Integer.valueOf(0), caseInfos.getVersion()); - } - - @Test - void testValidNameUcteID() { - EntsoeCaseInfos caseInfos = (EntsoeCaseInfos) createInfos(ID_UCTE_CASE_FILE_NAME); - assertEquals(ID_UCTE_CASE_FILE_NAME, caseInfos.getName()); - assertEquals("UCTE", caseInfos.getFormat()); - assertTrue(caseInfos.getDate().isEqual(EntsoeFileNameParser.parseDateTime(ID_UCTE_CASE_FILE_NAME.substring(0, 13)))); - assertEquals(Integer.valueOf(780), caseInfos.getForecastDistance()); - assertSame(EntsoeGeographicalCode.CH, caseInfos.getGeographicalCode()); - assertSame(Country.CH, caseInfos.getCountry()); - assertEquals(Integer.valueOf(2), caseInfos.getVersion()); - } - - @Test - void testValidNameUcte1D() { - EntsoeCaseInfos caseInfos = (EntsoeCaseInfos) createInfos(D1_UCTE_CASE_FILE_NAME); - assertEquals(D1_UCTE_CASE_FILE_NAME, caseInfos.getName()); - assertEquals("UCTE", caseInfos.getFormat()); - assertTrue(caseInfos.getDate().isEqual(EntsoeFileNameParser.parseDateTime(D1_UCTE_CASE_FILE_NAME.substring(0, 13)))); - assertEquals(Integer.valueOf(630), caseInfos.getForecastDistance()); - assertSame(EntsoeGeographicalCode.FR, caseInfos.getGeographicalCode()); - assertSame(Country.FR, caseInfos.getCountry()); - assertEquals(Integer.valueOf(0), caseInfos.getVersion()); - } - - @Test - void testValidNameUcte2D() { - EntsoeCaseInfos caseInfos = (EntsoeCaseInfos) createInfos(D2_UCTE_CASE_FILE_NAME); - assertEquals(D2_UCTE_CASE_FILE_NAME, caseInfos.getName()); - assertEquals("UCTE", caseInfos.getFormat()); - assertTrue(caseInfos.getDate().isEqual(EntsoeFileNameParser.parseDateTime(D2_UCTE_CASE_FILE_NAME.substring(0, 13)))); - assertEquals(Integer.valueOf(2730), caseInfos.getForecastDistance()); - assertSame(EntsoeGeographicalCode.D4, caseInfos.getGeographicalCode()); - assertSame(Country.DE, caseInfos.getCountry()); - assertEquals(Integer.valueOf(1), caseInfos.getVersion()); - } - - @Test - void testValidNameCgmes() { - CgmesCaseInfos caseInfos = (CgmesCaseInfos) createInfos(TEST_CGMES_CASE_FILE_NAME); - assertEquals(TEST_CGMES_CASE_FILE_NAME, caseInfos.getName()); - assertEquals("CGMES", caseInfos.getFormat()); - assertTrue(caseInfos.getDate().isEqual(CgmesFileNameParser.parseDateTime(TEST_CGMES_CASE_FILE_NAME.substring(0, 14)))); - assertEquals("2D", caseInfos.getBusinessProcess()); - assertEquals("RTEFRANCE", caseInfos.getTso()); - assertEquals(Integer.valueOf(1), caseInfos.getVersion()); - } - - @Test - void testNonValidNameEntsoe() { - CaseInfos caseInfos = createInfos(TEST_OTHER_CASE_FILE_NAME); - assertEquals(TEST_OTHER_CASE_FILE_NAME, caseInfos.getName()); - assertEquals("XIIDM", caseInfos.getFormat()); - } - - private CaseInfos createInfos(String fileName) { - Path casePath = Path.of(this.getClass().getResource("/" + fileName).getPath()); - String fileBaseName = casePath.getFileName().toString(); - String format = caseService.getFormat(casePath); - return caseService.createInfos(fileBaseName, UUID.randomUUID(), format); - } - - @Test - void testCreateDefaultCaseInfo() { - CaseInfos infos = caseService.createInfos(TEST_OTHER_CASE_FILE_NAME, UUID.randomUUID(), "UNKNOW"); - assertEquals(CaseInfos.class, infos.getClass()); - } - @Test void testFileNameIncorrect() { Path casePath = Path.of(this.getClass().getResource("/" + CASE_FILE_NAME_INCORRECT).getPath()); diff --git a/src/test/java/com/powsybl/caseserver/CaseInfosELRepositoryTests.java b/src/test/java/com/powsybl/caseserver/CaseInfosELRepositoryTests.java index 166f815..09db7a4 100644 --- a/src/test/java/com/powsybl/caseserver/CaseInfosELRepositoryTests.java +++ b/src/test/java/com/powsybl/caseserver/CaseInfosELRepositoryTests.java @@ -6,6 +6,7 @@ */ package com.powsybl.caseserver; +import com.powsybl.caseserver.service.CaseService; import com.powsybl.caseserver.dto.CaseInfos; import com.powsybl.caseserver.dto.cgmes.CgmesCaseInfos; import com.powsybl.caseserver.dto.entsoe.EntsoeCaseInfos; @@ -35,6 +36,9 @@ class CaseInfosELRepositoryTests { private static final String D4_UCTE_CASE_FILE_NAME = "20200430_1530_2D4_D41.uct"; private static final String TEST_CGMES_CASE_FILE_NAME = "20200424T1330Z_2D_RTEFRANCE_001.zip"; + private static final String UCTE_FORMAT = "UCTE"; + private static final String CGMES_FORMAT = "CGMES"; + @Autowired private CaseService caseService; @@ -43,13 +47,13 @@ class CaseInfosELRepositoryTests { @Test void testAddDeleteCaseInfos() { - EntsoeCaseInfos caseInfos1 = (EntsoeCaseInfos) caseInfosService.addCaseInfos(createInfos(SN_UCTE_CASE_FILE_NAME)); + EntsoeCaseInfos caseInfos1 = (EntsoeCaseInfos) caseInfosService.addCaseInfos(createInfos(SN_UCTE_CASE_FILE_NAME, UCTE_FORMAT)); Optional caseInfosAfter1 = caseInfosService.getCaseInfosByUuid(caseInfos1.getUuid().toString()); assertFalse(caseInfosAfter1.isEmpty()); assertEquals(caseInfos1, caseInfosAfter1.get()); assertThat(caseInfosAfter1.get()).usingRecursiveAssertion().isEqualTo(caseInfos1); - EntsoeCaseInfos caseInfos2 = (EntsoeCaseInfos) caseInfosService.addCaseInfos(createInfos(ID2_UCTE_CASE_FILE_NAME)); + EntsoeCaseInfos caseInfos2 = (EntsoeCaseInfos) caseInfosService.addCaseInfos(createInfos(ID2_UCTE_CASE_FILE_NAME, UCTE_FORMAT)); Optional caseInfosAfter2 = caseInfosService.getCaseInfosByUuid(caseInfos2.getUuid().toString()); assertFalse(caseInfosAfter2.isEmpty()); assertEquals(caseInfos2, caseInfosAfter2.get()); @@ -71,7 +75,7 @@ void testAddDeleteCaseInfos() { all = caseInfosService.getAllCaseInfos(); assertTrue(all.isEmpty()); - caseInfosService.recreateAllCaseInfos(List.of(createInfos(SN_UCTE_CASE_FILE_NAME), createInfos(TEST_CGMES_CASE_FILE_NAME))); + caseInfosService.recreateAllCaseInfos(List.of(createInfos(SN_UCTE_CASE_FILE_NAME, UCTE_FORMAT), createInfos(TEST_CGMES_CASE_FILE_NAME, CGMES_FORMAT))); all = caseInfosService.getAllCaseInfos(); assertEquals(2, all.size()); assertEquals(SN_UCTE_CASE_FILE_NAME, all.get(0).getName()); @@ -86,13 +90,13 @@ void searchCaseInfos() { List all = caseInfosService.getAllCaseInfos(); assertTrue(all.isEmpty()); - EntsoeCaseInfos ucte1 = (EntsoeCaseInfos) caseInfosService.addCaseInfos(createInfos(SN_UCTE_CASE_FILE_NAME)); - EntsoeCaseInfos ucte2 = (EntsoeCaseInfos) caseInfosService.addCaseInfos(createInfos(ID1_UCTE_CASE_FILE_NAME)); - EntsoeCaseInfos ucte3 = (EntsoeCaseInfos) caseInfosService.addCaseInfos(createInfos(ID2_UCTE_CASE_FILE_NAME)); - EntsoeCaseInfos ucte4 = (EntsoeCaseInfos) caseInfosService.addCaseInfos(createInfos(FO1_UCTE_CASE_FILE_NAME)); - EntsoeCaseInfos ucte5 = (EntsoeCaseInfos) caseInfosService.addCaseInfos(createInfos(FO2_UCTE_CASE_FILE_NAME)); - EntsoeCaseInfos ucte6 = (EntsoeCaseInfos) caseInfosService.addCaseInfos(createInfos(D4_UCTE_CASE_FILE_NAME)); - CgmesCaseInfos cgmes = (CgmesCaseInfos) caseInfosService.addCaseInfos(createInfos(TEST_CGMES_CASE_FILE_NAME)); + EntsoeCaseInfos ucte1 = (EntsoeCaseInfos) caseInfosService.addCaseInfos(createInfos(SN_UCTE_CASE_FILE_NAME, UCTE_FORMAT)); + EntsoeCaseInfos ucte2 = (EntsoeCaseInfos) caseInfosService.addCaseInfos(createInfos(ID1_UCTE_CASE_FILE_NAME, UCTE_FORMAT)); + EntsoeCaseInfos ucte3 = (EntsoeCaseInfos) caseInfosService.addCaseInfos(createInfos(ID2_UCTE_CASE_FILE_NAME, UCTE_FORMAT)); + EntsoeCaseInfos ucte4 = (EntsoeCaseInfos) caseInfosService.addCaseInfos(createInfos(FO1_UCTE_CASE_FILE_NAME, UCTE_FORMAT)); + EntsoeCaseInfos ucte5 = (EntsoeCaseInfos) caseInfosService.addCaseInfos(createInfos(FO2_UCTE_CASE_FILE_NAME, UCTE_FORMAT)); + EntsoeCaseInfos ucte6 = (EntsoeCaseInfos) caseInfosService.addCaseInfos(createInfos(D4_UCTE_CASE_FILE_NAME, UCTE_FORMAT)); + CgmesCaseInfos cgmes = (CgmesCaseInfos) caseInfosService.addCaseInfos(createInfos(TEST_CGMES_CASE_FILE_NAME, CGMES_FORMAT)); all = caseInfosService.searchCaseInfos("*"); assertFalse(all.isEmpty()); @@ -163,10 +167,9 @@ void searchCaseInfos() { assertTrue(list.size() == 1 && list.contains(ucte1)); } - private CaseInfos createInfos(String fileName) { + private CaseInfos createInfos(String fileName, String format) { Path casePath = Path.of(this.getClass().getResource("/" + fileName).getPath()); String fileBaseName = casePath.getFileName().toString(); - String format = caseService.getFormat(casePath); return caseService.createInfos(fileBaseName, UUID.randomUUID(), format); } } diff --git a/src/test/java/com/powsybl/caseserver/ContextConfigurationWithTestChannel.java b/src/test/java/com/powsybl/caseserver/ContextConfigurationWithTestChannel.java index f1adc6e..9d08ed8 100644 --- a/src/test/java/com/powsybl/caseserver/ContextConfigurationWithTestChannel.java +++ b/src/test/java/com/powsybl/caseserver/ContextConfigurationWithTestChannel.java @@ -10,16 +10,14 @@ import org.springframework.cloud.stream.binder.test.TestChannelBinderConfiguration; import org.springframework.test.context.ContextConfiguration; -import java.lang.annotation.ElementType; -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; -import java.lang.annotation.Target; +import java.lang.annotation.*; /** * @author Slimane Amar */ @Target(ElementType.TYPE) @Retention(RetentionPolicy.RUNTIME) -@ContextConfiguration(classes = {CaseApplication.class, TestChannelBinderConfiguration.class}) +@Inherited +@ContextConfiguration(classes = { CaseApplication.class, TestChannelBinderConfiguration.class }) public @interface ContextConfigurationWithTestChannel { } diff --git a/src/test/java/com/powsybl/caseserver/FsSupervisionControllerTest.java b/src/test/java/com/powsybl/caseserver/FsSupervisionControllerTest.java new file mode 100644 index 0000000..14e3b65 --- /dev/null +++ b/src/test/java/com/powsybl/caseserver/FsSupervisionControllerTest.java @@ -0,0 +1,39 @@ +/** + * Copyright (c) 2024, RTE (http://www.rte-france.com) + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ +package com.powsybl.caseserver; + +import com.google.common.jimfs.Configuration; +import com.google.common.jimfs.Jimfs; +import com.powsybl.caseserver.service.FsCaseService; +import com.powsybl.computation.ComputationManager; +import org.junit.jupiter.api.BeforeEach; +import org.mockito.Mockito; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.TestPropertySource; + +/** + * @author Jamal KHEYYAD + */ +@AutoConfigureMockMvc +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.MOCK, properties = {"case-store-directory=/cases"}) +@TestPropertySource(properties = {"storage.type=FS"}) +class FsSupervisionControllerTest extends AbstractSupervisionControllerTest { + + @Autowired + private FsCaseService fsCaseService; + + @BeforeEach + public void setUp() { + caseService = fsCaseService; + fileSystem = Jimfs.newFileSystem(Configuration.unix()); + ((FsCaseService) caseService).setFileSystem(fileSystem); + caseService.setComputationManager(Mockito.mock(ComputationManager.class)); + caseMetadataRepository.deleteAll(); + } +} diff --git a/src/test/java/com/powsybl/caseserver/S3SupervisionControllerTest.java b/src/test/java/com/powsybl/caseserver/S3SupervisionControllerTest.java new file mode 100644 index 0000000..562b736 --- /dev/null +++ b/src/test/java/com/powsybl/caseserver/S3SupervisionControllerTest.java @@ -0,0 +1,39 @@ +/** + * Copyright (c) 2024, RTE (http://www.rte-france.com) + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ +package com.powsybl.caseserver; + +import com.google.common.jimfs.Configuration; +import com.google.common.jimfs.Jimfs; +import com.powsybl.caseserver.service.MinioContainerConfig; +import com.powsybl.caseserver.service.S3CaseService; +import com.powsybl.computation.ComputationManager; +import org.junit.jupiter.api.BeforeEach; +import org.mockito.Mockito; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.TestPropertySource; + +/** + * @author Jamal KHEYYAD + */ +@AutoConfigureMockMvc +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.MOCK) +@TestPropertySource(properties = {"storage.type=S3"}) +class S3SupervisionControllerTest extends AbstractSupervisionControllerTest implements MinioContainerConfig { + + @Autowired + private S3CaseService s3CaseService; + + @BeforeEach + void setUp() { + caseService = s3CaseService; + fileSystem = Jimfs.newFileSystem(Configuration.unix()); + caseService.setComputationManager(Mockito.mock(ComputationManager.class)); + caseService.deleteAllCases(); + } +} diff --git a/src/test/java/com/powsybl/caseserver/ScheduledCaseCleanerTest.java b/src/test/java/com/powsybl/caseserver/ScheduledCaseCleanerTest.java index 9a90c1c..31f3868 100644 --- a/src/test/java/com/powsybl/caseserver/ScheduledCaseCleanerTest.java +++ b/src/test/java/com/powsybl/caseserver/ScheduledCaseCleanerTest.java @@ -6,6 +6,7 @@ */ package com.powsybl.caseserver; +import com.powsybl.caseserver.service.CaseService; import com.powsybl.caseserver.elasticsearch.CaseInfosRepository; import com.powsybl.caseserver.elasticsearch.DisableElasticsearch; import com.powsybl.caseserver.repository.CaseMetadataEntity; @@ -53,9 +54,9 @@ void cleanDB() { void test() { Instant now = Instant.now(); Instant yesterday = now.minus(1, ChronoUnit.DAYS); - CaseMetadataEntity shouldNotExpireEntity = new CaseMetadataEntity(UUID.randomUUID(), now.plus(1, ChronoUnit.HOURS), false); - CaseMetadataEntity shouldExpireEntity = new CaseMetadataEntity(UUID.randomUUID(), yesterday.plus(1, ChronoUnit.HOURS), false); - CaseMetadataEntity noExpireDateEntity = new CaseMetadataEntity(UUID.randomUUID(), null, false); + CaseMetadataEntity shouldNotExpireEntity = new CaseMetadataEntity(UUID.randomUUID(), now.plus(1, ChronoUnit.HOURS), false, "originalName", "compressionFormat", "format"); + CaseMetadataEntity shouldExpireEntity = new CaseMetadataEntity(UUID.randomUUID(), yesterday.plus(1, ChronoUnit.HOURS), false, "originalName", "compressionFormat", "format"); + CaseMetadataEntity noExpireDateEntity = new CaseMetadataEntity(UUID.randomUUID(), null, false, "originalName", "compressionFormat", "format"); caseMetadataRepository.save(shouldExpireEntity); caseMetadataRepository.save(shouldNotExpireEntity); caseMetadataRepository.save(noExpireDateEntity); diff --git a/src/test/java/com/powsybl/caseserver/datasource/AbstractCaseDataSourceControllerTest.java b/src/test/java/com/powsybl/caseserver/datasource/AbstractCaseDataSourceControllerTest.java new file mode 100644 index 0000000..45f91bc --- /dev/null +++ b/src/test/java/com/powsybl/caseserver/datasource/AbstractCaseDataSourceControllerTest.java @@ -0,0 +1,218 @@ +/** + * Copyright (c) 2023, RTE (http://www.rte-france.com) + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ +package com.powsybl.caseserver.datasource; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.powsybl.caseserver.elasticsearch.DisableElasticsearch; +import com.powsybl.caseserver.service.CaseService; +import com.powsybl.commons.datasource.DataSource; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.cloud.stream.function.StreamBridge; +import org.springframework.mock.web.MockMultipartFile; +import org.springframework.test.web.servlet.MockMvc; +import org.springframework.test.web.servlet.MvcResult; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.nio.file.Paths; +import java.util.Set; +import java.util.UUID; + +import static org.junit.Assert.assertEquals; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.*; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +/** + * @author Abdelsalem Hedhili + * @author Ghazwa Rehili + * @author Etienne Homer + */ +@DisableElasticsearch +public abstract class AbstractCaseDataSourceControllerTest { + + @MockBean + StreamBridge streamBridge; + + @Autowired + private MockMvc mvc; + + protected static CaseService caseService; + + @Value("${case-store-directory:#{systemProperties['user.home'].concat(\"/cases\")}}") + protected String rootDirectory; + + static final String CGMES_ZIP_NAME = "CGMES_v2415_MicroGridTestConfiguration_BC_BE_v2.zip"; + + static final String CGMES_FILE_NAME = "CGMES_v2415_MicroGridTestConfiguration_BC_BE_v2/MicroGridTestConfiguration_BC_BE_DL_V2.xml"; + + UUID cgmesCaseUuid; + + protected DataSource cgmesDataSource; + + private final ObjectMapper mapper = new ObjectMapper(); + + public static UUID importCase(String filename, String contentType) throws IOException { + UUID caseUUID; + try (InputStream inputStream = S3CaseDataSourceControllerTest.class.getResourceAsStream("/" + filename)) { + caseUUID = caseService.importCase(new MockMultipartFile(filename, filename, contentType, inputStream.readAllBytes()), false, false); + } + return caseUUID; + } + + @Test + public void testBaseName() throws Exception { + MvcResult mvcResult = mvc.perform(get("/v1/cases/{caseUuid}/datasource/baseName", cgmesCaseUuid)) + .andExpect(status().isOk()) + .andReturn(); + + assertEquals(cgmesDataSource.getBaseName(), mvcResult.getResponse().getContentAsString()); + } + + @Test + public void testListName() throws Exception { + MvcResult mvcResult = mvc.perform(get("/v1/cases/{caseUuid}/datasource/list", cgmesCaseUuid) + .param("regex", ".*")) + .andExpect(status().isOk()) + .andReturn(); + + Set nameList = mapper.readValue(mvcResult.getResponse().getContentAsString(), Set.class); + assertEquals(cgmesDataSource.listNames(".*"), nameList); + } + + @Test + public void testInputStreamWithFileName() throws Exception { + MvcResult mvcResult = mvc.perform(get("/v1/cases/{caseUuid}/datasource", cgmesCaseUuid) + .param("fileName", CGMES_FILE_NAME)) + .andExpect(status().isOk()) + .andReturn(); + + try (InputStreamReader isReader = new InputStreamReader(cgmesDataSource.newInputStream(CGMES_FILE_NAME), StandardCharsets.UTF_8)) { + BufferedReader reader = new BufferedReader(isReader); + StringBuilder datasourceResponse = new StringBuilder(); + String str; + while ((str = reader.readLine()) != null) { + datasourceResponse.append(str).append("\n"); + } + assertEquals(datasourceResponse.toString(), mvcResult.getResponse().getContentAsString()); + } + } + + private static String readDataSource(DataSource dataSource, String fileName) throws Exception { + try (InputStreamReader isReader = new InputStreamReader(dataSource.newInputStream(fileName), StandardCharsets.UTF_8)) { + BufferedReader reader = new BufferedReader(isReader); + StringBuilder datasourceResponse = new StringBuilder(); + String str; + while ((str = reader.readLine()) != null) { + datasourceResponse.append(str).append("\n"); + } + return datasourceResponse.toString(); + } + } + + @Test + public void testInputStreamWithZipFile() throws Exception { + String zipName = "LF.zip"; + String fileName = "LF.xml"; + UUID caseUuid = importCase(zipName, "application/zip"); + DataSource dataSource = DataSource.fromPath(Paths.get(S3CaseDataSourceControllerTest.class.getResource("/" + zipName).toURI())); + + MvcResult mvcResult = mvc.perform(get("/v1/cases/{caseUuid}/datasource", caseUuid) + .param("fileName", fileName)) + .andExpect(status().isOk()) + .andReturn(); + assertEquals(readDataSource(dataSource, fileName), mvcResult.getResponse().getContentAsString()); + } + + @Test + public void testInputStreamWithGZipFile() throws Exception { + String gzipName = "LF.xml.gz"; + String fileName = "LF.xml"; + UUID caseUuid = importCase(gzipName, "application/zip"); + DataSource dataSource = DataSource.fromPath(Paths.get(S3CaseDataSourceControllerTest.class.getResource("/" + gzipName).toURI())); + + MvcResult mvcResult = mvc.perform(get("/v1/cases/{caseUuid}/datasource", caseUuid) + .param("fileName", fileName)) + .andExpect(status().isOk()) + .andReturn(); + assertEquals(readDataSource(dataSource, fileName), mvcResult.getResponse().getContentAsString()); + } + + @Test + public void testInputStreamWithXiidmPlainFile() throws Exception { + String fileName = "LF.xml"; + UUID caseUuid = importCase(fileName, "application/zip"); + DataSource dataSource = DataSource.fromPath(Paths.get(S3CaseDataSourceControllerTest.class.getResource("/" + fileName).toURI())); + + MvcResult mvcResult = mvc.perform(get("/v1/cases/{caseUuid}/datasource", caseUuid) + .param("fileName", fileName)) + .andExpect(status().isOk()) + .andReturn(); + + assertEquals(readDataSource(dataSource, fileName), mvcResult.getResponse().getContentAsString()); + } + + @Test + public void testInputStreamWithSuffixExt() throws Exception { + String suffix = "/MicroGridTestConfiguration_BC_BE_DL_V2"; + String ext = "xml"; + MvcResult mvcResult = mvc.perform(get("/v1/cases/{caseUuid}/datasource", cgmesCaseUuid) + .param("suffix", suffix) + .param("ext", ext)) + .andExpect(status().isOk()) + .andReturn(); + + try (InputStreamReader isReader = new InputStreamReader(cgmesDataSource.newInputStream(suffix, ext), StandardCharsets.UTF_8)) { + BufferedReader reader = new BufferedReader(isReader); + StringBuilder datasourceResponse = new StringBuilder(); + String str; + while ((str = reader.readLine()) != null) { + datasourceResponse.append(str).append("\n"); + } + assertEquals(datasourceResponse.toString(), mvcResult.getResponse().getContentAsString()); + } + } + + @Test + public void testExistsWithFileName() throws Exception { + MvcResult mvcResult = mvc.perform(get("/v1/cases/{caseUuid}/datasource/exists", cgmesCaseUuid) + .param("fileName", CGMES_FILE_NAME)) + .andExpect(status().isOk()) + .andReturn(); + + Boolean res = mapper.readValue(mvcResult.getResponse().getContentAsString(), Boolean.class); + assertEquals(cgmesDataSource.exists(CGMES_FILE_NAME), res); + + mvcResult = mvc.perform(get("/v1/cases/{caseUuid}/datasource/exists", cgmesCaseUuid) + .param("fileName", "random")) + .andExpect(status().isOk()) + .andReturn(); + + res = mapper.readValue(mvcResult.getResponse().getContentAsString(), Boolean.class); + assertEquals(cgmesDataSource.exists("random"), res); + } + + @Test + public void testExistsWithSuffixExt() throws Exception { + String suffix = "random"; + String ext = "uct"; + MvcResult mvcResult = mvc.perform(get("/v1/cases/{caseUuid}/datasource/exists", cgmesCaseUuid) + .param("suffix", suffix) + .param("ext", ext)) + .andExpect(status().isOk()) + .andReturn(); + + Boolean res = mapper.readValue(mvcResult.getResponse().getContentAsString(), Boolean.class); + assertEquals(cgmesDataSource.exists(suffix, ext), res); + } + +} diff --git a/src/test/java/com/powsybl/caseserver/datasource/FsCaseDataSourceControllerTest.java b/src/test/java/com/powsybl/caseserver/datasource/FsCaseDataSourceControllerTest.java new file mode 100644 index 0000000..4b89b22 --- /dev/null +++ b/src/test/java/com/powsybl/caseserver/datasource/FsCaseDataSourceControllerTest.java @@ -0,0 +1,60 @@ +/** + * Copyright (c) 2023, RTE (http://www.rte-france.com) + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ +package com.powsybl.caseserver.datasource; + +import com.google.common.jimfs.Jimfs; +import com.powsybl.caseserver.ContextConfigurationWithTestChannel; +import com.powsybl.caseserver.service.FsCaseService; +import com.powsybl.commons.datasource.DataSource; +import org.junit.jupiter.api.BeforeEach; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.TestPropertySource; + +import java.io.IOException; +import java.io.InputStream; +import java.net.URISyntaxException; +import java.nio.file.*; +import java.util.UUID; + +/** + * @author Ghazwa Rehili + */ +@AutoConfigureMockMvc +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.MOCK) +@TestPropertySource(properties = {"storage.type=FS"}) +@ContextConfigurationWithTestChannel +class FsCaseDataSourceControllerTest extends AbstractCaseDataSourceControllerTest { + + @Autowired + protected FsCaseService fsCaseService; + + FileSystem fileSystem = Jimfs.newFileSystem(); + + @BeforeEach + void setUp() throws URISyntaxException, IOException { + caseService = fsCaseService; + cgmesCaseUuid = UUID.randomUUID(); + Path path = fileSystem.getPath(rootDirectory); + if (!Files.exists(path)) { + Files.createDirectories(path); + } + Path cgmesCaseDirectory = fileSystem.getPath(rootDirectory).resolve(cgmesCaseUuid.toString()); + if (!Files.exists(cgmesCaseDirectory)) { + Files.createDirectories(cgmesCaseDirectory); + } + + fsCaseService.setFileSystem(fileSystem); + //insert a cgmes in the FS + try (InputStream cgmesURL = getClass().getResourceAsStream("/" + CGMES_ZIP_NAME); + ) { + Files.copy(cgmesURL, cgmesCaseDirectory.resolve(CGMES_ZIP_NAME), StandardCopyOption.REPLACE_EXISTING); + } + cgmesDataSource = DataSource.fromPath(Paths.get(getClass().getResource("/" + CGMES_ZIP_NAME).toURI())); + } +} diff --git a/src/test/java/com/powsybl/caseserver/datasource/S3CaseDataSourceControllerTest.java b/src/test/java/com/powsybl/caseserver/datasource/S3CaseDataSourceControllerTest.java new file mode 100644 index 0000000..112ea55 --- /dev/null +++ b/src/test/java/com/powsybl/caseserver/datasource/S3CaseDataSourceControllerTest.java @@ -0,0 +1,43 @@ +/** + * Copyright (c) 2024, RTE (http://www.rte-france.com) + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ +package com.powsybl.caseserver.datasource; + +import com.powsybl.caseserver.ContextConfigurationWithTestChannel; +import com.powsybl.caseserver.service.MinioContainerConfig; +import com.powsybl.caseserver.service.S3CaseService; +import com.powsybl.commons.datasource.DataSource; +import org.junit.jupiter.api.BeforeEach; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.TestPropertySource; + +import java.io.*; +import java.net.URISyntaxException; +import java.nio.file.Paths; + +/** + * @author Ghazwa Rehili + * @author Etienne Homer + */ +@AutoConfigureMockMvc +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.MOCK) +@TestPropertySource(properties = {"storage.type=S3"}) +@ContextConfigurationWithTestChannel +class S3CaseDataSourceControllerTest extends AbstractCaseDataSourceControllerTest implements MinioContainerConfig { + + @Autowired + protected S3CaseService s3CaseService; + + @BeforeEach + void setUp() throws URISyntaxException, IOException { + caseService = s3CaseService; + cgmesCaseUuid = importCase(CGMES_ZIP_NAME, "application/zip"); + cgmesDataSource = DataSource.fromPath(Paths.get(S3CaseDataSourceControllerTest.class.getResource("/" + CGMES_ZIP_NAME).toURI())); + } + +} diff --git a/src/test/java/com/powsybl/caseserver/datasource/util/CaseDataSourceControllerTest.java b/src/test/java/com/powsybl/caseserver/datasource/util/CaseDataSourceControllerTest.java deleted file mode 100644 index 03848f9..0000000 --- a/src/test/java/com/powsybl/caseserver/datasource/util/CaseDataSourceControllerTest.java +++ /dev/null @@ -1,195 +0,0 @@ -/** - * Copyright (c) 2020, RTE (http://www.rte-france.com) - * This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. - */ -package com.powsybl.caseserver.datasource.util; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.jimfs.Jimfs; -import com.powsybl.caseserver.CaseApplication; -import com.powsybl.caseserver.CaseService; -import com.powsybl.caseserver.elasticsearch.CaseInfosRepository; -import com.powsybl.caseserver.repository.CaseMetadataRepository; -import com.powsybl.commons.datasource.DataSource; -import jakarta.persistence.EntityManager; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.test.autoconfigure.web.servlet.WebMvcTest; -import org.springframework.boot.test.mock.mockito.MockBean; -import org.springframework.cloud.stream.binder.test.TestChannelBinderConfiguration; -import org.springframework.cloud.stream.function.StreamBridge; -import org.springframework.test.context.ContextConfiguration; -import org.springframework.test.context.ContextHierarchy; -import org.springframework.test.web.servlet.MockMvc; -import org.springframework.test.web.servlet.MvcResult; -import org.springframework.web.servlet.config.annotation.EnableWebMvc; - -import java.io.BufferedReader; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.nio.charset.StandardCharsets; -import java.nio.file.*; -import java.util.Set; -import java.util.UUID; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; -import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; - -/** - * @author Abdelsalem Hedhili - */ -@EnableWebMvc -@WebMvcTest(value = CaseDataSourceController.class, properties = {"case-store-directory=test"}) -@ContextHierarchy({@ContextConfiguration(classes = {CaseApplication.class, TestChannelBinderConfiguration.class})}) -class CaseDataSourceControllerTest { - - private FileSystem fileSystem = Jimfs.newFileSystem(); - - @MockBean - private StreamBridge streamBridge; - - @MockBean - private EntityManager entityManager; - - @Autowired - private MockMvc mvc; - - @MockBean - private CaseMetadataRepository caseMetadataRepository; - - @MockBean - private CaseInfosRepository caseInfosRepository; - - @Autowired - private CaseService caseService; - - @Value("${case-store-directory:#{systemProperties['user.home'].concat(\"/cases\")}}") - private String rootDirectory; - - private String cgmesName = "CGMES_v2415_MicroGridTestConfiguration_BC_BE_v2.zip"; - private String fileName = "CGMES_v2415_MicroGridTestConfiguration_BC_BE_v2/MicroGridTestConfiguration_BC_BE_DL_V2.xml"; - - private static final UUID CASE_UUID = UUID.randomUUID(); - - private DataSource dataSource; - - @Autowired - private ObjectMapper mapper; - - @BeforeEach - void setUp() throws Exception { - Path path = fileSystem.getPath(rootDirectory); - if (!Files.exists(path)) { - Files.createDirectories(path); - } - Path caseDirectory = fileSystem.getPath(rootDirectory).resolve(CASE_UUID.toString()); - if (!Files.exists(caseDirectory)) { - Files.createDirectories(caseDirectory); - } - - caseService.setFileSystem(fileSystem); - //insert a cgmes in the FS - try (InputStream cgmesURL = getClass().getResourceAsStream("/" + cgmesName)) { - Path cgmes = caseDirectory.resolve(cgmesName); - Files.copy(cgmesURL, cgmes, StandardCopyOption.REPLACE_EXISTING); - } - dataSource = DataSource.fromPath(Paths.get(getClass().getResource("/" + cgmesName).toURI())); - } - - @Test - void testBaseName() throws Exception { - MvcResult mvcResult = mvc.perform(get("/v1/cases/{caseUuid}/datasource/baseName", CASE_UUID)) - .andExpect(status().isOk()) - .andReturn(); - - assertEquals(dataSource.getBaseName(), mvcResult.getResponse().getContentAsString()); - } - - @Test - void testListName() throws Exception { - MvcResult mvcResult = mvc.perform(get("/v1/cases/{caseUuid}/datasource/list", CASE_UUID) - .param("regex", ".*")) - .andExpect(status().isOk()) - .andReturn(); - - Set nameList = mapper.readValue(mvcResult.getResponse().getContentAsString(), Set.class); - assertEquals(dataSource.listNames(".*"), nameList); - } - - @Test - void testInputStreamWithFileName() throws Exception { - MvcResult mvcResult = mvc.perform(get("/v1/cases/{caseUuid}/datasource", CASE_UUID) - .param("fileName", fileName)) - .andExpect(status().isOk()) - .andReturn(); - - try (InputStreamReader isReader = new InputStreamReader(dataSource.newInputStream(fileName), StandardCharsets.UTF_8)) { - BufferedReader reader = new BufferedReader(isReader); - StringBuilder datasourceResponse = new StringBuilder(); - String str; - while ((str = reader.readLine()) != null) { - datasourceResponse.append(str).append("\n"); - } - assertEquals(datasourceResponse.toString(), mvcResult.getResponse().getContentAsString()); - } - } - - @Test - void testInputStreamWithSuffixExt() throws Exception { - String suffix = "/MicroGridTestConfiguration_BC_BE_DL_V2"; - String ext = "xml"; - MvcResult mvcResult = mvc.perform(get("/v1/cases/{caseUuid}/datasource", CASE_UUID) - .param("suffix", suffix) - .param("ext", ext)) - .andExpect(status().isOk()) - .andReturn(); - - try (InputStreamReader isReader = new InputStreamReader(dataSource.newInputStream(suffix, ext), StandardCharsets.UTF_8)) { - BufferedReader reader = new BufferedReader(isReader); - StringBuilder datasourceResponse = new StringBuilder(); - String str; - while ((str = reader.readLine()) != null) { - datasourceResponse.append(str).append("\n"); - } - assertEquals(datasourceResponse.toString(), mvcResult.getResponse().getContentAsString()); - } - } - - @Test - void testExistsWithFileName() throws Exception { - MvcResult mvcResult = mvc.perform(get("/v1/cases/{caseUuid}/datasource/exists", CASE_UUID) - .param("fileName", fileName)) - .andExpect(status().isOk()) - .andReturn(); - - Boolean res = mapper.readValue(mvcResult.getResponse().getContentAsString(), Boolean.class); - assertEquals(dataSource.exists(fileName), res); - - mvcResult = mvc.perform(get("/v1/cases/{caseUuid}/datasource/exists", CASE_UUID) - .param("fileName", "random")) - .andExpect(status().isOk()) - .andReturn(); - - res = mapper.readValue(mvcResult.getResponse().getContentAsString(), Boolean.class); - assertEquals(dataSource.exists("random"), res); - } - - @Test - void testExistsWithSuffixExt() throws Exception { - String suffix = "random"; - String ext = "uct"; - MvcResult mvcResult = mvc.perform(get("/v1/cases/{caseUuid}/datasource/exists", CASE_UUID) - .param("suffix", suffix) - .param("ext", ext)) - .andExpect(status().isOk()) - .andReturn(); - - Boolean res = mapper.readValue(mvcResult.getResponse().getContentAsString(), Boolean.class); - assertEquals(dataSource.exists(suffix, ext), res); - } -} diff --git a/src/test/java/com/powsybl/caseserver/CaseControllerTest.java b/src/test/java/com/powsybl/caseserver/service/AbstractCaseControllerTest.java similarity index 94% rename from src/test/java/com/powsybl/caseserver/CaseControllerTest.java rename to src/test/java/com/powsybl/caseserver/service/AbstractCaseControllerTest.java index b84dc12..8a30014 100644 --- a/src/test/java/com/powsybl/caseserver/CaseControllerTest.java +++ b/src/test/java/com/powsybl/caseserver/service/AbstractCaseControllerTest.java @@ -4,23 +4,19 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -package com.powsybl.caseserver; +package com.powsybl.caseserver.service; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.io.ByteStreams; -import com.google.common.jimfs.Configuration; -import com.google.common.jimfs.Jimfs; +import com.powsybl.caseserver.ContextConfigurationWithTestChannel; import com.powsybl.caseserver.dto.CaseInfos; import com.powsybl.caseserver.parsers.entsoe.EntsoeFileNameParser; import com.powsybl.caseserver.repository.CaseMetadataEntity; import com.powsybl.caseserver.repository.CaseMetadataRepository; import com.powsybl.caseserver.utils.TestUtils; -import com.powsybl.computation.ComputationManager; import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.mockito.Mockito; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc; @@ -61,48 +57,37 @@ @AutoConfigureMockMvc @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.MOCK, properties = {"case-store-directory=/cases"}) @ContextConfigurationWithTestChannel -class CaseControllerTest { +abstract class AbstractCaseControllerTest { private static final String TEST_CASE = "testCase.xiidm"; private static final String TEST_CASE_FORMAT = "XIIDM"; private static final String NOT_A_NETWORK = "notANetwork.txt"; private static final String STILL_NOT_A_NETWORK = "stillNotANetwork.xiidm"; - private static final String GET_CASE_URL = "/v1/cases/{caseUuid}"; private static final String GET_CASE_FORMAT_URL = "/v1/cases/{caseName}/format"; private static final UUID RANDOM_UUID = UUID.fromString("3e2b6777-fea5-4e76-9b6b-b68f151373ab"); @Autowired - private MockMvc mvc; + protected MockMvc mvc; - @Autowired - private CaseService caseService; + CaseService caseService; @Autowired - private CaseMetadataRepository caseMetadataRepository; + CaseMetadataRepository caseMetadataRepository; @Autowired - private OutputDestination outputDestination; + OutputDestination outputDestination; @Autowired private ObjectMapper mapper; - @Value("${case-store-directory}") - private String rootDirectory; + @Value("${case-store-directory:#{systemProperties['user.home'].concat(\"/cases\")}}") + String rootDirectory; - private FileSystem fileSystem; + FileSystem fileSystem; private final String caseImportDestination = "case.import.destination"; - @BeforeEach - void setUp() { - fileSystem = Jimfs.newFileSystem(Configuration.unix()); - caseService.setFileSystem(fileSystem); - caseService.setComputationManager(Mockito.mock(ComputationManager.class)); - caseMetadataRepository.deleteAll(); - outputDestination.clear(); - } - @AfterEach public void tearDown() throws Exception { fileSystem.close(); @@ -118,18 +103,11 @@ private void createStorageDir() throws IOException { } private static MockMultipartFile createMockMultipartFile(String fileName) throws IOException { - try (InputStream inputStream = CaseControllerTest.class.getResourceAsStream("/" + fileName)) { + try (InputStream inputStream = AbstractCaseControllerTest.class.getResourceAsStream("/" + fileName)) { return new MockMultipartFile("file", fileName, MediaType.TEXT_PLAIN_VALUE, inputStream); } } - @Test - void testStorageNotCreated() throws Exception { - // expect a fail since the storage dir. is not created - mvc.perform(delete("/v1/cases")) - .andExpect(status().isUnprocessableEntity()); - } - @Test void testDeleteCases() throws Exception { // create the storage dir @@ -454,6 +432,10 @@ void test() throws Exception { .andReturn(); String response = mvcResult.getResponse().getContentAsString(); assertTrue(response.contains("\"format\":\"XIIDM\"")); + + // delete all cases + mvc.perform(delete("/v1/cases")) + .andExpect(status().isOk()); } @Test @@ -497,31 +479,6 @@ private UUID importCase(String testCase, Boolean withExpiration) throws Exceptio return UUID.fromString(importedCase.substring(1, importedCase.length() - 1)); } - @Test - void validateCaseNameTest() { - CaseService.validateCaseName("test.xiidm"); - CaseService.validateCaseName("test-case.7zip"); - CaseService.validateCaseName("testcase1.7zip"); - CaseService.validateCaseName("testcase1.xiidm.gz"); - CaseService.validateCaseName("test..xiidm"); - - try { - CaseService.validateCaseName("test"); - fail(); - } catch (CaseException ignored) { - } - try { - CaseService.validateCaseName("../test.xiidm"); - fail(); - } catch (CaseException ignored) { - } - try { - CaseService.validateCaseName("test/xiidm"); - fail(); - } catch (CaseException ignored) { - } - } - @Test void searchCaseTest() throws Exception { // create the storage dir @@ -641,7 +598,6 @@ void searchCaseTest() throws Exception { assertTrue(response.contains("\"name\":\"20200103_0915_SN5_D80.UCT\"")); assertTrue(response.contains("\"name\":\"20200103_0915_135_CH2.UCT\"")); - String t = getDateSearchTerm("20200103_0915"); mvcResult = mvc.perform(get("/v1/cases/search") .param("q", getDateSearchTerm("20200103_0915"))) .andExpect(status().isOk()) diff --git a/src/test/java/com/powsybl/caseserver/service/CaseServiceTest.java b/src/test/java/com/powsybl/caseserver/service/CaseServiceTest.java new file mode 100644 index 0000000..d38e31a --- /dev/null +++ b/src/test/java/com/powsybl/caseserver/service/CaseServiceTest.java @@ -0,0 +1,140 @@ +/** + * Copyright (c) 2023, RTE (http://www.rte-france.com) + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ +package com.powsybl.caseserver.service; + +import com.powsybl.caseserver.CaseException; +import com.powsybl.caseserver.dto.CaseInfos; +import com.powsybl.caseserver.dto.cgmes.CgmesCaseInfos; +import com.powsybl.caseserver.dto.entsoe.EntsoeCaseInfos; +import com.powsybl.caseserver.elasticsearch.DisableElasticsearch; +import com.powsybl.caseserver.parsers.cgmes.CgmesFileNameParser; +import com.powsybl.caseserver.parsers.entsoe.EntsoeFileNameParser; +import com.powsybl.entsoe.util.EntsoeGeographicalCode; +import com.powsybl.iidm.network.Country; +import org.junit.jupiter.api.Test; +import org.junit.runner.RunWith; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.junit4.SpringRunner; + +import java.nio.file.Path; +import java.util.UUID; + +import static org.junit.Assert.*; + +/** + * @author Ghazwa Rehili + */ +@RunWith(SpringRunner.class) +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.NONE) +@DisableElasticsearch +class CaseServiceTest { + + @Autowired + CaseService caseService; + + private static final String SN_UCTE_CASE_FILE_NAME = "20200103_0915_SN5_D80.UCT"; + private static final String ID_UCTE_CASE_FILE_NAME = "20200424_1330_135_CH2.UCT"; + private static final String D1_UCTE_CASE_FILE_NAME = "20200110_0430_FO5_FR0.uct"; + private static final String D2_UCTE_CASE_FILE_NAME = "20200430_1530_2D4_D41.uct"; + private static final String TEST_CGMES_CASE_FILE_NAME = "20200424T1330Z_2D_RTEFRANCE_001.zip"; + private static final String TEST_OTHER_CASE_FILE_NAME = "testCase.xiidm"; + + @Test + void testCreateDefaultCaseInfo() { + CaseInfos infos = caseService.createInfos(TEST_OTHER_CASE_FILE_NAME, UUID.randomUUID(), "UNKNOW"); + assertEquals(infos.getClass(), CaseInfos.class); + } + + @Test + void validateCaseNameTest() { + caseService.validateCaseName("test.xiidm"); + caseService.validateCaseName("test-case.7zip"); + caseService.validateCaseName("testcase1.7zip"); + caseService.validateCaseName("testcase1.xiidm.gz"); + caseService.validateCaseName("test..xiidm"); + CaseException exception = assertThrows(CaseException.class, () -> caseService.validateCaseName("test")); + assertEquals(CaseException.Type.ILLEGAL_FILE_NAME, exception.getType()); + CaseException exception1 = assertThrows(CaseException.class, () -> caseService.validateCaseName("../test.xiidm")); + assertEquals(CaseException.Type.ILLEGAL_FILE_NAME, exception1.getType()); + CaseException exception2 = assertThrows(CaseException.class, () -> caseService.validateCaseName("test/xiidm")); + assertEquals(CaseException.Type.ILLEGAL_FILE_NAME, exception2.getType()); + } + + @Test + void testValidNameUcteSN() { + EntsoeCaseInfos caseInfos = (EntsoeCaseInfos) createInfos(SN_UCTE_CASE_FILE_NAME, "UCTE"); + assertEquals(SN_UCTE_CASE_FILE_NAME, caseInfos.getName()); + assertEquals("UCTE", caseInfos.getFormat()); + assertTrue(caseInfos.getDate().isEqual(EntsoeFileNameParser.parseDateTime(SN_UCTE_CASE_FILE_NAME.substring(0, 13)))); + assertEquals(Integer.valueOf(0), caseInfos.getForecastDistance()); + assertSame(EntsoeGeographicalCode.D8, caseInfos.getGeographicalCode()); + assertSame(Country.DE, caseInfos.getCountry()); + assertEquals(Integer.valueOf(0), caseInfos.getVersion()); + } + + @Test + void testValidNameUcteID() { + EntsoeCaseInfos caseInfos = (EntsoeCaseInfos) createInfos(ID_UCTE_CASE_FILE_NAME, "UCTE"); + assertEquals(ID_UCTE_CASE_FILE_NAME, caseInfos.getName()); + assertEquals("UCTE", caseInfos.getFormat()); + assertTrue(caseInfos.getDate().isEqual(EntsoeFileNameParser.parseDateTime(ID_UCTE_CASE_FILE_NAME.substring(0, 13)))); + assertEquals(Integer.valueOf(780), caseInfos.getForecastDistance()); + assertSame(EntsoeGeographicalCode.CH, caseInfos.getGeographicalCode()); + assertSame(Country.CH, caseInfos.getCountry()); + assertEquals(Integer.valueOf(2), caseInfos.getVersion()); + } + + @Test + void testValidNameUcte1D() { + EntsoeCaseInfos caseInfos = (EntsoeCaseInfos) createInfos(D1_UCTE_CASE_FILE_NAME, "UCTE"); + assertEquals(D1_UCTE_CASE_FILE_NAME, caseInfos.getName()); + assertEquals("UCTE", caseInfos.getFormat()); + assertTrue(caseInfos.getDate().isEqual(EntsoeFileNameParser.parseDateTime(D1_UCTE_CASE_FILE_NAME.substring(0, 13)))); + assertEquals(Integer.valueOf(630), caseInfos.getForecastDistance()); + assertSame(EntsoeGeographicalCode.FR, caseInfos.getGeographicalCode()); + assertSame(Country.FR, caseInfos.getCountry()); + assertEquals(Integer.valueOf(0), caseInfos.getVersion()); + } + + @Test + void testValidNameUcte2D() { + EntsoeCaseInfos caseInfos = (EntsoeCaseInfos) createInfos(D2_UCTE_CASE_FILE_NAME, "UCTE"); + assertEquals(D2_UCTE_CASE_FILE_NAME, caseInfos.getName()); + assertEquals("UCTE", caseInfos.getFormat()); + assertTrue(caseInfos.getDate().isEqual(EntsoeFileNameParser.parseDateTime(D2_UCTE_CASE_FILE_NAME.substring(0, 13)))); + assertEquals(Integer.valueOf(2730), caseInfos.getForecastDistance()); + assertSame(EntsoeGeographicalCode.D4, caseInfos.getGeographicalCode()); + assertSame(Country.DE, caseInfos.getCountry()); + assertEquals(Integer.valueOf(1), caseInfos.getVersion()); + } + + @Test + void testValidNameCgmes() { + CgmesCaseInfos caseInfos = (CgmesCaseInfos) createInfos(TEST_CGMES_CASE_FILE_NAME, "CGMES"); + assertEquals(TEST_CGMES_CASE_FILE_NAME, caseInfos.getName()); + assertEquals("CGMES", caseInfos.getFormat()); + assertTrue(caseInfos.getDate().isEqual(CgmesFileNameParser.parseDateTime(TEST_CGMES_CASE_FILE_NAME.substring(0, 14)))); + assertEquals("2D", caseInfos.getBusinessProcess()); + assertEquals("RTEFRANCE", caseInfos.getTso()); + assertEquals(Integer.valueOf(1), caseInfos.getVersion()); + } + + public void testNonValidNameEntsoe() { + CaseInfos caseInfos = createInfos(TEST_OTHER_CASE_FILE_NAME, "XIIDM"); + assertEquals(TEST_OTHER_CASE_FILE_NAME, caseInfos.getName()); + assertEquals("XIIDM", caseInfos.getFormat()); + } + + private CaseInfos createInfos(String fileName, String format) { + UUID caseUuid = UUID.randomUUID(); + Path casePath = Path.of(this.getClass().getResource("/" + fileName).getPath()); + String fileBaseName = casePath.getFileName().toString(); + return caseService.createInfos(fileBaseName, caseUuid, format); + } +} diff --git a/src/test/java/com/powsybl/caseserver/service/FsCaseControllerTest.java b/src/test/java/com/powsybl/caseserver/service/FsCaseControllerTest.java new file mode 100644 index 0000000..fad709c --- /dev/null +++ b/src/test/java/com/powsybl/caseserver/service/FsCaseControllerTest.java @@ -0,0 +1,47 @@ +/** + * Copyright (c) 2023, RTE (http://www.rte-france.com) + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ +package com.powsybl.caseserver.service; + +import com.google.common.jimfs.Configuration; +import com.google.common.jimfs.Jimfs; + +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.delete; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +import com.powsybl.computation.ComputationManager; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.test.context.TestPropertySource; + +/** + * @author Ghazwa Rehili + */ +@TestPropertySource(properties = {"storage.type=FS"}) +class FsCaseControllerTest extends AbstractCaseControllerTest { + + @Autowired + private FsCaseService fsCaseService; + + @BeforeEach + void setUp() { + caseService = fsCaseService; + fileSystem = Jimfs.newFileSystem(Configuration.unix()); + ((FsCaseService) caseService).setFileSystem(fileSystem); + caseService.setComputationManager(Mockito.mock(ComputationManager.class)); + caseMetadataRepository.deleteAll(); + outputDestination.clear(); + } + + @Test + void testStorageNotCreated() throws Exception { + // expect a fail since the storage dir. is not created + mvc.perform(delete("/v1/cases")).andExpect(status().isUnprocessableEntity()); + } + +} diff --git a/src/test/java/com/powsybl/caseserver/service/MinioContainerConfig.java b/src/test/java/com/powsybl/caseserver/service/MinioContainerConfig.java new file mode 100644 index 0000000..50300d2 --- /dev/null +++ b/src/test/java/com/powsybl/caseserver/service/MinioContainerConfig.java @@ -0,0 +1,53 @@ +/** + * Copyright (c) 2023, RTE (http://www.rte-france.com) + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ +package com.powsybl.caseserver.service; + +import org.springframework.test.context.DynamicPropertyRegistry; +import org.springframework.test.context.DynamicPropertySource; +import org.testcontainers.Testcontainers; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.wait.strategy.HttpWaitStrategy; + +import java.time.Duration; + +/** + * @author Ghazwa Rehili + */ +public interface MinioContainerConfig { + String MINIO_DOCKER_IMAGE_NAME = "minio/minio"; + String BUCKET_NAME = "bucket-gridsuite"; + // Just a fixed version, latest at the time of writing this + String MINIO_DOCKER_IMAGE_VERSION = "RELEASE.2023-09-27T15-22-50Z"; + int MINIO_PORT = 9000; + GenericContainer MINIO_CONTAINER = createMinioContainer(); + static GenericContainer createMinioContainer() { + try { + GenericContainer minioContainer = new GenericContainer( + String.format("%s:%s", MINIO_DOCKER_IMAGE_NAME, MINIO_DOCKER_IMAGE_VERSION)) + .withCommand("server /data") + .withExposedPorts(MINIO_PORT) + .waitingFor(new HttpWaitStrategy() + .forPath("/minio/health/ready") + .forPort(MINIO_PORT) + .withStartupTimeout(Duration.ofSeconds(10))); + minioContainer.start(); + minioContainer.execInContainer("mkdir", "/data/" + BUCKET_NAME); + return minioContainer; + } catch (Exception e) { + throw new RuntimeException("Failed to start minioContainer", e); + } + } + + @DynamicPropertySource + static void registerAwsProperties(DynamicPropertyRegistry registry) { + Integer mappedPort = MINIO_CONTAINER.getFirstMappedPort(); + Testcontainers.exposeHostPorts(mappedPort); + String minioContainerUrl = String.format("http://172.17.0.1:%s", mappedPort); + + registry.add("spring.cloud.aws.endpoint", () -> minioContainerUrl); + } +} diff --git a/src/test/java/com/powsybl/caseserver/service/S3CaseControllerTest.java b/src/test/java/com/powsybl/caseserver/service/S3CaseControllerTest.java new file mode 100644 index 0000000..92a8910 --- /dev/null +++ b/src/test/java/com/powsybl/caseserver/service/S3CaseControllerTest.java @@ -0,0 +1,34 @@ +/** + * Copyright (c) 2023, RTE (http://www.rte-france.com) + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ +package com.powsybl.caseserver.service; + +import com.google.common.jimfs.Configuration; +import com.google.common.jimfs.Jimfs; +import com.powsybl.computation.ComputationManager; +import org.junit.jupiter.api.BeforeEach; +import org.mockito.Mockito; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.test.context.TestPropertySource; + +/** + * @author Ghazwa Rehili + */ +@TestPropertySource(properties = {"storage.type=S3"}) +class S3CaseControllerTest extends AbstractCaseControllerTest implements MinioContainerConfig { + + @Autowired + private S3CaseService s3CaseService; + + @BeforeEach + void setUp() { + caseService = s3CaseService; + fileSystem = Jimfs.newFileSystem(Configuration.unix()); + caseService.setComputationManager(Mockito.mock(ComputationManager.class)); + caseService.deleteAllCases(); + outputDestination.clear(); + } +} diff --git a/src/test/resources/LF.xml b/src/test/resources/LF.xml new file mode 100644 index 0000000..25572ff --- /dev/null +++ b/src/test/resources/LF.xml @@ -0,0 +1,42 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/test/resources/LF.xml.gz b/src/test/resources/LF.xml.gz new file mode 100644 index 0000000000000000000000000000000000000000..91d6ecb4823f51fcc6bcb8865422f5f87814d7f3 GIT binary patch literal 804 zcmV+<1Ka!`iwFo&LK0^H158FPcx`L|y;j>!+dvR~hOe;vS~=N^<4Z)4KnW^>0tMPv zw6@pA!uGoB4IzKu*|lS*w1KJ=@{o*YX3w0Rn{$xoNja^!S_%1JJ(We(Otore-z!y5 z$K%y%HLAs`Sx3c4$ZXui_q=3d?K8%A#J2KeW-Hz(+lpDk=S=aLO%M(oj2#r*Vlt(4 zO1u$4!6$^L2-#LDBnM|T&l|<67!H845lb2K6Cq2cI>f{M14FM_M&PJ&QJH9Wu^Wai zNjxWr5O!Ss6-{{Hq!1Bv&l2IH*!qG~BjC&8z-FqhXUOEs@>8V`A&TeBOI4IRCvCT<}2)t`ICSA@!?BKisZsNaVMR`QqgCVljsWga9^Z7myL3 zucwU>VtOOA74|P%u7F#$6hHtLQg2fDeFA8MoZ)V_MjIg`8_h=O?oaxiYoeYexPQxb zWBANYx*b#N-7(ZR2nX6|pdIX_ zP2WbFzLM_W?Ax@Rt$aH}!Uy7p_u>9t%R635$t%^q zxixwDmc~^gpCOpmO{og|~BC0=_39S^Fv zEa$ixrlrE3ZhBt&=qd@mhE0OrlAnbYy_Z=XIsBK|d3~hYgB3-AGc(FJO1J5G+NgI3 zl*RlN^{sfYM1oy|_l`hiVf~Llk*?G~bL>AIlBw6(kQ=(_jCNSf{F{1C${g!mbh#$3 zJ^zWTMo;f~_wL89k6yg_XqWpcmqY!k>HIPG%I--AZ^=h4m9d z*FE}uP4D2tgDkg?Y*Y!?S#Pl-D9q%}HkY%j|2ZyBoUElg@$P{q_ZtfNeaaXvZ)XZ- zwyD#dExCS2ikFA5dGqI}iEHzHy3f2hefzwXMaT!PPYYLvZ(6Q>x9r)%)jv$nPiuAC z$>Qdv*Ek``=yBGEocZl)8Qg46wwvdkd(PgmZkpK}g*W2gO3q4({GK@fdFnc;vrn}x z9!-@073X1C%G@7d^HE%cwM~9Tc|`l1FXHb_Z{NMG-ClXaRIzdXFOG>P9EJ5uW1bnd zv)UDIc)x3oas8v>Uzb`Y^v@9ZbgN}Y{#)}c%oU&N)t)SMs|)_Q;{3+b0k3>4N@cGX z*S0J6AK2$TdBP&kQ-_``{dBQkL%w!x&An<}i~oDYVm>%>uM^NQRQ$Ty!~V{?Y>y|~ zyMI1<@+ApqQwxJxhWG@*`RkYW3MN%4ZxQ{bAHA+2daH;7gIG|0(2rWCjOIBH^R~N5 zD=JH$b$jv7|JU5?Mf22l)SoU({&B`{mu&FsC#Bh8k0n;Jtrqs{3F@|x$vyi~f6vM4 z4{Fi5pVaSdd3x9Iia7`St&9b-ZsiDygxey-fH|G#j6 xHzSiAGp@`e0nB6!42(d$q!Gk|Wi3`n)