S3 based case store (#11)

Signed-off-by: Etienne Homer <etiennehomer@gmail.com> Co-authored-by: Rehili Ghazwa <ghazwarhili@gmail.com> Co-authored-by: HARPER Jon <jon.harper87@gmail.com>
powsybl · Nov 12, 2024 · ed4c357 · ed4c357
1 parent 2c267b1
commit ed4c357
Show file tree

Hide file tree

Showing 42 changed files with 1,951 additions and 597 deletions.
diff --git a/pom.xml b/pom.xml
@@ -46,6 +46,7 @@
     <properties>
         <liquibase-hibernate-package>com.powsybl.caseserver.repository</liquibase-hibernate-package>
         <powsybl-ws-dependencies.version>2.15.0</powsybl-ws-dependencies.version>
+        <spring-cloud-aws-starter-s3>3.2.0</spring-cloud-aws-starter-s3>
     </properties>
 
     <build>
@@ -147,6 +148,11 @@
             <groupId>org.springframework.cloud</groupId>
             <artifactId>spring-cloud-stream-binder-rabbit</artifactId>
         </dependency>
+        <dependency>
+            <groupId>io.awspring.cloud</groupId>
+            <artifactId>spring-cloud-aws-starter-s3</artifactId>
+            <version>${spring-cloud-aws-starter-s3}</version>
+        </dependency>
         <dependency>
             <groupId>org.springframework.data</groupId>
             <artifactId>spring-data-elasticsearch</artifactId>

diff --git a/src/main/java/com/powsybl/caseserver/CaseController.java b/src/main/java/com/powsybl/caseserver/CaseController.java
@@ -7,6 +7,9 @@
 package com.powsybl.caseserver;
 
 import com.powsybl.caseserver.dto.CaseInfos;
+import com.powsybl.caseserver.elasticsearch.CaseInfosService;
+import com.powsybl.caseserver.service.CaseService;
+import com.powsybl.caseserver.service.MetadataService;
 import io.swagger.v3.oas.annotations.Operation;
 import io.swagger.v3.oas.annotations.media.Content;
 import io.swagger.v3.oas.annotations.media.Schema;
@@ -16,11 +19,9 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Qualifier;
 import org.springframework.context.annotation.ComponentScan;
-import org.springframework.http.ContentDisposition;
-import org.springframework.http.HttpHeaders;
-import org.springframework.http.MediaType;
-import org.springframework.http.ResponseEntity;
+import org.springframework.http.*;
 import org.springframework.web.bind.annotation.DeleteMapping;
 import org.springframework.web.bind.annotation.GetMapping;
 import org.springframework.web.bind.annotation.PathVariable;
@@ -33,7 +34,6 @@
 import org.springframework.web.multipart.MultipartFile;
 
 import java.io.IOException;
-import java.nio.file.Path;
 import java.util.List;
 import java.util.Map;
 import java.util.Properties;
@@ -54,14 +54,21 @@ public class CaseController {
     private static final Logger LOGGER = LoggerFactory.getLogger(CaseController.class);
 
     @Autowired
+    @Qualifier("storageService")
     private CaseService caseService;
 
+    @Autowired
+    private CaseInfosService caseInfosService;
+
+    @Autowired
+    private MetadataService metadataService;
+
     @GetMapping(value = "/cases")
     @Operation(summary = "Get all cases")
     //For maintenance purpose
     public ResponseEntity<List<CaseInfos>> getCases() {
         LOGGER.debug("getCases request received");
-        List<CaseInfos> cases = caseService.getCases(caseService.getStorageRootDir());
+        List<CaseInfos> cases = caseService.getCases();
         if (cases == null) {
             return ResponseEntity.noContent().build();
         }
@@ -72,30 +79,31 @@ public ResponseEntity<List<CaseInfos>> getCases() {
     @Operation(summary = "Get a case infos")
     public ResponseEntity<CaseInfos> getCaseInfos(@PathVariable("caseUuid") UUID caseUuid) {
         LOGGER.debug("getCaseInfos request received");
-        Path file = caseService.getCaseFile(caseUuid);
-        if (file == null) {
+        if (!caseService.caseExists(caseUuid)) {
             return ResponseEntity.noContent().build();
         }
-        CaseInfos caseInfos = caseService.getCase(file);
+        CaseInfos caseInfos = caseService.getCaseInfos(caseUuid);
         return ResponseEntity.ok().body(caseInfos);
     }
 
     @GetMapping(value = "/cases/{caseUuid}/format")
     @Operation(summary = "Get case Format")
     public ResponseEntity<String> getCaseFormat(@PathVariable("caseUuid") UUID caseUuid) {
         LOGGER.debug("getCaseFormat request received");
-        Path file = caseService.getCaseFile(caseUuid);
-        if (file == null) {
+        if (!caseService.caseExists(caseUuid)) {
             throw createDirectoryNotFound(caseUuid);
         }
-        String caseFormat = caseService.getFormat(file);
+        String caseFormat = caseService.getFormat(caseUuid);
         return ResponseEntity.ok().body(caseFormat);
     }
 
     @GetMapping(value = "/cases/{caseUuid}/name")
     @Operation(summary = "Get case name")
     public ResponseEntity<String> getCaseName(@PathVariable("caseUuid") UUID caseUuid) {
         LOGGER.debug("getCaseName request received");
+        if (!caseService.caseExists(caseUuid)) {
+            throw createDirectoryNotFound(caseUuid);
+        }
         String caseName = caseService.getCaseName(caseUuid);
         return ResponseEntity.ok().body(caseName);
     }
@@ -179,14 +187,17 @@ public ResponseEntity<UUID> duplicateCase(
         @ApiResponse(responseCode = "404", description = "Source case not found")})
     public ResponseEntity<Void> disableCaseExpiration(@PathVariable("caseUuid") UUID caseUuid) {
         LOGGER.debug("disableCaseExpiration request received for caseUuid = {}", caseUuid);
-        caseService.disableCaseExpiration(caseUuid);
+        metadataService.disableCaseExpiration(caseUuid);
         return ResponseEntity.ok().build();
     }
 
     @DeleteMapping(value = "/cases/{caseUuid}")
     @Operation(summary = "delete a case")
     public ResponseEntity<Void> deleteCase(@PathVariable("caseUuid") UUID caseUuid) {
         LOGGER.debug("deleteCase request received with parameter caseUuid = {}", caseUuid);
+        if (!caseService.caseExists(caseUuid)) {
+            throw createDirectoryNotFound(caseUuid);
+        }
         caseService.deleteCase(caseUuid);
         return ResponseEntity.ok().build();
     }
@@ -203,14 +214,14 @@ public ResponseEntity<Void> deleteCases() {
     @Operation(summary = "Search cases by metadata")
     public ResponseEntity<List<CaseInfos>> searchCases(@RequestParam(value = "q") String query) {
         LOGGER.debug("search cases request received");
-        List<CaseInfos> cases = caseService.searchCases(query);
+        List<CaseInfos> cases = caseInfosService.searchCaseInfos(query);
         return ResponseEntity.ok().body(cases);
     }
 
     @GetMapping(value = "/cases/metadata")
     @Operation(summary = "Get cases Metadata")
     public ResponseEntity<List<CaseInfos>> getMetadata(@RequestParam("ids") List<UUID> ids) {
-        LOGGER.debug("get Case metadata");
+        LOGGER.debug("get Cases metadata");
         return ResponseEntity.ok().body(caseService.getMetadata(ids));
     }
 }
diff --git a/src/main/java/com/powsybl/caseserver/CaseException.java b/src/main/java/com/powsybl/caseserver/CaseException.java
@@ -20,11 +20,17 @@ public final class CaseException extends RuntimeException {
 
     public enum Type {
         FILE_NOT_IMPORTABLE,
+        FILE_NOT_FOUND,
         STORAGE_DIR_NOT_CREATED,
         ILLEGAL_FILE_NAME,
         DIRECTORY_ALREADY_EXISTS,
         DIRECTORY_EMPTY,
         DIRECTORY_NOT_FOUND,
+        ORIGINAL_FILE_NOT_FOUND,
+        TEMP_FILE_INIT,
+        TEMP_FILE_PROCESS,
+        TEMP_DIRECTORY_CREATION,
+        ZIP_FILE_PROCESS,
         UNSUPPORTED_FORMAT
     }
 
@@ -35,7 +41,16 @@ private CaseException(Type type, String msg) {
         this.type = Objects.requireNonNull(type);
     }
 
-    public static CaseException createDirectoryAreadyExists(Path directory) {
+    public CaseException(Type type, String message, Throwable e) {
+        super(message, e);
+        this.type = type;
+    }
+
+    public Type getType() {
+        return type;
+    }
+
+    public static CaseException createDirectoryAreadyExists(String directory) {
         Objects.requireNonNull(directory);
         return new CaseException(Type.DIRECTORY_ALREADY_EXISTS, "A directory with the same name already exists: " + directory);
     }
@@ -50,11 +65,26 @@ public static CaseException createDirectoryNotFound(UUID uuid) {
         return new CaseException(Type.DIRECTORY_NOT_FOUND, "The directory with the following uuid doesn't exist: " + uuid);
     }
 
+    public static CaseException createOriginalFileNotFound(UUID uuid) {
+        Objects.requireNonNull(uuid);
+        return new CaseException(Type.ORIGINAL_FILE_NOT_FOUND, "The original file were not retrieved in the directory with the following uuid: " + uuid);
+    }
+
     public static CaseException createFileNotImportable(Path file) {
         Objects.requireNonNull(file);
         return new CaseException(Type.FILE_NOT_IMPORTABLE, "This file cannot be imported: " + file);
     }
 
+    public static CaseException createFileNotImportable(String file, Exception e) {
+        Objects.requireNonNull(file);
+        return new CaseException(Type.FILE_NOT_IMPORTABLE, "This file cannot be imported: " + file, e);
+    }
+
+    public static CaseException createFileNameNotFound(UUID uuid) {
+        Objects.requireNonNull(uuid);
+        return new CaseException(Type.FILE_NOT_FOUND, "The file name with the following uuid doesn't exist: " + uuid);
+    }
+
     public static CaseException createStorageNotInitialized(Path storageRootDir) {
         Objects.requireNonNull(storageRootDir);
         return new CaseException(Type.STORAGE_DIR_NOT_CREATED, "The storage is not initialized: " + storageRootDir);
@@ -65,6 +95,20 @@ public static CaseException createIllegalCaseName(String caseName) {
         return new CaseException(Type.ILLEGAL_FILE_NAME, "This is not an acceptable case name: " + caseName);
     }
 
+    public static CaseException createTempDirectory(UUID uuid, Exception e) {
+        Objects.requireNonNull(uuid);
+        return new CaseException(Type.TEMP_DIRECTORY_CREATION, "Error creating temporary directory: " + uuid, e);
+    }
+
+    public static CaseException createUInitTempFileError(UUID uuid, Throwable e) {
+        Objects.requireNonNull(uuid);
+        return new CaseException(Type.TEMP_FILE_INIT, "Error initializing temporary case file: " + uuid, e);
+    }
+
+    public static CaseException createCopyZipContentError(UUID uuid, Exception e) {
+        return new CaseException(Type.ZIP_FILE_PROCESS, "Error copying zip content file: " + uuid, e);
+    }
+
     public static CaseException createUnsupportedFormat(String format) {
         return new CaseException(Type.UNSUPPORTED_FORMAT, "The format: " + format + " is unsupported");
     }

diff --git a/src/main/java/com/powsybl/caseserver/ScheduledCaseCleaner.java b/src/main/java/com/powsybl/caseserver/ScheduledCaseCleaner.java
@@ -6,6 +6,7 @@
  */
 package com.powsybl.caseserver;
 
+import com.powsybl.caseserver.service.CaseService;
 import com.powsybl.caseserver.repository.CaseMetadataRepository;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;

diff --git a/src/main/java/com/powsybl/caseserver/SupervisionController.java b/src/main/java/com/powsybl/caseserver/SupervisionController.java
@@ -7,7 +7,8 @@
 package com.powsybl.caseserver;
 
 import com.powsybl.caseserver.elasticsearch.CaseInfosService;
-import com.powsybl.caseserver.services.SupervisionService;
+import com.powsybl.caseserver.service.CaseService;
+import com.powsybl.caseserver.service.SupervisionService;
 import io.swagger.v3.oas.annotations.Operation;
 import io.swagger.v3.oas.annotations.responses.ApiResponse;
 import io.swagger.v3.oas.annotations.responses.ApiResponses;

diff --git a/...source/util/CaseDataSourceController.java → .../datasource/CaseDataSourceController.java b/...source/util/CaseDataSourceController.java → .../datasource/CaseDataSourceController.java
@@ -1,15 +1,16 @@
 /**
- * Copyright (c) 2020, RTE (http://www.rte-france.com)
+ * Copyright (c) 2024, RTE (http://www.rte-france.com)
  * This Source Code Form is subject to the terms of the Mozilla Public
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/.
  */
-package com.powsybl.caseserver.datasource.util;
+package com.powsybl.caseserver.datasource;
 
 import com.powsybl.caseserver.CaseConstants;
 import io.swagger.v3.oas.annotations.Operation;
 import io.swagger.v3.oas.annotations.tags.Tag;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Qualifier;
 import org.springframework.context.annotation.ComponentScan;
 import org.springframework.http.MediaType;
 import org.springframework.http.ResponseEntity;
@@ -21,6 +22,8 @@
 
 /**
  * @author Abdelsalem Hedhili <abdelsalem.hedhili at rte-france.com>
+ * @author Ghazwa Rehili <ghazwa.rehili at rte-france.com>
+ * @author Etienne Homer <etienne.homer at rte-france.com>
  */
 @RestController
 @RequestMapping(value = "/" + CaseConstants.API_VERSION)
@@ -29,6 +32,7 @@
 public class CaseDataSourceController {
 
     @Autowired
+    @Qualifier("caseDataSourceService")
     private CaseDataSourceService caseDataSourceService;
 
     @GetMapping(value = "/cases/{caseUuid}/datasource/baseName")

diff --git a/src/main/java/com/powsybl/caseserver/datasource/CaseDataSourceService.java b/src/main/java/com/powsybl/caseserver/datasource/CaseDataSourceService.java
@@ -0,0 +1,28 @@
+/**
+ * Copyright (c) 2023, RTE (http://www.rte-france.com)
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ */
+package com.powsybl.caseserver.datasource;
+
+import java.util.Set;
+import java.util.UUID;
+
+/**
+ * @author Abdelsalem Hedhili <abdelsalem.hedhili at rte-france.com>
+ */
+public interface CaseDataSourceService {
+    String getBaseName(UUID caseUuid);
+
+    Boolean datasourceExists(UUID caseUuid, String suffix, String ext);
+
+    Boolean datasourceExists(UUID caseUuid, String fileName);
+
+    byte[] getInputStream(UUID caseUuid, String suffix, String ext);
+
+    byte[] getInputStream(UUID caseUuid, String fileName);
+
+    Set<String> listName(UUID caseUuid, String regex);
+
+}