Story#11963: migrate ingest application to api gateway mode, and fix …

…surrogation, and other bugs

api/api-gateway/src/main/resources/application-dev.yml

@@ -59,6 +59,7 @@ spring:
               /referential-api/profiles/**,
               /referential-api/externalparameters/**,
 
+
               /archive-search-api/ui/applications/**,
               /archive-search-api/externalparameters/**,
               /archive-search-api/security,
@@ -67,13 +68,15 @@ spring:
               /archive-search-api/users/analytics,
               /archive-search-api/logbooks/operations/**,
               /archive-search-api/accesscontracts/**,
+              /archive-search-api/subrogations/**,
 
               /pastis-api/ui/**,
               /pastis-api/security,
               /pastis-api/userinfos/**,
               /pastis-api/users/analytics,
               /pastis-api/logbooks/operations/**,
               /pastis-api/accesscontracts/**,
+              /pastis-api/subrogations/**,
 
               /collect-api/ui/applications/**,
               /collect-api/externalparameters/**,
@@ -82,7 +85,18 @@ spring:
               /collect-api/userinfos/**,
               /collect-api/users/analytics,
               /collect-api/logbooks/operations/**,
-              /collect-api/accesscontracts/**
+              /collect-api/accesscontracts/**,
+              /collect-api/subrogations/**,
+
+              /ingest-api/ui/applications/**,
+              /ingest-api/externalparameters/**,
+              /ingest-api/security,
+              /ingest-api/tenants/**,
+              /ingest-api/userinfos/**,
+              /ingest-api/users/analytics,
+              /ingest-api/logbooks/operations/**,
+              /ingest-api/accesscontracts/**,
+              /ingest-api/subrogations/**
 
           filters:
             # Portal IAM API
@@ -104,6 +118,7 @@ spring:
             # Archive Search IAM API
             - RewritePath=/archive-search-api/ui/(?<segment>.*),/iam/v1/$\{segment}
             - RewritePath=/archive-search-api/externalparameters(?<segment>.*),/iam/v1/externalparameters/me$\{segment},
+            - RewritePath=/archive-search-api/accesscontracts(?<segment>.*),/v1/accesscontracts$\{segment},
             - RewritePath=/archive-search-api/security(?<segment>.*),/iam/v1/security$\{segment},
             - RewritePath=/archive-search-api/userinfos(?<segment>.*),/iam/v1/userinfos$\{segment},
             - RewritePath=/archive-search-api/users(?<segment>.*),/iam/v1/users$\{segment},
@@ -124,6 +139,16 @@ spring:
             - RewritePath=/collect-api/users(?<segment>.*),/iam/v1/users$\{segment},
             - RewritePath=/collect-api/(?<segment>.*),/v1/$\{segment}
 
+
+            # Ingest IAM API
+            - RewritePath=/ingest-api/ui/(?<segment>.*),/iam/v1/$\{segment}
+            - RewritePath=/ingest-api/externalparameters(?<segment>.*),/iam/v1/externalparameters/me$\{segment},
+            - RewritePath=/ingest-api/logbooks/operations(?<segment>.*),/v1/logbooks/operations$\{segment},
+            - RewritePath=/ingest-api/security(?<segment>.*),/iam/v1/security$\{segment},
+            - RewritePath=/ingest-api/userinfos(?<segment>.*),/iam/v1/userinfos$\{segment},
+            - RewritePath=/ingest-api/users(?<segment>.*),/iam/v1/users$\{segment},
+            - RewritePath=/ingest-api/(?<segment>.*),/iam/v1/$\{segment}
+
         - id: referential-external-service
           uri: https://localhost:8087  # URL REFERENTIAL EXTERNAL
           predicates:
@@ -144,24 +169,27 @@ spring:
               /referential-api/profile/**,
               /referential-api/search/**,
               /referential-api/rules/**,
+              /referential-api/rule/**,
 
               /archive-search-api/security-profile/**,
               /archive-search-api/ontology/**,
               /archive-search-api/schemas,
 
-              /pastis-api/archival-profile,
-              /pastis-api/profile,
+              /pastis-api/archival-profile$,
+              /pastis-api/archival-profile/**,
+              /pastis-api/profile$,
+              /pastis-api/profile/**,
 
               /collect-api/ontology/**,
               /collect-api/search/**,
               /collect-api/schemas
 
           filters:
             - RewritePath=/referential-api/fileformat(?<segment>.*),/referential/v1/fileformats$\{segment}
-            - RewritePath=/referential-api/fileFormat(?<segment>.*),/referential/v1/fileformats$\{segment}
             - RewritePath=/referential-api/operation(?<segment>.*),/referential/v1/operations$\{segment}
             - RewritePath=/referential-api/search/filingplan(?<segment>.*),/units/filingplan$\{segment}
             - RewritePath=/referential-api/search/units(?<segment>.*),/units$\{segment}
+            - RewritePath=/referential-api/rule/(?<segment>.*),/referential/v1/rules/$\{segment}
             - RewritePath=/referential-api(?<segment>.*),/referential/v1$\{segment}
 
             # Archive Search Referential API
@@ -222,21 +250,15 @@ spring:
             - RewritePath=/collect-api/transactions/archive-units(?<segment>.*),/collect-api/v1/transactions$\{segment},
             - RewritePath=/collect-api/transactions(?<segment>.*),/collect-api/v1/transactions$\{segment},
 
-logging:
-  level:
-    reactor:
-      netty: debug
-    org:
-      springframework:
-        cloud:
-          gateway: debug
-
-management:
-  endpoints:
-    web:
-      exposure:
-        include: "*"
-  server:
-    port: 7070
-    ssl:
+        - id: ingest-external-service
+          uri: https://localhost:8088  # URL INGEST EXTERNAL
+          predicates:
+            - >
+              Path=
+              /ingest-api/ingest/**
+
+          filters:
+            # Ingest API
+            - RewritePath=/ingest-api/ingest/(?<segment>.*),/ingest/v1/$\{segment}
+    discovery:
       enabled: false

api/api-ingest/ingest-commons/src/main/java/fr/gouv/vitamui/ingest/common/rest/RestApi.java

@@ -27,9 +27,7 @@
 package fr.gouv.vitamui.ingest.common.rest;
 
 public class RestApi {
-    public static final String V1_INGEST = "/iam/v1/ingest";
+    public static final String V1_INGEST = "/v1/ingest";
     public static final String INGEST_REPORT_ODT = "/odtreport";
-    public static final String INGEST_UPLOAD_V2 = "/upload-v2";
-    public static final String INGEST_ATR = "/atr";
-    public static final String INGEST_MANIFEST = "/manifest";
+    public static final String INGEST_UPLOAD = "/upload";
 }

api/api-ingest/ingest-external-client/src/main/java/fr/gouv/vitamui/ingest/external/client/IngestExternalWebClient.java

@@ -36,72 +36,22 @@
  */
 package fr.gouv.vitamui.ingest.external.client;
 
-import fr.gouv.vitamui.commons.api.CommonConstants;
-import fr.gouv.vitamui.commons.api.exception.BadRequestException;
-import fr.gouv.vitamui.commons.api.exception.FileOperationException;
-import fr.gouv.vitamui.commons.api.logger.VitamUILogger;
-import fr.gouv.vitamui.commons.api.logger.VitamUILoggerFactory;
 import fr.gouv.vitamui.commons.rest.client.BaseWebClient;
 import fr.gouv.vitamui.commons.rest.client.ExternalHttpContext;
 import fr.gouv.vitamui.ingest.common.rest.RestApi;
-import org.springframework.http.HttpMethod;
-import org.springframework.util.LinkedMultiValueMap;
-import org.springframework.util.MultiValueMap;
-import org.springframework.web.reactive.function.client.ClientResponse;
 import org.springframework.web.reactive.function.client.WebClient;
 
-import java.io.IOException;
-import java.io.InputStream;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.nio.file.StandardCopyOption;
-import java.util.AbstractMap;
-import java.util.Optional;
-
 /**
  * Internal WebClient for Ingest operations.
  */
 public class IngestExternalWebClient extends BaseWebClient<ExternalHttpContext> {
 
-    private static final VitamUILogger LOGGER = VitamUILoggerFactory.getInstance(IngestExternalWebClient.class);
 
 
     public IngestExternalWebClient(final WebClient webClient, final String baseUrl) {
         super(webClient, baseUrl);
     }
 
-    public ClientResponse upload(final ExternalHttpContext context, InputStream in, String contextId, String action,
-        final String originalFilename) {
-        LOGGER.debug("[IngestExternalWebClient] upload file :  {}", originalFilename);
-        if (in == null) {
-            throw new FileOperationException("The uploaded file stream is null.");
-        }
-
-        final Path filePath =
-            Paths.get(System.getProperty(CommonConstants.VITAMUI_TEMP_DIRECTORY), context.getRequestId());
-        int length = 0;
-        try {
-            length = in.available();
-            Files.copy(in, filePath, StandardCopyOption.REPLACE_EXISTING);
-        } catch (IOException e) {
-            LOGGER
-                .debug("[IngestExternalWebClient] Error writing InputStream of length [{}] to temporary path {}",
-                    length,
-                    filePath.toAbsolutePath());
-            throw new BadRequestException("ERROR: InputStream writing error : ", e);
-        }
-
-        final MultiValueMap<String, String> headers = new LinkedMultiValueMap<>();
-        headers.add(CommonConstants.X_CONTEXT_ID, contextId);
-        headers.add(CommonConstants.X_ACTION, action);
-
-        return multipartDataFromFile(getPathUrl() + CommonConstants.INGEST_UPLOAD, HttpMethod.POST, context,
-            Optional.of(new AbstractMap.SimpleEntry<>(CommonConstants.MULTIPART_FILE_PARAM_NAME, filePath)),
-            headers);
-    }
-
-
     @Override
     public String getPathUrl() {
         return RestApi.V1_INGEST;

api/api-ingest/ingest-external-client/src/main/java/fr/gouv/vitamui/ingest/external/client/IngestStreamingExternalRestClient.java

@@ -64,7 +64,8 @@
 public class IngestStreamingExternalRestClient
     extends BasePaginatingAndSortingRestClient<LogbookOperationDto, ExternalHttpContext> {
 
-    private static final VitamUILogger LOGGER = VitamUILoggerFactory.getInstance(IngestStreamingExternalRestClient.class);
+    private static final VitamUILogger LOGGER =
+        VitamUILoggerFactory.getInstance(IngestStreamingExternalRestClient.class);
 
     public IngestStreamingExternalRestClient(final RestTemplate restTemplate, final String baseUrl) {
         super(restTemplate, baseUrl);
@@ -99,7 +100,7 @@ public ResponseEntity<Void> streamingUpload(final ExternalHttpContext context, S
         final String action) {
         LOGGER.debug("Calling upload using streaming process");
         final UriComponentsBuilder uriBuilder =
-            UriComponentsBuilder.fromHttpUrl(getUrl() + RestApi.INGEST_UPLOAD_V2);
+            UriComponentsBuilder.fromHttpUrl(getUrl() + RestApi.INGEST_UPLOAD);
 
         final MultiValueMap<String, String> headersList = new HttpHeaders();
         headersList.addAll(buildHeaders(context));

api/api-ingest/ingest-external/src/main/java/fr/gouv/vitamui/ingest/external/server/rest/IngestExternalController.java

@@ -96,14 +96,14 @@ public PaginatedValuesDto<LogbookOperationDto> getAllPaginated(@RequestParam fin
         @RequestParam(required = false) final Optional<String> orderBy,
         @RequestParam(required = false) final Optional<DirectionDto> direction)
         throws PreconditionFailedException, InvalidParseOperationException, IOException {
-        if(direction.isPresent()) {
+        if (direction.isPresent()) {
             SanityChecker.sanitizeCriteria(direction.get());
         }
-        if(orderBy.isPresent()) {
+        if (orderBy.isPresent()) {
             SanityChecker.checkSecureParameter(orderBy.get());
         }
         SanityChecker.sanitizeCriteria(criteria);
-        if(criteria.isPresent()) {
+        if (criteria.isPresent()) {
             SanityChecker.sanitizeCriteria(VitamUIUtils
                 .convertObjectFromJson(criteria.get(), Object.class));
         }
@@ -114,7 +114,8 @@ public PaginatedValuesDto<LogbookOperationDto> getAllPaginated(@RequestParam fin
 
     @Secured(ServicesData.ROLE_GET_INGEST)
     @GetMapping(CommonConstants.PATH_ID)
-    public LogbookOperationDto getOne(@PathVariable("id") final String id) throws PreconditionFailedException, InvalidParseOperationException {
+    public LogbookOperationDto getOne(@PathVariable("id") final String id)
+        throws PreconditionFailedException, InvalidParseOperationException {
         ParameterChecker.checkParameter("The Identifier is a mandatory parameter: ", id);
         SanityChecker.checkSecureParameter(id);
         LOGGER.debug("get One Ingest id={}", id);
@@ -133,7 +134,7 @@ public ResponseEntity<byte[]> generateODTReport(final @PathVariable("id") String
 
     @Secured(ServicesData.ROLE_CREATE_INGEST)
     @ApiOperation(value = "Upload an streaming SIP", consumes = MediaType.APPLICATION_OCTET_STREAM_VALUE)
-    @PostMapping(value = CommonConstants.INGEST_UPLOAD_V2, consumes = MediaType.APPLICATION_OCTET_STREAM_VALUE)
+    @PostMapping(value = CommonConstants.INGEST_UPLOAD, consumes = MediaType.APPLICATION_OCTET_STREAM_VALUE)
     public ResponseEntity<Void> streamingUpload(InputStream inputStream,
         @RequestHeader(value = CommonConstants.X_ACTION) final String action,
         @RequestHeader(value = CommonConstants.X_CONTEXT_ID) final String contextId,

api/api-ingest/ingest-external/src/main/resources/application-dev.yml

@@ -1,11 +1,11 @@
 debug: false
+
 spring:
   config:
     use-legacy-processing: true
   profiles:
     active: dev, swagger
 
-
   cloud:
     consul: #disabled for dev
       enabled: false
@@ -62,8 +62,23 @@ opentracing:
       host: localhost
       port: 6831
 
+
 logging:
   level:
     fr.gouv.vitamui: DEBUG
     fr.gouv.vitamui.iam.security: DEBUG
     #level.org.springframework.web.filter.CommonsRequestLoggingFilter: DEBUG
+
+---
+spring:
+  config:
+    activate:
+      on-profile: gateway
+
+gateway:
+  enabled: true
+
+server:
+  ssl:
+    client-auth: want
+    client-certificate-header-name: x-ssl-cert

api/api-ingest/ingest-external/src/main/resources/application.yml

@@ -10,7 +10,7 @@ spring:
     version: v1
   mvc:
     favicon:
-      enabled : false
+      enabled: false
   jackson:
     serialization:
       write-dates-as-timestamps: false
@@ -27,7 +27,12 @@ management:
     prometheus:
       enabled: true
 
+server:
+  tomcat:
+    max-swallow-size: 100GB
+    max-http-form-post-size: 100GB
+
 swagger:
- layer: api-external
- file-path: file:../../../tools/swagger/docs/api-external/ingest-external/swagger.json
+  layer: api-external
+  file-path: file:../../../tools/swagger/docs/api-external/ingest-external/swagger.json
 

api/api-ingest/ingest-internal-client/src/main/java/fr/gouv/vitamui/ingest/internal/client/IngestStreamingInternalRestClient.java

@@ -65,7 +65,8 @@
 public class IngestStreamingInternalRestClient
     extends BasePaginatingAndSortingRestClient<LogbookOperationDto, InternalHttpContext> {
 
-    private static final VitamUILogger LOGGER = VitamUILoggerFactory.getInstance(IngestStreamingInternalRestClient.class);
+    private static final VitamUILogger LOGGER =
+        VitamUILoggerFactory.getInstance(IngestStreamingInternalRestClient.class);
 
     public IngestStreamingInternalRestClient(final RestTemplate restTemplate, final String baseUrl) {
         super(restTemplate, baseUrl);
@@ -99,7 +100,7 @@ public ResponseEntity<Void> streamingUpload(final InternalHttpContext context, S
         final String action) {
         LOGGER.debug("Calling upload using streaming process");
         final UriComponentsBuilder uriBuilder =
-            UriComponentsBuilder.fromHttpUrl(getUrl() + RestApi.INGEST_UPLOAD_V2);
+            UriComponentsBuilder.fromHttpUrl(getUrl() + RestApi.INGEST_UPLOAD);
 
         final MultiValueMap<String, String> headersList = new HttpHeaders();
         headersList.addAll(buildHeaders(context));