psp-7928 make document upload block - to reduce number of failed uplo…

…ads.
bcgov · May 2, 2024 · 96a7243 · 96a7243
1 parent 40d8a14
commit 96a7243
Show file tree

Hide file tree

Showing 5 changed files with 33 additions and 4 deletions.
diff --git a/source/backend/api/Models/Configuration/MayanConfig.cs b/source/backend/api/Models/Configuration/MayanConfig.cs
@@ -9,5 +9,7 @@ public class MayanConfig
         public string ConnectionUser { get; set; }
 
         public string ConnectionPassword { get; set; }
+
+        public int UploadRetries { get; set; }
     }
 }
diff --git a/source/backend/api/Services/DocumentService.cs b/source/backend/api/Services/DocumentService.cs
@@ -1,17 +1,20 @@
+using System;
 using System.Collections.Generic;
+using System.Configuration;
 using System.IO;
 using System.Linq;
 using System.Net;
 using System.Security.Claims;
 using System.Threading.Tasks;
 using MapsterMapper;
 using Microsoft.AspNetCore.Http;
+using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.Logging;
 
 using Pims.Api.Helpers.Exceptions;
 using Pims.Api.Models.CodeTypes;
 using Pims.Api.Models.Concepts.Document;
-
+using Pims.Api.Models.Config;
 using Pims.Api.Models.Mayan;
 using Pims.Api.Models.Mayan.Document;
 using Pims.Api.Models.Requests.Document.UpdateMetadata;
@@ -23,6 +26,7 @@
 using Pims.Dal.Helpers.Extensions;
 using Pims.Dal.Repositories;
 using Pims.Dal.Security;
+using Polly;
 
 namespace Pims.Api.Services
 {
@@ -31,6 +35,7 @@ namespace Pims.Api.Services
     /// </summary>
     public class DocumentService : BaseService, IDocumentService
     {
+
         private static readonly string[] ValidExtensions =
         {
                 "txt",
@@ -55,6 +60,9 @@ public class DocumentService : BaseService, IDocumentService
                 "msg",
         };
 
+        private static readonly string MayanConfigSectionKey = "Mayan";
+        private readonly MayanConfig _config;
+
         private readonly IDocumentRepository documentRepository;
         private readonly IEdmsDocumentRepository documentStorageRepository;
         private readonly IDocumentTypeRepository documentTypeRepository;
@@ -63,6 +71,7 @@ public class DocumentService : BaseService, IDocumentService
 
         public DocumentService(
             ClaimsPrincipal user,
+            IConfiguration configuration,
             ILogger<DocumentService> logger,
             IDocumentRepository documentRepository,
             IEdmsDocumentRepository documentStorageRepository,
@@ -76,6 +85,8 @@ public DocumentService(
             this.documentTypeRepository = documentTypeRepository;
             this.avService = avService;
             this.mapper = mapper;
+            _config = new MayanConfig();
+            configuration.Bind(MayanConfigSectionKey, _config);
         }
 
         public IList<PimsDocumentTyp> GetPimsDocumentTypes()
@@ -133,7 +144,20 @@ public async Task<DocumentUploadResponse> UploadDocumentAsync(DocumentUploadRequ
             if (externalResponse.Status == ExternalResponseStatus.Success)
             {
                 var externalDocument = externalResponse.Payload;
+                if (externalDocument.FileLatest == null && _config.UploadRetries > 0)
+                {
+                    var retryPolicy = Policy<ExternalResponse<DocumentDetailModel>>
+                        .HandleResult(result => result.HttpStatusCode != HttpStatusCode.OK || result.Payload.FileLatest == null)
+                        .WaitAndRetryAsync(_config.UploadRetries, (int retry) => TimeSpan.FromSeconds(Math.Pow(2, retry)));
+                    var detail = await retryPolicy.ExecuteAsync(async () => await GetStorageDocumentDetail(externalDocument.Id));
+                    if(detail?.Payload?.FileLatest == null)
+                    {
+                        response.DocumentExternalResponse.Status = ExternalResponseStatus.Error;
+                        response.DocumentExternalResponse.Message = "Timed out waiting for Mayan to process document";
+                        return response;
+                    }
 
+                }
                 // Create metadata of document
                 if (uploadRequest.DocumentMetadata != null)
                 {

diff --git a/source/backend/api/appsettings.json b/source/backend/api/appsettings.json
@@ -98,7 +98,8 @@
   "Mayan": {
     "BaseUri": "[MAYAN_BASE_URI]",
     "ConnectionUser": "[MAYAN_USER]",
-    "ConnectionPassword": "[MAYAN_USER_PASSWORD]"
+    "ConnectionPassword": "[MAYAN_USER_PASSWORD]",
+    "UploadRetries": 4
   },
   "Cdogs": {
     "AuthEndpoint": "[AUTH_ENDPOINT]",

diff --git a/source/backend/tests/unit/api/Services/DocumentServiceTest.cs b/source/backend/tests/unit/api/Services/DocumentServiceTest.cs
@@ -28,6 +28,7 @@
 using Pims.Api.Models.Requests.Document.Upload;
 using Pims.Api.Models.Requests.Document.UpdateMetadata;
 using Pims.Api.Constants;
+using Microsoft.Extensions.Configuration;
 
 namespace Pims.Api.Test.Services
 {
@@ -59,7 +60,8 @@ private DocumentService CreateDocumentServiceWithPermissions(params Permissions[
         {
             var user = PrincipalHelper.CreateForPermission(permissions);
             this._helper.CreatePimsContext(user, true);
-            return this._helper.Create<DocumentService>();
+            var builder = new ConfigurationBuilder();
+            return this._helper.Create<DocumentService>(builder.Build());
         }
 
         [Fact]

diff --git a/tools/mayan-edms/docker-compose.yml b/tools/mayan-edms/docker-compose.yml
@@ -19,7 +19,7 @@ x-mayan-container: &mayan-container
     # - /opt/watch_folder:/watch_folder
 
 services:
-  app:
+  mayan-app:
     <<: *mayan-container
     profiles:
       - all_in_one