feat(DOSUpload): add retry failed upload

marceloavf · Oct 20, 2020 · f441ac0 · f441ac0
1 parent 964516c
commit f441ac0
Show file tree

Hide file tree

Showing 3 changed files with 51 additions and 17 deletions.
diff --git a/Tasks/DigitalOceanSpacesUpload/utils/Upload.ts b/Tasks/DigitalOceanSpacesUpload/utils/Upload.ts
@@ -6,8 +6,9 @@ import tl from './tl'
 import { Spaces } from '../common/Spaces'
 import { Parameters } from './Parameters'
 import { findFiles, getMimeTypes } from './utils'
-const { default: PQueue } = require('p-queue')
 import prettyBytes = require('pretty-bytes')
+const { default: PQueue } = require('p-queue')
+const pRetry = require('p-retry')
 
 interface NormalizePathParameters {
   filePath: string
@@ -50,6 +51,8 @@ export class Upload extends Spaces<Parameters> {
 
     const uploadQueue = new PQueue({ concurrency: 4 })
 
+    const errors: Error[] = []
+
     for (const filePath of files) {
       const targetPath = this.normalizeKeyPath({ ...this.params, filePath })
 
@@ -62,23 +65,36 @@ export class Upload extends Spaces<Parameters> {
 
       uploadQueue.add(async () => {
         try {
-          await this.uploadFile({
-            filePath,
-            targetPath,
-            digitalAcl: this.params.digitalAcl,
-            digitalBucket: this.params.digitalBucket,
-            contentType,
-          })
+          await pRetry(
+            async () =>
+              this.uploadFile({
+                filePath,
+                targetPath,
+                digitalAcl: this.params.digitalAcl,
+                digitalBucket: this.params.digitalBucket,
+                contentType,
+              }),
+            {
+              onFailedAttempt: (error: any) => {
+                console.log(
+                  `Failed uploading ${filePath}: Attempt ${error.attemptNumber} failed. There are ${error.retriesLeft} retries left.`
+                )
+              },
+              retries: 2,
+            }
+          )
           console.log(tl.loc('FileUploadCompleted', filePath, targetPath))
         } catch (error) {
+          errors.push(error.message)
           console.error(tl.loc('FileUploadFailed'), error)
-          throw error
         }
       })
     }
 
     await uploadQueue.onIdle()
 
+    if (!isEmpty(errors)) throw new Error(errors.toString())
+
     console.log(tl.loc('TaskCompleted'))
   }
 

diff --git a/Tests/DigitalOceanSpacesUpload/utils/Upload.spec.ts b/Tests/DigitalOceanSpacesUpload/utils/Upload.spec.ts
@@ -119,21 +119,24 @@ describe('DOSUpload', () => {
   })
 
   test('should throw an error when upload fails', async () => {
-    AWS.spyOn('S3', 'upload').mockReturnValue({
-      promise: () => Promise.reject('returned error'),
+    expect.assertions(3)
+    const uploadFn = AWS.spyOn('S3', 'upload').mockReturnValue({
+      promise: () => Promise.reject(new Error('returned error')),
       on: () => true,
     })
 
-    const upload = new Upload(baseParameters)
+    const upload = new Upload({
+      ...baseParameters,
+      digitalGlobExpressions: ['*.txt'],
+    })
 
     try {
       await upload.init()
     } catch (e) {
-      expect(e).toMatch('error')
-      expect(spyError.mock.calls[0]).toEqual([
-        'File upload failed',
-        'returned error',
-      ])
+      expect(uploadFn).toHaveBeenCalledTimes(6)
+      expect(e).toMatchSnapshot()
+      console.log(spyError.mock.calls)
+      expect(spyError.mock.calls).toMatchSnapshot()
     }
   })
 

diff --git a/Tests/DigitalOceanSpacesUpload/utils/__snapshots__/Upload.spec.ts.snap b/Tests/DigitalOceanSpacesUpload/utils/__snapshots__/Upload.spec.ts.snap
@@ -1,5 +1,20 @@
 // Jest Snapshot v1, https://goo.gl/fbAQLP
 
+exports[`DOSUpload should throw an error when upload fails 1`] = `[Error: returned error,returned error]`;
+
+exports[`DOSUpload should throw an error when upload fails 2`] = `
+Array [
+  Array [
+    "File upload failed",
+    [Error: returned error],
+  ],
+  Array [
+    "File upload failed",
+    [Error: returned error],
+  ],
+]
+`;
+
 exports[`DOSUpload should upload file successfully 1`] = `
 Array [
   Array [