diff --git a/Tasks/DigitalOceanSpacesUpload/utils/Upload.ts b/Tasks/DigitalOceanSpacesUpload/utils/Upload.ts index 325a63e..f3962e9 100644 --- a/Tasks/DigitalOceanSpacesUpload/utils/Upload.ts +++ b/Tasks/DigitalOceanSpacesUpload/utils/Upload.ts @@ -6,8 +6,9 @@ import tl from './tl' import { Spaces } from '../common/Spaces' import { Parameters } from './Parameters' import { findFiles, getMimeTypes } from './utils' -const { default: PQueue } = require('p-queue') import prettyBytes = require('pretty-bytes') +const { default: PQueue } = require('p-queue') +const pRetry = require('p-retry') interface NormalizePathParameters { filePath: string @@ -50,6 +51,8 @@ export class Upload extends Spaces { const uploadQueue = new PQueue({ concurrency: 4 }) + const errors: Error[] = [] + for (const filePath of files) { const targetPath = this.normalizeKeyPath({ ...this.params, filePath }) @@ -62,23 +65,36 @@ export class Upload extends Spaces { uploadQueue.add(async () => { try { - await this.uploadFile({ - filePath, - targetPath, - digitalAcl: this.params.digitalAcl, - digitalBucket: this.params.digitalBucket, - contentType, - }) + await pRetry( + async () => + this.uploadFile({ + filePath, + targetPath, + digitalAcl: this.params.digitalAcl, + digitalBucket: this.params.digitalBucket, + contentType, + }), + { + onFailedAttempt: (error: any) => { + console.log( + `Failed uploading ${filePath}: Attempt ${error.attemptNumber} failed. There are ${error.retriesLeft} retries left.` + ) + }, + retries: 2, + } + ) console.log(tl.loc('FileUploadCompleted', filePath, targetPath)) } catch (error) { + errors.push(error.message) console.error(tl.loc('FileUploadFailed'), error) - throw error } }) } await uploadQueue.onIdle() + if (!isEmpty(errors)) throw new Error(errors.toString()) + console.log(tl.loc('TaskCompleted')) } diff --git a/Tests/DigitalOceanSpacesUpload/utils/Upload.spec.ts b/Tests/DigitalOceanSpacesUpload/utils/Upload.spec.ts index 8ac362c..117ea7f 100644 --- a/Tests/DigitalOceanSpacesUpload/utils/Upload.spec.ts +++ b/Tests/DigitalOceanSpacesUpload/utils/Upload.spec.ts @@ -119,21 +119,24 @@ describe('DOSUpload', () => { }) test('should throw an error when upload fails', async () => { - AWS.spyOn('S3', 'upload').mockReturnValue({ - promise: () => Promise.reject('returned error'), + expect.assertions(3) + const uploadFn = AWS.spyOn('S3', 'upload').mockReturnValue({ + promise: () => Promise.reject(new Error('returned error')), on: () => true, }) - const upload = new Upload(baseParameters) + const upload = new Upload({ + ...baseParameters, + digitalGlobExpressions: ['*.txt'], + }) try { await upload.init() } catch (e) { - expect(e).toMatch('error') - expect(spyError.mock.calls[0]).toEqual([ - 'File upload failed', - 'returned error', - ]) + expect(uploadFn).toHaveBeenCalledTimes(6) + expect(e).toMatchSnapshot() + console.log(spyError.mock.calls) + expect(spyError.mock.calls).toMatchSnapshot() } }) diff --git a/Tests/DigitalOceanSpacesUpload/utils/__snapshots__/Upload.spec.ts.snap b/Tests/DigitalOceanSpacesUpload/utils/__snapshots__/Upload.spec.ts.snap index 661d365..8cec899 100644 --- a/Tests/DigitalOceanSpacesUpload/utils/__snapshots__/Upload.spec.ts.snap +++ b/Tests/DigitalOceanSpacesUpload/utils/__snapshots__/Upload.spec.ts.snap @@ -1,5 +1,20 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP +exports[`DOSUpload should throw an error when upload fails 1`] = `[Error: returned error,returned error]`; + +exports[`DOSUpload should throw an error when upload fails 2`] = ` +Array [ + Array [ + "File upload failed", + [Error: returned error], + ], + Array [ + "File upload failed", + [Error: returned error], + ], +] +`; + exports[`DOSUpload should upload file successfully 1`] = ` Array [ Array [