feat(DOSUpload): add advanced options to concurrency and retry limit

Extension/OVERVIEW.md

@@ -20,6 +20,11 @@ DigitalOcean Tools provide the ability to upload and delete objects from Digital
 - **Access Control (ACL):** The canned Access Control List (ACL) to apply to the uploaded content.
 - **Flatten Folders:** Flatten the folder structure and copy all files into the specified target folder in the bucket.
 
+Advanced Options:
+
+- **Concurrency limit:** Specify how many files to upload simultaneously.
+- **Retry failed limit:** Specify how many times to retry a failed upload.
+
 ## DigitalOcean Spaces Delete
 
 - **DigitalOcean Connection:** Set the service endpoint for your connection. It's based on AWS configuration (only Access Key ID and Secret Key ID is required).
@@ -31,7 +36,7 @@ DigitalOcean Tools provide the ability to upload and delete objects from Digital
 Semantic Version Filter Options:
 
 - **Enable:** Allow to filter based on [semantic version](https://semver.org/), avoiding newest versions from being deleted of the bucket prefix.
-- **How many versions to keep:** Specify how many versions to keep in the bucket, it will avoid deleting the newest ones.
+- **How many versions to keep:** Specify how many versions to keep in the bucket, it will avoid deleting the newest ones. *This option disables Filename patterns.*
 
 ## Install the extension to your account
 

Tasks/DigitalOceanSpacesDelete/task.json

@@ -78,7 +78,7 @@
       "type": "boolean",
       "label": "Enable",
       "defaultValue": "false",
-      "helpMarkDown": "Allow to filter based on semantic version, avoiding some newest versions from being deleted of the bucket prefix. Filename Patterns must match these files to be filtered. <br>This option disable Filename patterns."
+      "helpMarkDown": "Allow to filter based on semantic version, avoiding some newest versions from being deleted of the bucket prefix. Filename Patterns must match these files to be filtered. <br>This option disables Filename patterns."
     },
     {
       "groupName": "semver",

Tasks/DigitalOceanSpacesUpload/task.json

@@ -22,6 +22,13 @@
   ],
   "demands": [
   ],
+  "groups": [
+    {
+      "name": "advanced",
+      "displayName": "Advanced",
+      "isExpanded": false
+    }
+  ],
   "minimumAgentVersion": "1.91.0",
   "inputs": [
     {
@@ -104,6 +111,24 @@
       "defaultValue": "",
       "required": false,
       "helpMarkDown": "Sets a custom content type for the uploaded files. If a custom content type is not specified the task will apply built-in defaults for common file types (html, css, js, image files etc). This parameter can be used to override the built-in defaults.\n\n__Note:__ that any value is applied to __all__ files processed by the task."
+    },
+    {
+      "groupName": "advanced",
+      "name": "digitalQueueConcurrency",
+      "type": "string",
+      "label": "Concurrency limit",
+      "defaultValue": "4",
+      "required": true,
+      "helpMarkDown": "Specify how many files to upload simultaneously."
+    },
+    {
+      "groupName": "advanced",
+      "name": "digitalRetryFailed",
+      "type": "string",
+      "label": "Retry failed limit",
+      "defaultValue": "2",
+      "required": true,
+      "helpMarkDown": "Specify how many times to retry a failed upload."
     }
   ],
   "dataSourceBindings": [

Tasks/DigitalOceanSpacesUpload/utils/Parameters.ts

@@ -7,6 +7,8 @@ export class Parameters extends ParametersBase {
   public digitalAcl: string
   public digitalFlattenFolders: boolean
   public digitalContentType?: string
+  public digitalQueueConcurrency: string
+  public digitalRetryFailed: string
 
   constructor() {
     super()
@@ -20,6 +22,8 @@ export class Parameters extends ParametersBase {
       this.digitalSourceFolder = tl.getPathInput('digitalSourceFolder')
       this.digitalFlattenFolders = tl.getBoolInput('digitalFlattenFolders')
       this.digitalContentType = tl.getInput('digitalContentType')
+      this.digitalQueueConcurrency = tl.getInput('digitalQueueConcurrency')
+      this.digitalRetryFailed = tl.getInput('digitalRetryFailed')
     } catch (error) {
       throw new Error(error.message)
     }

Tasks/DigitalOceanSpacesUpload/utils/Upload.ts

@@ -49,7 +49,9 @@ export class Upload extends Spaces<Parameters> {
       return
     }
 
-    const uploadQueue = new PQueue({ concurrency: 4 })
+    const uploadQueue = new PQueue({
+      concurrency: parseInt(this.params.digitalQueueConcurrency, 10),
+    })
 
     const errors: Error[] = []
 
@@ -80,7 +82,7 @@ export class Upload extends Spaces<Parameters> {
                   `Failed uploading ${filePath}: Attempt ${error.attemptNumber} failed. There are ${error.retriesLeft} retries left.`
                 )
               },
-              retries: 2,
+              retries: parseInt(this.params.digitalRetryFailed, 10),
             }
           )
           console.log(tl.loc('FileUploadCompleted', filePath, targetPath))

Tests/DigitalOceanSpacesUpload/utils/Upload.spec.ts

@@ -22,6 +22,8 @@ describe('DOSUpload', () => {
     digitalRegion: 'test',
     digitalBucket: 'test',
     digitalCredentials: 'test',
+    digitalQueueConcurrency: '4',
+    digitalRetryFailed: '2',
   }
 
   afterEach(() => {
@@ -79,8 +81,11 @@ describe('DOSUpload', () => {
       },
       digitalFlattenFolders: false,
       digitalGlobExpressions: ['**'],
+      digitalQueueConcurrency: '4',
       digitalRegion: 'test',
+      digitalRetryFailed: '2',
       digitalSourceFolder: 'Tests/fixtures/',
+      filePath: 'Tests/fixtures/file1-v1.2.1.txt',
     }
 
     expect(normalizePaths).toHaveBeenNthCalledWith(1, {