feat(DOSDownload): add digitalocean spaces download task (#15)

BREAKING CHANGE: This release change and improve a lot of things in multiple tasks (Delete & Upload).

* docs: update links to use repository instead of wiki
* chore: upgrade packages and node to v14
* fix(DOSDelete): refactor filter files
* refactor(DOSUpload): use common filter files
* feat: add task to download from space
* fix(DOSDownload): get stream errors on pipe and before
* test(DOSDownload): add error tests
* chore: reduce code duplication on tests
* test: refactor parameters tests
* docs(readme): add tips and tweaks links
* chore: remove unused package
* chore(DOSUpload): add matcher package
* chore: remove inner packages to allow webpack implementation
* chore: add webpack
* fix(DOSUpload): remove unused resource path
* fix(DOSDownload): source & target path, icon, getInput instead of path

.gitignore

@@ -263,3 +263,13 @@ Common/**/*.d.ts
 coverage/
 
 .taskkey
+
+# Related to azure tasks
+dist
+Tasks/**/dependencies/
+
+*.LICENSE.txt
+Tasks/**/LICENSE.txt
+
+Tasks/**/lib.json
+Tasks/**/resources.resjson/**

Tasks/DigitalOceanSpacesDelete/utils/filterFiles.ts → Common/Node/utils/filterFiles.ts

@@ -1,18 +1,27 @@
 import AWS from 'aws-sdk'
 import { isEmpty } from 'lodash'
+import * as path from 'path'
 import matcher from 'matcher'
-import tl from './tl'
 
 interface FilterFilesOnList {
   digitalGlobExpressions: string[]
   listedObjects: AWS.S3.ListObjectsV2Output
   digitalTargetFolder?: string
+  tlLoc: (key: string, ...param: any[]) => void
 }
 
 interface SearchFilesOnBucket {
   digitalBucket: string
   s3Connection: AWS.S3
   digitalTargetFolder?: string
+  tlLoc: (key: string, ...param: any[]) => void
+}
+
+interface NormalizePathParameters {
+  filePath: string
+  digitalSourceFolder?: string
+  digitalFlattenFolders: boolean
+  digitalTargetFolder?: string
 }
 
 /**
@@ -21,7 +30,9 @@ interface SearchFilesOnBucket {
 export const filterFilesOnList = (
   parameters: FilterFilesOnList
 ): AWS.S3.ObjectIdentifier[] => {
-  console.log(tl.loc('FilteringFiles', parameters.digitalGlobExpressions))
+  console.log(
+    parameters.tlLoc('FilteringFiles', parameters.digitalGlobExpressions)
+  )
 
   const result: AWS.S3.ObjectIdentifier[] = parameters.listedObjects.Contents.map(
     ({ Key }) => {
@@ -30,13 +41,13 @@ export const filterFilesOnList = (
   ).filter(({ Key }) => {
     // If doesn't match, matcher will return an empty array that isEmpty will get it
     const itMatch = !isEmpty(matcher([Key], parameters.digitalGlobExpressions))
-    if (itMatch) console.log(tl.loc('MatchedFile', Key))
+    if (itMatch) console.log(parameters.tlLoc('MatchedFile', Key))
     return itMatch
   })
 
   if (isEmpty(result))
     console.log(
-      tl.loc(
+      parameters.tlLoc(
         'FilesNotMatched',
         parameters.digitalTargetFolder ? parameters.digitalTargetFolder : 'root'
       )
@@ -52,7 +63,7 @@ export const searchFilesOnBucket = async (
   parameters: SearchFilesOnBucket
 ): Promise<AWS.S3.ListObjectsV2Output> => {
   console.log(
-    tl.loc(
+    parameters.tlLoc(
       'SearchingFiles',
       parameters.digitalTargetFolder ? parameters.digitalTargetFolder : 'root'
     )
@@ -79,3 +90,30 @@ export const searchFilesOnBucket = async (
 
   return listObjects
 }
+
+export const normalizeKeyPathDestination = (
+  parameters: NormalizePathParameters
+): string => {
+  let relativePath = parameters.filePath.substring(
+    parameters.digitalSourceFolder.length
+  )
+
+  if (relativePath.startsWith(path.sep)) {
+    relativePath = relativePath.substr(1)
+  }
+
+  let targetPath
+
+  if (parameters.digitalFlattenFolders) {
+    const flatFileName = path.basename(parameters.filePath)
+    targetPath = parameters.digitalTargetFolder
+      ? path.join(parameters.digitalTargetFolder, flatFileName)
+      : flatFileName
+  } else {
+    targetPath = parameters.digitalTargetFolder
+      ? path.join(parameters.digitalTargetFolder, relativePath)
+      : relativePath
+  }
+
+  return targetPath.replace(/\\/g, '/')
+}

Extension/OVERVIEW.md

@@ -31,13 +31,29 @@ Advanced Options:
 - **DigitalOcean Region:** The DigitalOcean datacenter region code containing the Spaces resource(s), e.g. nyc3, ams3, sgp1, etc.
 - **Bucket Name:** The name of the bucket where the content will be deleted.
 - **Filename Patterns:** Glob patterns to select the file and folder content to be deleted.
-- **Target Folder:** The target folder (referred to as a key prefix in Spaces) in the bucket that contain the files. Root is assumed when empty, **be careful**.
+- **Target Folder:** The target folder (referred to as a key prefix in Spaces) in the bucket that contains the files. Root is assumed when empty, **be careful**.
 
 Semantic Version Filter Options:
 
 - **Enable:** Allow to filter based on [semantic version](https://semver.org/), avoiding newest versions from being deleted of the bucket prefix.
 - **How many versions to keep:** Specify how many versions to keep in the bucket, it will avoid deleting the newest ones. *This option disables Filename patterns.*
 
+## DigitalOcean Spaces Download
+
+- **DigitalOcean Connection:** Set the service endpoint for your connection. It's based on AWS configuration (only Access Key ID and Secret Key ID is required).
+- **DigitalOcean Region:** The DigitalOcean datacenter region code containing the Spaces resource(s), e.g. nyc3, ams3, sgp1, etc.
+- **Bucket Name:** The name of the bucket where the content will be downloaded.
+- **Source Folder:** The source folder (referred to as a key prefix in Spaces) that the content pattern(s) will be run against. If not set the root of the work area is assumed.
+- **Filename Patterns:** Glob patterns to select the file and folder content to be downloaded.
+- **Target Folder:** The target folder on your build host to contain the downloaded content.
+
+Advanced Options:
+
+- **Flatten Folders:** Flatten the folder structure and download all files into the specified target folder.
+- **Overwrite:** Enable to replace existing files in target folder. An error is thrown if it tries to replace existing files with overwrite disabled.
+- **Concurrency limit:** Specify how many files to download simultaneously.
+- **Retry failed limit:** Specify how many times to retry a failed download.
+
 ## Install the extension to your account
 
 You can find the latest stable version of the Azure DevOps Extension tasks on the [Visual Studio Marketplace](https://marketplace.visualstudio.com/items?itemName=marcelo-formentao.digitalocean-tools).
@@ -53,3 +69,7 @@ The [source](https://github.com/marceloavf/digitalocean-tools-vsts) for this ext
 ## Release Notes
 
 Please refer to our [release page on Github](https://github.com/marceloavf/digitalocean-tools-vsts/releases)
+
+## Tips & Tweaks
+
+- [**Performance Tips for DigitalOcean Spaces**](https://www.digitalocean.com/docs/spaces/resources/performance-tips/)

Extension/vss-extension.json

@@ -22,6 +22,7 @@
     "publish",
     "upload",
     "delete",
+    "download",
     "s3",
     "spaces",
     "node",

README.md

@@ -20,13 +20,18 @@ DigitalOcean Tools
 
 ## 🗃   Tasks included
 
-- 🔹   [**DigitalOcean Spaces Upload:**](https://github.com/marceloavf/digitalocean-tools-vsts/wiki/DigitalOcean-Spaces-Upload) Upload file(s) and folder(s) content to DigitalOcean Spaces Bucket.
-- 🔹   [**DigitalOcean Spaces Delete:**](https://github.com/marceloavf/digitalocean-tools-vsts/wiki/DigitalOcean-Spaces-Delete) Delete file(s) from DigitalOcean Spaces Bucket.
+- 🔹   [**DigitalOcean Spaces Upload:**](https://github.com/marceloavf/digitalocean-tools-vsts/blob/master/Extension/OVERVIEW.md#digitalocean-spaces-upload) Upload file(s) and folder(s) content to DigitalOcean Spaces Bucket.
+- 🔹   [**DigitalOcean Spaces Delete:**](https://github.com/marceloavf/digitalocean-tools-vsts/blob/master/Extension/OVERVIEW.md#digitalocean-spaces-delete) Delete file(s) from DigitalOcean Spaces Bucket.
+- 🔹   [**DigitalOcean Spaces Download:**](https://github.com/marceloavf/digitalocean-tools-vsts/blob/master/Extension/OVERVIEW.md#digitalocean-spaces-download) Download file(s) from DigitalOcean Spaces Bucket.
 
 ## 📟   Install the extension to your account
 
 You can find the latest stable version of the Azure DevOps Extension tasks on the [Visual Studio Marketplace](https://marketplace.visualstudio.com/items?itemName=marcelo-formentao.digitalocean-tools).
 
+## 🔮   Tips & Tweaks
+
+- [**Performance Tips for DigitalOcean Spaces**](https://www.digitalocean.com/docs/spaces/resources/performance-tips/)
+
 ## 🤝   Contribute
 
 Contributions, issues and feature requests are very welcome. Please make sure to read the [Contributing Guide](/CONTRIBUTING.md) before making a pull request.
@@ -35,6 +40,6 @@ Contributions, issues and feature requests are very welcome. Please make sure to
 
 Please refer to our [wiki page](https://github.com/marceloavf/digitalocean-tools-vsts/wiki/Known-Issues)
 
+## 📑   License
 
-## License
-[![FOSSA Status](https://app.fossa.com/api/projects/git%2Bgithub.com%2Fmarceloavf%2Fdigitalocean-tools-vsts.svg?type=large)](https://app.fossa.com/projects/git%2Bgithub.com%2Fmarceloavf%2Fdigitalocean-tools-vsts?ref=badge_large)
+[![FOSSA Status](https://app.fossa.com/api/projects/git%2Bgithub.com%2Fmarceloavf%2Fdigitalocean-tools-vsts.svg?type=large)](https://app.fossa.com/projects/git%2Bgithub.com%2Fmarceloavf%2Fdigitalocean-tools-vsts?ref=badge_large)

Tasks/DigitalOceanSpacesDelete/digitalOceanSpacesDelete.ts

@@ -1,4 +1,4 @@
-import tl from './utils/tl'
+import tl from './tl'
 import { Delete } from './utils/Delete'
 import { Parameters } from './utils/Parameters'
 

Tasks/DigitalOceanSpacesDelete/task.json

@@ -61,8 +61,7 @@
       "label": "Filename Patterns",
       "defaultValue": "**",
       "required": true,
-      "visibleRule": "digitalEnableSemver = false",
-      "helpMarkDown": "Glob patterns to select the file and folder content to be deleted. <br>Supports multiple lines of minimatch patterns and also exclude patterns."
+      "helpMarkDown": "Glob patterns to select the file and folder content to be deleted. <br>Supports multiple lines of minimatch patterns and also exclude patterns. <strong>NOTE:</strong> Filename Patterns has no effect when Semantic Version Filter is enabled."
     },
     {
       "name": "digitalTargetFolder",
@@ -94,7 +93,7 @@
   "dataSourceBindings": [
   ],
   "execution": {
-    "Node": {
+    "Node10": {
       "target": "index.js",
       "argumentFormat": ""
     }

Tasks/DigitalOceanSpacesUpload/utils/tl.ts → Tasks/DigitalOceanSpacesDelete/tl.ts

@@ -4,6 +4,6 @@ import * as path from 'path'
 /**
  * Set language loc file
  */
-tl.setResourcePath(path.join(__dirname, '..', 'task.json'))
+tl.setResourcePath(path.join(__dirname, './task.json'))
 
 export default tl

Tasks/DigitalOceanSpacesDelete/utils/Delete.ts

@@ -1,42 +1,41 @@
 import AWS from 'aws-sdk'
-import tl from './tl'
-import { Spaces } from '../common/Spaces'
+import tl from '../tl'
+import { Spaces } from '@Common/Spaces'
+import {
+  filterFilesOnList,
+  searchFilesOnBucket,
+} from '@Common/utils/filterFiles'
 import { Parameters } from './Parameters'
 import { getDeletableSemanticVersion } from './filterSemanticVersion'
-import { filterFilesOnList, searchFilesOnBucket } from './filterFiles'
 
 export class Delete extends Spaces<Parameters> {
   constructor(params: Parameters) {
     super(params)
   }
 
   public async init(): Promise<void> {
-    console.log(
-      tl.loc(
-        'DeletingFiles',
-        this.params.digitalTargetFolder
-          ? this.params.digitalTargetFolder
-          : 'root',
-        this.params.digitalBucket
+    try {
+      const baseTargetFolderMessage = this.params.digitalTargetFolder
+        ? this.params.digitalTargetFolder
+        : 'root'
+
+      console.log(
+        tl.loc(
+          'DeletingFiles',
+          baseTargetFolderMessage,
+          this.params.digitalBucket
+        )
       )
-    )
 
-    try {
       const listedObjects = await searchFilesOnBucket({
         digitalBucket: this.params.digitalBucket,
         s3Connection: this.s3Connection,
         digitalTargetFolder: this.params.digitalTargetFolder,
+        tlLoc: tl.loc,
       })
 
       if (listedObjects.Contents.length === 0) {
-        console.log(
-          tl.loc(
-            'FilesNotFound',
-            this.params.digitalTargetFolder
-              ? this.params.digitalTargetFolder
-              : 'root'
-          )
-        )
+        console.log(tl.loc('FilesNotFound', baseTargetFolderMessage))
         return
       }
 
@@ -52,6 +51,7 @@ export class Delete extends Spaces<Parameters> {
           digitalGlobExpressions: this.params.digitalGlobExpressions,
           listedObjects,
           digitalTargetFolder: this.params.digitalTargetFolder,
+          tlLoc: tl.loc,
         })
       }
 
@@ -67,9 +67,7 @@ export class Delete extends Spaces<Parameters> {
       console.log(
         tl.loc(
           'DeletingFilesCompleted',
-          this.params.digitalTargetFolder
-            ? this.params.digitalTargetFolder
-            : 'root',
+          baseTargetFolderMessage,
           this.params.digitalBucket
         )
       )

Tasks/DigitalOceanSpacesDelete/utils/Parameters.ts

@@ -1,5 +1,5 @@
-import { ParametersBase } from '../common/ParametersBase'
-import tl from './tl'
+import { ParametersBase } from '@Common/ParametersBase'
+import tl from '../tl'
 
 export class Parameters extends ParametersBase {
   public digitalGlobExpressions: string[]

Tasks/DigitalOceanSpacesDelete/utils/filterSemanticVersion.ts

@@ -1,7 +1,7 @@
 import AWS from 'aws-sdk'
 import { isEmpty, sortedUniq, dropRight, includes } from 'lodash'
 import * as semver from 'semver'
-import tl from './tl'
+import tl from '../tl'
 
 enum Sort {
   aBiggerThanB = -1,