fix: rework upload implementation, add tests and new configurations

.vscode/settings.json

@@ -1,3 +1,4 @@
 {
-    "editor.formatOnSave": true
-}
+    "editor.formatOnSave": true,
+    "jest.autoEnable": false,
+}

Common/Node/ParametersBase.ts

@@ -5,7 +5,7 @@ export class ParametersBase {
   public digitalRegion: string
   public digitalBucket: string
   public digitalTargetFolder?: string
-  private digitalCredentials: string
+  public digitalCredentials: string
 
   constructor() {
     try {

Common/Node/Spaces.ts

@@ -1,16 +1,16 @@
-import { Endpoint, S3 } from 'aws-sdk'
-import { ParametersBase } from './ParametersBase';
+import AWS from 'aws-sdk'
+import { ParametersBase } from './ParametersBase'
 
 export class Spaces<T> {
-  public endpoint: Endpoint
-  public s3Connection: S3
+  public endpoint: AWS.Endpoint
+  public s3Connection: AWS.S3
 
-  constructor(protected params: ParametersBase & T){
-    this.endpoint = new Endpoint(
+  constructor(protected params: ParametersBase & T) {
+    this.endpoint = new AWS.Endpoint(
       `${this.params.digitalRegion.toLowerCase()}.digitaloceanspaces.com`
     )
 
-    this.s3Connection = new S3({
+    this.s3Connection = new AWS.S3({
       endpoint: this.endpoint.host,
       accessKeyId: this.params.digitalEndpoint.parameters.username,
       secretAccessKey: this.params.digitalEndpoint.parameters.password,

Tasks/DigitalOceanSpacesDelete/utils/Delete.ts

@@ -1,4 +1,4 @@
-import { S3 } from 'aws-sdk'
+import AWS from 'aws-sdk'
 import { isEmpty, sortedUniq, dropRight, includes } from 'lodash'
 import * as matcher from 'matcher'
 import * as semver from 'semver'
@@ -36,7 +36,7 @@ export class Delete extends Spaces<Parameters> {
       if (this.params.digitalEnableSemver)
         filtedObjects = this.filterSemanticVersion(listedObjects)
 
-      const deleteParams: S3.DeleteObjectsRequest = {
+      const deleteParams: AWS.S3.DeleteObjectsRequest = {
         Bucket: this.params.digitalBucket,
         Delete: {
           Objects: filtedObjects,
@@ -69,11 +69,11 @@ export class Delete extends Spaces<Parameters> {
    * Get all files that match the glob pattern filter and return
    */
   private filterFiles(
-    listedObjects: S3.ListObjectsV2Output
-  ): S3.ObjectIdentifier[] {
+    listedObjects: AWS.S3.ListObjectsV2Output
+  ): AWS.S3.ObjectIdentifier[] {
     console.log(tl.loc('FilteringFiles', this.params.digitalGlobExpressions))
 
-    const result: S3.ObjectIdentifier[] = listedObjects.Contents.map(
+    const result: AWS.S3.ObjectIdentifier[] = listedObjects.Contents.map(
       ({ Key }) => {
         return { Key }
       }
@@ -102,7 +102,7 @@ export class Delete extends Spaces<Parameters> {
   /**
    * Get all files in the target folder and return
    */
-  private async searchFiles(): Promise<S3.ListObjectsV2Output> {
+  private async searchFiles(): Promise<AWS.S3.ListObjectsV2Output> {
     console.log(
       tl.loc(
         'SearchingFiles',
@@ -112,7 +112,7 @@ export class Delete extends Spaces<Parameters> {
       )
     )
 
-    const parameters: S3.ListObjectsV2Request = {
+    const parameters: AWS.S3.ListObjectsV2Request = {
       Bucket: this.params.digitalBucket,
       Prefix: this.params.digitalTargetFolder,
     }
@@ -133,8 +133,8 @@ export class Delete extends Spaces<Parameters> {
    * Making sure that only 'v1.0.0.exe' was deleted from the bucket
    */
   private filterSemanticVersion(
-    listedObjects: S3.ListObjectsV2Output
-  ): S3.ObjectIdentifier[] {
+    listedObjects: AWS.S3.ListObjectsV2Output
+  ): AWS.S3.ObjectIdentifier[] {
     console.log(tl.loc('SemverActive'))
 
     // Get version from Key and insert in a ordened list
@@ -172,7 +172,7 @@ export class Delete extends Spaces<Parameters> {
     }
 
     // Compare to the list, if not present, remove it from listedObjects to prevent from being deleted
-    const filteredListObjects: S3.ObjectIdentifier[] = listedObjects.Contents.map(
+    const filteredListObjects: AWS.S3.ObjectIdentifier[] = listedObjects.Contents.map(
       ({ Key }) => {
         return { Key }
       }

Tasks/DigitalOceanSpacesUpload/utils/Upload.ts

@@ -1,4 +1,4 @@
-import { S3 } from 'aws-sdk'
+import AWS from 'aws-sdk'
 import * as fs from 'fs'
 import { isEmpty } from 'lodash'
 import * as path from 'path'
@@ -234,28 +234,16 @@ export class Upload extends Spaces<Parameters> {
 
         console.log(tl.loc('UploadingFile', file, targetPath, contentType))
 
-        const params: S3.PutObjectRequest = {
+        const params: AWS.S3.PutObjectRequest = {
           Bucket: this.params.digitalBucket,
           ACL: this.params.digitalAcl,
           Key: targetPath,
           Body: fs.createReadStream(file),
           ContentType: contentType,
         }
 
-        const request: S3.ManagedUpload = this.s3Connection.upload(params)
+        await this.uploadFiles(params)
 
-        request.on('httpUploadProgress', (progress) => {
-          console.log(
-            tl.loc(
-              'FileUploadProgress',
-              prettyBytes(progress.loaded),
-              prettyBytes(progress.total),
-              Math.floor((progress.loaded / progress.total) * 100).toFixed(1)
-            )
-          )
-        })
-
-        const response: S3.ManagedUpload.SendData = await request.promise()
         console.log(tl.loc('FileUploadCompleted', file, targetPath))
       } catch (err) {
         console.error(tl.loc('FileUploadFailed'), err)
@@ -266,7 +254,28 @@ export class Upload extends Spaces<Parameters> {
     console.log(tl.loc('TaskCompleted'))
   }
 
-  private normalizeKeyPath(file: string): string {
+  async uploadFiles(
+    objectRequest: AWS.S3.PutObjectRequest
+  ): Promise<AWS.S3.ManagedUpload.SendData> {
+    const request: AWS.S3.ManagedUpload = this.s3Connection.upload(
+      objectRequest
+    )
+
+    request.on('httpUploadProgress', (progress) => {
+      console.log(
+        tl.loc(
+          'FileUploadProgress',
+          prettyBytes(progress.loaded),
+          prettyBytes(progress.total),
+          Math.floor((progress.loaded / progress.total) * 100).toFixed(1)
+        )
+      )
+    })
+
+    return request.promise()
+  }
+
+  normalizeKeyPath(file: string): string {
     let relativePath = file.substring(this.params.digitalSourceFolder.length)
 
     if (relativePath.startsWith(path.sep)) {

Tests/DigitalOceanSpacesUpload/utils/Upload.spec.ts

@@ -0,0 +1,51 @@
+import { Upload } from '@DOSUpload/utils/Upload.ts'
+// tslint:disable-next-line: no-var-requires
+const AWS = require('aws-sdk')
+
+describe('DOSUpload utils', () => {
+  afterEach(() => AWS.clearAllMocks())
+  it('Upload init', async () => {
+    const uploadFiles: jest.Mock = AWS.spyOn('S3', 'upload').mockReturnValue({
+      promise: () => Promise.resolve(),
+      on: () => Promise.resolve(),
+    })
+
+    const test = new Upload({
+      digitalSourceFolder: './Tests/fixtures/',
+      digitalGlobExpressions: ['**'],
+      digitalAcl: 'test',
+      digitalFlattenFolders: false,
+      digitalEndpoint: {
+        parameters: { username: 'test', password: 'test' },
+        scheme: 'test',
+      },
+      digitalRegion: 'test',
+      digitalBucket: 'test',
+      digitalCredentials: 'test',
+    })
+
+    const normalizePaths = jest.spyOn(test, 'normalizeKeyPath')
+
+    await test.init()
+
+    expect(uploadFiles.mock.calls).toMatchSnapshot()
+    expect(normalizePaths).toHaveBeenCalledTimes(3)
+    expect(normalizePaths).toHaveBeenNthCalledWith(
+      1,
+      'Tests/fixtures/file-v1.0.1.txt'
+    )
+    expect(normalizePaths).toHaveBeenNthCalledWith(
+      2,
+      'Tests/fixtures/file1-v1.2.1.txt'
+    )
+    expect(normalizePaths).toHaveBeenNthCalledWith(
+      3,
+      'Tests/fixtures/file2-v1.3.1.json'
+    )
+    expect(normalizePaths.mock.results).toEqual([
+      { type: 'return', value: 'file-v1.0.1.txt' },
+      { type: 'return', value: 'file1-v1.2.1.txt' },
+      { type: 'return', value: 'file2-v1.3.1.json' },
+    ])
+  })
+})