fix(DOSDelete): remove isTruncated and use eachPage instead, add tests

Tasks/DigitalOceanSpacesDelete/utils/Delete.ts

@@ -29,7 +29,14 @@ export class Delete extends Spaces<Parameters> {
       })
 
       if (listedObjects.Contents.length === 0) {
-        console.log(tl.loc('FilesNotFound', this.params.digitalTargetFolder))
+        console.log(
+          tl.loc(
+            'FilesNotFound',
+            this.params.digitalTargetFolder
+              ? this.params.digitalTargetFolder
+              : 'root'
+          )
+        )
         return
       }
 
@@ -57,13 +64,6 @@ export class Delete extends Spaces<Parameters> {
 
       await this.s3Connection.deleteObjects(deleteParams).promise()
 
-      // isTruncated means more objects to list
-      // WARNING: this parameter can enter inifite loop if Semver always clean all
-      // versions from being deleted!
-      // FIX: Instead of executing all again, get the point where it can continue deleteing files in the next page
-      // FIX: Maybe fixable using .eachPage() on listObjectsV2, so it'll have all data
-      if (listedObjects.IsTruncated) await this.init()
-
       console.log(
         tl.loc(
           'DeletingFilesCompleted',

Tasks/DigitalOceanSpacesDelete/utils/filterFiles.ts

@@ -62,10 +62,20 @@ export const searchFilesOnBucket = async (
     Bucket: parameters.digitalBucket,
     Prefix: parameters.digitalTargetFolder,
   }
+  const listObjects = new Promise((resolve, reject) => {
+    const items: AWS.S3.ListObjectsV2Output = { Contents: [] }
 
-  const listObjects = await parameters.s3Connection
-    .listObjectsV2(options)
-    .promise()
+    parameters.s3Connection
+      .listObjectsV2(options)
+      // TODO: Investigate deeply why its not well documented on aws-sdk part, specifically in V2
+      // @ts-ignore
+      .eachPage((err, data, done) => {
+        if (err) return reject(err)
+        if (!data) return resolve(items)
+        items.Contents.push(...data.Contents)
+        done()
+      })
+  })
 
   return listObjects
 }

Tests/DigitalOceanSpacesDelete/utils/Delete.spec.ts

@@ -0,0 +1,121 @@
+import { Delete } from '@DOSDelete/utils/Delete.ts'
+const AWS = require('aws-sdk')
+
+const spyLog = jest.spyOn(console, 'log')
+const spyError = jest.spyOn(console, 'error')
+
+describe('DOSDelete', () => {
+  const baseParameters = {
+    digitalGlobExpressions: ['**'],
+    digitalEndpoint: {
+      parameters: { username: 'test', password: 'test' },
+      scheme: 'test',
+    },
+    digitalRegion: 'test',
+    digitalBucket: 'test',
+    digitalCredentials: 'test',
+  }
+
+  afterEach(() => {
+    spyLog.mockClear()
+    spyError.mockClear()
+    AWS.clearAllMocks()
+  })
+  test('should delete files successfully', async () => {
+    const listFiles = AWS.spyOnEachPage('S3', 'listObjectsV2', [
+      { Contents: [{ Key: 'test/path/1.txt' }] },
+      { Contents: [{ Key: 'text/path/2.txt' }] },
+    ])
+
+    const deleteObjectsResponse = AWS.spyOnPromise('S3', 'deleteObjects', true)
+
+    const deleteFiles = new Delete({
+      ...baseParameters,
+      digitalEnableSemver: false,
+      digitalSemverKeepOnly: 0,
+    })
+
+    await deleteFiles.init()
+
+    expect(listFiles.mock.calls).toEqual([
+      [{ Bucket: 'test', Prefix: undefined }],
+    ])
+    expect(deleteObjectsResponse).toHaveBeenCalledWith({
+      Bucket: 'test',
+      Delete: {
+        Objects: [{ Key: 'test/path/1.txt' }, { Key: 'text/path/2.txt' }],
+      },
+    })
+  })
+  test('should show message when files not found', async () => {
+    const listFiles = AWS.spyOnEachPage('S3', 'listObjectsV2', [])
+
+    const deleteFiles = new Delete({
+      ...baseParameters,
+      digitalEnableSemver: false,
+      digitalSemverKeepOnly: 0,
+    })
+
+    await deleteFiles.init()
+
+    expect(listFiles.mock.calls).toEqual([
+      [{ Bucket: 'test', Prefix: undefined }],
+    ])
+    expect(spyLog.mock.calls).toEqual([
+      ["Deleting files from 'root' in bucket test"],
+      ["Searching 'root' prefix for files to delete"],
+      ["No files found at 'root'"],
+    ])
+  })
+
+  test('should throw an error when delete fails', async () => {
+    const listFiles = AWS.spyOnEachPage('S3', 'listObjectsV2', [
+      { Contents: [{ Key: 'test/path/1.txt' }] },
+      new Error('Not able to find files'),
+    ])
+
+    const deleteFiles = new Delete({
+      ...baseParameters,
+      digitalEnableSemver: false,
+      digitalSemverKeepOnly: 0,
+    })
+
+    try {
+      await deleteFiles.init()
+    } catch (error) {
+      expect(error).toEqual(new Error('Not able to find files'))
+      expect(spyError.mock.calls[0]).toEqual([
+        'Deleting files failed',
+        new Error('Not able to find files'),
+      ])
+    }
+  })
+
+  test('should delete files successfully using semantic version', async () => {
+    const listFiles = AWS.spyOnEachPage('S3', 'listObjectsV2', [
+      { Contents: [{ Key: 'test/path/v1.0.0.txt' }] },
+      { Contents: [{ Key: 'text/path/v2.0.0.txt' }] },
+    ])
+
+    const deleteObjectsResponse = AWS.spyOnPromise('S3', 'deleteObjects', true)
+
+    const deleteFiles = new Delete({
+      ...baseParameters,
+      digitalTargetFolder: 'test/',
+      digitalEnableSemver: true,
+      digitalSemverKeepOnly: 1,
+    })
+
+    await deleteFiles.init()
+
+    expect(listFiles.mock.calls).toEqual([
+      [{ Bucket: 'test', Prefix: 'test/' }],
+    ])
+    expect(deleteObjectsResponse).toHaveBeenCalledWith({
+      Bucket: 'test',
+      Delete: {
+        Objects: [{ Key: 'test/path/v1.0.0.txt' }],
+      },
+    })
+  })
+})

Tests/DigitalOceanSpacesDelete/utils/filterFiles.spec.ts

@@ -86,36 +86,58 @@ describe('filterFilesOnList', () => {
 
 describe('searchFilesOnBucket', () => {
   test('should return list of objects from S3 repository', async () => {
-    const list = AWS.spyOn('S3', 'listObjectsV2').mockReturnValue({
-      promise: () => Promise.resolve(['test/path/1.txt', 'text/path/2.txt']),
-    })
-
-    const s3 = new AWS.S3({ region: 'testRegion' })
+    const list = AWS.spyOnEachPage('S3', 'listObjectsV2', [
+      { Contents: ['test/path/1.txt'] },
+      { Contents: ['text/path/2.txt'] },
+    ])
 
     const searchFiles = await searchFilesOnBucket({
       digitalTargetFolder: 'test',
-      s3Connection: s3,
+      s3Connection: new AWS.S3({ region: 'testRegion' }),
       digitalBucket: 'testBucket',
     })
 
     expect(list).toHaveBeenCalledWith({ Bucket: 'testBucket', Prefix: 'test' })
-    expect(searchFiles).toEqual(['test/path/1.txt', 'text/path/2.txt'])
+    expect(searchFiles).toEqual({
+      Contents: ['test/path/1.txt', 'text/path/2.txt'],
+    })
     expect(spyLog.mock.calls).toEqual([
       ["Searching 'test' prefix for files to delete"],
     ])
 
+    AWS.clearAllMocks()
+
+    AWS.spyOnEachPage('S3', 'listObjectsV2', [
+      { Contents: ['test/path/1.txt'] },
+      { Contents: ['text/path/2.txt'] },
+    ])
+
     const searchFilesWithoutTarget = await searchFilesOnBucket({
-      s3Connection: s3,
+      s3Connection: new AWS.S3({ region: 'testRegion' }),
       digitalBucket: 'testBucket',
     })
 
-    expect(searchFilesWithoutTarget).toEqual([
-      'test/path/1.txt',
-      'text/path/2.txt',
-    ])
+    expect(searchFilesWithoutTarget).toEqual({
+      Contents: ['test/path/1.txt', 'text/path/2.txt'],
+    })
     expect(spyLog.mock.calls).toEqual([
       ["Searching 'test' prefix for files to delete"],
       ["Searching 'root' prefix for files to delete"],
     ])
+
+    AWS.clearAllMocks()
+
+    AWS.spyOnEachPage('S3', 'listObjectsV2', [
+      { Contents: ['test/path/1.txt'] },
+      new Error('foos'),
+    ])
+    try {
+      await searchFilesOnBucket({
+        s3Connection: new AWS.S3({ region: 'testRegion' }),
+        digitalBucket: 'testBucket',
+      })
+    } catch (error) {
+      expect(error).toEqual(new Error('foos'))
+    }
   })
 })