Add fixes file uploading with batch

whalemare · May 2, 2021 · b2f6984 · b2f6984
1 parent 8b8aeba
commit b2f6984
Show file tree

Hide file tree

Showing 5 changed files with 27 additions and 77 deletions.
diff --git a/__tests__/DropboxUploader.test.ts b/__tests__/DropboxUploader.test.ts
@@ -1,11 +1,6 @@
-import * as fsRaw from 'fs'
-
-const fs = fsRaw.promises
-
 import globby from 'globby'
 
 import { DropboxUploader } from '../src/upload/dropbox/DropboxUploader'
-import { uploadBatch } from '../src/upload/uploadBatch'
 
 /**
  * You can get new one at
@@ -81,37 +76,8 @@ describe('upload', () => {
     },
     60 * 1000,
   )
-
-  test(
-    'should upload batch files',
-    async () => {
-      const uploader = DropboxUploader.create({
-        accessToken: ACCESS_TOKEN,
-        logger: console,
-      })
-
-      await uploadBatch('src/**/*', async (file: string) => {
-        await uploader.uploadStream({
-          file: file,
-          destination: `/${file}`,
-          partSizeBytes: 500,
-          onProgress: (uploaded, total) => {
-            if (uploaded === 0) {
-              console.log(`onStart ${file}: ${uploaded}/${total}`)
-            } else if (uploaded === total) {
-              console.log(`onFinish ${file}: ${uploaded}/${total}`)
-            }
-          },
-        })
-      })
-
-      expect(true).toBeTruthy()
-    },
-    60 * 1000,
-  )
-
   test(
-    'upload files',
+    'upload batch files',
     async () => {
       const uploader = DropboxUploader.create({
         accessToken: ACCESS_TOKEN,

diff --git a/src/main.ts b/src/main.ts
@@ -1,9 +1,9 @@
 import { join } from 'path'
 
 import * as core from '@actions/core'
+import globby from 'globby'
 
 import { DropboxUploader } from './upload/dropbox/DropboxUploader'
-import { uploadBatch } from './upload/uploadBatch'
 import { getInputs } from './utils/getInputs'
 
 const { accessToken, file, destination, pattern, displayProgress = false, partSizeBytes = 1024 } = getInputs({
@@ -29,33 +29,29 @@ async function run() {
 
   if (pattern) {
     await core.group(`uploading batch ${pattern}`, async () => {
-      return uploadBatch(pattern, async (file) => {
-        const fileId = await dropbox.uploadStream({
-          file,
-          partSizeBytes: partSizeBytes,
-          destination: join(destination, file),
-          onProgress: displayProgress
-            ? (current, total) => {
-                const percent = Math.round((current / total) * 100)
-                core.info(`Uploading ${percent}%: ${file}`)
-              }
-            : undefined,
-        })
-        core.info(`Uploaded: ${file} -> ${fileId}`)
-        uploadedFiles.push(fileId)
+      const files = await globby(pattern)
+      await dropbox.uploadFiles(files, destination, {
+        onProgress: (current, total, file) => {
+          const percent = Math.round((current / total) * 100)
+          if (displayProgress) {
+            core.info(`Uploading ${percent}%: ${file}`)
+          } else if (percent === 100) {
+            core.info(`Uploaded: ${file}`)
+            files.push(file)
+          }
+        },
+        partSizeBytes: partSizeBytes,
       })
     })
   }
 
   if (file) {
-    const fileId = await dropbox.upload({
+    await dropbox.upload({
       file: file,
       destination: join(destination, file),
     })
-    uploadedFiles.push(fileId)
+    uploadedFiles.push(file)
   }
-
-  return uploadedFiles
 }
 
 run()

diff --git a/src/upload/dropbox/DropboxUploader.ts b/src/upload/dropbox/DropboxUploader.ts
@@ -114,7 +114,7 @@ export class DropboxUploader implements Uploader {
           close: false,
         })
       }
-      onProgress?.(uploaded, size)
+      onProgress?.(uploaded, size, file)
       uploaded += chunk.length
     }
     if (uploaded > 0) {
@@ -125,10 +125,10 @@ export class DropboxUploader implements Uploader {
         commit: { path: destination, mode: { '.tag': 'overwrite' } },
         contents: '',
       })
-      onProgress?.(uploaded, size)
+      onProgress?.(uploaded, size, file)
       return response.result.id
     } else {
-      onProgress?.(uploaded, size)
+      onProgress?.(uploaded, size, file)
       this.logger?.warn(`Skip ${file}, because it has empty content`)
       return ''
     }
@@ -140,7 +140,11 @@ export class DropboxUploader implements Uploader {
    * @param files
    * @param destination
    */
-  uploadFiles = async (files: string[], destination: string) => {
+  uploadFiles = async (
+    files: string[],
+    destination: string,
+    { onProgress, partSizeBytes = DROPBOX_MAX_BLOB_SIZE }: StreamUploader = {},
+  ) => {
     const sessions: {
       [k in string]: {
         sessionId: string
@@ -151,7 +155,7 @@ export class DropboxUploader implements Uploader {
       const fileStat = fsRaw.statSync(file)
       const fileSize = fileStat.size
       if (fileSize > 0) {
-        const fileStream = fsRaw.createReadStream(file, { highWaterMark: 1024 })
+        const fileStream = fsRaw.createReadStream(file, { highWaterMark: partSizeBytes })
 
         let sessionId: string | undefined = undefined
         let uploaded = 0
@@ -184,6 +188,7 @@ export class DropboxUploader implements Uploader {
             })
           }
           uploaded += chunk.length
+          onProgress?.(uploaded, fileSize, file)
         }
       }
     })

diff --git a/src/upload/dropbox/types/StreamUploader.ts b/src/upload/dropbox/types/StreamUploader.ts
@@ -1,5 +1,5 @@
 export interface StreamUploader {
-  onProgress?: (uploaded: number, total: number) => void
+  onProgress?: (uploaded: number, total: number, file: string) => void
 
   /**
    * Size of small blob part

diff --git a/src/upload/uploadBatch.ts b/src/upload/uploadBatch.ts