Add waiting

whalemare · May 2, 2021 · 8fde0e0 · 8fde0e0
1 parent 10b9371
commit 8fde0e0
Show file tree

Hide file tree

Showing 4 changed files with 60 additions and 46 deletions.
diff --git a/__tests__/DropboxUploader.test.ts b/__tests__/DropboxUploader.test.ts
@@ -84,9 +84,9 @@ describe('upload', () => {
         logger: console,
       })
 
-      const files = await globby('src/**/*')
+      const files = await globby('node_modules/**/*')
       try {
-        await uploader.uploadFiles(files, '/test')
+        await uploader.uploadFiles(files.slice(0, 300), '/upload')
       } catch (e) {
         console.error(e.error)
       }

diff --git a/__tests__/shouldRetry.test.ts b/__tests__/shouldRetry.test.ts
@@ -1,22 +1,29 @@
-import { retry } from '../src/utils/retry'
+import { shouldRetry } from '../src/utils/retry'
 
 it('retry return data when success', async () => {
   let counter = 0
-  const result = await retry(async () => {
-    counter++
-    return 'data'
-  })
+  const result = await shouldRetry(
+    async () => {
+      counter++
+      return 'data'
+    },
+    async () => true,
+  )
   expect(result).toBe('data')
   expect(counter).toBe(1) // should be only one request, because first is success
 })
 
 it(`retry should fail when to much errors`, async () => {
   let counter = 0
   try {
-    await retry(async () => {
-      counter++
-      throw Error('error ' + counter)
-    }, 3)
+    await shouldRetry(
+      async () => {
+        counter++
+        throw Error('error ' + counter)
+      },
+      async () => true,
+      3,
+    )
   } catch (e) {
     // eslint-disable-next-line jest/no-conditional-expect
     expect(e.message).toBe('error 4')

diff --git a/src/upload/dropbox/DropboxUploader.ts b/src/upload/dropbox/DropboxUploader.ts
@@ -165,43 +165,32 @@ export class DropboxUploader implements Uploader {
         for await (const chunk of fileStream) {
           const isLastChunk = uploaded + chunk.length === fileSize
 
-          this.logger?.debug(`
-            File ${file}
-            filesize: ${fileSize}
-            sessionId: ${sessionId}
-            uploaded: ${uploaded}
-            chunk: ${chunk.length}
-            isLastChunk: ${isLastChunk}
-          `)
+          // this.logger?.debug(`
+          //   File ${file}
+          //   filesize: ${fileSize}
+          //   sessionId: ${sessionId}
+          //   uploaded: ${uploaded}
+          //   chunk: ${chunk.length}
+          //   isLastChunk: ${isLastChunk}
+          // `)
 
           if (sessionId === undefined) {
-            await shouldRetry<any, { error?: { retry_after?: number } }>(
-              async () => {
-                sessionId = (await this.dropbox.filesUploadSessionStart({ contents: chunk, close: isLastChunk })).result
-                  .session_id
-                sessions[file] = {
-                  sessionId: sessionId,
-                  fileSize: fileSize,
-                }
-              },
-              async (error) => {
-                if (error.error?.retry_after) {
-                  this.logger?.warn(`Error: ${error}: wait ${error.error?.retry_after}`)
-                  await delay(error.error?.retry_after)
-                } else {
-                  await delay(100)
-                }
-
-                return true
-              },
-              5,
-            )
+            await retryWhenTooManyRequests(async () => {
+              sessionId = (await this.dropbox.filesUploadSessionStart({ contents: chunk, close: isLastChunk })).result
+                .session_id
+              sessions[file] = {
+                sessionId: sessionId,
+                fileSize: fileSize,
+              }
+            })
           } else {
-            await this.dropbox.filesUploadSessionAppendV2({
-              // @ts-ignore incorrect cursor typings here, that required `contents`, but crashed in runtime
-              cursor: { session_id: sessionId, offset: uploaded },
-              contents: chunk,
-              close: isLastChunk,
+            await retryWhenTooManyRequests(async () => {
+              return this.dropbox.filesUploadSessionAppendV2({
+                // @ts-ignore incorrect cursor typings here, that required `contents`, but crashed in runtime
+                cursor: { session_id: sessionId, offset: uploaded },
+                contents: chunk,
+                close: isLastChunk,
+              })
             })
           }
           uploaded += chunk.length
@@ -243,3 +232,21 @@ export class DropboxUploader implements Uploader {
 
   constructor(private dropbox: Dropbox, private logger?: Logger) {}
 }
+
+const retryWhenTooManyRequests = async <T>(func: () => Promise<T>) => {
+  return shouldRetry<any, { error?: { error?: { retry_after?: number } } }>(
+    func,
+    async (error) => {
+      console.warn(`error = ${JSON.stringify(error)}`)
+      if (error.error?.error?.retry_after) {
+        console.warn(`Error: ${error}: wait ${error.error?.error?.retry_after}`)
+        await delay(error.error?.error?.retry_after)
+      } else {
+        await delay(1000)
+      }
+
+      return true
+    },
+    5,
+  )
+}
diff --git a/src/utils/delay.ts b/src/utils/delay.ts
@@ -1,5 +1,5 @@
 export async function delay(timeout = 1000) {
   if (timeout === 0) return Promise.resolve()
-
+  console.warn(`delay ${timeout}`)
   return new Promise((resolver) => setTimeout(resolver, timeout))
 }