@uppy/aws-s3-multipart: add shouldUseMultipart option

packages/@uppy/aws-s3-multipart/src/MultipartUploader.js

@@ -43,6 +43,8 @@ class MultipartUploader {
 
   #onSuccess
 
+  #shouldUseMultipart
+
   #onReject = (err) => (err?.cause === pausingUploadReason ? null : this.#onError(err))
 
   constructor (data, options) {
@@ -57,25 +59,23 @@ class MultipartUploader {
     this.#file = options.file
     this.#onSuccess = this.options.onSuccess
     this.#onError = this.options.onError
+    this.#shouldUseMultipart = this.options.shouldUseMultipart
 
     this.#initChunks()
   }
 
   #initChunks () {
-    const desiredChunkSize = this.options.getChunkSize(this.#data)
-    // at least 5MB per request, at most 10k requests
     const fileSize = this.#data.size
-    const minChunkSize = Math.max(5 * MB, Math.ceil(fileSize / 10000))
-    const chunkSize = Math.max(desiredChunkSize, minChunkSize)
+    const shouldUseMultipart = typeof this.#shouldUseMultipart === 'function'
+      ? this.#shouldUseMultipart(this.#file, fileSize)
+      : Boolean(this.#shouldUseMultipart)
+
+    if (shouldUseMultipart) {
+      const desiredChunkSize = this.options.getChunkSize(this.#data)
+      // at least 5MB per request, at most 10k requests
+      const minChunkSize = Math.max(5 * MB, Math.ceil(fileSize / 10000))
+      const chunkSize = Math.max(desiredChunkSize, minChunkSize)
 
-    // Upload zero-sized files in one zero-sized chunk
-    if (this.#data.size === 0) {
-      this.#chunks = [{
-        getData: () => this.#data,
-        onProgress: this.#onPartProgress(0),
-        onComplete: this.#onPartComplete(0),
-      }]
-    } else {
       const arraySize = Math.ceil(fileSize / chunkSize)
       this.#chunks = Array(arraySize)
 
@@ -92,8 +92,16 @@ class MultipartUploader {
           getData,
           onProgress: this.#onPartProgress(j),
           onComplete: this.#onPartComplete(j),
+          shouldUseMultipart,
         }
       }
+    } else {
+      this.#chunks = [{
+        getData: () => this.#data,
+        onProgress: this.#onPartProgress(0),
+        onComplete: this.#onPartComplete(0),
+        shouldUseMultipart,
+      }]
     }
 
     this.#chunkState = this.#chunks.map(() => ({ uploaded: 0 }))
@@ -111,11 +119,8 @@ class MultipartUploader {
       .then(this.#onSuccess, this.#onReject)
   }
 
-  #onPartProgress = (index) => (ev) => {
-    if (!ev.lengthComputable) return
-
-    const sent = ev.loaded
-    this.#chunkState[index].uploaded = ensureInt(sent)
+  #onPartProgress = (index) => (bytes) => {
+    this.#chunkState[index].uploaded = ensureInt(bytes)
 
     const totalUploaded = this.#chunkState.reduce((n, c) => n + c.uploaded, 0)
     this.options.onProgress(totalUploaded, this.#data.size)

packages/@uppy/aws-s3-multipart/src/index.js

@@ -31,6 +31,8 @@ class HTTPCommunicationQueue {
 
   #fetchSignature
 
+  #getUploadParameters
+
   #listParts
 
   #previousRetryDelay
@@ -75,6 +77,9 @@ class HTTPCommunicationQueue {
     if ('uploadPartBytes' in options) {
       this.#uploadPartBytes = requests.wrapPromiseFunction(options.uploadPartBytes, { priority:Infinity })
     }
+    if ('getUploadParameters' in options) {
+      this.#getUploadParameters = requests.wrapPromiseFunction(options.getUploadParameters)
+    }
   }
 
   async #shouldRetry (err) {
@@ -190,8 +195,41 @@ class HTTPCommunicationQueue {
     await this.#abortMultipartUpload(file, awaitedResult)
   }
 
+  async #nonMultipartUpload (file, chunk, signal) {
+    const filename = file.meta.name
+    const { type } = file.meta
+    const metadata = undefined // TODO: add `allowedMetaFields` option to S3 multipart?
+
+    const query = new URLSearchParams({ filename, type, ...metadata })
+    const {
+      method = 'post',
+      url,
+      fields,
+      headers,
+    } = await this.#getUploadParameters(`s3/params?${query}`, { signal }).abortOn(signal)
+
+    const formData = new FormData()
+    Object.entries(fields).forEach(([key, value]) => formData.set(key, value))
+    const data = chunk.getData()
+    formData.set('file', data)
+
+    const { onProgress, onComplete } = chunk
+
+    return this.#uploadPartBytes({
+      signature: { url, headers, method },
+      body: formData,
+      size: data.size,
+      onProgress,
+      onComplete,
+      signal,
+    }).abortOn(signal)
+  }
+
   async uploadFile (file, chunks, signal) {
     throwIfAborted(signal)
+    if (chunks.length === 1 && !chunks[0].shouldUseMultipart) {
+      return this.#nonMultipartUpload(file, chunks[0], signal)
+    }
     const { uploadId, key } = await this.getUploadId(file, signal)
     throwIfAborted(signal)
     try {
@@ -206,6 +244,9 @@ class HTTPCommunicationQueue {
 
   async resumeUploadFile (file, chunks, signal) {
     throwIfAborted(signal)
+    if (chunks.length === 1) {
+      return this.#nonMultipartUpload(file, chunks[0], signal)
+    }
     const { uploadId, key } = await this.getUploadId(file, signal)
     throwIfAborted(signal)
     const alreadyUploadedParts = await this.#listParts(file, { uploadId, key, signal }).abortOn(signal)
@@ -240,7 +281,9 @@ class HTTPCommunicationQueue {
       try {
         return {
           PartNumber: partNumber,
-          ...await this.#uploadPartBytes({ signature, body: chunkData, onProgress, onComplete, signal }).abortOn(signal),
+          ...await this.#uploadPartBytes({
+            signature, body: chunkData, size: chunkData.size, onProgress, onComplete, signal,
+          }).abortOn(signal),
         }
       } catch (err) {
         if (!await this.#shouldRetry(err)) throw err
@@ -269,13 +312,17 @@ export default class AwsS3Multipart extends BasePlugin {
       // TODO: this is currently opt-in for backward compat, switch to opt-out in the next major
       allowedMetaFields: null,
       limit: 6,
+      shouldUseMultipart: true, // In the next major, the default should be switched as the following:
+      // eslint-disable-next-line no-bitwise
+      // shouldUseMultipart: (file, fileSize) => fileSize >> 10 >> 10 > 100,
       retryDelays: [0, 1000, 3000, 5000],
       createMultipartUpload: this.createMultipartUpload.bind(this),
       listParts: this.listParts.bind(this),
       abortMultipartUpload: this.abortMultipartUpload.bind(this),
       completeMultipartUpload: this.completeMultipartUpload.bind(this),
       signPart: this.signPart.bind(this),
       uploadPartBytes: AwsS3Multipart.uploadPartBytes,
+      getUploadParameters: (...args) => this.#client.get(...args),
       companionHeaders: {},
     }
 
@@ -397,7 +444,7 @@ export default class AwsS3Multipart extends BasePlugin {
       .then(assertServerError)
   }
 
-  static async uploadPartBytes ({ signature: { url, expires, headers }, body, onProgress, onComplete, signal }) {
+  static async uploadPartBytes ({ signature: { url, expires, headers, method = 'PUT' }, body, size, onProgress, onComplete, signal }) {
     throwIfAborted(signal)
 
     if (url == null) {
@@ -406,7 +453,7 @@ export default class AwsS3Multipart extends BasePlugin {
 
     return new Promise((resolve, reject) => {
       const xhr = new XMLHttpRequest()
-      xhr.open('PUT', url, true)
+      xhr.open(method, url, true)
       if (headers) {
         Object.keys(headers).forEach((key) => {
           xhr.setRequestHeader(key, headers[key])
@@ -425,7 +472,10 @@ export default class AwsS3Multipart extends BasePlugin {
       }
       signal.addEventListener('abort', onabort)
 
-      xhr.upload.addEventListener('progress', onProgress)
+      xhr.upload.addEventListener('progress', (ev) => {
+        if (!ev.lengthComputable) return
+        onProgress(ev.loaded)
+      })
 
       xhr.addEventListener('abort', () => {
         cleanup()
@@ -455,7 +505,7 @@ export default class AwsS3Multipart extends BasePlugin {
           return
         }
 
-        onProgress?.(body.size)
+        onProgress?.(size)
 
         // NOTE This must be allowed by CORS.
         const etag = ev.target.getResponseHeader('ETag')
@@ -550,6 +600,7 @@ export default class AwsS3Multipart extends BasePlugin {
         onPartComplete,
 
         file,
+        shouldUseMultipart: this.opts.shouldUseMultipart,
 
         ...file.s3Multipart,
       })

packages/@uppy/aws-s3-multipart/types/index.d.ts

@@ -43,6 +43,7 @@ export interface AwsS3MultipartOptions extends PluginOptions {
       opts: { uploadId: string; key: string; parts: AwsS3Part[]; signal: AbortSignal }
     ) => MaybePromise<{ location?: string }>
     limit?: number
+    shouldUseMultipart?: boolean | ((file: UppyFile, fileSize: number) => boolean)
     retryDelays?: number[] | null
 }