From 2d651896a93d4c2a67404479dce90ef01d79bb5a Mon Sep 17 00:00:00 2001 From: beajer <919060679@qq.com> Date: Thu, 25 Mar 2021 19:39:44 +0800 Subject: [PATCH] feat: provide a switch to generate upload file md5 (#945) --- README.md | 2 ++ lib/browser/client.js | 3 +-- lib/browser/managed-upload.js | 4 ++- lib/browser/object.js | 2 ++ lib/common/multipart.js | 1 + lib/common/utils/createRequest.js | 10 ++++--- lib/common/utils/createRequest.ts | 10 ++++--- test/browser/browser.test.js | 44 +++++++++++++++++++++++++++++-- 8 files changed, 63 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index aaaba6174..f3b298a82 100644 --- a/README.md +++ b/README.md @@ -1610,6 +1610,7 @@ parameters: - 'Content-Encoding' object content encoding for download, e.g.: `Content-Encoding: gzip` - 'Expires' expires time for download, an absolute date and time. e.g.: `Tue, 08 Dec 2020 13:49:43 GMT` - See more: [PutObject](https://help.aliyun.com/document_detail/31978.html#title-yxe-96d-x61) + - [disabledMD5] {Boolean} default true, it just work in Browser. if false,it means that MD5 is automatically calculated for uploaded files. **_NOTE:_** Synchronous computing tasks will block the main process Success will return the object information. @@ -3061,6 +3062,7 @@ parameters: - 'Expires' expires time for download, an absolute date and time. e.g.: `Tue, 08 Dec 2020 13:49:43 GMT` - **NOTE**: Some headers are [disabled in browser][disabled-browser-headers] - [timeout] {Number} Milliseconds before a request is considered to be timed out + - [disabledMD5] {Boolean} default true, it just work in Browser. if false,it means that MD5 is automatically calculated for uploaded files. **_NOTE:_** Synchronous computing tasks will block the main process Success will return: diff --git a/lib/browser/client.js b/lib/browser/client.js index 786984bda..782341f6c 100644 --- a/lib/browser/client.js +++ b/lib/browser/client.js @@ -71,7 +71,7 @@ Client.initOptions = function initOptions(options) { const opts = Object.assign({ secure: isHttpsWebProtocol(), // for browser compatibility disable fetch. - useFetch: false + useFetch: false, }, options); return _initOptions(opts); @@ -215,7 +215,6 @@ proto.request = async function (params) { async function request(params) { const reqParams = createRequest.call(this, params); - if (!this.options.useFetch) { reqParams.params.mode = 'disable-fetch'; } diff --git a/lib/browser/managed-upload.js b/lib/browser/managed-upload.js index 70b30442f..4bd59bda8 100644 --- a/lib/browser/managed-upload.js +++ b/lib/browser/managed-upload.js @@ -33,6 +33,7 @@ const proto = exports; */ proto.multipartUpload = async function multipartUpload(name, file, options = {}) { this.resetCancelFlag(); + options.disabledMD5 = options.disabledMD5 === undefined ? true : !!options.disabledMD5; if (options.checkpoint && options.checkpoint.uploadId) { if (file && isFile(file)) options.checkpoint.file = file; @@ -143,7 +144,8 @@ proto._resumeMultipart = async function _resumeMultipart(checkpoint, options) { let result; try { result = await self._uploadPart(name, uploadId, partNo, data, { - timeout: options.timeout + timeout: options.timeout, + disabledMD5: options.disabledMD5 }); } catch (error) { if (error.status === 404) { diff --git a/lib/browser/object.js b/lib/browser/object.js index 89cc8d4bc..a927c704d 100644 --- a/lib/browser/object.js +++ b/lib/browser/object.js @@ -58,6 +58,7 @@ proto.append = async function append(name, file, options) { proto.put = async function put(name, file, options) { let content; options = options || {}; + options.disabledMD5 = options.disabledMD5 === undefined ? true : !!options.disabledMD5; options.headers = options.headers || {}; name = this._objectName(name); if (isBuffer(file)) { @@ -83,6 +84,7 @@ proto.put = async function put(name, file, options) { const params = this._objectRequestParams(method, name, options); callback.encodeCallback(params, options); params.mime = options.mime; + params.disabledMD5 = options.disabledMD5; params.content = content; params.successStatuses = [200]; diff --git a/lib/common/multipart.js b/lib/common/multipart.js index ce8920108..fa546b4aa 100644 --- a/lib/common/multipart.js +++ b/lib/common/multipart.js @@ -243,6 +243,7 @@ proto._uploadPart = async function _uploadPart(name, uploadId, partNo, data, opt const isBrowserEnv = process && process.browser; isBrowserEnv ? (params.content = data.content) : (params.stream = data.stream); params.successStatuses = [200]; + params.disabledMD5 = options.disabledMD5; const result = await this.request(params); diff --git a/lib/common/utils/createRequest.js b/lib/common/utils/createRequest.js index df08aef3c..0c7cd2be6 100644 --- a/lib/common/utils/createRequest.js +++ b/lib/common/utils/createRequest.js @@ -49,10 +49,12 @@ function createRequest(params) { delHeader(headers, 'Content-Type'); } if (params.content) { - headers['Content-MD5'] = crypto - .createHash('md5') - .update(Buffer.from(params.content, 'utf8')) - .digest('base64'); + if (!params.disabledMD5) { + headers['Content-MD5'] = crypto + .createHash('md5') + .update(Buffer.from(params.content, 'utf8')) + .digest('base64'); + } if (!headers['Content-Length']) { headers['Content-Length'] = params.content.length; } diff --git a/lib/common/utils/createRequest.ts b/lib/common/utils/createRequest.ts index 353f3fbbb..d8ca67f36 100644 --- a/lib/common/utils/createRequest.ts +++ b/lib/common/utils/createRequest.ts @@ -65,10 +65,12 @@ export function createRequest(this: any, params) { } if (params.content) { - headers['Content-MD5'] = crypto - .createHash('md5') - .update(Buffer.from(params.content, 'utf8')) - .digest('base64'); + if (!params.disabledMD5) { + headers['Content-MD5'] = crypto + .createHash('md5') + .update(Buffer.from(params.content, 'utf8')) + .digest('base64'); + } if (!headers['Content-Length']) { headers['Content-Length'] = params.content.length; } diff --git a/test/browser/browser.test.js b/test/browser/browser.test.js index 08da54dd7..a474e583e 100644 --- a/test/browser/browser.test.js +++ b/test/browser/browser.test.js @@ -993,7 +993,6 @@ describe('browser', () => { it('should signature url for PUT', async () => { const putString = 'Hello World'; const contentMd5 = crypto1.createHash('md5').update(Buffer.from(putString, 'utf8')).digest('base64'); - console.log(contentMd5); const url = store.signatureUrl(name, { method: 'PUT', 'Content-Type': 'text/plain; charset=UTF-8', @@ -1731,7 +1730,6 @@ describe('browser', () => { } catch (err) { netErrs = err; } - console.log(netErrs); assert.strictEqual(netErrs.status, 0); assert.strictEqual(netErrs.name, 'abort'); store._uploadPart = uploadPart; @@ -2072,6 +2070,48 @@ describe('browser', () => { }); }); + describe('options.disabledMD5', () => { + const content = Array(100).fill(1).join(''); + const body = new Blob([content], { type: 'text/plain' }); + const MD5_VALUE = crypto1.createHash('md5').update(OSS.Buffer(content)).digest('base64'); + + it('should not calculate MD5 default when use put', async () => { + const store = oss(ossConfig); + const name = `${prefix}put/test-md5-0`; + const request = store.urllib.request; + let reqParams; + store.urllib.request = (url, params) => { + reqParams = params; + return request(url, params); + }; + await store.put(name, body); + assert.strictEqual(reqParams.headers['Content-MD5'], undefined); + await store.put(name, body, { disabledMD5: false }); + assert.strictEqual(reqParams.headers['Content-MD5'], MD5_VALUE); + store.urllib.request = request; + }); + + it('should not calculate MD5 default when use multipartUpload', async () => { + const store = oss(ossConfig); + const name = `${prefix}put/test-md5-2`; + const partSize = 100 * 1024; + const body2 = new File(Array(2 * partSize).fill(1), { type: 'text/plain' }); + const request = store.urllib.request; + let headerWithMD5Count = 0; + store.urllib.request = (url, params) => { + if (params.headers['Content-MD5'] && /partNumber=\d/.test(url)) { + headerWithMD5Count++; + } + return request(url, params); + }; + await store.multipartUpload(name, body2, { partSize }); + assert.strictEqual(headerWithMD5Count, 0); + await store.multipartUpload(name, body2, { disabledMD5: false, partSize }); + assert.strictEqual(headerWithMD5Count, 2); + store.urllib.request = request; + }); + }); + describe('test/retry.test.js', () => { let store; const RETRY_MAX = 3;