Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: multipart copy support #371

Merged
merged 8 commits into from
Feb 26, 2018
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
440 changes: 438 additions & 2 deletions README.md

Large diffs are not rendered by default.

9 changes: 9 additions & 0 deletions lib/browser/bucket.js
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,15 @@ proto.useBucket = function useBucket(name, region) {
return this;
};

proto.setBucket = function useBucket(name) {
this.options.bucket = name;
return this;
};

proto.getBucket = function getBucket() {
return this.options.bucket;
};

proto.putBucket = function* putBucket(name, region, options) {
var params = this._bucketRequestParams('PUT', name, '', options);
if (region) {
Expand Down
4 changes: 3 additions & 1 deletion lib/browser/client.js
Original file line number Diff line number Diff line change
Expand Up @@ -109,10 +109,12 @@ merge(proto, require('./object'));
// * Bucket operations
// */
// merge(proto, require('./bucket'));
//multipart upload
merge(proto, require('./managed_upload'));
/**
* Multipart operations
*/
merge(proto, require('./multipart'));
merge(proto, require('../common/multipart'));

/**
* Common module
Expand Down
168 changes: 3 additions & 165 deletions lib/browser/multipart.js → lib/browser/managed_upload.js
Original file line number Diff line number Diff line change
@@ -1,10 +1,7 @@
'use strict';

// var debug = require('debug')('ali-oss:multipart');
// var fs = require('fs');
var is = require('is-type-of');
// var destroy = require('destroy');
// var eoe = require('end-or-error');
var util = require('util');
var path = require('path');
var mime = require('mime');
Expand Down Expand Up @@ -61,7 +58,7 @@ proto.multipartUpload = function* multipartUpload(name, file, options) {
throw new Error('partSize must not be smaller than ' + minPartSize);
}

var result = yield this._initMultipartUpload(name, options);
var result = yield this.initMultipartUpload(name, options);
var uploadId = result.uploadId;
var partSize = this._getPartSize(fileSize, options.partSize);

Expand Down Expand Up @@ -111,7 +108,7 @@ proto._resumeMultipart = function* _resumeMultipart(checkpoint, options) {
size: pi.end - pi.start
};

var result = yield self._uploadPart(name, uploadId, partNo, data);
var result = yield self._uploadPart(name, uploadId, partNo, data, options);
doneParts.push({
number: partNo,
etag: result.res.headers.etag
Expand Down Expand Up @@ -163,169 +160,10 @@ proto._resumeMultipart = function* _resumeMultipart(checkpoint, options) {
throw err;
}
}
return yield this._completeMultipartUpload(name, uploadId, doneParts, options);
return yield this.completeMultipartUpload(name, uploadId, doneParts, options);
};

/**
* List the on-going multipart uploads
* https://help.aliyun.com/document_detail/31997.html
* @param {Object} options
* @return {Array} the multipart uploads
*/
proto.listUploads = function* listUploads(query, options) {
options = options || {};
options.subres = 'uploads';
var params = this._objectRequestParams('GET', '', options)
params.query = query;
params.xmlResponse = true;
params.successStatuses = [200];

var result = yield this.request(params);
var uploads = result.data.Upload || [];
if (!Array.isArray(uploads)) {
uploads = [uploads];
}
uploads = uploads.map(function (up) {
return {
name: up.Key,
uploadId: up.UploadId,
initiated: up.Initiated
};
});

return {
res: result.res,
uploads: uploads,
bucket: result.data.Bucket,
nextKeyMarker: result.data.NextKeyMarker,
nextUploadIdMarker: result.data.NextUploadIdMarker,
isTruncated: result.data.IsTruncated === 'true'
};
};

/**
* Abort a multipart upload transaction
* @param {String} name the object name
* @param {String} uploadId the upload id
* @param {Object} options
*/
proto.abortMultipartUpload = function* abortMultipartUpload(name, uploadId, options) {
this.cancel();
options = options || {};
options.subres = {uploadId: uploadId};
var params = this._objectRequestParams('DELETE', name, options);
params.successStatuses = [204];

var result = yield this.request(params);
return {
res: result.res
};
};

/**
* Initiate a multipart upload transaction
* @param {String} name the object name
* @param {Object} options
* @return {String} upload id
*/
proto._initMultipartUpload = function* _initMultipartUpload(name, options) {
options = options || {};
options.headers = options.headers || {};
this._convertMetaToHeaders(options.meta, options.headers);

options.subres = 'uploads';
var params = this._objectRequestParams('POST', name, options);
params.mime = options.mime;
params.xmlResponse = true;
params.successStatuses = [200];

var result = yield this.request(params);

return {
res: result.res,
bucket: result.data.Bucket,
name: result.data.Key,
uploadId: result.data.UploadId
};
};

/**
* Upload a part in a multipart upload transaction
* @param {String} name the object name
* @param {String} uploadId the upload id
* @param {Integer} partNo the part number
* @param {Object} data the body data
* @param {Object} options
*/
proto._uploadPart = function* _uploadPart(name, uploadId, partNo, data, options) {
options = options || {};
options.headers = {
'Content-Length': data.size
};

options.subres = {
partNumber: partNo,
uploadId: uploadId
};
var params = this._objectRequestParams('PUT', name, options);
params.mime = options.mime;
params.stream = data.stream;
params.successStatuses = [200];

var result = yield this.request(params);

data.stream = null;
params.stream = null;
return {
name: name,
etag: result.res.headers.etag,
res: result.res
};
};

/**
* Complete a multipart upload transaction
* @param {String} name the object name
* @param {String} uploadId the upload id
* @param {Array} parts the uploaded parts
* @param {Object} options
*/
proto._completeMultipartUpload = function* _completeMultipartUpload(name, uploadId, parts, options) {
parts.sort((a, b) => a.number - b.number);
var xml = '<?xml version="1.0" encoding="UTF-8"?>\n<CompleteMultipartUpload>\n';
for (var i = 0; i < parts.length; i++) {
var p = parts[i];
xml += '<Part>\n';
xml += '<PartNumber>' + p.number + '</PartNumber>\n';
xml += '<ETag>' + p.etag + '</ETag>\n';
xml += '</Part>\n';
}
xml += '</CompleteMultipartUpload>';

options = options || {};
options.subres = {uploadId: uploadId};
var params = this._objectRequestParams('POST', name, options);
params.mime = 'xml';
params.content = xml;
if (!(options.headers && options.headers['x-oss-callback'])) {
params.xmlResponse = true;
}
params.successStatuses = [200];
var result = yield this.request(params);

var ret = {
res: result.res,
bucket: params.bucket,
name: name,
etag: result.res.headers['etag']
};

if (options.headers && options.headers['x-oss-callback']) {
ret.data = JSON.parse(result.data.toString());
}

return ret;
};

is.file = function (file) {
return typeof(File) !== 'undefined' && file instanceof File;
Expand Down
9 changes: 9 additions & 0 deletions lib/bucket.js
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,15 @@ proto.useBucket = function useBucket(name, region) {
return this;
};

proto.setBucket = function useBucket(name) {
this.options.bucket = name;
return this;
};

proto.getBucket = function getBucket() {
return this.options.bucket;
};

proto.putBucket = function* putBucket(name, region, options) {
var params = this._bucketRequestParams('PUT', name, '', options);
if (region) {
Expand Down
15 changes: 11 additions & 4 deletions lib/client.js
Original file line number Diff line number Diff line change
Expand Up @@ -103,15 +103,22 @@ merge(proto, require('./object'));
* Bucket operations
*/
merge(proto, require('./bucket'));
/**
* Multipart operations
*/
merge(proto, require('./multipart'));
//multipart upload
merge(proto, require('./managed_upload'));
/**
* RTMP operations
*/
merge(proto, require('./rtmp'));

/**
* common multipart-copy support node and browser
*/
merge(proto, require('./common/multipart-copy.js'));
merge(proto, require('./common/thunkpool.js'));
/**
* Multipart operations
*/
merge(proto, require('./common/multipart'));
/**
* ImageClient class
*/
Expand Down
Loading