diff --git a/lib/common/util.js b/lib/common/util.js index b3ccf208a8c..f88e94d06ec 100644 --- a/lib/common/util.js +++ b/lib/common/util.js @@ -208,6 +208,52 @@ function getType(value) { return Object.prototype.toString.call(value).match(/\s(\w+)\]/)[1]; } +/** + * Used in an Array iterator usually, this will return the value of a property + * in an object by its name. + * + * @param {string} name - Name of the property to return. + * @return {function} + * + * @example + * var people = [ + * { + * name: 'Stephen', + * origin: 'USA', + * beenToNYC: false + * }, + * { + * name: 'Ryan', + * origin: 'Canada', + * beenToNYC: true + * } + * ]; + * + * var names = people.map(prop('name')); + * // [ + * // 'Stephen', + * // 'Ryan' + * // ] + * + * var peopleInUSA = people.filter(prop('beenToNYC')); + * // [ + * // { + * // name: 'Ryan', + * // origin: 'Canada', + * // beenToNYC: true + * // } + * // ] + */ +function prop(name) { + return function(item) { + if (name in item) { + return item[name]; + } + }; +} + +module.exports.prop = prop; + /** * Assign a value to a property in an Array iterator. * diff --git a/lib/storage/bucket.js b/lib/storage/bucket.js index 3689ea70a5f..ca427b0a9b7 100644 --- a/lib/storage/bucket.js +++ b/lib/storage/bucket.js @@ -202,6 +202,100 @@ function Bucket(storage, name) { /* jshint ignore:end */ } +/** + * Combine mutliple files into one new file. + * + * @throws if a non-array is provided as sources argument. + * @throws if less than two sources are provided. + * @throws if no destination is provided. + * @throws if a content type cannot be determined for the destination file. + * + * @param {string[]|module:storage/file[]} sources - The source files that will + * be combined. + * @param {string|module:storage/file} destination - The file you would like the + * source files combined into. + * @param {function=} callback - The callback function. + * + * @example + * var 2013logs = bucket.file('2013-logs.txt'); + * var 2014logs = bucket.file('2014-logs.txt'); + * + * var allLogs = bucket.file('all-logs.txt'); + * + * bucket.combine([ + * 2013logs, + * 2014logs + * ], allLogs, function(err, newFile) { + * // newFile === allLogs + * }); + */ +Bucket.prototype.combine = function(sources, destination, callback) { + if (!util.is(sources, 'array') || sources.length < 2) { + throw new Error('You must provide at least two source files.'); + } + + if (!destination) { + throw new Error('A destination file must be specified.'); + } + + var self = this; + + sources = sources.map(convertToFile); + destination = convertToFile(destination); + callback = callback || util.noop; + + if (!destination.metadata.contentType) { + var destinationContentType = mime.contentType(destination.name); + + if (destinationContentType) { + destination.metadata.contentType = destinationContentType; + } else { + throw new Error( + 'A content type could not be detected for the destination file.'); + } + } + + this.storage.makeAuthorizedRequest_({ + method: 'POST', + uri: util.format('{base}/{destBucket}/o/{destFile}/compose', { + base: STORAGE_BASE_URL, + destBucket: destination.bucket.name, + destFile: encodeURIComponent(destination.name) + }), + json: { + destination: { + contentType: destination.metadata.contentType + }, + sourceObjects: sources.map(function (source) { + var sourceObject = { + name: source.name + }; + + if (source.metadata && source.metadata.generation) { + sourceObject.generation = source.metadata.generation; + } + + return sourceObject; + }) + } + }, function(err) { + if (err) { + callback(err); + return; + } + + callback(null, destination); + }); + + function convertToFile(file) { + if (file instanceof File) { + return file; + } else { + return self.file(file); + } + } +}; + /** * Delete the bucket. * diff --git a/regression/storage.js b/regression/storage.js index af5946380d4..b61a37b713d 100644 --- a/regression/storage.js +++ b/regression/storage.js @@ -25,8 +25,11 @@ var fs = require('fs'); var request = require('request'); var through = require('through2'); var tmp = require('tmp'); +var util = require('../lib/common/util'); var uuid = require('node-uuid'); +var prop = util.prop; + var env = require('./env.js'); var storage = require('../lib/storage')(env); @@ -47,12 +50,14 @@ function deleteFiles(bucket, callback) { callback(err); return; } - async.map(files, function(file, next) { - file.delete(next); - }, callback); + async.map(files, deleteFile, callback); }); } +function deleteFile(file, callback) { + file.delete(callback); +} + function generateBucketName() { return 'gcloud-test-bucket-temp-' + uuid.v1(); } @@ -493,6 +498,41 @@ describe('storage', function() { }); }); + describe('combine files', function() { + it('should combine multiple files into one', function(done) { + var files = [ + { file: bucket.file('file-one.txt'), contents: '123' }, + { file: bucket.file('file-two.txt'), contents: '456' } + ]; + + async.each(files, createFile, function(err) { + assert.ifError(err); + + var sourceFiles = files.map(prop('file')); + var destinationFile = bucket.file('file-one-and-two.txt'); + + bucket.combine(sourceFiles, destinationFile, function(err) { + assert.ifError(err); + + destinationFile.download(function(err, contents) { + assert.ifError(err); + + assert.equal(contents, files.map(prop('contents')).join('')); + + async.each(sourceFiles.concat([destinationFile]), deleteFile, done); + }); + }); + }); + + function createFile(fileObject, cb) { + var ws = fileObject.file.createWriteStream(); + ws.on('error', cb); + ws.on('complete', cb.bind(null, null)); + ws.end(fileObject.contents); + } + }); + }); + describe('list files', function() { var filenames = ['CloudLogo1', 'CloudLogo2', 'CloudLogo3']; diff --git a/test/common/util.js b/test/common/util.js index 02d49d7c044..55d00259266 100644 --- a/test/common/util.js +++ b/test/common/util.js @@ -449,6 +449,18 @@ describe('common/util', function() { }); }); + describe('prop', function() { + it('should return objects that match the property name', function() { + var people = [ + { name: 'Stephen', origin: 'USA', beenToNYC: false }, + { name: 'Ryan', origin: 'Canada', beenToNYC: true } + ]; + + assert.deepEqual(people.map(util.prop('name')), ['Stephen', 'Ryan']); + assert.deepEqual(people.filter(util.prop('beenToNYC')), [people[1]]); + }); + }); + describe('propAssign', function() { it('should assign a property and value to an object', function() { var obj = {}; diff --git a/test/storage/bucket.js b/test/storage/bucket.js index 5718e196826..72235cc7128 100644 --- a/test/storage/bucket.js +++ b/test/storage/bucket.js @@ -19,6 +19,8 @@ 'use strict'; var assert = require('assert'); +var async = require('async'); +var mime = require('mime-types'); var mockery = require('mockery'); var request = require('request'); var stream = require('stream'); @@ -27,7 +29,7 @@ var util = require('../../lib/common/util.js'); function FakeFile(bucket, name, metadata) { this.bucket = bucket; this.name = name; - this.metadata = metadata; + this.metadata = metadata || {}; this.createWriteStream = function(options) { this.metadata = options.metadata; var ws = new stream.Writable(); @@ -93,6 +95,189 @@ describe('Bucket', function() { }); }); + describe('combine', function() { + it('should throw if invalid sources are not provided', function() { + var error = 'You must provide at least two source files.'; + + assert.throws(function() { + bucket.combine(); + }, new RegExp(error)); + + assert.throws(function() { + bucket.combine(['1']); + }, new RegExp(error)); + }); + + it('should throw if a destination is not provided', function() { + var error = 'A destination file must be specified.'; + + assert.throws(function() { + bucket.combine(['1', '2']); + }, new RegExp(error)); + }); + + it('should accept string or file input for sources', function(done) { + var file1 = bucket.file('1.txt'); + var file2 = '2.txt'; + + bucket.storage.makeAuthorizedRequest_ = function(reqOpts) { + assert.equal(reqOpts.json.sourceObjects[0].name, file1.name); + assert.equal(reqOpts.json.sourceObjects[1].name, file2); + done(); + }; + + bucket.combine([file1, file2], 'destination.txt'); + }); + + it('should accept string or file input for destination', function(done) { + var destinations = [ + 'destination.txt', + bucket.file('destination.txt') + ]; + + async.each(destinations, function(destination, next) { + bucket.storage.makeAuthorizedRequest_ = function(reqOpts) { + assert(reqOpts.uri.indexOf(bucket.name + '/o/destination.txt') > -1); + next(); + }; + + bucket.combine(['1', '2'], destination); + }, done); + }); + + it('should use content type from the destination metadata', function(done) { + var destination = 'destination.txt'; + + bucket.storage.makeAuthorizedRequest_ = function(reqOpts) { + assert.equal( + reqOpts.json.destination.contentType, + mime.contentType(destination) + ); + done(); + }; + + bucket.combine(['1', '2'], destination); + }); + + it('should use content type from the destination metadata', function(done) { + var destination = bucket.file('destination.txt'); + destination.metadata = { contentType: 'content-type' }; + + bucket.storage.makeAuthorizedRequest_ = function(reqOpts) { + assert.equal( + reqOpts.json.destination.contentType, + destination.metadata.contentType + ); + done(); + }; + + bucket.combine(['1', '2'], destination); + }); + + it('should detect dest content type if not in metadata', function(done) { + var destination = 'destination.txt'; + + bucket.storage.makeAuthorizedRequest_ = function(reqOpts) { + assert.equal( + reqOpts.json.destination.contentType, + mime.contentType(destination) + ); + done(); + }; + + bucket.combine(['1', '2'], destination); + }); + + it('should throw if content type cannot be determined', function() { + var error = + 'A content type could not be detected for the destination file.'; + + assert.throws(function() { + bucket.combine(['1', '2'], 'destination'); + }, new RegExp(error)); + }); + + it('should make correct API request', function(done) { + var sources = [bucket.file('1.txt'), bucket.file('2.txt')]; + var destination = bucket.file('destination.txt'); + + bucket.storage.makeAuthorizedRequest_ = function(reqOpts) { + var expectedUri = util.format('{base}/{bucket}/o/{file}/compose', { + base: 'https://www.googleapis.com/storage/v1/b', + bucket: destination.bucket.name, + file: encodeURIComponent(destination.name) + }); + + assert.equal(reqOpts.uri, expectedUri); + assert.deepEqual(reqOpts.json, { + destination: { contentType: mime.contentType(destination.name) }, + sourceObjects: [{ name: sources[0].name }, { name: sources[1].name }] + }); + done(); + }; + + bucket.combine(sources, destination); + }); + + it('should encode the destination file name', function(done) { + var sources = [bucket.file('1.txt'), bucket.file('2.txt')]; + var destination = 'needs encoding.jpg'; + + bucket.storage.makeAuthorizedRequest_ = function(reqOpts) { + assert.equal(reqOpts.uri.indexOf(destination), -1); + done(); + }; + + bucket.combine(sources, destination); + }); + + it('should send a source generation value if available', function(done) { + var sources = [bucket.file('1.txt'), bucket.file('2.txt')]; + sources[0].metadata = { generation: 1 }; + sources[1].metadata = { generation: 2 }; + + var destination = bucket.file('destination.txt'); + + bucket.storage.makeAuthorizedRequest_ = function(reqOpts) { + assert.deepEqual(reqOpts.json.sourceObjects, [ + { name: sources[0].name, generation: sources[0].metadata.generation }, + { name: sources[1].name, generation: sources[1].metadata.generation } + ]); + + done(); + }; + + bucket.combine(sources, destination); + }); + + it('should execute the callback', function(done) { + var sources = [bucket.file('1.txt'), bucket.file('2.txt')]; + var destination = 'destination.txt'; + + bucket.storage.makeAuthorizedRequest_ = function(reqOpts, callback) { + callback(); + }; + + bucket.combine(sources, destination, done); + }); + + it('should execute the callback with an error', function(done) { + var sources = [bucket.file('1.txt'), bucket.file('2.txt')]; + var destination = 'destination.txt'; + + var error = new Error('Error.'); + + bucket.storage.makeAuthorizedRequest_ = function(reqOpts, callback) { + callback(error); + }; + + bucket.combine(sources, destination, function(err) { + assert.equal(err, error); + done(); + }); + }); + }); + describe('delete', function() { it('should delete the bucket', function(done) { bucket.makeReq_ = function(method, path, query, body) {