storage: add gzip option

lib/storage/bucket.js

@@ -875,6 +875,8 @@ Bucket.prototype.setMetadata = function(metadata, callback) {
  * will be uploaded to the File object's bucket and under the File object's
  * name. Lastly, when this argument is omitted, the file is uploaded to your
  * bucket using the name of the local file.
+ * @param {boolean} options.gzip - Automatically gzip the file. This will set
+ * `options.metadata.contentEncoding` to `gzip`.
  * @param {object=} options.metadata - Metadata to set for your file.
  * @param {boolean=} options.resumable - Force a resumable upload. (default:
  * true for files larger than 5MB). Read more about resumable uploads
@@ -927,6 +929,17 @@ Bucket.prototype.setMetadata = function(metadata, callback) {
  * });
  *
  * //-
+ * // You can also have a file gzip'd on the fly.
+ * //-
+ * bucket.upload('index.html', { gzip: true }, function(err, file) {
+ * // Your bucket now contains:
+ * // - "index.html" (automatically compressed with gzip)
+ *
+ * // Downloading the file with `file.download` will automatically decode the
+ * // file.
+ * });
+ *
+ * //-
  * // You may also re-use a File object, {module:storage/file}, that references
  * // the file you wish to create or overwrite.
  * //-
@@ -990,7 +1003,8 @@ Bucket.prototype.upload = function(localPath, options, callback) {
  .pipe(newFile.createWriteStream({
  validation: options.validation,
  resumable: resumable,
- metadata: metadata
+ metadata: metadata,
+ gzip: options.gzip
  }))
  .on('error', function(err) {
  callback(err);

lib/storage/file.js

@@ -34,6 +34,7 @@ var request = require('request').defaults({
 });
 var streamEvents = require('stream-events');
 var through = require('through2');
+var zlib = require('zlib');
 
 /**
  * @type {module:storage/acl}
@@ -611,8 +612,10 @@ File.prototype.createReadStream = function(options) {
  * uploaded.
  *
  * @param {object=} options - Configuration object.
- * @param {object=} options.metadata - Set the metadata for this file.
- * @param {boolean=} options.resumable - Force a resumable upload. NOTE: When
+ * @param {boolean} options.gzip - Automatically gzip the file. This will set
+ * `options.metadata.contentEncoding` to `gzip`.
+ * @param {object} options.metadata - Set the metadata for this file.
+ * @param {boolean} options.resumable - Force a resumable upload. NOTE: When
  * working with streams, the file format and size is unknown until it's
  * completely consumed. Because of this, it's best for you to be explicit
  * for what makes sense given your input. Read more about resumable uploads
@@ -642,6 +645,24 @@ File.prototype.createReadStream = function(options) {
  * });
  *
  * //-
+ * // <h4>Uploading a File with gzip compression</h4>
+ * //-
+ * var fs = require('fs');
+ * var htmlFile = myBucket.file('index.html');
+ *
+ * fs.createReadStream('/Users/stephen/site/index.html')
+ * .pipe(htmlFile.createWriteStream({ gzip: true }))
+ * .on('error', function(err) {})
+ * .on('complete', function(metadata) {
+ * // The file upload is complete.
+ * });
+ *
+ * //-
+ * // Downloading the file with `createReadStream` will automatically decode the
+ * // file.
+ * //-
+ *
+ * //-
  * // <h4>Uploading a File with Metadata</h4>
  * //
  * // One last case you may run into is when you want to upload a file to your
@@ -661,13 +682,23 @@ File.prototype.createReadStream = function(options) {
  * }
  * }
  * }))
- * .on('error', function(err) {});
+ * .on('error', function(err) {})
+ * .on('complete', function(metadata) {
+ * // The file upload is complete.
+ * });
  */
 File.prototype.createWriteStream = function(options) {
  options = options || {};
 
  var that = this;
+
+ var gzip = options.gzip;
+
  var metadata = options.metadata || {};
+ if (gzip) {
+ metadata.contentEncoding = 'gzip';
+ }
+
  var validations = ['crc32c', 'md5'];
  var validation;
 
@@ -693,9 +724,11 @@ File.prototype.createWriteStream = function(options) {
  var localCrc32cHash;
  var localMd5Hash = crypto.createHash('md5');
 
- var dup = streamEvents(duplexify());
+ var writableStream = streamEvents(duplexify());
 
- var throughStream = through(function(chunk, enc, next) {
+ var throughStream = through();
+
+ var validationStream = through(function(chunk, enc, next) {
  if (crc32c) {
  localCrc32cHash = crc.calculate(chunk, localCrc32cHash);
  }
@@ -708,25 +741,30 @@ File.prototype.createWriteStream = function(options) {
  next();
  });
 
+ validationStream.on('end', function() {
+ if (crc32c) {
+ localCrc32cHash = new Buffer([localCrc32cHash]).toString('base64');
+ }
+
+ if (md5) {
+ localMd5Hash = localMd5Hash.digest('base64');
+ }
+ });
+
  throughStream
- .on('end', function() {
- if (crc32c) {
- localCrc32cHash = new Buffer([localCrc32cHash]).toString('base64');
- }
 
- if (md5) {
- localMd5Hash = localMd5Hash.digest('base64');
- }
- })
+ .pipe(gzip ? zlib.createGzip() : through())
+
+ .pipe(validationStream)
 
- .pipe(dup)
+ .pipe(writableStream)
 
  // Wait until we've received data to determine what upload technique to use.
  .once('writing', function() {
- if (util.is(options.resumable, 'boolean') && !options.resumable) {
- that.startSimpleUpload_(dup, metadata);
+ if (options.resumable === false) {
+ that.startSimpleUpload_(writableStream, metadata);
  } else {
- that.startResumableUpload_(dup, metadata);
+ that.startResumableUpload_(writableStream, metadata);
  }
  })
 

system-test/storage.js

@@ -42,8 +42,11 @@ var files = {
  big: {
  path: 'system-test/data/three-mb-file.tif'
  },
- gzip: {
+ html: {
  path: 'system-test/data/long-html-file.html'
+ },
+ gzip: {
+ path: 'system-test/data/long-html-file.html.gz'
  }
 };
 
@@ -535,16 +538,41 @@ describe('storage', function() {
  });
  });
 
+ it('should gzip a file on the fly and download it', function(done) {
+ var options = {
+ gzip: true
+ };
+
+ var expectedContents = fs.readFileSync(files.html.path, 'utf-8');
+
+ bucket.upload(files.html.path, options, function(err, file) {
+ assert.ifError(err);
+
+ file.download(function(err, contents) {
+ assert.ifError(err);
+ assert.strictEqual(contents.toString(), expectedContents);
+ file.delete(done);
+ });
+ });
+ });
+
  it('should upload a gzipped file and download it', function(done) {
  var options = {
  metadata: {
  contentEncoding: 'gzip'
  }
  };
 
+ var expectedContents = fs.readFileSync(files.html.path, 'utf-8');
+
  bucket.upload(files.gzip.path, options, function(err, file) {
  assert.ifError(err);
- file.download(done);
+
+ file.download(function(err, contents) {
+ assert.ifError(err);
+ assert.strictEqual(contents.toString(), expectedContents);
+ file.delete(done);
+ });
  });
  });
 

test/storage/bucket.js

@@ -960,6 +960,21 @@ describe('Bucket', function() {
  bucket.upload(filepath, options, assert.ifError);
  });
 
+ it('should allow specifying options.gzip', function(done) {
+ var fakeFile = new FakeFile(bucket, 'file-name');
+ var options = { destination: fakeFile, gzip: true };
+ fakeFile.createWriteStream = function(options) {
+ var ws = new stream.Writable();
+ ws.write = util.noop;
+ setImmediate(function() {
+ assert.strictEqual(options.gzip, true);
+ done();
+ });
+ return ws;
+ };
+ bucket.upload(filepath, options, assert.ifError);
+ });
+
  it('should allow specifying options.resumable', function(done) {
  var fakeFile = new FakeFile(bucket, 'file-name');
  var options = { destination: fakeFile, resumable: false };

test/storage/file.js

@@ -920,6 +920,17 @@ describe('File', function() {
  writable.write('data');
  });
 
+ it('should set metadata.contentEncoding with gzip', function(done) {
+ var writable = file.createWriteStream({ gzip: true });
+
+ file.startResumableUpload_ = function(stream, metadata) {
+ assert.strictEqual(metadata.contentEncoding, 'gzip');
+ done();
+ };
+
+ writable.write('data');
+ });
+
  describe('validation', function() {
  var data = 'test';