Merge pull request #698 from stephenplusplus/spp--storage-deleteFiles

storage: add deleteFiles()

docs/site/components/docs/docs.html

@@ -83,7 +83,10 @@ <h4 ng-show="method.params">Parameters</h4>
  <tbody>
  <tr
  ng-repeat="param in method.params"
- ng-class="{ 'param-optional': param.optional }">
+ ng-class="{
+ 'param-optional': param.optional,
+ 'param-nullable': param.nullable
+ }">
  <th scope="row" class="param">
  <span ng-if="param.subparam" class="param-parent">
  <div>{{param.name.split('.').slice(0, -1).join('.')}}</div>

docs/site/components/docs/docs.js

@@ -110,9 +110,19 @@ angular
  'bucket',
  'file',
  'job',
- 'table'
+ 'table',
+ 'index',
+ 'document',
+ 'field',
+ 'topic',
+ 'subscription'
  ];
- type = type.replace('=', '');
+
+ var arrayRegex = /Array\.<([^>]+)>/g;
+ type = type.replace('=', '')
+ .replace('?', '')
+ .replace(arrayRegex, '$1[]');
+
  if (CUSTOM_TYPES.indexOf(type.toLowerCase()) > -1) {
  if (types[index - 1]) {
  type = types[index - 1] + '/' + type;

docs/site/css/main.css

@@ -856,17 +856,27 @@ ul {
  padding-top: 0.7em;
 }
 
-.param-optional .param-types {
+.param-optional .param-types,
+.param-nullable .param-types {
  font-style: italic;
 }
 
-.param-optional .param-types:after {
- content: " (optional)";
+.param-optional .param-types:after,
+.param-nullable .param-types:after {
+ display: block;
  color: #aaa;
  font-style: italic;
  font-size: 85%;
 }
 
+.param-optional .param-types:after {
+ content: " (optional)";
+}
+
+.param-nullable .param-types:after {
+ content: " (may be null)";
+}
+
 .method-heading {
  position: relative;
 }

lib/storage/bucket.js

@@ -310,6 +310,124 @@ Bucket.prototype.delete = function(callback) {
  this.makeReq_('DELETE', '', null, true, callback);
 };
 
+/**
+ * Iterate over the bucket's files, calling `file.delete()` on each.
+ *
+ * <strong>This is not an atomic request.</strong> A delete attempt will be made
+ * for each file individually. Any one can fail, in which case only a portion of
+ * the files you intended to be deleted would have.
+ *
+ * Operations are performed in parallel, up to 10 at once. The first error
+ * breaks the loop and will execute the provided callback with it. Specify
+ * `{ force: true }` to suppress the errors until all files have had a chance to
+ * be processed.
+ *
+ * The `query` object passed as the first argument will also be passed to
+ * {module:storage/bucket#getFiles}.
+ *
+ * @param {object=} query - Query object. See {module:storage/bucket#getFiles}
+ * for all of the supported properties.
+ * @param {boolean} query.force - Supress errors until all files have been
+ * processed.
+ * @param {function} callback - The callback function.
+ * @param {?error|?error[]} callback.err - An API error or array of errors from
+ * files that were not able to be deleted.
+ *
+ * @example
+ * //-
+ * // Delete all of the files in the bucket.
+ * //-
+ * bucket.deleteFiles(function(err) {});
+ *
+ * //-
+ * // By default, if a file cannot be deleted, this method will stop deleting
+ * // files from your bucket. You can override this setting with `force: true`.
+ * //-
+ * bucket.deleteFiles({
+ * force: true
+ * }, function(errors) {
+ * // `errors`:
+ * // Array of errors if any occurred, otherwise null.
+ * });
+ *
+ * //-
+ * // The first argument to this method acts as a query to
+ * // {module:storage/bucket#getFiles}. As an example, you can delete files
+ * // which match a prefix.
+ * //-
+ * bucket.deleteFiles({
+ * prefix: 'images/'
+ * }, function(err) {
+ * if (!err) {
+ * // All files in the `images` directory have been deleted.
+ * }
+ * });
+ */
+Bucket.prototype.deleteFiles = function(query, callback) {
+ if (util.is(query, 'function')) {
+ callback = query;
+ query = {};
+ }
+
+ query = query || {};
+
+ var self = this;
+
+ var MAX_PARALLEL_LIMIT = 10;
+ var errors = [];
+
+ // Start deleting files, iteratively fetching more as necessary.
+ deleteFiles(query, function(err) {
+ if (err || errors.length > 0) {
+ callback(err || errors);
+ return;
+ }
+
+ callback(null);
+ });
+
+ function deleteFiles(query, callback) {
+ self.getFiles(query, function(err, files, nextQuery) {
+ if (err) {
+ callback(err);
+ return;
+ }
+
+ // Iterate through each file and attempt to delete it.
+ async.eachLimit(files, MAX_PARALLEL_LIMIT, deleteFile, function(err) {
+ if (err) {
+ callback(err);
+ return;
+ }
+
+ if (nextQuery) {
+ deleteFiles(nextQuery, callback);
+ return;
+ }
+
+ callback();
+ });
+ });
+ }
+
+ function deleteFile(file, callback) {
+ file.delete(function(err) {
+ if (err) {
+ if (query.force) {
+ errors.push(err);
+ callback();
+ return;
+ }
+
+ callback(err);
+ return;
+ }
+
+ callback();
+ });
+ }
+};
+
 /**
  * Create a File object. See {module:storage/file} to see how to handle
  * the different use cases you may have.
@@ -849,7 +967,7 @@ Bucket.prototype.makeAllFilesPublicPrivate_ = function(options, callback) {
  var updatedFiles = [];
 
  // Start processing files, iteratively fetching more as necessary.
- processFiles({}, function (err) {
+ processFiles({}, function(err) {
  if (err || errors.length > 0) {
  callback(err || errors, updatedFiles);
  return;

system-test/storage.js

@@ -42,27 +42,6 @@ var files = {
  }
 };
 
-function deleteVersionedFiles(bucket, callback) {
- bucket.getFiles({ versions: true }, function(err, files) {
- if (err) {
- callback(err);
- return;
- }
-
- async.each(files, deleteFile, callback);
- });
-}
-
-function deleteFiles(bucket, callback) {
- bucket.getFiles(function(err, files) {
- if (err) {
- callback(err);
- return;
- }
- async.map(files, deleteFile, callback);
- });
-}
-
 function deleteFile(file, callback) {
  file.delete(callback);
 }
@@ -100,7 +79,7 @@ describe('storage', function() {
  });
 
  after(function(done) {
- deleteFiles(bucket, function(err) {
+ bucket.deleteFiles(function(err) {
  assert.ifError(err);
  bucket.delete(done);
  });
@@ -221,7 +200,7 @@ describe('storage', function() {
  bucket.acl.default.delete({ entity: 'allUsers' }, next);
  },
  function(next) {
- deleteFiles(bucket, next);
+ bucket.deleteFiles(next);
  }
  ], done);
  });
@@ -278,7 +257,7 @@ describe('storage', function() {
  async.each(files, isFilePrivate, function(err) {
  assert.ifError(err);
 
- deleteFiles(bucket, done);
+ bucket.deleteFiles(done);
  });
  });
  });
@@ -686,7 +665,7 @@ describe('storage', function() {
  var filenames = ['CloudLogo1', 'CloudLogo2', 'CloudLogo3'];
 
  before(function(done) {
- deleteFiles(bucket, function(err) {
+ bucket.deleteFiles(function(err) {
  assert.ifError(err);
 
  var file = bucket.file(filenames[0]);
@@ -750,7 +729,7 @@ describe('storage', function() {
  });
 
  afterEach(function(done) {
- deleteVersionedFiles(versionedBucket, done);
+ versionedBucket.deleteFiles({ versions: true }, done);
  });
 
  after(function(done) {

test/storage/bucket.js

@@ -340,6 +340,125 @@ describe('Bucket', function() {
  });
  });
 
+ describe('deleteFiles', function() {
+ it('should get files from the bucket', function(done) {
+ var query = { a: 'b', c: 'd' };
+
+ bucket.getFiles = function(query_) {
+ assert.deepEqual(query_, query);
+ done();
+ };
+
+ bucket.deleteFiles(query, assert.ifError);
+ });
+
+ it('should process 10 files at a time', function(done) {
+ eachLimit_Override = function(arr, limit) {
+ assert.equal(limit, 10);
+ done();
+ };
+
+ bucket.getFiles = function(query, callback) {
+ callback(null, []);
+ };
+
+ bucket.deleteFiles({}, assert.ifError);
+ });
+
+ it('should delete the files', function(done) {
+ var timesCalled = 0;
+
+ var files = [
+ bucket.file('1'),
+ bucket.file('2')
+ ].map(util.propAssign('delete', function(callback) {
+ timesCalled++;
+ callback();
+ }));
+
+ bucket.getFiles = function(query, callback) {
+ callback(null, files);
+ };
+
+ bucket.deleteFiles({}, function(err) {
+ assert.ifError(err);
+ assert.equal(timesCalled, files.length);
+ done();
+ });
+ });
+
+ it('should get more files if more exist', function(done) {
+ var fakeNextQuery = { a: 'b', c: 'd' };
+
+ bucket.getFiles = function(query, callback) {
+ if (Object.keys(query).length === 0) {
+ // First time through, return a `nextQuery` value.
+ callback(null, [], fakeNextQuery);
+ } else {
+ // Second time through.
+ assert.deepEqual(query, fakeNextQuery);
+ done();
+ }
+ };
+
+ bucket.deleteFiles({}, assert.ifError);
+ });
+
+ it('should execute callback with error from getting files', function(done) {
+ var error = new Error('Error.');
+
+ bucket.getFiles = function(query, callback) {
+ callback(error);
+ };
+
+ bucket.deleteFiles({}, function(err) {
+ assert.strictEqual(err, error);
+ done();
+ });
+ });
+
+ it('should execute callback with error from deleting file', function(done) {
+ var error = new Error('Error.');
+
+ var files = [
+ bucket.file('1'),
+ bucket.file('2')
+ ].map(util.propAssign('delete', function(callback) {
+ callback(error);
+ }));
+
+ bucket.getFiles = function(query, callback) {
+ callback(null, files);
+ };
+
+ bucket.deleteFiles({}, function(err) {
+ assert.strictEqual(err, error);
+ done();
+ });
+ });
+
+ it('should execute callback with queued errors', function(done) {
+ var error = new Error('Error.');
+
+ var files = [
+ bucket.file('1'),
+ bucket.file('2')
+ ].map(util.propAssign('delete', function(callback) {
+ callback(error);
+ }));
+
+ bucket.getFiles = function(query, callback) {
+ callback(null, files);
+ };
+
+ bucket.deleteFiles({ force: true }, function(errs) {
+ assert.strictEqual(errs[0], error);
+ assert.strictEqual(errs[1], error);
+ done();
+ });
+ });
+ });
+
  describe('file', function() {
  var FILE_NAME = 'remote-file-name.jpg';
  var file;