Breaking: Convert to ReadableStream (closes #81) (#85)

gulpjs · Feb 21, 2017 · 2af4876 · 2af4876
1 parent 8a84056
commit 2af4876
Show file tree

Hide file tree

Showing 6 changed files with 364 additions and 115 deletions.
diff --git a/README.md b/README.md
@@ -30,7 +30,7 @@ You can pass any combination of globs. One caveat is that you can not only pass
 
 Returns a stream for multiple globs or filters.
 
-### Options
+#### Options
 
 - cwd
   - Default is `process.cwd()`
@@ -42,11 +42,10 @@ Returns a stream for multiple globs or filters.
 - allowEmpty
   - Default is `false`
   - If true, won't emit an error when a glob pointing at a single file fails to match
-- Any through2 related options are documented in [through2][through2-url]
 
 This argument is passed directly to [node-glob][node-glob-url] so check there for more options
 
-### Glob
+#### Glob
 
 ```js
 var stream = gs(['./**/*.js', '!./node_modules/**/*']);
@@ -64,16 +63,66 @@ would not exclude any files, but this would
 gulp.src(['*.js', '!b*.js'])
 ```
 
-## Related
+## Readable Stream
 
-- [globby][globby-url] - Non-streaming `glob` wrapper with support for multiple patterns.
+A ReadableStream interface is available by requiring `glob-stream/readable`.
+
+__Note: This is an advanced feature and you probably don't want to use it.__
+
+### `new ReadableGlobStream(singleGlob, negativesArray, options)`
+
+A constructor for a ReadableStream against a single glob string. An array of globs can be provided as the second argument and will remove matches from the result. Options are passed as the last argument. No argument juggling is provided, so all arguments must be provided (use an empty array if you have no negatives).
+
+#### Options
+
+##### `options.allowEmpty`
+
+Whether or not to error upon an empty singular glob.
+
+Type: `Boolean`
+
+Default: `false` (error upon no match)
+
+##### `options.highWaterMark`
+
+The highWaterMark of the ReadableStream. This is mostly exposed to test backpressure.
+
+Type: `Number`
+
+Default: `16`
+
+##### `options.root`
+
+The root path that the glob is resolved against.
+
+Type: `String`
+
+Default: `undefined` (use the filesystem root)
+
+##### `options.cwd`
+
+The current working directory that the glob is resolved against.
+
+Type: `String`
+
+Default: `process.cwd()`
+
+##### `options.base`
+
+The absolute segment of the glob path that isn't a glob. This value is attached to each glob object and is useful for relative pathing.
+
+Type: `String`
+
+Default: The absolute path segement before a glob starts (see [glob-parent][glob-parent-url])
+
+##### other
+
+Any glob-related options are documented in [node-glob][node-glob-url]. Those options are forwarded verbatim, with the exception of `root` and `ignore`. `root` is pre-resolved and `ignore` is overwritten by the `negativesArray` argument.
 
 ## License
 
 MIT
 
-[globby-url]: https://github.com/sindresorhus/globby
-[through2-url]: https://github.com/rvagg/through2
 [node-glob-url]: https://github.com/isaacs/node-glob
 [glob-parent-url]: https://github.com/es128/glob-parent
 

diff --git a/index.js b/index.js
@@ -1,16 +1,12 @@
 'use strict';
 
-var through2 = require('through2');
 var Combine = require('ordered-read-streams');
 var unique = require('unique-stream');
-
-var glob = require('glob');
 var pumpify = require('pumpify');
 var isNegatedGlob = require('is-negated-glob');
-var globParent = require('glob-parent');
-var resolveGlob = require('to-absolute-glob');
 var extend = require('extend');
-var removeTrailingSeparator = require('remove-trailing-separator');
+
+var GlobStream = require('./readable');
 
 function globStream(globs, opt) {
   if (!opt) {
@@ -46,7 +42,9 @@ function globStream(globs, opt) {
   var positives = [];
   var negatives = [];
 
-  globs.forEach(function(globString, index) {
+  globs.forEach(sortGlobs);
+
+  function sortGlobs(globString, index) {
     if (typeof globString !== 'string') {
       throw new Error('Invalid glob at index ' + index);
     }
@@ -58,17 +56,12 @@ function globStream(globs, opt) {
       index: index,
       glob: glob.pattern,
     });
-  });
+  }
 
   if (positives.length === 0) {
     throw new Error('Missing positive glob');
   }
 
-  // Only one positive glob no need to aggregate
-  if (positives.length === 1) {
-    return streamFromPositive(positives[0]);
-  }
-
   // Create all individual streams
   var streams = positives.map(streamFromPositive);
 
@@ -83,54 +76,8 @@ function globStream(globs, opt) {
       .filter(indexGreaterThan(positive.index))
       .map(toGlob)
       .concat(ignore);
-    return createStream(positive.glob, negativeGlobs, ourOpt);
-  }
-}
-
-function createStream(ourGlob, negatives, opt) {
-  function resolveNegatives(negative) {
-    return resolveGlob(negative, opt);
+    return new GlobStream(positive.glob, negativeGlobs, ourOpt);
   }
-
-  var ourOpt = extend({}, opt);
-  delete ourOpt.root;
-
-  var ourNegatives = negatives.map(resolveNegatives);
-  ourOpt.ignore = ourNegatives;
-
-  // Extract base path from glob
-  var basePath = ourOpt.base || getBasePath(ourGlob, opt);
-
-  // Remove path relativity to make globs make sense
-  ourGlob = resolveGlob(ourGlob, opt);
-
-  // Create globbing stuff
-  var globber = new glob.Glob(ourGlob, ourOpt);
-
-  // Create stream and map events from globber to it
-  var stream = through2.obj(ourOpt);
-
-  var found = false;
-
-  globber.on('error', stream.emit.bind(stream, 'error'));
-  globber.once('end', function() {
-    if (opt.allowEmpty !== true && !found && globIsSingular(globber)) {
-      stream.emit('error',
-        new Error('File not found with singular glob: ' + ourGlob));
-    }
-
-    stream.end();
-  });
-  globber.on('match', function(filename) {
-    found = true;
-
-    stream.write({
-      cwd: opt.cwd,
-      base: basePath,
-      path: removeTrailingSeparator(filename),
-    });
-  });
-  return stream;
 }
 
 function indexGreaterThan(index) {
@@ -143,19 +90,4 @@ function toGlob(obj) {
   return obj.glob;
 }
 
-function globIsSingular(glob) {
-  var globSet = glob.minimatch.set;
-  if (globSet.length !== 1) {
-    return false;
-  }
-
-  return globSet[0].every(function isString(value) {
-    return typeof value === 'string';
-  });
-}
-
-function getBasePath(ourGlob, opt) {
-  return globParent(resolveGlob(ourGlob, opt));
-}
-
 module.exports = globStream;
diff --git a/package.json b/package.json
@@ -31,8 +31,8 @@
     "is-negated-glob": "^1.0.0",
     "ordered-read-streams": "^1.0.0",
     "pumpify": "^1.3.5",
+    "readable-stream": "^2.1.5",
     "remove-trailing-separator": "^1.0.1",
-    "through2": "^0.6.0",
     "to-absolute-glob": "^2.0.0",
     "unique-stream": "^2.0.2"
   },

diff --git a/readable.js b/readable.js
@@ -0,0 +1,117 @@
+'use strict';
+
+var inherits = require('util').inherits;
+
+var glob = require('glob');
+var extend = require('extend');
+var Readable = require('readable-stream').Readable;
+var globParent = require('glob-parent');
+var toAbsoluteGlob = require('to-absolute-glob');
+var removeTrailingSeparator = require('remove-trailing-separator');
+
+var globErrMessage1 = 'File not found with singular glob: ';
+var globErrMessage2 = ' (if this was purposeful, use `allowEmpty` option)';
+
+function getBasePath(ourGlob, opt) {
+  return globParent(toAbsoluteGlob(ourGlob, opt));
+}
+
+function globIsSingular(glob) {
+  var globSet = glob.minimatch.set;
+  if (globSet.length !== 1) {
+    return false;
+  }
+
+  return globSet[0].every(function isString(value) {
+    return typeof value === 'string';
+  });
+}
+
+function GlobStream(ourGlob, negatives, opt) {
+  if (!(this instanceof GlobStream)) {
+    return new GlobStream(ourGlob, negatives, opt);
+  }
+
+  var ourOpt = extend({}, opt);
+
+  Readable.call(this, {
+    objectMode: true,
+    highWaterMark: ourOpt.highWaterMark || 16
+  });
+
+  // Delete `highWaterMark` after inheriting from Readable
+  delete ourOpt.highWaterMark;
+
+  var self = this;
+
+  function resolveNegatives(negative) {
+    return toAbsoluteGlob(negative, ourOpt);
+  }
+
+  var ourNegatives = negatives.map(resolveNegatives);
+  ourOpt.ignore = ourNegatives;
+
+  var cwd = ourOpt.cwd;
+  var allowEmpty = ourOpt.allowEmpty || false;
+
+  // Extract base path from glob
+  var basePath = ourOpt.base || getBasePath(ourGlob, ourOpt);
+
+  // Remove path relativity to make globs make sense
+  ourGlob = toAbsoluteGlob(ourGlob, ourOpt);
+  // Delete `root` after all resolving done
+  delete ourOpt.root;
+
+  var globber = new glob.Glob(ourGlob, ourOpt);
+  this._globber = globber;
+
+  var found = false;
+
+  globber.on('match', function(filepath) {
+    found = true;
+    var obj = {
+      cwd: cwd,
+      base: basePath,
+      path: removeTrailingSeparator(filepath),
+    };
+    if (!self.push(obj)) {
+      globber.pause();
+    }
+  });
+
+  globber.once('end', function() {
+    if (allowEmpty !== true && !found && globIsSingular(globber)) {
+      var err = new Error(globErrMessage1 + ourGlob + globErrMessage2);
+
+      return self.destroy(err);
+    }
+
+    self.push(null);
+  });
+
+  function onError(err) {
+    self.destroy(err);
+  }
+
+  globber.once('error', onError);
+}
+inherits(GlobStream, Readable);
+
+GlobStream.prototype._read = function() {
+  this._globber.resume();
+};
+
+GlobStream.prototype.destroy = function(err) {
+  var self = this;
+
+  this._globber.abort();
+
+  process.nextTick(function() {
+    if (err) {
+      self.emit('error', err);
+    }
+    self.emit('close');
+  });
+};
+
+module.exports = GlobStream;
diff --git a/test/main.js → test/index.js b/test/main.js → test/index.js
@@ -358,25 +358,6 @@ describe('glob-stream', function() {
     ], done);
   });
 
-  it('removes duplicates when used as a Transform stream', function(done) {
-    var expected = {
-      cwd: dir,
-      base: dir + '/fixtures',
-      path: dir + '/fixtures/test.coffee',
-    };
-
-    function assert(pathObjs) {
-      expect(pathObjs.length).toEqual(1);
-      expect(pathObjs[0]).toEqual(expected);
-    }
-
-    pipe([
-      globStream('./fixtures/test.coffee', { cwd: dir }),
-      globStream('./fixtures/*.coffee', { cwd: dir }),
-      concat(assert),
-    ], done);
-  });
-
   it('ignores dotfiles without dot option', function(done) {
     function assert(pathObjs) {
       expect(pathObjs.length).toEqual(0);
@@ -653,19 +634,6 @@ describe('glob-stream', function() {
       concat(assert),
     ], done);
   });
-
-  // TODO: remove this feature?
-  it('passes options to through2', function(done) {
-    function assert(err) {
-      expect(err).toMatch(/Invalid non-string\/buffer chunk/);
-      done();
-    }
-
-    pipe([
-      globStream('./fixtures/stuff/run.dmc', { cwd: dir, objectMode: false }),
-      concat(),
-    ], assert);
-  });
 });
 
 describe('options', function() {