From 2af4876478cda48f9e975fa9fee3f9e265a44ed7 Mon Sep 17 00:00:00 2001 From: Blaine Bublitz Date: Mon, 13 Feb 2017 15:14:51 -0700 Subject: [PATCH] Breaking: Convert to ReadableStream (closes #81) (#85) --- README.md | 63 +++++++++++-- index.js | 82 ++--------------- package.json | 2 +- readable.js | 117 ++++++++++++++++++++++++ test/{main.js => index.js} | 32 ------- test/readable.js | 183 +++++++++++++++++++++++++++++++++++++ 6 files changed, 364 insertions(+), 115 deletions(-) create mode 100644 readable.js rename test/{main.js => index.js} (95%) create mode 100644 test/readable.js diff --git a/README.md b/README.md index 12cba3a..567b165 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,7 @@ You can pass any combination of globs. One caveat is that you can not only pass Returns a stream for multiple globs or filters. -### Options +#### Options - cwd - Default is `process.cwd()` @@ -42,11 +42,10 @@ Returns a stream for multiple globs or filters. - allowEmpty - Default is `false` - If true, won't emit an error when a glob pointing at a single file fails to match -- Any through2 related options are documented in [through2][through2-url] This argument is passed directly to [node-glob][node-glob-url] so check there for more options -### Glob +#### Glob ```js var stream = gs(['./**/*.js', '!./node_modules/**/*']); @@ -64,16 +63,66 @@ would not exclude any files, but this would gulp.src(['*.js', '!b*.js']) ``` -## Related +## Readable Stream -- [globby][globby-url] - Non-streaming `glob` wrapper with support for multiple patterns. +A ReadableStream interface is available by requiring `glob-stream/readable`. + +__Note: This is an advanced feature and you probably don't want to use it.__ + +### `new ReadableGlobStream(singleGlob, negativesArray, options)` + +A constructor for a ReadableStream against a single glob string. An array of globs can be provided as the second argument and will remove matches from the result. Options are passed as the last argument. No argument juggling is provided, so all arguments must be provided (use an empty array if you have no negatives). + +#### Options + +##### `options.allowEmpty` + +Whether or not to error upon an empty singular glob. + +Type: `Boolean` + +Default: `false` (error upon no match) + +##### `options.highWaterMark` + +The highWaterMark of the ReadableStream. This is mostly exposed to test backpressure. + +Type: `Number` + +Default: `16` + +##### `options.root` + +The root path that the glob is resolved against. + +Type: `String` + +Default: `undefined` (use the filesystem root) + +##### `options.cwd` + +The current working directory that the glob is resolved against. + +Type: `String` + +Default: `process.cwd()` + +##### `options.base` + +The absolute segment of the glob path that isn't a glob. This value is attached to each glob object and is useful for relative pathing. + +Type: `String` + +Default: The absolute path segement before a glob starts (see [glob-parent][glob-parent-url]) + +##### other + +Any glob-related options are documented in [node-glob][node-glob-url]. Those options are forwarded verbatim, with the exception of `root` and `ignore`. `root` is pre-resolved and `ignore` is overwritten by the `negativesArray` argument. ## License MIT -[globby-url]: https://github.com/sindresorhus/globby -[through2-url]: https://github.com/rvagg/through2 [node-glob-url]: https://github.com/isaacs/node-glob [glob-parent-url]: https://github.com/es128/glob-parent diff --git a/index.js b/index.js index 7a8aa96..3c33a57 100644 --- a/index.js +++ b/index.js @@ -1,16 +1,12 @@ 'use strict'; -var through2 = require('through2'); var Combine = require('ordered-read-streams'); var unique = require('unique-stream'); - -var glob = require('glob'); var pumpify = require('pumpify'); var isNegatedGlob = require('is-negated-glob'); -var globParent = require('glob-parent'); -var resolveGlob = require('to-absolute-glob'); var extend = require('extend'); -var removeTrailingSeparator = require('remove-trailing-separator'); + +var GlobStream = require('./readable'); function globStream(globs, opt) { if (!opt) { @@ -46,7 +42,9 @@ function globStream(globs, opt) { var positives = []; var negatives = []; - globs.forEach(function(globString, index) { + globs.forEach(sortGlobs); + + function sortGlobs(globString, index) { if (typeof globString !== 'string') { throw new Error('Invalid glob at index ' + index); } @@ -58,17 +56,12 @@ function globStream(globs, opt) { index: index, glob: glob.pattern, }); - }); + } if (positives.length === 0) { throw new Error('Missing positive glob'); } - // Only one positive glob no need to aggregate - if (positives.length === 1) { - return streamFromPositive(positives[0]); - } - // Create all individual streams var streams = positives.map(streamFromPositive); @@ -83,54 +76,8 @@ function globStream(globs, opt) { .filter(indexGreaterThan(positive.index)) .map(toGlob) .concat(ignore); - return createStream(positive.glob, negativeGlobs, ourOpt); - } -} - -function createStream(ourGlob, negatives, opt) { - function resolveNegatives(negative) { - return resolveGlob(negative, opt); + return new GlobStream(positive.glob, negativeGlobs, ourOpt); } - - var ourOpt = extend({}, opt); - delete ourOpt.root; - - var ourNegatives = negatives.map(resolveNegatives); - ourOpt.ignore = ourNegatives; - - // Extract base path from glob - var basePath = ourOpt.base || getBasePath(ourGlob, opt); - - // Remove path relativity to make globs make sense - ourGlob = resolveGlob(ourGlob, opt); - - // Create globbing stuff - var globber = new glob.Glob(ourGlob, ourOpt); - - // Create stream and map events from globber to it - var stream = through2.obj(ourOpt); - - var found = false; - - globber.on('error', stream.emit.bind(stream, 'error')); - globber.once('end', function() { - if (opt.allowEmpty !== true && !found && globIsSingular(globber)) { - stream.emit('error', - new Error('File not found with singular glob: ' + ourGlob)); - } - - stream.end(); - }); - globber.on('match', function(filename) { - found = true; - - stream.write({ - cwd: opt.cwd, - base: basePath, - path: removeTrailingSeparator(filename), - }); - }); - return stream; } function indexGreaterThan(index) { @@ -143,19 +90,4 @@ function toGlob(obj) { return obj.glob; } -function globIsSingular(glob) { - var globSet = glob.minimatch.set; - if (globSet.length !== 1) { - return false; - } - - return globSet[0].every(function isString(value) { - return typeof value === 'string'; - }); -} - -function getBasePath(ourGlob, opt) { - return globParent(resolveGlob(ourGlob, opt)); -} - module.exports = globStream; diff --git a/package.json b/package.json index 822d841..766cf57 100644 --- a/package.json +++ b/package.json @@ -31,8 +31,8 @@ "is-negated-glob": "^1.0.0", "ordered-read-streams": "^1.0.0", "pumpify": "^1.3.5", + "readable-stream": "^2.1.5", "remove-trailing-separator": "^1.0.1", - "through2": "^0.6.0", "to-absolute-glob": "^2.0.0", "unique-stream": "^2.0.2" }, diff --git a/readable.js b/readable.js new file mode 100644 index 0000000..8d67130 --- /dev/null +++ b/readable.js @@ -0,0 +1,117 @@ +'use strict'; + +var inherits = require('util').inherits; + +var glob = require('glob'); +var extend = require('extend'); +var Readable = require('readable-stream').Readable; +var globParent = require('glob-parent'); +var toAbsoluteGlob = require('to-absolute-glob'); +var removeTrailingSeparator = require('remove-trailing-separator'); + +var globErrMessage1 = 'File not found with singular glob: '; +var globErrMessage2 = ' (if this was purposeful, use `allowEmpty` option)'; + +function getBasePath(ourGlob, opt) { + return globParent(toAbsoluteGlob(ourGlob, opt)); +} + +function globIsSingular(glob) { + var globSet = glob.minimatch.set; + if (globSet.length !== 1) { + return false; + } + + return globSet[0].every(function isString(value) { + return typeof value === 'string'; + }); +} + +function GlobStream(ourGlob, negatives, opt) { + if (!(this instanceof GlobStream)) { + return new GlobStream(ourGlob, negatives, opt); + } + + var ourOpt = extend({}, opt); + + Readable.call(this, { + objectMode: true, + highWaterMark: ourOpt.highWaterMark || 16 + }); + + // Delete `highWaterMark` after inheriting from Readable + delete ourOpt.highWaterMark; + + var self = this; + + function resolveNegatives(negative) { + return toAbsoluteGlob(negative, ourOpt); + } + + var ourNegatives = negatives.map(resolveNegatives); + ourOpt.ignore = ourNegatives; + + var cwd = ourOpt.cwd; + var allowEmpty = ourOpt.allowEmpty || false; + + // Extract base path from glob + var basePath = ourOpt.base || getBasePath(ourGlob, ourOpt); + + // Remove path relativity to make globs make sense + ourGlob = toAbsoluteGlob(ourGlob, ourOpt); + // Delete `root` after all resolving done + delete ourOpt.root; + + var globber = new glob.Glob(ourGlob, ourOpt); + this._globber = globber; + + var found = false; + + globber.on('match', function(filepath) { + found = true; + var obj = { + cwd: cwd, + base: basePath, + path: removeTrailingSeparator(filepath), + }; + if (!self.push(obj)) { + globber.pause(); + } + }); + + globber.once('end', function() { + if (allowEmpty !== true && !found && globIsSingular(globber)) { + var err = new Error(globErrMessage1 + ourGlob + globErrMessage2); + + return self.destroy(err); + } + + self.push(null); + }); + + function onError(err) { + self.destroy(err); + } + + globber.once('error', onError); +} +inherits(GlobStream, Readable); + +GlobStream.prototype._read = function() { + this._globber.resume(); +}; + +GlobStream.prototype.destroy = function(err) { + var self = this; + + this._globber.abort(); + + process.nextTick(function() { + if (err) { + self.emit('error', err); + } + self.emit('close'); + }); +}; + +module.exports = GlobStream; diff --git a/test/main.js b/test/index.js similarity index 95% rename from test/main.js rename to test/index.js index 446e5f1..d11e9f8 100644 --- a/test/main.js +++ b/test/index.js @@ -358,25 +358,6 @@ describe('glob-stream', function() { ], done); }); - it('removes duplicates when used as a Transform stream', function(done) { - var expected = { - cwd: dir, - base: dir + '/fixtures', - path: dir + '/fixtures/test.coffee', - }; - - function assert(pathObjs) { - expect(pathObjs.length).toEqual(1); - expect(pathObjs[0]).toEqual(expected); - } - - pipe([ - globStream('./fixtures/test.coffee', { cwd: dir }), - globStream('./fixtures/*.coffee', { cwd: dir }), - concat(assert), - ], done); - }); - it('ignores dotfiles without dot option', function(done) { function assert(pathObjs) { expect(pathObjs.length).toEqual(0); @@ -653,19 +634,6 @@ describe('glob-stream', function() { concat(assert), ], done); }); - - // TODO: remove this feature? - it('passes options to through2', function(done) { - function assert(err) { - expect(err).toMatch(/Invalid non-string\/buffer chunk/); - done(); - } - - pipe([ - globStream('./fixtures/stuff/run.dmc', { cwd: dir, objectMode: false }), - concat(), - ], assert); - }); }); describe('options', function() { diff --git a/test/readable.js b/test/readable.js new file mode 100644 index 0000000..fcf61c2 --- /dev/null +++ b/test/readable.js @@ -0,0 +1,183 @@ +'use strict'; + +var expect = require('expect'); +var miss = require('mississippi'); + +var stream = require('../readable'); + +// Need to wrap this to cause node-glob to emit an error +var fs = require('fs'); + +function deWindows(p) { + return p.replace(/\\/g, '/'); +} + +var pipe = miss.pipe; +var concat = miss.concat; +var through = miss.through; + +var dir = deWindows(__dirname); + +describe('readable stream', function() { + + it('emits an error if there are no matches', function(done) { + function assert(err) { + expect(err.message).toMatch(/^File not found with singular glob/g); + done(); + } + + pipe([ + stream('notfound', [], { cwd: dir }), + concat(), + ], assert); + }); + + it('throws an error if you try to write to it', function(done) { + var gs = stream('notfound', [], { cwd: dir }); + + try { + gs.write({}); + } catch (err) { + expect(err).toExist(); + done(); + } + }); + + it('does not throw an error if you push to it', function(done) { + var stub = { + cwd: dir, + base: dir, + path: dir, + }; + + var gs = stream('./fixtures/test.coffee', [], { cwd: dir }); + + gs.push(stub); + + function assert(pathObjs) { + expect(pathObjs.length).toEqual(2); + expect(pathObjs[0]).toEqual(stub); + } + + pipe([ + gs, + concat(assert), + ], done); + }); + + it('accepts a file path', function(done) { + var expected = { + cwd: dir, + base: dir + '/fixtures', + path: dir + '/fixtures/test.coffee', + }; + + function assert(pathObjs) { + expect(pathObjs.length).toBe(1); + expect(pathObjs[0]).toMatch(expected); + } + + pipe([ + stream('./fixtures/test.coffee', [], { cwd: dir }), + concat(assert), + ], done); + }); + + it('accepts a glob', function(done) { + var expected = [ + { + cwd: dir, + base: dir + '/fixtures', + path: dir + '/fixtures/has (parens)/test.dmc', + }, + { + cwd: dir, + base: dir + '/fixtures', + path: dir + '/fixtures/stuff/run.dmc', + }, + { + cwd: dir, + base: dir + '/fixtures', + path: dir + '/fixtures/stuff/test.dmc', + }, + ]; + + function assert(pathObjs) { + expect(pathObjs.length).toBe(3); + expect(pathObjs).toEqual(expected); + } + + pipe([ + stream('./fixtures/**/*.dmc', [], { cwd: dir }), + concat(assert), + ], done); + }); + + it('pauses the globber upon backpressure', function(done) { + var gs = stream('./fixtures/**/*.dmc', [], { cwd: dir, highWaterMark: 1 }); + + var spy = expect.spyOn(gs._globber, 'pause').andCallThrough(); + + function waiter(pathObj, _, cb) { + setTimeout(function() { + cb(null, pathObj); + }, 500); + } + + function assert(pathObjs) { + expect(pathObjs.length).toEqual(3); + expect(spy.calls.length).toEqual(2); + spy.restore(); + } + + pipe([ + gs, + through.obj({ highWaterMark: 1 }, waiter), + concat(assert), + ], done); + }); + + it('destroys the stream with an error if no match is found', function(done) { + var gs = stream('notfound', []); + + var spy = expect.spyOn(gs, 'destroy').andCallThrough(); + + function assert(err) { + spy.restore(); + expect(spy).toHaveBeenCalledWith(err); + expect(err).toMatch(/File not found with singular glob/); + done(); + } + + pipe([ + gs, + concat(), + ], assert); + }); + + it('destroys the stream if node-glob errors', function(done) { + var expectedError = new Error('Stubbed error'); + + var gs = stream('./fixtures/**/*.dmc', [], { cwd: dir, silent: true }); + + function stubError(dirpath, cb) { + cb(expectedError); + } + + var spy = expect.spyOn(gs, 'destroy').andCallThrough(); + var fsStub = expect.spyOn(fs, 'readdir').andCall(stubError); + + function assert(err) { + fsStub.restore(); + spy.restore(); + expect(spy).toHaveBeenCalledWith(err); + expect(err).toBe(expectedError); + done(); + } + + pipe([ + gs, + concat(), + ], assert); + }); +});