diff --git a/lib/codecs.js b/lib/codecs.js index 583205ee..12cee49b 100644 --- a/lib/codecs.js +++ b/lib/codecs.js @@ -3,7 +3,7 @@ /* eslint-disable new-cap */ var iconv = require('iconv-lite'); -var Transform = require('stream').Transform; +var Transform = require('streamx').Transform; var DEFAULT_ENCODING = require('./constants').DEFAULT_ENCODING; @@ -13,31 +13,28 @@ function Codec(codec, encoding) { this.bomAware = codec.bomAware || false; } - function getEncoder(codec) { return new codec.encoder(null, codec); } -Codec.prototype.encode = function(str) { +Codec.prototype.encode = function (str) { var encoder = getEncoder(this.codec); var buf = encoder.write(str); var end = encoder.end(); return end && end.length > 0 ? Buffer.concat(buf, end) : buf; }; -Codec.prototype.encodeStream = function() { +Codec.prototype.encodeStream = function () { var encoder = getEncoder(this.codec); return new Transform({ - objectMode: false, - decodeStrings: false, - transform: function(str, enc, cb) { + transform: function (str, cb) { var buf = encoder.write(str); if (buf && buf.length) { this.push(buf); } cb(); }, - flush: function(cb) { + flush: function (cb) { var buf = encoder.end(); if (buf && buf.length) { this.push(buf); @@ -47,41 +44,37 @@ Codec.prototype.encodeStream = function() { }); }; - function getDecoder(codec) { return new codec.decoder(null, codec); } -Codec.prototype.decode = function(buf) { +Codec.prototype.decode = function (buf) { var decoder = getDecoder(this.codec); var str = decoder.write(buf); var end = decoder.end(); return end ? str + end : str; }; -Codec.prototype.decodeStream = function() { +Codec.prototype.decodeStream = function () { var decoder = getDecoder(this.codec); return new Transform({ - objectMode: false, - encoding: DEFAULT_ENCODING, - transform: function(buf, enc, cb) { + transform: function (buf, cb) { var str = decoder.write(buf); if (str && str.length) { - this.push(str, DEFAULT_ENCODING); + this.push(str); } cb(); }, - flush: function(cb) { + flush: function (cb) { var str = decoder.end(); if (str && str.length) { - this.push(str, DEFAULT_ENCODING); + this.push(str); } cb(); }, }); }; - var cache = {}; function getCodec(encoding) { @@ -100,7 +93,6 @@ function getCodec(encoding) { return codec; } - // Pre-load default encoding getCodec(DEFAULT_ENCODING); diff --git a/package.json b/package.json index 31bd115d..ca8526f0 100644 --- a/package.json +++ b/package.json @@ -49,6 +49,7 @@ "mocha": "^8.4.0", "nyc": "^15.1.0", "parse-node-version": "^2.0.0", + "readable-stream": "3.6.0", "rimraf": "^3.0.2", "sinon": "^15.0.2" }, diff --git a/test/codecs.js b/test/codecs.js index 38fdabee..1c722036 100644 --- a/test/codecs.js +++ b/test/codecs.js @@ -2,14 +2,6 @@ var fs = require('graceful-fs'); var expect = require('expect'); -var stream = require('stream'); -var concat = require('concat-stream'); - -// TODO: This `from` should be replaced to `node:stream.Readable.from` -// if this package supports only >= v10.17.0 -var from = require('streamx').Readable.from; - -var pipeline = stream.pipeline; var getCodec = require('../lib/codecs'); var DEFAULT_ENCODING = require('../lib/constants').DEFAULT_ENCODING; @@ -23,208 +15,278 @@ var notBomContents = testConstants.notBomContents; var encodedInputPath = testConstants.encodedInputPath; var encodedContents = testConstants.encodedContents; -describe('codecs', function() { - - it('exports a function', function(done) { - expect(typeof getCodec).toEqual('function'); - done(); - }); - - it('returns undefined for unsupported encoding', function(done) { - var codec = getCodec('fubar42'); - expect(codec).toBe(undefined); - done(); - }); - - it('returns a proper codec for default encoding ' + DEFAULT_ENCODING, function(done) { - var codec = getCodec(DEFAULT_ENCODING); - testCodec(codec); - expect(codec.enc).toEqual(DEFAULT_ENCODING); - expect(codec.bomAware).toBe(true); - done(); - }); - - it('returns a proper codec for utf16be', function(done) { - var codec = getCodec('utf16be'); - testCodec(codec); - expect(codec.bomAware).toBe(true); - done(); - }); - - it('can decode bytes from utf16be encoding to a string (buffer)', function(done) { - var codec = getCodec('utf16be'); - var expected = notBomContents.replace('X', 'BE'); - - var result = codec.decode(fs.readFileSync(beNotBomInputPath)); - expect(result).toEqual(expect.anything()); - expect(typeof result).toEqual('string'); - expect(result.slice(2)).toEqual(expected); // Ignore leading garbage - done(); - }); - - it('can decode bytes from utf16be encoding to a string (stream)', function(done) { - var codec = getCodec('utf16be'); - var expected = notBomContents.replace('X', 'BE'); - - function assert(result) { +function suite(moduleName) { + var stream = require(moduleName); + + var from = stream.Readable.from; + + function concatString(fn, timeout) { + var collect = ''; + return new stream.Writable({ + objectMode: true, + write: function (chunk, enc, cb) { + if (typeof enc === 'function') { + cb = enc; + } + setTimeout(function () { + collect = collect + chunk; + cb(); + }, timeout || 1); + }, + final: function (cb) { + if (typeof fn === 'function') { + fn(collect); + } + + cb(); + }, + }); + } + + function concatBuffer(fn, timeout) { + var collect = Buffer.alloc(0); + return new stream.Writable({ + objectMode: true, + write: function (chunk, enc, cb) { + if (typeof enc === 'function') { + cb = enc; + } + setTimeout(function () { + collect = Buffer.concat([collect, chunk]); + cb(); + }, timeout || 1); + }, + final: function (cb) { + if (typeof fn === 'function') { + fn(collect); + } + + cb(); + }, + }); + } + + describe('codecs (with ' + moduleName + ')', function () { + it('exports a function', function (done) { + expect(typeof getCodec).toEqual('function'); + done(); + }); + + it('returns undefined for unsupported encoding', function (done) { + var codec = getCodec('fubar42'); + expect(codec).toBe(undefined); + done(); + }); + + it( + 'returns a proper codec for default encoding ' + DEFAULT_ENCODING, + function (done) { + var codec = getCodec(DEFAULT_ENCODING); + testCodec(codec); + expect(codec.enc).toEqual(DEFAULT_ENCODING); + expect(codec.bomAware).toBe(true); + done(); + } + ); + + it('returns a proper codec for utf16be', function (done) { + var codec = getCodec('utf16be'); + testCodec(codec); + expect(codec.bomAware).toBe(true); + done(); + }); + + it('can decode bytes from utf16be encoding to a string (buffer)', function (done) { + var codec = getCodec('utf16be'); + var expected = notBomContents.replace('X', 'BE'); + + var result = codec.decode(fs.readFileSync(beNotBomInputPath)); expect(result).toEqual(expect.anything()); expect(typeof result).toEqual('string'); expect(result.slice(2)).toEqual(expected); // Ignore leading garbage - } - - pipeline([ - fs.createReadStream(beNotBomInputPath), - codec.decodeStream(), - concat(assert), - ], done); - }); - - it('can encode a string to bytes in utf16be encoding (buffer)', function(done) { - var codec = getCodec('utf16be'); - var expected = fs.readFileSync(beNotBomInputPath); - - var result = codec.encode(notBomContents.replace('X', 'BE')); - expect(result).toEqual(expect.anything()); - expect(typeof result).toEqual('object'); - expect(Buffer.isBuffer(result)).toBe(true); - expect(result.toString()).toEqual(expected.slice(4).toString()); // Ignore leading garbage - done(); - }); - - it('can encode a string to bytes in utf16be encoding (stream)', function(done) { - var codec = getCodec('utf16be'); - var expected = fs.readFileSync(beNotBomInputPath); - - function assert(result) { + done(); + }); + + it('can decode bytes from utf16be encoding to a string (stream)', function (done) { + var codec = getCodec('utf16be'); + var expected = notBomContents.replace('X', 'BE'); + + function assert(result) { + expect(result).toEqual(expect.anything()); + expect(typeof result).toEqual('string'); + expect(result.slice(2)).toEqual(expected); // Ignore leading garbage + } + + stream.pipeline( + [ + fs.createReadStream(beNotBomInputPath), + codec.decodeStream(), + concatString(assert), + ], + done + ); + }); + + it('can encode a string to bytes in utf16be encoding (buffer)', function (done) { + var codec = getCodec('utf16be'); + var expected = fs.readFileSync(beNotBomInputPath); + + var result = codec.encode(notBomContents.replace('X', 'BE')); expect(result).toEqual(expect.anything()); expect(typeof result).toEqual('object'); expect(Buffer.isBuffer(result)).toBe(true); - expect(result.toString()).toEqual(expected.slice(4).toString()); // Ignore leading garbage - } - - pipeline([ - from([notBomContents.replace('X', 'BE')]), - codec.encodeStream(), - concat(assert), - ], done); - }); - - it('returns a proper codec for utf16le', function(done) { - var codec = getCodec('utf16le'); - testCodec(codec); - expect(codec.bomAware).toBe(true); - done(); - }); - - it('can decode bytes from utf16le encoding to a string (buffer)', function(done) { - var codec = getCodec('utf16le'); - var expected = notBomContents.replace('X', 'LE'); - - var result = codec.decode(fs.readFileSync(leNotBomInputPath)); - expect(result).toEqual(expect.anything()); - expect(typeof result).toEqual('string'); - expect(result.slice(2)).toEqual(expected); // Ignore leading garbage - done(); - }); - - it('can decode bytes from utf16le encoding to a string (stream)', function(done) { - var codec = getCodec('utf16le'); - var expected = notBomContents.replace('X', 'LE'); - - function assert(result) { + expect(result).toEqual(expected.slice(4)); // Ignore leading garbage + done(); + }); + + it('can encode a string to bytes in utf16be encoding (stream)', function (done) { + var codec = getCodec('utf16be'); + var expected = fs.readFileSync(beNotBomInputPath); + + function assert(result) { + expect(result).toEqual(expect.anything()); + expect(Buffer.isBuffer(result)).toBe(true); + expect(result).toEqual(expected.slice(4)); // Ignore leading garbage + } + + stream.pipeline( + [ + from([notBomContents.replace('X', 'BE')]), + codec.encodeStream(), + concatBuffer(assert), + ], + done + ); + }); + + it('returns a proper codec for utf16le', function (done) { + var codec = getCodec('utf16le'); + testCodec(codec); + expect(codec.bomAware).toBe(true); + done(); + }); + + it('can decode bytes from utf16le encoding to a string (buffer)', function (done) { + var codec = getCodec('utf16le'); + var expected = notBomContents.replace('X', 'LE'); + + var result = codec.decode(fs.readFileSync(leNotBomInputPath)); expect(result).toEqual(expect.anything()); expect(typeof result).toEqual('string'); expect(result.slice(2)).toEqual(expected); // Ignore leading garbage - } - - pipeline([ - fs.createReadStream(leNotBomInputPath), - codec.decodeStream(), - concat(assert), - ], done); - }); - - it('can encode a string to bytes in utf16le encoding (buffer)', function(done) { - var codec = getCodec('utf16le'); - var expected = fs.readFileSync(leNotBomInputPath); - - var result = codec.encode(notBomContents.replace('X', 'LE')); - expect(result).toEqual(expect.anything()); - expect(typeof result).toEqual('object'); - expect(Buffer.isBuffer(result)).toBe(true); - expect(result.toString()).toEqual(expected.slice(4).toString()); // Ignore leading garbage - done(); - }); - - it('can encode a string to bytes in utf16le encoding (stream)', function(done) { - var codec = getCodec('utf16le'); - var expected = fs.readFileSync(leNotBomInputPath); - - function assert(result) { + done(); + }); + + it('can decode bytes from utf16le encoding to a string (stream)', function (done) { + var codec = getCodec('utf16le'); + var expected = notBomContents.replace('X', 'LE'); + + function assert(result) { + expect(result).toEqual(expect.anything()); + expect(typeof result).toEqual('string'); + expect(result.slice(2)).toEqual(expected); // Ignore leading garbage + } + + stream.pipeline( + [ + fs.createReadStream(leNotBomInputPath), + codec.decodeStream(), + concatString(assert), + ], + done + ); + }); + + it('can encode a string to bytes in utf16le encoding (buffer)', function (done) { + var codec = getCodec('utf16le'); + var expected = fs.readFileSync(leNotBomInputPath); + + var result = codec.encode(notBomContents.replace('X', 'LE')); expect(result).toEqual(expect.anything()); expect(typeof result).toEqual('object'); expect(Buffer.isBuffer(result)).toBe(true); - expect(result.toString()).toEqual(expected.slice(4).toString()); // Ignore leading garbage - } - - pipeline([ - from([notBomContents.replace('X', 'LE')]), - codec.encodeStream(), - concat(assert), - ], done); - }); - - it('returns a proper codec for gb2312', function(done) { - var codec = getCodec('gb2312'); - testCodec(codec); - done(); + expect(result).toEqual(expected.slice(4)); // Ignore leading garbage + done(); + }); + + it('can encode a string to bytes in utf16le encoding (stream)', function (done) { + var codec = getCodec('utf16le'); + var expected = fs.readFileSync(leNotBomInputPath); + + function assert(result) { + expect(result).toEqual(expect.anything()); + expect(Buffer.isBuffer(result)).toBe(true); + expect(result).toEqual(expected.slice(4)); // Ignore leading garbage + } + + stream.pipeline( + [ + from([notBomContents.replace('X', 'LE')]), + codec.encodeStream(), + concatBuffer(assert), + ], + done + ); + }); + + it('returns a proper codec for gb2312', function (done) { + var codec = getCodec('gb2312'); + testCodec(codec); + done(); + }); + + it('can decode bytes from gb2312 encoding to a string (buffer)', function (done) { + var codec = getCodec('gb2312'); + var expected = encodedContents; + + var result = codec.decode(fs.readFileSync(encodedInputPath)); + expect(result).toEqual(expected); + done(); + }); + + it('can decode bytes from gb2312 encoding to a string (stream)', function (done) { + var codec = getCodec('gb2312'); + var expected = encodedContents; + + function assert(result) { + expect(result).toEqual(expected); + } + + stream.pipeline( + [ + fs.createReadStream(encodedInputPath), + codec.decodeStream(), + concatString(assert), + ], + done + ); + }); + + it('can encode a string to bytes in gb2312 encoding (buffer)', function (done) { + var codec = getCodec('gb2312'); + var expected = fs.readFileSync(encodedInputPath); + + var result = codec.encode(encodedContents); + expect(result).toEqual(expected); + done(); + }); + + it('can encode a string to bytes in gb2312 encoding (stream)', function (done) { + var codec = getCodec('gb2312'); + var expected = fs.readFileSync(encodedInputPath); + + function assert(result) { + expect(result).toEqual(expected); + } + + stream.pipeline( + [from([encodedContents]), codec.encodeStream(), concatBuffer(assert)], + done + ); + }); }); +} - it('can decode bytes from gb2312 encoding to a string (buffer)', function(done) { - var codec = getCodec('gb2312'); - var expected = encodedContents; - - var result = codec.decode(fs.readFileSync(encodedInputPath)); - expect(result.toString()).toEqual(expected.toString()); - done(); - }); - - it('can decode bytes from gb2312 encoding to a string (stream)', function(done) { - var codec = getCodec('gb2312'); - var expected = encodedContents; - - function assert(result) { - expect(result.toString()).toEqual(expected.toString()); - } - - pipeline([ - fs.createReadStream(encodedInputPath), - codec.decodeStream(), - concat(assert), - ], done); - }); - - it('can encode a string to bytes in gb2312 encoding (buffer)', function(done) { - var codec = getCodec('gb2312'); - var expected = fs.readFileSync(encodedInputPath); - - var result = codec.encode(encodedContents); - expect(result.toString()).toEqual(expected.toString()); - done(); - }); - - it('can encode a string to bytes in gb2312 encoding (stream)', function(done) { - var codec = getCodec('gb2312'); - var expected = fs.readFileSync(encodedInputPath); - - function assert(result) { - expect(result.toString()).toEqual(expected.toString()); - } - - pipeline([ - from([encodedContents]), - codec.encodeStream(), - concat(assert), - ], done); - }); -}); +suite('stream'); +suite('streamx'); +suite('readable-stream');