Breaking: Add encoding option (closes #23) (#287)

gulpjs · May 14, 2020 · 3c67763 · 3c67763
1 parent 1989914
commit 3c67763
Show file tree

Hide file tree

Showing 23 changed files with 1,106 additions and 45 deletions.
diff --git a/README.md b/README.md
@@ -102,6 +102,17 @@ Type: `Boolean`
 
 Default: `true`
 
+##### `options.encoding`
+
+Optionally transcode from the given encoding. The default is `'utf8'`. We use
+[iconv-lite], please refer to its Wiki for a list of supported encodings. You
+can set this to `false` to avoid any transcoding, and effectively just pass
+around raw binary data.
+
+Type: `String` or `Boolean`
+
+Default: `'utf8'`
+
 ##### `options.sourcemaps`
 
 Enables sourcemap support on files passed through the stream.  Will load inline sourcemaps and resolve sourcemap links from files.
@@ -196,6 +207,17 @@ Type: `Boolean`
 
 Default: `false` (always replace existing contents, if any)
 
+##### `options.encoding`
+
+Optionally transcode to the given encoding. The default is `'utf8'`. We use
+[iconv-lite], please refer to its Wiki for a list of supported encodings. You
+can set this to `false` to avoid any transcoding, and effectively just pass
+around raw binary data.
+
+Type: `String` or `Boolean`
+
+Default: `'utf8'`.
+
 ##### `options.sourcemaps`
 
 Enables sourcemap support on files passed through the stream.  Will write inline soucemaps if specified as `true`.
@@ -329,6 +351,7 @@ scenario.
 [gaze]: https://github.com/shama/gaze
 [glob-watcher]: https://github.com/wearefractal/glob-watcher
 [vinyl]: https://github.com/wearefractal/vinyl
+[iconv-lite]: https://github.com/ashtuchkin/iconv-lite
 
 [downloads-image]: http://img.shields.io/npm/dm/vinyl-fs.svg
 [npm-url]: https://www.npmjs.com/package/vinyl-fs

diff --git a/lib/codecs.js b/lib/codecs.js
@@ -0,0 +1,105 @@
+'use strict';
+
+/* eslint-disable new-cap */
+
+var iconv = require('iconv-lite');
+var through = require('through2');
+
+var DEFAULT_ENCODING = require('./constants').DEFAULT_ENCODING;
+
+function Codec(codec, encoding) {
+  this.codec = codec;
+  this.enc = codec.enc || encoding;
+  this.bomAware = codec.bomAware || false;
+}
+
+
+function getEncoder(codec) {
+  return new codec.encoder(null, codec);
+}
+
+Codec.prototype.encode = function(str) {
+  var encoder = getEncoder(this.codec);
+  var buf = encoder.write(str);
+  var end = encoder.end();
+  return end && end.length > 0 ? Buffer.concat(buf, end) : buf;
+};
+
+Codec.prototype.encodeStream = function() {
+  var encoder = getEncoder(this.codec);
+  return through(
+    { decodeStrings: false },
+    function(str, enc, cb) {
+      var buf = encoder.write(str);
+      if (buf && buf.length) {
+        this.push(buf);
+      }
+      cb();
+    },
+    function(cb) {
+      var buf = encoder.end();
+      if (buf && buf.length) {
+        this.push(buf);
+      }
+      cb();
+    }
+  );
+};
+
+
+function getDecoder(codec) {
+  return new codec.decoder(null, codec);
+}
+
+Codec.prototype.decode = function(buf) {
+  var decoder = getDecoder(this.codec);
+  var str = decoder.write(buf);
+  var end = decoder.end();
+  return end ? str + end : str;
+};
+
+Codec.prototype.decodeStream = function() {
+  var decoder = getDecoder(this.codec);
+  return through(
+    { encoding: DEFAULT_ENCODING },
+    function(buf, enc, cb) {
+      var str = decoder.write(buf);
+      if (str && str.length) {
+        this.push(str, DEFAULT_ENCODING);
+      }
+      cb();
+    },
+    function(cb) {
+      var str = decoder.end();
+      if (str && str.length) {
+        this.push(str, DEFAULT_ENCODING);
+      }
+      cb();
+    }
+  );
+};
+
+
+var cache = {};
+
+function getCodec(encoding) {
+  var codec = cache[encoding];
+  if (!!codec || !encoding || cache.hasOwnProperty(encoding)) {
+    return codec;
+  }
+
+  try {
+    codec = new Codec(iconv.getCodec(encoding), encoding);
+  } catch (err) {
+    // Unsupported codec
+  }
+
+  cache[encoding] = codec;
+  return codec;
+}
+
+
+// Pre-load default encoding
+getCodec(DEFAULT_ENCODING);
+
+module.exports = getCodec;
diff --git a/lib/constants.js b/lib/constants.js
@@ -3,4 +3,5 @@
 module.exports = {
   MASK_MODE: parseInt('7777', 8),
   DEFAULT_FILE_MODE: parseInt('0666', 8),
+  DEFAULT_ENCODING: 'utf8',
 };
diff --git a/lib/dest/options.js b/lib/dest/options.js
@@ -1,5 +1,7 @@
 'use strict';
 
+var DEFAULT_ENCODING = require('../constants').DEFAULT_ENCODING;
+
 var config = {
   cwd: {
     type: 'string',
@@ -22,6 +24,10 @@ var config = {
     type: 'boolean',
     default: false,
   },
+  encoding: {
+    type: ['string', 'boolean'],
+    default: DEFAULT_ENCODING,
+  },
   sourcemaps: {
     type: ['string', 'boolean'],
     default: false,

diff --git a/lib/dest/write-contents/write-buffer.js b/lib/dest/write-contents/write-buffer.js
@@ -1,18 +1,34 @@
 'use strict';
 
 var fo = require('../../file-operations');
+var getCodec = require('../../codecs');
+var DEFAULT_ENCODING = require('../../constants').DEFAULT_ENCODING;
 
 function writeBuffer(file, optResolver, onWritten) {
   var flags = fo.getFlags({
     overwrite: optResolver.resolve('overwrite', file),
     append: optResolver.resolve('append', file),
   });
+
+  var encoding = optResolver.resolve('encoding', file);
+  var codec = getCodec(encoding);
+  if (encoding && !codec) {
+    return onWritten(new Error('Unsupported encoding: ' + encoding));
+  }
+
   var opt = {
     mode: file.stat.mode,
     flags: flags,
   };
 
-  fo.writeFile(file.path, file.contents, opt, onWriteFile);
+  var contents = file.contents;
+
+  if (encoding && codec.enc !== DEFAULT_ENCODING) {
+    contents = getCodec(DEFAULT_ENCODING).decode(contents);
+    contents = codec.encode(contents);
+  }
+
+  fo.writeFile(file.path, contents, opt, onWriteFile);
 
   function onWriteFile(writeErr, fd) {
     if (writeErr) {

diff --git a/lib/dest/write-contents/write-stream.js b/lib/dest/write-contents/write-stream.js
@@ -1,13 +1,22 @@
 'use strict';
 
 var fo = require('../../file-operations');
+var getCodec = require('../../codecs');
+var DEFAULT_ENCODING = require('../../constants').DEFAULT_ENCODING;
 var readStream = require('../../src/read-contents/read-stream');
 
 function writeStream(file, optResolver, onWritten) {
   var flags = fo.getFlags({
     overwrite: optResolver.resolve('overwrite', file),
     append: optResolver.resolve('append', file),
   });
+
+  var encoding = optResolver.resolve('encoding', file);
+  var codec = getCodec(encoding);
+  if (encoding && !codec) {
+    return onWritten(new Error('Unsupported encoding: ' + encoding));
+  }
+
   var opt = {
     mode: file.stat.mode,
     // TODO: need to test this
@@ -17,12 +26,20 @@ function writeStream(file, optResolver, onWritten) {
   // TODO: is this the best API?
   var outStream = fo.createWriteStream(file.path, opt, onFlush);
 
+  var contents = file.contents;
+
+  if (encoding && encoding.enc !== DEFAULT_ENCODING) {
+    contents = contents
+      .pipe(getCodec(DEFAULT_ENCODING).decodeStream())
+      .pipe(codec.encodeStream());
+  }
+
   file.contents.once('error', onComplete);
   outStream.once('error', onComplete);
   outStream.once('finish', onComplete);
 
   // TODO: should this use a clone?
-  file.contents.pipe(outStream);
+  contents.pipe(outStream);
 
   function onComplete(streamErr) {
     // Cleanup event handlers before closing
@@ -45,8 +62,17 @@ function writeStream(file, optResolver, onWritten) {
     file.contents.removeListener('error', onComplete);
 
     // TODO: this is doing sync stuff & the callback seems unnecessary
-    // TODO: Replace the contents stream or use a clone?
-    readStream(file, complete);
+    readStream(file, { resolve: resolve }, complete);
+
+    function resolve(key) {
+      if (key === 'encoding') {
+        return encoding;
+      }
+      if (key === 'removeBOM') {
+        return false;
+      }
+      throw new Error('Eek! stub resolver doesn\'t have ' + key);
+    }
 
     function complete() {
       if (typeof fd !== 'number') {

diff --git a/lib/src/options.js b/lib/src/options.js
@@ -1,5 +1,7 @@
 'use strict';
 
+var DEFAULT_ENCODING = require('../constants').DEFAULT_ENCODING;
+
 var config = {
   buffer: {
     type: 'boolean',
@@ -16,6 +18,10 @@ var config = {
     type: 'boolean',
     default: true,
   },
+  encoding: {
+    type: ['string', 'boolean'],
+    default: DEFAULT_ENCODING,
+  },
   sourcemaps: {
     type: 'boolean',
     default: false,

diff --git a/lib/src/read-contents/index.js b/lib/src/read-contents/index.js
@@ -42,7 +42,7 @@ function readContents(optResolver) {
       if (readErr) {
         return callback(readErr);
       }
-      return callback(null, file);
+      callback(null, file);
     }
   }
 

diff --git a/lib/src/read-contents/read-buffer.js b/lib/src/read-contents/read-buffer.js
@@ -3,21 +3,39 @@
 var fs = require('graceful-fs');
 var removeBomBuffer = require('remove-bom-buffer');
 
+var getCodec = require('../../codecs');
+var DEFAULT_ENCODING = require('../../constants').DEFAULT_ENCODING;
+
 function bufferFile(file, optResolver, onRead) {
+  var encoding = optResolver.resolve('encoding', file);
+  var codec = getCodec(encoding);
+  if (encoding && !codec) {
+    return onRead(new Error('Unsupported encoding: ' + encoding));
+  }
+
   fs.readFile(file.path, onReadFile);
 
-  function onReadFile(readErr, data) {
+  function onReadFile(readErr, contents) {
     if (readErr) {
       return onRead(readErr);
     }
 
-    var removeBOM = optResolver.resolve('removeBOM', file);
-    if (removeBOM) {
-      file.contents = removeBomBuffer(data);
-    } else {
-      file.contents = data;
+    if (encoding) {
+      var removeBOM = codec.bomAware && optResolver.resolve('removeBOM', file);
+
+      if (codec.enc !== DEFAULT_ENCODING) {
+        contents = codec.decode(contents);
+        removeBOM = removeBOM && contents[0] === '\ufeff';
+        contents = getCodec(DEFAULT_ENCODING).encode(contents);
+      }
+
+      if (removeBOM) {
+        contents = removeBomBuffer(contents);
+      }
     }
 
+    file.contents = contents;
+
     onRead();
   }
 }

diff --git a/lib/src/read-contents/read-stream.js b/lib/src/read-contents/read-stream.js
@@ -3,23 +3,34 @@
 var fs = require('graceful-fs');
 var removeBomStream = require('remove-bom-stream');
 var lazystream = require('lazystream');
-var createResolver = require('resolve-options');
+
+var getCodec = require('../../codecs');
+var DEFAULT_ENCODING = require('../../constants').DEFAULT_ENCODING;
 
 function streamFile(file, optResolver, onRead) {
-  if (typeof optResolver === 'function') {
-    onRead = optResolver;
-    optResolver = createResolver();
+  var encoding = optResolver.resolve('encoding', file);
+  var codec = getCodec(encoding);
+  if (encoding && !codec) {
+    return onRead(new Error('Unsupported encoding: ' + encoding));
   }
 
   var filePath = file.path;
 
-  var removeBOM = optResolver.resolve('removeBOM', file);
-
   file.contents = new lazystream.Readable(function() {
     var contents = fs.createReadStream(filePath);
 
-    if (removeBOM) {
-      return contents.pipe(removeBomStream());
+    if (encoding) {
+      var removeBOM = codec.bomAware && optResolver.resolve('removeBOM', file);
+
+      if (codec.enc !== DEFAULT_ENCODING) {
+        contents = contents
+          .pipe(codec.decodeStream())
+          .pipe(getCodec(DEFAULT_ENCODING).encodeStream());
+      }
+
+      if (removeBOM) {
+        contents = contents.pipe(removeBomStream());
+      }
     }
 
     return contents;