port babel loader fs cache as the default caching engine

fs-cache.js

@@ -0,0 +1,196 @@
+/**
+ * Filesystem cache
+ *
+ * Given a file and a transform function, cache the result into files
+ * or retrieve the previously cached files if the given file is already known.
+ *
+ * @see https://github.com/babel/babel-loader/issues/34
+ * @see https://github.com/babel/babel-loader/pull/41
+ * @see https://github.com/babel/babel-loader/blob/master/src/fs-cache.js
+ */
+var crypto = require("crypto")
+var mkdirp = require("mkdirp")
+var findCacheDir = require("find-cache-dir")
+var fs = require("fs")
+var os = require("os")
+var path = require("path")
+var zlib = require("zlib")
+
+var defaultCacheDirectory = null // Lazily instantiated when needed
+
+/**
+ * Read the contents from the compressed file.
+ *
+ * @async
+ * @params {String} filename
+ * @params {Function} callback
+ */
+var read = function(filename, callback) {
+  return fs.readFile(filename, function(err, data) {
+    if (err) {
+      return callback(err)
+    }
+
+    return zlib.gunzip(data, function(err, content) {
+      var result = {}
+
+      if (err) {
+        return callback(err)
+      }
+
+      try {
+        result = JSON.parse(content)
+      } 
+      catch (e) {
+        return callback(e)
+      }
+
+      return callback(null, result)
+    })
+  })
+}
+
+/**
+ * Write contents into a compressed file.
+ *
+ * @async
+ * @params {String} filename
+ * @params {String} result
+ * @params {Function} callback
+ */
+var write = function(filename, result, callback) {
+  var content = JSON.stringify(result)
+
+  return zlib.gzip(content, function(err, data) {
+    if (err) {
+      return callback(err)
+    }
+
+    return fs.writeFile(filename, data, callback)
+  })
+}
+
+/**
+ * Build the filename for the cached file
+ *
+ * @params {String} source  File source code
+ * @params {Object} options Options used
+ *
+ * @return {String}
+ */
+var filename = function(source, identifier, options) {
+  var hash = crypto.createHash("SHA1")
+  var contents = JSON.stringify({
+    source: source,
+    options: options,
+    identifier: identifier,
+  })
+
+  hash.end(contents)
+
+  return hash.read().toString("hex") + ".json.gz"
+}
+
+/**
+ * Handle the cache
+ *
+ * @params {String} directory
+ * @params {Object} params
+ * @params {Function} callback
+ */
+var handleCache = function(directory, params, callback) {
+  var source = params.source
+  var options = params.options || {}
+  var transform = params.transform
+  var identifier = params.identifier
+  var shouldFallback = typeof params.directory !== "string" &&
+    directory !== os.tmpdir()
+
+  // Make sure the directory exists.
+  mkdirp(directory, function(err) {
+    // Fallback to tmpdir if node_modules folder not writable
+    if (err)
+      return shouldFallback
+        ? handleCache(os.tmpdir(), params, callback)
+        : callback(err)
+
+    var file = path.join(directory, filename(source, identifier, options))
+
+    return read(file, function(err, content) {
+      var result = {}
+      // No errors mean that the file was previously cached
+      // we just need to return it
+      if (!err) return callback(null, content)
+
+      // Otherwise just transform the file
+      // return it to the user asap and write it in cache
+      try {
+        result = transform(source, options)
+      } 
+      catch (error) {
+        return callback(error)
+      }
+
+      return write(file, result, function(err) {
+        // Fallback to tmpdir if node_modules folder not writable
+        if (err)
+          return shouldFallback
+            ? handleCache(os.tmpdir(), params, callback)
+            : callback(err)
+
+        callback(null, result)
+      })
+    })
+  })
+}
+
+/**
+ * Retrieve file from cache, or create a new one for future reads
+ *
+ * @async
+ * @param  {Object}   params
+ * @param  {String}   params.directory  Directory to store cached files
+ * @param  {String}   params.identifier Unique identifier to bust cache
+ * @param  {String}   params.source   Original contents of the file to be cached
+ * @param  {Object}   params.options  Options to be given to the transform fn
+ * @param  {Function} params.transform  Function that will transform the
+ *                                      original file and whose result will be
+ *                                      cached
+ *
+ * @param  {Function<err, result>} callback
+ *
+ * @example
+ *
+ *   cache({
+ *     directory: '.tmp/cache',
+ *     identifier: 'babel-loader-cachefile',
+ *     source: *source code from file*,
+ *     options: {
+ *       experimental: true,
+ *       runtime: true
+ *     },
+ *     transform: function(source, options) {
+ *       var content = *do what you need with the source*
+ *       return content
+ *     }
+ *   }, function(err, result) {
+ *
+ *   })
+ */
+
+module.exports = function(params, callback) {
+  var directory
+
+  if (typeof params.directory === "string") {
+    directory = params.directory
+  } 
+  else {
+    if (defaultCacheDirectory === null) {
+      defaultCacheDirectory = findCacheDir({
+        name: "eslint-loader",
+      }) || os.tmpdir()
+    }
+    directory = defaultCacheDirectory
+  }
+  handleCache(directory, params, callback)
+}

index.js

@@ -1,76 +1,21 @@
 var eslint = require("eslint")
 var assign = require("object-assign")
 var loaderUtils = require("loader-utils")
-var crypto = require("crypto")
-var fs = require("fs")
-var findCacheDir = require("find-cache-dir")
 var objectHash = require("object-hash")
-var os = require("os")
+var pkg = require("./package.json")
+var cache = require("./fs-cache")
 
 var engines = {}
-var rules = {}
-var cache = null
-var cachePath = null
 
 /**
- * linter
+ * printLinterOutput
  *
- * @param {String|Buffer} input JavaScript string
+ * @param {Object} eslint.executeOnText return value
  * @param {Object} config eslint configuration
  * @param {Object} webpack webpack instance
  * @return {void}
  */
-function lint(input, config, webpack) {
-  var resourcePath = webpack.resourcePath
-  var cwd = process.cwd()
-
-  // remove cwd from resource path in case webpack has been started from project
-  // root, to allow having relative paths in .eslintignore
-  if (resourcePath.indexOf(cwd) === 0) {
-    resourcePath = resourcePath.substr(cwd.length + 1)
-  }
-
-  // get engine
-  var configHash = objectHash(config)
-  var engine = engines[configHash]
-  var rulesHash = rules[configHash]
-
-  var res
-  // If cache is enable and the data are the same as in the cache, just
-  // use them
-  if (config.cache) {
-    // just get rules hash once per engine for performance reasons
-    if (!rulesHash) {
-      rulesHash = objectHash(engine.getConfigForFile(resourcePath))
-      rules[configHash] = rulesHash
-    }
-    var inputMD5 = crypto.createHash("md5").update(input).digest("hex")
-    if (
-      cache[resourcePath] &&
-      cache[resourcePath].hash === inputMD5 &&
-      cache[resourcePath].rules === rulesHash
-    ) {
-      res = cache[resourcePath].res
-    }
-  }
-
-  // Re-lint the text if the cache off or miss
-  if (!res) {
-    res = engine.executeOnText(input, resourcePath, true)
-
-    // Save new results in the cache
-    if (config.cache) {
-      cache[resourcePath] = {
-        hash: inputMD5,
-        rules: rulesHash,
-        res: res,
-      }
-      fs.writeFileSync(cachePath, JSON.stringify(cache))
-    }
-  }
-
-  // executeOnText ensure we will have res.results[0] only
-
+function printLinterOutput(res, config, webpack) {
   // skip ignored file warning
   if (!(
     res.warningCount === 1 &&
@@ -155,45 +100,73 @@ function lint(input, config, webpack) {
  * @return {void}
  */
 module.exports = function(input, map) {
+  var webpack = this
   var config = assign(
     // loader defaults
     {
       formatter: require("eslint/lib/formatters/stylish"),
+      cacheIdentifier: JSON.stringify({
+        "eslint-loader": pkg.version,
+        "eslint": eslint.version,
+      }),
     },
     // user defaults
     this.options.eslint || {},
     // loader query string
     loaderUtils.getOptions(this)
   )
-  this.cacheable()
+
+  var cacheDirectory = config.cacheDirectory
+  var cacheIdentifier = config.cacheIdentifier
+
+  delete config.cacheDirectory
+  delete config.cacheIdentifier
 
   // Create the engine only once per config
   var configHash = objectHash(config)
   if (!engines[configHash]) {
     engines[configHash] = new eslint.CLIEngine(config)
   }
 
-  // Read the cached information only once and if enable
-  if (cache === null) {
-    if (config.cache) {
-      var thunk = findCacheDir({
-        name: "eslint-loader",
-        thunk: true,
-        create: true,
-      })
-      cachePath = thunk("data.json") || os.tmpdir() + "/data.json"
-      try {
-        cache = require(cachePath)
-      }
-      catch (e) {
-        cache = {}
+  this.cacheable()
+
+  var resourcePath = webpack.resourcePath
+  var cwd = process.cwd()
+
+  // remove cwd from resource path in case webpack has been started from project
+  // root, to allow having relative paths in .eslintignore
+  if (resourcePath.indexOf(cwd) === 0) {
+    resourcePath = resourcePath.substr(cwd.length + 1)
+  }
+
+  var engine = engines[configHash]
+  // return early if cached
+  if (config.cache) {
+    var callback = this.async()
+    return cache({
+      directory: cacheDirectory,
+      identifier: cacheIdentifier,
+      options: config,
+      transform: function() {
+        return lint(engine, input, resourcePath)
+      },
+    }, function(err, res) {
+      if (err) { 
+        return callback(err)
       }
-    }
-    else {
-      cache = false
-    }
+      printLinterOutput(res, config, webpack)
+      return callback(null, input, map)
+    })
   }
 
-  lint(input, config, this)
-  this.callback(null, input, map)
+  printLinterOutput(
+    lint(engine, input, resourcePath), 
+    config, 
+    this
+  )
+  this.callback(input, map)
+}
+
+function lint(engine, input, resourcePath) {
+  engine.executeOnText(input, resourcePath, true)
 }