feat: add cache pruning (#386)

- Add `prune` configuration information to README

Prune, by default, caches that are older than 2 days after accumulating
50 megabytes of cache.

.travis.yml

@@ -10,15 +10,15 @@ matrix:
     - node_js: node
       env: NPM_SCRIPT=commitlint-travis
     - node_js: node
-      env: NPM_INSTALL_EXTRA="webpack@4 file-loader@1 html-webpack-plugin@3.2.0"
+      env: NPM_SCRIPT=test NPM_INSTALL_EXTRA="webpack@4 file-loader@1 html-webpack-plugin@3.2.0"
     - node_js: 8
-      env: NPM_INSTALL_EXTRA="webpack@3 file-loader@0.11 html-webpack-plugin@2.22.0"
+      env: NPM_SCRIPT=test NPM_INSTALL_EXTRA="webpack@3 file-loader@0.11 html-webpack-plugin@2.22.0"
     - node_js: 8
-      env: NPM_INSTALL_EXTRA="webpack@4 file-loader@1 html-webpack-plugin@3.2.0"
+      env: NPM_SCRIPT=test NPM_INSTALL_EXTRA="webpack@4 file-loader@1 html-webpack-plugin@3.2.0"
 
 before_script:
   - npm install ${NPM_INSTALL_EXTRA}
-script: npm run ${NPM_SCRIPT:test}
+script: npm run ${NPM_SCRIPT}
 cache:
   directories:
     - node_modules

README.md

@@ -44,6 +44,16 @@ new HardSourceWebpackPlugin({
     // 'debug', 'log', 'info', 'warn', or 'error'.
     level: 'debug',
   },
+  // Clean up large, old caches automatically.
+  cachePrune: {
+    // Caches younger than `maxAge` are not considered for deletion. They must
+    // be at least this (default: 2 days) old in milliseconds.
+    maxAge: 2 * 24 * 60 * 60 * 1000,
+    // All caches together must be larger than `sizeThreshold` before any
+    // caches will be deleted. Together they must be at least this
+    // (default: 50 MB) big in bytes.
+    sizeThreshold: 50 * 1024 * 1024
+  },
 }),
 ```
 
@@ -146,6 +156,18 @@ The level of log messages to report down to. Defaults to 'debug' when mode is 'n
 
 For example 'debug' reports all messages while 'warn' reports warn and error level messages.
 
+### `cachePrune`
+
+`hard-source` caches are by default created when the webpack configuration changes. Each cache holds a copy of all the data to create a build so they can become quite large. Once a cache is considered "old enough" that it is unlikely to be reused `hard-source` will delete it to free up space automatically.
+
+#### `maxAge`
+
+Caches older than `maxAge` in milliseconds are considered for automatic deletion.
+
+#### `sizeThreshold`
+
+For caches to be deleted, all of them together must total more than this threshold.
+
 ## Troubleshooting
 
 ### Configuration changes are not being detected

index.js

@@ -241,6 +241,15 @@ class HardSourceWebpackPlugin {
       'relativeHelpers',
     ]);
 
+    if (configHashInDirectory) {
+      const PruneCachesSystem = require('./lib/SystemPruneCaches');
+
+      new PruneCachesSystem(
+        path.dirname(cacheDirPath),
+        options.cachePrune,
+      ).apply(compiler);
+    }
+
     function runReadOrReset(_compiler) {
       logger.unlock();
 

lib/ChalkLoggerPlugin.js

@@ -27,6 +27,15 @@ const messages = {
     short: value =>
       `Reading from cache ${value.data.configHash.substring(0, 8)}...`,
   },
+  'caches--delete-old': {
+    short: value =>
+      `Deleted ${value.data.deletedSizeMB} MB. Using ${
+        value.data.sizeMB
+      } MB of disk space.`,
+  },
+  'caches--keep': {
+    short: value => `Using ${value.data.sizeMB} MB of disk space.`,
+  },
   'environment--inputs': {
     short: value =>
       `Tracking node dependencies with: ${value.data.inputs.join(', ')}.`,

lib/SystemPruneCaches.js

@@ -0,0 +1,163 @@
+const { readdir: _readdir, stat: _stat } = require('fs');
+const { basename, join } = require('path');
+
+const _rimraf = require('rimraf');
+
+const logMessages = require('./util/log-messages');
+const pluginCompat = require('./util/plugin-compat');
+const promisify = require('./util/promisify');
+
+const readdir = promisify(_readdir);
+const rimraf = promisify(_rimraf);
+const stat = promisify(_stat);
+
+const directorySize = async dir => {
+  const _stat = await stat(dir);
+  if (_stat.isFile()) {
+    return _stat.size;
+  }
+
+  if (_stat.isDirectory()) {
+    const names = await readdir(dir);
+    let size = 0;
+    for (const name of names) {
+      size += await directorySize(join(dir, name));
+    }
+    return size;
+  }
+
+  return 0;
+};
+
+class CacheInfo {
+  constructor(id = '') {
+    this.id = id;
+    this.lastModified = 0;
+    this.size = 0;
+  }
+
+  static async fromDirectory(dir) {
+    const info = new CacheInfo(basename(dir));
+    info.lastModified = new Date(
+      (await stat(join(dir, 'stamp'))).mtime,
+    ).getTime();
+    info.size = await directorySize(dir);
+    return info;
+  }
+
+  static async fromDirectoryChildren(dir) {
+    const children = [];
+    const names = await readdir(dir);
+    for (const name of names) {
+      children.push(await CacheInfo.fromDirectory(join(dir, name)));
+    }
+    return children;
+  }
+}
+
+// Compilers for webpack with multiple parallel configurations might try to
+// delete caches at the same time. Mutex lock the process of pruning to keep
+// from multiple pruning runs from colliding with each other.
+let deleteLock = null;
+
+class PruneCachesSystem {
+  constructor(cacheRoot, options = {}) {
+    this.cacheRoot = cacheRoot;
+
+    this.options = Object.assign(
+      {
+        // Caches younger than `maxAge` are not considered for deletion. They
+        // must be at least this (default: 2 days) old in milliseconds.
+        maxAge: 2 * 24 * 60 * 60 * 1000,
+        // All caches together must be larger than `sizeThreshold` before any
+        // caches will be deleted. Together they must be at least this
+        // (default: 50 MB) big in bytes.
+        sizeThreshold: 50 * 1024 * 1024,
+      },
+      options,
+    );
+  }
+
+  apply(compiler) {
+    const compilerHooks = pluginCompat.hooks(compiler);
+
+    const deleteOldCaches = async () => {
+      while (deleteLock !== null) {
+        await deleteLock;
+      }
+
+      let resolveLock;
+
+      let infos;
+      try {
+        deleteLock = new Promise(resolve => {
+          resolveLock = resolve;
+        });
+
+        infos = await CacheInfo.fromDirectoryChildren(this.cacheRoot);
+
+        // Sort lastModified in descending order. More recently modified at the
+        // beginning of the array.
+        infos.sort((a, b) => b.lastModified - a.lastModified);
+
+        const totalSize = infos.reduce((carry, info) => carry + info.size, 0);
+        const oldInfos = infos.filter(
+          info => info.lastModified < Date.now() - this.options.maxAge,
+        );
+        const oldTotalSize = oldInfos.reduce(
+          (carry, info) => carry + info.size,
+          0,
+        );
+
+        if (oldInfos.length > 0 && totalSize > this.options.sizeThreshold) {
+          const newInfos = infos.filter(
+            info => info.lastModified >= Date.now() - this.options.maxAge,
+          );
+
+          for (const info of oldInfos) {
+            rimraf(join(this.cacheRoot, info.id));
+          }
+
+          const newTotalSize = newInfos.reduce(
+            (carry, info) => carry + info.size,
+            0,
+          );
+
+          logMessages.deleteOldCaches(compiler, {
+            infos,
+            totalSize,
+            newInfos,
+            newTotalSize,
+            oldInfos,
+            oldTotalSize,
+          });
+        } else {
+          logMessages.keepCaches(compiler, {
+            infos,
+            totalSize,
+          });
+        }
+      } catch (error) {
+        if (error.code !== 'ENOENT') {
+          throw error;
+        }
+      } finally {
+        if (typeof resolveLock === 'function') {
+          deleteLock = null;
+          resolveLock();
+        }
+      }
+    };
+
+    compilerHooks.watchRun.tapPromise(
+      'HardSource - PruneCachesSystem',
+      deleteOldCaches,
+    );
+    compilerHooks.run.tapPromise(
+      'HardSource - PruneCachesSystem',
+      deleteOldCaches,
+    );
+  }
+}
+
+module.exports = PruneCachesSystem;

lib/util/log-messages.js

@@ -91,6 +91,35 @@ exports.configHashBuildWith = (compiler, { cacheDirPath, configHash }) => {
   );
 };
 
+exports.deleteOldCaches = (compiler, { newTotalSize, oldTotalSize }) => {
+  const loggerCore = logCore(compiler);
+  const sizeMB = Math.ceil(newTotalSize / 1024 / 1024);
+  const deletedSizeMB = Math.ceil(oldTotalSize / 1024 / 1024);
+  loggerCore.log(
+    {
+      id: 'caches--delete-old',
+      size: newTotalSize,
+      sizeMB,
+      deletedSize: oldTotalSize,
+      deletedSizeMB,
+    },
+    `HardSourceWebpackPlugin is using ${sizeMB} MB of disk space after deleting ${deletedSizeMB} MB.`,
+  );
+};
+
+exports.keepCaches = (compiler, { totalSize }) => {
+  const loggerCore = logCore(compiler);
+  const sizeMB = Math.ceil(totalSize / 1024 / 1024);
+  loggerCore.log(
+    {
+      id: 'caches--keep',
+      size: totalSize,
+      sizeMB,
+    },
+    `HardSourceWebpackPlugin is using ${sizeMB} MB of disk space.`,
+  );
+};
+
 exports.environmentInputs = (compiler, { inputs }) => {
   const loggerCore = logCore(compiler);
   loggerCore.log(

tests/fixtures/hard-source-prune/config-hash

@@ -0,0 +1 @@
+b

tests/fixtures/hard-source-prune/fib/index.js

@@ -0,0 +1,3 @@
+module.exports = function(n) {
+  return n + (n > 0 ? n - 2 : 0);
+};

tests/fixtures/hard-source-prune/index.js

@@ -0,0 +1,3 @@
+var fib = require('./fib');
+
+console.log(fib(3));

tests/fixtures/hard-source-prune/webpack.config.js

@@ -0,0 +1,27 @@
+var fs = require('fs');
+
+var HardSourceWebpackPlugin = require('../../..');
+
+module.exports = {
+  context: __dirname,
+  entry: './index.js',
+  output: {
+    path: __dirname + '/tmp',
+    filename: 'main.js',
+  },
+  plugins: [
+    new HardSourceWebpackPlugin({
+      cacheDirectory: 'cache/[confighash]',
+      configHash: function(config) {
+        return fs.readFileSync(__dirname + '/config-hash', 'utf8');
+      },
+      environmentHash: {
+        root: __dirname + '/../../..',
+      },
+      cachePrune: {
+        maxAge: -2000,
+        sizeThreshold: 0,
+      },
+    }),
+  ],
+};

tests/hard-source.js

@@ -278,4 +278,13 @@ describe('hard-source features', function() {
   itCompilesTwice('hard-source-exclude-plugin');
   itCompilesHardModules('hard-source-exclude-plugin', ['./index.js', '!./fib.js']);
 
+  itCompilesChange('hard-source-prune', {
+    'config-hash': 'a',
+  }, {
+    'config-hash': 'b',
+  }, function(output) {
+    expect(fs.readdirSync(__dirname + '/fixtures/hard-source-prune/tmp/cache'))
+    .to.have.length(1);
+  });
+
 });