fs: implement mkdir recursive (mkdirp)

Implements mkdirp functionality in node_file.cc. The Benefit
of implementing in C++ layer is that the logic is more easily
shared between the Promise and callback implementation and
there are notable performance improvements.

This commit is part of the Tooling Group Initiative.

Refs: nodejs/user-feedback#70

PR-URL: #21875
Reviewed-By: Anna Henningsen <anna@addaleax.net>
Reviewed-By: Matteo Collina <matteo.collina@gmail.com>
Reviewed-By: Jon Moss <me@jonathanmoss.me>
Reviewed-By: Ron Korving <ron@ronkorving.nl>
Reviewed-By: James M Snell <jasnell@gmail.com>
Reviewed-By: Michael Dawson <michael_dawson@ca.ibm.com>
Reviewed-By: Anatoli Papirovski <apapirovski@mac.com>
Reviewed-By: Sam Ruby <rubys@intertwingly.net>
Reviewed-By: Joyee Cheung <joyeec9h3@gmail.com>

Author: Benjamin Coe

benchmark/fs/bench-mkdirp.js

@@ -0,0 +1,23 @@
+'use strict';
+
+const common = require('../common');
+const fs = require('fs');
+const tmpdir = require('../../test/common/tmpdir');
+tmpdir.refresh();
+let dirc = 0;
+
+const bench = common.createBenchmark(main, {
+  n: [1e4],
+});
+
+function main({ n }) {
+  bench.start();
+  (function r(cntr) {
+    if (cntr-- <= 0)
+      return bench.end(n);
+    const pathname = `${tmpdir.path}/${++dirc}/${++dirc}/${++dirc}/${++dirc}`;
+    fs.mkdir(pathname, { createParents: true }, (err) => {
+      r(cntr);
+    });
+  }(n));
+}

doc/api/fs.md

@@ -2139,7 +2139,7 @@ changes:
 
 Synchronous lstat(2).
 
-## fs.mkdir(path[, mode], callback)
+## fs.mkdir(path[, options], callback)
 <!-- YAML
 added: v0.1.8
 changes:
@@ -2158,16 +2158,29 @@ changes:
 -->
 
 * `path` {string|Buffer|URL}
-* `mode` {integer} Not supported on Windows. **Default:** `0o777`.
+* `options` {Object|integer}
+  * `recursive` {boolean} **Default:** `false`
+  * `mode` {integer} Not supported on Windows. **Default:** `0o777`.
 * `callback` {Function}
   * `err` {Error}
 
 Asynchronously creates a directory. No arguments other than a possible exception
 are given to the completion callback.
 
+The optional `options` argument can be an integer specifying mode (permission
+and sticky bits), or an object with a `mode` property and a `recursive`
+property indicating whether parent folders should be created.
+
+```js
+// Creates /tmp/a/apple, regardless of whether `/tmp` and /tmp/a exist.
+fs.mkdir('/tmp/a/apple', { recursive: true }, (err) => {
+  if (err) throw err;
+});
+```
+
 See also: mkdir(2).
 
-## fs.mkdirSync(path[, mode])
+## fs.mkdirSync(path[, options])
 <!-- YAML
 added: v0.1.21
 changes:
@@ -2178,7 +2191,9 @@ changes:
 -->
 
 * `path` {string|Buffer|URL}
-* `mode` {integer} Not supported on Windows. **Default:** `0o777`.
+* `options` {Object|integer}
+  * `recursive` {boolean} **Default:** `false`
+  * `mode` {integer} Not supported on Windows. **Default:** `0o777`.
 
 Synchronously creates a directory. Returns `undefined`.
 This is the synchronous version of [`fs.mkdir()`][].
@@ -4100,18 +4115,24 @@ changes:
 Asynchronous lstat(2). The `Promise` is resolved with the [`fs.Stats`][] object
 for the given symbolic link `path`.
 
-### fsPromises.mkdir(path[, mode])
+### fsPromises.mkdir(path[, options])
 <!-- YAML
 added: v10.0.0
 -->
 
 * `path` {string|Buffer|URL}
-* `mode` {integer} **Default:** `0o777`
+* `options` {Object|integer}
+  * `recursive` {boolean} **Default:** `false`
+  * `mode` {integer} Not supported on Windows. **Default:** `0o777`.
 * Returns: {Promise}
 
 Asynchronously creates a directory then resolves the `Promise` with no
 arguments upon success.
 
+The optional `options` argument can be an integer specifying mode (permission
+and sticky bits), or an object with a `mode` property and a `recursive`
+property indicating whether parent folders should be created.
+
 ### fsPromises.mkdtemp(prefix[, options])
 <!-- YAML
 added: v10.0.0
@@ -4749,7 +4770,7 @@ the file contents.
 [`fs.ftruncate()`]: #fs_fs_ftruncate_fd_len_callback
 [`fs.futimes()`]: #fs_fs_futimes_fd_atime_mtime_callback
 [`fs.lstat()`]: #fs_fs_lstat_path_options_callback
-[`fs.mkdir()`]: #fs_fs_mkdir_path_mode_callback
+[`fs.mkdir()`]: #fs_fs_mkdir_path_options_callback
 [`fs.mkdtemp()`]: #fs_fs_mkdtemp_prefix_options_callback
 [`fs.open()`]: #fs_fs_open_path_flags_mode_callback
 [`fs.read()`]: #fs_fs_read_fd_buffer_offset_length_position_callback

lib/fs.js

@@ -705,29 +705,48 @@ function fsyncSync(fd) {
   handleErrorFromBinding(ctx);
 }
 
-function mkdir(path, mode, callback) {
+function mkdir(path, options, callback) {
+  if (typeof options === 'function') {
+    callback = options;
+    options = {};
+  } else if (typeof options === 'number' || typeof options === 'string') {
+    options = { mode: options };
+  }
+  const {
+    recursive = false,
+    mode = 0o777
+  } = options || {};
+  callback = makeCallback(callback);
   path = getPathFromURL(path);
-  validatePath(path);
 
-  if (arguments.length < 3) {
-    callback = makeCallback(mode);
-    mode = 0o777;
-  } else {
-    callback = makeCallback(callback);
-    mode = validateMode(mode, 'mode', 0o777);
-  }
+  validatePath(path);
+  if (typeof recursive !== 'boolean')
+    throw new ERR_INVALID_ARG_TYPE('recursive', 'boolean', recursive);
 
   const req = new FSReqWrap();
   req.oncomplete = callback;
-  binding.mkdir(pathModule.toNamespacedPath(path), mode, req);
+  binding.mkdir(pathModule.toNamespacedPath(path),
+                validateMode(mode, 'mode', 0o777), recursive, req);
 }
 
-function mkdirSync(path, mode) {
+function mkdirSync(path, options) {
+  if (typeof options === 'number' || typeof options === 'string') {
+    options = { mode: options };
+  }
   path = getPathFromURL(path);
+  const {
+    recursive = false,
+    mode = 0o777
+  } = options || {};
+
   validatePath(path);
-  mode = validateMode(mode, 'mode', 0o777);
+  if (typeof recursive !== 'boolean')
+    throw new ERR_INVALID_ARG_TYPE('recursive', 'boolean', recursive);
+
   const ctx = { path };
-  binding.mkdir(pathModule.toNamespacedPath(path), mode, undefined, ctx);
+  binding.mkdir(pathModule.toNamespacedPath(path),
+                validateMode(mode, 'mode', 0o777), recursive, undefined,
+                ctx);
   handleErrorFromBinding(ctx);
 }
 

lib/internal/fs/promises.js

@@ -287,11 +287,23 @@ async function fsync(handle) {
   return binding.fsync(handle.fd, kUsePromises);
 }
 
-async function mkdir(path, mode) {
+async function mkdir(path, options) {
+  if (typeof options === 'number' || typeof options === 'string') {
+    options = { mode: options };
+  }
+  const {
+    recursive = false,
+    mode = 0o777
+  } = options || {};
   path = getPathFromURL(path);
+
   validatePath(path);
-  mode = validateMode(mode, 'mode', 0o777);
-  return binding.mkdir(pathModule.toNamespacedPath(path), mode, kUsePromises);
+  if (typeof recursive !== 'boolean')
+    throw new ERR_INVALID_ARG_TYPE('recursive', 'boolean', recursive);
+
+  return binding.mkdir(pathModule.toNamespacedPath(path),
+                       validateMode(mode, 'mode', 0o777), recursive,
+                       kUsePromises);
 }
 
 async function readdir(path, options) {

src/node_file.cc

@@ -76,6 +76,16 @@ using v8::Value;
 # define MIN(a, b) ((a) < (b) ? (a) : (b))
 #endif
 
+#ifndef S_ISDIR
+# define S_ISDIR(mode)  (((mode) & S_IFMT) == S_IFDIR)
+#endif
+
+#ifdef __POSIX__
+const char* kPathSeparator = "/";
+#else
+const char* kPathSeparator = "\\/";
+#endif
+
 #define GET_OFFSET(a) ((a)->IsNumber() ? (a).As<Integer>()->Value() : -1)
 #define TRACE_NAME(name) "fs.sync." #name
 #define GET_TRACE_ENABLED                                                  \
@@ -1236,28 +1246,162 @@ static void RMDir(const FunctionCallbackInfo<Value>& args) {
   }
 }
 
+int MKDirpSync(uv_loop_t* loop, uv_fs_t* req, const std::string& path, int mode,
+               uv_fs_cb cb = nullptr) {
+  FSContinuationData continuation_data(req, mode, cb);
+  continuation_data.PushPath(std::move(path));
+
+  while (continuation_data.paths.size() > 0) {
+    std::string next_path = continuation_data.PopPath();
+    int err = uv_fs_mkdir(loop, req, next_path.c_str(), mode, nullptr);
+    while (true) {
+      switch (err) {
+        case 0:
+          if (continuation_data.paths.size() == 0) {
+            return 0;
+          }
+          break;
+        case UV_ENOENT: {
+          std::string dirname = next_path.substr(0,
+                                        next_path.find_last_of(kPathSeparator));
+          if (dirname != next_path) {
+            continuation_data.PushPath(std::move(next_path));
+            continuation_data.PushPath(std::move(dirname));
+          } else if (continuation_data.paths.size() == 0) {
+            err = UV_EEXIST;
+            continue;
+          }
+          break;
+        }
+        case UV_EPERM: {
+          return err;
+        }
+        default:
+          uv_fs_req_cleanup(req);
+          err = uv_fs_stat(loop, req, next_path.c_str(), nullptr);
+          if (err == 0 && !S_ISDIR(req->statbuf.st_mode)) return UV_EEXIST;
+          if (err < 0) return err;
+          break;
+      }
+      break;
+    }
+    uv_fs_req_cleanup(req);
+  }
+
+  return 0;
+}
+
+int MKDirpAsync(uv_loop_t* loop,
+                uv_fs_t* req,
+                const char* path,
+                int mode,
+                uv_fs_cb cb) {
+  FSReqBase* req_wrap = FSReqBase::from_req(req);
+  // on the first iteration of algorithm, stash state information.
+  if (req_wrap->continuation_data == nullptr) {
+    req_wrap->continuation_data = std::unique_ptr<FSContinuationData>{
+      new FSContinuationData(req, mode, cb)};
+    req_wrap->continuation_data->PushPath(std::move(path));
+  }
+
+  // on each iteration of algorithm, mkdir directory on top of stack.
+  std::string next_path = req_wrap->continuation_data->PopPath();
+  int err = uv_fs_mkdir(loop, req, next_path.c_str(), mode,
+                        uv_fs_callback_t{[](uv_fs_t* req) {
+    FSReqBase* req_wrap = FSReqBase::from_req(req);
+    Environment* env = req_wrap->env();
+    uv_loop_t* loop = env->event_loop();
+    std::string path = req->path;
+    int err = req->result;
+
+    while (true) {
+      switch (err) {
+        case 0: {
+          if (req_wrap->continuation_data->paths.size() == 0) {
+            req_wrap->continuation_data->Done(0);
+          } else {
+            uv_fs_req_cleanup(req);
+            MKDirpAsync(loop, req, path.c_str(),
+                        req_wrap->continuation_data->mode, nullptr);
+          }
+          break;
+        }
+        case UV_ENOENT: {
+          std::string dirname = path.substr(0,
+                                            path.find_last_of(kPathSeparator));
+          if (dirname != path) {
+            req_wrap->continuation_data->PushPath(std::move(path));
+            req_wrap->continuation_data->PushPath(std::move(dirname));
+          } else if (req_wrap->continuation_data->paths.size() == 0) {
+            err = UV_EEXIST;
+            continue;
+          }
+          uv_fs_req_cleanup(req);
+          MKDirpAsync(loop, req, path.c_str(),
+                      req_wrap->continuation_data->mode, nullptr);
+          break;
+        }
+        case UV_EPERM: {
+          req_wrap->continuation_data->Done(err);
+          break;
+        }
+        default:
+          if (err == UV_EEXIST &&
+              req_wrap->continuation_data->paths.size() > 0) {
+            uv_fs_req_cleanup(req);
+            MKDirpAsync(loop, req, path.c_str(),
+                        req_wrap->continuation_data->mode, nullptr);
+          } else {
+            // verify that the path pointed to is actually a directory.
+            uv_fs_req_cleanup(req);
+            int err = uv_fs_stat(loop, req, path.c_str(),
+                                 uv_fs_callback_t{[](uv_fs_t* req) {
+              FSReqBase* req_wrap = FSReqBase::from_req(req);
+              int err = req->result;
+              if (err == 0 && !S_ISDIR(req->statbuf.st_mode)) err = UV_EEXIST;
+              req_wrap->continuation_data->Done(err);
+            }});
+            if (err < 0) req_wrap->continuation_data->Done(err);
+          }
+          break;
+      }
+      break;
+    }
+  }});
+
+  return err;
+}
+
 static void MKDir(const FunctionCallbackInfo<Value>& args) {
   Environment* env = Environment::GetCurrent(args);
 
   const int argc = args.Length();
-  CHECK_GE(argc, 3);
+  CHECK_GE(argc, 4);
 
   BufferValue path(env->isolate(), args[0]);
   CHECK_NOT_NULL(*path);
 
   CHECK(args[1]->IsInt32());
   const int mode = args[1].As<Int32>()->Value();
 
-  FSReqBase* req_wrap_async = GetReqWrap(env, args[2]);
+  CHECK(args[2]->IsBoolean());
+  bool mkdirp = args[2]->IsTrue();
+
+  FSReqBase* req_wrap_async = GetReqWrap(env, args[3]);
   if (req_wrap_async != nullptr) {  // mkdir(path, mode, req)
-    AsyncCall(env, req_wrap_async, args, "mkdir", UTF8, AfterNoArgs,
-              uv_fs_mkdir, *path, mode);
+    AsyncCall(env, req_wrap_async, args, "mkdir", UTF8,
+              AfterNoArgs, mkdirp ? MKDirpAsync : uv_fs_mkdir, *path, mode);
   } else {  // mkdir(path, mode, undefined, ctx)
-    CHECK_EQ(argc, 4);
+    CHECK_EQ(argc, 5);
     FSReqWrapSync req_wrap_sync;
     FS_SYNC_TRACE_BEGIN(mkdir);
-    SyncCall(env, args[3], &req_wrap_sync, "mkdir",
-             uv_fs_mkdir, *path, mode);
+    if (mkdirp) {
+      SyncCall(env, args[4], &req_wrap_sync, "mkdir",
+               MKDirpSync, *path, mode);
+    } else {
+      SyncCall(env, args[4], &req_wrap_sync, "mkdir",
+               uv_fs_mkdir, *path, mode);
+    }
     FS_SYNC_TRACE_END(mkdir);
   }
 }