move fs ops to standalone crate

Still a couple of things to figure out here

1. I would like to standardize providing permissions access to standalone ops.
2. We need a way to pass error kind that is outwardly extensible. Maybe just strings, since we are using json?

Cargo.toml

@@ -3,4 +3,6 @@ members = [
   "cli",
   "core",
   "tools/hyper_hello",
+  "ops/dispatch_json",
+  "ops/fs",
 ]

cli/Cargo.toml

@@ -17,6 +17,8 @@ edition = "2018"
 
 [dependencies]
 deno = { path = "../core" }
+deno_dispatch_json = { path = "../ops/dispatch_json" }
+deno_ops_fs = { path = "../ops/fs" }
 
 ansi_term = "0.12.0"
 atty = "0.2.13"

cli/compilers/ts.rs

@@ -639,9 +639,9 @@ impl TsCompiler {
 #[cfg(test)]
 mod tests {
   use super::*;
-  use crate::fs as deno_fs;
   use crate::tokio_util;
   use deno::ModuleSpecifier;
+  use deno_ops_fs::fs as deno_fs;
   use std::path::PathBuf;
   use tempfile::TempDir;
 

cli/disk_cache.rs

@@ -1,4 +1,4 @@
-use crate::fs as deno_fs;
+use deno_ops_fs::fs as deno_fs;
 use std::ffi::OsStr;
 use std::fs;
 use std::path::Component;

cli/file_fetcher.rs

@@ -620,7 +620,7 @@ impl SourceCodeHeaders {
 #[cfg(test)]
 mod tests {
   use super::*;
-  use crate::fs as deno_fs;
+  use deno_ops_fs::fs as deno_fs;
   use tempfile::TempDir;
 
   impl SourceFileFetcher {

cli/flags.rs

@@ -1,12 +1,12 @@
 // Copyright 2018-2019 the Deno authors. All rights reserved. MIT license.
-use crate::fs as deno_fs;
 use clap::App;
 use clap::AppSettings;
 use clap::Arg;
 use clap::ArgMatches;
 use clap::Shell;
 use clap::SubCommand;
 use deno::ModuleSpecifier;
+use deno_ops_fs::fs as deno_fs;
 use log::Level;
 use std;
 use std::str;

cli/ops/files.rs

@@ -1,11 +1,11 @@
 // Copyright 2018-2019 the Deno authors. All rights reserved. MIT license.
 use super::dispatch_json::{wrap_json_op, Deserialize, JsonOp};
 use crate::deno_error;
-use crate::fs as deno_fs;
 use crate::resources;
 use crate::state::DenoOpDispatcher;
 use crate::state::ThreadSafeState;
 use deno::*;
+use deno_ops_fs::fs as deno_fs;
 use futures::Future;
 use std;
 use std::convert::From;

cli/ops/mod.rs

@@ -1,15 +1,16 @@
 // Copyright 2018-2019 the Deno authors. All rights reserved. MIT license.
 use crate::state::ThreadSafeState;
 use deno::*;
+use deno_ops_fs as fs;
 use std::sync::Arc;
 
 mod compiler;
+//  TODO(afinch7) remove this.
 mod dispatch_json;
 mod dispatch_minimal;
 mod errors;
 mod fetch;
 mod files;
-mod fs;
 mod io;
 mod metrics;
 mod net;
@@ -46,23 +47,30 @@ pub fn setup_dispatcher_registry(state: ThreadSafeState) -> Arc<OpDisReg> {
   registry.register_op(OP_NAMESPACE, state.wrap_op(files::OpClose));
   registry.register_op(OP_NAMESPACE, state.wrap_op(files::OpSeek));
 
+  let state_ = state.clone();
+  let state__ = state.clone();
+  let fs_state = fs::TSFsOpsState::new(
+    move |filename| state_.check_read(filename),
+    move |filename| state__.check_write(filename),
+  );
+
   // Fs
-  registry.register_op(OP_NAMESPACE, state.wrap_op(fs::OpChdir));
-  registry.register_op(OP_NAMESPACE, state.wrap_op(fs::OpMkdir));
-  registry.register_op(OP_NAMESPACE, state.wrap_op(fs::OpChmod));
-  registry.register_op(OP_NAMESPACE, state.wrap_op(fs::OpChown));
-  registry.register_op(OP_NAMESPACE, state.wrap_op(fs::OpRemove));
-  registry.register_op(OP_NAMESPACE, state.wrap_op(fs::OpCopyFile));
-  registry.register_op(OP_NAMESPACE, state.wrap_op(fs::OpStat));
-  registry.register_op(OP_NAMESPACE, state.wrap_op(fs::OpReadDir));
-  registry.register_op(OP_NAMESPACE, state.wrap_op(fs::OpRename));
-  registry.register_op(OP_NAMESPACE, state.wrap_op(fs::OpLink));
-  registry.register_op(OP_NAMESPACE, state.wrap_op(fs::OpSymlink));
-  registry.register_op(OP_NAMESPACE, state.wrap_op(fs::OpReadLink));
-  registry.register_op(OP_NAMESPACE, state.wrap_op(fs::OpTruncate));
-  registry.register_op(OP_NAMESPACE, state.wrap_op(fs::OpMakeTempDir));
-  registry.register_op(OP_NAMESPACE, state.wrap_op(fs::OpUtime));
-  registry.register_op(OP_NAMESPACE, state.wrap_op(fs::OpCwd));
+  registry.register_op(OP_NAMESPACE, fs_state.wrap_op(fs::OpChdir));
+  registry.register_op(OP_NAMESPACE, fs_state.wrap_op(fs::OpMkdir));
+  registry.register_op(OP_NAMESPACE, fs_state.wrap_op(fs::OpChmod));
+  registry.register_op(OP_NAMESPACE, fs_state.wrap_op(fs::OpChown));
+  registry.register_op(OP_NAMESPACE, fs_state.wrap_op(fs::OpRemove));
+  registry.register_op(OP_NAMESPACE, fs_state.wrap_op(fs::OpCopyFile));
+  registry.register_op(OP_NAMESPACE, fs_state.wrap_op(fs::OpStat));
+  registry.register_op(OP_NAMESPACE, fs_state.wrap_op(fs::OpReadDir));
+  registry.register_op(OP_NAMESPACE, fs_state.wrap_op(fs::OpRename));
+  registry.register_op(OP_NAMESPACE, fs_state.wrap_op(fs::OpLink));
+  registry.register_op(OP_NAMESPACE, fs_state.wrap_op(fs::OpSymlink));
+  registry.register_op(OP_NAMESPACE, fs_state.wrap_op(fs::OpReadLink));
+  registry.register_op(OP_NAMESPACE, fs_state.wrap_op(fs::OpTruncate));
+  registry.register_op(OP_NAMESPACE, fs_state.wrap_op(fs::OpMakeTempDir));
+  registry.register_op(OP_NAMESPACE, fs_state.wrap_op(fs::OpUtime));
+  registry.register_op(OP_NAMESPACE, fs_state.wrap_op(fs::OpCwd));
 
   // Io
   registry.register_op(OP_NAMESPACE, state.wrap_op(io::OpRead));

cli/ops/os.rs

@@ -1,12 +1,12 @@
 // Copyright 2018-2019 the Deno authors. All rights reserved. MIT license.
 use super::dispatch_json::{wrap_json_op, Deserialize, JsonOp};
 use crate::ansi;
-use crate::fs as deno_fs;
 use crate::state::DenoOpDispatcher;
 use crate::state::ThreadSafeState;
 use crate::version;
 use atty;
 use deno::*;
+use deno_ops_fs::fs as deno_fs;
 use log;
 use std::collections::HashMap;
 use std::env;

ops/dispatch_json/Cargo.toml

@@ -0,0 +1,22 @@
+[package]
+name = "deno_dispatch_json"
+version = "0.16.0"
+edition = "2018"
+authors = ["The deno authors <bertbelder@nodejs.org>"]
+license = "MIT"
+readme = "README.md"
+repository = "https://github.com/denoland/deno"
+
+[lib]
+path = "lib.rs"
+
+[dependencies]
+deno = { path = "../../core" }
+
+log = "0.4.8"
+futures = "0.1.28"
+serde = { version = "1.0.99", features = ["derive"] }
+serde_derive = "1.0.99"
+serde_json = { version = "1.0.40", features = [ "preserve_order" ] }
+tokio-executor = "0.1.8"
+tokio-threadpool = "0.1.15"

ops/dispatch_json/lib.rs

@@ -0,0 +1,139 @@
+// Copyright 2018-2019 the Deno authors. All rights reserved. MIT license.
+#[macro_use]
+extern crate log;
+
+use deno::*;
+use futures::Future;
+use futures::Poll;
+pub use serde_derive::Deserialize;
+use serde_json::json;
+pub use serde_json::Value;
+
+pub type AsyncJsonOp = Box<dyn Future<Item = Value, Error = ErrBox> + Send>;
+
+pub enum JsonOp {
+  Sync(Value),
+  Async(AsyncJsonOp),
+}
+
+fn json_err(err: ErrBox) -> Value {
+  json!({
+    "message": err.to_string(),
+  })
+}
+
+fn serialize_result(
+  promise_id: Option<u64>,
+  result: Result<Value, ErrBox>,
+) -> Buf {
+  let value = match result {
+    Ok(v) => json!({ "ok": v, "promiseId": promise_id }),
+    Err(err) => json!({ "err": json_err(err), "promiseId": promise_id }),
+  };
+  let mut vec = serde_json::to_vec(&value).unwrap();
+  debug!("JSON response pre-align, len={}", vec.len());
+  // Align to 32bit word, padding with the space character.
+  vec.resize((vec.len() + 3usize) & !3usize, b' ');
+  vec.into_boxed_slice()
+}
+
+#[derive(Deserialize)]
+#[serde(rename_all = "camelCase")]
+struct AsyncArgs {
+  promise_id: Option<u64>,
+}
+
+pub fn wrap_json_op<D>(
+  d: D,
+  control: &[u8],
+  zero_copy: Option<PinnedBuf>,
+) -> CoreOp
+where
+  D: FnOnce(Value, Option<PinnedBuf>) -> Result<JsonOp, ErrBox>,
+{
+  let async_args: AsyncArgs = serde_json::from_slice(control).unwrap();
+  let promise_id = async_args.promise_id;
+  let is_sync = promise_id.is_none();
+
+  let result = serde_json::from_slice(control)
+    .map_err(ErrBox::from)
+    .and_then(move |args| d(args, zero_copy));
+
+  match result {
+    Ok(JsonOp::Sync(sync_value)) => {
+      assert!(promise_id.is_none());
+      CoreOp::Sync(serialize_result(promise_id, Ok(sync_value)))
+    }
+    Ok(JsonOp::Async(fut)) => {
+      assert!(promise_id.is_some());
+      let fut2 = Box::new(fut.then(move |result| -> Result<Buf, ()> {
+        Ok(serialize_result(promise_id, result))
+      }));
+      CoreOp::Async(fut2)
+    }
+    Err(sync_err) => {
+      let buf = serialize_result(promise_id, Err(sync_err));
+      if is_sync {
+        CoreOp::Sync(buf)
+      } else {
+        CoreOp::Async(Box::new(futures::future::ok(buf)))
+      }
+    }
+  }
+}
+
+// This is just type conversion. Implement From trait?
+// See https://github.com/tokio-rs/tokio/blob/ffd73a64e7ec497622b7f939e38017afe7124dc4/tokio-fs/src/lib.rs#L76-L85
+fn convert_blocking_json<F>(f: F) -> Poll<Value, ErrBox>
+where
+  F: FnOnce() -> Result<Value, ErrBox>,
+{
+  use futures::Async::*;
+  match tokio_threadpool::blocking(f) {
+    Ok(Ready(Ok(v))) => Ok(Ready(v)),
+    Ok(Ready(Err(err))) => Err(err),
+    Ok(NotReady) => Ok(NotReady),
+    Err(err) => panic!("blocking error {}", err),
+  }
+}
+
+pub fn blocking_json<F>(is_sync: bool, f: F) -> Result<JsonOp, ErrBox>
+where
+  F: 'static + Send + FnOnce() -> Result<Value, ErrBox>,
+{
+  if is_sync {
+    Ok(JsonOp::Sync(f()?))
+  } else {
+    Ok(JsonOp::Async(Box::new(futures::sync::oneshot::spawn(
+      poll_fn(move || convert_blocking_json(f)),
+      &tokio_executor::DefaultExecutor::current(),
+    ))))
+  }
+}
+
+/// `futures::future::poll_fn` only support `F: FnMut()->Poll<T, E>`
+/// However, we require that `F: FnOnce()->Poll<T, E>`.
+/// Therefore, we created our version of `poll_fn`.
+fn poll_fn<T, E, F>(f: F) -> PollFn<F>
+where
+  F: FnOnce() -> Poll<T, E>,
+{
+  PollFn { inner: Some(f) }
+}
+
+struct PollFn<F> {
+  inner: Option<F>,
+}
+
+impl<T, E, F> Future for PollFn<F>
+where
+  F: FnOnce() -> Poll<T, E>,
+{
+  type Item = T;
+  type Error = E;
+
+  fn poll(&mut self) -> Poll<T, E> {
+    let f = self.inner.take().expect("Inner fn has been taken.");
+    f()
+  }
+}

ops/fs/Cargo.toml

@@ -0,0 +1,27 @@
+[package]
+name = "deno_ops_fs"
+version = "0.16.0"
+edition = "2018"
+authors = ["The deno authors <bertbelder@nodejs.org>"]
+license = "MIT"
+readme = "README.md"
+repository = "https://github.com/denoland/deno"
+
+[lib]
+path = "lib.rs"
+
+[dependencies]
+deno = { path = "../../core" }
+deno_dispatch_json = { path = "../dispatch_json" }
+
+log = "0.4.8"
+rand = "0.6.5"
+remove_dir_all = "0.5.2"
+serde = { version = "1.0.99", features = ["derive"] }
+serde_derive = "1.0.99"
+serde_json = { version = "1.0.40", features = [ "preserve_order" ] }
+url = "2.1.0"
+utime = "0.2.1"
+
+[target.'cfg(unix)'.dependencies]
+nix = "0.14.1"