Merge pull request #452 from jiangliu/storage-v0.6

Prepare for publishing nydus-storage and nydus-rafs

Cargo.toml

@@ -20,7 +20,7 @@ path = "src/lib.rs"
 
 
 [dependencies]
-rlimit = "0.3.0"
+rlimit = "0.8.3"
 log = "0.4.8"
 libc = "0.2"
 vmm-sys-util = "0.9.0"
@@ -31,7 +31,7 @@ regex = "1.5.5"
 serde = { version = "1.0.110", features = ["serde_derive", "rc"] }
 serde_json = "1.0.51"
 serde_with = { version = "1.6.0", features = ["macros"] }
-sha2 = "0.9.1"
+sha2 = "0.10.2"
 lazy_static = "1.4.0"
 xattr = "0.2.2"
 nix = "0.23.1"
@@ -42,11 +42,12 @@ vm-memory = { version = "0.7.0", features = ["backend-mmap"], optional = true }
 chrono = "0.4.19"
 openssl = { version = "0.10.38", features = ["vendored"] }
 hyperlocal = "0.8.0"
-tokio = { version = "1.16.1", features = ["macros"] }
+tokio = { version = "1.18.2", features = ["macros"] }
 hyper = "0.14.11"
 # pin rand_core to bring in fix for RUSTSEC-2021-0023
 rand_core = "0.6.2"
 tar = "0.4.38"
+mio = { version = "0.8", features = ["os-poll", "os-ext"]}
 
 fuse-backend-rs = { version = "0.9.0", optional = true }
 vhost = { version = "0.3.0", features = ["vhost-user-slave"], optional = true }
@@ -57,11 +58,10 @@ virtio-queue = { version = "0.1.0", optional = true }
 nydus-api = { path = "api" }
 nydus-app = { path = "app" }
 nydus-error = { path = "error" }
-nydus-utils = { path = "utils" }
-rafs = { path = "rafs", features = ["backend-registry", "backend-oss"] }
-storage = { path = "storage" }
+nydus-rafs = { version = "0.1.0", path = "rafs", features = ["backend-registry", "backend-oss"] }
+nydus-storage = { version = "0.5.0", path = "storage" }
+nydus-utils = { version = "0.2.0", path = "utils" }
 blobfs = { path = "blobfs", features = ["virtiofs"], optional = true }
-mio = { version = "0.8", features = ["os-poll", "os-ext"]}
 
 [dev-dependencies]
 sendfd = "0.3.3"

api/src/http.rs

@@ -483,7 +483,7 @@ fn handle_http_request(
     let mut response = match uri_parsed {
         Ok(uri) => match HTTP_ROUTES.routes.get(uri.path()) {
             Some(route) => route
-                .handle_request(&request, &|r| {
+                .handle_request(request, &|r| {
                     kick_api_server(api_notifier.clone(), to_api, from_api, r)
                 })
                 .unwrap_or_else(|err| error_response(err, StatusCode::BadRequest)),

blobfs/Cargo.toml

@@ -16,14 +16,14 @@ libc = "0.2"
 vm-memory = { version = "0.7.0" }
 fuse-backend-rs = { version = "0.9.0" }
 
-rafs = { path = "../rafs" }
 nydus-error = { path = "../error" }
-storage = { path = "../storage", features = ["backend-localfs"] }
+nydus-rafs = { version = "0.1.0", path = "../rafs" }
+nydus-storage = { version = "0.5.0", path = "../storage", features = ["backend-localfs"] }
 
 [features]
-virtiofs = [ "fuse-backend-rs/virtiofs", "rafs/virtio-fs" ]
-backend-oss = ["rafs/backend-oss"]
-backend-registry = ["rafs/backend-registry"]
+virtiofs = [ "fuse-backend-rs/virtiofs", "nydus-rafs/virtio-fs" ]
+backend-oss = ["nydus-rafs/backend-oss"]
+backend-registry = ["nydus-rafs/backend-registry"]
 
 [dev-dependencies]
 nydus-app = { version = "0.2", path = "../app" }

blobfs/src/lib.rs

@@ -20,7 +20,7 @@ use fuse_backend_rs::{
     passthrough::PassthroughFs,
 };
 use nydus_error::{einval, eother};
-use rafs::{
+use nydus_rafs::{
     fs::{Rafs, RafsConfig},
     RafsIoRead,
 };
@@ -45,7 +45,7 @@ use std::sync::{Arc, Mutex};
 use std::thread;
 
 #[cfg(feature = "virtiofs")]
-use storage::device::BlobPrefetchRequest;
+use nydus_storage::device::BlobPrefetchRequest;
 use vm_memory::ByteValued;
 
 mod sync_io;
@@ -94,23 +94,14 @@ impl FromStr for BlobOndemandConfig {
 }
 
 /// Options that configure the behavior of the blobfs fuse file system.
-#[derive(Debug, Clone, PartialEq)]
+#[derive(Default, Debug, Clone, PartialEq)]
 pub struct Config {
     /// Blobfs config is embedded with passthrough config
     pub ps_config: PassthroughConfig,
     /// This provides on demand config of blob management.
     pub blob_ondemand_cfg: String,
 }
 
-impl Default for Config {
-    fn default() -> Self {
-        Config {
-            ps_config: PassthroughConfig::default(),
-            blob_ondemand_cfg: Default::default(),
-        }
-    }
-}
-
 #[allow(dead_code)]
 struct RafsHandle {
     rafs: Arc<Mutex<Option<Rafs>>>,

blobfs/src/sync_io.rs

@@ -13,14 +13,14 @@ use fuse_backend_rs::api::CreateIn;
 use fuse_backend_rs::transport::FsCacheReqHandler;
 use nydus_error::eacces;
 #[cfg(feature = "virtiofs")]
+use nydus_storage::device::BlobPrefetchRequest;
+#[cfg(feature = "virtiofs")]
 use std::cmp::min;
 use std::ffi::CStr;
 use std::io;
 #[cfg(feature = "virtiofs")]
 use std::path::Path;
 use std::time::Duration;
-#[cfg(feature = "virtiofs")]
-use storage::device::BlobPrefetchRequest;
 
 impl BlobFs {
     #[cfg(feature = "virtiofs")]
@@ -50,7 +50,7 @@ impl BlobFs {
 
         let blob_file = Self::open_file(
             libc::AT_FDCWD,
-            &blob_id_full_path.as_path(),
+            blob_id_full_path.as_path(),
             libc::O_PATH | libc::O_NOFOLLOW | libc::O_CLOEXEC,
             0,
         )

error/Cargo.toml

@@ -1,11 +1,11 @@
 [package]
 name = "nydus-error"
-version = "0.2.0"
-description = "Error handling utilities for Nydus"
-readme = "README.md"
-repository = "https://github.com/dragonflyoss/image-service"
+version = "0.2.1"
+description = "Error handling utilities for Nydus Image Service"
 authors = ["The Nydus Developers"]
 license = "Apache-2.0 OR BSD-3-Clause"
+homepage = "https://nydus.dev/"
+repository = "https://github.com/dragonflyoss/image-service"
 edition = "2018"
 
 [dependencies]
@@ -15,3 +15,7 @@ libc = "0.2"
 log = "0.4"
 serde = { version = "1.0.110", features = ["serde_derive", "rc"] }
 serde_json = "1.0.53"
+
+[package.metadata.docs.rs]
+all-features = true
+targets = ["x86_64-unknown-linux-gnu", "aarch64-unknown-linux-gnu", "aarch64-apple-darwin"]

error/LICENSE-BSD-3-Clause

@@ -0,0 +1 @@
+../LICENSE-BSD-3-Clause

error/README.md

@@ -54,4 +54,4 @@ fn record_error(size: usize) {
 
 ## License
 
-This code is licensed under [Apache-2.0](LICENSE).
+This code is licensed under [Apache-2.0](LICENSE-APACHE) or [BSD-3-Clause](LICENSE-BSD-3-Clause).

error/src/logger.rs

@@ -87,12 +87,12 @@ mod tests {
         let mut left = 16;
         while left >= 0 {
             let r = holder.push(error_msg);
-            assert_eq!(r.is_ok(), true);
+            assert!(r.is_ok());
             left -= 1;
         }
 
-        assert_eq!(holder.total_errors <= 10, true);
-        assert_eq!(holder.total_size <= 80, true);
+        assert!(holder.total_errors <= 10);
+        assert!(holder.total_size <= 80);
 
         let mut multi = 10;
         let mut error_msg_long = "".to_string();

misc/musl-static/Dockerfile

@@ -1,4 +1,4 @@
-FROM clux/muslrust:1.52.1
+FROM clux/muslrust:1.61.0
 
 ARG ARCH=x86_64
 

misc/nydus-smoke/Dockerfile

@@ -1,4 +1,4 @@
-FROM rust:1.52.1
+FROM rust:1.61.0
 ARG ARCH=x86_64
 
 RUN mkdir /root/.cargo/
@@ -9,4 +9,4 @@ RUN apt update && apt install -y tree
 
 WORKDIR /nydus-rs
 
-CMD make fusedev-release smoke
+CMD make fusedev-release smoke

rafs/Cargo.toml

@@ -1,9 +1,12 @@
 [package]
-name = "rafs"
+name = "nydus-rafs"
 version = "0.1.0"
+description = "The RAFS filesystem format for Nydus Image Service"
 authors = ["The Nydus Developers"]
-edition = "2018"
 license = "Apache-2.0 OR BSD-3-Clause"
+homepage = "https://nydus.dev/"
+repository = "https://github.com/dragonflyoss/image-service"
+edition = "2018"
 
 [dependencies]
 anyhow = "1.0.35"
@@ -14,7 +17,7 @@ bitflags = "1.2.1"
 blake3 = "1.0"
 flate2 = { version = "1.0", features = ["miniz-sys"], default-features = false }
 futures = "0.3"
-hmac = { version = "0.8.1", optional = true }
+hmac = { version = "0.12.1", optional = true }
 lazy_static = "1.4.0"
 libc = "0.2"
 log = "0.4"
@@ -23,16 +26,16 @@ nix = "0.23.1"
 serde = { version = "1.0.110", features = ["serde_derive", "rc"] }
 serde_json = "1.0.53"
 serde_with = { version = "1.6.0", features = ["macros"] }
-sha2 = { version = "0.9.1" }
-sha-1 = { version = "0.9.1", optional = true }
+sha2 = { version = "0.10.2" }
+sha-1 = { version = "0.10.0", optional = true }
 spmc = "0.3.0"
 url = { version = "2.1.1", optional = true }
 vm-memory = "0.7.0"
 fuse-backend-rs = { version = "0.9.0" }
 
-nydus-utils = { path = "../utils" }
-nydus-error = { path = "../error" }
-storage = { path = "../storage", features = ["backend-localfs"] }
+nydus-error = { version = "0.2.0", path = "../error" }
+nydus-storage = { version = "0.5.0", path = "../storage", features = ["backend-localfs"] }
+nydus-utils = { version = "0.2.0", path = "../utils" }
 
 [dev-dependencies]
 vmm-sys-util = "0.9.0"
@@ -42,5 +45,9 @@ assert_matches = "1.5.0"
 fusedev = ["fuse-backend-rs/fusedev"]
 virtio-fs = ["fuse-backend-rs/virtiofs", "vm-memory/backend-mmap"]
 vhost-user-fs = ["fuse-backend-rs/vhost-user-fs"]
-backend-oss = ["storage/backend-oss"]
-backend-registry = ["storage/backend-registry"]
+backend-oss = ["nydus-storage/backend-oss"]
+backend-registry = ["nydus-storage/backend-registry"]
+
+[package.metadata.docs.rs]
+all-features = true
+targets = ["x86_64-unknown-linux-gnu", "aarch64-unknown-linux-gnu", "aarch64-apple-darwin"]

rafs/LICENSE-APACHE

@@ -0,0 +1 @@
+../LICENSE-APACHE

rafs/LICENSE-BSD-3-Clause

@@ -0,0 +1 @@
+../LICENSE-BSD-3-Clause

rafs/README.md

@@ -0,0 +1,17 @@
+# nydus-storage
+
+The RAFS Fuse filesystem for [Nydus Image Service](https://nydus.dev/).
+
+## Support
+
+**Platforms**:
+- x86_64
+- aarch64
+
+**Operating Systems**:
+- Linux
+- MacOS
+
+## License
+
+This code is licensed under [Apache-2.0](LICENSE-APACHE) or [BSD-3-Clause](LICENSE-BSD-3-Clause).

rafs/src/fs.rs

@@ -131,9 +131,9 @@ macro_rules! trim_backend_config {
     ($config:expr, $($i:expr),*) => {
         let mut _n :&mut serde_json::Value = &mut $config["device"]["backend"]["config"];
         if let serde_json::Value::Object(ref mut m) = _n {
-            $(if m.contains_key($i) { m[$i].take();} else {()};)*
+            $(if m.contains_key($i) { m[$i].take();} )*
         }
-    };
+    }
 }
 
 /// Rafs storage backend configuration information.
@@ -1029,7 +1029,7 @@ pub(crate) mod tests {
     #[test]
     fn it_should_enable_xattr() {
         let rafs = new_rafs_backend();
-        assert_eq!(rafs.xattr_supported(), true);
+        assert!(rafs.xattr_supported());
     }
 
     #[test]

rafs/src/lib.rs

@@ -35,7 +35,7 @@ extern crate bitflags;
 #[macro_use]
 extern crate nydus_error;
 #[macro_use]
-extern crate storage;
+extern crate nydus_storage as storage;
 
 use std::any::Any;
 use std::fmt::{Debug, Display, Formatter, Result as FmtResult};
@@ -75,7 +75,7 @@ impl std::error::Error for RafsError {
 
 impl Display for RafsError {
     fn fmt(&self, f: &mut Formatter) -> FmtResult {
-        write!(f, "{}", format!("{:?}", self))?;
+        write!(f, "{:?}", self)?;
         Ok(())
     }
 }

rafs/src/metadata/cached_v5.rs

@@ -846,7 +846,7 @@ mod cached_tests {
         assert_eq!(cached_chunk.uncompress_offset(), 0);
         let c_xattr = cached_inode.get_xattrs().unwrap();
         for k in c_xattr.iter() {
-            let k = OsStr::from_bytes(&k);
+            let k = OsStr::from_bytes(k);
             let v = cached_inode.get_xattr(k).unwrap();
             assert_eq!(xattr.get(k).cloned().unwrap(), v.unwrap());
         }
@@ -990,7 +990,7 @@ mod cached_tests {
 
         assert_eq!(sb.max_inode, RAFS_ROOT_INODE);
         assert_eq!(sb.s_inodes.len(), 0);
-        assert_eq!(sb.validate_digest, true);
+        assert!(sb.validate_digest);
 
         let mut inode = CachedInodeV5::new(sb.s_blob.clone(), sb.s_meta.clone());
         inode.i_ino = 1;

rafs/src/metadata/direct_v5.rs

@@ -108,6 +108,11 @@ struct DirectMappingState {
     validate_digest: bool,
 }
 
+// Safe to Send/Sync because the underlying data structures are readonly
+unsafe impl Send for DirectMappingState {}
+
+unsafe impl Sync for DirectMappingState {}
+
 impl DirectMappingState {
     fn new(meta: &RafsSuperMeta, validate_digest: bool) -> Self {
         DirectMappingState {
@@ -180,11 +185,6 @@ pub struct DirectSuperBlockV5 {
     state: ArcSwap<DirectMappingState>,
 }
 
-// Safe to Send/Sync because the underlying data structures are readonly
-unsafe impl Send for DirectSuperBlockV5 {}
-
-unsafe impl Sync for DirectSuperBlockV5 {}
-
 impl DirectSuperBlockV5 {
     /// Create a new instance of `DirectSuperBlockV5`.
     pub fn new(meta: &RafsSuperMeta, validate_digest: bool) -> Self {

rafs/src/metadata/direct_v6.rs

@@ -61,10 +61,6 @@ use storage::device::{
 };
 use storage::utils::readahead;
 
-// Safe to Send/Sync because the underlying data structures are readonly
-unsafe impl Send for DirectSuperBlockV6 {}
-unsafe impl Sync for DirectSuperBlockV6 {}
-
 fn err_invalidate_data(rafs_err: RafsError) -> std::io::Error {
     std::io::Error::new(std::io::ErrorKind::InvalidData, rafs_err)
 }
@@ -86,6 +82,10 @@ struct DirectMappingState {
     validate_digest: bool,
 }
 
+// Safe to Send/Sync because the underlying data structures are readonly
+unsafe impl Send for DirectMappingState {}
+unsafe impl Sync for DirectMappingState {}
+
 impl DirectMappingState {
     fn new(meta: &RafsSuperMeta, validate_digest: bool) -> Self {
         DirectMappingState {