diff --git a/Cargo.lock b/Cargo.lock index 6356ee7b6ba..dbb1432bb46 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -208,9 +208,9 @@ checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc" [[package]] name = "cpufeatures" -version = "0.2.2" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59a6001667ab124aebae2a495118e11d30984c3a653e99d86d58971708cf5e4b" +checksum = "28d997bd5e24a5928dd43e46dc529867e207907fe0b239c3477d924f7f2ca320" dependencies = [ "libc", ] @@ -224,16 +224,6 @@ dependencies = [ "cfg-if", ] -[[package]] -name = "crossbeam-utils" -version = "0.8.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d82ee10ce34d7bc12c2122495e7593a9c41347ecdd64185af4ecf72cb1a7f83" -dependencies = [ - "cfg-if", - "once_cell", -] - [[package]] name = "crypto-common" version = "0.1.6" @@ -244,16 +234,6 @@ dependencies = [ "typenum", ] -[[package]] -name = "dashmap" -version = "4.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e77a43b28d0668df09411cb0bc9a8c2adc40f9a048afe863e05fd43251e8e39c" -dependencies = [ - "cfg-if", - "num_cpus", -] - [[package]] name = "dbs-uhttp" version = "0.3.1" @@ -474,12 +454,6 @@ version = "0.3.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57c66a976bf5909d801bbef33416c41372779507e7a6b3a5e25e4749c58f776a" -[[package]] -name = "futures-timer" -version = "3.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e64b03909df88034c26dc1547e8970b91f98bdb65165d6a4e9110d94263dbb2c" - [[package]] name = "futures-util" version = "0.3.21" @@ -516,7 +490,7 @@ checksum = "4eb1a864a501629691edf6c15a593b7a51eebaa1e8468e9ddc623de7c9b58ec6" dependencies = [ "cfg-if", "libc", - "wasi 0.11.0+wasi-snapshot-preview1", + "wasi", ] [[package]] @@ -531,23 +505,6 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574" -[[package]] -name = "governor" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7df0ee4b237afb71e99f7e2fbd840ffec2d6c4bb569f69b2af18aa1f63077d38" -dependencies = [ - "dashmap", - "futures", - "futures-timer", - "no-std-compat", - "nonzero_ext", - "parking_lot", - "quanta", - "rand", - "smallvec", -] - [[package]] name = "h2" version = "0.3.13" @@ -589,13 +546,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" [[package]] -name = "hmac" -version = "0.12.1" +name = "hmac-sha1-compact" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" -dependencies = [ - "digest", -] +checksum = "d103cfecf6edf3f7d1dc7c5ab64e99488c0f8d11786e43b40873e66e8489d014" [[package]] name = "http" @@ -763,6 +717,17 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" +[[package]] +name = "leaky-bucket" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28d6c39b1cdaa514a898cd9cfcfbdb03a220264fb9ee08931495291cdda3fe7e" +dependencies = [ + "parking_lot", + "tokio", + "tracing", +] + [[package]] name = "libc" version = "0.2.126" @@ -808,15 +773,6 @@ dependencies = [ "libc", ] -[[package]] -name = "mach" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b823e83b2affd8f40a9ee8c29dbc56404c1e34cd2710921f2801e2cf29527afa" -dependencies = [ - "libc", -] - [[package]] name = "matches" version = "0.1.9" @@ -861,7 +817,7 @@ checksum = "57ee1c23c7c63b0c9250c339ffdc69255f110b298b901b9f6c82547b7b87caaf" dependencies = [ "libc", "log", - "wasi 0.11.0+wasi-snapshot-preview1", + "wasi", "windows-sys", ] @@ -895,18 +851,6 @@ dependencies = [ "memoffset", ] -[[package]] -name = "no-std-compat" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b93853da6d84c2e3c7d730d6473e8817692dd89be387eb01b94d7f108ecb5b8c" - -[[package]] -name = "nonzero_ext" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38bf9645c8b145698bb0b18a4637dcacbc421ea49bef2317e4fd8065a387cf21" - [[package]] name = "num_cpus" version = "1.13.1" @@ -1000,7 +944,6 @@ dependencies = [ "blake3", "fuse-backend-rs", "futures", - "hmac", "lazy_static", "libc", "log", @@ -1012,10 +955,7 @@ dependencies = [ "nydus-utils", "serde", "serde_json", - "sha-1", - "sha2", "spmc", - "url", "vm-memory", "vmm-sys-util", ] @@ -1070,17 +1010,14 @@ dependencies = [ name = "nydus-storage" version = "0.5.0" dependencies = [ - "anyhow", "arc-swap", "base64", "bitflags", - "dbs-uhttp", "fuse-backend-rs", - "futures", - "governor", - "hmac", + "hmac-sha1-compact", "httpdate", "lazy_static", + "leaky-bucket", "libc", "log", "nix", @@ -1090,8 +1027,6 @@ dependencies = [ "reqwest", "serde", "serde_json", - "sha-1", - "sha2", "tokio", "url", "vm-memory", @@ -1190,27 +1125,25 @@ dependencies = [ [[package]] name = "parking_lot" -version = "0.11.2" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" +checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" dependencies = [ - "instant", "lock_api", "parking_lot_core", ] [[package]] name = "parking_lot_core" -version = "0.8.5" +version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d76e8e1493bcac0d2766c42737f34458f1c8c50c0d23bcb24ea953affb273216" +checksum = "09a279cbf25cb0757810394fbc1e359949b59e348145c643a939a525692e6929" dependencies = [ "cfg-if", - "instant", "libc", "redox_syscall", "smallvec", - "winapi", + "windows-sys", ] [[package]] @@ -1272,22 +1205,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "quanta" -version = "0.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20afe714292d5e879d8b12740aa223c6a88f118af41870e8b6196e39a02238a8" -dependencies = [ - "crossbeam-utils", - "libc", - "mach", - "once_cell", - "raw-cpuid", - "wasi 0.10.0+wasi-snapshot-preview1", - "web-sys", - "winapi", -] - [[package]] name = "quote" version = "1.0.20" @@ -1327,15 +1244,6 @@ dependencies = [ "getrandom", ] -[[package]] -name = "raw-cpuid" -version = "10.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "738bc47119e3eeccc7e94c4a506901aea5e7b4944ecd0829cbebf4af04ceda12" -dependencies = [ - "bitflags", -] - [[package]] name = "redox_syscall" version = "0.2.13" @@ -1551,22 +1459,11 @@ dependencies = [ "serde", ] -[[package]] -name = "sha-1" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "028f48d513f9678cda28f6e4064755b3fbb2af6acd672f2c209b62323f7aea0f" -dependencies = [ - "cfg-if", - "cpufeatures", - "digest", -] - [[package]] name = "sha2" -version = "0.10.2" +version = "0.10.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55deaec60f81eefe3cce0dc50bda92d6d8e88f2a27df7c5033b42afeb1ed2676" +checksum = "cf9db03534dff993187064c4e0c05a5708d2a9728ace9a8959b77bedf415dac5" dependencies = [ "cfg-if", "cpufeatures", @@ -1806,9 +1703,21 @@ checksum = "a400e31aa60b9d44a52a8ee0343b5b18566b03a8321e0d321f695cf56e940160" dependencies = [ "cfg-if", "pin-project-lite", + "tracing-attributes", "tracing-core", ] +[[package]] +name = "tracing-attributes" +version = "0.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11c75893af559bc8e10716548bdef5cb2b983f8e637db9d0e15126b61b484ee2" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "tracing-core" version = "0.1.28" @@ -1962,12 +1871,6 @@ dependencies = [ "try-lock", ] -[[package]] -name = "wasi" -version = "0.10.0+wasi-snapshot-preview1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" - [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" diff --git a/deny.toml b/deny.toml index ba302db4f76..a7c446e0dc8 100644 --- a/deny.toml +++ b/deny.toml @@ -48,12 +48,6 @@ notice = "warn" # A list of advisory IDs to ignore. Note that ignored advisories will still # output a note when they are encountered. ignore = [ - # stderrlog needs to fix it - "RUSTSEC-2020-0071", - # stderrlog needs to fix it - "RUSTSEC-2020-0159", - # stderrlog needs to fix it - "RUSTSEC-2022-0006", ] # Threshold for security vulnerabilities, any vulnerability with a CVSS score # lower than the range specified will be ignored. Note that ignored advisories @@ -80,6 +74,7 @@ allow = [ "BSD-3-Clause", "BSD-2-Clause", "CC0-1.0", + "ISC", "Unicode-DFS-2016", ] # List of explictly disallowed licenses diff --git a/rafs/Cargo.toml b/rafs/Cargo.toml index 44794901b4c..663e4d325b8 100644 --- a/rafs/Cargo.toml +++ b/rafs/Cargo.toml @@ -15,7 +15,6 @@ base64 = { version = "0.13.0", optional = true } bitflags = "1.2.1" blake3 = "1.0" futures = "0.3" -hmac = { version = "0.12.1", optional = true } lazy_static = "1.4.0" libc = "0.2" log = "0.4" @@ -23,10 +22,7 @@ lz4-sys = "1.9.4" nix = "0.24" serde = { version = "1.0.110", features = ["serde_derive", "rc"] } serde_json = "1.0.53" -sha2 = { version = "0.10.2" } -sha-1 = { version = "0.10.0", optional = true } spmc = "0.3.0" -url = { version = "2.1.1", optional = true } vm-memory = "0.9" fuse-backend-rs = { version = "0.9" } diff --git a/storage/Cargo.toml b/storage/Cargo.toml index b44a125103a..bd366045fea 100644 --- a/storage/Cargo.toml +++ b/storage/Cargo.toml @@ -9,29 +9,22 @@ repository = "https://github.com/dragonflyoss/image-service" edition = "2018" [dependencies] -anyhow = "1.0.35" arc-swap = "1.5" base64 = { version = "0.13.0", optional = true } bitflags = "1.2.1" -dbs-uhttp = { version = "0.3.0" } -futures = "0.3" -# pin governor to avoid multi versions of hashbrown -governor = "=0.4.1" -hmac = { version = "0.12.1", optional = true } +hmac-sha1-compact = { version = "1.1.1", optional = true } httpdate = { version = "1.0", optional = true } lazy_static = "1.4.0" +leaky-bucket = "0.12.1" libc = "0.2" log = "0.4.8" nix = "0.24" reqwest = { version = "0.11.11", features = ["blocking", "json"], optional = true } -serde = { version = "1.0.110", features = ["serde_derive", "rc"] } +serde = { version = "1.0.110", features = ["serde_derive", "rc"], optional = true } serde_json = "1.0.53" -sha2 = { version = "0.10.2", optional = true } -sha-1 = { version = "0.10.0", optional = true } tokio = { version = "1.19.0", features = ["rt", "rt-multi-thread", "sync", "time"] } url = { version = "2.1.1", optional = true } vm-memory = "0.9" -vmm-sys-util = "0.10" fuse-backend-rs = { version = "0.9" } nydus-api = { version = "0.1", path = "../api" } @@ -39,11 +32,12 @@ nydus-utils = { version = "0.3", path = "../utils" } nydus-error = { version = "0.2", path = "../error" } [dev-dependencies] +vmm-sys-util = "0.10" [features] -backend-localfs = ["sha2"] -backend-oss = ["base64", "httpdate", "reqwest", "sha-1", "sha2", "hmac", "url"] -backend-registry = ["base64", "reqwest", "sha2", "url"] +backend-localfs = [] +backend-oss = ["base64", "httpdate", "hmac-sha1-compact", "reqwest"] +backend-registry = ["base64", "reqwest", "serde", "url"] [package.metadata.docs.rs] all-features = true diff --git a/storage/src/backend/oss.rs b/storage/src/backend/oss.rs index 71c60beb7d7..885e1524ff8 100644 --- a/storage/src/backend/oss.rs +++ b/storage/src/backend/oss.rs @@ -8,10 +8,9 @@ use std::io::{Error, Result}; use std::sync::Arc; use std::time::SystemTime; -use hmac::{Hmac, Mac}; +use hmac_sha1_compact::HMAC; use reqwest::header::{HeaderMap, CONTENT_LENGTH}; use reqwest::Method; -use sha1::Sha1; use nydus_api::http::OssConfig; use nydus_utils::metrics::BackendMetrics; @@ -22,8 +21,6 @@ use crate::backend::{BackendError, BackendResult, BlobBackend, BlobReader}; const HEADER_DATE: &str = "Date"; const HEADER_AUTHORIZATION: &str = "Authorization"; -type HmacSha1 = Hmac; - /// Error codes related to OSS storage backend. #[derive(Debug)] pub enum OssError { @@ -108,10 +105,8 @@ impl OssState { data.insert(4, canonicalized_oss_headers.as_str()); } let data = data.join("\n"); - let mut mac = - HmacSha1::new_from_slice(self.access_key_secret.as_bytes()).map_err(|e| einval!(e))?; - mac.update(data.as_bytes()); - let signature = base64::encode(&mac.finalize().into_bytes()); + let digest = HMAC::mac(data.as_bytes(), self.access_key_secret.as_bytes()); + let signature = base64::encode(&digest); let authorization = format!("OSS {}:{}", self.access_key_id, signature); diff --git a/storage/src/cache/worker.rs b/storage/src/cache/worker.rs index 09d674df0ef..ae77c529035 100644 --- a/storage/src/cache/worker.rs +++ b/storage/src/cache/worker.rs @@ -4,23 +4,19 @@ // SPDX-License-Identifier: Apache-2.0 use std::io::Result; -use std::num::NonZeroU32; -use std::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, AtomicUsize, Ordering}; use std::sync::Arc; use std::thread; use std::time::Duration; -use tokio::time::interval; - -use governor::clock::QuantaClock; -use governor::state::{InMemoryState, NotKeyed}; -use governor::{Quota, RateLimiter}; -use nydus_utils::metrics::{BlobcacheMetrics, Metric}; -use tokio::runtime::Runtime; -use tokio::sync::Semaphore; +use leaky_bucket::RateLimiter; use nydus_api::http::BlobPrefetchConfig; use nydus_utils::async_helper::with_runtime; +use nydus_utils::metrics::{BlobcacheMetrics, Metric}; use nydus_utils::mpmc::Channel; +use tokio::runtime::Runtime; +use tokio::sync::Semaphore; +use tokio::time::interval; use crate::cache::{BlobCache, BlobIoRange}; use crate::factory::ASYNC_RUNTIME; @@ -86,7 +82,8 @@ pub(crate) struct AsyncWorkerMgr { prefetch_config: Arc, prefetch_delayed: AtomicU64, prefetch_inflight: AtomicU32, - prefetch_limiter: Option>>, + prefetch_consumed: AtomicUsize, + prefetch_limiter: Option>, } impl AsyncWorkerMgr { @@ -95,21 +92,21 @@ impl AsyncWorkerMgr { metrics: Arc, prefetch_config: Arc, ) -> Result { - // If the given value is less than maximum blob chunk size, it exceeds burst size of the - // limiter ending up with throttling all throughput, so ensure bandwidth is bigger than - // the maximum chunk size. - let tweaked_bw_limit = if prefetch_config.bandwidth_rate != 0 { - std::cmp::max(RAFS_MAX_CHUNK_SIZE as u32, prefetch_config.bandwidth_rate) - } else { - 0 + let prefetch_limiter = match prefetch_config.bandwidth_rate { + 0 => None, + v => { + // If the given value is less than maximum blob chunk size, it exceeds burst size of the + // limiter ending up with throttling all throughput, so ensure bandwidth is bigger than + // the maximum chunk size. + let limit = std::cmp::max(RAFS_MAX_CHUNK_SIZE as usize, v as usize); + let limiter = RateLimiter::builder() + .initial(limit) + .refill(limit / 10) + .interval(Duration::from_millis(100)) + .build(); + Some(Arc::new(limiter)) + } }; - let prefetch_limiter = NonZeroU32::new(tweaked_bw_limit).map(|v| { - info!( - "storage: prefetch bandwidth will be limited at {}Bytes/S", - v - ); - Arc::new(RateLimiter::direct(Quota::per_second(v))) - }); Ok(AsyncWorkerMgr { metrics, @@ -122,6 +119,7 @@ impl AsyncWorkerMgr { prefetch_config, prefetch_delayed: AtomicU64::new(0), prefetch_inflight: AtomicU32::new(0), + prefetch_consumed: AtomicUsize::new(0), prefetch_limiter, }) } @@ -182,12 +180,8 @@ impl AsyncWorkerMgr { /// Consume network bandwidth budget for prefetching. pub fn consume_prefetch_budget(&self, size: u32) { if self.prefetch_inflight.load(Ordering::Relaxed) > 0 { - if let Some(v) = NonZeroU32::new(size) { - // Try to consume budget but ignore result. - if let Some(limiter) = self.prefetch_limiter.as_ref() { - let _ = limiter.check_n(v); - } - } + self.prefetch_consumed + .fetch_add(size as usize, Ordering::AcqRel); } } @@ -297,17 +291,15 @@ impl AsyncWorkerMgr { }; if size > 0 { - let size = std::cmp::min(size, u32::MAX as u64) as u32; - // Safe to unwrap because we have checked that size is not zero. - let cells = NonZeroU32::new(size).unwrap(); - if limiter.check_n(cells).is_err() { + let size = (self.prefetch_consumed.swap(0, Ordering::AcqRel)) + .saturating_add(size as usize); + let max = limiter.max(); + let size = std::cmp::min(size, max.saturating_add(max)); + let cap = limiter.balance(); + if cap < size { self.prefetch_delayed.fetch_add(1, Ordering::Relaxed); - if let Err(e) = limiter.until_n_ready(cells).await { - // `InsufficientCapacity` is the only possible error - // Have to give up to avoid dead-loop - error!("{}: give up rate-limiting", e); - } } + limiter.acquire(size).await; } } } @@ -471,14 +463,20 @@ mod tests { assert_eq!(mgr.prefetch_inflight.load(Ordering::Acquire), 0); assert!(mgr - .send_prefetch_message(AsyncPrefetchMessage::RateLimiter(0x1000001)) + .send_prefetch_message(AsyncPrefetchMessage::RateLimiter(0x1000000)) + .is_ok()); + assert!(mgr + .send_prefetch_message(AsyncPrefetchMessage::RateLimiter(0x1000000)) .is_ok()); assert!(mgr .send_prefetch_message(AsyncPrefetchMessage::RateLimiter(u64::MAX)) .is_ok()); - thread::sleep(Duration::from_secs(4)); - assert_eq!(mgr.prefetch_delayed.load(Ordering::Acquire), 2); - assert_eq!(mgr.prefetch_inflight.load(Ordering::Acquire), 0); + assert_eq!(mgr.prefetch_inflight.load(Ordering::Acquire), 3); + thread::sleep(Duration::from_secs(1)); + assert_eq!(mgr.prefetch_inflight.load(Ordering::Acquire), 2); + thread::sleep(Duration::from_secs(3)); + assert!(mgr.prefetch_inflight.load(Ordering::Acquire) >= 1); + assert!(mgr.prefetch_delayed.load(Ordering::Acquire) >= 1); mgr.stop(); assert_eq!(mgr.workers.load(Ordering::Acquire), 0); diff --git a/storage/src/lib.rs b/storage/src/lib.rs index 930461e8aee..2909d756f55 100644 --- a/storage/src/lib.rs +++ b/storage/src/lib.rs @@ -43,7 +43,6 @@ extern crate log; extern crate bitflags; #[macro_use] extern crate nydus_error; -extern crate dbs_uhttp; use std::fmt::{Display, Formatter}; @@ -52,7 +51,7 @@ pub mod cache; pub mod device; pub mod factory; pub mod meta; -pub mod remote; +//pub mod remote; #[cfg(test)] pub(crate) mod test; pub mod utils;