storage: Add LocalHttpProxy backend

LocalHttpProxy is a storage backend driver to access blobs stored in the local machine through a http proxy server over unix socket.

`LocalHttpProxy` uses two API endpoints to access the blobs:
- `HEAD /` to get the blob size
- `GET /` to read the blob

The http proxy server should respect [the `Range` header](https://www.rfc-editor.org/rfc/rfc9110.html#name-range) to compute the offset and length of the blob.

Signed-off-by: Nan Li <loheagn@icloud.com>

Cargo.toml

@@ -66,7 +66,7 @@ nydus-rafs = { version = "0.1.0", path = "rafs", features = [
     "backend-oss",
     "backend-s3",
 ] }
-nydus-storage = { version = "0.5.0", path = "storage" }
+nydus-storage = { version = "0.5.0", path = "storage", features = ["backend-local-http-proxy"] }
 nydus-utils = { version = "0.3.0", path = "utils" }
 nydus-blobfs = { version = "0.1.0", path = "blobfs", features = [
     "virtiofs",

api/src/config.rs

@@ -238,6 +238,8 @@ pub struct BackendConfigV2 {
     pub s3: Option<S3Config>,
     /// Configuration for container registry backend.
     pub registry: Option<RegistryConfig>,
+    /// Configuration for local http proxy.
+    pub local_http_proxy: Option<LocalHttpProxyConfig>,
 }
 
 impl BackendConfigV2 {
@@ -276,6 +278,7 @@ impl BackendConfigV2 {
                 }
                 None => return false,
             },
+
             _ => return false,
         }
 
@@ -325,6 +328,17 @@ impl BackendConfigV2 {
                 .ok_or_else(|| einval!("no configuration information for registry"))
         }
     }
+
+    /// Get configuration information for local http proxy
+    pub fn get_local_http_proxy_config(&self) -> Result<&LocalHttpProxyConfig> {
+        if &self.backend_type != "local-http-proxy" {
+            Err(einval!("backend type is not 'local-http-proxy'"))
+        } else {
+            self.local_http_proxy
+                .as_ref()
+                .ok_or_else(|| einval!("no configuration information for local-http-proxy"))
+        }
+    }
 }
 
 /// Configuration information for localfs storage backend.
@@ -427,6 +441,14 @@ pub struct S3Config {
     pub mirrors: Vec<MirrorConfig>,
 }
 
+/// ContentProxyProxy configuration information to access blobs.
+#[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)]
+pub struct LocalHttpProxyConfig {
+    pub socket_path: String,
+    #[serde(default = "default_local_http_proxy_thread_num")]
+    pub thread_num: usize,
+}
+
 /// Container registry configuration information to access blobs.
 #[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)]
 pub struct RegistryConfig {
@@ -878,6 +900,10 @@ fn default_rafs_mode() -> String {
     "direct".to_string()
 }
 
+fn default_local_http_proxy_thread_num() -> usize {
+    1
+}
+
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 // For backward compatibility
 ////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -904,6 +930,7 @@ impl TryFrom<&BackendConfig> for BackendConfigV2 {
             oss: None,
             s3: None,
             registry: None,
+            local_http_proxy: None,
         };
 
         match value.backend_type.as_str() {

storage/Cargo.toml

@@ -17,6 +17,8 @@ hmac = { version = "0.12.1", optional = true }
 hmac-sha1-compact = { version = "1.1.1", optional = true }
 http = { version = "0.2.8", optional = true }
 httpdate = { version = "1.0", optional = true }
+hyper = {version = "0.14.11", optional =  true}
+hyperlocal = {version = "0.8.0", optional = true}
 lazy_static = "1.4.0"
 leaky-bucket = "0.12.1"
 libc = "0.2"
@@ -28,7 +30,7 @@ serde_json = "1.0.53"
 sha2 = { version = "0.10.2", optional = true }
 tar = "0.4.38"
 time = { version = "0.3.14", features = ["formatting"], optional = true }
-tokio = { version = "1.19.0", features = ["rt", "rt-multi-thread", "sync", "time"] }
+tokio = { version = "1.19.0", features = ["macros", "rt", "rt-multi-thread", "sync", "time"] }
 url = { version = "2.1.1", optional = true }
 vm-memory = "0.9"
 fuse-backend-rs = "0.10"
@@ -47,6 +49,7 @@ backend-localfs = []
 backend-oss = ["base64", "httpdate", "hmac-sha1-compact", "reqwest", "url"]
 backend-registry = ["base64", "reqwest", "url"]
 backend-s3 = ["base64", "hmac", "http", "reqwest", "sha2", "time", "url"]
+backend-local-http-proxy = ["hyper", "hyperlocal", "http"]
 
 [package.metadata.docs.rs]
 all-features = true