storage: add new backend HttpProxy

This patch adds a new storage backend `HttpProxy` which can access blobs through a http proxy server. The http proxy server can be local (using unix socket) or remote (using `https://` or using `http://`). `HttpProxy` uses two API endpoints to access the blobs: - `HEAD /path/to/blobs` to get the blob size - `GET /path/to/blobs` to read the blob The http proxy server should respect [the `Range` header](https://www.rfc-editor.org/rfc/rfc9110.html#name-range) to compute the offset and length of the blob. The example config files for this new backend may be: ```jsonc // for remote usage { "backend": { "type": "http-proxy", "config": { "addr": "http://127.0.0.1:9977", "path": "/namespace/<repo>/blobs" } } } ``` or ```jsonc // for local unix socket { "backend": { "type": "http-proxy", "config": { "addr": "/path/to/unix/socket/file" } } } ``` There is also a test in `http_proxy.rs` to make sure `HttpProxy` works well, which setups a simple http server and generates a `HttpProxy` backend to get contents from the server. Signed-off-by: Nan Li <loheagn@icloud.com>
dragonflyoss · Feb 3, 2023 · a000744 · a000744
1 parent 462b083
commit a000744
Show file tree

Hide file tree

Showing 13 changed files with 692 additions and 3 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -67,6 +67,7 @@ nydus-rafs = { version = "0.2.0", path = "rafs", features = [
     "backend-registry",
     "backend-oss",
     "backend-s3",
+    "backend-http-proxy",
 ] }
 nydus-storage = { version = "0.6.0", path = "storage" }
 nydus-utils = { version = "0.4.0", path = "utils" }

diff --git a/api/src/config.rs b/api/src/config.rs
@@ -238,6 +238,8 @@ pub struct BackendConfigV2 {
     pub s3: Option<S3Config>,
     /// Configuration for container registry backend.
     pub registry: Option<RegistryConfig>,
+    /// Configuration for local http proxy.
+    pub http_proxy: Option<HttpProxyConfig>,
 }
 
 impl BackendConfigV2 {
@@ -276,6 +278,28 @@ impl BackendConfigV2 {
                 }
                 None => return false,
             },
+
+            "http-proxy" => match self.http_proxy.as_ref() {
+                Some(v) => {
+                    let is_valid_unix_socket_path = |path: &str| {
+                        let path = Path::new(path);
+                        path.is_absolute() && path.exists()
+                    };
+                    if v.addr.is_empty()
+                        || !(v.addr.starts_with("http://")
+                            || v.addr.starts_with("https://")
+                            || is_valid_unix_socket_path(&v.addr))
+                    {
+                        return false;
+                    }
+
+                    // check if v.path is valid url path format
+                    if Path::new(&v.path).join("any_blob_id").to_str().is_none() {
+                        return false;
+                    }
+                }
+                None => return false,
+            },
             _ => return false,
         }
 
@@ -325,6 +349,17 @@ impl BackendConfigV2 {
                 .ok_or_else(|| einval!("no configuration information for registry"))
         }
     }
+
+    /// Get configuration information for http proxy
+    pub fn get_http_proxy_config(&self) -> Result<&HttpProxyConfig> {
+        if &self.backend_type != "http-proxy" {
+            Err(einval!("backend type is not 'http-proxy'"))
+        } else {
+            self.http_proxy
+                .as_ref()
+                .ok_or_else(|| einval!("no configuration information for http-proxy"))
+        }
+    }
 }
 
 /// Configuration information for localfs storage backend.
@@ -427,6 +462,35 @@ pub struct S3Config {
     pub mirrors: Vec<MirrorConfig>,
 }
 
+/// Http proxy configuration information to access blobs.
+#[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)]
+pub struct HttpProxyConfig {
+    /// Address of http proxy server, like `http://xxx.xxx` or `https://xxx.xxx` or `/path/to/unix.sock`.
+    pub addr: String,
+    /// Path to access the blobs, like `/<_namespace>/<_repo>/blobs`.
+    /// If the http proxy server is over unix socket, this field will be ignored.
+    #[serde(default)]
+    pub path: String,
+    /// Skip SSL certificate validation for HTTPS scheme.
+    #[serde(default)]
+    pub skip_verify: bool,
+    /// Drop the read request once http request timeout, in seconds.
+    #[serde(default = "default_http_timeout")]
+    pub timeout: u32,
+    /// Drop the read request once http connection timeout, in seconds.
+    #[serde(default = "default_http_timeout")]
+    pub connect_timeout: u32,
+    /// Retry count when read request failed.
+    #[serde(default)]
+    pub retry_limit: u8,
+    /// Enable HTTP proxy for the read request.
+    #[serde(default)]
+    pub proxy: ProxyConfig,
+    /// Enable mirrors for the read request.
+    #[serde(default)]
+    pub mirrors: Vec<MirrorConfig>,
+}
+
 /// Container registry configuration information to access blobs.
 #[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)]
 pub struct RegistryConfig {
@@ -904,6 +968,7 @@ impl TryFrom<&BackendConfig> for BackendConfigV2 {
             oss: None,
             s3: None,
             registry: None,
+            http_proxy: None,
         };
 
         match value.backend_type.as_str() {

diff --git a/blobfs/Cargo.toml b/blobfs/Cargo.toml
@@ -31,6 +31,7 @@ virtiofs = ["fuse-backend-rs/virtiofs", "nydus-rafs/virtio-fs"]
 baekend-s3 = ["nydus-rafs/backend-s3"]
 backend-oss = ["nydus-rafs/backend-oss"]
 backend-registry = ["nydus-rafs/backend-registry"]
+backend-http-proxy = ["nydus-rafs/backend-http-proxy"]
 
 [package.metadata.docs.rs]
 all-features = true

diff --git a/clib/Cargo.toml b/clib/Cargo.toml
@@ -23,3 +23,4 @@ nydus-rafs = { version = "0.2.0", path = "../rafs" }
 baekend-s3 = ["nydus-rafs/backend-s3"]
 backend-oss = ["nydus-rafs/backend-oss"]
 backend-registry = ["nydus-rafs/backend-registry"]
+backend-http-proxy = ["nydus-rafs/backend-http-proxy"]
diff --git a/docs/nydusd.md b/docs/nydusd.md
@@ -293,6 +293,51 @@ Currently, the mirror mode is only tested in the registry backend, and in theory
 }
 ```
 
+#### HTTP proxy backend
+
+The `HttpProxy` backend can access blobs through a http proxy server which can be local (using unix socket) or remote (using `https://` or using `http://`).
+
+`HttpProxy` uses two API endpoints to access the blobs:
+- `HEAD /path/to/blobs` to get the blob size
+- `GET /path/to/blobs` to read the blob
+
+The http proxy server should respect [the `Range` header](https://www.rfc-editor.org/rfc/rfc9110.html#name-range) to compute the offset and length of the blob.
+
+The example config files for the `HttpProxy` backend may be:
+
+```
+// for remote usage
+{
+  "device": {
+      "backend": {
+      "type": "http-proxy",
+      "config": {
+        "addr": "http://127.0.0.1:9977",
+        "path": "/namespace/<repo>/blobs"
+      }
+    }
+  }
+}
+```
+
+or
+
+```
+// for remote usage
+{
+  "device": {
+      "backend": {
+      "type": "http-proxy",
+      "config": {
+        "addr": "/path/to/unix.sock",
+      }
+    }
+  }
+}
+```
+
+The `HttpProxy` backend also supports the `Proxy` and `Mirrors` configurations for remote usage like the `Registry backend` described above.
+
 ### Mount Bootstrap Via API
 
 To mount a bootstrap via api, first launch nydusd without a bootstrap:

diff --git a/rafs/Cargo.toml b/rafs/Cargo.toml
@@ -42,6 +42,7 @@ vhost-user-fs = ["fuse-backend-rs/vhost-user-fs"]
 backend-oss = ["nydus-storage/backend-oss"]
 backend-s3 = ["nydus-storage/backend-s3"]
 backend-registry = ["nydus-storage/backend-registry"]
+backend-http-proxy = ["nydus-storage/backend-http-proxy"]
 
 [package.metadata.docs.rs]
 all-features = true

diff --git a/storage/Cargo.toml b/storage/Cargo.toml
@@ -16,6 +16,8 @@ hex = "0.4.3"
 hmac = { version = "0.12.1", optional = true }
 http = { version = "0.2.8", optional = true }
 httpdate = { version = "1.0", optional = true }
+hyper = {version = "0.14.11", optional =  true}
+hyperlocal = {version = "0.8.0", optional = true}
 lazy_static = "1.4.0"
 leaky-bucket = "0.12.1"
 libc = "0.2"
@@ -27,7 +29,7 @@ serde_json = "1.0.53"
 sha2 = { version = "0.10.2", optional = true }
 tar = "0.4.38"
 time = { version = "0.3.14", features = ["formatting"], optional = true }
-tokio = { version = "1.19.0", features = ["rt", "rt-multi-thread", "sync", "time"] }
+tokio = { version = "1.19.0", features = ["macros", "rt", "rt-multi-thread", "sync", "time"] }
 url = { version = "2.1.1", optional = true }
 vm-memory = "0.9"
 fuse-backend-rs = "0.10"
@@ -47,6 +49,7 @@ backend-localfs = []
 backend-oss = ["base64", "httpdate", "hmac", "sha1", "reqwest", "url"]
 backend-registry = ["base64", "reqwest", "url"]
 backend-s3 = ["base64", "hmac", "http", "reqwest", "sha2", "time", "url"]
+backend-http-proxy = ["hyper", "hyperlocal", "http", "reqwest", "url"]
 
 [package.metadata.docs.rs]
 all-features = true

diff --git a/storage/src/backend/connection.rs b/storage/src/backend/connection.rs
@@ -23,7 +23,7 @@ use reqwest::{
     Method, StatusCode, Url,
 };
 
-use nydus_api::{MirrorConfig, OssConfig, ProxyConfig, RegistryConfig, S3Config};
+use nydus_api::{HttpProxyConfig, MirrorConfig, OssConfig, ProxyConfig, RegistryConfig, S3Config};
 use url::ParseError;
 
 const HEADER_AUTHORIZATION: &str = "Authorization";
@@ -128,6 +128,19 @@ impl From<RegistryConfig> for ConnectionConfig {
     }
 }
 
+impl From<HttpProxyConfig> for ConnectionConfig {
+    fn from(c: HttpProxyConfig) -> ConnectionConfig {
+        ConnectionConfig {
+            proxy: c.proxy,
+            mirrors: c.mirrors,
+            skip_verify: c.skip_verify,
+            timeout: c.timeout,
+            connect_timeout: c.connect_timeout,
+            retry_limit: c.retry_limit,
+        }
+    }
+}
+
 /// HTTP request data with progress callback.
 #[derive(Clone)]
 pub struct Progress<R> {