feat(services/webdav): support redirection when get 302/307 response during read operation

.github/workflows/service_test_webdav.yml

@@ -112,3 +112,38 @@ jobs:
           OPENDAL_WEBDAV_ENDPOINT: http://127.0.0.1:8080
           OPENDAL_WEBDAV_USERNAME: bar
           OPENDAL_WEBDAV_PASSWORD: bar
+
+  nginx_with_redirect:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os:
+          - ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Setup Rust toolchain
+        uses: ./.github/actions/setup
+
+      - name: Install nginx full for dav_ext modules
+        run: sudo apt install nginx-full
+
+      - name: Start nginx
+        shell: bash
+        working-directory: core
+        run: |
+          mkdir -p /tmp/static
+          mkdir -p /var/lib/nginx
+          chmod a+rw /tmp/static/ # make nginx worker has permission to operate it
+          sudo chmod 777 /var/lib/nginx/body # make nginx worker has permission to operate it
+          nginx -c `pwd`/src/services/webdav/fixtures/nginx.conf
+
+      - name: Test with redirect
+        shell: bash
+        working-directory: core
+        run: |
+          cargo test webdav -- --show-output
+        env:
+          RUST_BACKTRACE: full
+          RUST_LOG: debug
+          OPENDAL_WEBDAV_TEST: on
+          OPENDAL_WEBDAV_ENDPOINT: http://127.0.0.1:8081

core/src/services/webdav/backend.rs

@@ -15,6 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
+use std::borrow::Cow;
 use std::collections::HashMap;
 use std::fmt::Debug;
 use std::fmt::Formatter;
@@ -27,6 +28,8 @@ use http::Request;
 use http::Response;
 use http::StatusCode;
 use log::debug;
+use percent_encoding::percent_decode_str;
+use reqwest::Url;
 
 use super::error::parse_error;
 use super::list_response::Multistatus;
@@ -199,7 +202,7 @@ impl Builder for WebdavBuilder {
             Some(v) => v,
             None => {
                 return Err(Error::new(ErrorKind::ConfigInvalid, "endpoint is empty")
-                    .with_context("service", Scheme::Webdav))
+                    .with_context("service", Scheme::Webdav));
             }
         };
 
@@ -299,17 +302,81 @@ impl Accessor for WebdavBackend {
     }
 
     async fn read(&self, path: &str, args: OpRead) -> Result<(RpRead, Self::Reader)> {
-        let resp = self.webdav_get(path, args.range()).await?;
-
-        let status = resp.status();
-
-        match status {
-            StatusCode::OK | StatusCode::PARTIAL_CONTENT => {
-                let meta = parse_into_metadata(path, resp.headers())?;
-                Ok((RpRead::with_metadata(meta), resp.into_body()))
+        let mut remaining_retry_times = 3;
+        // if response indicates that we should redirect
+        // then modify this variable
+        let mut override_endpoint: Option<String> = None;
+        // sometimes path will also changed according to redirect
+        // response
+        let mut override_path: String = path.to_string();
+
+        debug!(
+            "will read with maximum {} times to retry: path={}",
+            &path, remaining_retry_times
+        );
+        while remaining_retry_times > 0 {
+            debug!("remaining retry times: {}", remaining_retry_times);
+            let resp = self
+                .webdav_get(&override_path, args.range(), override_endpoint.clone())
+                .await?;
+            let status = resp.status();
+            match status {
+                StatusCode::OK | StatusCode::PARTIAL_CONTENT => {
+                    let meta = parse_into_metadata(path, resp.headers())?;
+                    return Ok((RpRead::with_metadata(meta), resp.into_body()));
+                }
+                StatusCode::FOUND | StatusCode::TEMPORARY_REDIRECT => {
+                    // if server returns redirect HTTP status, then redirect it
+                    let redirected_url = parse_location(resp.headers())?
+                        // no location means invalid redirect response
+                        .ok_or_else(|| {
+                            Error::new(
+                                ErrorKind::Unexpected,
+                                "no location header in redirect response.",
+                            )
+                            .with_operation(Operation::Read)
+                        })?;
+                    debug!(
+                        "received status code 302/307, will redirect request, url: {}",
+                        redirected_url
+                    );
+
+                    // first the url in location should be valid
+                    let redirected_url = Url::parse(redirected_url).map_err(|e| {
+                        Error::new(
+                            ErrorKind::Unexpected,
+                            &format!("redirected url({redirected_url}) is not valid."),
+                        )
+                        .with_operation(Operation::Read)
+                        .set_source(e)
+                    })?;
+
+                    // basic security check, the redirected url should have the same origin with original url
+                    // if not, it will not send request with auth
+                    let path = redirected_url.path();
+                    // url escape decode to avoid special case
+                    let path = percent_decode_str(path)
+                        .decode_utf8()
+                        .unwrap_or_else(|_| Cow::from(path))
+                        .into_owned();
+                    // if root is the prefix of path, then remove it
+                    // this is for the case that redirect only change the origin of url
+                    override_path = path.strip_prefix(&self.root).unwrap_or(&path).to_string();
+                    override_endpoint = Some(redirected_url.origin().unicode_serialization())
+                }
+                _ => return Err(parse_error(resp).await?),
             }
-            _ => Err(parse_error(resp).await?),
+
+            remaining_retry_times -= 1
         }
+        return Err(Error::new(
+            ErrorKind::Unexpected,
+            &format!(
+                "reach maximum retry times for requesting redirected endpoint({:?}), path({})",
+                override_endpoint, override_path,
+            ),
+        )
+        .with_operation(Operation::Read));
     }
 
     async fn write(&self, path: &str, args: OpWrite) -> Result<(RpWrite, Self::Writer)> {
@@ -449,15 +516,22 @@ impl WebdavBackend {
         &self,
         path: &str,
         range: BytesRange,
+        override_endpoint: Option<String>,
     ) -> Result<Response<IncomingAsyncBody>> {
         let p = build_rooted_abs_path(&self.root, path);
-
-        let url: String = format!("{}{}", self.endpoint, percent_encode_path(&p));
+        // user can give one new endpoint to override default endpoint
+        // this case happens when receive redirect response from server
+        let endpoint = override_endpoint.unwrap_or_else(|| self.endpoint.clone());
+        // if the override endpoint differs from original endpoint
+        // we will not send auth to server due to security issue.
+        let send_auth = endpoint.eq(&self.endpoint);
+        let url: String = format!("{}{}", endpoint, percent_encode_path(&p));
 
         let mut req = Request::get(&url);
 
-        if let Some(auth) = &self.authorization {
-            req = req.header(header::AUTHORIZATION, auth.clone())
+        match &self.authorization {
+            Some(auth) if send_auth => req = req.header(header::AUTHORIZATION, auth.clone()),
+            _ => (),
         }
 
         if !range.is_full() {

core/src/services/webdav/fixtures/nginx.conf

@@ -8,6 +8,23 @@ events {
 }
 
 http {
+  # the following configuration is for redirect test
+  server {
+    listen       127.0.0.1:8081;
+    server_name  localhost;
+    access_log   /tmp/forward-access.log;
+    error_log    /tmp/forward-error.log;
+
+    location / {
+      if ($request_method = GET) {
+        return 302 http://$server_name:8080$request_uri;
+      }
+      client_max_body_size 1024M;
+      # forward all other requests to port 8080
+      proxy_pass http://127.0.0.1:8080;
+    }
+  }
+
   server {
     listen       127.0.0.1:8080;
     server_name  localhost;