Handle S3 virtual host request type

object_store/src/aws/client.rs

@@ -197,14 +197,15 @@ pub struct S3Config {
     pub region: String,
     pub endpoint: String,
     pub bucket: String,
+    pub bucket_endpoint: String,
     pub credentials: CredentialProvider,
     pub retry_config: RetryConfig,
     pub allow_http: bool,
 }
 
 impl S3Config {
     fn path_url(&self, path: &Path) -> String {
-        format!("{}/{}/{}", self.endpoint, self.bucket, encode_path(path))
+        format!("{}/{}", self.bucket_endpoint, encode_path(path))
     }
 }
 
@@ -342,7 +343,7 @@ impl S3Client {
         token: Option<&str>,
     ) -> Result<(ListResult, Option<String>)> {
         let credential = self.get_credential().await?;
-        let url = format!("{}/{}", self.config.endpoint, self.config.bucket);
+        let url = self.config.bucket_endpoint.clone();
 
         let mut query = Vec::with_capacity(4);
 
@@ -398,12 +399,7 @@ impl S3Client {
 
     pub async fn create_multipart(&self, location: &Path) -> Result<MultipartId> {
         let credential = self.get_credential().await?;
-        let url = format!(
-            "{}/{}/{}?uploads=",
-            self.config.endpoint,
-            self.config.bucket,
-            encode_path(location)
-        );
+        let url = format!("{}?uploads=", self.config.path_url(location),);
 
         let response = self
             .client

object_store/src/aws/mod.rs

@@ -357,6 +357,7 @@ pub struct AmazonS3Builder {
     retry_config: RetryConfig,
     allow_http: bool,
     imdsv1_fallback: bool,
+    virtual_hosted_style_request: bool,
     metadata_endpoint: Option<String>,
 }
 
@@ -446,10 +447,13 @@ impl AmazonS3Builder {
     }
 
     /// Sets the endpoint for communicating with AWS S3. Default value
-    /// is based on region.
+    /// is based on region. The `endpoint` field should be consistent with
+    /// the field `virtual_hosted_style_request'.
     ///
     /// For example, this might be set to `"http://localhost:4566:`
     /// for testing against a localstack instance.
+    /// If `virtual_hosted_style_request` is set to true then `endpoint`
+    /// should have bucket name included.
     pub fn with_endpoint(mut self, endpoint: impl Into<String>) -> Self {
         self.endpoint = Some(endpoint.into());
         self
@@ -469,6 +473,23 @@ impl AmazonS3Builder {
         self
     }
 
+    /// Sets if virtual hosted style request has to be used.
+    /// If `virtual_hosted_style_request` is :
+    /// * false (default):  Path style request is used
+    /// * true:  Virtual hosted style request is used
+    ///
+    /// If the `endpoint` is provided then it should be
+    /// consistent with `virtual_hosted_style_request`.
+    /// i.e. if `virtual_hosted_style_request` is set to true
+    /// then `endpoint` should have bucket name included.
+    pub fn with_virtual_hosted_style_request(
+        mut self,
+        virtual_hosted_style_request: bool,
+    ) -> Self {
+        self.virtual_hosted_style_request = virtual_hosted_style_request;
+        self
+    }
+
     /// Set the retry configuration
     pub fn with_retry(mut self, retry_config: RetryConfig) -> Self {
         self.retry_config = retry_config;
@@ -568,14 +589,29 @@ impl AmazonS3Builder {
             },
         };
 
-        let endpoint = self
-            .endpoint
-            .unwrap_or_else(|| format!("https://s3.{}.amazonaws.com", region));
+        let endpoint: String;
+        let bucket_endpoint: String;
+
+        //If `endpoint` is provided then its assumed to be consistent with
+        // `virutal_hosted_style_request`. i.e. if `virtual_hosted_style_request` is true then
+        // `endpoint` should have bucket name included.
+        if self.virtual_hosted_style_request {
+            endpoint = self.endpoint.unwrap_or_else(|| {
+                format!("https://{}.s3.{}.amazonaws.com", bucket, region)
+            });
+            bucket_endpoint = endpoint.clone();
+        } else {
+            endpoint = self
+                .endpoint
+                .unwrap_or_else(|| format!("https://s3.{}.amazonaws.com", region));
+            bucket_endpoint = format!("{}/{}", endpoint, bucket);
+        }
 
         let config = S3Config {
             region,
             endpoint,
             bucket,
+            bucket_endpoint,
             credentials,
             retry_config: self.retry_config,
             allow_http: self.allow_http,
@@ -674,6 +710,16 @@ mod tests {
                     config
                 };
 
+                let config = if let Some(virtual_hosted_style_request) =
+                    env::var("OBJECT_STORE_VIRTUAL_HOSTED_STYLE_REQUEST").ok()
+                {
+                    config.with_virtual_hosted_style_request(
+                        virtual_hosted_style_request.trim().parse().unwrap(),
+                    )
+                } else {
+                    config
+                };
+
                 config
             }
         }};