Storage Initializer: Support virtual path style in S3 (kubeflow#2887)

* s3 to support virtual path style Signed-off-by: Lize Cai <lize.cai@sap.com> * fix unit tests Signed-off-by: Lize Cai <lize.cai@sap.com> * use lower() to do case insensitive check Signed-off-by: Lize Cai <lize.cai@sap.com> --------- Signed-off-by: Lize Cai <lize.cai@sap.com>
magdalenakuhn17 · May 12, 2023 · 8618d32 · 8618d32
1 parent 5900a68
commit 8618d32
Show file tree

Hide file tree

Showing 2 changed files with 26 additions and 8 deletions.
diff --git a/python/kserve/kserve/storage/storage.py b/python/kserve/kserve/storage/storage.py
@@ -136,12 +136,21 @@ def _update_with_storage_spec():
 
     @staticmethod
     def get_S3_config():
+        # default s3 config
+        c = Config()
+
         # anon environment variable defined in s3_secret.go
-        anon = ("True" == os.getenv("awsAnonymousCredential", "false").capitalize())
+        anon = ("true" == os.getenv("awsAnonymousCredential", "false").lower())
+        # S3UseVirtualBucket environment variable defined in s3_secret.go
+        # use virtual hosted-style URLs if enabled
+        virtual = ("true" == os.getenv("S3_USER_VIRTUAL_BUCKET", "false").lower())
+
         if anon:
-            return Config(signature_version=UNSIGNED)
-        else:
-            return None
+            c = c.merge(Config(signature_version=UNSIGNED))
+        if virtual:
+            c = c.merge(Config(s3={"addressing_style": "virtual"}))
+
+        return c
 
     @staticmethod
     def _download_s3(uri, temp_dir: str):

diff --git a/python/kserve/kserve/storage/test/test_s3_storage.py b/python/kserve/kserve/storage/test/test_s3_storage.py
@@ -141,20 +141,21 @@ def test_full_name_key_root_bucket_dir(mock_storage):
 
 
 def test_get_S3_config():
+    DEFAULT_CONFIG = Config()
     ANON_CONFIG = Config(signature_version=UNSIGNED)
-    DEFAULT_CONFIG = None
+    VIRTUAL_CONFIG = Config(s3={"addressing_style": "virtual"})
 
     with mock.patch.dict(os.environ, {}):
         config1 = Storage.get_S3_config()
-    assert config1 == DEFAULT_CONFIG
+    assert vars(config1) == vars(DEFAULT_CONFIG)
 
     with mock.patch.dict(os.environ, {"awsAnonymousCredential": "False"}):
         config2 = Storage.get_S3_config()
-    assert config2 == DEFAULT_CONFIG
+    assert vars(config2) == vars(DEFAULT_CONFIG)
 
     with mock.patch.dict(os.environ, AWS_TEST_CREDENTIALS):
         config3 = Storage.get_S3_config()
-    assert config3 == DEFAULT_CONFIG
+    assert vars(config3) == vars(DEFAULT_CONFIG)
 
     with mock.patch.dict(os.environ, {"awsAnonymousCredential": "True"}):
         config4 = Storage.get_S3_config()
@@ -165,3 +166,11 @@ def test_get_S3_config():
     with mock.patch.dict(os.environ, credentials_and_anon):
         config5 = Storage.get_S3_config()
     assert config5.signature_version == ANON_CONFIG.signature_version
+
+    with mock.patch.dict(os.environ, {"S3_USER_VIRTUAL_BUCKET": "False"}):
+        config6 = Storage.get_S3_config()
+    assert vars(config6) == vars(DEFAULT_CONFIG)
+
+    with mock.patch.dict(os.environ, {"S3_USER_VIRTUAL_BUCKET": "True"}):
+        config7 = Storage.get_S3_config()
+    assert config7.s3["addressing_style"] == VIRTUAL_CONFIG.s3["addressing_style"]