feat(connector): allow custom s3 endpoint (#8246)

risingwavelabs · Mar 1, 2023 · c75863a · c75863a
1 parent 76af020
commit c75863a
Show file tree

Hide file tree

Showing 3 changed files with 8 additions and 0 deletions.
diff --git a/src/connector/src/source/filesystem/s3/enumerator.rs b/src/connector/src/source/filesystem/s3/enumerator.rs
@@ -149,6 +149,7 @@ mod tests {
             match_pattern: Some("happy[0-9].csv".to_owned()),
             access: None,
             secret: None,
+            endpoint_url: None,
         };
         let mut enumerator = S3SplitEnumerator::new(props.clone()).await.unwrap();
         let splits = enumerator.list_splits().await.unwrap();

diff --git a/src/connector/src/source/filesystem/s3/mod.rs b/src/connector/src/source/filesystem/s3/mod.rs
@@ -33,6 +33,8 @@ pub struct S3Properties {
     pub access: Option<String>,
     #[serde(rename = "s3.credentials.secret", default)]
     pub secret: Option<String>,
+    #[serde(rename = "s3.endpoint_url")]
+    endpoint_url: Option<String>,
 }
 
 impl From<S3Properties> for HashMap<String, String> {
@@ -46,6 +48,10 @@ impl From<S3Properties> for HashMap<String, String> {
         if props.secret.is_some() {
             m.insert("secret_access".to_owned(), props.secret.unwrap());
         }
+        if props.endpoint_url.is_some() {
+            m.insert("endpoint_url".to_owned(), props.endpoint_url.unwrap());
+        }
+
         m
     }
 }
diff --git a/src/connector/src/source/filesystem/s3/source/reader.rs b/src/connector/src/source/filesystem/s3/source/reader.rs
@@ -231,6 +231,7 @@ mod tests {
             match_pattern: None,
             access: None,
             secret: None,
+            endpoint_url: None,
         };
         let mut enumerator = S3SplitEnumerator::new(props.clone()).await.unwrap();
         let splits = enumerator.list_splits().await.unwrap();