Skip to content

Commit

Permalink
Split hdfs code path in parse_url_opts to separate function
Browse files Browse the repository at this point in the history
  • Loading branch information
SchutteJan committed Jul 15, 2024
1 parent 1f57962 commit 1797a95
Showing 1 changed file with 21 additions and 15 deletions.
36 changes: 21 additions & 15 deletions kernel/src/engine/default/storage.rs
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
#[cfg(feature = "cloud")]
use hdfs_native_object_store::HdfsObjectStore;
use object_store::parse_url_opts as object_store_parse_url_opts;
use object_store::parse_url_opts as parse_url_opts_object_store;
use object_store::path::Path;
use object_store::{Error, ObjectStore};
use url::Url;
Expand All @@ -11,19 +11,25 @@ where
K: AsRef<str>,
V: Into<String>,
{
#[cfg(feature = "cloud")]
{
let scheme = url.scheme();
if scheme == "hdfs" || scheme == "viewfs" {
let options_map = options
.into_iter()
.map(|(k, v)| (k.as_ref().to_string(), v.into()))
.collect();
let store = HdfsObjectStore::with_config(url.as_str(), options_map)?;
let path = Path::parse(url.path())?;
return Ok((Box::new(store), path));
}
match url.scheme() {
#[cfg(feature = "cloud")]
"hdfs" | "viewfs" => parse_url_opts_hdfs_native(url, options),
_ => parse_url_opts_object_store(url, options)
}

object_store_parse_url_opts(url, options)
}

#[cfg(feature = "cloud")]
pub fn parse_url_opts_hdfs_native<I, K, V>(url: &Url, options: I) -> Result<(Box<dyn ObjectStore>, Path), Error>
where
I: IntoIterator<Item = (K, V)>,
K: AsRef<str>,
V: Into<String>,
{
let options_map = options
.into_iter()
.map(|(k, v)| (k.as_ref().to_string(), v.into()))
.collect();
let store = HdfsObjectStore::with_config(url.as_str(), options_map)?;
let path = Path::parse(url.path())?;
return Ok((Box::new(store), path));
}

0 comments on commit 1797a95

Please sign in to comment.