From db0fd6195e75cfb0f347f23be13064b83b33b41e Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Wed, 13 Nov 2024 23:36:23 +0800 Subject: [PATCH] refactor: Remove metakey concept Signed-off-by: Xuanwo --- bin/oay/src/services/s3/service.rs | 2 - bindings/java/src/error.rs | 2 +- bindings/java/src/lib.rs | 85 +++----- bindings/java/src/operator_input_stream.rs | 2 +- core/src/layers/complete.rs | 14 +- core/src/layers/mime_guess.rs | 5 +- core/src/raw/ops.rs | 23 --- core/src/services/fs/backend.rs | 8 +- core/src/services/fs/lister.rs | 75 ++----- core/src/services/webdav/core.rs | 2 +- core/src/types/entry.rs | 120 ----------- core/src/types/list.rs | 190 +++--------------- core/src/types/metadata.rs | 167 --------------- core/src/types/mod.rs | 1 - core/src/types/operator/blocking_operator.rs | 103 ---------- core/src/types/operator/operator.rs | 155 -------------- core/src/types/operator/operator_functions.rs | 28 --- core/src/types/operator/operator_futures.rs | 43 ---- core/tests/behavior/async_list.rs | 126 ------------ core/tests/behavior/blocking_list.rs | 91 --------- integrations/cloud_filter/src/lib.rs | 3 +- integrations/compat/Cargo.toml | 2 +- integrations/fuse3/src/file_system.rs | 2 - integrations/object_store/src/store.rs | 16 +- integrations/object_store/src/utils.rs | 16 +- 25 files changed, 80 insertions(+), 1201 deletions(-) diff --git a/bin/oay/src/services/s3/service.rs b/bin/oay/src/services/s3/service.rs index b30bae284b6d..db40cafb7f8f 100644 --- a/bin/oay/src/services/s3/service.rs +++ b/bin/oay/src/services/s3/service.rs @@ -26,7 +26,6 @@ use axum::routing::get; use axum::Router; use chrono::SecondsFormat; use futures_util::StreamExt; -use opendal::Metakey; use opendal::Operator; use serde::Deserialize; use serde::Serialize; @@ -100,7 +99,6 @@ async fn handle_list_objects( .op .lister_with(¶ms.prefix) .start_after(¶ms.start_after) - .metakey(Metakey::Mode | Metakey::LastModified | Metakey::Etag | Metakey::ContentLength) .await? .chunks(1000); diff --git a/bindings/java/src/error.rs b/bindings/java/src/error.rs index 598050463a95..eb5cf30d8ee2 100644 --- a/bindings/java/src/error.rs +++ b/bindings/java/src/error.rs @@ -83,7 +83,7 @@ impl From for Error { impl From for Error { fn from(err: jni::errors::Error) -> Self { - opendal::Error::new(ErrorKind::Unexpected, &err.to_string()).into() + opendal::Error::new(ErrorKind::Unexpected, err.to_string()).into() } } diff --git a/bindings/java/src/lib.rs b/bindings/java/src/lib.rs index a7a67334b05b..47113ae74a11 100644 --- a/bindings/java/src/lib.rs +++ b/bindings/java/src/lib.rs @@ -28,7 +28,6 @@ use opendal::Capability; use opendal::Entry; use opendal::EntryMode; use opendal::Metadata; -use opendal::Metakey; use opendal::OperatorInfo; mod async_operator; @@ -51,7 +50,7 @@ fn make_presigned_request<'a>(env: &mut JNIEnv<'a>, req: PresignedRequest) -> Re for (k, v) in req.header().iter() { let key = k.to_string(); let value = v.to_str().map_err(|err| { - opendal::Error::new(opendal::ErrorKind::Unexpected, &err.to_string()) + opendal::Error::new(opendal::ErrorKind::Unexpected, err.to_string()) })?; map.insert(key, value.to_owned()); } @@ -143,72 +142,36 @@ fn make_metadata<'a>(env: &mut JNIEnv<'a>, metadata: Metadata) -> Result 2, }; - let metakey = metadata.metakey(); - - let contains_metakey = |k| metakey.contains(k) || metakey.contains(Metakey::Complete); - - let last_modified = if contains_metakey(Metakey::LastModified) { - metadata.last_modified().map_or_else( - || Ok::, error::Error>(JObject::null()), - |v| { - Ok(env - .call_static_method( - "java/time/Instant", - "ofEpochSecond", - "(JJ)Ljava/time/Instant;", - &[ - JValue::Long(v.timestamp()), - JValue::Long(v.timestamp_subsec_nanos() as jlong), - ], - )? - .l()?) - }, - )? - } else { - JObject::null() - }; + let last_modified = metadata.last_modified().map_or_else( + || Ok::, error::Error>(JObject::null()), + |v| { + Ok(env + .call_static_method( + "java/time/Instant", + "ofEpochSecond", + "(JJ)Ljava/time/Instant;", + &[ + JValue::Long(v.timestamp()), + JValue::Long(v.timestamp_subsec_nanos() as jlong), + ], + )? + .l()?) + }, + )?; - let cache_control = if contains_metakey(Metakey::CacheControl) { - convert::string_to_jstring(env, metadata.cache_control())? - } else { - JObject::null() - }; + let cache_control = convert::string_to_jstring(env, metadata.cache_control())?; - let content_disposition = if contains_metakey(Metakey::ContentDisposition) { - convert::string_to_jstring(env, metadata.content_disposition())? - } else { - JObject::null() - }; + let content_disposition = convert::string_to_jstring(env, metadata.content_disposition())?; - let content_md5 = if contains_metakey(Metakey::ContentMd5) { - convert::string_to_jstring(env, metadata.content_md5())? - } else { - JObject::null() - }; + let content_md5 = convert::string_to_jstring(env, metadata.content_md5())?; - let content_type = if contains_metakey(Metakey::ContentType) { - convert::string_to_jstring(env, metadata.content_type())? - } else { - JObject::null() - }; + let content_type = convert::string_to_jstring(env, metadata.content_type())?; - let etag = if contains_metakey(Metakey::Etag) { - convert::string_to_jstring(env, metadata.etag())? - } else { - JObject::null() - }; + let etag = convert::string_to_jstring(env, metadata.etag())?; - let version = if contains_metakey(Metakey::Version) { - convert::string_to_jstring(env, metadata.version())? - } else { - JObject::null() - }; + let version = convert::string_to_jstring(env, metadata.version())?; - let content_length = if contains_metakey(Metakey::ContentLength) { - metadata.content_length() as jlong - } else { - -1 - }; + let content_length = metadata.content_length() as jlong; let result = env .new_object( diff --git a/bindings/java/src/operator_input_stream.rs b/bindings/java/src/operator_input_stream.rs index 7af8f31a30fe..105d15154d12 100644 --- a/bindings/java/src/operator_input_stream.rs +++ b/bindings/java/src/operator_input_stream.rs @@ -87,7 +87,7 @@ fn intern_read_next_bytes( match reader .next() .transpose() - .map_err(|err| opendal::Error::new(opendal::ErrorKind::Unexpected, &err.to_string()))? + .map_err(|err| opendal::Error::new(opendal::ErrorKind::Unexpected, err.to_string()))? { None => Ok(JObject::null().into_raw()), Some(content) => { diff --git a/core/src/layers/complete.rs b/core/src/layers/complete.rs index ad26e5b7bcc4..2d178763852d 100644 --- a/core/src/layers/complete.rs +++ b/core/src/layers/complete.rs @@ -209,12 +209,7 @@ impl CompleteAccessor { } // Forward to underlying storage directly since we don't know how to handle stat dir. - self.inner.stat(path, args).await.map(|v| { - v.map_metadata(|m| { - let bit = m.metakey(); - m.with_metakey(bit | Metakey::Complete) - }) - }) + self.inner.stat(path, args).await } fn complete_blocking_stat(&self, path: &str, args: OpStat) -> Result { @@ -258,12 +253,7 @@ impl CompleteAccessor { } // Forward to underlying storage directly since we don't know how to handle stat dir. - self.inner.blocking_stat(path, args).map(|v| { - v.map_metadata(|m| { - let bit = m.metakey(); - m.with_metakey(bit | Metakey::Complete) - }) - }) + self.inner.blocking_stat(path, args) } async fn complete_list( diff --git a/core/src/layers/mime_guess.rs b/core/src/layers/mime_guess.rs index 7a4bc08b8aa2..1a4f460e168a 100644 --- a/core/src/layers/mime_guess.rs +++ b/core/src/layers/mime_guess.rs @@ -161,7 +161,6 @@ impl LayeredAccess for MimeGuessAccessor { mod tests { use super::*; use crate::services::Memory; - use crate::Metakey; use crate::Operator; const DATA: &str = "test"; @@ -196,7 +195,7 @@ mod tests { Some(CUSTOM) ); - let entries = op.list_with("").metakey(Metakey::Complete).await.unwrap(); + let entries = op.list_with("").await.unwrap(); assert_eq!(entries[0].metadata().content_type(), Some(HTML)); assert_eq!(entries[1].metadata().content_type(), None); assert_eq!(entries[2].metadata().content_type(), Some(CUSTOM)); @@ -222,7 +221,7 @@ mod tests { .unwrap(); assert_eq!(op.stat("test2.html").unwrap().content_type(), Some(CUSTOM)); - let entries = op.list_with("").metakey(Metakey::Complete).call().unwrap(); + let entries = op.list_with("").call().unwrap(); assert_eq!(entries[0].metadata().content_type(), Some(HTML)); assert_eq!(entries[1].metadata().content_type(), None); assert_eq!(entries[2].metadata().content_type(), Some(CUSTOM)); diff --git a/core/src/raw/ops.rs b/core/src/raw/ops.rs index 5875160b068f..69615e0c9466 100644 --- a/core/src/raw/ops.rs +++ b/core/src/raw/ops.rs @@ -24,7 +24,6 @@ use std::time::Duration; use crate::raw::*; use crate::*; -use flagset::FlagSet; /// Args for `create` operation. /// @@ -85,13 +84,6 @@ pub struct OpList { /// /// Default to `false`. recursive: bool, - /// Metakey is used to control which meta should be returned. - /// - /// Lister will make sure the result for specified meta is **known**: - /// - /// - `Some(v)` means exist. - /// - `None` means services doesn't have this meta. - metakey: FlagSet, /// The concurrent of stat operations inside list operation. /// Users could use this to control the number of concurrent stat operation when metadata is unknown. /// @@ -115,8 +107,6 @@ impl Default for OpList { limit: None, start_after: None, recursive: false, - // By default, we want to know what's the mode of this entry. - metakey: Metakey::Mode.into(), concurrent: 1, version: false, } @@ -167,19 +157,6 @@ impl OpList { self.recursive } - /// Change the metakey of this list operation. - /// - /// The default metakey is `Metakey::Mode`. - pub fn with_metakey(mut self, metakey: impl Into>) -> Self { - self.metakey = metakey.into(); - self - } - - /// Get the current metakey. - pub fn metakey(&self) -> FlagSet { - self.metakey - } - /// Change the concurrent of this list operation. /// /// The default concurrent is 1. diff --git a/core/src/services/fs/backend.rs b/core/src/services/fs/backend.rs index a3c40ead1330..2572361e35dd 100644 --- a/core/src/services/fs/backend.rs +++ b/core/src/services/fs/backend.rs @@ -342,7 +342,7 @@ impl Access for FsBackend { } } - async fn list(&self, path: &str, arg: OpList) -> Result<(RpList, Self::Lister)> { + async fn list(&self, path: &str, _: OpList) -> Result<(RpList, Self::Lister)> { let p = self.core.root.join(path.trim_end_matches('/')); let f = match tokio::fs::read_dir(&p).await { @@ -356,7 +356,7 @@ impl Access for FsBackend { } }; - let rd = FsLister::new(&self.core.root, path, f, arg); + let rd = FsLister::new(&self.core.root, path, f); Ok((RpList::default(), Some(rd))) } @@ -511,7 +511,7 @@ impl Access for FsBackend { } } - fn blocking_list(&self, path: &str, arg: OpList) -> Result<(RpList, Self::BlockingLister)> { + fn blocking_list(&self, path: &str, _: OpList) -> Result<(RpList, Self::BlockingLister)> { let p = self.core.root.join(path.trim_end_matches('/')); let f = match std::fs::read_dir(p) { @@ -525,7 +525,7 @@ impl Access for FsBackend { } }; - let rd = FsLister::new(&self.core.root, path, f, arg); + let rd = FsLister::new(&self.core.root, path, f); Ok((RpList::default(), Some(rd))) } diff --git a/core/src/services/fs/lister.rs b/core/src/services/fs/lister.rs index adafd2c9dd38..fe9e024991b5 100644 --- a/core/src/services/fs/lister.rs +++ b/core/src/services/fs/lister.rs @@ -21,7 +21,6 @@ use std::path::PathBuf; use crate::raw::*; use crate::EntryMode; use crate::Metadata; -use crate::Metakey; use crate::Result; pub struct FsLister

{ @@ -30,17 +29,14 @@ pub struct FsLister

{ current_path: Option, rd: P, - - op: OpList, } impl

FsLister

{ - pub fn new(root: &Path, path: &str, rd: P, arg: OpList) -> Self { + pub fn new(root: &Path, path: &str, rd: P) -> Self { Self { root: root.to_owned(), current_path: Some(path.to_string()), rd, - op: arg, } } } @@ -71,38 +67,15 @@ impl oio::List for FsLister { .replace('\\', "/"), ); - let default_meta = self.op.metakey() == Metakey::Mode; - - let metadata = if default_meta { - let ft = de.file_type().await.map_err(new_std_io_error)?; - if ft.is_file() { - Metadata::new(EntryMode::FILE) - } else if ft.is_dir() { - Metadata::new(EntryMode::DIR) - } else { - Metadata::new(EntryMode::Unknown) - } - } else { - let fs_meta = de.metadata().await.map_err(new_std_io_error)?; - let mut meta = if fs_meta.file_type().is_file() { - Metadata::new(EntryMode::FILE) - } else if fs_meta.file_type().is_dir() { - Metadata::new(EntryMode::DIR) - } else { - Metadata::new(EntryMode::Unknown) - }; - meta.set_content_length(fs_meta.len()); - meta.set_last_modified(fs_meta.modified().map_err(new_std_io_error)?.into()); - meta - }; - - let entry = if metadata.is_dir() { + let ft = de.file_type().await.map_err(new_std_io_error)?; + let entry = if ft.is_dir() { // Make sure we are returning the correct path. - oio::Entry::new(&format!("{rel_path}/"), metadata) + oio::Entry::new(&format!("{rel_path}/"), Metadata::new(EntryMode::DIR)) + } else if ft.is_file() { + oio::Entry::new(&rel_path, Metadata::new(EntryMode::FILE)) } else { - oio::Entry::new(&rel_path, metadata) + oio::Entry::new(&rel_path, Metadata::new(EntryMode::Unknown)) }; - Ok(Some(entry)) } } @@ -129,36 +102,14 @@ impl oio::BlockingList for FsLister { .replace('\\', "/"), ); - let default_meta = self.op.metakey() == Metakey::Mode; - - let metadata = if default_meta { - let ft = de.file_type().map_err(new_std_io_error)?; - if ft.is_file() { - Metadata::new(EntryMode::FILE) - } else if ft.is_dir() { - Metadata::new(EntryMode::DIR) - } else { - Metadata::new(EntryMode::Unknown) - } - } else { - let fs_meta = de.metadata().map_err(new_std_io_error)?; - let mut meta = if fs_meta.file_type().is_file() { - Metadata::new(EntryMode::FILE) - } else if fs_meta.file_type().is_dir() { - Metadata::new(EntryMode::DIR) - } else { - Metadata::new(EntryMode::Unknown) - }; - meta.set_content_length(fs_meta.len()); - meta.set_last_modified(fs_meta.modified().map_err(new_std_io_error)?.into()); - meta - }; - - let entry = if metadata.is_dir() { + let ft = de.file_type().map_err(new_std_io_error)?; + let entry = if ft.is_dir() { // Make sure we are returning the correct path. - oio::Entry::new(&format!("{rel_path}/"), metadata) + oio::Entry::new(&format!("{rel_path}/"), Metadata::new(EntryMode::DIR)) + } else if ft.is_file() { + oio::Entry::new(&rel_path, Metadata::new(EntryMode::FILE)) } else { - oio::Entry::new(&rel_path, metadata) + oio::Entry::new(&rel_path, Metadata::new(EntryMode::Unknown)) }; Ok(Some(entry)) diff --git a/core/src/services/webdav/core.rs b/core/src/services/webdav/core.rs index 41026454e587..71efd0a960ab 100644 --- a/core/src/services/webdav/core.rs +++ b/core/src/services/webdav/core.rs @@ -412,7 +412,7 @@ pub fn parse_propstat(propstat: &Propstat) -> Result { m.set_last_modified(parse_datetime_from_rfc2822(getlastmodified)?); // the storage services have returned all the properties - Ok(m.with_metakey(Metakey::Complete)) + Ok(m) } #[derive(Deserialize, Debug, PartialEq, Eq, Clone, Default)] diff --git a/core/src/types/entry.rs b/core/src/types/entry.rs index 5e699deaf2c4..9fe4510806fe 100644 --- a/core/src/types/entry.rs +++ b/core/src/types/entry.rs @@ -19,44 +19,6 @@ use crate::raw::*; use crate::*; /// Entry returned by [`Lister`] or [`BlockingLister`] to represent a path and it's relative metadata. -/// -/// # Notes -/// -/// Entry returned by [`Lister`] or [`BlockingLister`] may carry some already known metadata. -/// Lister by default only make sure that `Mode` is fetched. To make sure the entry contains -/// metadata you want, please use `list_with` or `lister_with` and `metakey`. -/// -/// For example: -/// -/// ```no_run -/// # use anyhow::Result; -/// use opendal::EntryMode; -/// use opendal::Metakey; -/// use opendal::Operator; -/// # async fn test(op: Operator) -> Result<()> { -/// let mut entries = op -/// .list_with("dir/") -/// .metakey(Metakey::ContentLength | Metakey::LastModified) -/// .await?; -/// for entry in entries { -/// let meta = entry.metadata(); -/// match meta.mode() { -/// EntryMode::FILE => { -/// println!( -/// "Handling file {} with size {}", -/// entry.path(), -/// meta.content_length() -/// ) -/// } -/// EntryMode::DIR => { -/// println!("Handling dir {}", entry.path()) -/// } -/// EntryMode::Unknown => continue, -/// } -/// } -/// # Ok(()) -/// # } -/// ``` #[derive(Clone, Debug)] pub struct Entry { /// Path of this entry. @@ -96,93 +58,11 @@ impl Entry { } /// Fetch metadata of this entry. - /// - /// # Notes - /// - /// Metadata only guaranteed to have results of `metakey` (which default to `Metakey::Mode`). - /// - /// - `Some(T)` means the metadata is valid. - /// - `None` means the metadata is not provided by services. - /// - /// Visiting a metadata that not covered by `metakey` could result in panic. - /// - /// # Examples - /// - /// Please use `metakey` to specify the metadata you want, for example: - /// - /// ```no_run - /// # use anyhow::Result; - /// use opendal::EntryMode; - /// use opendal::Metakey; - /// use opendal::Operator; - /// # async fn test(op: Operator) -> Result<()> { - /// let mut entries = op - /// .list_with("dir/") - /// .metakey(Metakey::ContentLength | Metakey::LastModified) - /// .await?; - /// for entry in entries { - /// let meta = entry.metadata(); - /// match meta.mode() { - /// EntryMode::FILE => { - /// println!( - /// "Handling file {} with size {}", - /// entry.path(), - /// meta.content_length() - /// ) - /// } - /// EntryMode::DIR => { - /// println!("Handling dir {}", entry.path()) - /// } - /// EntryMode::Unknown => continue, - /// } - /// } - /// # Ok(()) - /// # } - /// ``` pub fn metadata(&self) -> &Metadata { &self.metadata } /// Consume this entry to get it's path and metadata. - /// - /// # Notes - /// - /// Metadata only guaranteed to have results of `metakey` (which default to `Metakey::Mode`). - /// - /// - `Some(T)` means the metadata is valid. - /// - `None` means the metadata is not provided by services. - /// - /// Visiting a metadata that not covered by `metakey` could result in panic. - /// - /// # Examples - /// - /// Please use `metakey` to specify the metadata you want, for example: - /// - /// ```no_run - /// # use anyhow::Result; - /// use opendal::EntryMode; - /// use opendal::Metakey; - /// use opendal::Operator; - /// # async fn test(op: Operator) -> Result<()> { - /// let mut entries = op - /// .list_with("dir/") - /// .metakey(Metakey::ContentLength | Metakey::LastModified) - /// .await?; - /// for entry in entries { - /// let (path, meta) = entry.into_parts(); - /// match meta.mode() { - /// EntryMode::FILE => { - /// println!("Handling file {} with size {}", path, meta.content_length()) - /// } - /// EntryMode::DIR => { - /// println!("Handling dir {}", path) - /// } - /// EntryMode::Unknown => continue, - /// } - /// } - /// # Ok(()) - /// # } - /// ``` pub fn into_parts(self) -> (String, Metadata) { (self.path, self.metadata) } diff --git a/core/src/types/list.rs b/core/src/types/list.rs index 8b3bb341b4fe..e53781dc1e04 100644 --- a/core/src/types/list.rs +++ b/core/src/types/list.rs @@ -15,16 +15,12 @@ // specific language governing permissions and limitations // under the License. -use std::cmp; -use std::future::Future; use std::pin::Pin; use std::task::ready; use std::task::Context; use std::task::Poll; -use flagset::FlagSet; use futures::Stream; -use futures::StreamExt; use crate::raw::*; use crate::*; @@ -32,93 +28,15 @@ use crate::*; /// Lister is designed to list entries at given path in an asynchronous /// manner. /// -/// Users can construct Lister by [`Operator::lister`] or [`Operator::lister_with`], and can use `metakey` along with list. -/// For example, suppose you need to access `content_length`, you can bring the corresponding field in metakey when listing: -/// `op.list_with("dir/").metakey(Metakey::ContentLength).await?;`. -/// /// - Lister implements `Stream>`. /// - Lister will return `None` if there is no more entries or error has been returned. pub struct Lister { - acc: Accessor, lister: Option, - /// required_metakey is the metakey required by users. - required_metakey: FlagSet, fut: Option>)>>, - - /// tasks is used to store tasks that are run in concurrent. - /// - /// TODO: maybe we should move logic inside? - tasks: ConcurrentFutures, errored: bool, } -/// StatTask is used to store the task that is run in concurrent. -/// -/// # Note for clippy -/// -/// Clippy will raise error for this enum like the following: -/// -/// ```shell -/// error: large size difference between variants -/// --> core/src/types/list.rs:64:1 -/// | -/// 64 | / enum StatTask { -/// 65 | | /// BoxFuture is used to store the join handle of spawned task. -/// 66 | | Handle(BoxFuture<(String, Result)>), -/// | | -------------------------------------------- the second-largest variant contains at least 0 bytes -/// 67 | | /// KnownEntry is used to store the entry that already contains the required metakey. -/// 68 | | KnownEntry(Option), -/// | | ------------------------- the largest variant contains at least 264 bytes -/// 69 | | } -/// | |_^ the entire enum is at least 0 bytes -/// | -/// = help: for further information visit https://rust-lang.github.io/rust-clippy/master/index.html#large_enum_variant -/// = note: `-D clippy::large-enum-variant` implied by `-D warnings` -/// = help: to override `-D warnings` add `#[allow(clippy::large_enum_variant)]` -/// help: consider boxing the large fields to reduce the total size of the enum -/// | -/// 68 | KnownEntry(Box>), -/// | ~~~~~~~~~~~~~~~~~~ -/// ``` -/// But this lint is wrong since it doesn't take the generic param JoinHandle into account. In fact, they have exactly -/// the same size: -/// -/// ```rust -/// use std::mem::size_of; -/// -/// use opendal::Entry; -/// use opendal::Result; -/// -/// assert_eq!(304, size_of::<(String, Result)>()); -/// assert_eq!(304, size_of::>()); -/// ``` -/// -/// So let's ignore this lint: -#[allow(clippy::large_enum_variant)] -enum StatTask { - /// Stating is used to store the join handle of spawned task. - /// - /// TODO: Replace with static future type after rust supported. - Stating(BoxedStaticFuture<(String, Result)>), - /// Known is used to store the entry that already contains the required metakey. - Known(Option<(String, Metadata)>), -} - -impl Future for StatTask { - type Output = (String, Result); - - fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { - match self.get_mut() { - StatTask::Stating(fut) => Pin::new(fut).poll(cx), - StatTask::Known(entry) => { - let (path, metadata) = entry.take().expect("entry should not be None"); - Poll::Ready((path, Ok(metadata))) - } - } - } -} - /// # Safety /// /// Lister will only be accessed by `&mut Self` @@ -127,18 +45,12 @@ unsafe impl Sync for Lister {} impl Lister { /// Create a new lister. pub(crate) async fn create(acc: Accessor, path: &str, args: OpList) -> Result { - let required_metakey = args.metakey(); - let concurrent = cmp::max(1, args.concurrent()); - let (_, lister) = acc.list(path, args).await?; Ok(Self { - acc, lister: Some(lister), - required_metakey, fut: None, - tasks: ConcurrentFutures::new(concurrent), errored: false, }) } @@ -153,60 +65,33 @@ impl Stream for Lister { return Poll::Ready(None); } - // Trying to pull more tasks if there are more space. - if self.tasks.has_remaining() { - // Building future if we have a lister available. - if let Some(mut lister) = self.lister.take() { - let fut = async move { - let res = lister.next_dyn().await; - (lister, res) - }; - self.fut = Some(Box::pin(fut)); - } + if let Some(mut lister) = self.lister.take() { + let fut = async move { + let res = lister.next_dyn().await; + (lister, res) + }; + self.fut = Some(Box::pin(fut)); + } - if let Some(fut) = self.fut.as_mut() { - if let Poll::Ready((lister, entry)) = fut.as_mut().poll(cx) { - self.lister = Some(lister); - self.fut = None; + if let Some(fut) = self.fut.as_mut() { + let (lister, entry) = ready!(fut.as_mut().poll(cx)); + self.lister = Some(lister); + self.fut = None; - match entry { - Ok(Some(oe)) => { - let (path, metadata) = oe.into_entry().into_parts(); - if metadata.contains_metakey(self.required_metakey) { - self.tasks - .push_back(StatTask::Known(Some((path, metadata)))); - } else { - let acc = self.acc.clone(); - let fut = async move { - let res = acc.stat(&path, OpStat::default()).await; - (path, res.map(|rp| rp.into_metadata())) - }; - self.tasks.push_back(StatTask::Stating(Box::pin(fut))); - } - } - Ok(None) => { - self.lister = None; - } - Err(err) => { - self.errored = true; - return Poll::Ready(Some(Err(err))); - } - } + return match entry { + Ok(Some(oe)) => Poll::Ready(Some(Ok(oe.into_entry()))), + Ok(None) => { + self.lister = None; + Poll::Ready(None) } - } - } - - // Try to poll tasks - if let Some((path, rp)) = ready!(self.tasks.poll_next_unpin(cx)) { - let metadata = rp?; - return Poll::Ready(Some(Ok(Entry::new(path, metadata)))); + Err(err) => { + self.errored = true; + Poll::Ready(Some(Err(err))) + } + }; } - if self.lister.is_some() || self.fut.is_some() { - Poll::Pending - } else { - Poll::Ready(None) - } + Poll::Ready(None) } } @@ -218,10 +103,6 @@ impl Stream for Lister { /// - Lister implements `Iterator>`. /// - Lister will return `None` if there is no more entries or error has been returned. pub struct BlockingLister { - acc: Accessor, - /// required_metakey is the metakey required by users. - required_metakey: FlagSet, - lister: oio::BlockingLister, errored: bool, } @@ -234,20 +115,15 @@ unsafe impl Sync for BlockingLister {} impl BlockingLister { /// Create a new lister. pub(crate) fn create(acc: Accessor, path: &str, args: OpList) -> Result { - let required_metakey = args.metakey(); let (_, lister) = acc.blocking_list(path, args)?; Ok(Self { - acc, - required_metakey, - lister, errored: false, }) } } -/// TODO: we can implement next_chunk. impl Iterator for BlockingLister { type Item = Result; @@ -257,28 +133,14 @@ impl Iterator for BlockingLister { return None; } - let entry = match self.lister.next() { - Ok(Some(entry)) => entry, - Ok(None) => return None, + match self.lister.next() { + Ok(Some(entry)) => Some(Ok(entry.into_entry())), + Ok(None) => None, Err(err) => { self.errored = true; - return Some(Err(err)); + Some(Err(err)) } - }; - - let (path, metadata) = entry.into_entry().into_parts(); - if metadata.contains_metakey(self.required_metakey) { - return Some(Ok(Entry::new(path, metadata))); } - - let metadata = match self.acc.blocking_stat(&path, OpStat::default()) { - Ok(rp) => rp.into_metadata(), - Err(err) => { - self.errored = true; - return Some(Err(err)); - } - }; - Some(Ok(Entry::new(path, metadata))) } } diff --git a/core/src/types/metadata.rs b/core/src/types/metadata.rs index 48bfe3cd65e3..7314f25db5fd 100644 --- a/core/src/types/metadata.rs +++ b/core/src/types/metadata.rs @@ -18,8 +18,6 @@ use std::collections::HashMap; use chrono::prelude::*; -use flagset::flags; -use flagset::FlagSet; use crate::raw::*; use crate::*; @@ -33,9 +31,6 @@ use crate::*; /// a.k.a., `Entry`'s content length could be `None`. #[derive(Debug, Clone, Eq, PartialEq)] pub struct Metadata { - /// metakey stores current key store. - metakey: FlagSet, - mode: EntryMode, cache_control: Option, @@ -54,16 +49,7 @@ pub struct Metadata { impl Metadata { /// Create a new metadata pub fn new(mode: EntryMode) -> Self { - // Mode is required to be set for metadata. - let mut metakey: FlagSet = Metakey::Mode.into(); - // If mode is dir, we should always mark it as complete. - if mode.is_dir() { - metakey |= Metakey::Complete - } - Self { - metakey, - mode, cache_control: None, @@ -79,38 +65,8 @@ impl Metadata { } } - /// Get the metakey from metadata. - /// - /// This value describes which metadata has been set. - pub fn metakey(&self) -> FlagSet { - self.metakey - } - - /// Set metakey with given. - pub(crate) fn with_metakey(mut self, metakey: impl Into>) -> Self { - self.metakey = metakey.into(); - self - } - - /// Check if the metadata already contains given metakey. - pub(crate) fn contains_metakey(&self, metakey: impl Into>) -> bool { - let input_metakey = metakey.into(); - - // If meta already contains complete, we don't need to check. - if self.metakey.contains(Metakey::Complete) { - return true; - } - - self.metakey.contains(input_metakey) - } - /// mode represent this entry's mode. pub fn mode(&self) -> EntryMode { - debug_assert!( - self.metakey.contains(Metakey::Mode) || self.metakey.contains(Metakey::Complete), - "visiting not set metadata: mode, maybe a bug" - ); - self.mode } @@ -127,14 +83,12 @@ impl Metadata { /// Set mode for entry. pub fn set_mode(&mut self, v: EntryMode) -> &mut Self { self.mode = v; - self.metakey |= Metakey::Mode; self } /// Set mode for entry. pub fn with_mode(mut self, v: EntryMode) -> Self { self.mode = v; - self.metakey |= Metakey::Mode; self } @@ -145,13 +99,6 @@ impl Metadata { /// This value is only available when calling on result of `stat` or `list` with /// [`Metakey::CacheControl`], otherwise this method returns `None`. pub fn cache_control(&self) -> Option<&str> { - #[cfg(feature = "tests")] - debug_assert!( - self.metakey.contains(Metakey::CacheControl) - || self.metakey.contains(Metakey::Complete), - "visiting not set metadata: cache_control, maybe a bug" - ); - self.cache_control.as_deref() } @@ -161,7 +108,6 @@ impl Metadata { /// Refer to [MDN Cache-Control](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Cache-Control) for more information. pub fn set_cache_control(&mut self, v: &str) -> &mut Self { self.cache_control = Some(v.to_string()); - self.metakey |= Metakey::CacheControl; self } @@ -171,7 +117,6 @@ impl Metadata { /// Refer to [MDN Cache-Control](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Cache-Control) for more information. pub fn with_cache_control(mut self, v: String) -> Self { self.cache_control = Some(v); - self.metakey |= Metakey::CacheControl; self } @@ -185,26 +130,18 @@ impl Metadata { /// This value is only available when calling on result of `stat` or `list` with /// [`Metakey::ContentLength`], otherwise it will panic. pub fn content_length(&self) -> u64 { - debug_assert!( - self.metakey.contains(Metakey::ContentLength) - || self.metakey.contains(Metakey::Complete), - "visiting not set metadata: content_length, maybe a bug" - ); - self.content_length.unwrap_or_default() } /// Set content length of this entry. pub fn set_content_length(&mut self, v: u64) -> &mut Self { self.content_length = Some(v); - self.metakey |= Metakey::ContentLength; self } /// Set content length of this entry. pub fn with_content_length(mut self, v: u64) -> Self { self.content_length = Some(v); - self.metakey |= Metakey::ContentLength; self } @@ -218,12 +155,6 @@ impl Metadata { /// This value is only available when calling on result of `stat` or `list` with /// [`Metakey::ContentMd5`], otherwise this method returns `None`. pub fn content_md5(&self) -> Option<&str> { - #[cfg(feature = "tests")] - debug_assert!( - self.metakey.contains(Metakey::ContentMd5) || self.metakey.contains(Metakey::Complete), - "visiting not set metadata: content_md5, maybe a bug" - ); - self.content_md5.as_deref() } @@ -233,7 +164,6 @@ impl Metadata { /// And removed by [RFC 7231](https://www.rfc-editor.org/rfc/rfc7231). pub fn set_content_md5(&mut self, v: &str) -> &mut Self { self.content_md5 = Some(v.to_string()); - self.metakey |= Metakey::ContentMd5; self } @@ -243,7 +173,6 @@ impl Metadata { /// And removed by [RFC 7231](https://www.rfc-editor.org/rfc/rfc7231). pub fn with_content_md5(mut self, v: String) -> Self { self.content_md5 = Some(v); - self.metakey |= Metakey::ContentMd5; self } @@ -254,12 +183,6 @@ impl Metadata { /// This value is only available when calling on result of `stat` or `list` with /// [`Metakey::ContentType`], otherwise this method returns `None`. pub fn content_type(&self) -> Option<&str> { - #[cfg(feature = "tests")] - debug_assert!( - self.metakey.contains(Metakey::ContentType) || self.metakey.contains(Metakey::Complete), - "visiting not set metadata: content_type, maybe a bug" - ); - self.content_type.as_deref() } @@ -268,7 +191,6 @@ impl Metadata { /// Content Type is defined by [RFC 9110](https://httpwg.org/specs/rfc9110.html#field.content-type). pub fn set_content_type(&mut self, v: &str) -> &mut Self { self.content_type = Some(v.to_string()); - self.metakey |= Metakey::ContentType; self } @@ -277,7 +199,6 @@ impl Metadata { /// Content Type is defined by [RFC 9110](https://httpwg.org/specs/rfc9110.html#field.content-type). pub fn with_content_type(mut self, v: String) -> Self { self.content_type = Some(v); - self.metakey |= Metakey::ContentType; self } @@ -288,13 +209,6 @@ impl Metadata { /// This value is only available when calling on result of `stat` or `list` with /// [`Metakey::ContentRange`], otherwise this method returns `None`. pub fn content_range(&self) -> Option { - #[cfg(feature = "tests")] - debug_assert!( - self.metakey.contains(Metakey::ContentRange) - || self.metakey.contains(Metakey::Complete), - "visiting not set metadata: content_range, maybe a bug" - ); - self.content_range } @@ -303,7 +217,6 @@ impl Metadata { /// Content Range is defined by [RFC 9110](https://httpwg.org/specs/rfc9110.html#field.content-range). pub fn set_content_range(&mut self, v: BytesContentRange) -> &mut Self { self.content_range = Some(v); - self.metakey |= Metakey::ContentRange; self } @@ -312,7 +225,6 @@ impl Metadata { /// Content Range is defined by [RFC 9110](https://httpwg.org/specs/rfc9110.html#field.content-range). pub fn with_content_range(mut self, v: BytesContentRange) -> Self { self.content_range = Some(v); - self.metakey |= Metakey::ContentRange; self } @@ -326,13 +238,6 @@ impl Metadata { /// This value is only available when calling on result of `stat` or `list` with /// [`Metakey::LastModified`], otherwise this method returns `None`. pub fn last_modified(&self) -> Option> { - #[cfg(feature = "tests")] - debug_assert!( - self.metakey.contains(Metakey::LastModified) - || self.metakey.contains(Metakey::Complete), - "visiting not set metadata: last_modified, maybe a bug" - ); - self.last_modified } @@ -342,7 +247,6 @@ impl Metadata { /// Refer to [MDN Last-Modified](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Last-Modified) for more information. pub fn set_last_modified(&mut self, v: DateTime) -> &mut Self { self.last_modified = Some(v); - self.metakey |= Metakey::LastModified; self } @@ -352,7 +256,6 @@ impl Metadata { /// Refer to [MDN Last-Modified](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Last-Modified) for more information. pub fn with_last_modified(mut self, v: DateTime) -> Self { self.last_modified = Some(v); - self.metakey |= Metakey::LastModified; self } @@ -371,12 +274,6 @@ impl Metadata { /// This value is only available when calling on result of `stat` or `list` with /// [`Metakey::Etag`], otherwise this method returns `None`. pub fn etag(&self) -> Option<&str> { - #[cfg(feature = "tests")] - debug_assert!( - self.metakey.contains(Metakey::Etag) || self.metakey.contains(Metakey::Complete), - "visiting not set metadata: etag, maybe a bug" - ); - self.etag.as_deref() } @@ -393,7 +290,6 @@ impl Metadata { /// `"` is part of etag, don't trim it before setting. pub fn set_etag(&mut self, v: &str) -> &mut Self { self.etag = Some(v.to_string()); - self.metakey |= Metakey::Etag; self } @@ -410,7 +306,6 @@ impl Metadata { /// `"` is part of etag, don't trim it before setting. pub fn with_etag(mut self, v: String) -> Self { self.etag = Some(v); - self.metakey |= Metakey::Etag; self } @@ -429,13 +324,6 @@ impl Metadata { /// This value is only available when calling on result of `stat` or `list` with /// [`Metakey::ContentDisposition`], otherwise this method returns `None`. pub fn content_disposition(&self) -> Option<&str> { - #[cfg(feature = "tests")] - debug_assert!( - self.metakey.contains(Metakey::ContentDisposition) - || self.metakey.contains(Metakey::Complete), - "visiting not set metadata: content_disposition, maybe a bug" - ); - self.content_disposition.as_deref() } @@ -452,7 +340,6 @@ impl Metadata { /// - "attachment; filename=\"filename.jpg\"" pub fn with_content_disposition(mut self, v: String) -> Self { self.content_disposition = Some(v); - self.metakey |= Metakey::ContentDisposition; self } @@ -469,7 +356,6 @@ impl Metadata { /// - "attachment; filename=\"filename.jpg\"" pub fn set_content_disposition(&mut self, v: &str) -> &mut Self { self.content_disposition = Some(v.to_string()); - self.metakey |= Metakey::ContentDisposition; self } @@ -482,12 +368,6 @@ impl Metadata { /// This value is only available when calling on result of `stat` or `list` with /// [`Metakey::Version`], otherwise this method returns `None`. pub fn version(&self) -> Option<&str> { - #[cfg(feature = "tests")] - debug_assert!( - self.metakey.contains(Metakey::Version) || self.metakey.contains(Metakey::Complete), - "visiting not set metadata: version, maybe a bug" - ); - self.version.as_deref() } @@ -498,7 +378,6 @@ impl Metadata { /// This field may come out from the version control system, like object versioning in AWS S3. pub fn with_version(mut self, v: String) -> Self { self.version = Some(v); - self.metakey |= Metakey::Version; self } @@ -509,7 +388,6 @@ impl Metadata { /// This field may come out from the version control system, like object versioning in AWS S3. pub fn set_version(&mut self, v: &str) -> &mut Self { self.version = Some(v.to_string()); - self.metakey |= Metakey::Version; self } @@ -524,51 +402,6 @@ impl Metadata { /// Set user defined metadata of this entry pub fn with_user_metadata(&mut self, data: HashMap) -> &mut Self { self.user_metadata = Some(data); - self.metakey |= Metakey::UserMetaData; self } } - -flags! { - /// Metakey describes the metadata keys that can be stored - /// or queried. - /// - /// ## For store - /// - /// Internally, we will store a flag set of Metakey to check - /// whether we have set some key already. - /// - /// ## For query - /// - /// At user side, we will allow user to query the metadata. If - /// the meta has been stored, we will return directly. If no, we will - /// call `stat` internally to fetch the metadata. - pub enum Metakey: u64 { - /// The special metadata key that used to mark this entry - /// already contains all metadata. - Complete, - - /// Key for mode. - Mode, - /// Key for cache control. - CacheControl, - /// Key for content disposition. - ContentDisposition, - /// Key for content length. - ContentLength, - /// Key for content md5. - ContentMd5, - /// Key for content range. - ContentRange, - /// Key for content type. - ContentType, - /// Key for etag. - Etag, - /// Key for last modified. - LastModified, - /// Key for version. - Version, - /// Key for user metadata - UserMetaData, - } -} diff --git a/core/src/types/mod.rs b/core/src/types/mod.rs index 602cf8c557f7..448a5f430245 100644 --- a/core/src/types/mod.rs +++ b/core/src/types/mod.rs @@ -26,7 +26,6 @@ pub use entry::Entry; mod metadata; pub use metadata::Metadata; -pub use metadata::Metakey; mod read; pub use read::*; diff --git a/core/src/types/operator/blocking_operator.rs b/core/src/types/operator/blocking_operator.rs index 38a95357f653..9ea08fb81f4e 100644 --- a/core/src/types/operator/blocking_operator.rs +++ b/core/src/types/operator/blocking_operator.rs @@ -133,12 +133,6 @@ impl BlockingOperator { impl BlockingOperator { /// Get given path's metadata. /// - /// # Notes - /// - /// For fetch metadata of entries returned by [`BlockingLister`], it's better to - /// use [`BlockingOperator::list_with`] and [`BlockingOperator::lister_with`] with `metakey` - /// query like `Metakey::ContentLength | Metakey::LastModified` so that we can avoid extra requests. - /// /// # Behavior /// /// ## Services that support `create_dir` @@ -190,13 +184,6 @@ impl BlockingOperator { /// Get given path's metadata with extra options. /// - /// # Notes - /// - /// For fetch metadata of entries returned by [`Lister`], it's better to use - /// [`Operator::list_with`] and [`Operator::lister_with`] with `metakey` query like - /// `Metakey::ContentLength | Metakey::LastModified` - /// so that we can avoid extra requests. - /// /// # Behavior /// /// ## Services that support `create_dir` @@ -383,7 +370,6 @@ impl BlockingOperator { /// # use anyhow::Result; /// use opendal::BlockingOperator; /// use opendal::EntryMode; - /// use opendal::Metakey; /// # fn test(op: BlockingOperator) -> Result<()> { /// let bs = op.read_with("path/to/file").range(0..10).call()?; /// # Ok(()) @@ -439,7 +425,6 @@ impl BlockingOperator { /// # use anyhow::Result; /// use opendal::BlockingOperator; /// use opendal::EntryMode; - /// use opendal::Metakey; /// # fn test(op: BlockingOperator) -> Result<()> { /// let r = op /// .reader_with("path/to/file") @@ -695,7 +680,6 @@ impl BlockingOperator { /// # use anyhow::Result; /// use opendal::BlockingOperator; /// use opendal::EntryMode; - /// use opendal::Metakey; /// # fn test(op: BlockingOperator) -> Result<()> { /// let mut w = op.writer_with("path/to/file").call()?; /// w.write(vec![0; 4096])?; @@ -903,18 +887,12 @@ impl BlockingOperator { /// In order to avoid this, you can use [`BlockingOperator::lister`] to list entries in /// a streaming way. /// - /// ## Reuse Metadata - /// - /// The only metadata that is guaranteed to be available is the `Mode`. - /// For fetching more metadata, please use [`BlockingOperator::list_with`] and `metakey`. - /// /// # Examples /// /// ```no_run /// # use anyhow::Result; /// use opendal::BlockingOperator; /// use opendal::EntryMode; - /// use opendal::Metakey; /// # fn test(op: BlockingOperator) -> Result<()> { /// let mut entries = op.list("path/to/dir/")?; /// for entry in entries { @@ -948,11 +926,6 @@ impl BlockingOperator { /// In order to avoid this, you can use [`BlockingOperator::lister`] to list entries in /// a streaming way. /// - /// ## Reuse Metadata - /// - /// The only metadata that is guaranteed to be available is the `Mode`. - /// For fetching more metadata, please specify the `metakey`. - /// /// # Examples /// /// ## List entries with prefix @@ -963,7 +936,6 @@ impl BlockingOperator { /// # use anyhow::Result; /// use opendal::BlockingOperator; /// use opendal::EntryMode; - /// use opendal::Metakey; /// # fn test(op: BlockingOperator) -> Result<()> { /// let mut entries = op.list_with("prefix/").recursive(true).call()?; /// for entry in entries { @@ -980,38 +952,6 @@ impl BlockingOperator { /// # Ok(()) /// # } /// ``` - /// - /// ## List entries with metakey for more metadata - /// - /// ```no_run - /// # use anyhow::Result; - /// use opendal::BlockingOperator; - /// use opendal::EntryMode; - /// use opendal::Metakey; - /// # fn test(op: BlockingOperator) -> Result<()> { - /// let mut entries = op - /// .list_with("dir/") - /// .metakey(Metakey::ContentLength | Metakey::LastModified) - /// .call()?; - /// for entry in entries { - /// let meta = entry.metadata(); - /// match meta.mode() { - /// EntryMode::FILE => { - /// println!( - /// "Handling file {} with size {}", - /// entry.path(), - /// meta.content_length() - /// ) - /// } - /// EntryMode::DIR => { - /// println!("Handling dir {}", entry.path()) - /// } - /// EntryMode::Unknown => continue, - /// } - /// } - /// # Ok(()) - /// # } - /// ``` pub fn list_with(&self, path: &str) -> FunctionList { let path = normalize_path(path); @@ -1040,11 +980,6 @@ impl BlockingOperator { /// all entries recursively, use [`BlockingOperator::lister_with`] and `delimiter("")` /// instead. /// - /// ## Metadata - /// - /// The only metadata that is guaranteed to be available is the `Mode`. - /// For fetching more metadata, please use [`BlockingOperator::lister_with`] and `metakey`. - /// /// # Examples /// /// ```no_run @@ -1053,7 +988,6 @@ impl BlockingOperator { /// use futures::TryStreamExt; /// use opendal::BlockingOperator; /// use opendal::EntryMode; - /// use opendal::Metakey; /// # fn test(op: BlockingOperator) -> Result<()> { /// let mut ds = op.lister("path/to/dir/")?; /// for de in ds { @@ -1091,7 +1025,6 @@ impl BlockingOperator { /// use futures::TryStreamExt; /// use opendal::BlockingOperator; /// use opendal::EntryMode; - /// use opendal::Metakey; /// # fn test(op: BlockingOperator) -> Result<()> { /// let mut ds = op /// .lister_with("path/to/dir/") @@ -1122,7 +1055,6 @@ impl BlockingOperator { /// use futures::TryStreamExt; /// use opendal::BlockingOperator; /// use opendal::EntryMode; - /// use opendal::Metakey; /// # fn test(op: BlockingOperator) -> Result<()> { /// let mut ds = op.lister_with("path/to/dir/").recursive(true).call()?; /// for entry in ds { @@ -1140,41 +1072,6 @@ impl BlockingOperator { /// # Ok(()) /// # } /// ``` - /// - /// ## List files with required metadata - /// - /// ```no_run - /// # use anyhow::Result; - /// # use futures::io; - /// use futures::TryStreamExt; - /// use opendal::BlockingOperator; - /// use opendal::EntryMode; - /// use opendal::Metakey; - /// # fn test(op: BlockingOperator) -> Result<()> { - /// let mut ds = op - /// .lister_with("path/to/dir/") - /// .metakey(Metakey::ContentLength | Metakey::LastModified) - /// .call()?; - /// for entry in ds { - /// let entry = entry?; - /// let meta = entry.metadata(); - /// match meta.mode() { - /// EntryMode::FILE => { - /// println!( - /// "Handling file {} with size {}", - /// entry.path(), - /// meta.content_length() - /// ) - /// } - /// EntryMode::DIR => { - /// println!("Handling dir {}", entry.path()) - /// } - /// EntryMode::Unknown => continue, - /// } - /// } - /// # Ok(()) - /// # } - /// ``` pub fn lister_with(&self, path: &str) -> FunctionLister { let path = normalize_path(path); diff --git a/core/src/types/operator/operator.rs b/core/src/types/operator/operator.rs index 84fb6e0aa695..fc8b31d25389 100644 --- a/core/src/types/operator/operator.rs +++ b/core/src/types/operator/operator.rs @@ -180,12 +180,6 @@ impl Operator { /// [`Operator::stat`] is a wrapper of [`Operator::stat_with`] without any options. To use extra /// options like `if_match` and `if_none_match`, please use [`Operator::stat_with`] instead. /// - /// ## Reuse Metadata - /// - /// For fetch metadata of entries returned by [`Lister`], it's better to use - /// [`Operator::list_with`] and [`Operator::lister_with`] with `metakey` query like - /// `Metakey::ContentLength | Metakey::LastModified` so that we can avoid extra stat requests. - /// /// # Examples /// /// ## Check if file exists @@ -211,14 +205,6 @@ impl Operator { /// Get given path's metadata with extra options. /// - /// # Notes - /// - /// ## Reuse Metadata - /// - /// For fetch metadata of entries returned by [`Lister`], it's better to use - /// [`Operator::list_with`] and [`Operator::lister_with`] with `metakey` query like - /// `Metakey::ContentLength | Metakey::LastModified` so that we can avoid extra requests. - /// /// # Options /// /// ## `if_match` @@ -1546,11 +1532,6 @@ impl Operator { /// In order to avoid this, you can use [`Operator::lister`] to list entries in /// a streaming way. /// - /// ## Reuse Metadata - /// - /// The only metadata that is guaranteed to be available is the `Mode`. - /// For fetching more metadata, please use [`Operator::list_with`] and `metakey`. - /// /// # Examples /// /// ## List entries under a dir @@ -1560,7 +1541,6 @@ impl Operator { /// ```no_run /// # use anyhow::Result; /// use opendal::EntryMode; - /// use opendal::Metakey; /// use opendal::Operator; /// # async fn test(op: Operator) -> Result<()> { /// let mut entries = op.list("path/to/dir/").await?; @@ -1590,7 +1570,6 @@ impl Operator { /// ```no_run /// # use anyhow::Result; /// use opendal::EntryMode; - /// use opendal::Metakey; /// use opendal::Operator; /// # async fn test(op: Operator) -> Result<()> { /// let mut entries = op.list("path/to/prefix").await?; @@ -1663,51 +1642,6 @@ impl Operator { /// # } /// ``` /// - /// ## `metakey` - /// - /// Specify the metadata that required to be fetched in entries. - /// - /// If `metakey` is not set, we will fetch only the entry's `mode`. Otherwise, we will retrieve - /// the required metadata from storage services. Even if `metakey` is specified, the metadata - /// may still be `None`, indicating that the storage service does not supply this information. - /// - /// Some storage services like `s3` could return more metadata like `content-length` and - /// `last-modified`. By using `metakey`, we can fetch those metadata without an extra `stat` call. - /// Please pick up the metadata you need to reduce the extra `stat` cost. - /// - /// This example shows how to list entries with `content-length` and `last-modified` metadata: - /// - /// ```no_run - /// # use anyhow::Result; - /// use opendal::EntryMode; - /// use opendal::Metakey; - /// use opendal::Operator; - /// # async fn test(op: Operator) -> Result<()> { - /// let mut entries = op - /// .list_with("dir/") - /// // Make sure content-length and last-modified been fetched. - /// .metakey(Metakey::ContentLength | Metakey::LastModified) - /// .await?; - /// for entry in entries { - /// let meta = entry.metadata(); - /// match meta.mode() { - /// EntryMode::FILE => { - /// println!( - /// "Handling file {} with size {}", - /// entry.path(), - /// meta.content_length() - /// ) - /// } - /// EntryMode::DIR => { - /// println!("Handling dir {}", entry.path()) - /// } - /// EntryMode::Unknown => continue, - /// } - /// } - /// # Ok(()) - /// # } - /// ``` - /// /// # Examples /// /// ## List all entries recursively @@ -1717,7 +1651,6 @@ impl Operator { /// ```no_run /// # use anyhow::Result; /// use opendal::EntryMode; - /// use opendal::Metakey; /// use opendal::Operator; /// # async fn test(op: Operator) -> Result<()> { /// let mut entries = op.list_with("path/to/dir/").recursive(true).await?; @@ -1743,7 +1676,6 @@ impl Operator { /// ```no_run /// # use anyhow::Result; /// use opendal::EntryMode; - /// use opendal::Metakey; /// use opendal::Operator; /// # async fn test(op: Operator) -> Result<()> { /// let mut entries = op.list_with("path/to/prefix").recursive(true).await?; @@ -1789,11 +1721,6 @@ impl Operator { /// all entries recursively, use [`Operator::lister_with`] and `recursive(true)` /// instead. /// - /// ## Reuse Metadata - /// - /// The only metadata that is guaranteed to be available is the `Mode`. - /// For fetching more metadata, please use [`Operator::lister_with`] and `metakey`. - /// /// # Examples /// /// ```no_run @@ -1801,7 +1728,6 @@ impl Operator { /// # use futures::io; /// use futures::TryStreamExt; /// use opendal::EntryMode; - /// use opendal::Metakey; /// use opendal::Operator; /// # async fn test(op: Operator) -> Result<()> { /// let mut ds = op.lister("path/to/dir/").await?; @@ -1866,52 +1792,6 @@ impl Operator { /// # } /// ``` /// - /// ## `metakey` - /// - /// Specify the metadata that required to be fetched in entries. - /// - /// If `metakey` is not set, we will fetch only the entry's `mode`. Otherwise, we will retrieve - /// the required metadata from storage services. Even if `metakey` is specified, the metadata - /// may still be `None`, indicating that the storage service does not supply this information. - /// - /// Some storage services like `s3` could return more metadata like `content-length` and - /// `last-modified`. By using `metakey`, we can fetch those metadata without an extra `stat` call. - /// Please pick up the metadata you need to reduce the extra `stat` cost. - /// - /// This example shows how to list entries with `content-length` and `last-modified` metadata: - /// - /// ```no_run - /// # use anyhow::Result; - /// use futures::TryStreamExt; - /// use opendal::EntryMode; - /// use opendal::Metakey; - /// use opendal::Operator; - /// # async fn test(op: Operator) -> Result<()> { - /// let mut lister = op - /// .lister_with("dir/") - /// // Make sure content-length and last-modified been fetched. - /// .metakey(Metakey::ContentLength | Metakey::LastModified) - /// .await?; - /// while let Some(mut entry) = lister.try_next().await? { - /// let meta = entry.metadata(); - /// match meta.mode() { - /// EntryMode::FILE => { - /// println!( - /// "Handling file {} with size {}", - /// entry.path(), - /// meta.content_length() - /// ) - /// } - /// EntryMode::DIR => { - /// println!("Handling dir {}", entry.path()) - /// } - /// EntryMode::Unknown => continue, - /// } - /// } - /// # Ok(()) - /// # } - /// ``` - /// /// # Examples /// /// ## List all files recursively @@ -1920,7 +1800,6 @@ impl Operator { /// # use anyhow::Result; /// use futures::TryStreamExt; /// use opendal::EntryMode; - /// use opendal::Metakey; /// use opendal::Operator; /// # async fn test(op: Operator) -> Result<()> { /// let mut lister = op.lister_with("path/to/dir/").recursive(true).await?; @@ -1938,40 +1817,6 @@ impl Operator { /// # Ok(()) /// # } /// ``` - /// - /// ## List files with required metadata - /// - /// ```no_run - /// # use anyhow::Result; - /// # use futures::io; - /// use futures::TryStreamExt; - /// use opendal::EntryMode; - /// use opendal::Metakey; - /// use opendal::Operator; - /// # async fn test(op: Operator) -> Result<()> { - /// let mut ds = op - /// .lister_with("path/to/dir/") - /// .metakey(Metakey::ContentLength | Metakey::LastModified) - /// .await?; - /// while let Some(mut entry) = ds.try_next().await? { - /// let meta = entry.metadata(); - /// match meta.mode() { - /// EntryMode::FILE => { - /// println!( - /// "Handling file {} with size {}", - /// entry.path(), - /// meta.content_length() - /// ) - /// } - /// EntryMode::DIR => { - /// println!("Handling dir {}", entry.path()) - /// } - /// EntryMode::Unknown => continue, - /// } - /// } - /// # Ok(()) - /// # } - /// ``` pub fn lister_with(&self, path: &str) -> FutureLister>> { let path = normalize_path(path); diff --git a/core/src/types/operator/operator_functions.rs b/core/src/types/operator/operator_functions.rs index dc5209af39e6..cdfcc555da3e 100644 --- a/core/src/types/operator/operator_functions.rs +++ b/core/src/types/operator/operator_functions.rs @@ -21,8 +21,6 @@ use std::ops::RangeBounds; -use flagset::FlagSet; - use crate::raw::*; use crate::*; @@ -277,19 +275,6 @@ impl FunctionList { self } - /// Metakey is used to control which meta should be returned. - /// - /// Lister will make sure the result for specified meta is **known**: - /// - /// - `Some(v)` means exist. - /// - `None` means services doesn't have this meta. - /// - /// The default metakey is `Metakey::Mode`. - pub fn metakey(mut self, v: impl Into>) -> Self { - self.0 = self.0.map_args(|args| args.with_metakey(v)); - self - } - /// Call the function to consume all the input and generate a /// result. pub fn call(self) -> Result> { @@ -330,19 +315,6 @@ impl FunctionLister { self } - /// Metakey is used to control which meta should be returned. - /// - /// Lister will make sure the result for specified meta is **known**: - /// - /// - `Some(v)` means exist. - /// - `None` means services doesn't have this meta. - /// - /// The default metakey is `Metakey::Mode`. - pub fn metakey(mut self, v: impl Into>) -> Self { - self.0 = self.0.map_args(|args| args.with_metakey(v)); - self - } - /// Call the function to consume all the input and generate a /// result. pub fn call(self) -> Result { diff --git a/core/src/types/operator/operator_futures.rs b/core/src/types/operator/operator_futures.rs index 17f1cb77c749..53d89f594eb1 100644 --- a/core/src/types/operator/operator_futures.rs +++ b/core/src/types/operator/operator_futures.rs @@ -24,7 +24,6 @@ use std::future::IntoFuture; use std::ops::RangeBounds; use std::time::Duration; -use flagset::FlagSet; use futures::Future; use crate::raw::*; @@ -465,27 +464,6 @@ impl>>> FutureList { self.map(|args| args.with_recursive(v)) } - /// Metakey is used to control which meta should be returned. - /// - /// Lister will make sure the result for specified meta is **known**: - /// - /// - `Some(v)` means exist. - /// - `None` means services doesn't have this meta. - /// - /// The default metakey is `Metakey::Mode`. - pub fn metakey(self, v: impl Into>) -> Self { - self.map(|args| args.with_metakey(v)) - } - - /// Concurrent is used to control the number of concurrent stat requests. - /// - /// If concurrent is set to <=1, the lister will perform stat requests sequentially. - /// - /// The default concurrent is 1. - pub fn concurrent(self, v: usize) -> Self { - self.map(|args| args.with_concurrent(v)) - } - /// The version is used to control whether the object versions should be returned. /// /// - If `false`, list operation will not return with object versions @@ -528,27 +506,6 @@ impl>> FutureLister { self.map(|args| args.with_recursive(v)) } - /// Metakey is used to control which meta should be returned. - /// - /// Lister will make sure the result for specified meta is **known**: - /// - /// - `Some(v)` means exist. - /// - `None` means services doesn't have this meta. - /// - /// The default metakey is `Metakey::Mode`. - pub fn metakey(self, v: impl Into>) -> Self { - self.map(|args| args.with_metakey(v)) - } - - /// Concurrent is used to control the number of concurrent stat requests. - /// - /// If concurrent is set to <=1, the lister will perform stat requests sequentially. - /// - /// The default concurrent is 1. - pub fn concurrent(self, v: usize) -> Self { - self.map(|args| args.with_concurrent(v)) - } - /// The version is used to control whether the object versions should be returned. /// /// - If `false`, list operation will not return with object versions diff --git a/core/tests/behavior/async_list.rs b/core/tests/behavior/async_list.rs index 69157cdc155c..9e9367b1ab8c 100644 --- a/core/tests/behavior/async_list.rs +++ b/core/tests/behavior/async_list.rs @@ -34,8 +34,6 @@ pub fn tests(op: &Operator, tests: &mut Vec) { op, test_check, test_list_dir, - test_list_dir_with_metakey, - test_list_dir_with_metakey_complete, test_list_prefix, test_list_rich_dir, test_list_empty_dir, @@ -94,95 +92,6 @@ pub async fn test_list_dir(op: Operator) -> Result<()> { Ok(()) } -/// List dir with metakey -pub async fn test_list_dir_with_metakey(op: Operator) -> Result<()> { - let parent = uuid::Uuid::new_v4().to_string(); - let path = format!("{parent}/{}", uuid::Uuid::new_v4()); - debug!("Generate a random file: {}", &path); - let (content, size) = gen_bytes(op.info().full_capability()); - - op.write(&path, content).await.expect("write must succeed"); - - let mut obs = op - .lister_with(&format!("{parent}/")) - .metakey( - Metakey::Mode - | Metakey::CacheControl - | Metakey::ContentDisposition - | Metakey::ContentLength - | Metakey::ContentMd5 - | Metakey::ContentRange - | Metakey::ContentType - | Metakey::Etag - | Metakey::LastModified - | Metakey::Version, - ) - .await?; - let mut found = false; - while let Some(de) = obs.try_next().await? { - let meta = de.metadata(); - if de.path() == path { - assert_eq!(meta.mode(), EntryMode::FILE); - assert_eq!(meta.content_length(), size as u64); - - // We don't care about the value, we just to check there is no panic. - let _ = meta.cache_control(); - let _ = meta.content_disposition(); - let _ = meta.content_md5(); - let _ = meta.content_range(); - let _ = meta.content_type(); - let _ = meta.etag(); - let _ = meta.last_modified(); - let _ = meta.version(); - - found = true - } - } - assert!(found, "file should be found in list"); - - op.delete(&path).await.expect("delete must succeed"); - Ok(()) -} - -/// List dir with metakey complete -pub async fn test_list_dir_with_metakey_complete(op: Operator) -> Result<()> { - let parent = uuid::Uuid::new_v4().to_string(); - let path = format!("{parent}/{}", uuid::Uuid::new_v4()); - debug!("Generate a random file: {}", &path); - let (content, size) = gen_bytes(op.info().full_capability()); - - op.write(&path, content).await.expect("write must succeed"); - - let mut obs = op - .lister_with(&format!("{parent}/")) - .metakey(Metakey::Complete) - .await?; - let mut found = false; - while let Some(de) = obs.try_next().await? { - let meta = de.metadata(); - if de.path() == path { - assert_eq!(meta.mode(), EntryMode::FILE); - assert_eq!(meta.content_length(), size as u64); - - // We don't care about the value, we just to check there is no panic. - let _ = meta.cache_control(); - let _ = meta.content_disposition(); - let _ = meta.content_md5(); - let _ = meta.content_range(); - let _ = meta.content_type(); - let _ = meta.etag(); - let _ = meta.last_modified(); - let _ = meta.version(); - - found = true - } - } - assert!(found, "file should be found in list"); - - op.delete(&path).await.expect("delete must succeed"); - Ok(()) -} - /// List prefix should return newly created file. pub async fn test_list_prefix(op: Operator) -> Result<()> { let path = uuid::Uuid::new_v4().to_string(); @@ -228,23 +137,6 @@ pub async fn test_list_rich_dir(op: Operator) -> Result<()> { assert_eq!(actual, expected); - // List concurrently. - let mut objects = op - .lister_with(parent) - .limit(5) - .concurrent(5) - .metakey(Metakey::Complete) - .await?; - let mut actual = vec![]; - while let Some(o) = objects.try_next().await? { - let path = o.path().to_string(); - actual.push(path) - } - expected.sort_unstable(); - actual.sort_unstable(); - - assert_eq!(actual, expected); - op.remove_all(parent).await?; Ok(()) } @@ -752,24 +644,6 @@ pub async fn test_list_with_version_and_limit(op: Operator) -> Result<()> { assert_eq!(actual, expected); - // List concurrently. - let mut objects = op - .lister_with(parent) - .version(true) - .limit(5) - .concurrent(5) - .metakey(Metakey::Complete) - .await?; - let mut actual = vec![]; - while let Some(o) = objects.try_next().await? { - let path = o.path().to_string(); - actual.push(path) - } - expected.sort_unstable(); - actual.sort_unstable(); - - assert_eq!(actual, expected); - op.remove_all(parent).await?; Ok(()) } diff --git a/core/tests/behavior/blocking_list.rs b/core/tests/behavior/blocking_list.rs index 225d19cad494..80866b8a77b6 100644 --- a/core/tests/behavior/blocking_list.rs +++ b/core/tests/behavior/blocking_list.rs @@ -29,8 +29,6 @@ pub fn tests(op: &Operator, tests: &mut Vec) { tests.extend(blocking_trials!( op, test_blocking_list_dir, - test_blocking_list_dir_with_metakey, - test_blocking_list_dir_with_metakey_complete, test_blocking_list_non_exist_dir, test_blocking_list_dir_with_recursive, test_blocking_list_dir_with_recursive_no_trailing_slash, @@ -68,95 +66,6 @@ pub fn test_blocking_list_dir(op: BlockingOperator) -> Result<()> { Ok(()) } -/// List dir with metakey -pub fn test_blocking_list_dir_with_metakey(op: BlockingOperator) -> Result<()> { - let parent = uuid::Uuid::new_v4().to_string(); - let path = format!("{parent}/{}", uuid::Uuid::new_v4()); - debug!("Generate a random file: {}", &path); - let (content, size) = gen_bytes(op.info().full_capability()); - - op.write(&path, content).expect("write must succeed"); - - let mut obs = op - .lister_with(&format!("{parent}/")) - .metakey( - Metakey::Mode - | Metakey::CacheControl - | Metakey::ContentDisposition - | Metakey::ContentLength - | Metakey::ContentMd5 - | Metakey::ContentRange - | Metakey::ContentType - | Metakey::Etag - | Metakey::LastModified - | Metakey::Version, - ) - .call()?; - let mut found = false; - while let Some(de) = obs.next().transpose()? { - let meta = de.metadata(); - if de.path() == path { - assert_eq!(meta.mode(), EntryMode::FILE); - assert_eq!(meta.content_length(), size as u64); - - // We don't care about the value, we just to check there is no panic. - let _ = meta.cache_control(); - let _ = meta.content_disposition(); - let _ = meta.content_md5(); - let _ = meta.content_range(); - let _ = meta.content_type(); - let _ = meta.etag(); - let _ = meta.last_modified(); - let _ = meta.version(); - - found = true - } - } - assert!(found, "file should be found in list"); - - op.delete(&path).expect("delete must succeed"); - Ok(()) -} - -/// List dir with metakey complete -pub fn test_blocking_list_dir_with_metakey_complete(op: BlockingOperator) -> Result<()> { - let parent = uuid::Uuid::new_v4().to_string(); - let path = format!("{parent}/{}", uuid::Uuid::new_v4()); - debug!("Generate a random file: {}", &path); - let (content, size) = gen_bytes(op.info().full_capability()); - - op.write(&path, content).expect("write must succeed"); - - let mut obs = op - .lister_with(&format!("{parent}/")) - .metakey(Metakey::Complete) - .call()?; - let mut found = false; - while let Some(de) = obs.next().transpose()? { - let meta = de.metadata(); - if de.path() == path { - assert_eq!(meta.mode(), EntryMode::FILE); - assert_eq!(meta.content_length(), size as u64); - - // We don't care about the value, we just to check there is no panic. - let _ = meta.cache_control(); - let _ = meta.content_disposition(); - let _ = meta.content_md5(); - let _ = meta.content_range(); - let _ = meta.content_type(); - let _ = meta.etag(); - let _ = meta.last_modified(); - let _ = meta.version(); - - found = true - } - } - assert!(found, "file should be found in list"); - - op.delete(&path).expect("delete must succeed"); - Ok(()) -} - /// List non exist dir should return nothing. pub fn test_blocking_list_non_exist_dir(op: BlockingOperator) -> Result<()> { let dir = format!("{}/", uuid::Uuid::new_v4()); diff --git a/integrations/cloud_filter/src/lib.rs b/integrations/cloud_filter/src/lib.rs index eb5518b6efad..24ace0324144 100644 --- a/integrations/cloud_filter/src/lib.rs +++ b/integrations/cloud_filter/src/lib.rs @@ -101,7 +101,7 @@ use cloud_filter::{ }; use file::FileBlob; use futures::StreamExt; -use opendal::{Entry, Metakey, Operator}; +use opendal::{Entry, Operator}; const BUF_SIZE: usize = 65536; @@ -209,7 +209,6 @@ impl Filter for CloudFilter { let mut entries = self .op .lister_with(&remote_path.to_string_lossy().replace('\\', "/")) - .metakey(Metakey::LastModified | Metakey::ContentLength) .await .map_err(|e| { log::warn!("failed to list files: {}", e); diff --git a/integrations/compat/Cargo.toml b/integrations/compat/Cargo.toml index 2fe51ad15550..fc400782748a 100644 --- a/integrations/compat/Cargo.toml +++ b/integrations/compat/Cargo.toml @@ -36,7 +36,7 @@ v0_50_to_v0_49 = ["dep:opendal_v0_49", "dep:opendal_v0_50"] [dependencies] async-trait = "0.1" opendal_v0_49 = { package = "opendal", version = "0.49", optional = true } -opendal_v0_50 = { package = "opendal", version = "0.50", optional = true, path = "../../core" } +opendal_v0_50 = { package = "opendal", version = "0.50", optional = true } [dev-dependencies] tokio = { version = "1.41", features = ["full"] } diff --git a/integrations/fuse3/src/file_system.rs b/integrations/fuse3/src/file_system.rs index 18855787171c..2811149b2177 100644 --- a/integrations/fuse3/src/file_system.rs +++ b/integrations/fuse3/src/file_system.rs @@ -33,7 +33,6 @@ use opendal::raw::normalize_path; use opendal::EntryMode; use opendal::ErrorKind; use opendal::Metadata; -use opendal::Metakey; use opendal::Operator; use sharded_slab::Slab; use tokio::sync::Mutex; @@ -717,7 +716,6 @@ impl PathFilesystem for Filesystem { let children = self .op .lister_with(&path) - .metakey(Metakey::ContentLength | Metakey::LastModified | Metakey::Mode) .await .map_err(opendal_error2errno)? .filter_map(move |entry| { diff --git a/integrations/object_store/src/store.rs b/integrations/object_store/src/store.rs index 1339f1bb78fe..c5ba9a6750f7 100644 --- a/integrations/object_store/src/store.rs +++ b/integrations/object_store/src/store.rs @@ -39,8 +39,8 @@ use object_store::PutOptions; use object_store::PutPayload; use object_store::PutResult; use object_store::{GetOptions, UploadPart}; +use opendal::Buffer; use opendal::Writer; -use opendal::{Buffer, Metakey}; use opendal::{Operator, OperatorInfo}; use tokio::sync::{Mutex, Notify}; @@ -103,16 +103,6 @@ impl OpendalStore { inner: op, } } - - /// The metakey that requested by object_store, should align with its meta. - #[inline] - fn metakey() -> flagset::FlagSet { - Metakey::Mode - | Metakey::LastModified - | Metakey::ContentLength - | Metakey::Etag - | Metakey::Version - } } impl Debug for OpendalStore { @@ -308,7 +298,6 @@ impl ObjectStore for OpendalStore { let stream = self .inner .lister_with(&path) - .metakey(Self::metakey()) .recursive(true) .await .map_err(|err| format_object_store_error(err, &path))?; @@ -338,7 +327,6 @@ impl ObjectStore for OpendalStore { self.inner .lister_with(&path) .start_after(offset.as_ref()) - .metakey(Self::metakey()) .recursive(true) .into_future() .into_send() @@ -350,7 +338,6 @@ impl ObjectStore for OpendalStore { } else { self.inner .lister_with(&path) - .metakey(Self::metakey()) .recursive(true) .into_future() .into_send() @@ -372,7 +359,6 @@ impl ObjectStore for OpendalStore { let mut stream = self .inner .lister_with(&path) - .metakey(Self::metakey()) .into_future() .into_send() .await diff --git a/integrations/object_store/src/utils.rs b/integrations/object_store/src/utils.rs index 5bfdc7d23fc0..5eb4a10f73bc 100644 --- a/integrations/object_store/src/utils.rs +++ b/integrations/object_store/src/utils.rs @@ -17,7 +17,7 @@ use futures::Stream; use object_store::ObjectMeta; -use opendal::{Entry, Metadata, Metakey}; +use opendal::{Entry, Metadata}; use std::future::IntoFuture; /// Conditionally add the `Send` marker trait for the wrapped type. @@ -51,22 +51,12 @@ pub fn format_object_store_error(err: opendal::Error, path: &str) -> object_stor /// Format `opendal::Metadata` to `object_store::ObjectMeta`. pub fn format_object_meta(path: &str, meta: &Metadata) -> ObjectMeta { - let version = match meta.metakey().contains(Metakey::Version) { - true => meta.version().map(|x| x.to_string()), - false => None, - }; - - let e_tag = match meta.metakey().contains(Metakey::Etag) { - true => meta.etag().map(|x| x.to_string()), - false => None, - }; - ObjectMeta { location: path.into(), last_modified: meta.last_modified().unwrap_or_default(), size: meta.content_length() as usize, - e_tag, - version, + e_tag: meta.etag().map(|x| x.to_string()), + version: meta.version().map(|x| x.to_string()), } }