feat: Upgrade OpenDAL to v0.16.0

Cargo.toml

@@ -2,7 +2,6 @@
 members = [
     # Binaries
     "src/binaries",
-
     # Common
     "src/common/arrow",
     "src/common/base",
@@ -18,7 +17,6 @@ members = [
     "src/common/metrics",
     "src/common/tracing",
     "src/common/storage",
-
     # Query
     "src/query/ast",
     "src/query/codegen",
@@ -50,7 +48,6 @@ members = [
     "src/query/users",
     # databend-query
     "src/query/service",
-
     # Meta
     "src/meta/api",
     "src/meta/app",
@@ -64,7 +61,6 @@ members = [
     "src/meta/protos",
     # databend-meta
     "src/meta/service",
-
     # Hack
     # This crate is used to hack our cargo.lock to resovle CVEs and so on.
     "src/workspace-hack",
@@ -106,3 +102,9 @@ gimli = { opt-level = 3 }
 miniz_oxide = { opt-level = 3 }
 object = { opt-level = 3 }
 rustc-demangle = { opt-level = 3 }
+
+[patch.crates-io]
+# http-types doesn't play well with the new errors api.
+# This fork address this problem, and should be removed once we upgrade to
+# new toolchain.
+http-types = { git = "https://github.com/Xuanwo/http-types", rev = "106fc6d" }

src/binaries/Cargo.toml

@@ -47,8 +47,9 @@ databend-meta = { path = "../meta/service" }
 databend-query = { path = "../query/service" }
 
 # Crates.io dependencies
-anyhow = "1.0.58"
+anyhow = "1.0.64"
 clap = { version = "3.2.5", features = ["derive", "env"] }
+openraft = { git = "https://github.com/datafuselabs/openraft", tag = "v0.7.2" }
 sentry = "0.27.0"
 serde = { version = "1.0.137", features = ["derive"] }
 serde_json = "1.0.81"
@@ -57,8 +58,6 @@ tonic = "0.7.2"
 tracing = "0.1.35"
 url = "2.2.2"
 
-openraft = { git = "https://github.com/drmingdrmer/openraft", rev = "v0.7.2-alpha.4" }
-
 [[bin]]
 name = "databend-meta"
 path = "meta/main.rs"

src/common/base/Cargo.toml

@@ -46,4 +46,4 @@ tracing = "0.1.35"
 uuid = { version = "1.1.2", features = ["serde", "v4"] }
 
 [dev-dependencies]
-anyhow = "1.0.58"
+anyhow = "1.0.64"

src/common/building/Cargo.toml

@@ -11,7 +11,7 @@ doctest = false
 test = false
 
 [dependencies]
-anyhow = "1.0.58"
+anyhow = "1.0.64"
 cargo-license = "0.5.1"
 cargo_metadata = "0.15.0"
 git2 = { version = "0.14.4", default-features = false }

src/common/contexts/Cargo.toml

@@ -12,4 +12,4 @@ test = false
 common-base = { path = "../base" }
 
 async-trait = "0.1.56"
-opendal = { version = "0.14.1", features = ["layers-retry"] }
+opendal = { version = "0.16.0", features = ["layers-retry"] }

src/common/exception/Cargo.toml

@@ -13,14 +13,14 @@ test = false
 [dependencies] # In alphabetical order
 common-arrow = { path = "../arrow" }
 
-anyhow = "1.0.58"
+anyhow = "1.0.64"
 bincode = { version = "2.0.0-rc.1", features = ["serde", "std", "alloc"] }
 octocrab = "0.16.0"
 paste = "1.0.7"
 prost = "0.10.4"
 serde = { version = "1.0.137", features = ["derive"] }
 serde_json = "1.0.81"
-thiserror = "=1.0.31"
+thiserror = "1"
 time = "0.3.10"
 tonic = "0.7.2"
 

src/common/grpc/Cargo.toml

@@ -18,12 +18,12 @@ common-exception = { path = "../exception" }
 # Github dependencies
 
 # Crates.io dependencies
-anyerror = "=0.1.6"
+anyerror = "=0.1.7"
 hyper = "0.14.19"
 jwt-simple = "0.11.0"
 once_cell = "1.12.0"
 serde = { version = "1.0.137", features = ["derive"] }
-thiserror = "=1.0.31"
+thiserror = "1"
 tonic = { version = "0.7.2", features = ["transport", "codegen", "prost", "tls-roots", "tls"] }
 tracing = "0.1.35"
 trust-dns-resolver = { version = "0.21.2", features = ["system-config"] }

src/common/metrics/Cargo.toml

@@ -24,7 +24,7 @@ serde = { version = "1.0.137", features = ["derive"] }
 tracing = "0.1.35"
 
 [dev-dependencies]
-anyhow = "1.0.58"
+anyhow = "1.0.64"
 
 [dev-dependencies.tokio]
 default-features = false

src/common/storage/Cargo.toml

@@ -14,13 +14,12 @@ common-base = { path = "../base" }
 common-contexts = { path = "../contexts" }
 common-exception = { path = "../exception" }
 
-anyhow = "1.0.58"
-backon = "0.0.2"
+anyhow = "1.0.64"
+backon = "0.1.0"
 globiter = "0.1.0"
 once_cell = "1.12.0"
-opendal = { version = "0.14.1", features = [
+opendal = { version = "0.16.0", features = [
     "layers-retry",
-    "services-http",
     "layers-tracing",
     "layers-metrics",
     "compress",

src/common/storage/src/operator.rs

@@ -21,6 +21,7 @@ use common_base::base::Singleton;
 use common_contexts::DalRuntime;
 use common_exception::ErrorCode;
 use once_cell::sync::OnceCell;
+use opendal::layers::ImmutableIndexLayer;
 use opendal::layers::LoggingLayer;
 use opendal::layers::MetricsLayer;
 use opendal::layers::RetryLayer;
@@ -75,7 +76,7 @@ pub fn init_azblob_operator(cfg: &StorageAzblobConfig) -> Result<Operator> {
     builder.account_name(&cfg.account_name);
     builder.account_key(&cfg.account_key);
 
-    Ok(Operator::new(builder.build()?))
+    Ok(Operator::new(builder.build()?).layer(LoggingLayer))
 }
 
 /// init_fs_operator will init a opendal fs operator.
@@ -88,7 +89,7 @@ pub fn init_fs_operator(cfg: &StorageFsConfig) -> Result<Operator> {
     }
     builder.root(&path);
 
-    Ok(Operator::new(builder.build()?))
+    Ok(Operator::new(builder.build()?).layer(LoggingLayer))
 }
 
 /// init_gcs_operator will init a opendal gcs operator.
@@ -102,7 +103,7 @@ pub fn init_gcs_operator(cfg: &StorageGcsConfig) -> Result<Operator> {
         .credential(&cfg.credential)
         .build()?;
 
-    Ok(Operator::new(accessor))
+    Ok(Operator::new(accessor).layer(LoggingLayer))
 }
 
 /// init_hdfs_operator will init an opendal hdfs operator.
@@ -118,7 +119,7 @@ pub fn init_hdfs_operator(cfg: &super::StorageHdfsConfig) -> Result<Operator> {
     // Root
     builder.root(&cfg.root);
 
-    Ok(Operator::new(builder.build()?))
+    Ok(Operator::new(builder.build()?).layer(LoggingLayer))
 }
 
 pub fn init_http_operator(cfg: &StorageHttpConfig) -> Result<Operator> {
@@ -127,17 +128,29 @@ pub fn init_http_operator(cfg: &StorageHttpConfig) -> Result<Operator> {
     // Endpoint.
     builder.endpoint(&cfg.endpoint_url);
 
-    // Update index.
-    builder.extend_index(cfg.paths.iter().map(|v| v.as_str()));
+    // HTTP Service is read-only and doesn't support list operation.
+    // ImmutableIndexLayer will build an in-memory immutable index for it.
+    let mut immutable_layer = ImmutableIndexLayer::default();
+    let files: Vec<String> = cfg
+        .paths
+        .iter()
+        .map(|v| v.trim_start_matches('/').to_string())
+        .collect();
+    // TODO: should be replace by `immutable_layer.extend_iter()` after fix
+    for i in files {
+        immutable_layer.insert(i);
+    }
 
-    Ok(Operator::new(builder.build()?))
+    Ok(Operator::new(builder.build()?)
+        .layer(LoggingLayer)
+        .layer(immutable_layer))
 }
 
 /// init_memory_operator will init a opendal memory operator.
 pub fn init_memory_operator() -> Result<Operator> {
     let mut builder = memory::Builder::default();
 
-    Ok(Operator::new(builder.build()?))
+    Ok(Operator::new(builder.build()?).layer(LoggingLayer))
 }
 
 /// init_s3_operator will init a opendal s3 operator with input s3 config.
@@ -170,7 +183,7 @@ pub fn init_s3_operator(cfg: &StorageS3Config) -> Result<Operator> {
         builder.enable_virtual_host_style();
     }
 
-    Ok(Operator::new(builder.build()?))
+    Ok(Operator::new(builder.build()?).layer(LoggingLayer))
 }
 
 /// init_obs_operator will init a opendal obs operator with input obs config.
@@ -186,7 +199,7 @@ pub fn init_obs_operator(cfg: &StorageObsConfig) -> Result<Operator> {
     builder.access_key_id(&cfg.access_key_id);
     builder.secret_access_key(&cfg.secret_access_key);
 
-    Ok(Operator::new(builder.build()?))
+    Ok(Operator::new(builder.build()?).layer(LoggingLayer))
 }
 
 pub struct StorageOperator;

src/meta/api/Cargo.toml

@@ -22,11 +22,11 @@ common-proto-conv = { path = "../proto-conv" }
 common-protos = { path = "../protos" }
 common-tracing = { path = "../../common/tracing" }
 
-anyhow = "1.0.58"
+anyhow = "1.0.64"
 async-trait = "0.1.56"
 enumflags2 = { version = "0.7.5", features = ["serde"] }
 maplit = "1.0.2"
 serde_json = "1.0.81"
-thiserror = "=1.0.31"
+thiserror = "1"
 tonic = { version = "0.7.2", features = ["transport", "codegen", "prost", "tls-roots", "tls"] }
 tracing = "0.1.35"

src/meta/client/Cargo.toml

@@ -35,7 +35,7 @@ rand = "0.8.5"
 semver = "1.0.10"
 serde = { version = "1.0.137", features = ["derive"] }
 serde_json = "1.0.81"
-thiserror = "=1.0.31"
+thiserror = "1"
 tonic = { version = "0.7.2", features = ["transport", "codegen", "prost", "tls-roots", "tls"] }
 tracing = "0.1.35"
 

src/meta/embedded/Cargo.toml

@@ -31,4 +31,4 @@ tempfile = "3.3.0"
 tracing = "0.1.35"
 
 [dev-dependencies]
-anyhow = "1.0.58"
+anyhow = "1.0.64"

src/meta/proto-conv/Cargo.toml

@@ -18,14 +18,14 @@ common-protos = { path = "../protos" }
 common-storage = { path = "../../common/storage" }
 
 num = "0.4.0"
-thiserror = "=1.0.31"
+thiserror = "1"
 
 enumflags2 = { version = "0.7.5", features = ["serde"] }
 serde_json = "1.0.81"
 
 [build-dependencies]
 
 [dev-dependencies]
-anyhow = "1.0.58"
+anyhow = "1.0.64"
 maplit = "1.0.2"
 pretty_assertions = "1.2.1"

src/meta/raft-store/Cargo.toml

@@ -24,7 +24,7 @@ common-meta-types = { path = "../types" }
 common-tracing = { path = "../../common/tracing" }
 
 # crates.io deps
-anyhow = "1.0.58"
+anyhow = "1.0.64"
 async-trait = "0.1.56"
 bytes = "1.1.0"
 derive_more = "0.99.17"

src/meta/service/Cargo.toml

@@ -41,8 +41,8 @@ common-tracing = { path = "../../common/tracing" }
 sled = { git = "https://github.com/datafuse-extras/sled", tag = "v0.34.7-datafuse.1", default-features = false }
 
 # Crates.io dependencies
-anyerror = "=0.1.6"
-anyhow = "1.0.58"
+anyerror = "=0.1.7"
+anyhow = "1.0.64"
 async-trait = "0.1.56"
 clap = { version = "3.2.5", features = ["derive", "env"] }
 futures = "0.3.21"

src/meta/sled-store/Cargo.toml

@@ -17,10 +17,10 @@ io-uring = ["sled/io_uring"]
 [dependencies]
 common-meta-types = { path = "../types" }
 
-openraft = { git = "https://github.com/drmingdrmer/openraft", rev = "v0.7.2-alpha.4" }
+openraft = { git = "https://github.com/datafuselabs/openraft", tag = "v0.7.2" }
 sled = { git = "https://github.com/datafuse-extras/sled", tag = "v0.34.7-datafuse.1", default-features = false }
 
-anyhow = "1.0.58"
+anyhow = "1.0.64"
 byteorder = "1.4.3"
 once_cell = "1.12.0"
 serde = { version = "1.0.137", features = ["derive"] }

src/meta/types/Cargo.toml

@@ -15,10 +15,10 @@ common-datavalues = { path = "../../query/datavalues" }
 common-exception = { path = "../../common/exception" }
 common-storage = { path = "../../common/storage" }
 
-openraft = { git = "https://github.com/drmingdrmer/openraft", rev = "v0.7.2-alpha.4" }
+openraft = { git = "https://github.com/datafuselabs/openraft", tag = "v0.7.2" }
 sled = { git = "https://github.com/datafuse-extras/sled", tag = "v0.34.7-datafuse.1", default-features = false }
 
-anyerror = "=0.1.6"
+anyerror = "=0.1.7"
 derive_more = "0.99.17"
 enumflags2 = { version = "0.7.5", features = ["serde"] }
 hex = "0.4.3"
@@ -31,7 +31,7 @@ serde = { version = "1.0.137", features = ["derive"] }
 serde_json = "1.0.81"
 sha1 = "0.10.1"
 sha2 = "0.10.2"
-thiserror = "=1.0.31"
+thiserror = "1"
 tonic = { version = "0.7.2", features = ["transport", "codegen", "prost", "tls-roots", "tls"] }
 
 [build-dependencies]
@@ -40,5 +40,5 @@ prost-build = "0.10.4"
 tonic-build = "0.7.2"
 
 [dev-dependencies]
-anyhow = "1.0.58"
+anyhow = "1.0.64"
 regex = "1.5.6"

src/query/ast/Cargo.toml

@@ -31,7 +31,7 @@ nom = "7.1.1"
 nom-rule = "0.3.0"
 pratt = "0.3.0"
 pretty = "0.11.3"
-thiserror = "=1.0.31"
+thiserror = "1"
 url = "2.2.2"
 
 [dev-dependencies]

src/query/catalog/Cargo.toml

@@ -26,5 +26,5 @@ common-users = { path = "../users" }
 
 async-trait = "0.1.56"
 dyn-clone = "1.0.6"
-opendal = { version = "0.14.1", features = ["layers-retry"] }
+opendal = { version = "0.16.0", features = ["layers-retry"] }
 parking_lot = "0.12.1"

src/query/config/Cargo.toml

@@ -21,7 +21,7 @@ common-users = { path = "../users" }
 clap = { version = "3.2.5", features = ["derive", "env"] }
 hex = "0.4.3"
 once_cell = "1.12.0"
-opendal = { version = "0.14.1", features = ["layers-retry", "compress"], optional = true }
+opendal = { version = "0.16.0", features = ["layers-retry", "compress"], optional = true }
 semver = "1.0.10"
 serde = { version = "1.0.137", features = ["derive"] }
 serfig = "0.0.2"

src/query/functions-v2/tests/it/scalars/testdata/string.txt

@@ -2764,12 +2764,12 @@ evaluation:
 | Row 2  | "c"         | "ccc"   |
 +--------+-------------+---------+
 evaluation (internal):
-+--------+------------------------------------------------------------------------------------+
-| Column | Data                                                                               |
-+--------+------------------------------------------------------------------------------------+
-| a      | StringColumn { data: [97, 98, 99], offsets: [0, 1, 2, 3] }                         |
-| Output | StringColumn { data: [97, 97, 97, 98, 98, 98, 99, 99, 99], offsets: [0, 3, 6, 9] } |
-+--------+------------------------------------------------------------------------------------+
++--------+--------------------------------------------------------------------+
+| Column | Data                                                               |
++--------+--------------------------------------------------------------------+
+| a      | StringColumn { data: 0x616263, offsets: [0, 1, 2, 3] }             |
+| Output | StringColumn { data: 0x616161626262636363, offsets: [0, 3, 6, 9] } |
++--------+--------------------------------------------------------------------+
 
 
 error: 

src/query/pipeline/sources/Cargo.toml

@@ -23,5 +23,5 @@ common-streams = { path = "../../streams" }
 async-trait = { version = "0.1.0", package = "async-trait-fn" }
 futures = "0.3.21"
 futures-util = "0.3.21"
-opendal = { version = "0.14.1", features = ["layers-retry", "compress"] }
+opendal = { version = "0.16.0", features = ["layers-retry", "compress"] }
 parking_lot = "0.12.1"