diff --git a/Cargo.lock b/Cargo.lock index cd4c66d0df..ae01a248ff 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -74,6 +74,10 @@ dependencies = [ "alloc-no-stdlib", ] +[[package]] +name = "alloc_tracker" +version = "1.2.5-alpha" + [[package]] name = "analytic_engine" version = "1.2.5-alpha" @@ -86,15 +90,21 @@ dependencies = [ "base64 0.13.1", "bytes", "ceresdbproto", + "codec", "common_types", - "common_util", "datafusion", "env_logger", + "future_cancel", "futures 0.3.28", + "generic_error", + "hex", + "id_allocator", "lazy_static", "log", "lru", + "macros", "message_queue", + "metric_ext", "object_store 1.2.5-alpha", "parquet", "parquet_ext", @@ -104,13 +114,17 @@ dependencies = [ "rand 0.7.3", "remote_engine_client", "router", + "runtime", "serde", + "size_ext", "skiplist", "smallvec", "snafu 0.6.10", "table_engine", "table_kv", "tempfile", + "test_util", + "time_ext", "tokio", "trace_metric", "wal", @@ -609,21 +623,25 @@ dependencies = [ "base64 0.13.1", "clap 3.2.23", "common_types", - "common_util", "criterion", "env_logger", "futures 0.3.28", "log", + "macros", "object_store 1.2.5-alpha", "parquet", "parquet_ext", "pprof 0.10.1", "rand 0.7.3", + "runtime", "serde", + "size_ext", "snafu 0.6.10", "table_engine", "table_kv", + "time_ext", "tokio", + "toml_ext", "trace_metric", "wal", "zstd 0.12.3+zstd.1.5.2", @@ -988,10 +1006,12 @@ version = "1.2.5-alpha" dependencies = [ "async-trait", "common_types", - "common_util", + "generic_error", "log", + "macros", "snafu 0.6.10", "table_engine", + "time_ext", ] [[package]] @@ -1003,8 +1023,9 @@ dependencies = [ "catalog", "cluster", "common_types", - "common_util", + "generic_error", "log", + "macros", "meta_client", "snafu 0.6.10", "system_catalog", @@ -1030,7 +1051,6 @@ dependencies = [ "catalog_impls", "clap 3.2.23", "cluster", - "common_util", "df_operator", "etcd-client", "interpreters", @@ -1038,14 +1058,17 @@ dependencies = [ "logger", "meta_client", "moka", + "panic_ext", "proxy", "query_engine", "router", + "runtime", "serde", "server", "signal-hook", "table_engine", "toml 0.7.3", + "toml_ext", "tracing_util", "vergen", ] @@ -1242,15 +1265,18 @@ dependencies = [ "catalog", "ceresdbproto", "common_types", - "common_util", "etcd-client", + "generic_error", "log", + "macros", "meta_client", "prost", + "runtime", "serde", "serde_json", "snafu 0.6.10", "table_engine", + "time_ext", "tokio", "wal", ] @@ -1264,6 +1290,15 @@ dependencies = [ "cc", ] +[[package]] +name = "codec" +version = "1.2.5-alpha" +dependencies = [ + "common_types", + "macros", + "snafu 0.6.10", +] + [[package]] name = "codespan-reporting" version = "0.11.1" @@ -1296,8 +1331,8 @@ dependencies = [ "bytes_ext", "ceresdbproto", "chrono", - "common_util", "datafusion", + "macros", "murmur3", "paste 1.0.12", "prost", @@ -1308,39 +1343,6 @@ dependencies = [ "sqlparser", ] -[[package]] -name = "common_util" -version = "1.2.5-alpha" -dependencies = [ - "arrow", - "backtrace", - "ceresdbproto", - "chrono", - "common_types", - "crossbeam-utils 0.8.15", - "env_logger", - "futures 0.3.28", - "gag", - "hex", - "lazy_static", - "libc", - "log", - "logger", - "nix 0.22.3", - "pin-project-lite", - "prometheus 0.12.0", - "serde", - "serde_json", - "slog", - "slog-global", - "snafu 0.6.10", - "tempfile", - "time 0.3.20", - "tokio", - "tokio-test", - "toml 0.7.3", -] - [[package]] name = "concurrent-queue" version = "2.1.0" @@ -2001,9 +2003,10 @@ dependencies = [ "bincode", "chrono", "common_types", - "common_util", "datafusion", + "generic_error", "hyperloglog", + "macros", "smallvec", "snafu 0.6.10", ] @@ -2326,6 +2329,15 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" +[[package]] +name = "future_cancel" +version = "1.2.5-alpha" +dependencies = [ + "futures 0.3.28", + "runtime", + "tokio", +] + [[package]] name = "futures" version = "0.1.31" @@ -2495,6 +2507,10 @@ dependencies = [ "version_check", ] +[[package]] +name = "generic_error" +version = "1.2.5-alpha" + [[package]] name = "getrandom" version = "0.1.16" @@ -2798,6 +2814,14 @@ dependencies = [ "cxx-build", ] +[[package]] +name = "id_allocator" +version = "1.2.5-alpha" +dependencies = [ + "generic_error", + "tokio", +] + [[package]] name = "ident_case" version = "1.0.1" @@ -2891,18 +2915,21 @@ dependencies = [ "async-trait", "catalog", "catalog_impls", + "codec", "common_types", - "common_util", "datafusion", "datafusion-proto", "df_operator", + "generic_error", "log", + "macros", "meta_client", "query_engine", "query_frontend", "regex", "snafu 0.6.10", "table_engine", + "test_util", "tokio", ] @@ -3335,6 +3362,10 @@ dependencies = [ "libc", ] +[[package]] +name = "macros" +version = "1.2.5-alpha" + [[package]] name = "matchers" version = "0.1.0" @@ -3413,12 +3444,13 @@ version = "0.1.0" dependencies = [ "async-trait", "chrono", - "common_util", "futures 0.3.28", "log", + "macros", "rskafka", "serde", "snafu 0.6.10", + "time_ext", "tokio", "uuid 1.3.3", ] @@ -3430,20 +3462,32 @@ dependencies = [ "async-trait", "ceresdbproto", "common_types", - "common_util", "futures 0.3.28", + "generic_error", "log", + "macros", "prost", "reqwest", "serde", "serde_json", "snafu 0.6.10", "table_engine", + "time_ext", "tokio", "tonic 0.8.3", "url", ] +[[package]] +name = "metric_ext" +version = "1.2.5-alpha" +dependencies = [ + "crossbeam-utils 0.8.15", + "serde", + "serde_json", + "time_ext", +] + [[package]] name = "mime" version = "0.3.17" @@ -3921,22 +3965,27 @@ dependencies = [ "chrono", "clru", "common_types", - "common_util", "crc", "futures 0.3.28", + "generic_error", "lazy_static", "log", "lru", + "macros", "object_store 0.5.6", + "partitioned_lock", "prometheus 0.12.0", "prometheus-static-metric", "prost", "rand 0.7.3", + "runtime", "serde", "serde_json", + "size_ext", "snafu 0.6.10", "table_kv", "tempfile", + "time_ext", "tokio", "twox-hash", "uuid 1.3.3", @@ -4042,6 +4091,20 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" +[[package]] +name = "panic_ext" +version = "1.2.5-alpha" +dependencies = [ + "backtrace", + "gag", + "libc", + "log", + "logger", + "nix 0.22.3", + "slog", + "slog-global", +] + [[package]] name = "parking" version = "2.0.0" @@ -4138,9 +4201,9 @@ dependencies = [ "arrow_ext", "async-trait", "bytes", - "common_util", "datafusion", "futures 0.3.28", + "generic_error", "log", "object_store 1.2.5-alpha", "parquet", @@ -4162,14 +4225,23 @@ version = "1.2.5-alpha" dependencies = [ "async-trait", "common_types", - "common_util", "futures 0.3.28", + "generic_error", "lazy_static", + "macros", "prometheus 0.12.0", "snafu 0.6.10", "table_engine", ] +[[package]] +name = "partitioned_lock" +version = "1.2.5-alpha" +dependencies = [ + "common_types", + "tokio", +] + [[package]] name = "paste" version = "0.1.18" @@ -4719,10 +4791,10 @@ dependencies = [ "clru", "cluster", "common_types", - "common_util", "datafusion", "df_operator", "futures 0.3.28", + "generic_error", "http", "influxdb-line-protocol", "interpreters", @@ -4731,7 +4803,7 @@ dependencies = [ "json_pretty", "lazy_static", "log", - "logger", + "macros", "meta_client", "paste 1.0.12", "prom-remote-api", @@ -4741,6 +4813,7 @@ dependencies = [ "query_engine", "query_frontend", "router", + "runtime", "serde", "serde_json", "snafu 0.6.10", @@ -4748,6 +4821,8 @@ dependencies = [ "sqlparser", "system_catalog", "table_engine", + "time_ext", + "timed_task", "tokio", "tokio-stream", "tonic 0.8.3", @@ -4810,16 +4885,18 @@ dependencies = [ "async-trait", "chrono", "common_types", - "common_util", "datafusion", "df_operator", "futures 0.3.28", + "generic_error", "iox_query", "log", + "macros", "query_frontend", "serde", "snafu 0.6.10", "table_engine", + "time_ext", "tokio", ] @@ -4832,17 +4909,19 @@ dependencies = [ "catalog", "ceresdbproto", "cluster", + "codec", "common_types", - "common_util", "datafusion", "datafusion-proto", "df_operator", + "generic_error", "hashbrown 0.12.3", "influxdb_influxql_parser", "iox_query_influxql", "itertools", "lazy_static", "log", + "macros", "paste 1.0.12", "prom-remote-api", "regex", @@ -5140,13 +5219,16 @@ dependencies = [ "async-trait", "ceresdbproto", "common_types", - "common_util", "futures 0.3.28", + "generic_error", "log", + "macros", "router", + "runtime", "serde", "snafu 0.6.10", "table_engine", + "time_ext", "tokio", "tonic 0.8.3", ] @@ -5267,13 +5349,15 @@ dependencies = [ "ceresdbproto", "cluster", "common_types", - "common_util", + "generic_error", "log", + "macros", "meta_client", "moka", "serde", "snafu 0.6.10", "table_engine", + "time_ext", "tokio", "twox-hash", ] @@ -5300,6 +5384,19 @@ dependencies = [ "tracing", ] +[[package]] +name = "runtime" +version = "1.2.5-alpha" +dependencies = [ + "lazy_static", + "macros", + "pin-project-lite", + "prometheus 0.12.0", + "snafu 0.6.10", + "tokio", + "tokio-test", +] + [[package]] name = "rust-crypto" version = "0.2.36" @@ -5623,16 +5720,17 @@ dependencies = [ "clru", "cluster", "common_types", - "common_util", "datafusion", "df_operator", "futures 0.3.28", + "generic_error", "http", "influxdb-line-protocol", "interpreters", "lazy_static", "log", "logger", + "macros", "meta_client", "opensrv-mysql", "partition_table_engine", @@ -5647,12 +5745,15 @@ dependencies = [ "query_frontend", "remote_engine_client", "router", + "runtime", "serde", "serde_json", + "size_ext", "snafu 0.6.10", "spin 0.9.8", "sqlparser", "table_engine", + "time_ext", "tokio", "tokio-stream", "tonic 0.8.3", @@ -5776,6 +5877,14 @@ dependencies = [ "serde", ] +[[package]] +name = "size_ext" +version = "1.2.5-alpha" +dependencies = [ + "serde", + "toml 0.7.3", +] + [[package]] name = "skeptic" version = "0.13.7" @@ -6159,10 +6268,12 @@ dependencies = [ "async-trait", "catalog", "ceresdbproto", + "codec", "common_types", - "common_util", "futures 0.3.28", + "generic_error", "log", + "macros", "prost", "snafu 0.6.10", "table_engine", @@ -6179,15 +6290,16 @@ dependencies = [ "async-trait", "ceresdbproto", "common_types", - "common_util", "datafusion", "datafusion-proto", "df_operator", - "env_logger", "futures 0.3.28", + "generic_error", "itertools", "log", + "macros", "prost", + "runtime", "serde", "smallvec", "snafu 0.6.10", @@ -6199,12 +6311,13 @@ dependencies = [ name = "table_kv" version = "1.2.5-alpha" dependencies = [ - "common_util", "log", + "macros", "obkv-table-client-rs", "rand 0.7.3", "serde", "snafu 0.6.10", + "time_ext", ] [[package]] @@ -6271,6 +6384,28 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "test_util" +version = "1.2.5-alpha" +dependencies = [ + "arrow", + "backtrace", + "ceresdbproto", + "chrono", + "common_types", + "env_logger", + "futures 0.3.28", + "lazy_static", + "libc", + "log", + "pin-project-lite", + "serde", + "serde_json", + "snafu 0.6.10", + "tempfile", + "tokio", +] + [[package]] name = "textwrap" version = "0.11.0" @@ -6366,6 +6501,29 @@ dependencies = [ "time-core", ] +[[package]] +name = "time_ext" +version = "1.2.5-alpha" +dependencies = [ + "ceresdbproto", + "chrono", + "common_types", + "macros", + "serde", + "serde_json", + "snafu 0.6.10", + "toml 0.7.3", +] + +[[package]] +name = "timed_task" +version = "1.2.5-alpha" +dependencies = [ + "log", + "runtime", + "tokio", +] + [[package]] name = "tiny-keccak" version = "2.0.2" @@ -6545,6 +6703,17 @@ dependencies = [ "winnow", ] +[[package]] +name = "toml_ext" +version = "1.2.5-alpha" +dependencies = [ + "macros", + "serde", + "snafu 0.6.10", + "tempfile", + "toml 0.7.3", +] + [[package]] name = "tonic" version = "0.8.3" @@ -6626,14 +6795,14 @@ dependencies = [ "anyhow", "clap 3.2.23", "common_types", - "common_util", - "env_logger", "futures 0.3.28", "num_cpus", "object_store 1.2.5-alpha", "parquet", "parquet_ext", + "runtime", "table_engine", + "time_ext", "tokio", ] @@ -6993,21 +7162,26 @@ dependencies = [ "async-trait", "ceresdbproto", "chrono", + "codec", "common_types", - "common_util", - "env_logger", "futures 0.3.28", + "generic_error", "log", + "macros", "message_queue", "prost", "rand 0.8.5", "rocksdb", + "runtime", "serde", "serde_json", + "size_ext", "smallvec", "snafu 0.6.10", "table_kv", "tempfile", + "time_ext", + "timed_task", "tokio", "uuid 1.3.3", ] diff --git a/Cargo.toml b/Cargo.toml index be9a86bd69..c82c27e73e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,17 +18,30 @@ members = [ "catalog_impls", "cluster", "common_types", - "common_util", + "components/alloc_tracker", "components/arena", "components/arrow_ext", "components/bytes_ext", + "components/codec", + "components/future_cancel", + "components/id_allocator", "components/logger", + "components/macros", "components/message_queue", + "components/metric_ext", "components/object_store", + "components/panic_ext", "components/parquet_ext", + "components/partitioned_lock", "components/profile", + "components/runtime", + "components/size_ext", "components/skiplist", "components/table_kv", + "components/test_util", + "components/time_ext", + "components/timed_task", + "components/toml_ext", "components/trace_metric", "components/trace_metric_derive", "components/trace_metric_derive_tests", @@ -75,7 +88,6 @@ clru = "0.6.1" cluster = { path = "cluster" } criterion = "0.3" common_types = { path = "common_types" } -common_util = { path = "common_util" } datafusion = { git = "https://github.com/ceresdb/arrow-datafusion.git", rev = "acb5d97a8a8de5296989740f97db3773fe3aa45a" } datafusion-proto = { git = "https://github.com/ceresdb/arrow-datafusion.git", rev = "acb5d97a8a8de5296989740f97db3773fe3aa45a" } df_operator = { path = "df_operator" } @@ -93,6 +105,7 @@ influxql-query = { git = "https://github.com/CeresDB/influxql", rev = "b65a125b9 influxql-schema = { git = "https://github.com/CeresDB/influxql", rev = "b65a125b9cdfa3121a3c8843bc48441b91049e31", package = "schema" } interpreters = { path = "interpreters" } itertools = "0.10.5" +macros = { path = "components/macros" } meta_client = { path = "meta_client" } object_store = { path = "components/object_store" } partition_table_engine = { path = "partition_table_engine" } @@ -107,24 +120,31 @@ prometheus-static-metric = "0.5" prost = "0.11" proxy = { path = "proxy" } query_engine = { path = "query_engine" } +query_frontend = { path = "query_frontend" } rand = "0.7" remote_engine_client = { path = "remote_engine_client" } reqwest = { version = "0.11", default-features = false, features = ["rustls-tls", "json"] } router = { path = "router" } +runtime = { path = "components/runtime" } snafu = { version = "0.6.10", features = ["backtraces"] } serde = "1.0" serde_json = "1.0.60" server = { path = "server" } +size_ext = { path = "components/size_ext" } smallvec = "1.6" slog = "2.7" spin = "0.9.6" -query_frontend = { path = "query_frontend" } sqlparser = { version = "0.33", features = ["serde"] } system_catalog = { path = "system_catalog" } table_engine = { path = "table_engine" } table_kv = { path = "components/table_kv" } tempfile = "3.1.0" +test_util = { path = "components/test_util", features = ["test"] } +time_ext = { path = "components/time_ext" } toml = "0.7" +toml_ext = { path = "components/toml_ext" } +generic_error = { path = "components/generic_error" } +partitioned_lock = { path = "components/partitioned_lock" } tracing_util = { path = "components/tracing_util" } trace_metric = { path = "components/trace_metric" } trace_metric_derive = { path = "components/trace_metric_derive" } @@ -134,6 +154,14 @@ tokio = { version = "1.25", features = ["full"] } wal = { path = "wal" } message_queue = { path = "components/message_queue" } zstd = { version = "0.12", default-features = false } +id_allocator = { path = "components/id_allocator" } +panic_ext = { path = "components/panic_ext" } +timed_task = { path = "components/timed_task" } +future_cancel = { path = "components/future_cancel" } +alloc_tracker = { path = "components/alloc_tracker" } +metric_ext = { path = "components/metric_ext" } +codec = { path = "components/codec" } +hex = "0.4.3" [dependencies] analytic_engine = { workspace = true } @@ -141,7 +169,6 @@ catalog = { workspace = true } catalog_impls = { workspace = true } clap = { workspace = true } cluster = { workspace = true } -common_util = { workspace = true } df_operator = { workspace = true } etcd-client = { workspace = true } interpreters = { workspace = true } @@ -149,14 +176,17 @@ log = { workspace = true } logger = { workspace = true } meta_client = { workspace = true } moka = { version = "0.10", features = ["future"] } +panic_ext = { workspace = true } proxy = { workspace = true } query_engine = { workspace = true } router = { workspace = true } +runtime = { workspace = true } serde = { workspace = true } server = { workspace = true } signal-hook = "0.3" table_engine = { workspace = true } toml = { workspace = true } +toml_ext = { workspace = true } tracing_util = { workspace = true } [build-dependencies] diff --git a/analytic_engine/Cargo.toml b/analytic_engine/Cargo.toml index 694103701d..26a3d0a827 100644 --- a/analytic_engine/Cargo.toml +++ b/analytic_engine/Cargo.toml @@ -23,14 +23,20 @@ async-trait = { workspace = true } base64 = { workspace = true } bytes = { workspace = true } ceresdbproto = { workspace = true } +codec = { workspace = true } common_types = { workspace = true } -common_util = { workspace = true } datafusion = { workspace = true } +future_cancel = { workspace = true } futures = { workspace = true } +generic_error = { workspace = true } +hex = { workspace = true } +id_allocator = { workspace = true } lazy_static = { workspace = true } log = { workspace = true } lru = { workspace = true } +macros = { workspace = true } message_queue = { workspace = true } +metric_ext = { workspace = true } object_store = { workspace = true } parquet = { workspace = true } parquet_ext = { workspace = true } @@ -38,13 +44,16 @@ prometheus = { workspace = true } prost = { workspace = true } remote_engine_client = { workspace = true } router = { workspace = true } +runtime = { workspace = true } serde = { workspace = true } +size_ext = { workspace = true } skiplist = { path = "../components/skiplist" } smallvec = { workspace = true } snafu = { workspace = true } table_engine = { workspace = true } table_kv = { workspace = true } tempfile = { workspace = true, optional = true } +time_ext = { workspace = true } tokio = { workspace = true } trace_metric = { workspace = true } wal = { workspace = true } @@ -52,9 +61,9 @@ xorfilter-rs = { workspace = true } [dev-dependencies] common_types = { workspace = true, features = ["test"] } -common_util = { workspace = true, features = ["test"] } env_logger = { workspace = true } pin-project-lite = { workspace = true } rand = { workspace = true } tempfile = { workspace = true } +test_util = { workspace = true } wal = { workspace = true, features = ["test"] } diff --git a/analytic_engine/src/compaction/mod.rs b/analytic_engine/src/compaction/mod.rs index 407e6a4e76..1b383814b4 100644 --- a/analytic_engine/src/compaction/mod.rs +++ b/analytic_engine/src/compaction/mod.rs @@ -5,9 +5,10 @@ use std::{collections::HashMap, fmt, str::FromStr, sync::Arc}; use common_types::COMPACTION_STRATEGY; -use common_util::config::{ReadableSize, TimeUnit}; use serde::{Deserialize, Serialize}; +use size_ext::ReadableSize; use snafu::{ensure, Backtrace, GenerateBacktrace, ResultExt, Snafu}; +use time_ext::TimeUnit; use tokio::sync::oneshot; use crate::{ diff --git a/analytic_engine/src/compaction/picker.rs b/analytic_engine/src/compaction/picker.rs index f2f38f8261..ecda034687 100644 --- a/analytic_engine/src/compaction/picker.rs +++ b/analytic_engine/src/compaction/picker.rs @@ -9,9 +9,10 @@ use std::{ }; use common_types::time::Timestamp; -use common_util::{config::TimeUnit, define_result}; use log::{debug, info}; +use macros::define_result; use snafu::Snafu; +use time_ext::TimeUnit; use crate::{ compaction::{ @@ -688,7 +689,7 @@ mod tests { tests::build_schema, time::{TimeRange, Timestamp}, }; - use common_util::hash_map; + use macros::hash_map; use tokio::sync::mpsc; use super::*; diff --git a/analytic_engine/src/compaction/scheduler.rs b/analytic_engine/src/compaction/scheduler.rs index 620bfa83ed..834d77e249 100644 --- a/analytic_engine/src/compaction/scheduler.rs +++ b/analytic_engine/src/compaction/scheduler.rs @@ -14,16 +14,14 @@ use std::{ use async_trait::async_trait; use common_types::request_id::RequestId; -use common_util::{ - config::{ReadableDuration, ReadableSize}, - define_result, - runtime::{JoinHandle, Runtime}, - time::DurationExt, -}; use log::{debug, error, info, warn}; +use macros::define_result; +use runtime::{JoinHandle, Runtime}; use serde::{Deserialize, Serialize}; +use size_ext::ReadableSize; use snafu::{ResultExt, Snafu}; use table_engine::table::TableId; +use time_ext::{DurationExt, ReadableDuration}; use tokio::{ sync::{ mpsc::{self, error::TrySendError, Receiver, Sender}, @@ -49,7 +47,7 @@ use crate::{ #[derive(Debug, Snafu)] pub enum Error { #[snafu(display("Failed to join compaction schedule worker, err:{}", source))] - JoinWorker { source: common_util::runtime::Error }, + JoinWorker { source: runtime::Error }, } define_result!(Error); @@ -647,7 +645,7 @@ impl ScheduleWorker { for table_data in &tables_buf { let last_flush_time = table_data.last_flush_time(); let flush_deadline_ms = last_flush_time + self.max_unflushed_duration.as_millis_u64(); - let now_ms = common_util::time::current_time_millis(); + let now_ms = time_ext::current_time_millis(); if now_ms > flush_deadline_ms { info!( "Scheduled flush is triggered, table:{}, last_flush_time:{last_flush_time}ms, max_unflushed_duration:{:?}", diff --git a/analytic_engine/src/engine.rs b/analytic_engine/src/engine.rs index 528c84334d..840cb1155d 100644 --- a/analytic_engine/src/engine.rs +++ b/analytic_engine/src/engine.rs @@ -5,7 +5,7 @@ use std::sync::Arc; use async_trait::async_trait; -use common_util::error::BoxError; +use generic_error::BoxError; use log::{error, info}; use snafu::{OptionExt, ResultExt}; use table_engine::{ diff --git a/analytic_engine/src/instance/alter.rs b/analytic_engine/src/instance/alter.rs index 81ffb58b31..6103e4b37d 100644 --- a/analytic_engine/src/instance/alter.rs +++ b/analytic_engine/src/instance/alter.rs @@ -4,7 +4,7 @@ use std::collections::HashMap; -use common_util::error::BoxError; +use generic_error::BoxError; use log::info; use snafu::{ensure, ResultExt}; use table_engine::table::AlterSchemaRequest; diff --git a/analytic_engine/src/instance/create.rs b/analytic_engine/src/instance/create.rs index b971102cb7..e150eff351 100644 --- a/analytic_engine/src/instance/create.rs +++ b/analytic_engine/src/instance/create.rs @@ -2,7 +2,7 @@ //! Create table logic of instance -use common_util::error::BoxError; +use generic_error::BoxError; use log::info; use snafu::{OptionExt, ResultExt}; use table_engine::engine::CreateTableRequest; diff --git a/analytic_engine/src/instance/engine.rs b/analytic_engine/src/instance/engine.rs index e32fa064d5..202c847d0a 100644 --- a/analytic_engine/src/instance/engine.rs +++ b/analytic_engine/src/instance/engine.rs @@ -5,7 +5,8 @@ use std::sync::Arc; use common_types::schema::Version; -use common_util::{define_result, error::GenericError}; +use generic_error::GenericError; +use macros::define_result; use snafu::{Backtrace, OptionExt, Snafu}; use table_engine::{ engine::{CloseTableRequest, CreateTableRequest, DropTableRequest, OpenShardRequest}, diff --git a/analytic_engine/src/instance/flush_compaction.rs b/analytic_engine/src/instance/flush_compaction.rs index 0992427c02..1b6d3465a0 100644 --- a/analytic_engine/src/instance/flush_compaction.rs +++ b/analytic_engine/src/instance/flush_compaction.rs @@ -12,20 +12,17 @@ use common_types::{ time::TimeRange, SequenceNumber, }; -use common_util::{ - config::ReadableDuration, - define_result, - error::{BoxError, GenericError}, - runtime::{Runtime, RuntimeRef}, - time, -}; use futures::{ channel::{mpsc, mpsc::channel}, stream, SinkExt, TryStreamExt, }; +use generic_error::{BoxError, GenericError}; use log::{debug, error, info}; +use macros::define_result; +use runtime::{Runtime, RuntimeRef}; use snafu::{Backtrace, ResultExt, Snafu}; use table_engine::predicate::Predicate; +use time_ext::{self, ReadableDuration}; use tokio::{sync::oneshot, time::Instant}; use wal::manager::WalLocation; @@ -126,7 +123,7 @@ pub enum Error { ChannelSend { source: mpsc::SendError }, #[snafu(display("Runtime join error, source:{}", source))] - RuntimeJoin { source: common_util::runtime::Error }, + RuntimeJoin { source: runtime::Error }, #[snafu(display("Other failure, msg:{}.\nBacktrace:\n{:?}", msg, backtrace))] Other { msg: String, backtrace: Backtrace }, @@ -278,7 +275,7 @@ impl FlushTask { })?; self.table_data - .set_last_flush_time(time::current_time_millis()); + .set_last_flush_time(time_ext::current_time_millis()); info!( "Instance flush memtables done, table:{}, table_id:{}, request_id:{}, cost:{}ms", diff --git a/analytic_engine/src/instance/mod.rs b/analytic_engine/src/instance/mod.rs index 6546a3e97f..0ea9abf51b 100644 --- a/analytic_engine/src/instance/mod.rs +++ b/analytic_engine/src/instance/mod.rs @@ -21,13 +21,11 @@ pub(crate) mod write; use std::sync::Arc; use common_types::table::TableId; -use common_util::{ - define_result, - error::{BoxError, GenericError}, - runtime::Runtime, -}; +use generic_error::{BoxError, GenericError}; use log::{error, info}; +use macros::define_result; use mem_collector::MemUsageCollector; +use runtime::Runtime; use snafu::{ResultExt, Snafu}; use table_engine::{engine::EngineRuntimes, table::FlushRequest}; use tokio::sync::oneshot::{self, error::RecvError}; diff --git a/analytic_engine/src/instance/read.rs b/analytic_engine/src/instance/read.rs index 18195a966d..40192b263c 100644 --- a/analytic_engine/src/instance/read.rs +++ b/analytic_engine/src/instance/read.rs @@ -15,9 +15,10 @@ use common_types::{ schema::RecordSchema, time::TimeRange, }; -use common_util::{define_result, error::BoxError}; use futures::stream::Stream; +use generic_error::BoxError; use log::debug; +use macros::define_result; use snafu::{ResultExt, Snafu}; use table_engine::{ stream::{ diff --git a/analytic_engine/src/instance/serial_executor.rs b/analytic_engine/src/instance/serial_executor.rs index 4a84404e2c..4c8db96e41 100644 --- a/analytic_engine/src/instance/serial_executor.rs +++ b/analytic_engine/src/instance/serial_executor.rs @@ -8,10 +8,11 @@ use std::{ time::Instant, }; -use common_util::{runtime::Runtime, time::InstantExt}; use futures::Future; use log::{error, warn}; +use runtime::Runtime; use table_engine::table::TableId; +use time_ext::InstantExt; use tokio::sync::{ oneshot, watch::{self, Receiver, Sender}, diff --git a/analytic_engine/src/instance/wal_replayer.rs b/analytic_engine/src/instance/wal_replayer.rs index c1e6fd96b9..5e888b3c7c 100644 --- a/analytic_engine/src/instance/wal_replayer.rs +++ b/analytic_engine/src/instance/wal_replayer.rs @@ -10,7 +10,7 @@ use std::{ use async_trait::async_trait; use common_types::{schema::IndexInWriterSchema, table::ShardId}; -use common_util::error::BoxError; +use generic_error::BoxError; use lazy_static::lazy_static; use log::{debug, error, info, trace}; use prometheus::{exponential_buckets, register_histogram, Histogram}; diff --git a/analytic_engine/src/instance/write.rs b/analytic_engine/src/instance/write.rs index 219e86102c..2cbf0e0ea2 100644 --- a/analytic_engine/src/instance/write.rs +++ b/analytic_engine/src/instance/write.rs @@ -3,13 +3,14 @@ //! Write logic of instance use ceresdbproto::{schema as schema_pb, table_requests}; +use codec::row; use common_types::{ bytes::ByteVec, row::{RowGroup, RowGroupSlicer}, schema::{IndexInWriterSchema, Schema}, }; -use common_util::{codec::row, define_result}; use log::{debug, error, info, trace, warn}; +use macros::define_result; use smallvec::SmallVec; use snafu::{ensure, Backtrace, ResultExt, Snafu}; use table_engine::table::WriteRequest; @@ -96,9 +97,7 @@ pub enum Error { }, #[snafu(display("Failed to encode row group, err:{}", source))] - EncodeRowGroup { - source: common_util::codec::row::Error, - }, + EncodeRowGroup { source: codec::row::Error }, #[snafu(display("Failed to update sequence of memtable, err:{}", source))] UpdateMemTableSequence { source: crate::memtable::Error }, diff --git a/analytic_engine/src/lib.rs b/analytic_engine/src/lib.rs index 025845afbe..ee11ad7e5e 100644 --- a/analytic_engine/src/lib.rs +++ b/analytic_engine/src/lib.rs @@ -22,12 +22,13 @@ pub mod table_meta_set_impl; #[cfg(any(test, feature = "test"))] pub mod tests; -use common_util::config::{ReadableDuration, ReadableSize}; use manifest::details::Options as ManifestOptions; use message_queue::kafka::config::Config as KafkaConfig; use object_store::config::StorageOptions; use serde::{Deserialize, Serialize}; +use size_ext::ReadableSize; use table_kv::config::ObkvConfig; +use time_ext::ReadableDuration; use wal::{ message_queue_impl::config::Config as MessageQueueWalConfig, rocks_impl::config::Config as RocksDBWalConfig, table_kv_impl::model::NamespaceConfig, diff --git a/analytic_engine/src/manifest/details.rs b/analytic_engine/src/manifest/details.rs index f495ce5aeb..2d9f6ad8c5 100644 --- a/analytic_engine/src/manifest/details.rs +++ b/analytic_engine/src/manifest/details.rs @@ -14,18 +14,16 @@ use std::{ use async_trait::async_trait; use ceresdbproto::manifest as manifest_pb; -use common_util::{ - config::ReadableDuration, - define_result, - error::{BoxError, GenericError, GenericResult}, -}; +use generic_error::{BoxError, GenericError, GenericResult}; use log::{debug, info, warn}; +use macros::define_result; use object_store::{ObjectStoreRef, Path}; use parquet::data_type::AsBytes; use prost::Message; use serde::{Deserialize, Serialize}; use snafu::{Backtrace, ResultExt, Snafu}; use table_engine::table::TableId; +use time_ext::ReadableDuration; use tokio::sync::Mutex; use wal::{ kv_encoder::LogBatchEncoder, @@ -704,10 +702,11 @@ mod tests { use common_types::{ column_schema, datum::DatumKind, schema, schema::Schema, table::DEFAULT_SHARD_ID, }; - use common_util::{runtime, runtime::Runtime, tests::init_log_for_test}; use futures::future::BoxFuture; use object_store::LocalFileSystem; + use runtime::Runtime; use table_engine::table::{SchemaId, TableId, TableSeqGenerator}; + use test_util::tests::init_log_for_test; use wal::rocks_impl::manager::Builder as WalBuilder; use super::*; diff --git a/analytic_engine/src/manifest/meta_edit.rs b/analytic_engine/src/manifest/meta_edit.rs index 2e5935569c..734de5b3be 100644 --- a/analytic_engine/src/manifest/meta_edit.rs +++ b/analytic_engine/src/manifest/meta_edit.rs @@ -10,7 +10,7 @@ use common_types::{ schema::{Schema, Version}, SequenceNumber, }; -use common_util::define_result; +use macros::define_result; use prost::Message; use snafu::{Backtrace, OptionExt, ResultExt, Snafu}; use table_engine::table::TableId; diff --git a/analytic_engine/src/manifest/meta_snapshot.rs b/analytic_engine/src/manifest/meta_snapshot.rs index 2b42e3d52c..3e7b7578a6 100644 --- a/analytic_engine/src/manifest/meta_snapshot.rs +++ b/analytic_engine/src/manifest/meta_snapshot.rs @@ -2,8 +2,8 @@ //! Meta data of manifest. -use common_util::define_result; use log::debug; +use macros::define_result; use snafu::{ensure, Backtrace, Snafu}; use crate::{ diff --git a/analytic_engine/src/manifest/mod.rs b/analytic_engine/src/manifest/mod.rs index 4fbf1fba07..c8b0daa50e 100644 --- a/analytic_engine/src/manifest/mod.rs +++ b/analytic_engine/src/manifest/mod.rs @@ -10,7 +10,7 @@ use std::{fmt, sync::Arc}; use async_trait::async_trait; use common_types::table::ShardId; -use common_util::error::GenericResult; +use generic_error::GenericResult; use table_engine::table::TableId; use crate::{manifest::meta_edit::MetaEditRequest, space::SpaceId}; diff --git a/analytic_engine/src/memtable/factory.rs b/analytic_engine/src/memtable/factory.rs index 0867bba2da..c38293992d 100644 --- a/analytic_engine/src/memtable/factory.rs +++ b/analytic_engine/src/memtable/factory.rs @@ -1,4 +1,4 @@ -// Copyright 2022 CeresDB Project Authors. Licensed under Apache-2.0. +// Copyright 2022-2023 CeresDB Project Authors. Licensed under Apache-2.0. //! MemTable factory @@ -6,7 +6,7 @@ use std::{fmt, sync::Arc}; use arena::CollectorRef; use common_types::{schema::Schema, SequenceNumber}; -use common_util::define_result; +use macros::define_result; use snafu::Snafu; use crate::memtable::MemTableRef; diff --git a/analytic_engine/src/memtable/key.rs b/analytic_engine/src/memtable/key.rs index b352976359..ee4ea58fd1 100644 --- a/analytic_engine/src/memtable/key.rs +++ b/analytic_engine/src/memtable/key.rs @@ -1,4 +1,4 @@ -// Copyright 2022 CeresDB Project Authors. Licensed under Apache-2.0. +// Copyright 2022-2023 CeresDB Project Authors. Licensed under Apache-2.0. //! Memtable key //! @@ -13,24 +13,20 @@ use std::mem; use bytes::BufMut; +use codec::{memcomparable::MemComparable, Decoder, Encoder}; use common_types::{ bytes::{BytesMut, SafeBuf, SafeBufMut}, row::Row, schema::Schema, SequenceNumber, }; -use common_util::{ - codec::{memcomparable::MemComparable, Decoder, Encoder}, - define_result, -}; +use macros::define_result; use snafu::{ensure, Backtrace, ResultExt, Snafu}; #[derive(Debug, Snafu)] pub enum Error { #[snafu(display("Failed to encode key datum, err:{}", source))] - EncodeKeyDatum { - source: common_util::codec::memcomparable::Error, - }, + EncodeKeyDatum { source: codec::memcomparable::Error }, #[snafu(display("Failed to encode sequence, err:{}", source))] EncodeSequence { source: common_types::bytes::Error }, diff --git a/analytic_engine/src/memtable/mod.rs b/analytic_engine/src/memtable/mod.rs index 7414f7e323..ed3173e704 100644 --- a/analytic_engine/src/memtable/mod.rs +++ b/analytic_engine/src/memtable/mod.rs @@ -16,7 +16,8 @@ use common_types::{ schema::{IndexInWriterSchema, Schema}, SequenceNumber, }; -use common_util::{define_result, error::GenericError}; +use generic_error::GenericError; +use macros::define_result; use snafu::{Backtrace, Snafu}; use trace_metric::MetricsCollector; @@ -34,9 +35,7 @@ pub enum Error { DecodeInternalKey { source: crate::memtable::key::Error }, #[snafu(display("Failed to decode row, err:{}", source))] - DecodeRow { - source: common_util::codec::row::Error, - }, + DecodeRow { source: codec::row::Error }, #[snafu(display("Failed to append row to batch builder, err:{}", source))] AppendRow { diff --git a/analytic_engine/src/memtable/skiplist/iter.rs b/analytic_engine/src/memtable/skiplist/iter.rs index bb3d11d980..6bdb9b6419 100644 --- a/analytic_engine/src/memtable/skiplist/iter.rs +++ b/analytic_engine/src/memtable/skiplist/iter.rs @@ -5,6 +5,7 @@ use std::{cmp::Ordering, iter::Rev, ops::Bound, time::Instant}; use arena::{Arena, BasicStats}; +use codec::row; use common_types::{ bytes::{Bytes, BytesMut}, projected_schema::{ProjectedSchema, RowProjector}, @@ -13,10 +14,11 @@ use common_types::{ schema::Schema, SequenceNumber, }; -use common_util::{codec::row, error::BoxError, time::InstantExt}; +use generic_error::BoxError; use log::trace; use skiplist::{ArenaSlice, IterRef, Skiplist}; use snafu::ResultExt; +use time_ext::InstantExt; use crate::memtable::{ key::{self, KeySequence}, diff --git a/analytic_engine/src/memtable/skiplist/mod.rs b/analytic_engine/src/memtable/skiplist/mod.rs index ec06a98efd..d940bf42ea 100644 --- a/analytic_engine/src/memtable/skiplist/mod.rs +++ b/analytic_engine/src/memtable/skiplist/mod.rs @@ -12,13 +12,14 @@ use std::{ }; use arena::{Arena, BasicStats}; +use codec::Encoder; use common_types::{ bytes::Bytes, row::{contiguous::ContiguousRowWriter, Row}, schema::Schema, SequenceNumber, }; -use common_util::{codec::Encoder, error::BoxError}; +use generic_error::BoxError; use log::{debug, trace}; use skiplist::{KeyComparator, Skiplist}; use snafu::{ensure, ResultExt}; @@ -204,6 +205,7 @@ mod tests { use std::{ops::Bound, sync::Arc}; use arena::NoopCollector; + use codec::memcomparable::MemComparable; use common_types::{ bytes::ByteVec, datum::Datum, @@ -213,7 +215,6 @@ mod tests { tests::{build_row, build_schema}, time::Timestamp, }; - use common_util::codec::memcomparable::MemComparable; use super::*; use crate::memtable::{ diff --git a/analytic_engine/src/payload.rs b/analytic_engine/src/payload.rs index bbf5083e43..e773e15545 100644 --- a/analytic_engine/src/payload.rs +++ b/analytic_engine/src/payload.rs @@ -1,17 +1,15 @@ -// Copyright 2022 CeresDB Project Authors. Licensed under Apache-2.0. +// Copyright 2022-2023 CeresDB Project Authors. Licensed under Apache-2.0. //! Payloads to write to wal use ceresdbproto::{manifest as manifest_pb, table_requests}; +use codec::{row::WalRowDecoder, Decoder}; use common_types::{ bytes::{Buf, BufMut, SafeBuf, SafeBufMut}, row::{RowGroup, RowGroupBuilder}, schema::Schema, }; -use common_util::{ - codec::{row::WalRowDecoder, Decoder}, - define_result, -}; +use macros::define_result; use prost::Message; use snafu::{Backtrace, OptionExt, ResultExt, Snafu}; use wal::log_batch::{Payload, PayloadDecoder}; @@ -49,9 +47,7 @@ pub enum Error { DecodeSchema { source: common_types::schema::Error }, #[snafu(display("Failed to decode row, err:{}", source))] - DecodeRow { - source: common_util::codec::row::Error, - }, + DecodeRow { source: codec::row::Error }, #[snafu(display( "Table schema is not found in the write request.\nBacktrace:\n{}", diff --git a/analytic_engine/src/row_iter/chain.rs b/analytic_engine/src/row_iter/chain.rs index c9f7fbf7db..5c17f84289 100644 --- a/analytic_engine/src/row_iter/chain.rs +++ b/analytic_engine/src/row_iter/chain.rs @@ -10,9 +10,10 @@ use common_types::{ projected_schema::ProjectedSchema, record_batch::RecordBatchWithKey, request_id::RequestId, schema::RecordSchemaWithKey, }; -use common_util::{define_result, error::GenericError}; use futures::StreamExt; +use generic_error::GenericError; use log::debug; +use macros::define_result; use snafu::{ResultExt, Snafu}; use table_engine::{predicate::PredicateRef, table::TableId}; use trace_metric::{MetricsCollector, TraceMetricWhenDrop}; diff --git a/analytic_engine/src/row_iter/dedup.rs b/analytic_engine/src/row_iter/dedup.rs index 241efca465..3837ce9485 100644 --- a/analytic_engine/src/row_iter/dedup.rs +++ b/analytic_engine/src/row_iter/dedup.rs @@ -1,4 +1,4 @@ -// Copyright 2022 CeresDB Project Authors. Licensed under Apache-2.0. +// Copyright 2022-2023 CeresDB Project Authors. Licensed under Apache-2.0. use std::cmp::Ordering; @@ -9,11 +9,9 @@ use common_types::{ row::{Row, RowViewOnBatch, RowWithMeta}, schema::RecordSchemaWithKey, }; -use common_util::{ - define_result, - error::{BoxError, GenericError}, -}; +use generic_error::{BoxError, GenericError}; use log::{info, trace}; +use macros::define_result; use snafu::{ResultExt, Snafu}; use crate::row_iter::{IterOptions, RecordBatchWithKeyIterator}; diff --git a/analytic_engine/src/row_iter/merge.rs b/analytic_engine/src/row_iter/merge.rs index b00dfc6e41..13a303638e 100644 --- a/analytic_engine/src/row_iter/merge.rs +++ b/analytic_engine/src/row_iter/merge.rs @@ -1,4 +1,4 @@ -// Copyright 2022 CeresDB Project Authors. Licensed under Apache-2.0. +// Copyright 2022-2023 CeresDB Project Authors. Licensed under Apache-2.0. use std::{ cmp, @@ -18,9 +18,10 @@ use common_types::{ schema::RecordSchemaWithKey, SequenceNumber, }; -use common_util::{define_result, error::GenericError}; use futures::{stream::FuturesUnordered, StreamExt}; +use generic_error::GenericError; use log::{debug, trace}; +use macros::define_result; use snafu::{ensure, Backtrace, ResultExt, Snafu}; use table_engine::{predicate::PredicateRef, table::TableId}; use trace_metric::{MetricsCollector, TraceMetricWhenDrop}; diff --git a/analytic_engine/src/row_iter/mod.rs b/analytic_engine/src/row_iter/mod.rs index 843c145912..dabd2fd2ae 100644 --- a/analytic_engine/src/row_iter/mod.rs +++ b/analytic_engine/src/row_iter/mod.rs @@ -1,11 +1,11 @@ -// Copyright 2022 CeresDB Project Authors. Licensed under Apache-2.0. +// Copyright 2022-2023 CeresDB Project Authors. Licensed under Apache-2.0. //! Iterators for row. use async_stream::try_stream; use async_trait::async_trait; use common_types::{record_batch::RecordBatchWithKey, schema::RecordSchemaWithKey}; -use common_util::error::BoxError; +use generic_error::BoxError; use crate::sst::writer::RecordBatchStream; diff --git a/analytic_engine/src/row_iter/record_batch_stream.rs b/analytic_engine/src/row_iter/record_batch_stream.rs index ec8f0b9a48..ca733239a7 100644 --- a/analytic_engine/src/row_iter/record_batch_stream.rs +++ b/analytic_engine/src/row_iter/record_batch_stream.rs @@ -1,4 +1,4 @@ -// Copyright 2022 CeresDB Project Authors. Licensed under Apache-2.0. +// Copyright 2022-2023 CeresDB Project Authors. Licensed under Apache-2.0. use std::{ ops::{Bound, Not}, @@ -13,10 +13,6 @@ use arrow::{ use common_types::{ projected_schema::ProjectedSchema, record_batch::RecordBatchWithKey, SequenceNumber, }; -use common_util::{ - define_result, - error::{BoxError, GenericError}, -}; use datafusion::{ common::ToDFSchema, error::DataFusionError, @@ -25,6 +21,8 @@ use datafusion::{ physical_plan::PhysicalExpr, }; use futures::stream::{self, Stream, StreamExt}; +use generic_error::{BoxError, GenericError}; +use macros::define_result; use snafu::{Backtrace, OptionExt, ResultExt, Snafu}; use table_engine::{predicate::Predicate, table::TableId}; use trace_metric::MetricsCollector; diff --git a/analytic_engine/src/row_iter/tests.rs b/analytic_engine/src/row_iter/tests.rs index 7af07ff778..be533ccfc0 100644 --- a/analytic_engine/src/row_iter/tests.rs +++ b/analytic_engine/src/row_iter/tests.rs @@ -10,7 +10,7 @@ use common_types::{ }, schema::{IndexInWriterSchema, RecordSchemaWithKey, Schema}, }; -use common_util::define_result; +use macros::define_result; use snafu::Snafu; use crate::row_iter::RecordBatchWithKeyIterator; diff --git a/analytic_engine/src/sampler.rs b/analytic_engine/src/sampler.rs index 9a72011eec..e6463816bb 100644 --- a/analytic_engine/src/sampler.rs +++ b/analytic_engine/src/sampler.rs @@ -1,4 +1,4 @@ -// Copyright 2022 CeresDB Project Authors. Licensed under Apache-2.0. +// Copyright 2022-2023 CeresDB Project Authors. Licensed under Apache-2.0. //! Segment duration sampler. @@ -9,7 +9,7 @@ use std::{ }; use common_types::time::{TimeRange, Timestamp}; -use common_util::define_result; +use macros::define_result; use snafu::{ensure, Backtrace, Snafu}; use crate::table_options; diff --git a/analytic_engine/src/setup.rs b/analytic_engine/src/setup.rs index 940ce16579..ab2c7563c5 100644 --- a/analytic_engine/src/setup.rs +++ b/analytic_engine/src/setup.rs @@ -5,8 +5,8 @@ use std::{num::NonZeroUsize, path::Path, pin::Pin, sync::Arc}; use async_trait::async_trait; -use common_util::define_result; use futures::Future; +use macros::define_result; use message_queue::kafka::kafka_impl::KafkaImpl; use object_store::{ aliyun, @@ -68,7 +68,7 @@ pub enum Error { OpenObkv { source: table_kv::obkv::Error }, #[snafu(display("Failed to execute in runtime, err:{}", source))] - RuntimeExec { source: common_util::runtime::Error }, + RuntimeExec { source: runtime::Error }, #[snafu(display("Failed to open object store, err:{}", source))] OpenObjectStore { diff --git a/analytic_engine/src/sst/factory.rs b/analytic_engine/src/sst/factory.rs index 4100073133..9089d28521 100644 --- a/analytic_engine/src/sst/factory.rs +++ b/analytic_engine/src/sst/factory.rs @@ -1,4 +1,4 @@ -// Copyright 2022 CeresDB Project Authors. Licensed under Apache-2.0. +// Copyright 2022-2023 CeresDB Project Authors. Licensed under Apache-2.0. //! Factory for different kinds sst writer and reader. @@ -6,8 +6,9 @@ use std::{fmt::Debug, sync::Arc}; use async_trait::async_trait; use common_types::projected_schema::ProjectedSchema; -use common_util::{define_result, runtime::Runtime}; +use macros::define_result; use object_store::{ObjectStoreRef, Path}; +use runtime::Runtime; use snafu::{ResultExt, Snafu}; use table_engine::predicate::PredicateRef; use trace_metric::MetricsCollector; diff --git a/analytic_engine/src/sst/file.rs b/analytic_engine/src/sst/file.rs index 1b7d5a160c..ff8827a40b 100644 --- a/analytic_engine/src/sst/file.rs +++ b/analytic_engine/src/sst/file.rs @@ -18,13 +18,11 @@ use common_types::{ time::{TimeRange, Timestamp}, SequenceNumber, }; -use common_util::{ - define_result, - metric::Meter, - runtime::{JoinHandle, Runtime}, -}; use log::{error, info, warn}; +use macros::define_result; +use metric_ext::Meter; use object_store::ObjectStoreRef; +use runtime::{JoinHandle, Runtime}; use snafu::{ResultExt, Snafu}; use table_engine::table::TableId; use tokio::sync::{ @@ -38,7 +36,7 @@ use crate::{space::SpaceId, sst::manager::FileId, table::sst_util, table_options #[derive(Debug, Snafu)] pub enum Error { #[snafu(display("Failed to join purger, err:{}", source))] - StopPurger { source: common_util::runtime::Error }, + StopPurger { source: runtime::Error }, } define_result!(Error); diff --git a/analytic_engine/src/sst/header.rs b/analytic_engine/src/sst/header.rs index 56ff61b1ce..1bc1aea3d5 100644 --- a/analytic_engine/src/sst/header.rs +++ b/analytic_engine/src/sst/header.rs @@ -1,9 +1,9 @@ -// Copyright 2022 CeresDB Project Authors. Licensed under Apache-2.0. +// Copyright 2022-2023 CeresDB Project Authors. Licensed under Apache-2.0. // The header parser for one sst. use bytes::Bytes; -use common_util::define_result; +use macros::define_result; use object_store::{ObjectStoreRef, Path}; use parquet::data_type::AsBytes; use snafu::{Backtrace, ResultExt, Snafu}; diff --git a/analytic_engine/src/sst/meta_data/mod.rs b/analytic_engine/src/sst/meta_data/mod.rs index c1aade735f..5462b87949 100644 --- a/analytic_engine/src/sst/meta_data/mod.rs +++ b/analytic_engine/src/sst/meta_data/mod.rs @@ -1,4 +1,4 @@ -// Copyright 2022 CeresDB Project Authors. Licensed under Apache-2.0. +// Copyright 2022-2023 CeresDB Project Authors. Licensed under Apache-2.0. pub mod cache; @@ -6,7 +6,7 @@ use std::sync::Arc; use ceresdbproto::sst as sst_pb; use common_types::{schema::Schema, time::TimeRange, SequenceNumber}; -use common_util::define_result; +use macros::define_result; use snafu::{Backtrace, OptionExt, ResultExt, Snafu}; use table_engine::table::TableId; diff --git a/analytic_engine/src/sst/parquet/async_reader.rs b/analytic_engine/src/sst/parquet/async_reader.rs index 3092ccd3b2..2a05f70241 100644 --- a/analytic_engine/src/sst/parquet/async_reader.rs +++ b/analytic_engine/src/sst/parquet/async_reader.rs @@ -17,11 +17,6 @@ use common_types::{ projected_schema::{ProjectedSchema, RowProjector}, record_batch::{ArrowRecordBatchProjector, RecordBatchWithKey}, }; -use common_util::{ - error::{BoxError, GenericResult}, - runtime::{AbortOnDropMany, JoinHandle, Runtime}, - time::InstantExt, -}; use datafusion::{ common::ToDFSchema, physical_expr::{create_physical_expr, execution_props::ExecutionProps}, @@ -31,6 +26,7 @@ use datafusion::{ }, }; use futures::{Stream, StreamExt}; +use generic_error::{BoxError, GenericResult}; use log::{debug, error}; use object_store::{ObjectStoreRef, Path}; use parquet::{ @@ -38,8 +34,10 @@ use parquet::{ file::metadata::RowGroupMetaData, }; use parquet_ext::{meta_data::ChunkReader, reader::ObjectStoreReader}; +use runtime::{AbortOnDropMany, JoinHandle, Runtime}; use snafu::ResultExt; use table_engine::predicate::PredicateRef; +use time_ext::InstantExt; use tokio::sync::mpsc::{self, Receiver, Sender}; use trace_metric::{MetricsCollector, TraceMetricWhenDrop}; diff --git a/analytic_engine/src/sst/parquet/encoding.rs b/analytic_engine/src/sst/parquet/encoding.rs index 47d3b8c2dd..bfc1087f4d 100644 --- a/analytic_engine/src/sst/parquet/encoding.rs +++ b/analytic_engine/src/sst/parquet/encoding.rs @@ -16,11 +16,9 @@ use common_types::{ datum::DatumKind, schema::{ArrowSchema, ArrowSchemaRef, DataType, Field, Schema}, }; -use common_util::{ - define_result, - error::{BoxError, GenericError}, -}; +use generic_error::{BoxError, GenericError}; use log::trace; +use macros::define_result; use parquet::{ arrow::AsyncArrowWriter, basic::Compression, diff --git a/analytic_engine/src/sst/parquet/hybrid.rs b/analytic_engine/src/sst/parquet/hybrid.rs index 3d155bc01b..95bdf03008 100644 --- a/analytic_engine/src/sst/parquet/hybrid.rs +++ b/analytic_engine/src/sst/parquet/hybrid.rs @@ -1,4 +1,4 @@ -// Copyright 2022 CeresDB Project Authors. Licensed under Apache-2.0. +// Copyright 2022-2023 CeresDB Project Authors. Licensed under Apache-2.0. use std::{collections::BTreeMap, sync::Arc}; @@ -16,7 +16,7 @@ use common_types::{ datum::DatumKind, schema::{ArrowSchemaRef, DataType, Field, Schema}, }; -use common_util::error::BoxError; +use generic_error::BoxError; use snafu::{Backtrace, ResultExt, Snafu}; use crate::sst::writer::{EncodeRecordBatch, Result}; diff --git a/analytic_engine/src/sst/parquet/meta_data.rs b/analytic_engine/src/sst/parquet/meta_data.rs index 4a6778b7f3..c1bf746a8d 100644 --- a/analytic_engine/src/sst/parquet/meta_data.rs +++ b/analytic_engine/src/sst/parquet/meta_data.rs @@ -1,4 +1,4 @@ -// Copyright 2022 CeresDB Project Authors. Licensed under Apache-2.0. +// Copyright 2022-2023 CeresDB Project Authors. Licensed under Apache-2.0. // MetaData for SST based on parquet. @@ -12,7 +12,7 @@ use common_types::{ time::TimeRange, SequenceNumber, }; -use common_util::define_result; +use macros::define_result; use snafu::{Backtrace, OptionExt, ResultExt, Snafu}; use xorfilter::xor8::{Xor8, Xor8Builder}; @@ -369,11 +369,9 @@ impl From for MetaData { impl fmt::Debug for ParquetMetaData { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - use common_util::byte::encode; - f.debug_struct("ParquetMetaData") - .field("min_key", &encode(&self.min_key)) - .field("max_key", &encode(&self.max_key)) + .field("min_key", &hex::encode(&self.min_key)) + .field("max_key", &hex::encode(&self.max_key)) .field("time_range", &self.time_range) .field("max_sequence", &self.max_sequence) .field("schema", &self.schema) diff --git a/analytic_engine/src/sst/parquet/writer.rs b/analytic_engine/src/sst/parquet/writer.rs index 0e1b5f3658..fe4b088a71 100644 --- a/analytic_engine/src/sst/parquet/writer.rs +++ b/analytic_engine/src/sst/parquet/writer.rs @@ -1,12 +1,12 @@ -// Copyright 2022 CeresDB Project Authors. Licensed under Apache-2.0. +// Copyright 2022-2023 CeresDB Project Authors. Licensed under Apache-2.0. //! Sst writer implementation based on parquet. use async_trait::async_trait; use common_types::{record_batch::RecordBatchWithKey, request_id::RequestId}; -use common_util::error::BoxError; use datafusion::parquet::basic::Compression; use futures::StreamExt; +use generic_error::BoxError; use log::{debug, error}; use object_store::{ObjectStoreRef, Path}; use snafu::ResultExt; @@ -336,14 +336,12 @@ mod tests { tests::{build_row, build_row_for_dictionary, build_schema, build_schema_with_dictionary}, time::{TimeRange, Timestamp}, }; - use common_util::{ - runtime::{self, Runtime}, - tests::init_log_for_test, - }; use futures::stream; use object_store::LocalFileSystem; + use runtime::{self, Runtime}; use table_engine::predicate::Predicate; use tempfile::tempdir; + use test_util::tests::init_log_for_test; use super::*; use crate::{ diff --git a/analytic_engine/src/sst/reader.rs b/analytic_engine/src/sst/reader.rs index 029b0aa34a..7bf392723b 100644 --- a/analytic_engine/src/sst/reader.rs +++ b/analytic_engine/src/sst/reader.rs @@ -9,7 +9,8 @@ use futures::Stream; use crate::sst::meta_data::SstMetaData; pub mod error { - use common_util::{define_result, error::GenericError}; + use generic_error::GenericError; + use macros::define_result; use snafu::{Backtrace, Snafu}; #[derive(Debug, Snafu)] diff --git a/analytic_engine/src/sst/writer.rs b/analytic_engine/src/sst/writer.rs index 5f436bea5e..f4483c07ef 100644 --- a/analytic_engine/src/sst/writer.rs +++ b/analytic_engine/src/sst/writer.rs @@ -1,4 +1,4 @@ -// Copyright 2022 CeresDB Project Authors. Licensed under Apache-2.0. +// Copyright 2022-2023 CeresDB Project Authors. Licensed under Apache-2.0. //! Sst writer trait definition @@ -10,13 +10,14 @@ use common_types::{ record_batch::RecordBatchWithKey, request_id::RequestId, schema::Schema, time::TimeRange, SequenceNumber, }; -use common_util::error::GenericError; use futures::Stream; +use generic_error::GenericError; use crate::table_options::StorageFormat; pub mod error { - use common_util::{define_result, error::GenericError}; + use generic_error::GenericError; + use macros::define_result; use snafu::{Backtrace, Snafu}; #[derive(Debug, Snafu)] diff --git a/analytic_engine/src/table/data.rs b/analytic_engine/src/table/data.rs index 1257e6c1b8..f705a34943 100644 --- a/analytic_engine/src/table/data.rs +++ b/analytic_engine/src/table/data.rs @@ -24,12 +24,10 @@ use common_types::{ time::{TimeRange, Timestamp}, SequenceNumber, }; -use common_util::{ - define_result, - error::{GenericError, GenericResult}, - id_allocator::IdAllocator, -}; +use generic_error::{GenericError, GenericResult}; +use id_allocator::IdAllocator; use log::{debug, info}; +use macros::define_result; use object_store::Path; use snafu::{Backtrace, OptionExt, ResultExt, Snafu}; use table_engine::table::TableId; @@ -677,11 +675,11 @@ pub mod tests { use arena::NoopCollector; use common_types::{datum::DatumKind, table::DEFAULT_SHARD_ID}; - use common_util::config::ReadableDuration; use table_engine::{ engine::{CreateTableRequest, TableState}, table::SchemaId, }; + use time_ext::ReadableDuration; use super::*; use crate::{ diff --git a/analytic_engine/src/table/mod.rs b/analytic_engine/src/table/mod.rs index f1df20a4b8..7678692898 100644 --- a/analytic_engine/src/table/mod.rs +++ b/analytic_engine/src/table/mod.rs @@ -14,9 +14,10 @@ use common_types::{ schema::Schema, time::TimeRange, }; -use common_util::{error::BoxError, future_cancel::CancellationSafeFuture}; use datafusion::{common::Column, logical_expr::Expr}; +use future_cancel::CancellationSafeFuture; use futures::TryStreamExt; +use generic_error::BoxError; use log::{error, warn}; use snafu::{ensure, OptionExt, ResultExt}; use table_engine::{ diff --git a/analytic_engine/src/table/version.rs b/analytic_engine/src/table/version.rs index 604be2c08d..373c5ec813 100644 --- a/analytic_engine/src/table/version.rs +++ b/analytic_engine/src/table/version.rs @@ -17,7 +17,7 @@ use common_types::{ time::{TimeRange, Timestamp}, SequenceNumber, }; -use common_util::define_result; +use macros::define_result; use snafu::{ensure, Backtrace, ResultExt, Snafu}; use crate::{ diff --git a/analytic_engine/src/table/version_edit.rs b/analytic_engine/src/table/version_edit.rs index 27ba78e9cb..60777c4488 100644 --- a/analytic_engine/src/table/version_edit.rs +++ b/analytic_engine/src/table/version_edit.rs @@ -1,4 +1,4 @@ -// Copyright 2022 CeresDB Project Authors. Licensed under Apache-2.0. +// Copyright 2022-2023 CeresDB Project Authors. Licensed under Apache-2.0. //! Version edits @@ -6,7 +6,7 @@ use std::convert::TryFrom; use ceresdbproto::manifest as manifest_pb; use common_types::{time::TimeRange, SequenceNumber}; -use common_util::define_result; +use macros::define_result; use snafu::{Backtrace, OptionExt, ResultExt, Snafu}; use crate::{ diff --git a/analytic_engine/src/table_meta_set_impl.rs b/analytic_engine/src/table_meta_set_impl.rs index 38952b4e74..dbfebc20d3 100644 --- a/analytic_engine/src/table_meta_set_impl.rs +++ b/analytic_engine/src/table_meta_set_impl.rs @@ -4,7 +4,8 @@ use std::{fmt, num::NonZeroUsize, sync::Arc}; -use common_util::{error::BoxError, id_allocator::IdAllocator}; +use generic_error::BoxError; +use id_allocator::IdAllocator; use log::debug; use snafu::{OptionExt, ResultExt}; use table_engine::table::TableId; diff --git a/analytic_engine/src/table_options.rs b/analytic_engine/src/table_options.rs index a1399b91d2..279f559acb 100644 --- a/analytic_engine/src/table_options.rs +++ b/analytic_engine/src/table_options.rs @@ -1,4 +1,4 @@ -// Copyright 2022 CeresDB Project Authors. Licensed under Apache-2.0. +// Copyright 2022-2023 CeresDB Project Authors. Licensed under Apache-2.0. //! Constants for table options. @@ -10,14 +10,12 @@ use common_types::{ NUM_ROWS_PER_ROW_GROUP, OPTION_KEY_ENABLE_TTL, SEGMENT_DURATION, STORAGE_FORMAT, TTL, UPDATE_MODE, WRITE_BUFFER_SIZE, }; -use common_util::{ - config::{ReadableDuration, ReadableSize, TimeUnit}, - define_result, - time::{parse_duration, DurationExt}, -}; use datafusion::parquet::basic::Compression as ParquetCompression; +use macros::define_result; use serde::{Deserialize, Serialize}; +use size_ext::ReadableSize; use snafu::{Backtrace, GenerateBacktrace, OptionExt, ResultExt, Snafu}; +use time_ext::{parse_duration, DurationExt, ReadableDuration, TimeUnit}; use crate::compaction::{ self, CompactionStrategy, SizeTieredCompactionOptions, TimeWindowCompactionOptions, @@ -58,7 +56,7 @@ const MAX_NUM_ROWS_PER_ROW_GROUP: usize = 10_000_000; pub enum Error { #[snafu(display("Failed to parse duration, err:{}.\nBacktrace:\n{}", source, backtrace))] ParseDuration { - source: common_util::time::Error, + source: time_ext::Error, backtrace: Backtrace, }, diff --git a/analytic_engine/src/tests/compaction_test.rs b/analytic_engine/src/tests/compaction_test.rs index 72a519c4ac..2a2d367286 100644 --- a/analytic_engine/src/tests/compaction_test.rs +++ b/analytic_engine/src/tests/compaction_test.rs @@ -85,7 +85,7 @@ fn test_table_compact_current_segment(engine_context: T) ) .await; - common_util::tests::init_log_for_test(); + test_util::tests::init_log_for_test(); // Trigger a compaction. test_ctx.compact_table(compact_test_table1).await; diff --git a/analytic_engine/src/tests/table.rs b/analytic_engine/src/tests/table.rs index 6a1201cbbb..fa3968ebba 100644 --- a/analytic_engine/src/tests/table.rs +++ b/analytic_engine/src/tests/table.rs @@ -15,13 +15,13 @@ use common_types::{ table::DEFAULT_SHARD_ID, time::Timestamp, }; -use common_util::config::ReadableDuration; use table_engine::{ self, engine::{CreateTableRequest, TableState}, predicate::Predicate, table::{GetRequest, ReadOptions, ReadOrder, ReadRequest, SchemaId, TableId, TableSeq}, }; +use time_ext::ReadableDuration; use trace_metric::MetricsCollector; use crate::{table_options, tests::row_util}; diff --git a/analytic_engine/src/tests/util.rs b/analytic_engine/src/tests/util.rs index 0cc8fb94e3..a74b354dba 100644 --- a/analytic_engine/src/tests/util.rs +++ b/analytic_engine/src/tests/util.rs @@ -11,13 +11,10 @@ use common_types::{ table::{ShardId, DEFAULT_SHARD_ID}, time::Timestamp, }; -use common_util::{ - config::{ReadableDuration, ReadableSize}, - runtime, -}; use futures::stream::StreamExt; use log::info; use object_store::config::{LocalOptions, ObjectStoreOptions, StorageOptions}; +use size_ext::ReadableSize; use table_engine::{ engine::{ CreateTableRequest, DropTableRequest, EngineRuntimes, OpenShardRequest, OpenTableRequest, @@ -29,6 +26,7 @@ use table_engine::{ }, }; use tempfile::TempDir; +use time_ext::ReadableDuration; use crate::{ setup::{EngineBuilder, MemWalsOpener, OpenedWals, RocksDBWalsOpener, WalsOpener}, diff --git a/benchmarks/Cargo.toml b/benchmarks/Cargo.toml index cfec6d7976..ff85ba24bf 100644 --- a/benchmarks/Cargo.toml +++ b/benchmarks/Cargo.toml @@ -17,20 +17,24 @@ arrow = { workspace = true } base64 = { workspace = true } clap = { workspace = true } common_types = { workspace = true } -common_util = { workspace = true } env_logger = { workspace = true } futures = { workspace = true } log = { workspace = true } +macros = { workspace = true } object_store = { workspace = true } parquet = { workspace = true } parquet_ext = { workspace = true } pprof = { version = "0.10", features = ["flamegraph", "criterion"] } rand = { workspace = true } +runtime = { workspace = true } serde = { workspace = true } +size_ext = { workspace = true } snafu = { workspace = true } table_engine = { workspace = true } table_kv = { workspace = true } +time_ext = { workspace = true } tokio = { workspace = true } +toml_ext = { workspace = true } trace_metric = { workspace = true } wal = { workspace = true } zstd = { workspace = true } diff --git a/benchmarks/src/bin/sst-tools.rs b/benchmarks/src/bin/sst-tools.rs index 1665dec4c5..10f7b7c7d5 100644 --- a/benchmarks/src/bin/sst-tools.rs +++ b/benchmarks/src/bin/sst-tools.rs @@ -1,4 +1,4 @@ -// Copyright 2022 CeresDB Project Authors. Licensed under Apache-2.0. +// Copyright 2022-2023 CeresDB Project Authors. Licensed under Apache-2.0. use std::sync::Arc; @@ -7,7 +7,6 @@ use benchmarks::{ util, }; use clap::{App, Arg}; -use common_util::toml; use log::info; use serde::Deserialize; @@ -31,7 +30,7 @@ impl Default for Config { fn config_from_path(path: &str) -> Config { let mut toml_buf = String::new(); - toml::parse_toml_from_path(path, &mut toml_buf).expect("Failed to parse config.") + toml_ext::parse_toml_from_path(path, &mut toml_buf).expect("Failed to parse config.") } fn main() { diff --git a/benchmarks/src/config.rs b/benchmarks/src/config.rs index 5d65a3b1cd..e0f83f419a 100644 --- a/benchmarks/src/config.rs +++ b/benchmarks/src/config.rs @@ -1,4 +1,4 @@ -// Copyright 2022 CeresDB Project Authors. Licensed under Apache-2.0. +// Copyright 2022-2023 CeresDB Project Authors. Licensed under Apache-2.0. //! Benchmark configs. @@ -6,15 +6,13 @@ use std::env; use analytic_engine::{space::SpaceId, sst::manager::FileId}; use common_types::time::{TimeRange, Timestamp}; -use common_util::{ - config::{ReadableDuration, ReadableSize}, - toml, -}; use serde::Deserialize; +use size_ext::ReadableSize; use table_engine::{ predicate::{PredicateBuilder, PredicateRef}, table::TableId, }; +use time_ext::ReadableDuration; const BENCH_CONFIG_PATH_KEY: &str = "ANALYTIC_BENCH_CONFIG_PATH"; @@ -35,7 +33,7 @@ pub fn bench_config_from_env() -> BenchConfig { }; let mut toml_buf = String::new(); - toml::parse_toml_from_path(&path, &mut toml_buf).expect("Failed to parse config.") + toml_ext::parse_toml_from_path(&path, &mut toml_buf).expect("Failed to parse config.") } #[derive(Deserialize)] diff --git a/benchmarks/src/merge_memtable_bench.rs b/benchmarks/src/merge_memtable_bench.rs index 4974877d1e..d7da24356b 100644 --- a/benchmarks/src/merge_memtable_bench.rs +++ b/benchmarks/src/merge_memtable_bench.rs @@ -1,4 +1,4 @@ -// Copyright 2022 CeresDB Project Authors. Licensed under Apache-2.0. +// Copyright 2022-2023 CeresDB Project Authors. Licensed under Apache-2.0. //! Merge memtable bench. @@ -31,9 +31,9 @@ use arena::NoopCollector; use common_types::{ projected_schema::ProjectedSchema, request_id::RequestId, schema::Schema, time::TimeRange, }; -use common_util::runtime::Runtime; use log::info; use object_store::{LocalFileSystem, ObjectStoreRef}; +use runtime::Runtime; use table_engine::{predicate::Predicate, table::TableId}; use crate::{config::MergeMemTableBenchConfig, util}; diff --git a/benchmarks/src/merge_sst_bench.rs b/benchmarks/src/merge_sst_bench.rs index c744ac3138..916d131154 100644 --- a/benchmarks/src/merge_sst_bench.rs +++ b/benchmarks/src/merge_sst_bench.rs @@ -24,9 +24,9 @@ use analytic_engine::{ table::sst_util, }; use common_types::{projected_schema::ProjectedSchema, request_id::RequestId, schema::Schema}; -use common_util::runtime::Runtime; use log::info; use object_store::{LocalFileSystem, ObjectStoreRef}; +use runtime::Runtime; use table_engine::{predicate::Predicate, table::TableId}; use tokio::sync::mpsc::{self, UnboundedReceiver}; diff --git a/benchmarks/src/parquet_bench.rs b/benchmarks/src/parquet_bench.rs index a39b659838..17b6bb8625 100644 --- a/benchmarks/src/parquet_bench.rs +++ b/benchmarks/src/parquet_bench.rs @@ -1,4 +1,4 @@ -// Copyright 2022 CeresDB Project Authors. Licensed under Apache-2.0. +// Copyright 2022-2023 CeresDB Project Authors. Licensed under Apache-2.0. //! Parquet bench. @@ -6,13 +6,13 @@ use std::{io::Cursor, sync::Arc, time::Instant}; use analytic_engine::sst::meta_data::cache::MetaCacheRef; use common_types::schema::Schema; -use common_util::runtime::Runtime; use futures::StreamExt; use log::info; use object_store::{LocalFileSystem, ObjectStoreRef, Path}; use parquet::arrow::{ arrow_reader::ParquetRecordBatchReaderBuilder, ParquetRecordBatchStreamBuilder, }; +use runtime::Runtime; use table_engine::predicate::PredicateRef; use crate::{config::SstBenchConfig, util}; diff --git a/benchmarks/src/sst_bench.rs b/benchmarks/src/sst_bench.rs index c57af0721b..8666df816e 100644 --- a/benchmarks/src/sst_bench.rs +++ b/benchmarks/src/sst_bench.rs @@ -1,4 +1,4 @@ -// Copyright 2022 CeresDB Project Authors. Licensed under Apache-2.0. +// Copyright 2022-2023 CeresDB Project Authors. Licensed under Apache-2.0. //! SST bench. @@ -12,10 +12,10 @@ use analytic_engine::sst::{ meta_data::cache::{MetaCache, MetaCacheRef}, }; use common_types::{projected_schema::ProjectedSchema, schema::Schema}; -use common_util::runtime::Runtime; use futures::stream::StreamExt; use log::info; use object_store::{LocalFileSystem, ObjectStoreRef, Path}; +use runtime::Runtime; use crate::{config::SstBenchConfig, util}; diff --git a/benchmarks/src/sst_tools.rs b/benchmarks/src/sst_tools.rs index 8f3a6c1f09..4b4a777f44 100644 --- a/benchmarks/src/sst_tools.rs +++ b/benchmarks/src/sst_tools.rs @@ -1,4 +1,4 @@ -// Copyright 2022 CeresDB Project Authors. Licensed under Apache-2.0. +// Copyright 2022-2023 CeresDB Project Authors. Licensed under Apache-2.0. //! Tools to generate SST. @@ -26,10 +26,10 @@ use analytic_engine::{ table_options::{Compression, StorageFormatHint}, }; use common_types::{projected_schema::ProjectedSchema, request_id::RequestId}; -use common_util::runtime::Runtime; use futures::TryStreamExt; use log::info; use object_store::{LocalFileSystem, ObjectStoreRef, Path}; +use runtime::Runtime; use serde::Deserialize; use table_engine::{predicate::Predicate, table::TableId}; use tokio::sync::mpsc; diff --git a/benchmarks/src/util.rs b/benchmarks/src/util.rs index 9d3d44c50c..6520b6e7e0 100644 --- a/benchmarks/src/util.rs +++ b/benchmarks/src/util.rs @@ -1,4 +1,4 @@ -// Copyright 2022 CeresDB Project Authors. Licensed under Apache-2.0. +// Copyright 2022-2023 CeresDB Project Authors. Licensed under Apache-2.0. //! Utilities. @@ -26,13 +26,11 @@ use common_types::{ projected_schema::ProjectedSchema, schema::{IndexInWriterSchema, Schema}, }; -use common_util::{ - define_result, - runtime::{self, Runtime}, -}; use futures::stream::StreamExt; +use macros::define_result; use object_store::{ObjectStoreRef, Path}; use parquet::file::footer; +use runtime::Runtime; use snafu::{ResultExt, Snafu}; use table_engine::{predicate::Predicate, table::TableId}; use wal::log_batch::Payload; diff --git a/benchmarks/src/wal_write_bench.rs b/benchmarks/src/wal_write_bench.rs index 42d4e36e1e..6ca7c7af5f 100644 --- a/benchmarks/src/wal_write_bench.rs +++ b/benchmarks/src/wal_write_bench.rs @@ -4,8 +4,8 @@ use std::sync::Arc; -use common_util::runtime::Runtime; use rand::prelude::*; +use runtime::Runtime; use table_kv::memory::MemoryImpl; use wal::{ kv_encoder::LogBatchEncoder, diff --git a/catalog/Cargo.toml b/catalog/Cargo.toml index 3316502a15..94a5c817c9 100644 --- a/catalog/Cargo.toml +++ b/catalog/Cargo.toml @@ -13,7 +13,9 @@ workspace = true [dependencies] async-trait = { workspace = true } common_types = { workspace = true } -common_util = { workspace = true } +generic_error = { workspace = true } log = { workspace = true } +macros = { workspace = true } snafu = { workspace = true } table_engine = { workspace = true } +time_ext = { workspace = true } diff --git a/catalog/src/lib.rs b/catalog/src/lib.rs index 33519c94a8..095c9b1a94 100644 --- a/catalog/src/lib.rs +++ b/catalog/src/lib.rs @@ -2,9 +2,6 @@ //! Common traits and types about catalog (schema) -#[macro_use] -extern crate common_util; - pub mod consts; pub mod manager; pub mod schema; @@ -13,7 +10,8 @@ pub mod table_operator; use std::sync::Arc; use async_trait::async_trait; -use common_util::error::GenericError; +use generic_error::GenericError; +use macros::define_result; use snafu::{Backtrace, Snafu}; use crate::schema::{NameRef, SchemaRef}; diff --git a/catalog/src/manager.rs b/catalog/src/manager.rs index 21a8b83816..5d209c2c66 100644 --- a/catalog/src/manager.rs +++ b/catalog/src/manager.rs @@ -1,9 +1,10 @@ -// Copyright 2022 CeresDB Project Authors. Licensed under Apache-2.0. +// Copyright 2022-2023 CeresDB Project Authors. Licensed under Apache-2.0. //! Catalog manager use std::sync::Arc; +use macros::define_result; use snafu::Snafu; use crate::{schema::NameRef, CatalogRef}; diff --git a/catalog/src/schema.rs b/catalog/src/schema.rs index f604f17fec..127622b702 100644 --- a/catalog/src/schema.rs +++ b/catalog/src/schema.rs @@ -6,7 +6,8 @@ use std::{collections::HashMap, sync::Arc}; use async_trait::async_trait; use common_types::{column_schema::ColumnSchema, table::ShardId}; -use common_util::error::GenericError; +use generic_error::GenericError; +use macros::define_result; use snafu::{Backtrace, Snafu}; use table_engine::{ engine::{self, TableEngineRef, TableState}, diff --git a/catalog/src/table_operator.rs b/catalog/src/table_operator.rs index c31e0b1f68..d162168224 100644 --- a/catalog/src/table_operator.rs +++ b/catalog/src/table_operator.rs @@ -2,10 +2,11 @@ use std::time::Instant; -use common_util::{error::BoxError, time::InstantExt}; +use generic_error::BoxError; use log::{error, info, warn}; use snafu::{OptionExt, ResultExt}; use table_engine::{engine, table::TableRef}; +use time_ext::InstantExt; use crate::{ manager::ManagerRef, diff --git a/catalog_impls/Cargo.toml b/catalog_impls/Cargo.toml index 156d696a29..e1ebcb4f3c 100644 --- a/catalog_impls/Cargo.toml +++ b/catalog_impls/Cargo.toml @@ -15,8 +15,9 @@ async-trait = { workspace = true } catalog = { workspace = true } cluster = { workspace = true } common_types = { workspace = true } -common_util = { workspace = true } +generic_error = { workspace = true } log = { workspace = true } +macros = { workspace = true } meta_client = { workspace = true } snafu = { workspace = true } system_catalog = { workspace = true } diff --git a/catalog_impls/src/table_based.rs b/catalog_impls/src/table_based.rs index 6254a2df02..394322e07b 100644 --- a/catalog_impls/src/table_based.rs +++ b/catalog_impls/src/table_based.rs @@ -19,8 +19,9 @@ use catalog::{ }, Catalog, CatalogRef, }; -use common_util::{define_result, error::BoxError}; +use generic_error::BoxError; use log::{debug, info}; +use macros::define_result; use snafu::{ensure, Backtrace, OptionExt, ResultExt, Snafu}; use system_catalog::sys_catalog_table::{ self, CreateCatalogRequest, CreateSchemaRequest, SysCatalogTable, VisitOptions, diff --git a/catalog_impls/src/volatile.rs b/catalog_impls/src/volatile.rs index fda87ae401..1f9e429fc3 100644 --- a/catalog_impls/src/volatile.rs +++ b/catalog_impls/src/volatile.rs @@ -22,7 +22,7 @@ use catalog::{ }; use cluster::shard_set::ShardSet; use common_types::schema::SchemaName; -use common_util::error::BoxError; +use generic_error::BoxError; use log::{debug, info}; use meta_client::{types::AllocSchemaIdRequest, MetaClientRef}; use snafu::{ensure, OptionExt, ResultExt}; diff --git a/cluster/Cargo.toml b/cluster/Cargo.toml index 6b40f42382..b317ea7a35 100644 --- a/cluster/Cargo.toml +++ b/cluster/Cargo.toml @@ -15,14 +15,17 @@ async-trait = { workspace = true } catalog = { workspace = true } ceresdbproto = { workspace = true } common_types = { workspace = true } -common_util = { workspace = true } etcd-client = { workspace = true } +generic_error = { workspace = true } log = { workspace = true } +macros = { workspace = true } meta_client = { workspace = true } prost = { workspace = true } +runtime = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } snafu = { workspace = true } table_engine = { workspace = true } +time_ext = { workspace = true } tokio = { workspace = true } wal = { workspace = true } diff --git a/cluster/src/cluster_impl.rs b/cluster/src/cluster_impl.rs index f1859a7fa1..0f139bd433 100644 --- a/cluster/src/cluster_impl.rs +++ b/cluster/src/cluster_impl.rs @@ -7,11 +7,8 @@ use std::{ use async_trait::async_trait; use common_types::table::ShardId; -use common_util::{ - error::BoxError, - runtime::{JoinHandle, Runtime}, -}; use etcd_client::ConnectOptions; +use generic_error::BoxError; use log::{error, info, warn}; use meta_client::{ types::{ @@ -20,6 +17,7 @@ use meta_client::{ }, MetaClientRef, }; +use runtime::{JoinHandle, Runtime}; use snafu::{ensure, OptionExt, ResultExt}; use tokio::{ sync::mpsc::{self, Sender}, diff --git a/cluster/src/config.rs b/cluster/src/config.rs index e47cee016d..030e62ba40 100644 --- a/cluster/src/config.rs +++ b/cluster/src/config.rs @@ -3,11 +3,11 @@ use std::time::Duration; use common_types::schema::TIMESTAMP_COLUMN; -use common_util::config::ReadableDuration; use etcd_client::ConnectOptions; use meta_client::meta_impl::MetaClientConfig; use serde::{Deserialize, Serialize}; use table_engine::ANALYTIC_ENGINE_TYPE; +use time_ext::ReadableDuration; #[derive(Debug, Clone, Deserialize, Serialize)] #[serde(default)] diff --git a/cluster/src/lib.rs b/cluster/src/lib.rs index 524fd918da..6561c6b718 100644 --- a/cluster/src/lib.rs +++ b/cluster/src/lib.rs @@ -14,7 +14,8 @@ use std::sync::Arc; use async_trait::async_trait; use common_types::schema::SchemaName; -use common_util::{define_result, error::GenericError}; +use generic_error::GenericError; +use macros::define_result; use meta_client::types::{ ClusterNodesRef, RouteTablesRequest, RouteTablesResponse, ShardId, ShardInfo, ShardVersion, }; diff --git a/cluster/src/shard_lock_manager.rs b/cluster/src/shard_lock_manager.rs index c1b7f957b9..809b49b3dd 100644 --- a/cluster/src/shard_lock_manager.rs +++ b/cluster/src/shard_lock_manager.rs @@ -9,15 +9,13 @@ use std::{ use ceresdbproto::meta_event::ShardLockValue; use common_types::{bytes::Bytes, table::ShardId}; -use common_util::{ - define_result, - runtime::{JoinHandle, RuntimeRef}, -}; use etcd_client::{ Client, Compare, CompareOp, LeaseKeepAliveStream, LeaseKeeper, PutOptions, Txn, TxnOp, }; use log::{debug, error, info, warn}; +use macros::define_result; use prost::Message; +use runtime::{JoinHandle, RuntimeRef}; use snafu::{ensure, Backtrace, ResultExt, Snafu}; use tokio::sync::{oneshot, RwLock as AsyncRwLock}; diff --git a/cluster/src/shard_operation.rs b/cluster/src/shard_operation.rs index f7d1a8006d..d04e5e3ab6 100644 --- a/cluster/src/shard_operation.rs +++ b/cluster/src/shard_operation.rs @@ -9,7 +9,7 @@ use std::sync::Arc; use async_trait::async_trait; use common_types::table::ShardId; -use common_util::error::{BoxError, GenericResult}; +use generic_error::{BoxError, GenericResult}; use wal::manager::WalManagerRef; #[async_trait] diff --git a/cluster/src/shard_operator.rs b/cluster/src/shard_operator.rs index 76e3189bbc..6ebed011f1 100644 --- a/cluster/src/shard_operator.rs +++ b/cluster/src/shard_operator.rs @@ -9,7 +9,7 @@ use catalog::{ }, table_operator::TableOperator, }; -use common_util::error::BoxError; +use generic_error::BoxError; use log::info; use snafu::ResultExt; use table_engine::{ diff --git a/common_types/Cargo.toml b/common_types/Cargo.toml index 3ad2ac2777..73a196d620 100644 --- a/common_types/Cargo.toml +++ b/common_types/Cargo.toml @@ -24,6 +24,7 @@ bytes_ext = { workspace = true } ceresdbproto = { workspace = true } chrono = { workspace = true } datafusion = { workspace = true, optional = true } +macros = { workspace = true } murmur3 = "0.4.1" paste = { workspace = true } prost = { workspace = true } @@ -32,6 +33,3 @@ serde = { workspace = true } serde_json = { workspace = true } snafu = { workspace = true } sqlparser = { workspace = true } - -[dev-dependencies] -common_util = { workspace = true } diff --git a/common_types/src/column_schema.rs b/common_types/src/column_schema.rs index 7aa067751f..fee89dd329 100644 --- a/common_types/src/column_schema.rs +++ b/common_types/src/column_schema.rs @@ -1,4 +1,4 @@ -// Copyright 2022 CeresDB Project Authors. Licensed under Apache-2.0. +// Copyright 2022-2023 CeresDB Project Authors. Licensed under Apache-2.0. //! Schema of column @@ -538,7 +538,7 @@ impl From for schema_pb::ColumnSchema { #[cfg(test)] mod tests { - use common_util::hash_map; + use macros::hash_map; use sqlparser::ast::Value; use super::*; diff --git a/common_types/src/hex.rs b/common_types/src/hex.rs index 8ae595c562..c8e8be26ac 100644 --- a/common_types/src/hex.rs +++ b/common_types/src/hex.rs @@ -1,7 +1,7 @@ // Copyright 2022-2023 CeresDB Project Authors. Licensed under Apache-2.0. -// TODO: move this module to common_util package after remove the common_types -// from the dependencies of the common_util. +// TODO: move this module to test_util package after remove the common_types +// from the dependencies of the test_util. /// Try to decode bytes from hex literal string. /// diff --git a/common_types/src/row/mod.rs b/common_types/src/row/mod.rs index 05a47e3819..46ee38e654 100644 --- a/common_types/src/row/mod.rs +++ b/common_types/src/row/mod.rs @@ -88,7 +88,7 @@ pub enum Error { }, } -// Do not depend on common_util crates +// Do not depend on test_util crates pub type Result = std::result::Result; // TODO(yingwen): diff --git a/common_util/src/byte.rs b/common_util/src/byte.rs deleted file mode 100644 index a27053d951..0000000000 --- a/common_util/src/byte.rs +++ /dev/null @@ -1,3 +0,0 @@ -// Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. - -pub use hex::encode; diff --git a/common_util/src/record_batch.rs b/common_util/src/record_batch.rs deleted file mode 100644 index de8035e5b8..0000000000 --- a/common_util/src/record_batch.rs +++ /dev/null @@ -1,22 +0,0 @@ -// Copyright 2022 CeresDB Project Authors. Licensed under Apache-2.0. - -use arrow::util::pretty; -use common_types::record_batch::RecordBatch; - -/// A helper function to assert record batch. -pub fn assert_record_batches_eq(expected: &[&str], record_batches: Vec) { - let arrow_record_batch = record_batches - .into_iter() - .map(|record| record.into_arrow_record_batch()) - .collect::>(); - - let expected_lines: Vec = expected.iter().map(|&s| s.into()).collect(); - let formatted = pretty::pretty_format_batches(arrow_record_batch.as_slice()) - .unwrap() - .to_string(); - let actual_lines: Vec<&str> = formatted.trim().lines().collect(); - assert_eq!( - expected_lines, actual_lines, - "\n\nexpected:\n\n{expected_lines:#?}\nactual:\n\n{actual_lines:#?}\n\n" - ); -} diff --git a/common_util/src/time.rs b/common_util/src/time.rs deleted file mode 100644 index 8748da6110..0000000000 --- a/common_util/src/time.rs +++ /dev/null @@ -1,125 +0,0 @@ -// Copyright 2022 CeresDB Project Authors. Licensed under Apache-2.0. - -//! Time utilities - -// TODO(yingwen): Move to common_types ? - -use std::{ - convert::TryInto, - time::{Duration, Instant, UNIX_EPOCH}, -}; - -use chrono::{DateTime, Utc}; -use common_types::time::Timestamp; -use snafu::{Backtrace, GenerateBacktrace, Snafu}; - -use crate::config::ReadableDuration; - -#[derive(Debug, Snafu)] -#[allow(clippy::enum_variant_names)] -pub enum Error { - #[snafu(display("Failed to parse duration, err:{}.\nBacktrace:\n{}", err, backtrace))] - ParseDuration { err: String, backtrace: Backtrace }, -} - -define_result!(Error); - -pub trait DurationExt { - /// Convert into u64. - /// - /// Returns u64::MAX if overflow - fn as_millis_u64(&self) -> u64; -} - -impl DurationExt for Duration { - #[inline] - fn as_millis_u64(&self) -> u64 { - match self.as_millis().try_into() { - Ok(v) => v, - Err(_) => u64::MAX, - } - } -} - -pub trait InstantExt { - fn saturating_elapsed(&self) -> Duration; - - /// Check whether this instant is reached - fn check_deadline(&self) -> bool; -} - -impl InstantExt for Instant { - fn saturating_elapsed(&self) -> Duration { - Instant::now().saturating_duration_since(*self) - } - - fn check_deadline(&self) -> bool { - self.saturating_elapsed().is_zero() - } -} - -#[inline] -pub fn secs_to_nanos(s: u64) -> u64 { - s * 1_000_000_000 -} - -#[inline] -pub fn current_time_millis() -> u64 { - Utc::now().timestamp_millis() as u64 -} - -#[inline] -pub fn current_as_rfc3339() -> String { - Utc::now().to_rfc3339() -} - -#[inline] -pub fn format_as_ymdhms(unix_timestamp: i64) -> String { - let dt = DateTime::::from(UNIX_EPOCH + Duration::from_millis(unix_timestamp as u64)); - dt.format("%Y-%m-%d %H:%M:%S").to_string() -} - -pub fn try_to_millis(ts: i64) -> Option { - // https://help.aliyun.com/document_detail/60683.html - if (4294968..=4294967295).contains(&ts) { - return Some(Timestamp::new(ts * 1000)); - } - if (4294967296..=9999999999999).contains(&ts) { - return Some(Timestamp::new(ts)); - } - None -} - -pub fn parse_duration(v: &str) -> Result { - v.parse::() - .map_err(|err| Error::ParseDuration { - err, - backtrace: Backtrace::generate(), - }) -} - -#[cfg(test)] -mod tests { - use std::thread; - - use super::*; - #[test] - fn test_as_mills_u64() { - let d = Duration::from_millis(100); - assert_eq!(100, d.as_millis_u64()); - - let d = Duration::from_secs(100); - assert_eq!(100000, d.as_millis_u64()); - } - - #[test] - fn test_saturating_elapsed() { - let ins = Instant::now(); - let one_hundred_mills = Duration::from_millis(100); - let error = 10; - thread::sleep(one_hundred_mills); - assert!(ins.saturating_elapsed().as_millis_u64() - 100 < error); - thread::sleep(one_hundred_mills); - assert!(ins.saturating_elapsed().as_millis_u64() - 200 < 2 * error); - } -} diff --git a/components/alloc_tracker/Cargo.toml b/components/alloc_tracker/Cargo.toml new file mode 100644 index 0000000000..c81a3fe600 --- /dev/null +++ b/components/alloc_tracker/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "alloc_tracker" + +[package.version] +workspace = true + +[package.authors] +workspace = true + +[package.edition] +workspace = true diff --git a/common_util/src/alloc_tracker.rs b/components/alloc_tracker/src/lib.rs similarity index 98% rename from common_util/src/alloc_tracker.rs rename to components/alloc_tracker/src/lib.rs index 7e0979cb0f..242fc04a48 100644 --- a/common_util/src/alloc_tracker.rs +++ b/components/alloc_tracker/src/lib.rs @@ -1,4 +1,4 @@ -// Copyright 2022 CeresDB Project Authors. Licensed under Apache-2.0. +// Copyright 2022-2023 CeresDB Project Authors. Licensed under Apache-2.0. //! Alloc tracker diff --git a/components/codec/Cargo.toml b/components/codec/Cargo.toml new file mode 100644 index 0000000000..36b426b321 --- /dev/null +++ b/components/codec/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "codec" + +[package.version] +workspace = true + +[package.authors] +workspace = true + +[package.edition] +workspace = true + +[dependencies] +# In alphabetical order +common_types = { workspace = true, features = ["test"] } +macros = { workspace = true } +snafu = { workspace = true } diff --git a/common_util/src/codec/compact/bytes.rs b/components/codec/src/compact/bytes.rs similarity index 97% rename from common_util/src/codec/compact/bytes.rs rename to components/codec/src/compact/bytes.rs index 1b3346e388..09e68ba4f9 100644 --- a/common_util/src/codec/compact/bytes.rs +++ b/components/codec/src/compact/bytes.rs @@ -1,4 +1,4 @@ -// Copyright 2022 CeresDB Project Authors. Licensed under Apache-2.0. +// Copyright 2022-2023 CeresDB Project Authors. Licensed under Apache-2.0. //! Bytes format @@ -7,7 +7,7 @@ use std::convert::TryFrom; use common_types::bytes::{Buf, BufMut, Bytes, BytesMut, SafeBuf, SafeBufMut}; use snafu::{ensure, ResultExt}; -use crate::codec::{ +use crate::{ compact::{ DecodeEmptyValue, DecodeValue, DecodeVarint, EncodeValue, EncodeVarint, Error, MemCompactDecoder, MemCompactEncoder, Result, SkipDecodedValue, TryIntoUsize, diff --git a/common_util/src/codec/compact/datum.rs b/components/codec/src/compact/datum.rs similarity index 99% rename from common_util/src/codec/compact/datum.rs rename to components/codec/src/compact/datum.rs index 3ee2794477..bde3f4c722 100644 --- a/common_util/src/codec/compact/datum.rs +++ b/components/codec/src/compact/datum.rs @@ -1,4 +1,4 @@ -// Copyright 2022 CeresDB Project Authors. Licensed under Apache-2.0. +// Copyright 2022-2023 CeresDB Project Authors. Licensed under Apache-2.0. //! Datum compact codec @@ -10,7 +10,7 @@ use common_types::{ }; use snafu::ResultExt; -use crate::codec::{ +use crate::{ compact::{EncodeKey, Error, MemCompactDecoder, MemCompactEncoder, Result}, consts, DecodeTo, Encoder, }; diff --git a/common_util/src/codec/compact/float.rs b/components/codec/src/compact/float.rs similarity index 96% rename from common_util/src/codec/compact/float.rs rename to components/codec/src/compact/float.rs index 89f965104d..8e3652bc89 100644 --- a/common_util/src/codec/compact/float.rs +++ b/components/codec/src/compact/float.rs @@ -1,11 +1,11 @@ -// Copyright 2022 CeresDB Project Authors. Licensed under Apache-2.0. +// Copyright 2022-2023 CeresDB Project Authors. Licensed under Apache-2.0. use std::mem; use common_types::bytes::{SafeBuf, SafeBufMut}; use snafu::ResultExt; -use crate::codec::{ +use crate::{ compact::{DecodeValue, EncodeValue, Error, MemCompactDecoder, MemCompactEncoder, Result}, DecodeTo, Encoder, }; diff --git a/common_util/src/codec/compact/mod.rs b/components/codec/src/compact/mod.rs similarity index 92% rename from common_util/src/codec/compact/mod.rs rename to components/codec/src/compact/mod.rs index 99a3319f41..28f97c3aef 100644 --- a/common_util/src/codec/compact/mod.rs +++ b/components/codec/src/compact/mod.rs @@ -1,4 +1,4 @@ -// Copyright 2022 CeresDB Project Authors. Licensed under Apache-2.0. +// Copyright 2022-2023 CeresDB Project Authors. Licensed under Apache-2.0. //! Mem compact format codec @@ -10,9 +10,10 @@ mod float; mod number; use common_types::bytes::SafeBuf; +use macros::define_result; use snafu::{ensure, Backtrace, ResultExt, Snafu}; -use crate::codec::consts; +use crate::consts; #[derive(Debug, Snafu)] pub enum Error { @@ -23,10 +24,10 @@ pub enum Error { EncodeValue { source: common_types::bytes::Error }, #[snafu(display("Failed to encode varint, err:{}", source))] - EncodeVarint { source: crate::codec::varint::Error }, + EncodeVarint { source: crate::varint::Error }, #[snafu(display("Failed to decode varint, err:{}", source))] - DecodeVarint { source: crate::codec::varint::Error }, + DecodeVarint { source: crate::varint::Error }, #[snafu(display("Failed to decode key, err:{}", source))] DecodeKey { source: common_types::bytes::Error }, diff --git a/common_util/src/codec/compact/number.rs b/components/codec/src/compact/number.rs similarity index 97% rename from common_util/src/codec/compact/number.rs rename to components/codec/src/compact/number.rs index f79419783d..929d792fba 100644 --- a/common_util/src/codec/compact/number.rs +++ b/components/codec/src/compact/number.rs @@ -1,11 +1,11 @@ -// Copyright 2022 CeresDB Project Authors. Licensed under Apache-2.0. +// Copyright 2022-2023 CeresDB Project Authors. Licensed under Apache-2.0. //! Number format use common_types::bytes::{Buf, SafeBufMut}; use snafu::ResultExt; -use crate::codec::{ +use crate::{ compact::{DecodeVarint, EncodeVarint, Error, MemCompactDecoder, MemCompactEncoder, Result}, consts, varint, DecodeTo, Encoder, }; diff --git a/common_util/src/codec/consts.rs b/components/codec/src/consts.rs similarity index 89% rename from common_util/src/codec/consts.rs rename to components/codec/src/consts.rs index 843985eec6..28c63b394d 100644 --- a/common_util/src/codec/consts.rs +++ b/components/codec/src/consts.rs @@ -1,4 +1,4 @@ -// Copyright 2022 CeresDB Project Authors. Licensed under Apache-2.0. +// Copyright 2022-2023 CeresDB Project Authors. Licensed under Apache-2.0. //! Common constants used in codec diff --git a/common_util/src/codec/mod.rs b/components/codec/src/lib.rs similarity index 92% rename from common_util/src/codec/mod.rs rename to components/codec/src/lib.rs index dcabf4eba6..5281cbbc49 100644 --- a/common_util/src/codec/mod.rs +++ b/components/codec/src/lib.rs @@ -1,4 +1,4 @@ -// Copyright 2022 CeresDB Project Authors. Licensed under Apache-2.0. +// Copyright 2022-2023 CeresDB Project Authors. Licensed under Apache-2.0. //! Data encoding diff --git a/common_util/src/codec/memcomparable/bytes.rs b/components/codec/src/memcomparable/bytes.rs similarity index 99% rename from common_util/src/codec/memcomparable/bytes.rs rename to components/codec/src/memcomparable/bytes.rs index ad0fb9c15f..d60139c8bd 100644 --- a/common_util/src/codec/memcomparable/bytes.rs +++ b/components/codec/src/memcomparable/bytes.rs @@ -1,11 +1,11 @@ -// Copyright 2022 CeresDB Project Authors. Licensed under Apache-2.0. +// Copyright 2022-2023 CeresDB Project Authors. Licensed under Apache-2.0. //! Bytes format use common_types::bytes::{Buf, BufMut, Bytes, BytesMut, SafeBuf, SafeBufMut}; use snafu::{ensure, ResultExt}; -use crate::codec::{ +use crate::{ memcomparable::{ DecodeValueGroup, DecodeValueMarker, DecodeValuePadding, EncodeValue, Error, MemComparable, Result, SkipPadding, diff --git a/common_util/src/codec/memcomparable/datum.rs b/components/codec/src/memcomparable/datum.rs similarity index 99% rename from common_util/src/codec/memcomparable/datum.rs rename to components/codec/src/memcomparable/datum.rs index c469b4bbb9..48a0210797 100644 --- a/common_util/src/codec/memcomparable/datum.rs +++ b/components/codec/src/memcomparable/datum.rs @@ -1,4 +1,4 @@ -// Copyright 2022 CeresDB Project Authors. Licensed under Apache-2.0. +// Copyright 2022-2023 CeresDB Project Authors. Licensed under Apache-2.0. //! Datum comparable codec @@ -12,7 +12,7 @@ use common_types::{ }; use snafu::ResultExt; -use crate::codec::{ +use crate::{ consts, memcomparable::{EncodeKey, Error, MemComparable, Result, UnsupportedKind}, DecodeTo, Encoder, diff --git a/common_util/src/codec/memcomparable/mod.rs b/components/codec/src/memcomparable/mod.rs similarity index 96% rename from common_util/src/codec/memcomparable/mod.rs rename to components/codec/src/memcomparable/mod.rs index a9116c1e1d..01d4cc49ae 100644 --- a/common_util/src/codec/memcomparable/mod.rs +++ b/components/codec/src/memcomparable/mod.rs @@ -1,4 +1,4 @@ -// Copyright 2022 CeresDB Project Authors. Licensed under Apache-2.0. +// Copyright 2022-2023 CeresDB Project Authors. Licensed under Apache-2.0. //! Mem comparable format codec @@ -13,6 +13,7 @@ use common_types::{ bytes::{BytesMut, SafeBuf}, datum::DatumKind, }; +use macros::define_result; use snafu::{ensure, Backtrace, ResultExt, Snafu}; #[derive(Debug, Snafu)] diff --git a/common_util/src/codec/memcomparable/number.rs b/components/codec/src/memcomparable/number.rs similarity index 98% rename from common_util/src/codec/memcomparable/number.rs rename to components/codec/src/memcomparable/number.rs index d3dc01643a..0ce412a432 100644 --- a/common_util/src/codec/memcomparable/number.rs +++ b/components/codec/src/memcomparable/number.rs @@ -1,11 +1,11 @@ -// Copyright 2022 CeresDB Project Authors. Licensed under Apache-2.0. +// Copyright 2022-2023 CeresDB Project Authors. Licensed under Apache-2.0. //! Number format use common_types::bytes::{SafeBuf, SafeBufMut}; use snafu::ResultExt; -use crate::codec::{ +use crate::{ consts, memcomparable::{DecodeValue, EncodeValue, Error, MemComparable, Result}, DecodeTo, Encoder, diff --git a/common_util/src/codec/row.rs b/components/codec/src/row.rs similarity index 96% rename from common_util/src/codec/row.rs rename to components/codec/src/row.rs index ba707d58bf..873f4a779a 100644 --- a/common_util/src/codec/row.rs +++ b/components/codec/src/row.rs @@ -1,4 +1,4 @@ -// Copyright 2022 CeresDB Project Authors. Licensed under Apache-2.0. +// Copyright 2022-2023 CeresDB Project Authors. Licensed under Apache-2.0. //! Row encoding utils //! @@ -13,9 +13,10 @@ use common_types::{ row::{Row, RowGroup}, schema::{IndexInWriterSchema, Schema}, }; +use macros::define_result; use snafu::{ResultExt, Snafu}; -use crate::codec::{ +use crate::{ compact::{MemCompactDecoder, MemCompactEncoder}, DecodeTo, Decoder, Encoder, }; @@ -24,14 +25,10 @@ use crate::codec::{ #[snafu(visibility(pub(crate)))] pub enum Error { #[snafu(display("Failed to encode row datum, err:{}", source))] - EncodeRowDatum { - source: crate::codec::compact::Error, - }, + EncodeRowDatum { source: crate::compact::Error }, #[snafu(display("Failed to decode row datum, err:{}", source))] - DecodeRowDatum { - source: crate::codec::compact::Error, - }, + DecodeRowDatum { source: crate::compact::Error }, } define_result!(Error); @@ -209,7 +206,7 @@ pub fn key_prefix_next(key: &[u8]) -> BytesMut { mod test { use common_types::schema::IndexInWriterSchema; - use crate::codec::{ + use crate::{ row::{WalRowDecoder, WalRowEncoder}, Decoder, Encoder, }; diff --git a/common_util/src/codec/varint.rs b/components/codec/src/varint.rs similarity index 98% rename from common_util/src/codec/varint.rs rename to components/codec/src/varint.rs index a10b9058fa..df7cab672a 100644 --- a/common_util/src/codec/varint.rs +++ b/components/codec/src/varint.rs @@ -1,7 +1,8 @@ -// Copyright 2022 CeresDB Project Authors. Licensed under Apache-2.0. +// Copyright 2022-2023 CeresDB Project Authors. Licensed under Apache-2.0. //! Varint for codec whose test is covered by compact/number.rs use common_types::bytes::{Buf, SafeBuf, SafeBufMut}; +use macros::define_result; use snafu::{Backtrace, ResultExt, Snafu}; #[derive(Debug, Snafu)] diff --git a/components/future_cancel/Cargo.toml b/components/future_cancel/Cargo.toml new file mode 100644 index 0000000000..8e9fef5f37 --- /dev/null +++ b/components/future_cancel/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "future_cancel" + +[package.version] +workspace = true + +[package.authors] +workspace = true + +[package.edition] +workspace = true + +[dependencies] +futures = { workspace = true } +runtime = { workspace = true } + +[dev-dependencies] +tokio = { workspace = true } diff --git a/common_util/src/future_cancel.rs b/components/future_cancel/src/lib.rs similarity index 98% rename from common_util/src/future_cancel.rs rename to components/future_cancel/src/lib.rs index 93b8eaf9bc..ffbcd7d96e 100644 --- a/common_util/src/future_cancel.rs +++ b/components/future_cancel/src/lib.rs @@ -13,8 +13,7 @@ use std::{ }; use futures::future::BoxFuture; - -use crate::runtime::RuntimeRef; +use runtime::RuntimeRef; /// Wrapper around a future that cannot be cancelled. /// @@ -106,10 +105,10 @@ mod tests { time::Duration, }; + use runtime::Builder; use tokio::sync::Barrier; use super::*; - use crate::runtime::Builder; fn rt() -> RuntimeRef { let rt = Builder::default() diff --git a/components/generic_error/Cargo.toml b/components/generic_error/Cargo.toml new file mode 100644 index 0000000000..19d21ea900 --- /dev/null +++ b/components/generic_error/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "generic_error" + +[package.version] +workspace = true + +[package.authors] +workspace = true + +[package.edition] +workspace = true diff --git a/common_util/src/error.rs b/components/generic_error/src/lib.rs similarity index 86% rename from common_util/src/error.rs rename to components/generic_error/src/lib.rs index c2ffc2d52a..59e871326c 100644 --- a/common_util/src/error.rs +++ b/components/generic_error/src/lib.rs @@ -1,4 +1,4 @@ -// Copyright 2022 CeresDB Project Authors. Licensed under Apache-2.0. +// Copyright 2022-2023 CeresDB Project Authors. Licensed under Apache-2.0. pub type GenericError = Box; pub type GenericResult = std::result::Result; diff --git a/components/id_allocator/Cargo.toml b/components/id_allocator/Cargo.toml new file mode 100644 index 0000000000..b2ec5d9030 --- /dev/null +++ b/components/id_allocator/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "id_allocator" + +[package.version] +workspace = true + +[package.authors] +workspace = true + +[package.edition] +workspace = true + +[dependencies] +# In alphabetical order +generic_error = { workspace = true } +tokio = { workspace = true } diff --git a/common_util/src/id_allocator.rs b/components/id_allocator/src/lib.rs similarity index 98% rename from common_util/src/id_allocator.rs rename to components/id_allocator/src/lib.rs index b9eb9bfa7e..57e21b1aba 100644 --- a/common_util/src/id_allocator.rs +++ b/components/id_allocator/src/lib.rs @@ -2,10 +2,9 @@ use std::future::Future; +use generic_error::GenericResult; use tokio::sync::RwLock; -use crate::error::GenericResult; - struct Inner { last_id: u64, max_id: u64, diff --git a/components/macros/Cargo.toml b/components/macros/Cargo.toml new file mode 100644 index 0000000000..de205d4607 --- /dev/null +++ b/components/macros/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "macros" + +[package.version] +workspace = true + +[package.authors] +workspace = true + +[package.edition] +workspace = true diff --git a/common_util/src/macros.rs b/components/macros/src/lib.rs similarity index 92% rename from common_util/src/macros.rs rename to components/macros/src/lib.rs index 343bd1fee1..2e5cbf384a 100644 --- a/common_util/src/macros.rs +++ b/components/macros/src/lib.rs @@ -1,4 +1,4 @@ -// Copyright 2022 CeresDB Project Authors. Licensed under Apache-2.0. +// Copyright 2022-2023 CeresDB Project Authors. Licensed under Apache-2.0. //! Contains all needed macros diff --git a/components/message_queue/Cargo.toml b/components/message_queue/Cargo.toml index 4766330828..657e6c5a28 100644 --- a/components/message_queue/Cargo.toml +++ b/components/message_queue/Cargo.toml @@ -8,11 +8,12 @@ workspace = true [dependencies] async-trait = { workspace = true } chrono = { workspace = true } -common_util = { workspace = true } futures = { workspace = true } log = { workspace = true } +macros = { workspace = true } serde = { workspace = true } snafu = { workspace = true } +time_ext = { workspace = true } tokio = { workspace = true } [dependencies.rskafka] diff --git a/components/message_queue/src/kafka/config.rs b/components/message_queue/src/kafka/config.rs index 24629442ed..df6e3e4673 100644 --- a/components/message_queue/src/kafka/config.rs +++ b/components/message_queue/src/kafka/config.rs @@ -2,8 +2,8 @@ //! Kafka implementation's config -use common_util::config::ReadableDuration; use serde::{Deserialize, Serialize}; +use time_ext::ReadableDuration; /// Generic client config that is used for consumers, producers as well as admin /// operations (like "create topic"). diff --git a/components/message_queue/src/kafka/kafka_impl.rs b/components/message_queue/src/kafka/kafka_impl.rs index 0fdfb462b5..4a270449e9 100644 --- a/components/message_queue/src/kafka/kafka_impl.rs +++ b/components/message_queue/src/kafka/kafka_impl.rs @@ -9,9 +9,9 @@ use std::{ }; use async_trait::async_trait; -use common_util::define_result; use futures::StreamExt; use log::info; +use macros::define_result; use rskafka::{ client::{ consumer::{StartOffset as KafkaStartOffset, StreamConsumer, StreamConsumerBuilder}, diff --git a/components/metric_ext/Cargo.toml b/components/metric_ext/Cargo.toml new file mode 100644 index 0000000000..d796e523b1 --- /dev/null +++ b/components/metric_ext/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "metric_ext" + +[package.version] +workspace = true + +[package.authors] +workspace = true + +[package.edition] +workspace = true + +[dependencies] +crossbeam-utils = "0.8.7" +# In alphabetical order +serde = { workspace = true } +serde_json = { workspace = true } +time_ext = { workspace = true } diff --git a/common_util/src/metric.rs b/components/metric_ext/src/lib.rs similarity index 97% rename from common_util/src/metric.rs rename to components/metric_ext/src/lib.rs index 8eedfdb933..2afc31c91d 100644 --- a/common_util/src/metric.rs +++ b/components/metric_ext/src/lib.rs @@ -1,4 +1,4 @@ -// Copyright 2022 CeresDB Project Authors. Licensed under Apache-2.0. +// Copyright 2022-2023 CeresDB Project Authors. Licensed under Apache-2.0. /// Copied from https://github.com/sunng87/metriki/blob/master/metriki-core/src/metrics/meter.rs /// But supports 1 hour and 2 hour rate. @@ -11,8 +11,6 @@ use serde::ser::SerializeMap; #[cfg(feature = "ser")] use serde::{Serialize, Serializer}; -use crate::time; - /// Meters are used to calculate rate of an event. #[derive(Debug)] pub struct Meter { @@ -95,7 +93,7 @@ impl ExponentiallyWeightedMovingAverage { fn new(alpha: f64, interval_secs: u64) -> ExponentiallyWeightedMovingAverage { ExponentiallyWeightedMovingAverage { alpha, - interval_nanos: time::secs_to_nanos(interval_secs), + interval_nanos: time_ext::secs_to_nanos(interval_secs), uncounted: AtomicCell::new(0), rate: AtomicCell::new(None), @@ -120,7 +118,7 @@ impl ExponentiallyWeightedMovingAverage { fn get_rate(&self) -> f64 { if let Some(rate) = self.rate.load() { - rate * time::secs_to_nanos(1) as f64 + rate * time_ext::secs_to_nanos(1) as f64 } else { 0f64 } diff --git a/components/object_store/Cargo.toml b/components/object_store/Cargo.toml index 2d2738ae95..f74349a185 100644 --- a/components/object_store/Cargo.toml +++ b/components/object_store/Cargo.toml @@ -17,20 +17,25 @@ ceresdbproto = { workspace = true } chrono = { workspace = true } clru = { workspace = true } common_types = { workspace = true } -common_util = { workspace = true } crc = "3.0.0" futures = { workspace = true } +generic_error = { workspace = true } lazy_static = { workspace = true } log = { workspace = true } lru = { workspace = true } +macros = { workspace = true } +partitioned_lock = { workspace = true } prometheus = { workspace = true } prometheus-static-metric = { workspace = true } prost = { workspace = true } rand = { workspace = true } +runtime = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } +size_ext = { workspace = true } snafu = { workspace = true } table_kv = { workspace = true } +time_ext = { workspace = true } tokio = { workspace = true } twox-hash = "1.6" upstream = { package = "object_store", version = "0.5.6", features = [ "aws" ] } diff --git a/components/object_store/src/config.rs b/components/object_store/src/config.rs index f9558c2736..2dc5086123 100644 --- a/components/object_store/src/config.rs +++ b/components/object_store/src/config.rs @@ -1,10 +1,11 @@ -// Copyright 2022 CeresDB Project Authors. Licensed under Apache-2.0. +// Copyright 2022-2023 CeresDB Project Authors. Licensed under Apache-2.0. use std::time::Duration; -use common_util::config::{ReadableDuration, ReadableSize}; use serde::{Deserialize, Serialize}; +use size_ext::ReadableSize; use table_kv::config::ObkvConfig; +use time_ext::ReadableDuration; #[derive(Debug, Clone, Deserialize, Serialize)] #[serde(default)] diff --git a/components/object_store/src/disk_cache.rs b/components/object_store/src/disk_cache.rs index ef5814964f..6c0d8aa515 100644 --- a/components/object_store/src/disk_cache.rs +++ b/components/object_store/src/disk_cache.rs @@ -12,14 +12,15 @@ use std::{collections::BTreeMap, fmt::Display, ops::Range, sync::Arc}; use async_trait::async_trait; use bytes::{Bytes, BytesMut}; use common_types::hash::SeaHasherBuilder; -use common_util::{partitioned_lock::PartitionedMutexAsync, time::current_as_rfc3339}; use crc::{Crc, CRC_32_ISCSI}; use futures::stream::BoxStream; use log::{debug, error, info}; use lru::LruCache; +use partitioned_lock::PartitionedMutexAsync; use prost::Message; use serde::{Deserialize, Serialize}; use snafu::{ensure, Backtrace, ResultExt, Snafu}; +use time_ext; use tokio::{ fs::{File, OpenOptions}, io::{AsyncReadExt, AsyncWrite, AsyncWriteExt}, @@ -332,7 +333,7 @@ impl DiskCacheStore { if metadata.len() == 0 { let manifest = Manifest { page_size, - create_at: current_as_rfc3339(), + create_at: time_ext::current_as_rfc3339(), version: CURRENT_VERSION, }; diff --git a/components/object_store/src/mem_cache.rs b/components/object_store/src/mem_cache.rs index d0421f20b9..deba4c4004 100644 --- a/components/object_store/src/mem_cache.rs +++ b/components/object_store/src/mem_cache.rs @@ -1,4 +1,4 @@ -// Copyright 2022 CeresDB Project Authors. Licensed under Apache-2.0. +// Copyright 2022-2023 CeresDB Project Authors. Licensed under Apache-2.0. //! An implementation of ObjectStore, which support //! 1. Cache based on memory, and support evict based on memory usage @@ -15,8 +15,9 @@ use async_trait::async_trait; use bytes::Bytes; use clru::{CLruCache, CLruCacheConfig, WeightScale}; use common_types::hash::{ahash::RandomState, build_fixed_seed_ahasher_builder}; -use common_util::{define_result, partitioned_lock::PartitionedMutex}; use futures::stream::BoxStream; +use macros::define_result; +use partitioned_lock::PartitionedMutex; use snafu::{OptionExt, Snafu}; use tokio::io::AsyncWrite; use upstream::{ diff --git a/components/object_store/src/metrics.rs b/components/object_store/src/metrics.rs index be6516aac0..32a71d894b 100644 --- a/components/object_store/src/metrics.rs +++ b/components/object_store/src/metrics.rs @@ -1,15 +1,15 @@ -// Copyright 2022 CeresDB Project Authors. Licensed under Apache-2.0. +// Copyright 2022-2023 CeresDB Project Authors. Licensed under Apache-2.0. use std::{fmt::Display, ops::Range, sync::Arc, thread, time::Instant}; use async_trait::async_trait; use bytes::Bytes; -use common_util::runtime::Runtime; use futures::stream::BoxStream; use lazy_static::lazy_static; use log::trace; use prometheus::{exponential_buckets, register_histogram_vec, HistogramVec}; use prometheus_static_metric::make_static_metric; +use runtime::Runtime; use tokio::io::AsyncWrite; use upstream::{ path::Path, Error as StoreError, GetResult, ListResult, MultipartId, ObjectMeta, ObjectStore, diff --git a/components/object_store/src/obkv/meta.rs b/components/object_store/src/obkv/meta.rs index 6aaf445124..2339fc2249 100644 --- a/components/object_store/src/obkv/meta.rs +++ b/components/object_store/src/obkv/meta.rs @@ -2,10 +2,8 @@ use std::{ops::Range, str, sync::Arc, time}; -use common_util::{ - define_result, - error::{BoxError, GenericError}, -}; +use generic_error::{BoxError, GenericError}; +use macros::define_result; use serde::{Deserialize, Serialize}; use snafu::{ensure, Backtrace, ResultExt, Snafu}; use table_kv::{ScanContext, ScanIter, TableKv, WriteBatch, WriteContext}; diff --git a/components/object_store/src/obkv/mod.rs b/components/object_store/src/obkv/mod.rs index b989cfe180..af7aff5394 100644 --- a/components/object_store/src/obkv/mod.rs +++ b/components/object_store/src/obkv/mod.rs @@ -15,11 +15,11 @@ use std::{ use async_trait::async_trait; use bytes::Bytes; use chrono::{DateTime, TimeZone, Utc}; -use common_util::error::{BoxError, GenericError}; use futures::{ stream::{BoxStream, FuturesOrdered}, StreamExt, }; +use generic_error::{BoxError, GenericError}; use log::debug; use snafu::{ensure, Backtrace, ResultExt, Snafu}; use table_kv::{ScanContext, ScanIter, TableKv, WriteBatch, WriteContext}; @@ -795,9 +795,9 @@ mod test { use std::sync::Arc; use bytes::Bytes; - use common_util::runtime::{Builder, Runtime}; use futures::StreamExt; use rand::{thread_rng, Rng}; + use runtime::{Builder, Runtime}; use table_kv::memory::MemoryImpl; use tokio::io::AsyncWriteExt; use upstream::{path::Path, ObjectStore}; diff --git a/components/panic_ext/Cargo.toml b/components/panic_ext/Cargo.toml new file mode 100644 index 0000000000..0b49fcf672 --- /dev/null +++ b/components/panic_ext/Cargo.toml @@ -0,0 +1,27 @@ +[package] +name = "panic_ext" + +[package.version] +workspace = true + +[package.authors] +workspace = true + +[package.edition] +workspace = true + +[dependencies.slog-global] +version = "0.1" +git = "https://github.com/tikv/slog-global.git" +rev = "d592f88e4dbba5eb439998463054f1a44fbf17b9" + +[dependencies] +backtrace = "0.3.9" +libc = "0.2" +log = { workspace = true } +logger = { workspace = true } + +[dev-dependencies] +gag = "1.0" +nix = "0.22" +slog = { workspace = true } diff --git a/common_util/src/panic.rs b/components/panic_ext/src/lib.rs similarity index 98% rename from common_util/src/panic.rs rename to components/panic_ext/src/lib.rs index ba6a59a757..05e278d58a 100644 --- a/common_util/src/panic.rs +++ b/components/panic_ext/src/lib.rs @@ -1,4 +1,4 @@ -// Copyright 2022 CeresDB Project Authors. Licensed under Apache-2.0. +// Copyright 2022-2023 CeresDB Project Authors. Licensed under Apache-2.0. use std::thread; @@ -88,7 +88,7 @@ mod tests { }; use slog::{self, Drain, Level, OwnedKVList, Record}; - use crate::panic::set_panic_hook; + use crate::set_panic_hook; /// Create a child process and wait to get its exit code. fn run_and_wait_child_process(child: impl Fn()) -> Result { diff --git a/components/parquet_ext/Cargo.toml b/components/parquet_ext/Cargo.toml index ba31703d18..6c024bdb65 100644 --- a/components/parquet_ext/Cargo.toml +++ b/components/parquet_ext/Cargo.toml @@ -15,9 +15,9 @@ arrow = { workspace = true } arrow_ext = { workspace = true } async-trait = { workspace = true } bytes = { workspace = true } -common_util = { workspace = true } datafusion = { workspace = true } futures = { workspace = true } +generic_error = { workspace = true } log = { workspace = true } object_store = { workspace = true } parquet = { workspace = true } diff --git a/components/parquet_ext/src/meta_data.rs b/components/parquet_ext/src/meta_data.rs index 56bf777ff8..4c984134d5 100644 --- a/components/parquet_ext/src/meta_data.rs +++ b/components/parquet_ext/src/meta_data.rs @@ -4,7 +4,7 @@ use std::{ops::Range, sync::Arc}; use async_trait::async_trait; use bytes::Bytes; -use common_util::error::GenericResult; +use generic_error::GenericResult; use parquet::{ arrow::{arrow_reader::ArrowReaderOptions, ParquetRecordBatchStreamBuilder}, errors::{ParquetError, Result}, diff --git a/components/partitioned_lock/Cargo.toml b/components/partitioned_lock/Cargo.toml new file mode 100644 index 0000000000..045865a8ff --- /dev/null +++ b/components/partitioned_lock/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "partitioned_lock" + +[package.version] +workspace = true + +[package.authors] +workspace = true + +[package.edition] +workspace = true + +[dependencies] +tokio = { workspace = true } + +[dev-dependencies] +common_types = { workspace = true } diff --git a/common_util/src/partitioned_lock.rs b/components/partitioned_lock/src/lib.rs similarity index 98% rename from common_util/src/partitioned_lock.rs rename to components/partitioned_lock/src/lib.rs index ba4baf43c0..e42c9f8c3a 100644 --- a/common_util/src/partitioned_lock.rs +++ b/components/partitioned_lock/src/lib.rs @@ -1,4 +1,4 @@ -// Copyright 2022 CeresDB Project Authors. Licensed under Apache-2.0. +// Copyright 2022-2023 CeresDB Project Authors. Licensed under Apache-2.0. //! Partitioned locks @@ -7,8 +7,6 @@ use std::{ sync::{Mutex, MutexGuard, RwLock, RwLockReadGuard, RwLockWriteGuard}, }; -use tokio; - /// Simple partitioned `RwLock` pub struct PartitionedRwLock where @@ -171,6 +169,7 @@ where mod tests { use std::collections::HashMap; + // TODO: remove this importing. use common_types::hash::{build_fixed_seed_ahasher_builder, SeaHasherBuilder}; use super::*; diff --git a/components/runtime/Cargo.toml b/components/runtime/Cargo.toml new file mode 100644 index 0000000000..aa9f11c2bb --- /dev/null +++ b/components/runtime/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "runtime" + +[package.version] +workspace = true + +[package.authors] +workspace = true + +[package.edition] +workspace = true + +[dependencies] +lazy_static = { workspace = true } +macros = { workspace = true } +pin-project-lite = { workspace = true } +prometheus = { workspace = true } +snafu = { workspace = true } +tokio = { workspace = true } + +[dev-dependencies] +tokio-test = "0.4.2" diff --git a/common_util/src/runtime/mod.rs b/components/runtime/src/lib.rs similarity index 99% rename from common_util/src/runtime/mod.rs rename to components/runtime/src/lib.rs index 407577b097..499c4de36d 100644 --- a/common_util/src/runtime/mod.rs +++ b/components/runtime/src/lib.rs @@ -8,14 +8,16 @@ use std::{ task::{Context, Poll}, }; +use macros::define_result; +use metrics::Metrics; use pin_project_lite::pin_project; use snafu::{Backtrace, GenerateBacktrace, ResultExt, Snafu}; use tokio::{ runtime::{Builder as RuntimeBuilder, Runtime as TokioRuntime}, task::{JoinError, JoinHandle as TokioJoinHandle}, }; + mod metrics; -use metrics::Metrics; // TODO(yingwen): Use opaque error type #[derive(Debug, Snafu)] diff --git a/common_util/src/runtime/metrics.rs b/components/runtime/src/metrics.rs similarity index 94% rename from common_util/src/runtime/metrics.rs rename to components/runtime/src/metrics.rs index 4f82494093..58ae763374 100644 --- a/common_util/src/runtime/metrics.rs +++ b/components/runtime/src/metrics.rs @@ -1,4 +1,4 @@ -// Copyright 2022 CeresDB Project Authors. Licensed under Apache-2.0. +// Copyright 2022-2023 CeresDB Project Authors. Licensed under Apache-2.0. use lazy_static::lazy_static; use prometheus::{register_int_gauge_vec, IntGauge, IntGaugeVec}; diff --git a/components/size_ext/Cargo.toml b/components/size_ext/Cargo.toml new file mode 100644 index 0000000000..7d8bc3c8f8 --- /dev/null +++ b/components/size_ext/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "size_ext" + +[package.version] +workspace = true + +[package.authors] +workspace = true + +[package.edition] +workspace = true + +[dependencies] +serde = { workspace = true } + +[dev-dependencies] +toml = { workspace = true } diff --git a/components/size_ext/src/lib.rs b/components/size_ext/src/lib.rs new file mode 100644 index 0000000000..cba72537e2 --- /dev/null +++ b/components/size_ext/src/lib.rs @@ -0,0 +1,299 @@ +// Copyright 2022-2023 CeresDB Project Authors. Licensed under Apache-2.0. + +// Copyright 2016 TiKV Project Authors. Licensed under Apache-2.0. + +//! Configure utils + +//This module is forked from tikv and remove unnecessary code. +//https://github.com/tikv/tikv/blob/HEAD/src/util/config.rs +use std::{ + fmt::{self, Write}, + ops::{Div, Mul}, + str::{self, FromStr}, +}; + +use serde::{ + de::{self, Unexpected, Visitor}, + Deserialize, Deserializer, Serialize, Serializer, +}; + +const UNIT: u64 = 1; + +const BINARY_DATA_MAGNITUDE: u64 = 1024; +pub const B: u64 = UNIT; +pub const KIB: u64 = UNIT * BINARY_DATA_MAGNITUDE; +pub const MIB: u64 = KIB * BINARY_DATA_MAGNITUDE; +pub const GIB: u64 = MIB * BINARY_DATA_MAGNITUDE; +pub const TIB: u64 = GIB * BINARY_DATA_MAGNITUDE; +pub const PIB: u64 = TIB * BINARY_DATA_MAGNITUDE; + +#[derive(Clone, Debug, Copy, PartialEq, Eq, PartialOrd)] +pub struct ReadableSize(pub u64); + +impl ReadableSize { + pub const fn kb(count: u64) -> ReadableSize { + ReadableSize(count * KIB) + } + + pub const fn mb(count: u64) -> ReadableSize { + ReadableSize(count * MIB) + } + + pub const fn gb(count: u64) -> ReadableSize { + ReadableSize(count * GIB) + } + + pub const fn as_mb(self) -> u64 { + self.0 / MIB + } + + pub const fn as_byte(self) -> u64 { + self.0 + } +} + +impl Div for ReadableSize { + type Output = ReadableSize; + + fn div(self, rhs: u64) -> ReadableSize { + ReadableSize(self.0 / rhs) + } +} + +impl Div for ReadableSize { + type Output = u64; + + fn div(self, rhs: ReadableSize) -> u64 { + self.0 / rhs.0 + } +} + +impl Mul for ReadableSize { + type Output = ReadableSize; + + fn mul(self, rhs: u64) -> ReadableSize { + ReadableSize(self.0 * rhs) + } +} + +impl Serialize for ReadableSize { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + let size = self.0; + let mut buffer = String::new(); + if size == 0 { + write!(buffer, "{size}KiB").unwrap(); + } else if size % PIB == 0 { + write!(buffer, "{}PiB", size / PIB).unwrap(); + } else if size % TIB == 0 { + write!(buffer, "{}TiB", size / TIB).unwrap(); + } else if size % GIB == 0 { + write!(buffer, "{}GiB", size / GIB).unwrap(); + } else if size % MIB == 0 { + write!(buffer, "{}MiB", size / MIB).unwrap(); + } else if size % KIB == 0 { + write!(buffer, "{}KiB", size / KIB).unwrap(); + } else { + return serializer.serialize_u64(size); + } + serializer.serialize_str(&buffer) + } +} + +impl FromStr for ReadableSize { + type Err = String; + + // This method parses value in binary unit. + fn from_str(s: &str) -> Result { + let size_str = s.trim(); + if size_str.is_empty() { + return Err(format!("{s:?} is not a valid size.")); + } + + if !size_str.is_ascii() { + return Err(format!("ASCII string is expected, but got {s:?}")); + } + + // size: digits and '.' as decimal separator + let size_len = size_str + .to_string() + .chars() + .take_while(|c| char::is_ascii_digit(c) || ['.', 'e', 'E', '-', '+'].contains(c)) + .count(); + + // unit: alphabetic characters + let (size, unit) = size_str.split_at(size_len); + + let unit = match unit.trim() { + "K" | "KB" | "KiB" => KIB, + "M" | "MB" | "MiB" => MIB, + "G" | "GB" | "GiB" => GIB, + "T" | "TB" | "TiB" => TIB, + "P" | "PB" | "PiB" => PIB, + "B" | "" => UNIT, + _ => { + return Err(format!( + "only B, KB, KiB, MB, MiB, GB, GiB, TB, TiB, PB, and PiB are supported: {s:?}" + )); + } + }; + + match size.parse::() { + Ok(n) => Ok(ReadableSize((n * unit as f64) as u64)), + Err(_) => Err(format!("invalid size string: {s:?}")), + } + } +} + +impl<'de> Deserialize<'de> for ReadableSize { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + struct SizeVisitor; + + impl<'de> Visitor<'de> for SizeVisitor { + type Value = ReadableSize; + + fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("valid size") + } + + fn visit_i64(self, size: i64) -> Result + where + E: de::Error, + { + if size >= 0 { + self.visit_u64(size as u64) + } else { + Err(E::invalid_value(Unexpected::Signed(size), &self)) + } + } + + fn visit_u64(self, size: u64) -> Result + where + E: de::Error, + { + Ok(ReadableSize(size)) + } + + fn visit_str(self, size_str: &str) -> Result + where + E: de::Error, + { + size_str.parse().map_err(E::custom) + } + } + + deserializer.deserialize_any(SizeVisitor) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_readable_size() { + let s = ReadableSize::kb(2); + assert_eq!(s.0, 2048); + assert_eq!(s.as_mb(), 0); + let s = ReadableSize::mb(2); + assert_eq!(s.0, 2 * 1024 * 1024); + assert_eq!(s.as_mb(), 2); + let s = ReadableSize::gb(2); + assert_eq!(s.0, 2 * 1024 * 1024 * 1024); + assert_eq!(s.as_mb(), 2048); + + assert_eq!((ReadableSize::mb(2) / 2).0, MIB); + assert_eq!((ReadableSize::mb(1) / 2).0, 512 * KIB); + assert_eq!(ReadableSize::mb(2) / ReadableSize::kb(1), 2048); + } + + #[test] + fn test_parse_readable_size() { + #[derive(Serialize, Deserialize)] + struct SizeHolder { + s: ReadableSize, + } + + let legal_cases = vec![ + (0, "0KiB"), + (2 * KIB, "2KiB"), + (4 * MIB, "4MiB"), + (5 * GIB, "5GiB"), + (7 * TIB, "7TiB"), + (11 * PIB, "11PiB"), + ]; + for (size, exp) in legal_cases { + let c = SizeHolder { + s: ReadableSize(size), + }; + let res_str = toml::to_string(&c).unwrap(); + let exp_str = format!("s = {exp:?}\n"); + assert_eq!(res_str, exp_str); + let res_size: SizeHolder = toml::from_str(&exp_str).unwrap(); + assert_eq!(res_size.s.0, size); + } + + let c = SizeHolder { + s: ReadableSize(512), + }; + let res_str = toml::to_string(&c).unwrap(); + assert_eq!(res_str, "s = 512\n"); + let res_size: SizeHolder = toml::from_str(&res_str).unwrap(); + assert_eq!(res_size.s.0, c.s.0); + + let decode_cases = vec![ + (" 0.5 PB", PIB / 2), + ("0.5 TB", TIB / 2), + ("0.5GB ", GIB / 2), + ("0.5MB", MIB / 2), + ("0.5KB", KIB / 2), + ("0.5P", PIB / 2), + ("0.5T", TIB / 2), + ("0.5G", GIB / 2), + ("0.5M", MIB / 2), + ("0.5K", KIB / 2), + ("23", 23), + ("1", 1), + ("1024B", KIB), + // units with binary prefixes + (" 0.5 PiB", PIB / 2), + ("1PiB", PIB), + ("0.5 TiB", TIB / 2), + ("2 TiB", TIB * 2), + ("0.5GiB ", GIB / 2), + ("787GiB ", GIB * 787), + ("0.5MiB", MIB / 2), + ("3MiB", MIB * 3), + ("0.5KiB", KIB / 2), + ("1 KiB", KIB), + // scientific notation + ("0.5e6 B", B * 500000), + ("0.5E6 B", B * 500000), + ("1e6B", B * 1000000), + ("8E6B", B * 8000000), + ("8e7", B * 80000000), + ("1e-1MB", MIB / 10), + ("1e+1MB", MIB * 10), + ("0e+10MB", 0), + ]; + for (src, exp) in decode_cases { + let src = format!("s = {src:?}"); + let res: SizeHolder = toml::from_str(&src).unwrap(); + assert_eq!(res.s.0, exp); + } + + let illegal_cases = vec![ + "0.5kb", "0.5kB", "0.5Kb", "0.5k", "0.5g", "b", "gb", "1b", "B", "1K24B", " 5_KB", + "4B7", "5M_", + ]; + for src in illegal_cases { + let src_str = format!("s = {src:?}"); + assert!(toml::from_str::(&src_str).is_err(), "{}", src); + } + } +} diff --git a/components/table_kv/Cargo.toml b/components/table_kv/Cargo.toml index e7eaf8e667..f0206a0e73 100644 --- a/components/table_kv/Cargo.toml +++ b/components/table_kv/Cargo.toml @@ -11,12 +11,12 @@ workspace = true workspace = true [dependencies] -common_util = { workspace = true } log = { workspace = true } +macros = { workspace = true } obkv-table-client-rs = { git = "https://github.com/oceanbase/obkv-table-client-rs.git", rev = "f7c7a2c866fc83af4862b02447538f2abc5e8839" } serde = { workspace = true } - snafu = { workspace = true } +time_ext = { workspace = true } [dev-dependencies] rand = { workspace = true } diff --git a/components/table_kv/src/config.rs b/components/table_kv/src/config.rs index 919177ee2d..e77ed4d909 100644 --- a/components/table_kv/src/config.rs +++ b/components/table_kv/src/config.rs @@ -1,9 +1,9 @@ -// Copyright 2022 CeresDB Project Authors. Licensed under Apache-2.0. +// Copyright 2022-2023 CeresDB Project Authors. Licensed under Apache-2.0. //! Config of table kv. -use common_util::config::ReadableDuration; use serde::{Deserialize, Serialize}; +use time_ext::ReadableDuration; // TODO: use test conf to control which environments to test. const TEST_FULL_USER_NAME: &str = "user_name"; diff --git a/components/table_kv/src/memory.rs b/components/table_kv/src/memory.rs index fd01c01473..5998e976c2 100644 --- a/components/table_kv/src/memory.rs +++ b/components/table_kv/src/memory.rs @@ -1,4 +1,4 @@ -// Copyright 2022 CeresDB Project Authors. Licensed under Apache-2.0. +// Copyright 2022-2023 CeresDB Project Authors. Licensed under Apache-2.0. //! Memory table kv, mainly for test. @@ -8,7 +8,7 @@ use std::{ sync::{Arc, Mutex}, }; -use common_util::define_result; +use macros::define_result; use snafu::{Backtrace, OptionExt, Snafu}; use crate::{ diff --git a/components/table_kv/src/obkv.rs b/components/table_kv/src/obkv.rs index 0fc0d73772..d4ded97181 100644 --- a/components/table_kv/src/obkv.rs +++ b/components/table_kv/src/obkv.rs @@ -1,11 +1,11 @@ -// Copyright 2022 CeresDB Project Authors. Licensed under Apache-2.0. +// Copyright 2022-2023 CeresDB Project Authors. Licensed under Apache-2.0. //! Obkv implementation. use std::{collections::HashMap, error::Error as StdError, fmt}; -use common_util::define_result; use log::{error, info}; +use macros::define_result; use obkv::{ payloads::ObTableBatchOperation, Builder, ObTableClient, QueryResultSet, RunningMode, Table, TableOpResult, TableQuery, Value, diff --git a/common_util/Cargo.toml b/components/test_util/Cargo.toml similarity index 66% rename from common_util/Cargo.toml rename to components/test_util/Cargo.toml index bef0814525..f42a4dc534 100644 --- a/common_util/Cargo.toml +++ b/components/test_util/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "common_util" +name = "test_util" [package.version] workspace = true @@ -20,32 +20,17 @@ backtrace = "0.3.9" ceresdbproto = { workspace = true } chrono = { workspace = true } common_types = { workspace = true, features = ["test"] } -crossbeam-utils = "0.8.7" env_logger = { workspace = true, optional = true } futures = { workspace = true } -hex = "0.4.3" lazy_static = { workspace = true } libc = "0.2" log = { workspace = true } -logger = { workspace = true } pin-project-lite = { workspace = true } -prometheus = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } snafu = { workspace = true } -time = "0.3" tokio = { workspace = true } -toml = { workspace = true } - -[dev-dependencies.slog-global] -version = "0.1" -git = "https://github.com/tikv/slog-global.git" -rev = "d592f88e4dbba5eb439998463054f1a44fbf17b9" [dev-dependencies] env_logger = { workspace = true } -gag = "1.0" -nix = "0.22" -slog = { workspace = true } tempfile = { workspace = true } -tokio-test = "0.4.2" diff --git a/common_util/src/lib.rs b/components/test_util/src/lib.rs similarity index 51% rename from common_util/src/lib.rs rename to components/test_util/src/lib.rs index 4414591f88..9702a45442 100644 --- a/common_util/src/lib.rs +++ b/components/test_util/src/lib.rs @@ -2,32 +2,13 @@ //! Common utils shared by the whole project -// We need to define mod with macro_use before other mod so that other -// mods in this crate can use the macros -#[macro_use] -pub mod macros; - -// TODO(yingwen): Move some mod into components as a crate -pub mod alloc_tracker; -pub mod byte; -pub mod codec; -pub mod config; -pub mod error; -pub mod future_cancel; -pub mod id_allocator; -pub mod metric; -pub mod panic; -pub mod partitioned_lock; -pub mod record_batch; -pub mod runtime; -pub mod time; -pub mod timed_task; -pub mod toml; - #[cfg(any(test, feature = "test"))] pub mod tests { use std::{io::Write, sync::Once}; + use arrow::util::pretty; + use common_types::record_batch::RecordBatch; + static INIT_LOG: Once = Once::new(); pub fn init_log_for_test() { @@ -47,4 +28,22 @@ pub mod tests { .init(); }); } + + /// A helper function to assert record batch. + pub fn assert_record_batches_eq(expected: &[&str], record_batches: Vec) { + let arrow_record_batch = record_batches + .into_iter() + .map(|record| record.into_arrow_record_batch()) + .collect::>(); + + let expected_lines: Vec = expected.iter().map(|&s| s.into()).collect(); + let formatted = pretty::pretty_format_batches(arrow_record_batch.as_slice()) + .unwrap() + .to_string(); + let actual_lines: Vec<&str> = formatted.trim().lines().collect(); + assert_eq!( + expected_lines, actual_lines, + "\n\nexpected:\n\n{expected_lines:#?}\nactual:\n\n{actual_lines:#?}\n\n" + ); + } } diff --git a/components/time_ext/Cargo.toml b/components/time_ext/Cargo.toml new file mode 100644 index 0000000000..ce78222c73 --- /dev/null +++ b/components/time_ext/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "time_ext" + +[package.version] +workspace = true + +[package.authors] +workspace = true + +[package.edition] +workspace = true + +[dependencies] +# In alphabetical order +ceresdbproto = { workspace = true } +chrono = { workspace = true } +common_types = { workspace = true } +macros = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +snafu = { workspace = true } + +[dev-dependencies] +toml = { workspace = true } diff --git a/common_util/src/config.rs b/components/time_ext/src/lib.rs similarity index 58% rename from common_util/src/config.rs rename to components/time_ext/src/lib.rs index df552928fb..37f0fbaaac 100644 --- a/common_util/src/config.rs +++ b/components/time_ext/src/lib.rs @@ -1,37 +1,40 @@ -// Copyright 2022 CeresDB Project Authors. Licensed under Apache-2.0. +// Copyright 2022-2023 CeresDB Project Authors. Licensed under Apache-2.0. -// Copyright 2016 TiKV Project Authors. Licensed under Apache-2.0. +//! Time utilities -//! Configure utils +// TODO(yingwen): Move to common_types ? -//This module is forked from tikv and remove unnecessary code. -//https://github.com/tikv/tikv/blob/HEAD/src/util/config.rs use std::{ + convert::TryInto, fmt::{self, Write}, ops::{Add, AddAssign, Div, DivAssign, Mul, MulAssign, Sub, SubAssign}, - str::{self, FromStr}, - time::Duration, + str::FromStr, + time::{Duration, Instant, UNIX_EPOCH}, }; use ceresdbproto::manifest as manifest_pb; +use chrono::{DateTime, Utc}; +use common_types::time::Timestamp; +use macros::define_result; use serde::{ - de::{self, Unexpected, Visitor}, + de::{self, Visitor}, Deserialize, Deserializer, Serialize, Serializer, }; +use snafu::{Backtrace, GenerateBacktrace, Snafu}; -const UNIT: u64 = 1; +#[derive(Debug, Snafu)] +#[allow(clippy::enum_variant_names)] +pub enum Error { + #[snafu(display("Failed to parse duration, err:{}.\nBacktrace:\n{}", err, backtrace))] + ParseDuration { err: String, backtrace: Backtrace }, +} -const BINARY_DATA_MAGNITUDE: u64 = 1024; -pub const B: u64 = UNIT; -pub const KIB: u64 = UNIT * BINARY_DATA_MAGNITUDE; -pub const MIB: u64 = KIB * BINARY_DATA_MAGNITUDE; -pub const GIB: u64 = MIB * BINARY_DATA_MAGNITUDE; -pub const TIB: u64 = GIB * BINARY_DATA_MAGNITUDE; -pub const PIB: u64 = TIB * BINARY_DATA_MAGNITUDE; +define_result!(Error); const TIME_MAGNITUDE_1: u64 = 1000; const TIME_MAGNITUDE_2: u64 = 60; const TIME_MAGNITUDE_3: u64 = 24; +const UNIT: u64 = 1; const MS: u64 = UNIT; const SECOND: u64 = MS * TIME_MAGNITUDE_1; const MINUTE: u64 = SECOND * TIME_MAGNITUDE_2; @@ -92,10 +95,10 @@ impl From for TimeUnit { impl FromStr for TimeUnit { type Err = String; - fn from_str(tu_str: &str) -> Result { + fn from_str(tu_str: &str) -> std::result::Result { let tu_str = tu_str.trim(); if !tu_str.is_ascii() { - return Err(format!("unexpect ascii string: {tu_str}")); + return Err(format!("unexpected ascii string: {tu_str}")); } match tu_str.to_lowercase().as_str() { @@ -126,168 +129,70 @@ impl fmt::Display for TimeUnit { } } -#[derive(Clone, Debug, Copy, PartialEq, Eq, PartialOrd)] -pub struct ReadableSize(pub u64); +pub trait DurationExt { + /// Convert into u64. + /// + /// Returns u64::MAX if overflow + fn as_millis_u64(&self) -> u64; +} -impl ReadableSize { - pub const fn kb(count: u64) -> ReadableSize { - ReadableSize(count * KIB) +impl DurationExt for Duration { + #[inline] + fn as_millis_u64(&self) -> u64 { + match self.as_millis().try_into() { + Ok(v) => v, + Err(_) => u64::MAX, + } } +} - pub const fn mb(count: u64) -> ReadableSize { - ReadableSize(count * MIB) - } +pub trait InstantExt { + fn saturating_elapsed(&self) -> Duration; - pub const fn gb(count: u64) -> ReadableSize { - ReadableSize(count * GIB) - } + /// Check whether this instant is reached + fn check_deadline(&self) -> bool; +} - pub const fn as_mb(self) -> u64 { - self.0 / MIB +impl InstantExt for Instant { + fn saturating_elapsed(&self) -> Duration { + Instant::now().saturating_duration_since(*self) } - pub const fn as_byte(self) -> u64 { - self.0 + fn check_deadline(&self) -> bool { + self.saturating_elapsed().is_zero() } } -impl Div for ReadableSize { - type Output = ReadableSize; - - fn div(self, rhs: u64) -> ReadableSize { - ReadableSize(self.0 / rhs) - } +#[inline] +pub fn secs_to_nanos(s: u64) -> u64 { + s * 1_000_000_000 } -impl Div for ReadableSize { - type Output = u64; - - fn div(self, rhs: ReadableSize) -> u64 { - self.0 / rhs.0 - } +#[inline] +pub fn current_time_millis() -> u64 { + Utc::now().timestamp_millis() as u64 } -impl Mul for ReadableSize { - type Output = ReadableSize; - - fn mul(self, rhs: u64) -> ReadableSize { - ReadableSize(self.0 * rhs) - } +#[inline] +pub fn current_as_rfc3339() -> String { + Utc::now().to_rfc3339() } -impl Serialize for ReadableSize { - fn serialize(&self, serializer: S) -> Result - where - S: Serializer, - { - let size = self.0; - let mut buffer = String::new(); - if size == 0 { - write!(buffer, "{size}KiB").unwrap(); - } else if size % PIB == 0 { - write!(buffer, "{}PiB", size / PIB).unwrap(); - } else if size % TIB == 0 { - write!(buffer, "{}TiB", size / TIB).unwrap(); - } else if size % GIB == 0 { - write!(buffer, "{}GiB", size / GIB).unwrap(); - } else if size % MIB == 0 { - write!(buffer, "{}MiB", size / MIB).unwrap(); - } else if size % KIB == 0 { - write!(buffer, "{}KiB", size / KIB).unwrap(); - } else { - return serializer.serialize_u64(size); - } - serializer.serialize_str(&buffer) - } +#[inline] +pub fn format_as_ymdhms(unix_timestamp: i64) -> String { + let dt = DateTime::::from(UNIX_EPOCH + Duration::from_millis(unix_timestamp as u64)); + dt.format("%Y-%m-%d %H:%M:%S").to_string() } -impl FromStr for ReadableSize { - type Err = String; - - // This method parses value in binary unit. - fn from_str(s: &str) -> Result { - let size_str = s.trim(); - if size_str.is_empty() { - return Err(format!("{s:?} is not a valid size.")); - } - - if !size_str.is_ascii() { - return Err(format!("ASCII string is expected, but got {s:?}")); - } - - // size: digits and '.' as decimal separator - let size_len = size_str - .to_string() - .chars() - .take_while(|c| char::is_ascii_digit(c) || ['.', 'e', 'E', '-', '+'].contains(c)) - .count(); - - // unit: alphabetic characters - let (size, unit) = size_str.split_at(size_len); - - let unit = match unit.trim() { - "K" | "KB" | "KiB" => KIB, - "M" | "MB" | "MiB" => MIB, - "G" | "GB" | "GiB" => GIB, - "T" | "TB" | "TiB" => TIB, - "P" | "PB" | "PiB" => PIB, - "B" | "" => UNIT, - _ => { - return Err(format!( - "only B, KB, KiB, MB, MiB, GB, GiB, TB, TiB, PB, and PiB are supported: {s:?}" - )); - } - }; - - match size.parse::() { - Ok(n) => Ok(ReadableSize((n * unit as f64) as u64)), - Err(_) => Err(format!("invalid size string: {s:?}")), - } +pub fn try_to_millis(ts: i64) -> Option { + // https://help.aliyun.com/document_detail/60683.html + if (4294968..=4294967295).contains(&ts) { + return Some(Timestamp::new(ts * 1000)); } -} - -impl<'de> Deserialize<'de> for ReadableSize { - fn deserialize(deserializer: D) -> Result - where - D: Deserializer<'de>, - { - struct SizeVisitor; - - impl<'de> Visitor<'de> for SizeVisitor { - type Value = ReadableSize; - - fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - formatter.write_str("valid size") - } - - fn visit_i64(self, size: i64) -> Result - where - E: de::Error, - { - if size >= 0 { - self.visit_u64(size as u64) - } else { - Err(E::invalid_value(Unexpected::Signed(size), &self)) - } - } - - fn visit_u64(self, size: u64) -> Result - where - E: de::Error, - { - Ok(ReadableSize(size)) - } - - fn visit_str(self, size_str: &str) -> Result - where - E: de::Error, - { - size_str.parse().map_err(E::custom) - } - } - - deserializer.deserialize_any(SizeVisitor) + if (4294967296..=9999999999999).contains(&ts) { + return Some(Timestamp::new(ts)); } + None } #[derive(Clone, Copy, Debug, PartialEq, Eq, Ord, PartialOrd, Default)] @@ -365,10 +270,10 @@ impl From for ReadableDuration { impl FromStr for ReadableDuration { type Err = String; - fn from_str(dur_str: &str) -> Result { + fn from_str(dur_str: &str) -> std::result::Result { let dur_str = dur_str.trim(); if !dur_str.is_ascii() { - return Err(format!("unexpect ascii string: {dur_str}")); + return Err(format!("unexpected ascii string: {dur_str}")); } let err_msg = "valid duration, only d, h, m, s, ms are supported.".to_owned(); let mut left = dur_str.as_bytes(); @@ -394,7 +299,7 @@ impl FromStr for ReadableDuration { return Err("d, h, m, s, ms should occur in given order.".to_owned()); } // do we need to check 12h360m? - let number_str = unsafe { str::from_utf8_unchecked(first) }; + let number_str = unsafe { std::str::from_utf8_unchecked(first) }; dur += match number_str.trim().parse::() { Ok(n) => n * unit as f64, Err(_) => return Err(err_msg), @@ -484,7 +389,7 @@ impl fmt::Display for ReadableDuration { } impl Serialize for ReadableDuration { - fn serialize(&self, serializer: S) -> Result + fn serialize(&self, serializer: S) -> std::result::Result where S: Serializer, { @@ -495,7 +400,7 @@ impl Serialize for ReadableDuration { } impl<'de> Deserialize<'de> for ReadableDuration { - fn deserialize(deserializer: D) -> Result + fn deserialize(deserializer: D) -> std::result::Result where D: Deserializer<'de>, { @@ -508,7 +413,7 @@ impl<'de> Deserialize<'de> for ReadableDuration { formatter.write_str("valid duration") } - fn visit_str(self, dur_str: &str) -> Result + fn visit_str(self, dur_str: &str) -> std::result::Result where E: de::Error, { @@ -520,110 +425,37 @@ impl<'de> Deserialize<'de> for ReadableDuration { } } +pub fn parse_duration(v: &str) -> Result { + v.parse::() + .map_err(|err| Error::ParseDuration { + err, + backtrace: Backtrace::generate(), + }) +} + #[cfg(test)] mod tests { - use super::*; + use std::thread; + use super::*; #[test] - fn test_readable_size() { - let s = ReadableSize::kb(2); - assert_eq!(s.0, 2048); - assert_eq!(s.as_mb(), 0); - let s = ReadableSize::mb(2); - assert_eq!(s.0, 2 * 1024 * 1024); - assert_eq!(s.as_mb(), 2); - let s = ReadableSize::gb(2); - assert_eq!(s.0, 2 * 1024 * 1024 * 1024); - assert_eq!(s.as_mb(), 2048); - - assert_eq!((ReadableSize::mb(2) / 2).0, MIB); - assert_eq!((ReadableSize::mb(1) / 2).0, 512 * KIB); - assert_eq!(ReadableSize::mb(2) / ReadableSize::kb(1), 2048); + fn test_as_mills_u64() { + let d = Duration::from_millis(100); + assert_eq!(100, d.as_millis_u64()); + + let d = Duration::from_secs(100); + assert_eq!(100000, d.as_millis_u64()); } #[test] - fn test_parse_readable_size() { - #[derive(Serialize, Deserialize)] - struct SizeHolder { - s: ReadableSize, - } - - let legal_cases = vec![ - (0, "0KiB"), - (2 * KIB, "2KiB"), - (4 * MIB, "4MiB"), - (5 * GIB, "5GiB"), - (7 * TIB, "7TiB"), - (11 * PIB, "11PiB"), - ]; - for (size, exp) in legal_cases { - let c = SizeHolder { - s: ReadableSize(size), - }; - let res_str = toml::to_string(&c).unwrap(); - let exp_str = format!("s = {exp:?}\n"); - assert_eq!(res_str, exp_str); - let res_size: SizeHolder = toml::from_str(&exp_str).unwrap(); - assert_eq!(res_size.s.0, size); - } - - let c = SizeHolder { - s: ReadableSize(512), - }; - let res_str = toml::to_string(&c).unwrap(); - assert_eq!(res_str, "s = 512\n"); - let res_size: SizeHolder = toml::from_str(&res_str).unwrap(); - assert_eq!(res_size.s.0, c.s.0); - - let decode_cases = vec![ - (" 0.5 PB", PIB / 2), - ("0.5 TB", TIB / 2), - ("0.5GB ", GIB / 2), - ("0.5MB", MIB / 2), - ("0.5KB", KIB / 2), - ("0.5P", PIB / 2), - ("0.5T", TIB / 2), - ("0.5G", GIB / 2), - ("0.5M", MIB / 2), - ("0.5K", KIB / 2), - ("23", 23), - ("1", 1), - ("1024B", KIB), - // units with binary prefixes - (" 0.5 PiB", PIB / 2), - ("1PiB", PIB), - ("0.5 TiB", TIB / 2), - ("2 TiB", TIB * 2), - ("0.5GiB ", GIB / 2), - ("787GiB ", GIB * 787), - ("0.5MiB", MIB / 2), - ("3MiB", MIB * 3), - ("0.5KiB", KIB / 2), - ("1 KiB", KIB), - // scientific notation - ("0.5e6 B", B * 500000), - ("0.5E6 B", B * 500000), - ("1e6B", B * 1000000), - ("8E6B", B * 8000000), - ("8e7", B * 80000000), - ("1e-1MB", MIB / 10), - ("1e+1MB", MIB * 10), - ("0e+10MB", 0), - ]; - for (src, exp) in decode_cases { - let src = format!("s = {src:?}"); - let res: SizeHolder = toml::from_str(&src).unwrap(); - assert_eq!(res.s.0, exp); - } - - let illegal_cases = vec![ - "0.5kb", "0.5kB", "0.5Kb", "0.5k", "0.5g", "b", "gb", "1b", "B", "1K24B", " 5_KB", - "4B7", "5M_", - ]; - for src in illegal_cases { - let src_str = format!("s = {src:?}"); - assert!(toml::from_str::(&src_str).is_err(), "{}", src); - } + fn test_saturating_elapsed() { + let ins = Instant::now(); + let one_hundred_mills = Duration::from_millis(100); + let error = 10; + thread::sleep(one_hundred_mills); + assert!(ins.saturating_elapsed().as_millis_u64() - 100 < error); + thread::sleep(one_hundred_mills); + assert!(ins.saturating_elapsed().as_millis_u64() - 200 < 2 * error); } #[test] diff --git a/components/timed_task/Cargo.toml b/components/timed_task/Cargo.toml new file mode 100644 index 0000000000..7d869ebacc --- /dev/null +++ b/components/timed_task/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "timed_task" + +[package.version] +workspace = true + +[package.authors] +workspace = true + +[package.edition] +workspace = true + +[dependencies] +log = { workspace = true } +runtime = { workspace = true } +tokio = { workspace = true } diff --git a/common_util/src/timed_task.rs b/components/timed_task/src/lib.rs similarity index 94% rename from common_util/src/timed_task.rs rename to components/timed_task/src/lib.rs index 580e222f43..c8a343bf88 100644 --- a/common_util/src/timed_task.rs +++ b/components/timed_task/src/lib.rs @@ -1,10 +1,11 @@ -// Copyright 2022 CeresDB Project Authors. Licensed under Apache-2.0. +// Copyright 2022-2023 CeresDB Project Authors. Licensed under Apache-2.0. //! Timed background tasks. use std::{future::Future, time::Duration}; use log::info; +use runtime::{self, JoinHandle, Runtime}; use tokio::{ sync::{ mpsc::{self, UnboundedReceiver, UnboundedSender}, @@ -13,8 +14,6 @@ use tokio::{ time, }; -use crate::runtime::{JoinHandle, Runtime}; - /// A task to run periodically. pub struct TimedTask { name: String, @@ -79,7 +78,7 @@ pub struct TaskHandle { impl TaskHandle { /// Explicit stop the task and wait util the task exits. - pub async fn stop_task(&self) -> std::result::Result<(), crate::runtime::Error> { + pub async fn stop_task(&self) -> std::result::Result<(), runtime::Error> { self.notify_exit(); let handle = self.handle.lock().await.take(); diff --git a/components/toml_ext/Cargo.toml b/components/toml_ext/Cargo.toml new file mode 100644 index 0000000000..2c1288ff80 --- /dev/null +++ b/components/toml_ext/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "toml_ext" + +[package.version] +workspace = true + +[package.authors] +workspace = true + +[package.edition] +workspace = true + +[dependencies] +macros = { workspace = true } +serde = { workspace = true } +snafu = { workspace = true } +toml = { workspace = true } + +[dev-dependencies] +tempfile = { workspace = true } diff --git a/common_util/src/toml.rs b/components/toml_ext/src/lib.rs similarity index 95% rename from common_util/src/toml.rs rename to components/toml_ext/src/lib.rs index f7c72c3467..60d8ec03ef 100644 --- a/common_util/src/toml.rs +++ b/components/toml_ext/src/lib.rs @@ -1,9 +1,10 @@ -// Copyright 2022 CeresDB Project Authors. Licensed under Apache-2.0. +// Copyright 2022-2023 CeresDB Project Authors. Licensed under Apache-2.0. //! Toml config utilities. use std::{fs::File, io::Read}; +use macros::define_result; use serde::de; use snafu::{Backtrace, ResultExt, Snafu}; diff --git a/df_operator/Cargo.toml b/df_operator/Cargo.toml index 86fe1fc3e8..4cd4fbf116 100644 --- a/df_operator/Cargo.toml +++ b/df_operator/Cargo.toml @@ -16,8 +16,9 @@ base64 = { workspace = true } bincode = "1.3" chrono = { workspace = true } common_types = { workspace = true } -common_util = { workspace = true } datafusion = { workspace = true } +generic_error = { workspace = true } hyperloglog = { git = "https://github.com/jedisct1/rust-hyperloglog.git", rev = "ed1b9b915072ba90c6b93fbfbba30c03215ba682", features = ["with_serde"] } +macros = { workspace = true } smallvec = { workspace = true } snafu = { workspace = true } diff --git a/df_operator/src/aggregate.rs b/df_operator/src/aggregate.rs index 45540c6e1a..756ffdd7b3 100644 --- a/df_operator/src/aggregate.rs +++ b/df_operator/src/aggregate.rs @@ -1,16 +1,17 @@ -// Copyright 2022 CeresDB Project Authors. Licensed under Apache-2.0. +// Copyright 2022-2023 CeresDB Project Authors. Licensed under Apache-2.0. //! Aggregate functions. use std::{fmt, ops::Deref}; use arrow::array::ArrayRef as DfArrayRef; -use common_util::{define_result, error::GenericError}; use datafusion::{ error::{DataFusionError, Result as DfResult}, physical_plan::Accumulator as DfAccumulator, scalar::ScalarValue as DfScalarValue, }; +use generic_error::GenericError; +use macros::define_result; use snafu::Snafu; use crate::functions::{ScalarValue, ScalarValueRef}; diff --git a/df_operator/src/functions.rs b/df_operator/src/functions.rs index 8d918d42b5..0470706847 100644 --- a/df_operator/src/functions.rs +++ b/df_operator/src/functions.rs @@ -1,4 +1,4 @@ -// Copyright 2022 CeresDB Project Authors. Licensed under Apache-2.0. +// Copyright 2022-2023 CeresDB Project Authors. Licensed under Apache-2.0. //! Functions. @@ -9,7 +9,6 @@ use std::{ use arrow::datatypes::DataType; use common_types::{column::ColumnBlock, datum::DatumKind}; -use common_util::{define_result, error::GenericError}; use datafusion::{ error::DataFusionError, logical_expr::{ @@ -19,6 +18,8 @@ use datafusion::{ physical_plan::ColumnarValue as DfColumnarValue, scalar::ScalarValue as DfScalarValue, }; +use generic_error::GenericError; +use macros::define_result; use smallvec::SmallVec; use snafu::{ResultExt, Snafu}; diff --git a/df_operator/src/registry.rs b/df_operator/src/registry.rs index 34e0af7051..b8bdb63902 100644 --- a/df_operator/src/registry.rs +++ b/df_operator/src/registry.rs @@ -1,10 +1,10 @@ -// Copyright 2022 CeresDB Project Authors. Licensed under Apache-2.0. +// Copyright 2022-2023 CeresDB Project Authors. Licensed under Apache-2.0. //! Function registry. use std::{collections::HashMap, sync::Arc}; -use common_util::define_result; +use macros::define_result; use snafu::{ensure, Backtrace, Snafu}; use crate::{scalar::ScalarUdf, udaf::AggregateUdf, udfs}; diff --git a/df_operator/src/udfs/thetasketch_distinct.rs b/df_operator/src/udfs/thetasketch_distinct.rs index 581a41d39c..a9c619804a 100644 --- a/df_operator/src/udfs/thetasketch_distinct.rs +++ b/df_operator/src/udfs/thetasketch_distinct.rs @@ -1,4 +1,4 @@ -// Copyright 2022 CeresDB Project Authors. Licensed under Apache-2.0. +// Copyright 2022-2023 CeresDB Project Authors. Licensed under Apache-2.0. //! thetasketch_distinct() udaf. @@ -6,8 +6,9 @@ use std::fmt; use arrow::datatypes::DataType; use common_types::datum::DatumKind; -use common_util::{define_result, error::BoxError}; +use generic_error::BoxError; use hyperloglog::HyperLogLog; +use macros::define_result; use snafu::{ensure, OptionExt, ResultExt, Snafu}; use crate::{ diff --git a/df_operator/src/udfs/time_bucket.rs b/df_operator/src/udfs/time_bucket.rs index 29d2932aff..9eb13bc3fc 100644 --- a/df_operator/src/udfs/time_bucket.rs +++ b/df_operator/src/udfs/time_bucket.rs @@ -1,4 +1,4 @@ -// Copyright 2022 CeresDB Project Authors. Licensed under Apache-2.0. +// Copyright 2022-2023 CeresDB Project Authors. Licensed under Apache-2.0. //! time_bucket UDF. @@ -10,7 +10,8 @@ use common_types::{ datum::{Datum, DatumKind}, time::Timestamp, }; -use common_util::{define_result, error::BoxError}; +use generic_error::BoxError; +use macros::define_result; use snafu::{ensure, OptionExt, ResultExt, Snafu}; use crate::{ diff --git a/interpreters/Cargo.toml b/interpreters/Cargo.toml index 962cd45fcc..da80389852 100644 --- a/interpreters/Cargo.toml +++ b/interpreters/Cargo.toml @@ -15,12 +15,14 @@ arrow = { workspace = true } # In alphabetical order async-trait = { workspace = true } catalog = { workspace = true } +codec = { workspace = true } common_types = { workspace = true } -common_util = { workspace = true } datafusion = { workspace = true } datafusion-proto = { workspace = true } df_operator = { workspace = true } +generic_error = { workspace = true } log = { workspace = true } +macros = { workspace = true } meta_client = { workspace = true } query_engine = { workspace = true } query_frontend = { workspace = true } @@ -32,4 +34,5 @@ table_engine = { workspace = true } analytic_engine = { workspace = true, features = ["test"] } catalog_impls = { workspace = true } query_frontend = { workspace = true, features = ["test"] } +test_util = { workspace = true } tokio = { workspace = true } diff --git a/interpreters/src/alter_table.rs b/interpreters/src/alter_table.rs index fdc8747bdc..2722e2fef1 100644 --- a/interpreters/src/alter_table.rs +++ b/interpreters/src/alter_table.rs @@ -7,7 +7,7 @@ use common_types::{ column_schema::{self, ColumnSchema}, schema::{self, Schema}, }; -use common_util::define_result; +use macros::define_result; use query_frontend::plan::{AlterTableOperation, AlterTablePlan}; use snafu::{ensure, ResultExt, Snafu}; use table_engine::table::AlterSchemaRequest; diff --git a/interpreters/src/context.rs b/interpreters/src/context.rs index ba6e9cb192..4d9483616a 100644 --- a/interpreters/src/context.rs +++ b/interpreters/src/context.rs @@ -1,10 +1,11 @@ -// Copyright 2022 CeresDB Project Authors. Licensed under Apache-2.0. +// Copyright 2022-2023 CeresDB Project Authors. Licensed under Apache-2.0. //! Interpreter context use std::{sync::Arc, time::Instant}; use common_types::request_id::RequestId; +use macros::define_result; use query_engine::context::{Context as QueryContext, ContextRef as QueryContextRef}; use snafu::Snafu; diff --git a/interpreters/src/create.rs b/interpreters/src/create.rs index 3ce44f1374..2ac84cd94b 100644 --- a/interpreters/src/create.rs +++ b/interpreters/src/create.rs @@ -3,6 +3,7 @@ //! Interpreter for create statements use async_trait::async_trait; +use macros::define_result; use query_frontend::plan::CreateTablePlan; use snafu::{ResultExt, Snafu}; use table_engine::engine::TableEngineRef; diff --git a/interpreters/src/describe.rs b/interpreters/src/describe.rs index cf415cc6e0..f3ae3ddf35 100644 --- a/interpreters/src/describe.rs +++ b/interpreters/src/describe.rs @@ -8,6 +8,7 @@ use arrow::{ record_batch::RecordBatch, }; use async_trait::async_trait; +use macros::define_result; use query_engine::executor::RecordBatchVec; use query_frontend::plan::DescribeTablePlan; use snafu::{ResultExt, Snafu}; diff --git a/interpreters/src/drop.rs b/interpreters/src/drop.rs index 13d4f9e1bc..c710870720 100644 --- a/interpreters/src/drop.rs +++ b/interpreters/src/drop.rs @@ -3,6 +3,7 @@ //! Interpreter for drop statements use async_trait::async_trait; +use macros::define_result; use query_frontend::plan::DropTablePlan; use snafu::{ResultExt, Snafu}; use table_engine::engine::TableEngineRef; diff --git a/interpreters/src/exists.rs b/interpreters/src/exists.rs index 9736589eff..a0b01a16d6 100644 --- a/interpreters/src/exists.rs +++ b/interpreters/src/exists.rs @@ -8,6 +8,7 @@ use arrow::{ record_batch::RecordBatch, }; use async_trait::async_trait; +use macros::define_result; use query_engine::executor::RecordBatchVec; use query_frontend::plan::ExistsTablePlan; use snafu::{ResultExt, Snafu}; diff --git a/interpreters/src/insert.rs b/interpreters/src/insert.rs index 782b07b6f3..649aef5481 100644 --- a/interpreters/src/insert.rs +++ b/interpreters/src/insert.rs @@ -10,6 +10,7 @@ use std::{ use arrow::{array::ArrayRef, error::ArrowError, record_batch::RecordBatch}; use async_trait::async_trait; +use codec::{compact::MemCompactEncoder, Encoder}; use common_types::{ column::{ColumnBlock, ColumnBlockBuilder}, column_schema::ColumnId, @@ -17,7 +18,6 @@ use common_types::{ hash::hash64, row::RowGroup, }; -use common_util::codec::{compact::MemCompactEncoder, Encoder}; use datafusion::{ common::ToDFSchema, error::DataFusionError, @@ -28,6 +28,7 @@ use datafusion::{ }, }; use df_operator::visitor::find_columns_by_expr; +use macros::define_result; use query_frontend::plan::InsertPlan; use snafu::{OptionExt, ResultExt, Snafu}; use table_engine::table::{TableRef, WriteRequest}; @@ -64,9 +65,7 @@ pub enum Error { WriteTable { source: table_engine::table::Error }, #[snafu(display("Failed to encode tsid, err:{}", source))] - EncodeTsid { - source: common_util::codec::compact::Error, - }, + EncodeTsid { source: codec::compact::Error }, #[snafu(display("Failed to convert arrow array to column block, err:{}", source))] ConvertColumnBlock { source: common_types::column::Error }, diff --git a/interpreters/src/interpreter.rs b/interpreters/src/interpreter.rs index 9ae0f73dc1..78e30fba70 100644 --- a/interpreters/src/interpreter.rs +++ b/interpreters/src/interpreter.rs @@ -1,8 +1,9 @@ -// Copyright 2022 CeresDB Project Authors. Licensed under Apache-2.0. +// Copyright 2022-2023 CeresDB Project Authors. Licensed under Apache-2.0. //! Interpreter trait use async_trait::async_trait; +use macros::define_result; use query_engine::executor::RecordBatchVec; use snafu::Snafu; diff --git a/interpreters/src/lib.rs b/interpreters/src/lib.rs index d504bedb6d..8c82cfd674 100644 --- a/interpreters/src/lib.rs +++ b/interpreters/src/lib.rs @@ -1,4 +1,4 @@ -// Copyright 2022 CeresDB Project Authors. Licensed under Apache-2.0. +// Copyright 2022-2023 CeresDB Project Authors. Licensed under Apache-2.0. //! Interpreters of query/insert/update/delete commands //! @@ -6,9 +6,6 @@ #![feature(string_remove_matches)] -#[macro_use] -extern crate common_util; - pub mod alter_table; pub mod context; pub mod create; diff --git a/interpreters/src/select.rs b/interpreters/src/select.rs index 026e819f08..7db558ff03 100644 --- a/interpreters/src/select.rs +++ b/interpreters/src/select.rs @@ -4,6 +4,7 @@ use async_trait::async_trait; use log::debug; +use macros::define_result; use query_engine::executor::{Executor, Query}; use query_frontend::plan::QueryPlan; use snafu::{ResultExt, Snafu}; diff --git a/interpreters/src/show.rs b/interpreters/src/show.rs index f35d3331cf..0f59b33cf6 100644 --- a/interpreters/src/show.rs +++ b/interpreters/src/show.rs @@ -9,6 +9,7 @@ use arrow::{ }; use async_trait::async_trait; use catalog::{manager::ManagerRef, schema::Schema, Catalog}; +use macros::define_result; use query_frontend::{ ast::ShowCreateObject, plan::{QueryType, ShowCreatePlan, ShowPlan, ShowTablesPlan}, diff --git a/interpreters/src/table_manipulator/meta_based.rs b/interpreters/src/table_manipulator/meta_based.rs index e8cd63b1a6..6d58520e66 100644 --- a/interpreters/src/table_manipulator/meta_based.rs +++ b/interpreters/src/table_manipulator/meta_based.rs @@ -2,7 +2,7 @@ use async_trait::async_trait; use common_types::schema::SchemaEncoder; -use common_util::error::BoxError; +use generic_error::BoxError; use log::info; use meta_client::{ types::{CreateTableRequest, DropTableRequest, PartitionTableInfo}, diff --git a/interpreters/src/table_manipulator/mod.rs b/interpreters/src/table_manipulator/mod.rs index edc19e4f29..559567560f 100644 --- a/interpreters/src/table_manipulator/mod.rs +++ b/interpreters/src/table_manipulator/mod.rs @@ -3,7 +3,8 @@ use std::sync::Arc; use async_trait::async_trait; -use common_util::error::GenericError; +use generic_error::GenericError; +use macros::define_result; use query_frontend::plan::{CreateTablePlan, DropTablePlan}; use snafu::{Backtrace, Snafu}; use table_engine::engine::TableEngineRef; diff --git a/interpreters/src/tests.rs b/interpreters/src/tests.rs index 98d1c33a6f..3773b23478 100644 --- a/interpreters/src/tests.rs +++ b/interpreters/src/tests.rs @@ -143,7 +143,7 @@ where "| 7461676b32 | 2021-12-02T07:00:34 | 100.0 | hello3 | 2022-10-11 | 11:10:10.234 |", "+------------+---------------------+--------+--------+------------+--------------+", ]; - common_util::record_batch::assert_record_batches_eq(&expected, records); + test_util::tests::assert_record_batches_eq(&expected, records); let sql = "select count(*) from test_table"; let output = self.sql_to_output(sql).await?; @@ -155,7 +155,7 @@ where "| 2 |", "+-----------------+", ]; - common_util::record_batch::assert_record_batches_eq(&expected, records); + test_util::tests::assert_record_batches_eq(&expected, records); Ok(()) } @@ -182,7 +182,7 @@ where "| field4 | time | false | true | false | false |", "+--------+-----------+------------+-------------+--------+---------------+", ]; - common_util::record_batch::assert_record_batches_eq(&expected, records); + test_util::tests::assert_record_batches_eq(&expected, records); } async fn test_exists_table(&self) { @@ -196,7 +196,7 @@ where "| 1 |", "+--------+", ]; - common_util::record_batch::assert_record_batches_eq(&expected, records); + test_util::tests::assert_record_batches_eq(&expected, records); } async fn test_insert_table(&self) { @@ -262,7 +262,7 @@ where "| 7461676b32 | 2021-12-02T07:00:34 | 10 | 20 | 3 | 10 | 12 |", "+------------+---------------------+--------+--------+--------+--------+--------+", ]; - common_util::record_batch::assert_record_batches_eq(&expected, records); + test_util::tests::assert_record_batches_eq(&expected, records); } async fn test_select_table(&self) { @@ -282,7 +282,7 @@ where "| test_table | CREATE TABLE `test_table` (`key1` varbinary NOT NULL, `key2` timestamp NOT NULL, `field1` double, `field2` string, `field3` date, `field4` time, PRIMARY KEY(key1,key2), TIMESTAMP KEY(key2)) ENGINE=Analytic |", "+------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+" ]; - common_util::record_batch::assert_record_batches_eq(&expected, records); + test_util::tests::assert_record_batches_eq(&expected, records); } async fn test_alter_table(&self) { @@ -338,7 +338,7 @@ where #[tokio::test] async fn test_interpreters_rocks() { - common_util::tests::init_log_for_test(); + test_util::tests::init_log_for_test(); let rocksdb_ctx = RocksDBEngineBuildContext::default(); test_interpreters(rocksdb_ctx).await; } diff --git a/meta_client/Cargo.toml b/meta_client/Cargo.toml index 0ee8e6ae4b..5acee6dbd4 100644 --- a/meta_client/Cargo.toml +++ b/meta_client/Cargo.toml @@ -14,15 +14,17 @@ workspace = true async-trait = { workspace = true } ceresdbproto = { workspace = true } common_types = { workspace = true } -common_util = { workspace = true } futures = { workspace = true } +generic_error = { workspace = true } log = { workspace = true } +macros = { workspace = true } prost = { workspace = true } reqwest = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } snafu = { workspace = true } table_engine = { workspace = true } +time_ext = { workspace = true } tokio = { workspace = true } tonic = { workspace = true } url = "2.2" diff --git a/meta_client/src/lib.rs b/meta_client/src/lib.rs index 13bb5f9c85..ab56cd9136 100644 --- a/meta_client/src/lib.rs +++ b/meta_client/src/lib.rs @@ -3,7 +3,8 @@ use std::sync::Arc; use async_trait::async_trait; -use common_util::{define_result, error::GenericError}; +use generic_error::GenericError; +use macros::define_result; use snafu::{Backtrace, Snafu}; use types::{ AllocSchemaIdRequest, AllocSchemaIdResponse, CreateTableRequest, CreateTableResponse, diff --git a/meta_client/src/load_balance.rs b/meta_client/src/load_balance.rs index 707fb08d98..dbb5c37cd2 100644 --- a/meta_client/src/load_balance.rs +++ b/meta_client/src/load_balance.rs @@ -1,8 +1,8 @@ -// Copyright 2022 CeresDB Project Authors. Licensed under Apache-2.0. +// Copyright 2022-2023 CeresDB Project Authors. Licensed under Apache-2.0. //! Load balancer -use common_util::define_result; +use macros::define_result; use rand::Rng; use snafu::{Backtrace, Snafu}; diff --git a/meta_client/src/meta_impl.rs b/meta_client/src/meta_impl.rs index ebc9b7bd0d..f6370b85ab 100644 --- a/meta_client/src/meta_impl.rs +++ b/meta_client/src/meta_impl.rs @@ -7,10 +7,11 @@ use ceresdbproto::{ common::ResponseHeader, meta_service::{self, ceresmeta_rpc_service_client::CeresmetaRpcServiceClient}, }; -use common_util::{config::ReadableDuration, error::BoxError}; +use generic_error::BoxError; use log::{debug, info}; use serde::{Deserialize, Serialize}; use snafu::{OptionExt, ResultExt}; +use time_ext::ReadableDuration; use crate::{ types::{ diff --git a/meta_client/src/types.rs b/meta_client/src/types.rs index 6c54e29a5e..b47a5325d4 100644 --- a/meta_client/src/types.rs +++ b/meta_client/src/types.rs @@ -8,10 +8,11 @@ use common_types::{ schema::{SchemaId, SchemaName}, table::{TableId, TableName}, }; -use common_util::{config::ReadableDuration, error::BoxError}; +use generic_error::BoxError; use serde::{Deserialize, Serialize}; use snafu::{OptionExt, ResultExt}; use table_engine::partition::PartitionInfo; +use time_ext::ReadableDuration; use crate::{Convert, Error, MissingShardInfo, MissingTableInfo, Result}; pub type ClusterNodesRef = Arc>; diff --git a/partition_table_engine/Cargo.toml b/partition_table_engine/Cargo.toml index 1a69ad327b..36350357a9 100644 --- a/partition_table_engine/Cargo.toml +++ b/partition_table_engine/Cargo.toml @@ -13,9 +13,10 @@ workspace = true [dependencies] async-trait = { workspace = true } common_types = { workspace = true } -common_util = { workspace = true } futures = { workspace = true } +generic_error = { workspace = true } lazy_static = { workspace = true } +macros = { workspace = true } prometheus = { workspace = true } snafu = { workspace = true } table_engine = { workspace = true } diff --git a/partition_table_engine/src/error.rs b/partition_table_engine/src/error.rs index 1b18e79e46..10e7175868 100644 --- a/partition_table_engine/src/error.rs +++ b/partition_table_engine/src/error.rs @@ -1,6 +1,7 @@ // Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. -use common_util::{define_result, error::GenericError}; +use generic_error::GenericError; +use macros::define_result; use snafu::Snafu; define_result!(Error); diff --git a/partition_table_engine/src/lib.rs b/partition_table_engine/src/lib.rs index 1cebd9aff5..ac5dfb02a0 100644 --- a/partition_table_engine/src/lib.rs +++ b/partition_table_engine/src/lib.rs @@ -9,7 +9,7 @@ mod partition; use std::sync::Arc; use async_trait::async_trait; -use common_util::error::BoxError; +use generic_error::BoxError; use snafu::{OptionExt, ResultExt}; use table_engine::{ engine::{ diff --git a/partition_table_engine/src/partition.rs b/partition_table_engine/src/partition.rs index 8a6160fadc..2b286065e5 100644 --- a/partition_table_engine/src/partition.rs +++ b/partition_table_engine/src/partition.rs @@ -9,8 +9,8 @@ use common_types::{ row::{Row, RowGroupBuilder}, schema::Schema, }; -use common_util::error::BoxError; use futures::{stream::FuturesUnordered, StreamExt}; +use generic_error::BoxError; use snafu::ResultExt; use table_engine::{ partition::{ diff --git a/proxy/Cargo.toml b/proxy/Cargo.toml index 07ff869ddb..5c4f9791bf 100644 --- a/proxy/Cargo.toml +++ b/proxy/Cargo.toml @@ -20,10 +20,10 @@ ceresdbproto = { workspace = true } clru = { workspace = true } cluster = { workspace = true } common_types = { workspace = true } -common_util = { workspace = true } datafusion = { workspace = true } df_operator = { workspace = true } futures = { workspace = true } +generic_error = { workspace = true } http = "0.2" influxdb-line-protocol = "1.0" influxql-query = { workspace = true } @@ -31,7 +31,7 @@ interpreters = { workspace = true } itertools = { workspace = true } lazy_static = { workspace = true } log = { workspace = true } -logger = { workspace = true } +macros = { workspace = true } meta_client = { workspace = true } paste = { workspace = true } prom-remote-api = { workspace = true, features = ["warp"] } @@ -41,12 +41,15 @@ prost = { workspace = true } query_engine = { workspace = true } query_frontend = { workspace = true } router = { workspace = true } +runtime = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } snafu = { workspace = true } spin = { workspace = true } sqlparser = { workspace = true } table_engine = { workspace = true } +time_ext = { workspace = true } +timed_task = { workspace = true } tokio = { workspace = true } tokio-stream = { version = "0.1", features = ["net"] } tonic = { workspace = true } diff --git a/proxy/src/context.rs b/proxy/src/context.rs index 8745e3043d..21be9864d2 100644 --- a/proxy/src/context.rs +++ b/proxy/src/context.rs @@ -4,7 +4,7 @@ use std::time::Duration; -use common_util::define_result; +use macros::define_result; use snafu::{ensure, Backtrace, Snafu}; #[allow(clippy::enum_variant_names)] diff --git a/proxy/src/error.rs b/proxy/src/error.rs index db9f36acd1..4000803d1d 100644 --- a/proxy/src/error.rs +++ b/proxy/src/error.rs @@ -1,8 +1,9 @@ // Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. use ceresdbproto::common::ResponseHeader; -use common_util::{define_result, error::GenericError}; +use generic_error::GenericError; use http::StatusCode; +use macros::define_result; use snafu::{Backtrace, Snafu}; use crate::error_util; diff --git a/proxy/src/forward.rs b/proxy/src/forward.rs index e1765dbfda..75ba715c07 100644 --- a/proxy/src/forward.rs +++ b/proxy/src/forward.rs @@ -11,11 +11,12 @@ use async_trait::async_trait; use ceresdbproto::storage::{ storage_service_client::StorageServiceClient, RequestContext, RouteRequest, }; -use common_util::{config::ReadableDuration, define_result}; use log::{debug, error, warn}; +use macros::define_result; use router::{endpoint::Endpoint, RouterRef}; use serde::{Deserialize, Serialize}; use snafu::{Backtrace, ResultExt, Snafu}; +use time_ext::ReadableDuration; use tonic::{ metadata::errors::InvalidMetadataValue, transport::{self, Channel}, diff --git a/proxy/src/grpc/prom_query.rs b/proxy/src/grpc/prom_query.rs index 8368921fc0..205ef30707 100644 --- a/proxy/src/grpc/prom_query.rs +++ b/proxy/src/grpc/prom_query.rs @@ -17,7 +17,7 @@ use common_types::{ request_id::RequestId, schema::{RecordSchema, TSID_COLUMN}, }; -use common_util::error::BoxError; +use generic_error::BoxError; use http::StatusCode; use interpreters::interpreter::Output; use log::info; diff --git a/proxy/src/grpc/sql_query.rs b/proxy/src/grpc/sql_query.rs index e40bbee236..496b94b923 100644 --- a/proxy/src/grpc/sql_query.rs +++ b/proxy/src/grpc/sql_query.rs @@ -13,8 +13,8 @@ use ceresdbproto::{ }, }; use common_types::record_batch::RecordBatch; -use common_util::error::BoxError; use futures::{stream, stream::BoxStream, FutureExt, StreamExt}; +use generic_error::BoxError; use http::StatusCode; use interpreters::interpreter::Output; use log::{error, warn}; diff --git a/proxy/src/handlers/error.rs b/proxy/src/handlers/error.rs index f810b4d337..91bf84f5b6 100644 --- a/proxy/src/handlers/error.rs +++ b/proxy/src/handlers/error.rs @@ -2,7 +2,7 @@ //! Error of handlers -use common_util::define_result; +use macros::define_result; use snafu::{Backtrace, Snafu}; use warp::reject::Reject; diff --git a/proxy/src/hotspot.rs b/proxy/src/hotspot.rs index 32904b3d5a..235beb7d67 100644 --- a/proxy/src/hotspot.rs +++ b/proxy/src/hotspot.rs @@ -6,10 +6,11 @@ use std::{fmt::Write, sync::Arc, time::Duration}; use ceresdbproto::storage::{ PrometheusQueryRequest, RequestContext, SqlQueryRequest, WriteRequest, }; -use common_util::{runtime::Runtime, timed_task::TimedTask}; use log::{info, warn}; +use runtime::Runtime; use serde::{Deserialize, Serialize}; use spin::Mutex as SpinMutex; +use timed_task::TimedTask; use tokio::sync::mpsc::{self, Sender}; use crate::{hotspot_lru::HotspotLru, util}; @@ -308,7 +309,7 @@ mod test { WriteTableRequest, }, }; - use common_util::runtime::Builder; + use runtime::Builder; fn new_runtime() -> Arc { let runtime = Builder::default() diff --git a/proxy/src/http/prom.rs b/proxy/src/http/prom.rs index 4d90bdf8b4..56c717aa97 100644 --- a/proxy/src/http/prom.rs +++ b/proxy/src/http/prom.rs @@ -22,7 +22,7 @@ use common_types::{ request_id::RequestId, schema::{RecordSchema, TSID_COLUMN}, }; -use common_util::{error::BoxError, time::InstantExt}; +use generic_error::BoxError; use http::StatusCode; use interpreters::interpreter::Output; use log::debug; @@ -37,6 +37,7 @@ use query_frontend::{ provider::CatalogMetaProvider, }; use snafu::{ensure, OptionExt, ResultExt}; +use time_ext::InstantExt; use warp::reject; use crate::{ diff --git a/proxy/src/http/sql.rs b/proxy/src/http/sql.rs index 67ec394042..1116931aba 100644 --- a/proxy/src/http/sql.rs +++ b/proxy/src/http/sql.rs @@ -11,7 +11,7 @@ use common_types::{ datum::{Datum, DatumKind}, record_batch::RecordBatch, }; -use common_util::error::BoxError; +use generic_error::BoxError; use interpreters::interpreter::Output; use query_engine::executor::{Executor as QueryExecutor, RecordBatchVec}; use serde::{ diff --git a/proxy/src/influxdb/mod.rs b/proxy/src/influxdb/mod.rs index 142d7bcc6b..93fcbcae26 100644 --- a/proxy/src/influxdb/mod.rs +++ b/proxy/src/influxdb/mod.rs @@ -12,7 +12,7 @@ use ceresdbproto::storage::{ RequestContext as GrpcRequestContext, WriteRequest as GrpcWriteRequest, }; use common_types::request_id::RequestId; -use common_util::{error::BoxError, time::InstantExt}; +use generic_error::BoxError; use http::StatusCode; use interpreters::interpreter::Output; use log::{debug, info}; @@ -22,6 +22,7 @@ use query_frontend::{ provider::CatalogMetaProvider, }; use snafu::{ensure, ResultExt}; +use time_ext::InstantExt; use crate::{ context::RequestContext, diff --git a/proxy/src/influxdb/types.rs b/proxy/src/influxdb/types.rs index d090842b07..05dd441b3a 100644 --- a/proxy/src/influxdb/types.rs +++ b/proxy/src/influxdb/types.rs @@ -12,7 +12,7 @@ use common_types::{ column_schema::ColumnSchema, datum::Datum, record_batch::RecordBatch, schema::RecordSchema, time::Timestamp, }; -use common_util::error::BoxError; +use generic_error::BoxError; use http::Method; use influxdb_line_protocol::FieldValue; use interpreters::interpreter::Output; diff --git a/proxy/src/lib.rs b/proxy/src/lib.rs index 1bf8da40f6..c6775c24f3 100644 --- a/proxy/src/lib.rs +++ b/proxy/src/lib.rs @@ -44,16 +44,12 @@ use ceresdbproto::storage::{ PrometheusRemoteQueryResponse, Route, RouteRequest, }; use common_types::{request_id::RequestId, table::DEFAULT_SHARD_ID, ENABLE_TTL, TTL}; -use common_util::{ - error::BoxError, - runtime::Runtime, - time::{current_time_millis, parse_duration}, -}; use datafusion::{ prelude::{Column, Expr}, scalar::ScalarValue, }; use futures::FutureExt; +use generic_error::BoxError; use influxql_query::logical_optimizer::range_predicate::find_time_range; use interpreters::{ context::Context as InterpreterContext, @@ -64,6 +60,7 @@ use log::{error, info}; use query_engine::executor::Executor as QueryExecutor; use query_frontend::plan::Plan; use router::{endpoint::Endpoint, Router}; +use runtime::Runtime; use snafu::{OptionExt, ResultExt}; use table_engine::{ engine::{EngineRuntimes, TableState}, @@ -71,6 +68,7 @@ use table_engine::{ table::{TableId, TableRef}, PARTITION_TABLE_ENGINE_TYPE, }; +use time_ext::{current_time_millis, parse_duration}; use tonic::{transport::Channel, IntoRequest}; use crate::{ diff --git a/proxy/src/limiter.rs b/proxy/src/limiter.rs index d1ae663e21..95a74f89e9 100644 --- a/proxy/src/limiter.rs +++ b/proxy/src/limiter.rs @@ -2,8 +2,8 @@ use std::{collections::HashSet, sync::RwLock}; -use common_util::define_result; use datafusion::logical_expr::logical_plan::LogicalPlan; +use macros::define_result; use query_frontend::plan::Plan; use serde::{Deserialize, Serialize}; use snafu::{Backtrace, Snafu}; diff --git a/proxy/src/opentsdb/types.rs b/proxy/src/opentsdb/types.rs index ca9cce9c9d..0018f19a6e 100644 --- a/proxy/src/opentsdb/types.rs +++ b/proxy/src/opentsdb/types.rs @@ -9,11 +9,12 @@ use bytes::Bytes; use ceresdbproto::storage::{ value, Field, FieldGroup, Tag, Value as ProtoValue, WriteSeriesEntry, WriteTableRequest, }; -use common_util::{error::BoxError, time::try_to_millis}; +use generic_error::BoxError; use http::StatusCode; use serde::Deserialize; use serde_json::from_slice; use snafu::{OptionExt, ResultExt}; +use time_ext::try_to_millis; use crate::error::{ErrNoCause, ErrWithCause, Result}; diff --git a/proxy/src/read.rs b/proxy/src/read.rs index d29e50480e..0eace2b73a 100644 --- a/proxy/src/read.rs +++ b/proxy/src/read.rs @@ -8,8 +8,8 @@ use ceresdbproto::storage::{ storage_service_client::StorageServiceClient, RequestContext, SqlQueryRequest, SqlQueryResponse, }; use common_types::request_id::RequestId; -use common_util::{error::BoxError, time::InstantExt}; use futures::FutureExt; +use generic_error::BoxError; use http::StatusCode; use interpreters::interpreter::Output; use log::{error, info, warn}; @@ -21,6 +21,7 @@ use query_frontend::{ }; use router::endpoint::Endpoint; use snafu::{ensure, ResultExt}; +use time_ext::InstantExt; use tonic::{transport::Channel, IntoRequest}; use crate::{ diff --git a/proxy/src/schema_config_provider/mod.rs b/proxy/src/schema_config_provider/mod.rs index 2b3da3c1a4..3de7a5edf8 100644 --- a/proxy/src/schema_config_provider/mod.rs +++ b/proxy/src/schema_config_provider/mod.rs @@ -5,7 +5,7 @@ use std::sync::Arc; use cluster::config::SchemaConfig; -use common_util::define_result; +use macros::define_result; use snafu::Snafu; pub mod cluster_based; diff --git a/proxy/src/write.rs b/proxy/src/write.rs index fc2f88d4d9..08be43edfe 100644 --- a/proxy/src/write.rs +++ b/proxy/src/write.rs @@ -22,8 +22,8 @@ use common_types::{ schema::Schema, time::Timestamp, }; -use common_util::error::BoxError; use futures::{future::BoxFuture, stream::FuturesUnordered, FutureExt, StreamExt}; +use generic_error::BoxError; use http::StatusCode; use interpreters::interpreter::Output; use log::{debug, error, info}; @@ -45,8 +45,7 @@ use crate::{ Context, Proxy, }; -type WriteResponseFutures<'a> = - Vec>>>; +type WriteResponseFutures<'a> = Vec>>>; #[derive(Debug)] pub struct WriteContext { @@ -391,7 +390,7 @@ impl Proxy { async fn collect_write_response( &self, - futures: Vec>>>, + futures: Vec>>>, ) -> Result { let mut futures: FuturesUnordered<_> = futures.into_iter().collect(); let mut success = 0; diff --git a/query_engine/Cargo.toml b/query_engine/Cargo.toml index 796a4433ba..1e9c65d498 100644 --- a/query_engine/Cargo.toml +++ b/query_engine/Cargo.toml @@ -16,14 +16,16 @@ arrow = { workspace = true } async-trait = { workspace = true } chrono = { workspace = true } common_types = { workspace = true } -common_util = { workspace = true } datafusion = { workspace = true } df_operator = { workspace = true } futures = { workspace = true } +generic_error = { workspace = true } influxql-query = { workspace = true } log = { workspace = true } +macros = { workspace = true } query_frontend = { workspace = true } serde = { workspace = true } snafu = { workspace = true } table_engine = { workspace = true } +time_ext = { workspace = true } tokio = { workspace = true } diff --git a/query_engine/src/df_execution_extension/prom_align.rs b/query_engine/src/df_execution_extension/prom_align.rs index 305abcbc6b..a7eccea8e5 100644 --- a/query_engine/src/df_execution_extension/prom_align.rs +++ b/query_engine/src/df_execution_extension/prom_align.rs @@ -31,6 +31,7 @@ use datafusion::{ }; use futures::{Stream, StreamExt}; use log::debug; +use macros::define_result; use query_frontend::promql::{AlignParameter, ColumnNames, Func as PromFunc}; use snafu::{OptionExt, ResultExt, Snafu}; @@ -943,7 +944,7 @@ impl AlignFunc for IrateFunc { } } -/// This function is not in Promtheus' functions list. +/// This function is not in Prometheus' functions list. /// /// It simulates the behavior of `Instant Selector` by finding the newest point /// from the input. Thus `Instant Selector` can be represented by [PromAlignOp] diff --git a/query_engine/src/executor.rs b/query_engine/src/executor.rs index 2fb29a3541..3be3aeeb6d 100644 --- a/query_engine/src/executor.rs +++ b/query_engine/src/executor.rs @@ -6,13 +6,14 @@ use std::{sync::Arc, time::Instant}; use async_trait::async_trait; use common_types::record_batch::RecordBatch; -use common_util::time::InstantExt; use datafusion::prelude::SessionContext; use futures::TryStreamExt; use log::{debug, info}; +use macros::define_result; use query_frontend::{plan::QueryPlan, provider::CatalogProviderAdapter}; use snafu::{Backtrace, ResultExt, Snafu}; use table_engine::stream::SendableRecordBatchStream; +use time_ext::InstantExt; use crate::{ config::Config, diff --git a/query_engine/src/lib.rs b/query_engine/src/lib.rs index 56cc58c55c..e466752663 100644 --- a/query_engine/src/lib.rs +++ b/query_engine/src/lib.rs @@ -1,4 +1,4 @@ -// Copyright 2022 CeresDB Project Authors. Licensed under Apache-2.0. +// Copyright 2022-2023 CeresDB Project Authors. Licensed under Apache-2.0. //! Query engine //! @@ -7,9 +7,6 @@ // TODO(yingwen): Maybe renamed to query_executor or query_backend? // TODO(yingwen): Use datafusion or fuse-query as query backend -#[macro_use] -extern crate common_util; - pub mod config; pub mod context; pub mod df_execution_extension; diff --git a/query_engine/src/logical_optimizer/mod.rs b/query_engine/src/logical_optimizer/mod.rs index 6ed4bc7e36..3cb6612f4d 100644 --- a/query_engine/src/logical_optimizer/mod.rs +++ b/query_engine/src/logical_optimizer/mod.rs @@ -8,6 +8,7 @@ pub mod tests; pub mod type_conversion; use datafusion::{error::DataFusionError, prelude::SessionContext}; +use macros::define_result; use query_frontend::plan::QueryPlan; use snafu::{Backtrace, ResultExt, Snafu}; diff --git a/query_engine/src/physical_optimizer/mod.rs b/query_engine/src/physical_optimizer/mod.rs index 96007d0409..6724309aac 100644 --- a/query_engine/src/physical_optimizer/mod.rs +++ b/query_engine/src/physical_optimizer/mod.rs @@ -9,6 +9,7 @@ use datafusion::{ error::DataFusionError, physical_optimizer::optimizer::PhysicalOptimizerRule, prelude::SessionContext, }; +use macros::define_result; use query_frontend::plan::QueryPlan; use snafu::{Backtrace, ResultExt, Snafu}; diff --git a/query_engine/src/physical_plan.rs b/query_engine/src/physical_plan.rs index c128a0da21..a72de74c1f 100644 --- a/query_engine/src/physical_plan.rs +++ b/query_engine/src/physical_plan.rs @@ -1,4 +1,4 @@ -// Copyright 2022 CeresDB Project Authors. Licensed under Apache-2.0. +// Copyright 2022-2023 CeresDB Project Authors. Licensed under Apache-2.0. //! Physical execution plan @@ -17,6 +17,7 @@ use datafusion::{ }, prelude::SessionContext, }; +use macros::define_result; use snafu::{Backtrace, ResultExt, Snafu}; use table_engine::stream::{FromDfStream, SendableRecordBatchStream}; diff --git a/query_frontend/Cargo.toml b/query_frontend/Cargo.toml index 9832c23f29..4107d303db 100644 --- a/query_frontend/Cargo.toml +++ b/query_frontend/Cargo.toml @@ -20,11 +20,12 @@ async-trait = { workspace = true } catalog = { workspace = true } ceresdbproto = { workspace = true } cluster = { workspace = true } +codec = { workspace = true } common_types = { workspace = true } -common_util = { workspace = true } datafusion = { workspace = true } datafusion-proto = { workspace = true } df_operator = { workspace = true } +generic_error = { workspace = true } hashbrown = { version = "0.12", features = ["raw"] } influxql-logical-planner = { workspace = true } influxql-parser = { workspace = true } @@ -32,6 +33,7 @@ influxql-schema = { workspace = true } itertools = { workspace = true } lazy_static = { workspace = true } log = { workspace = true } +macros = { workspace = true } paste = { workspace = true } prom-remote-api = { workspace = true } regex = "1" diff --git a/query_frontend/src/frontend.rs b/query_frontend/src/frontend.rs index ecdf8e17f2..826ed25994 100644 --- a/query_frontend/src/frontend.rs +++ b/query_frontend/src/frontend.rs @@ -7,8 +7,9 @@ use std::{sync::Arc, time::Instant}; use ceresdbproto::{prometheus::Expr as PromExpr, storage::WriteTableRequest}; use cluster::config::SchemaConfig; use common_types::request_id::RequestId; -use common_util::error::GenericError; +use generic_error::GenericError; use influxql_parser::statement::Statement as InfluxqlStatement; +use macros::define_result; use prom_remote_api::types::Query as PromRemoteQuery; use snafu::{Backtrace, OptionExt, ResultExt, Snafu}; use sqlparser::ast::{SetExpr, Statement as SqlStatement, TableFactor}; diff --git a/query_frontend/src/influxql/mod.rs b/query_frontend/src/influxql/mod.rs index b5b6e625eb..3f47cb7a6d 100644 --- a/query_frontend/src/influxql/mod.rs +++ b/query_frontend/src/influxql/mod.rs @@ -5,7 +5,8 @@ pub mod planner; pub mod error { - use common_util::error::GenericError; + use generic_error::GenericError; + use macros::define_result; use snafu::{Backtrace, Snafu}; #[derive(Debug, Snafu)] @@ -35,5 +36,6 @@ pub mod error { #[snafu(display("Unimplemented influxql statement, msg:{}", msg))] Unimplemented { msg: String }, } + define_result!(Error); } diff --git a/query_frontend/src/influxql/planner.rs b/query_frontend/src/influxql/planner.rs index bfa9fbbf47..d83324f158 100644 --- a/query_frontend/src/influxql/planner.rs +++ b/query_frontend/src/influxql/planner.rs @@ -5,10 +5,10 @@ use std::{cell::OnceCell, sync::Arc}; use arrow::datatypes::SchemaRef as ArrowSchemaRef; -use common_util::error::BoxError; use datafusion::{ error::DataFusionError, logical_expr::TableSource, sql::planner::ContextProvider, }; +use generic_error::BoxError; use influxql_logical_planner::plan::{ ceresdb_schema_to_influxdb, InfluxQLToLogicalPlan, SchemaProvider, }; diff --git a/query_frontend/src/lib.rs b/query_frontend/src/lib.rs index 360cf7545c..bc234eec2d 100644 --- a/query_frontend/src/lib.rs +++ b/query_frontend/src/lib.rs @@ -5,9 +5,6 @@ //! //! Parse sql into logical plan that can be handled by interpreters -#[macro_use] -extern crate common_util; - pub mod ast; pub mod container; pub mod frontend; diff --git a/query_frontend/src/parser.rs b/query_frontend/src/parser.rs index 77f4c35056..a5381c2eec 100644 --- a/query_frontend/src/parser.rs +++ b/query_frontend/src/parser.rs @@ -5,6 +5,7 @@ //! Some codes are copied from datafusion: use log::debug; +use macros::define_result; use paste::paste; use sqlparser::{ ast::{ diff --git a/query_frontend/src/partition.rs b/query_frontend/src/partition.rs index 3b4950346f..d8965d6066 100644 --- a/query_frontend/src/partition.rs +++ b/query_frontend/src/partition.rs @@ -2,9 +2,9 @@ //! Parse partition statement to partition info -use common_util::error::BoxError; use datafusion::{logical_expr::Expr, prelude::Column}; use datafusion_proto::bytes::Serializeable; +use generic_error::BoxError; use snafu::ResultExt; use sqlparser::ast::Expr as SqlExpr; use table_engine::partition::{ diff --git a/query_frontend/src/plan.rs b/query_frontend/src/plan.rs index da7c236eea..d9179d0d4c 100644 --- a/query_frontend/src/plan.rs +++ b/query_frontend/src/plan.rs @@ -10,10 +10,10 @@ use std::{ }; use common_types::{column_schema::ColumnSchema, row::RowGroup, schema::Schema}; -use common_util::define_result; use datafusion::logical_expr::{ expr::Expr as DfLogicalExpr, logical_plan::LogicalPlan as DataFusionLogicalPlan, }; +use macros::define_result; use snafu::Snafu; use table_engine::{partition::PartitionInfo, table::TableRef}; diff --git a/query_frontend/src/planner.rs b/query_frontend/src/planner.rs index de8327da70..b355f6cd1b 100644 --- a/query_frontend/src/planner.rs +++ b/query_frontend/src/planner.rs @@ -26,7 +26,6 @@ use common_types::{ row::{RowGroup, RowGroupBuilder}, schema::{self, Builder as SchemaBuilder, Schema, TSID_COLUMN}, }; -use common_util::error::GenericError; use datafusion::{ common::{DFField, DFSchema}, error::DataFusionError, @@ -37,8 +36,10 @@ use datafusion::{ ResolvedTableReference, }, }; +use generic_error::GenericError; use influxql_parser::statement::Statement as InfluxqlStatement; use log::{debug, trace}; +use macros::define_result; use prom_remote_api::types::Query as PromRemoteQuery; use snafu::{ensure, Backtrace, OptionExt, ResultExt, Snafu}; use sqlparser::ast::{ diff --git a/query_frontend/src/promql/error.rs b/query_frontend/src/promql/error.rs index 221503f076..e558ef31f5 100644 --- a/query_frontend/src/promql/error.rs +++ b/query_frontend/src/promql/error.rs @@ -2,6 +2,7 @@ use ceresdbproto::prometheus::sub_expr::OperatorType; use datafusion::error::DataFusionError; +use macros::define_result; use snafu::{Backtrace, Snafu}; #[allow(clippy::large_enum_variant)] diff --git a/query_frontend/src/promql/pushdown.rs b/query_frontend/src/promql/pushdown.rs index fbb09c6b79..9f708e45d4 100644 --- a/query_frontend/src/promql/pushdown.rs +++ b/query_frontend/src/promql/pushdown.rs @@ -3,6 +3,7 @@ use std::convert::TryFrom; use common_types::time::{TimeRange, Timestamp}; +use macros::define_result; use snafu::Snafu; #[derive(Debug, Snafu)] diff --git a/query_frontend/src/promql/udf.rs b/query_frontend/src/promql/udf.rs index 1f6ce79799..71429cfa31 100644 --- a/query_frontend/src/promql/udf.rs +++ b/query_frontend/src/promql/udf.rs @@ -9,8 +9,8 @@ use arrow::{ array::{ArrayRef, BooleanArray, StringArray, UInt64Array}, datatypes::DataType, }; +use codec::{compact::MemCompactEncoder, Encoder}; use common_types::hash::hash64; -use common_util::codec::{compact::MemCompactEncoder, Encoder}; use datafusion::{ error::{DataFusionError, Result as DataFusionResult}, logical_expr::{create_udf, Expr, Volatility}, diff --git a/query_frontend/src/provider.rs b/query_frontend/src/provider.rs index b8033ab13b..19cd84c470 100644 --- a/query_frontend/src/provider.rs +++ b/query_frontend/src/provider.rs @@ -16,6 +16,7 @@ use datafusion::{ sql::planner::ContextProvider, }; use df_operator::{registry::FunctionRegistry, scalar::ScalarUdf, udaf::AggregateUdf}; +use macros::define_result; use snafu::{OptionExt, ResultExt, Snafu}; use table_engine::{provider::TableProviderAdapter, table::TableRef}; diff --git a/remote_engine_client/Cargo.toml b/remote_engine_client/Cargo.toml index 425d9d01d2..f41cc091a1 100644 --- a/remote_engine_client/Cargo.toml +++ b/remote_engine_client/Cargo.toml @@ -15,12 +15,15 @@ arrow_ext = { workspace = true } async-trait = { workspace = true } ceresdbproto = { workspace = true } common_types = { workspace = true } -common_util = { workspace = true } futures = { workspace = true } +generic_error = { workspace = true } log = { workspace = true } +macros = { workspace = true } router = { workspace = true } +runtime = { workspace = true } serde = { workspace = true } snafu = { workspace = true } table_engine = { workspace = true } +time_ext = { workspace = true } tokio = { workspace = true } tonic = { workspace = true } diff --git a/remote_engine_client/src/client.rs b/remote_engine_client/src/client.rs index 333bf05617..a8090ae359 100644 --- a/remote_engine_client/src/client.rs +++ b/remote_engine_client/src/client.rs @@ -20,10 +20,11 @@ use ceresdbproto::{ use common_types::{ projected_schema::ProjectedSchema, record_batch::RecordBatch, schema::RecordSchema, }; -use common_util::{error::BoxError, runtime::Runtime}; use futures::{Stream, StreamExt}; +use generic_error::BoxError; use log::info; use router::RouterRef; +use runtime::Runtime; use snafu::{ensure, OptionExt, ResultExt}; use table_engine::{ remote::model::{ diff --git a/remote_engine_client/src/config.rs b/remote_engine_client/src/config.rs index dac2d4dea1..b898b7591b 100644 --- a/remote_engine_client/src/config.rs +++ b/remote_engine_client/src/config.rs @@ -1,12 +1,12 @@ -// Copyright 2022 CeresDB Project Authors. Licensed under Apache-2.0. +// Copyright 2022-2023 CeresDB Project Authors. Licensed under Apache-2.0. //! Config for [Client] use std::str::FromStr; use arrow_ext::ipc::CompressOptions; -use common_util::config::ReadableDuration; use serde::{Deserialize, Serialize}; +use time_ext::ReadableDuration; #[derive(Debug, Clone, Deserialize, Serialize)] #[serde(default)] diff --git a/remote_engine_client/src/lib.rs b/remote_engine_client/src/lib.rs index 902b33c96b..de94965ce6 100644 --- a/remote_engine_client/src/lib.rs +++ b/remote_engine_client/src/lib.rs @@ -18,10 +18,11 @@ use std::{ use async_trait::async_trait; use common_types::{record_batch::RecordBatch, schema::RecordSchema}; -use common_util::{error::BoxError, runtime::Runtime}; pub use config::Config; use futures::{Stream, StreamExt}; +use generic_error::BoxError; use router::RouterRef; +use runtime::Runtime; use snafu::ResultExt; use table_engine::{ remote::{ @@ -35,7 +36,8 @@ use table_engine::{ use self::client::{Client, ClientReadRecordBatchStream}; pub mod error { - use common_util::{define_result, error::GenericError}; + use generic_error::GenericError; + use macros::define_result; use snafu::{Backtrace, Snafu}; use table_engine::remote::model::TableIdentifier; diff --git a/router/Cargo.toml b/router/Cargo.toml index 2a7aaa5922..40c19f64fc 100644 --- a/router/Cargo.toml +++ b/router/Cargo.toml @@ -15,12 +15,14 @@ async-trait = { workspace = true } ceresdbproto = { workspace = true } cluster = { workspace = true } common_types = { workspace = true } -common_util = { workspace = true } +generic_error = { workspace = true } log = { workspace = true } +macros = { workspace = true } meta_client = { workspace = true } moka = { version = "0.10", features = ["future"] } serde = { workspace = true } snafu = { workspace = true } table_engine = { workspace = true } +time_ext = { workspace = true } tokio = { workspace = true } twox-hash = "1.6" diff --git a/router/src/cluster_based.rs b/router/src/cluster_based.rs index 59820003b7..b9b8e73b1f 100644 --- a/router/src/cluster_based.rs +++ b/router/src/cluster_based.rs @@ -5,7 +5,7 @@ use async_trait::async_trait; use ceresdbproto::storage::{Route, RouteRequest}; use cluster::ClusterRef; -use common_util::error::BoxError; +use generic_error::BoxError; use log::trace; use meta_client::types::RouteTablesRequest; use moka::future::Cache; @@ -167,10 +167,10 @@ mod tests { shard_lock_manager::ShardLockManagerRef, shard_set::ShardRef, Cluster, ClusterNodesResp, }; use common_types::table::ShardId; - use common_util::config::ReadableDuration; use meta_client::types::{ NodeShard, RouteEntry, RouteTablesResponse, ShardInfo, ShardRole::Leader, TableInfo, }; + use time_ext::ReadableDuration; use super::*; diff --git a/router/src/endpoint.rs b/router/src/endpoint.rs index 91d8784fc7..246965e073 100644 --- a/router/src/endpoint.rs +++ b/router/src/endpoint.rs @@ -1,11 +1,11 @@ -// Copyright 2022 CeresDB Project Authors. Licensed under Apache-2.0. +// Copyright 2022-2023 CeresDB Project Authors. Licensed under Apache-2.0. //! Endpoint definition use std::str::FromStr; use ceresdbproto::storage; -use common_util::error::GenericError; +use generic_error::GenericError; use serde::{Deserialize, Serialize}; #[derive(Debug, Clone, Deserialize, PartialEq, Eq, Hash, Serialize)] diff --git a/router/src/lib.rs b/router/src/lib.rs index 61ccbde0c7..4617d09fa2 100644 --- a/router/src/lib.rs +++ b/router/src/lib.rs @@ -9,11 +9,12 @@ use std::{sync::Arc, time::Duration}; use async_trait::async_trait; use ceresdbproto::storage::{Route, RouteRequest}; pub use cluster_based::ClusterBasedRouter; -use common_util::{config::ReadableDuration, define_result}; +use macros::define_result; use meta_client::types::TableInfo; pub use rule_based::{RuleBasedRouter, RuleList}; use serde::{Deserialize, Serialize}; use snafu::{Backtrace, Snafu}; +use time_ext::ReadableDuration; #[derive(Snafu, Debug)] #[snafu(visibility(pub))] diff --git a/server/Cargo.toml b/server/Cargo.toml index eb5c523af5..cc9939384b 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -21,16 +21,17 @@ ceresdbproto = { workspace = true } clru = { workspace = true } cluster = { workspace = true } common_types = { workspace = true } -common_util = { workspace = true } datafusion = { workspace = true } df_operator = { workspace = true } futures = { workspace = true } +generic_error = { workspace = true } http = "0.2" influxdb-line-protocol = "1.0" interpreters = { workspace = true } lazy_static = { workspace = true } log = { workspace = true } logger = { workspace = true } +macros = { workspace = true } meta_client = { workspace = true } opensrv-mysql = "0.1.0" partition_table_engine = { workspace = true } @@ -45,12 +46,15 @@ query_engine = { workspace = true } query_frontend = { workspace = true } remote_engine_client = { workspace = true } router = { workspace = true } +runtime = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } +size_ext = { workspace = true } snafu = { workspace = true } spin = { workspace = true } sqlparser = { workspace = true } table_engine = { workspace = true } +time_ext = { workspace = true } tokio = { workspace = true } tokio-stream = { version = "0.1", features = ["net"] } tonic = { workspace = true } diff --git a/server/src/config.rs b/server/src/config.rs index 9bbf4b0af6..318d9b0f1d 100644 --- a/server/src/config.rs +++ b/server/src/config.rs @@ -6,7 +6,6 @@ use std::collections::HashMap; use cluster::config::SchemaConfig; use common_types::schema::TIMESTAMP_COLUMN; -use common_util::config::{ReadableDuration, ReadableSize}; use meta_client::types::ShardId; use proxy::{forward, hotspot}; use router::{ @@ -14,7 +13,9 @@ use router::{ rule_based::{ClusterView, RuleList}, }; use serde::{Deserialize, Serialize}; +use size_ext::ReadableSize; use table_engine::ANALYTIC_ENGINE_TYPE; +use time_ext::ReadableDuration; #[derive(Clone, Debug, Default, Deserialize, Serialize)] #[serde(default)] diff --git a/server/src/grpc/meta_event_service/error.rs b/server/src/grpc/meta_event_service/error.rs index 4ab65a070c..d0a11ceb15 100644 --- a/server/src/grpc/meta_event_service/error.rs +++ b/server/src/grpc/meta_event_service/error.rs @@ -3,7 +3,8 @@ //! Error definitions for meta event service. use ceresdbproto::common::ResponseHeader; -use common_util::{define_result, error::GenericError}; +use generic_error::GenericError; +use macros::define_result; use snafu::Snafu; use crate::error_util; diff --git a/server/src/grpc/meta_event_service/mod.rs b/server/src/grpc/meta_event_service/mod.rs index 165f15b4c6..69f76c49a6 100644 --- a/server/src/grpc/meta_event_service/mod.rs +++ b/server/src/grpc/meta_event_service/mod.rs @@ -25,14 +25,16 @@ use cluster::{ ClusterRef, }; use common_types::{schema::SchemaEncoder, table::ShardId}; -use common_util::{error::BoxError, runtime::Runtime, time::InstantExt}; +use generic_error::BoxError; use log::{error, info, warn}; use meta_client::types::{ShardInfo, TableInfo}; use paste::paste; use proxy::instance::InstanceRef; use query_engine::executor::Executor as QueryExecutor; +use runtime::Runtime; use snafu::{OptionExt, ResultExt}; use table_engine::{engine::TableEngineRef, ANALYTIC_ENGINE_TYPE}; +use time_ext::InstantExt; use tonic::Response; use crate::grpc::{ diff --git a/server/src/grpc/mod.rs b/server/src/grpc/mod.rs index a2b59599fb..9a556132eb 100644 --- a/server/src/grpc/mod.rs +++ b/server/src/grpc/mod.rs @@ -17,13 +17,10 @@ use ceresdbproto::{ }; use cluster::ClusterRef; use common_types::column_schema; -use common_util::{ - define_result, - error::GenericError, - runtime::{JoinHandle, Runtime}, -}; use futures::FutureExt; +use generic_error::GenericError; use log::{info, warn}; +use macros::define_result; use proxy::{ forward, instance::InstanceRef, @@ -31,6 +28,7 @@ use proxy::{ Proxy, }; use query_engine::executor::Executor as QueryExecutor; +use runtime::{JoinHandle, Runtime}; use snafu::{Backtrace, OptionExt, ResultExt, Snafu}; use table_engine::engine::EngineRuntimes; use tokio::sync::oneshot::{self, Sender}; diff --git a/server/src/grpc/remote_engine_service/error.rs b/server/src/grpc/remote_engine_service/error.rs index ff74f3d30c..01f0c4ded0 100644 --- a/server/src/grpc/remote_engine_service/error.rs +++ b/server/src/grpc/remote_engine_service/error.rs @@ -1,9 +1,10 @@ -// Copyright 2022 CeresDB Project Authors. Licensed under Apache-2.0. +// Copyright 2022-2023 CeresDB Project Authors. Licensed under Apache-2.0. //! Error definitions for meta event service. use ceresdbproto::common::ResponseHeader; -use common_util::{define_result, error::GenericError}; +use generic_error::GenericError; +use macros::define_result; use snafu::Snafu; use crate::error_util; diff --git a/server/src/grpc/remote_engine_service/mod.rs b/server/src/grpc/remote_engine_service/mod.rs index bd2401628c..bc6f8535d3 100644 --- a/server/src/grpc/remote_engine_service/mod.rs +++ b/server/src/grpc/remote_engine_service/mod.rs @@ -16,8 +16,8 @@ use ceresdbproto::{ storage::{arrow_payload, ArrowPayload}, }; use common_types::record_batch::RecordBatch; -use common_util::{error::BoxError, time::InstantExt}; use futures::stream::{self, BoxStream, StreamExt}; +use generic_error::BoxError; use log::{error, info}; use proxy::instance::InstanceRef; use query_engine::executor::Executor as QueryExecutor; @@ -26,6 +26,7 @@ use table_engine::{ engine::EngineRuntimes, remote::model::TableIdentifier, stream::PartitionedStreams, table::TableRef, }; +use time_ext::InstantExt; use tokio::sync::mpsc; use tokio_stream::wrappers::ReceiverStream; use tonic::{Request, Response, Status}; diff --git a/server/src/grpc/storage_service/mod.rs b/server/src/grpc/storage_service/mod.rs index ca758146ca..35a85449ea 100644 --- a/server/src/grpc/storage_service/mod.rs +++ b/server/src/grpc/storage_service/mod.rs @@ -18,12 +18,12 @@ use ceresdbproto::{ SqlQueryRequest, SqlQueryResponse, WriteRequest, WriteResponse, }, }; -use common_util::time::InstantExt; use futures::{stream, stream::BoxStream, StreamExt}; use http::StatusCode; use proxy::{Context, Proxy, FORWARDED_FROM}; use query_engine::executor::Executor as QueryExecutor; use table_engine::engine::EngineRuntimes; +use time_ext::InstantExt; use crate::grpc::metrics::GRPC_HANDLER_DURATION_HISTOGRAM_VEC; diff --git a/server/src/http.rs b/server/src/http.rs index ef991036a3..9c9b88e5f5 100644 --- a/server/src/http.rs +++ b/server/src/http.rs @@ -10,12 +10,10 @@ use std::{ use analytic_engine::setup::OpenedWals; use cluster::ClusterRef; use common_types::bytes::Bytes; -use common_util::{ - error::{BoxError, GenericError}, - runtime::Runtime, -}; +use generic_error::{BoxError, GenericError}; use log::{error, info}; use logger::RuntimeLevel; +use macros::define_result; use profile::Profiler; use prom_remote_api::web; use proxy::{ @@ -29,6 +27,7 @@ use proxy::{ }; use query_engine::executor::Executor as QueryExecutor; use router::endpoint::Endpoint; +use runtime::Runtime; use serde::Serialize; use snafu::{Backtrace, OptionExt, ResultExt, Snafu}; use table_engine::{engine::EngineRuntimes, table::FlushRequest}; @@ -89,7 +88,7 @@ pub enum Error { }, #[snafu(display("Fail to join async task, err:{}.", source))] - JoinAsyncTask { source: common_util::runtime::Error }, + JoinAsyncTask { source: runtime::Error }, #[snafu(display( "Failed to parse ip addr, ip:{}, err:{}.\nBacktrace:\n{}", diff --git a/server/src/lib.rs b/server/src/lib.rs index 62caafedf3..d2d58bf156 100644 --- a/server/src/lib.rs +++ b/server/src/lib.rs @@ -5,9 +5,6 @@ // TODO(yingwen): // Borrow some ideas from tikv: https://github.com/tikv/tikv/blob/dc8ce2cf6a8904cb3dad556f71b11bac3531689b/src/server/service/kv.rs#L51 -#[macro_use] -extern crate common_util; - pub mod config; mod consts; mod error_util; diff --git a/server/src/local_tables.rs b/server/src/local_tables.rs index ac5467441f..16d1dfc13d 100644 --- a/server/src/local_tables.rs +++ b/server/src/local_tables.rs @@ -7,10 +7,8 @@ use catalog::{ table_operator::TableOperator, }; use common_types::table::DEFAULT_SHARD_ID; -use common_util::{ - define_result, - error::{BoxError, GenericError}, -}; +use generic_error::{BoxError, GenericError}; +use macros::define_result; use snafu::{Backtrace, ResultExt, Snafu}; use table_engine::table::TableInfo; diff --git a/server/src/mysql/error.rs b/server/src/mysql/error.rs index 9ac798f22b..fa04c7ad19 100644 --- a/server/src/mysql/error.rs +++ b/server/src/mysql/error.rs @@ -1,6 +1,7 @@ // Copyright 2022-2023 CeresDB Project Authors. Licensed under Apache-2.0. -use common_util::error::GenericError; +use generic_error::GenericError; +use macros::define_result; use snafu::{Backtrace, Snafu}; #[derive(Debug, Snafu)] diff --git a/server/src/mysql/service.rs b/server/src/mysql/service.rs index fe4526ac75..296bb34c7b 100644 --- a/server/src/mysql/service.rs +++ b/server/src/mysql/service.rs @@ -2,11 +2,11 @@ use std::{net::SocketAddr, sync::Arc, time::Duration}; -use common_util::runtime::JoinHandle; use log::{error, info}; use opensrv_mysql::AsyncMysqlIntermediary; use proxy::Proxy; use query_engine::executor::Executor as QueryExecutor; +use runtime::JoinHandle; use table_engine::engine::EngineRuntimes; use tokio::sync::oneshot::{self, Receiver, Sender}; diff --git a/server/src/mysql/worker.rs b/server/src/mysql/worker.rs index 95cd2a9d9e..30f518a214 100644 --- a/server/src/mysql/worker.rs +++ b/server/src/mysql/worker.rs @@ -2,7 +2,7 @@ use std::{marker::PhantomData, sync::Arc, time::Duration}; -use common_util::error::BoxError; +use generic_error::BoxError; use interpreters::interpreter::Output; use log::{error, info}; use opensrv_mysql::{AsyncMysqlShim, ErrorKind, QueryResultWriter, StatementMetaWriter}; diff --git a/server/src/server.rs b/server/src/server.rs index 83c029aa23..04005bee64 100644 --- a/server/src/server.rs +++ b/server/src/server.rs @@ -11,6 +11,7 @@ use df_operator::registry::FunctionRegistryRef; use interpreters::table_manipulator::TableManipulatorRef; use log::{info, warn}; use logger::RuntimeLevel; +use macros::define_result; use partition_table_engine::PartitionTableEngine; use proxy::{ instance::{Instance, InstanceRef}, diff --git a/src/bin/ceresdb-server.rs b/src/bin/ceresdb-server.rs index b9e527ba83..26f0473371 100644 --- a/src/bin/ceresdb-server.rs +++ b/src/bin/ceresdb-server.rs @@ -9,7 +9,6 @@ use ceresdb::{ setup, }; use clap::{App, Arg}; -use common_util::{panic, toml}; use log::info; /// By this environment variable, the address of current node can be overridden. @@ -63,7 +62,7 @@ fn main() { let mut config = match matches.value_of("config") { Some(path) => { let mut toml_buf = String::new(); - toml::parse_toml_from_path(path, &mut toml_buf).expect("Failed to parse config.") + toml_ext::parse_toml_from_path(path, &mut toml_buf).expect("Failed to parse config.") } None => Config::default(), }; @@ -85,7 +84,7 @@ fn main() { // Setup tracing. let _writer_guard = setup::setup_tracing(&config); - panic::set_panic_hook(false); + panic_ext::set_panic_hook(false); // Log version. info!("version:{}", version); diff --git a/src/setup.rs b/src/setup.rs index ac2db96e22..35fa70e213 100644 --- a/src/setup.rs +++ b/src/setup.rs @@ -12,7 +12,6 @@ use analytic_engine::{ use catalog::{manager::ManagerRef, schema::OpenOptions, table_operator::TableOperator}; use catalog_impls::{table_based::TableBasedManager, volatile, CatalogManagerImpl}; use cluster::{cluster_impl::ClusterImpl, config::ClusterConfig, shard_set::ShardSet}; -use common_util::runtime; use df_operator::registry::FunctionRegistryImpl; use interpreters::table_manipulator::{catalog_based, meta_based}; use log::info; diff --git a/system_catalog/Cargo.toml b/system_catalog/Cargo.toml index ca58403126..37807fb4e4 100644 --- a/system_catalog/Cargo.toml +++ b/system_catalog/Cargo.toml @@ -16,10 +16,12 @@ arrow = { workspace = true } async-trait = { workspace = true } catalog = { workspace = true } ceresdbproto = { workspace = true } +codec = { workspace = true } common_types = { workspace = true } -common_util = { workspace = true } futures = { workspace = true } +generic_error = { workspace = true } log = { workspace = true } +macros = { workspace = true } prost = { workspace = true } snafu = { workspace = true } table_engine = { workspace = true } diff --git a/system_catalog/src/sys_catalog_table.rs b/system_catalog/src/sys_catalog_table.rs index 46e7ebe7bb..8c3703a835 100644 --- a/system_catalog/src/sys_catalog_table.rs +++ b/system_catalog/src/sys_catalog_table.rs @@ -7,6 +7,7 @@ use std::{collections::HashMap, mem}; use async_trait::async_trait; use catalog::consts; use ceresdbproto::sys_catalog::{CatalogEntry, SchemaEntry, TableEntry}; +use codec::{memcomparable::MemComparable, Encoder}; use common_types::{ bytes::{BufMut, Bytes, BytesMut, SafeBuf, SafeBufMut}, column_schema, @@ -19,12 +20,9 @@ use common_types::{ table::DEFAULT_SHARD_ID, time::Timestamp, }; -use common_util::{ - codec::{memcomparable::MemComparable, Encoder}, - define_result, -}; use futures::TryStreamExt; use log::{debug, info, warn}; +use macros::define_result; use prost::Message; use snafu::{ensure, Backtrace, OptionExt, ResultExt, Snafu}; use table_engine::{ @@ -112,9 +110,7 @@ pub enum Error { EncodeKeyHeader { source: common_types::bytes::Error }, #[snafu(display("Failed to encode entry body, err:{}", source))] - EncodeKeyBody { - source: common_util::codec::memcomparable::Error, - }, + EncodeKeyBody { source: codec::memcomparable::Error }, #[snafu(display("Failed to encode table key type, err:{}", source))] EncodeTableKeyType { source: common_types::bytes::Error }, diff --git a/system_catalog/src/tables.rs b/system_catalog/src/tables.rs index 21a219c65f..3d8989cbc9 100644 --- a/system_catalog/src/tables.rs +++ b/system_catalog/src/tables.rs @@ -1,4 +1,4 @@ -// Copyright 2022 CeresDB Project Authors. Licensed under Apache-2.0. +// Copyright 2022-2023 CeresDB Project Authors. Licensed under Apache-2.0. /// implementation of system table: Tables /// For example `SELECT * FROM system.public.tables` @@ -15,7 +15,7 @@ use common_types::{ schema::Schema, time::Timestamp, }; -use common_util::error::BoxError; +use generic_error::BoxError; use snafu::ResultExt; use table_engine::{ stream::SendableRecordBatchStream, diff --git a/table_engine/Cargo.toml b/table_engine/Cargo.toml index 45acf34084..078d3ebbc5 100644 --- a/table_engine/Cargo.toml +++ b/table_engine/Cargo.toml @@ -17,19 +17,18 @@ arrow_ext = { workspace = true } async-trait = { workspace = true } ceresdbproto = { workspace = true } common_types = { workspace = true } -common_util = { workspace = true } datafusion = { workspace = true } datafusion-proto = { workspace = true } df_operator = { workspace = true } futures = { workspace = true } +generic_error = { workspace = true } itertools = { workspace = true } log = { workspace = true } +macros = { workspace = true } prost = { workspace = true } +runtime = { workspace = true } serde = { workspace = true } smallvec = { workspace = true } snafu = { workspace = true } tokio = { workspace = true } trace_metric = { workspace = true } - -[dev-dependencies] -env_logger = { workspace = true } diff --git a/table_engine/src/engine.rs b/table_engine/src/engine.rs index 168bd79b4d..70062ba49c 100644 --- a/table_engine/src/engine.rs +++ b/table_engine/src/engine.rs @@ -10,10 +10,9 @@ use common_types::{ schema::Schema, table::{ShardId, DEFAULT_SHARD_ID}, }; -use common_util::{ - error::{GenericError, GenericResult}, - runtime::RuntimeRef, -}; +use generic_error::{GenericError, GenericResult}; +use macros::define_result; +use runtime::RuntimeRef; use snafu::{ensure, Backtrace, Snafu}; use crate::{ diff --git a/table_engine/src/lib.rs b/table_engine/src/lib.rs index a6141c3821..415db062d9 100644 --- a/table_engine/src/lib.rs +++ b/table_engine/src/lib.rs @@ -2,9 +2,6 @@ //! Table engine facade, provides read/write interfaces of table -#[macro_use] -extern crate common_util; - pub mod engine; pub mod memory; pub mod partition; diff --git a/table_engine/src/memory.rs b/table_engine/src/memory.rs index 80d092609c..37abb234e5 100644 --- a/table_engine/src/memory.rs +++ b/table_engine/src/memory.rs @@ -18,8 +18,8 @@ use common_types::{ row::{Row, RowGroup}, schema::{RecordSchema, Schema}, }; -use common_util::error::BoxError; use futures::stream::Stream; +use generic_error::BoxError; use snafu::{OptionExt, ResultExt}; use crate::{ diff --git a/table_engine/src/partition/mod.rs b/table_engine/src/partition/mod.rs index 4e3d45548b..83bb91acc1 100644 --- a/table_engine/src/partition/mod.rs +++ b/table_engine/src/partition/mod.rs @@ -6,6 +6,7 @@ pub mod rule; use ceresdbproto::cluster::partition_info::Info; use common_types::bytes::Bytes; +use macros::define_result; use snafu::{Backtrace, Snafu}; const PARTITION_TABLE_PREFIX: &str = "__"; diff --git a/table_engine/src/predicate.rs b/table_engine/src/predicate.rs index 2ba37d195e..d348c81acc 100644 --- a/table_engine/src/predicate.rs +++ b/table_engine/src/predicate.rs @@ -1,4 +1,4 @@ -// Copyright 2022 CeresDB Project Authors. Licensed under Apache-2.0. +// Copyright 2022-2023 CeresDB Project Authors. Licensed under Apache-2.0. //! Predict for query table. //! Reference to: https://github.com/influxdata/influxdb_iox/blob/29b10413051f8c4a2193e8633aa133e45b0e505a/query/src/predicate.rs @@ -9,13 +9,14 @@ use common_types::{ schema::Schema, time::{TimeRange, Timestamp}, }; -use common_util::error::{BoxError, GenericError}; use datafusion::{ logical_expr::{Expr, Operator}, scalar::ScalarValue, }; use datafusion_proto::bytes::Serializeable; +use generic_error::{BoxError, GenericError}; use log::debug; +use macros::define_result; use snafu::{Backtrace, OptionExt, ResultExt, Snafu}; #[derive(Debug, Snafu)] diff --git a/table_engine/src/remote/mod.rs b/table_engine/src/remote/mod.rs index 507583209d..5aa8ae3f65 100644 --- a/table_engine/src/remote/mod.rs +++ b/table_engine/src/remote/mod.rs @@ -7,7 +7,8 @@ pub mod model; use std::sync::Arc; use async_trait::async_trait; -use common_util::{define_result, error::GenericError}; +use generic_error::GenericError; +use macros::define_result; use model::{ReadRequest, WriteRequest}; use snafu::Snafu; diff --git a/table_engine/src/remote/model.rs b/table_engine/src/remote/model.rs index 4119cb3311..94d24bb78e 100644 --- a/table_engine/src/remote/model.rs +++ b/table_engine/src/remote/model.rs @@ -18,7 +18,8 @@ use common_types::{ row::{RowGroup, RowGroupBuilder}, schema::Schema, }; -use common_util::error::{BoxError, GenericError, GenericResult}; +use generic_error::{BoxError, GenericError, GenericResult}; +use macros::define_result; use snafu::{ensure, Backtrace, OptionExt, ResultExt, Snafu}; use crate::{ diff --git a/table_engine/src/stream.rs b/table_engine/src/stream.rs index 6f85dd57fc..7479ed6b6f 100644 --- a/table_engine/src/stream.rs +++ b/table_engine/src/stream.rs @@ -1,4 +1,4 @@ -// Copyright 2022 CeresDB Project Authors. Licensed under Apache-2.0. +// Copyright 2022-2023 CeresDB Project Authors. Licensed under Apache-2.0. //! Table record stream @@ -10,10 +10,6 @@ use std::{ use arrow::{datatypes::SchemaRef, record_batch::RecordBatch as ArrowRecordBatch}; use common_types::{record_batch::RecordBatch, schema::RecordSchema}; -use common_util::{ - define_result, - error::{BoxError, GenericError}, -}; use datafusion::{ error::{DataFusionError, Result as DataFusionResult}, physical_plan::{ @@ -22,6 +18,8 @@ use datafusion::{ }, }; use futures::stream::Stream; +use generic_error::{BoxError, GenericError}; +use macros::define_result; use snafu::{Backtrace, ResultExt, Snafu}; // TODO(yingwen): Classify the error. diff --git a/table_engine/src/table.rs b/table_engine/src/table.rs index 821bdb6195..482903c25c 100644 --- a/table_engine/src/table.rs +++ b/table_engine/src/table.rs @@ -22,7 +22,8 @@ use common_types::{ row::{Row, RowGroup}, schema::{RecordSchemaWithKey, Schema, Version}, }; -use common_util::error::{BoxError, GenericError}; +use generic_error::{BoxError, GenericError}; +use macros::define_result; use serde::Deserialize; use snafu::{Backtrace, OptionExt, ResultExt, Snafu}; use trace_metric::MetricsCollector; diff --git a/tools/Cargo.toml b/tools/Cargo.toml index 980fe16e58..36701622a2 100644 --- a/tools/Cargo.toml +++ b/tools/Cargo.toml @@ -15,12 +15,12 @@ analytic_engine = { workspace = true } anyhow = { version = "1.0", features = ["backtrace"] } clap = { workspace = true, features = ["derive"] } common_types = { workspace = true } -common_util = { workspace = true } -env_logger = { workspace = true } futures = { workspace = true } num_cpus = "1.15.0" object_store = { workspace = true } parquet = { workspace = true } parquet_ext = { workspace = true } +runtime = { workspace = true } table_engine = { workspace = true } +time_ext = { workspace = true } tokio = { workspace = true } diff --git a/tools/src/bin/sst-convert.rs b/tools/src/bin/sst-convert.rs index 37de204e2b..d101253ca1 100644 --- a/tools/src/bin/sst-convert.rs +++ b/tools/src/bin/sst-convert.rs @@ -1,4 +1,4 @@ -// Copyright 2022 CeresDB Project Authors. Licensed under Apache-2.0. +// Copyright 2022-2023 CeresDB Project Authors. Licensed under Apache-2.0. //! A cli to convert ssts between different options @@ -17,9 +17,9 @@ use analytic_engine::{ use anyhow::{Context, Result}; use clap::Parser; use common_types::{projected_schema::ProjectedSchema, request_id::RequestId}; -use common_util::runtime::{self, Runtime}; use futures::stream::StreamExt; use object_store::{LocalFileSystem, Path}; +use runtime::Runtime; use table_engine::predicate::Predicate; use tools::sst_util; diff --git a/tools/src/bin/sst-metadata.rs b/tools/src/bin/sst-metadata.rs index a463ce09c9..8bbe969651 100644 --- a/tools/src/bin/sst-metadata.rs +++ b/tools/src/bin/sst-metadata.rs @@ -7,13 +7,11 @@ use std::{collections::HashMap, fmt, str::FromStr, sync::Arc}; use analytic_engine::sst::{meta_data::cache::MetaData, parquet::async_reader::ChunkReaderAdapter}; use anyhow::{Context, Result}; use clap::Parser; -use common_util::{ - runtime::{self, Runtime}, - time::format_as_ymdhms, -}; use futures::StreamExt; use object_store::{LocalFileSystem, ObjectMeta, ObjectStoreRef, Path}; use parquet_ext::{meta_data::fetch_parquet_metadata, reader::ObjectStoreReader}; +use runtime::Runtime; +use time_ext::format_as_ymdhms; use tokio::{runtime::Handle, task::JoinSet}; #[derive(Parser, Debug)] diff --git a/wal/Cargo.toml b/wal/Cargo.toml index c027187c24..852d61271a 100644 --- a/wal/Cargo.toml +++ b/wal/Cargo.toml @@ -21,22 +21,27 @@ features = ["portable"] async-trait = { workspace = true } ceresdbproto = { workspace = true } chrono = { workspace = true } +codec = { workspace = true } common_types = { workspace = true } -common_util = { workspace = true } futures = { workspace = true, features = ["async-await"], optional = true } +generic_error = { workspace = true } log = { workspace = true } +macros = { workspace = true } message_queue = { workspace = true } prost = { workspace = true } +runtime = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } +size_ext = { workspace = true } smallvec = { workspace = true } snafu = { workspace = true } table_kv = { workspace = true } tempfile = { workspace = true, optional = true } +time_ext = { workspace = true } +timed_task = { workspace = true } tokio = { workspace = true } uuid = { version = "1.3", features = ["v4"], optional = true } [dev-dependencies] -env_logger = { workspace = true } futures = { workspace = true, features = ["async-await"] } rand = "0.8.5" diff --git a/wal/src/kv_encoder.rs b/wal/src/kv_encoder.rs index 728f53b73e..f0ea2a86c3 100644 --- a/wal/src/kv_encoder.rs +++ b/wal/src/kv_encoder.rs @@ -2,16 +2,14 @@ //! Common Encoding for Wal logs +use codec::{Decoder, Encoder}; use common_types::{ bytes::{self, Buf, BufMut, BytesMut, SafeBuf, SafeBufMut}, table::TableId, SequenceNumber, }; -use common_util::{ - codec::{Decoder, Encoder}, - define_result, - error::{BoxError, GenericError}, -}; +use generic_error::{BoxError, GenericError}; +use macros::define_result; use snafu::{ensure, Backtrace, ResultExt, Snafu}; use crate::{ diff --git a/wal/src/manager.rs b/wal/src/manager.rs index 5d41585b34..ef8a6a8f7a 100644 --- a/wal/src/manager.rs +++ b/wal/src/manager.rs @@ -1,4 +1,4 @@ -// Copyright 2022 CeresDB Project Authors. Licensed under Apache-2.0. +// Copyright 2022-2023 CeresDB Project Authors. Licensed under Apache-2.0. //! WalManager abstraction @@ -10,14 +10,16 @@ use common_types::{ table::{TableId, DEFAULT_SHARD_ID}, MAX_SEQUENCE_NUMBER, MIN_SEQUENCE_NUMBER, }; -use common_util::{error::BoxError, runtime::Runtime}; pub use error::*; +use generic_error::BoxError; +use runtime::Runtime; use snafu::ResultExt; use crate::log_batch::{LogEntry, LogWriteBatch, PayloadDecoder}; pub mod error { - use common_util::{define_result, error::GenericError}; + use generic_error::GenericError; + use macros::define_result; use snafu::{Backtrace, Snafu}; use crate::manager::{RegionId, WalLocation}; @@ -126,7 +128,7 @@ pub mod error { }, #[snafu(display("Failed to execute in runtime, err:{}", source))] - RuntimeExec { source: common_util::runtime::Error }, + RuntimeExec { source: runtime::Error }, #[snafu(display("Encountered unknown error, msg:{}.\nBacktrace:\n{}", msg, backtrace))] Unknown { msg: String, backtrace: Backtrace }, @@ -461,7 +463,7 @@ mod tests { use std::{collections::VecDeque, sync::Arc}; use async_trait::async_trait; - use common_util::runtime::{self, Runtime}; + use runtime::{self, Runtime}; use super::*; use crate::{log_batch::LogEntry, tests::util::TestPayloadDecoder}; diff --git a/wal/src/message_queue_impl/config.rs b/wal/src/message_queue_impl/config.rs index 665c26b4f3..7504da69fb 100644 --- a/wal/src/message_queue_impl/config.rs +++ b/wal/src/message_queue_impl/config.rs @@ -1,9 +1,9 @@ -// Copyright 2022 CeresDB Project Authors. Licensed under Apache-2.0. +// Copyright 2022-2023 CeresDB Project Authors. Licensed under Apache-2.0. //! Config for wal on message queue -use common_util::config::ReadableDuration; use serde::{Deserialize, Serialize}; +use time_ext::ReadableDuration; // TODO: add more needed config items. #[derive(Debug, Clone, Serialize, Deserialize)] diff --git a/wal/src/message_queue_impl/encoding.rs b/wal/src/message_queue_impl/encoding.rs index 9e99b9ade9..1e9ec14e8e 100644 --- a/wal/src/message_queue_impl/encoding.rs +++ b/wal/src/message_queue_impl/encoding.rs @@ -6,12 +6,10 @@ use ceresdbproto::wal_on_mq::{ table_meta_data::SafeDeleteOffset, RegionMetaSnapshot as RegionMetaSnapshotPb, TableMetaData as TableMetaDataPb, }; +use codec::{Decoder, Encoder}; use common_types::bytes::{self, Buf, BufMut, BytesMut, SafeBuf, SafeBufMut}; -use common_util::{ - codec::{Decoder, Encoder}, - define_result, - error::{BoxError, GenericError}, -}; +use generic_error::{BoxError, GenericError}; +use macros::define_result; use prost::Message; use snafu::{ensure, Backtrace, ResultExt, Snafu}; diff --git a/wal/src/message_queue_impl/log_cleaner.rs b/wal/src/message_queue_impl/log_cleaner.rs index 074a01b29a..7e3137c83b 100644 --- a/wal/src/message_queue_impl/log_cleaner.rs +++ b/wal/src/message_queue_impl/log_cleaner.rs @@ -4,11 +4,9 @@ use std::sync::Arc; -use common_util::{ - define_result, - error::{BoxError, GenericError}, -}; +use generic_error::{BoxError, GenericError}; use log::info; +use macros::define_result; use message_queue::{MessageQueue, Offset}; use snafu::{ensure, Backtrace, ResultExt, Snafu}; diff --git a/wal/src/message_queue_impl/namespace.rs b/wal/src/message_queue_impl/namespace.rs index 317eb9c023..5c22e33701 100644 --- a/wal/src/message_queue_impl/namespace.rs +++ b/wal/src/message_queue_impl/namespace.rs @@ -5,14 +5,12 @@ use std::{collections::HashMap, fmt, sync::Arc, time::Duration}; use common_types::SequenceNumber; -use common_util::{ - define_result, - runtime::Runtime, - timed_task::{TaskHandle, TimedTask}, -}; use log::{debug, error, info}; +use macros::define_result; use message_queue::{ConsumeIterator, MessageQueue}; +use runtime::Runtime; use snafu::{ensure, Backtrace, OptionExt, ResultExt, Snafu}; +use timed_task::{TaskHandle, TimedTask}; use tokio::sync::RwLock; use crate::{ @@ -108,7 +106,7 @@ pub enum Error { #[snafu(display("Failed to close namespace, namespace:{namespace}, err:{source}"))] Close { namespace: String, - source: common_util::runtime::Error, + source: runtime::Error, }, } diff --git a/wal/src/message_queue_impl/region.rs b/wal/src/message_queue_impl/region.rs index f2d6a285ba..711090f6c8 100644 --- a/wal/src/message_queue_impl/region.rs +++ b/wal/src/message_queue_impl/region.rs @@ -5,11 +5,9 @@ use std::{cmp, sync::Arc}; use common_types::{table::TableId, SequenceNumber}; -use common_util::{ - define_result, - error::{BoxError, GenericError}, -}; +use generic_error::{BoxError, GenericError}; use log::{debug, info}; +use macros::define_result; use message_queue::{ConsumeIterator, MessageQueue, Offset, OffsetType, StartOffset}; use snafu::{ensure, Backtrace, OptionExt, ResultExt, Snafu}; use tokio::sync::{Mutex, RwLock}; diff --git a/wal/src/message_queue_impl/region_context.rs b/wal/src/message_queue_impl/region_context.rs index 80c0f70ac4..ad369cc574 100644 --- a/wal/src/message_queue_impl/region_context.rs +++ b/wal/src/message_queue_impl/region_context.rs @@ -9,11 +9,9 @@ use std::{ }; use common_types::{bytes::BytesMut, table::TableId, SequenceNumber}; -use common_util::{ - define_result, - error::{BoxError, GenericError}, -}; +use generic_error::{BoxError, GenericError}; use log::{debug, warn}; +use macros::define_result; use message_queue::{MessageQueue, Offset}; use snafu::{ensure, Backtrace, OptionExt, ResultExt, Snafu}; use tokio::sync::{Mutex, RwLock}; diff --git a/wal/src/message_queue_impl/snapshot_synchronizer.rs b/wal/src/message_queue_impl/snapshot_synchronizer.rs index 5e61c0ee3a..4d52412967 100644 --- a/wal/src/message_queue_impl/snapshot_synchronizer.rs +++ b/wal/src/message_queue_impl/snapshot_synchronizer.rs @@ -1,15 +1,13 @@ -// Copyright 2022 CeresDB Project Authors. Licensed under Apache-2.0. +// Copyright 2022-2023 CeresDB Project Authors. Licensed under Apache-2.0. //! Snapshot synchronizer use std::sync::Arc; use common_types::bytes::BytesMut; -use common_util::{ - define_result, - error::{BoxError, GenericError}, -}; +use generic_error::{BoxError, GenericError}; use log::debug; +use macros::define_result; use message_queue::MessageQueue; use snafu::{ensure, Backtrace, ResultExt, Snafu}; diff --git a/wal/src/message_queue_impl/wal.rs b/wal/src/message_queue_impl/wal.rs index faf1dac8f7..bef2bca859 100644 --- a/wal/src/message_queue_impl/wal.rs +++ b/wal/src/message_queue_impl/wal.rs @@ -6,8 +6,9 @@ use std::sync::Arc; use async_trait::async_trait; use common_types::SequenceNumber; -use common_util::{error::BoxError, runtime::Runtime}; +use generic_error::BoxError; use message_queue::{ConsumeIterator, MessageQueue}; +use runtime::Runtime; use snafu::ResultExt; use crate::{ diff --git a/wal/src/rocks_impl/config.rs b/wal/src/rocks_impl/config.rs index 966720f094..00d3dd6998 100644 --- a/wal/src/rocks_impl/config.rs +++ b/wal/src/rocks_impl/config.rs @@ -2,8 +2,8 @@ //! RocksDB Config -use common_util::config::ReadableSize; use serde::{Deserialize, Serialize}; +use size_ext::ReadableSize; #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(default)] diff --git a/wal/src/rocks_impl/manager.rs b/wal/src/rocks_impl/manager.rs index 2e4e70cf00..242dce017d 100644 --- a/wal/src/rocks_impl/manager.rs +++ b/wal/src/rocks_impl/manager.rs @@ -17,12 +17,13 @@ use async_trait::async_trait; use common_types::{ bytes::BytesMut, table::TableId, SequenceNumber, MAX_SEQUENCE_NUMBER, MIN_SEQUENCE_NUMBER, }; -use common_util::{error::BoxError, runtime::Runtime}; +use generic_error::BoxError; use log::{debug, info, warn}; use rocksdb::{ rocksdb_options::ColumnFamilyDescriptor, ColumnFamilyOptions, DBCompactionStyle, DBIterator, DBOptions, FifoCompactionOptions, ReadOptions, SeekKey, Statistics, Writable, WriteBatch, DB, }; +use runtime::Runtime; use snafu::ResultExt; use tokio::sync::Mutex; diff --git a/wal/src/table_kv_impl/encoding.rs b/wal/src/table_kv_impl/encoding.rs index 7c44f10f6d..fe0ba4324d 100644 --- a/wal/src/table_kv_impl/encoding.rs +++ b/wal/src/table_kv_impl/encoding.rs @@ -1,12 +1,13 @@ -// Copyright 2022 CeresDB Project Authors. Licensed under Apache-2.0. +// Copyright 2022-2023 CeresDB Project Authors. Licensed under Apache-2.0. //! Constants and utils for encoding. use chrono::{TimeZone, Utc}; use common_types::{table::TableId, time::Timestamp}; -use common_util::{config::ReadableDuration, define_result}; +use macros::define_result; use snafu::Snafu; use table_kv::{KeyBoundary, ScanRequest}; +use time_ext::ReadableDuration; #[derive(Debug, Snafu)] pub enum Error { diff --git a/wal/src/table_kv_impl/mod.rs b/wal/src/table_kv_impl/mod.rs index 1260f36452..ddd9914bc5 100644 --- a/wal/src/table_kv_impl/mod.rs +++ b/wal/src/table_kv_impl/mod.rs @@ -4,7 +4,7 @@ use std::sync::Arc; -use common_util::runtime::Runtime; +use runtime::Runtime; pub mod encoding; pub mod model; diff --git a/wal/src/table_kv_impl/model.rs b/wal/src/table_kv_impl/model.rs index afd0fb50dc..c0e7e23fb9 100644 --- a/wal/src/table_kv_impl/model.rs +++ b/wal/src/table_kv_impl/model.rs @@ -1,14 +1,15 @@ -// Copyright 2022 CeresDB Project Authors. Licensed under Apache-2.0. +// Copyright 2022-2023 CeresDB Project Authors. Licensed under Apache-2.0. //! Data models for TableKv based wal. use std::{str, time::Duration}; use common_types::{table::TableId, time::Timestamp}; -use common_util::{config::ReadableDuration, define_result}; +use macros::define_result; use serde::{Deserialize, Serialize}; use snafu::{ensure, Backtrace, ResultExt, Snafu}; use table_kv::ScanContext; +use time_ext::ReadableDuration; use crate::{ manager::SequenceNumber, diff --git a/wal/src/table_kv_impl/namespace.rs b/wal/src/table_kv_impl/namespace.rs index 7ece23953c..c1280a64ef 100644 --- a/wal/src/table_kv_impl/namespace.rs +++ b/wal/src/table_kv_impl/namespace.rs @@ -10,18 +10,16 @@ use std::{ }; use common_types::{table::TableId, time::Timestamp}; -use common_util::{ - config::ReadableDuration, - define_result, - error::{BoxError, GenericError}, - runtime::Runtime, - timed_task::{TaskHandle, TimedTask}, -}; +use generic_error::{BoxError, GenericError}; use log::{debug, error, info, trace, warn}; +use macros::define_result; +use runtime::Runtime; use snafu::{Backtrace, OptionExt, ResultExt, Snafu}; use table_kv::{ ScanContext as KvScanContext, ScanIter, TableError, TableKv, WriteBatch, WriteContext, }; +use time_ext::ReadableDuration; +use timed_task::{TaskHandle, TimedTask}; use crate::{ kv_encoder::CommonLogKey, @@ -187,7 +185,7 @@ pub enum Error { #[snafu(display("Failed to stop task, namespace:{}, err:{}", namespace, source))] StopTask { namespace: String, - source: common_util::runtime::Error, + source: runtime::Error, }, #[snafu(display( @@ -1470,7 +1468,7 @@ mod tests { use std::sync::Arc; use common_types::{bytes::BytesMut, table::DEFAULT_SHARD_ID}; - use common_util::runtime::{Builder, Runtime}; + use runtime::{Builder, Runtime}; use table_kv::{memory::MemoryImpl, KeyBoundary, ScanContext, ScanRequest}; use super::*; diff --git a/wal/src/table_kv_impl/table_unit.rs b/wal/src/table_kv_impl/table_unit.rs index 835a16e38c..74fd7aa741 100644 --- a/wal/src/table_kv_impl/table_unit.rs +++ b/wal/src/table_kv_impl/table_unit.rs @@ -14,12 +14,10 @@ use std::{ }; use common_types::{bytes::BytesMut, table::TableId}; -use common_util::{ - define_result, - error::{BoxError, GenericError}, - runtime::Runtime, -}; +use generic_error::{BoxError, GenericError}; use log::debug; +use macros::define_result; +use runtime::{self, Runtime}; use snafu::{ensure, Backtrace, OptionExt, ResultExt, Snafu}; use table_kv::{ KeyBoundary, ScanContext, ScanIter, ScanRequest, TableError, TableKv, WriteBatch, WriteContext, @@ -105,7 +103,7 @@ pub enum Error { }, #[snafu(display("Failed to execute in runtime, err:{}", source))] - RuntimeExec { source: common_util::runtime::Error }, + RuntimeExec { source: runtime::Error }, #[snafu(display("Failed to delete table, region_id:{}, err:{}", region_id, source))] Delete { diff --git a/wal/src/table_kv_impl/wal.rs b/wal/src/table_kv_impl/wal.rs index 78856a03b4..ab367fa39f 100644 --- a/wal/src/table_kv_impl/wal.rs +++ b/wal/src/table_kv_impl/wal.rs @@ -6,7 +6,7 @@ use std::{fmt, str, sync::Arc}; use async_trait::async_trait; use common_types::SequenceNumber; -use common_util::error::BoxError; +use generic_error::BoxError; use log::info; use snafu::ResultExt; use table_kv::TableKv; diff --git a/wal/src/tests/util.rs b/wal/src/tests/util.rs index ef0efd26ac..431199de66 100644 --- a/wal/src/tests/util.rs +++ b/wal/src/tests/util.rs @@ -15,14 +15,12 @@ use common_types::{ table::TableId, SequenceNumber, }; -use common_util::{ - config::ReadableDuration, - runtime::{self, Runtime}, -}; use message_queue::kafka::{config::Config as KafkaConfig, kafka_impl::KafkaImpl}; +use runtime::{self, Runtime}; use snafu::Snafu; use table_kv::memory::MemoryImpl; use tempfile::TempDir; +use time_ext::ReadableDuration; use crate::{ kv_encoder::LogBatchEncoder,