From 8eb0fd087199935d10bd23213c4f078cda3dc214 Mon Sep 17 00:00:00 2001 From: jiacai2050 Date: Wed, 21 Aug 2024 17:51:08 +0800 Subject: [PATCH] feat: add metric engine structure --- Cargo.lock | 61 +++++++++++++++ Cargo.toml | 2 + src/horaedb/Cargo.toml | 1 + src/horaedb/src/setup.rs | 1 + src/metric_engine/Cargo.toml | 100 +++++++++++++++++++++++++ src/metric_engine/README.md | 0 src/metric_engine/src/lib.rs | 3 + src/metric_engine/src/table_storage.rs | 19 +++++ 8 files changed, 187 insertions(+) create mode 100644 src/metric_engine/Cargo.toml create mode 100644 src/metric_engine/README.md create mode 100644 src/metric_engine/src/lib.rs create mode 100644 src/metric_engine/src/table_storage.rs diff --git a/Cargo.lock b/Cargo.lock index 0a2d3a0f76..2ac3fc51bc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3144,6 +3144,7 @@ dependencies = [ "interpreters", "logger", "meta_client", + "metric_engine", "moka", "panic_ext", "proxy", @@ -4167,6 +4168,66 @@ dependencies = [ "url", ] +[[package]] +name = "metric_engine" +version = "2.0.0" +dependencies = [ + "anyhow", + "arc-swap 1.6.0", + "arena", + "arrow 49.0.0", + "async-stream", + "async-trait", + "atomic_enum", + "base64 0.13.1", + "bytes_ext", + "codec", + "common_types", + "datafusion", + "env_logger", + "future_ext", + "futures 0.3.28", + "generic_error", + "hash_ext", + "hex", + "horaedbproto 2.0.0", + "hyperloglog", + "id_allocator", + "itertools 0.10.5", + "lazy_static", + "logger", + "lru 0.7.8", + "macros", + "message_queue", + "metric_ext", + "object_store 2.0.0", + "parquet", + "parquet_ext", + "pin-project-lite", + "prometheus 0.12.0", + "prost 0.11.8", + "rand 0.8.5", + "remote_engine_client", + "router", + "runtime", + "sampling_cache", + "serde", + "size_ext", + "skiplist", + "smallvec", + "snafu 0.6.10", + "table_engine", + "table_kv", + "tempfile", + "test_util", + "thiserror", + "time_ext", + "tokio", + "trace_metric", + "wal", + "xorfilter-rs", +] + [[package]] name = "metric_ext" version = "2.0.0" diff --git a/Cargo.toml b/Cargo.toml index c0e62ebf8b..148f6c45f4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -29,6 +29,7 @@ members = [ "integration_tests", "integration_tests/sdk/rust", "src/analytic_engine", + "src/metric_engine", "src/benchmarks", "src/catalog", "src/catalog_impls", @@ -90,6 +91,7 @@ arrow = { version = "49.0.0", features = ["prettyprint"] } arrow_ipc = { version = "49.0.0" } arrow_ext = { path = "src/components/arrow_ext" } analytic_engine = { path = "src/analytic_engine" } +metric_engine = { path = "src/metric_engine" } anyhow = { version = "1.0" } arena = { path = "src/components/arena" } async-stream = "0.3.4" diff --git a/src/horaedb/Cargo.toml b/src/horaedb/Cargo.toml index 2abfa49e17..65bcdeb9f7 100644 --- a/src/horaedb/Cargo.toml +++ b/src/horaedb/Cargo.toml @@ -51,6 +51,7 @@ logger = { workspace = true } meta_client = { workspace = true } moka = { version = "0.10", features = ["future"] } panic_ext = { workspace = true } +metric_engine = { workspace = true } proxy = { workspace = true } query_engine = { workspace = true } router = { workspace = true } diff --git a/src/horaedb/src/setup.rs b/src/horaedb/src/setup.rs index 9bdb46daf9..58a0788bb7 100644 --- a/src/horaedb/src/setup.rs +++ b/src/horaedb/src/setup.rs @@ -31,6 +31,7 @@ use df_operator::registry::{FunctionRegistry, FunctionRegistryImpl}; use interpreters::table_manipulator::{catalog_based, meta_based}; use logger::{info, warn, RuntimeLevel}; use meta_client::{meta_impl, types::NodeMetaInfo}; +use metric_engine::*; use proxy::{ limiter::Limiter, schema_config_provider::{ diff --git a/src/metric_engine/Cargo.toml b/src/metric_engine/Cargo.toml new file mode 100644 index 0000000000..d314fcb30a --- /dev/null +++ b/src/metric_engine/Cargo.toml @@ -0,0 +1,100 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[package] +name = "metric_engine" + +[package.license] +workspace = true + +[package.version] +workspace = true + +[package.authors] +workspace = true + +[package.edition] +workspace = true + +[features] +test = ["tempfile"] +wal-table-kv = ["wal/wal-table-kv"] +wal-message-queue = ["wal/wal-message-queue"] +wal-rocksdb = ["wal/wal-rocksdb"] +wal-local-storage = ["wal/wal-local-storage"] + +[dependencies] +# In alphabetical order +anyhow = { workspace = true } +arc-swap = "1.4.0" +arena = { workspace = true } +arrow = { workspace = true } +async-stream = { workspace = true } +async-trait = { workspace = true } +atomic_enum = { workspace = true } +base64 = { workspace = true } +bytes_ext = { workspace = true } +codec = { workspace = true } +common_types = { workspace = true } +datafusion = { workspace = true } +future_ext = { workspace = true } +futures = { workspace = true } +generic_error = { workspace = true } +hash_ext = { workspace = true } +hex = { workspace = true } +horaedbproto = { workspace = true } +hyperloglog = { workspace = true } +id_allocator = { workspace = true } +itertools = { workspace = true } +lazy_static = { workspace = true } +logger = { workspace = true } +lru = { workspace = true } +macros = { workspace = true } +message_queue = { workspace = true } +metric_ext = { workspace = true } +object_store = { workspace = true } +parquet = { workspace = true } +parquet_ext = { workspace = true } +prometheus = { workspace = true } +prost = { workspace = true } +remote_engine_client = { workspace = true } +router = { workspace = true } +runtime = { workspace = true } +sampling_cache = { workspace = true } +serde = { workspace = true } +size_ext = { workspace = true } +skiplist = { path = "../components/skiplist" } +smallvec = { workspace = true } +snafu = { workspace = true } +table_engine = { workspace = true } +table_kv = { workspace = true } +tempfile = { workspace = true, optional = true } +thiserror = { workspace = true } +time_ext = { workspace = true } +tokio = { workspace = true } +trace_metric = { workspace = true } +wal = { workspace = true } +xorfilter-rs = { workspace = true } + +[dev-dependencies] +common_types = { workspace = true, features = ["test"] } +env_logger = { workspace = true } +pin-project-lite = { workspace = true } +rand = { workspace = true } +tempfile = { workspace = true } +test_util = { workspace = true } +wal = { workspace = true, features = ["wal-message-queue", "wal-rocksdb", "wal-table-kv"] } diff --git a/src/metric_engine/README.md b/src/metric_engine/README.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/metric_engine/src/lib.rs b/src/metric_engine/src/lib.rs new file mode 100644 index 0000000000..0aa5802aa4 --- /dev/null +++ b/src/metric_engine/src/lib.rs @@ -0,0 +1,3 @@ +//! Storage Engine for metrics. + +pub mod table_storage; diff --git a/src/metric_engine/src/table_storage.rs b/src/metric_engine/src/table_storage.rs new file mode 100644 index 0000000000..1c949cc564 --- /dev/null +++ b/src/metric_engine/src/table_storage.rs @@ -0,0 +1,19 @@ +use wal::manager::WalManagerRef; + +pub struct SSTable { + id: u64, +} + +/// Storage to represent different components of the system. +/// Such as: metrics, series, indexes, etc. +/// +/// Columns for design: +/// metrics: {MetricName}-{MetricID}-{FieldName} +/// series: {TSID}-{SeriesKey} +/// index: {TagKey}-{TagValue}-{TSID} +pub struct TableStorage { + name: String, + id: u64, + wal: WalManagerRef, + sstables: Vec, +}