From e8cad7ca8ebb527986433a740724545c4f657ba8 Mon Sep 17 00:00:00 2001 From: "chunshao.rcs" Date: Mon, 3 Apr 2023 18:41:17 +0800 Subject: [PATCH 1/9] feat: support partition table with proxy --- Cargo.lock | 1 + analytic_engine/src/instance/create.rs | 52 ++--- analytic_engine/src/manifest/meta_update.rs | 20 +- analytic_engine/src/table/data.rs | 2 +- catalog/src/schema.rs | 12 +- catalog_impls/src/table_based.rs | 2 +- catalog_impls/src/volatile.rs | 30 +-- cluster/src/cluster_impl.rs | 16 +- cluster/src/lib.rs | 5 +- .../src/table_manipulator/catalog_based.rs | 1 + meta_client/src/lib.rs | 5 +- meta_client/src/meta_impl.rs | 4 +- meta_client/src/types.rs | 50 +++-- remote_engine_client/src/lib.rs | 2 +- router/src/cluster_based.rs | 25 +-- router/src/lib.rs | 16 +- router/src/rule_based.rs | 23 +-- server/Cargo.toml | 1 + server/src/grpc/meta_event_service/mod.rs | 13 +- server/src/grpc/mod.rs | 11 +- server/src/proxy/forward.rs | 41 +++- server/src/proxy/grpc/route.rs | 13 +- server/src/proxy/grpc/sql_query.rs | 182 +++++++++++++++--- server/src/proxy/mod.rs | 9 + server/src/server.rs | 5 +- sql/src/frontend.rs | 52 ++++- table_engine/src/partition/mod.rs | 4 +- 27 files changed, 437 insertions(+), 160 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a1e56378ea..df74835fd3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5626,6 +5626,7 @@ dependencies = [ "prometheus-static-metric", "prost", "query_engine", + "remote_engine_client", "router", "serde", "serde_json", diff --git a/analytic_engine/src/instance/create.rs b/analytic_engine/src/instance/create.rs index 8cb0ef9524..1d199d948c 100644 --- a/analytic_engine/src/instance/create.rs +++ b/analytic_engine/src/instance/create.rs @@ -1,4 +1,4 @@ -// Copyright 2022 CeresDB Project Authors. Licensed under Apache-2.0. +// Copyright 2022-2023 CeresDB Project Authors. Licensed under Apache-2.0. //! Create table logic of instance @@ -105,30 +105,32 @@ impl Instance { return Ok(table_data); }; - // Store table info into meta - let update_req = { - let meta_update = MetaUpdate::AddTable(AddTableMeta { - space_id: space.id, - table_id: table_data.id, - table_name: table_data.name.clone(), - schema: table_data.schema(), - opts: table_data.table_options().as_ref().clone(), - partition_info: table_data.partition_info.clone(), - }); - MetaUpdateRequest { - shard_info: table_data.shard_info, - meta_update, - } - }; - self.space_store - .manifest - .store_update(update_req) - .await - .context(WriteManifest { - space_id: space.id, - table: &table_data.name, - table_id: table_data.id, - })?; + // Partition table is not stored in manifest. + if table_data.partition_info.is_none() { + // Store table info into meta + let update_req = { + let meta_update = MetaUpdate::AddTable(AddTableMeta { + space_id: space.id, + table_id: table_data.id, + table_name: table_data.name.clone(), + schema: table_data.schema(), + opts: table_data.table_options().as_ref().clone(), + }); + MetaUpdateRequest { + shard_info: table_data.shard_info, + meta_update, + } + }; + self.space_store + .manifest + .store_update(update_req) + .await + .context(WriteManifest { + space_id: space.id, + table: &table_data.name, + table_id: table_data.id, + })?; + } space.insert_table(table_data.clone()); Ok(table_data) diff --git a/analytic_engine/src/manifest/meta_update.rs b/analytic_engine/src/manifest/meta_update.rs index adcf0f7a5f..7e641da45d 100644 --- a/analytic_engine/src/manifest/meta_update.rs +++ b/analytic_engine/src/manifest/meta_update.rs @@ -1,4 +1,4 @@ -// Copyright 2022 CeresDB Project Authors. Licensed under Apache-2.0. +// Copyright 2022-2023 CeresDB Project Authors. Licensed under Apache-2.0. //! Update to meta @@ -13,7 +13,7 @@ use common_types::{ use common_util::define_result; use prost::Message; use snafu::{Backtrace, OptionExt, ResultExt, Snafu}; -use table_engine::{partition::PartitionInfo, table::TableId}; +use table_engine::table::TableId; use wal::log_batch::{Payload, PayloadDecoder}; use crate::{ @@ -42,11 +42,6 @@ pub enum Error { #[snafu(display("Failed to convert schema, err:{}", source))] ConvertSchema { source: common_types::schema::Error }, - #[snafu(display("Failed to convert partition info, err:{}", source))] - ConvertPartitionInfo { - source: table_engine::partition::Error, - }, - #[snafu(display("Empty table schema.\nBacktrace:\n{}", backtrace))] EmptyTableSchema { backtrace: Backtrace }, @@ -155,19 +150,17 @@ pub struct AddTableMeta { pub schema: Schema, // Options needed to persist pub opts: TableOptions, - pub partition_info: Option, } impl From for manifest_pb::AddTableMeta { fn from(v: AddTableMeta) -> Self { - let partition_info = v.partition_info.map(|v| v.into()); manifest_pb::AddTableMeta { space_id: v.space_id, table_id: v.table_id.as_u64(), table_name: v.table_name, schema: Some(schema_pb::TableSchema::from(&v.schema)), options: Some(manifest_pb::TableOptions::from(v.opts)), - partition_info, + partition_info: None, } } } @@ -178,12 +171,6 @@ impl TryFrom for AddTableMeta { fn try_from(src: manifest_pb::AddTableMeta) -> Result { let table_schema = src.schema.context(EmptyTableSchema)?; let opts = src.options.context(EmptyTableOptions)?; - let partition_info = match src.partition_info { - Some(partition_info) => { - Some(PartitionInfo::try_from(partition_info).context(ConvertPartitionInfo)?) - } - None => None, - }; Ok(Self { space_id: src.space_id, @@ -191,7 +178,6 @@ impl TryFrom for AddTableMeta { table_name: src.table_name, schema: Schema::try_from(table_schema).context(ConvertSchema)?, opts: TableOptions::try_from(opts).context(ConvertTableOptions)?, - partition_info, }) } } diff --git a/analytic_engine/src/table/data.rs b/analytic_engine/src/table/data.rs index cd1599be53..a50dcf787f 100644 --- a/analytic_engine/src/table/data.rs +++ b/analytic_engine/src/table/data.rs @@ -256,7 +256,7 @@ impl TableData { dropped: AtomicBool::new(false), metrics, shard_info: TableShardInfo::new(shard_id), - partition_info: add_meta.partition_info, + partition_info: None, }) } diff --git a/catalog/src/schema.rs b/catalog/src/schema.rs index 172bb42892..2dbcad69be 100644 --- a/catalog/src/schema.rs +++ b/catalog/src/schema.rs @@ -193,6 +193,8 @@ pub struct CreateTableRequest { pub schema_id: SchemaId, /// Table name pub table_name: String, + /// Table id + pub table_id: Option, /// Table schema pub table_schema: common_types::schema::Schema, /// Table engine type @@ -208,7 +210,15 @@ pub struct CreateTableRequest { } impl CreateTableRequest { - pub fn into_engine_create_request(self, table_id: TableId) -> engine::CreateTableRequest { + pub fn into_engine_create_request( + self, + table_id: Option, + ) -> engine::CreateTableRequest { + let table_id = match (self.table_id, table_id) { + (Some(v), _) => v, + (None, Some(v)) => v, + (None, None) => TableId::MIN, + }; engine::CreateTableRequest { catalog_name: self.catalog_name, schema_name: self.schema_name, diff --git a/catalog_impls/src/table_based.rs b/catalog_impls/src/table_based.rs index 5dba9dfee0..6de5f195b7 100644 --- a/catalog_impls/src/table_based.rs +++ b/catalog_impls/src/table_based.rs @@ -733,7 +733,7 @@ impl Schema for SchemaImpl { // Create table let table_id = self.alloc_table_id(&request.table_name).await?; - let request = request.into_engine_create_request(table_id); + let request = request.into_engine_create_request(Some(table_id)); let table_name = request.table_name.clone(); let table = opts .table_engine diff --git a/catalog_impls/src/volatile.rs b/catalog_impls/src/volatile.rs index ff8fb7dfb5..7210865ddf 100644 --- a/catalog_impls/src/volatile.rs +++ b/catalog_impls/src/volatile.rs @@ -1,4 +1,4 @@ -// Copyright 2022 CeresDB Project Authors. Licensed under Apache-2.0. +// Copyright 2022-2023 CeresDB Project Authors. Licensed under Apache-2.0. //! A volatile catalog implementation used for storing information about table //! and schema in memory. @@ -302,19 +302,21 @@ impl Schema for SchemaImpl { } // Do real create table. - let table_with_shards = self - .shard_tables_cache - .find_table_by_name( - &request.catalog_name, - &request.schema_name, - &request.table_name, - ) - .with_context(|| schema::CreateTable { - request: request.clone(), - msg: "table with shards is not found in the ShardTableManager", - })?; - - let request = request.into_engine_create_request(table_with_shards.table_info.id.into()); + if request.partition_info.is_none() { + let _ = self + .shard_tables_cache + .find_table_by_name( + &request.catalog_name, + &request.schema_name, + &request.table_name, + ) + .with_context(|| schema::CreateTable { + request: request.clone(), + msg: format!("table with shards is not found in the ShardTableManager, catalog_name:{}, schema_name:{}, table_name:{}", + request.catalog_name,request.schema_name,request.table_name), + })?; + } + let request = request.into_engine_create_request(None); // Table engine is able to handle duplicate table creation. let table = opts diff --git a/cluster/src/cluster_impl.rs b/cluster/src/cluster_impl.rs index a2d43ebb1e..9008aceb7f 100644 --- a/cluster/src/cluster_impl.rs +++ b/cluster/src/cluster_impl.rs @@ -1,4 +1,4 @@ -// Copyright 2022 CeresDB Project Authors. Licensed under Apache-2.0. +// Copyright 2022-2023 CeresDB Project Authors. Licensed under Apache-2.0. use std::{ sync::{Arc, Mutex, RwLock}, @@ -33,7 +33,7 @@ use tokio::{ use crate::{ config::ClusterConfig, shard_tables_cache::ShardTablesCache, topology::ClusterTopology, - Cluster, ClusterNodesNotFound, ClusterNodesResp, MetaClientFailure, OpenShard, + Cluster, ClusterNodesNotFound, ClusterNodesResp, Internal, MetaClientFailure, OpenShard, OpenShardWithCause, Result, ShardNotFound, TableNotFound, }; @@ -287,7 +287,11 @@ impl Inner { self.shard_tables_cache.try_insert_table_to_shard( update_shard_info.prev_version, ShardInfo::from(curr_shard_info), - TableInfo::from(table_info), + TableInfo::try_from(table_info) + .box_err() + .context(Internal { + msg: "Failed to parse tableInfo", + })?, ) } @@ -309,7 +313,11 @@ impl Inner { self.shard_tables_cache.try_remove_table_from_shard( update_shard_info.prev_version, ShardInfo::from(curr_shard_info), - TableInfo::from(table_info), + TableInfo::try_from(table_info) + .box_err() + .context(Internal { + msg: "Failed to parse tableInfo", + })?, ) } } diff --git a/cluster/src/lib.rs b/cluster/src/lib.rs index 004aadf178..5aa2ed09ff 100644 --- a/cluster/src/lib.rs +++ b/cluster/src/lib.rs @@ -1,4 +1,4 @@ -// Copyright 2022 CeresDB Project Authors. Licensed under Apache-2.0. +// Copyright 2022-2023 CeresDB Project Authors. Licensed under Apache-2.0. //! Cluster sub-crate includes serval functionalities for supporting CeresDB //! server to running in the distribute mode. Including: @@ -34,6 +34,9 @@ pub mod topology; #[derive(Debug, Snafu)] #[snafu(visibility = "pub")] pub enum Error { + #[snafu(display("{msg}, err:{source}"))] + Internal { msg: String, source: GenericError }, + #[snafu(display("Build meta client failed, err:{}.", source))] BuildMetaClient { source: meta_client::Error }, diff --git a/interpreters/src/table_manipulator/catalog_based.rs b/interpreters/src/table_manipulator/catalog_based.rs index c4f931b265..c718874e2f 100644 --- a/interpreters/src/table_manipulator/catalog_based.rs +++ b/interpreters/src/table_manipulator/catalog_based.rs @@ -77,6 +77,7 @@ impl TableManipulator for TableManipulatorImpl { schema_name: schema.name().to_string(), schema_id: schema.id(), table_name: table.clone(), + table_id: None, table_schema, engine, options, diff --git a/meta_client/src/lib.rs b/meta_client/src/lib.rs index e20ad4c735..13bb5f9c85 100644 --- a/meta_client/src/lib.rs +++ b/meta_client/src/lib.rs @@ -1,4 +1,4 @@ -// Copyright 2022 CeresDB Project Authors. Licensed under Apache-2.0. +// Copyright 2022-2023 CeresDB Project Authors. Licensed under Apache-2.0. use std::sync::Arc; @@ -18,6 +18,9 @@ pub mod types; #[derive(Debug, Snafu)] #[snafu(visibility = "pub")] pub enum Error { + #[snafu(display("{msg}, err:{source}"))] + Convert { msg: String, source: GenericError }, + #[snafu(display("Missing shard info, msg:{}.\nBacktrace:\n{}", msg, backtrace))] MissingShardInfo { msg: String, backtrace: Backtrace }, diff --git a/meta_client/src/meta_impl.rs b/meta_client/src/meta_impl.rs index dec07ee510..ebc9b7bd0d 100644 --- a/meta_client/src/meta_impl.rs +++ b/meta_client/src/meta_impl.rs @@ -1,4 +1,4 @@ -// Copyright 2022 CeresDB Project Authors. Licensed under Apache-2.0. +// Copyright 2022-2023 CeresDB Project Authors. Licensed under Apache-2.0. use std::sync::Arc; @@ -153,7 +153,7 @@ impl MetaClient for MetaClientImpl { info!("Meta client finish dropping table, resp:{:?}", pb_resp); check_response_header(&pb_resp.header)?; - Ok(DropTableResponse::from(pb_resp)) + DropTableResponse::try_from(pb_resp) } async fn get_tables_of_shards( diff --git a/meta_client/src/types.rs b/meta_client/src/types.rs index 7b60691102..0d389452a5 100644 --- a/meta_client/src/types.rs +++ b/meta_client/src/types.rs @@ -1,4 +1,4 @@ -// Copyright 2022 CeresDB Project Authors. Licensed under Apache-2.0. +// Copyright 2022-2023 CeresDB Project Authors. Licensed under Apache-2.0. use std::{collections::HashMap, sync::Arc}; @@ -8,12 +8,12 @@ use common_types::{ schema::{SchemaId, SchemaName}, table::{TableId, TableName}, }; -use common_util::config::ReadableDuration; +use common_util::{config::ReadableDuration, error::BoxError}; use serde::Deserialize; -use snafu::OptionExt; +use snafu::{OptionExt, ResultExt}; use table_engine::partition::PartitionInfo; -use crate::{Error, MissingShardInfo, MissingTableInfo, Result}; +use crate::{Convert, Error, MissingShardInfo, MissingTableInfo, Result}; pub type ClusterNodesRef = Arc>; #[derive(Debug, Clone)] @@ -87,16 +87,30 @@ pub struct TableInfo { pub name: String, pub schema_id: SchemaId, pub schema_name: String, + pub partition_info: Option, } -impl From for TableInfo { - fn from(pb_table_info: meta_service_pb::TableInfo) -> Self { - TableInfo { +impl TryFrom for TableInfo { + type Error = Error; + + fn try_from(pb_table_info: meta_service_pb::TableInfo) -> Result { + let partition_info = match pb_table_info.partition_info { + Some(partition_info) => Some( + PartitionInfo::try_from(partition_info) + .box_err() + .context(Convert { + msg: "Failed to parse partition", + })?, + ), + None => None, + }; + Ok(TableInfo { id: pb_table_info.id, name: pb_table_info.name, schema_id: pb_table_info.schema_id, schema_name: pb_table_info.schema_name, - } + partition_info, + }) } } @@ -274,8 +288,8 @@ impl TryFrom for TablesOfShard { tables: pb_tables_of_shard .tables .into_iter() - .map(Into::into) - .collect(), + .map(TryInto::::try_into) + .collect::>>()?, }) } } @@ -340,7 +354,7 @@ impl TryFrom for CreateTableResponse { })?; Ok(Self { - created_table: TableInfo::from(pb_table_info), + created_table: TableInfo::try_from(pb_table_info)?, shard_info: ShardInfo::from(pb_shard_info), }) } @@ -363,11 +377,13 @@ impl From for meta_service_pb::DropTableRequest { } } -impl From for DropTableResponse { - fn from(pb_resp: meta_service_pb::DropTableResponse) -> Self { - Self { - dropped_table: pb_resp.dropped_table.map(TableInfo::from), - } +impl TryFrom for DropTableResponse { + type Error = Error; + + fn try_from(pb_resp: meta_service_pb::DropTableResponse) -> Result { + Ok(Self { + dropped_table: pb_resp.dropped_table.map(TableInfo::try_from).transpose()?, + }) } } @@ -441,7 +457,7 @@ impl TryFrom for RouteEntry { msg: "table info is missing in route entry", })?; Ok(RouteEntry { - table: TableInfo::from(table_info), + table: TableInfo::try_from(table_info)?, node_shards, }) } diff --git a/remote_engine_client/src/lib.rs b/remote_engine_client/src/lib.rs index 0dcba650eb..1682d419d6 100644 --- a/remote_engine_client/src/lib.rs +++ b/remote_engine_client/src/lib.rs @@ -18,7 +18,7 @@ use std::{ use async_trait::async_trait; use common_types::{record_batch::RecordBatch, schema::RecordSchema}; use common_util::error::BoxError; -use config::Config; +pub use config::Config; use futures::{Stream, StreamExt}; use router::RouterRef; use snafu::ResultExt; diff --git a/router/src/cluster_based.rs b/router/src/cluster_based.rs index 38f31ab53e..7b2bb47db9 100644 --- a/router/src/cluster_based.rs +++ b/router/src/cluster_based.rs @@ -1,20 +1,22 @@ -// Copyright 2022 CeresDB Project Authors. Licensed under Apache-2.0. +// Copyright 2022-2023 CeresDB Project Authors. Licensed under Apache-2.0. //! A router based on the [`cluster::Cluster`]. use async_trait::async_trait; -use ceresdbproto::storage::{Route, RouteRequest}; +use ceresdbproto::storage::RouteRequest; use cluster::ClusterRef; use common_util::error::BoxError; -use meta_client::types::RouteTablesRequest; +use meta_client::types::{RouteTablesRequest, TableInfo}; use moka::future::Cache; use snafu::ResultExt; -use crate::{endpoint::Endpoint, OtherWithCause, ParseEndpoint, Result, RouteCacheConfig, Router}; +use crate::{ + endpoint::Endpoint, OtherWithCause, ParseEndpoint, Result, RouteCacheConfig, RouteData, Router, +}; pub struct ClusterBasedRouter { cluster: ClusterRef, - cache: Option>, + cache: Option>, } impl ClusterBasedRouter { @@ -36,7 +38,7 @@ impl ClusterBasedRouter { /// route table from local cache, return cache routes and tables which are /// not in cache - fn route_from_cache(&self, tables: Vec, routes: &mut Vec) -> Vec { + fn route_from_cache(&self, tables: Vec, routes: &mut Vec) -> Vec { let mut miss = vec![]; if let Some(cache) = &self.cache { @@ -56,18 +58,19 @@ impl ClusterBasedRouter { } /// Make a route according to the table name and the raw endpoint. -fn make_route(table_name: &str, endpoint: &str) -> Result { +fn make_route(table: TableInfo, endpoint: &str) -> Result { let endpoint: Endpoint = endpoint.parse().context(ParseEndpoint { endpoint })?; - Ok(Route { - table: table_name.to_string(), + Ok(RouteData { + table_name: table.name.clone(), + table: Some(table), endpoint: Some(endpoint.into()), }) } #[async_trait] impl Router for ClusterBasedRouter { - async fn route(&self, req: RouteRequest) -> Result> { + async fn route(&self, req: RouteRequest) -> Result> { let req_ctx = req.context.unwrap(); // Firstly route table from local cache. @@ -96,7 +99,7 @@ impl Router for ClusterBasedRouter { for (table_name, route_entry) in route_resp.entries { for node_shard in route_entry.node_shards { if node_shard.shard_info.is_leader() { - let route = make_route(&table_name, &node_shard.endpoint)?; + let route = make_route(route_entry.table.clone(), &node_shard.endpoint)?; if let Some(cache) = &self.cache { // There may be data race here, and it is acceptable currently. cache.insert(table_name.clone(), route.clone()).await; diff --git a/router/src/lib.rs b/router/src/lib.rs index e2fa1e752f..5ff6bca616 100644 --- a/router/src/lib.rs +++ b/router/src/lib.rs @@ -1,4 +1,4 @@ -// Copyright 2022 CeresDB Project Authors. Licensed under Apache-2.0. +// Copyright 2022-2023 CeresDB Project Authors. Licensed under Apache-2.0. pub mod cluster_based; pub mod endpoint; @@ -7,13 +7,16 @@ pub mod rule_based; use std::{sync::Arc, time::Duration}; use async_trait::async_trait; -use ceresdbproto::storage::{Route, RouteRequest}; +use ceresdbproto::storage::RouteRequest; pub use cluster_based::ClusterBasedRouter; use common_util::{config::ReadableDuration, define_result}; +use meta_client::types::TableInfo; pub use rule_based::{RuleBasedRouter, RuleList}; use serde::{Deserialize, Serialize}; use snafu::{Backtrace, Snafu}; +use crate::endpoint::Endpoint; + #[derive(Snafu, Debug)] #[snafu(visibility(pub))] pub enum Error { @@ -59,9 +62,16 @@ define_result!(Error); pub type RouterRef = Arc; +#[derive(Debug, Clone)] +pub struct RouteData { + pub table_name: String, + pub table: Option, + pub endpoint: Option, +} + #[async_trait] pub trait Router { - async fn route(&self, req: RouteRequest) -> Result>; + async fn route(&self, req: RouteRequest) -> Result>; } #[derive(Clone, Debug, Deserialize, Serialize)] diff --git a/router/src/rule_based.rs b/router/src/rule_based.rs index 608e0fbb9b..b330d9de14 100644 --- a/router/src/rule_based.rs +++ b/router/src/rule_based.rs @@ -1,18 +1,18 @@ -// Copyright 2022 CeresDB Project Authors. Licensed under Apache-2.0. +// Copyright 2022-2023 CeresDB Project Authors. Licensed under Apache-2.0. //! A router based on rules. use std::collections::HashMap; use async_trait::async_trait; -use ceresdbproto::storage::{self, Route, RouteRequest}; +use ceresdbproto::storage::RouteRequest; use cluster::config::SchemaConfig; use log::info; use meta_client::types::ShardId; use serde::{Deserialize, Serialize}; use snafu::{ensure, OptionExt}; -use crate::{endpoint::Endpoint, hash, Result, RouteNotFound, Router, ShardNotFound}; +use crate::{endpoint::Endpoint, hash, Result, RouteData, RouteNotFound, Router, ShardNotFound}; pub type ShardNodes = HashMap; @@ -138,7 +138,7 @@ impl RuleBasedRouter { #[async_trait] impl Router for RuleBasedRouter { - async fn route(&self, req: RouteRequest) -> Result> { + async fn route(&self, req: RouteRequest) -> Result> { let req_ctx = req.context.unwrap(); let schema = &req_ctx.database; if let Some(shard_nodes) = self.cluster_view.schema_shards.get(schema) { @@ -150,18 +150,19 @@ impl Router for RuleBasedRouter { // TODO(yingwen): Better way to get total shard number let total_shards = shard_nodes.len(); let mut route_results = Vec::with_capacity(req.tables.len()); - for table in req.tables { - let shard_id = Self::route_table(&table, rule_list_opt, total_shards); + for table_name in req.tables { + let shard_id = Self::route_table(&table_name, rule_list_opt, total_shards); let endpoint = shard_nodes.get(&shard_id).with_context(|| ShardNotFound { schema, - table: &table, + table: &table_name, })?; - let pb_endpoint = storage::Endpoint::from(endpoint.clone()); - let route = Route { - table, - endpoint: Some(pb_endpoint), + // let pb_endpoint = storage::Endpoint::from(endpoint.clone()); + let route = RouteData { + table_name, + table: None, + endpoint: Some(endpoint.to_owned()), }; route_results.push(route); } diff --git a/server/Cargo.toml b/server/Cargo.toml index c61569ff11..94bfa89965 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -40,6 +40,7 @@ prometheus = { workspace = true } prometheus-static-metric = { workspace = true } prost = { workspace = true } query_engine = { workspace = true } +remote_engine_client = { workspace = true } router = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } diff --git a/server/src/grpc/meta_event_service/mod.rs b/server/src/grpc/meta_event_service/mod.rs index 196ebef9ae..6effa5c91f 100644 --- a/server/src/grpc/meta_event_service/mod.rs +++ b/server/src/grpc/meta_event_service/mod.rs @@ -353,7 +353,7 @@ async fn handle_create_table_on_shard( code: StatusCode::BadRequest, msg: "current shard info is missing ine CreateTableOnShardRequest", })?; - let table = request.table_info.context(ErrNoCause { + let table_info = request.table_info.context(ErrNoCause { code: StatusCode::BadRequest, msg: "table info is missing in the CreateTableOnShardRequest", })?; @@ -361,7 +361,7 @@ async fn handle_create_table_on_shard( // Create the table by catalog manager afterwards. let default_catalog = ctx.default_catalog()?; - let schema = find_schema(default_catalog, &table.schema_name)?; + let schema = find_schema(default_catalog, &table_info.schema_name)?; let table_schema = SchemaEncoder::default() .decode(&request.encoded_schema) @@ -374,7 +374,7 @@ async fn handle_create_table_on_shard( ), })?; - let partition_info = match table.partition_info { + let partition_info = match table_info.partition_info { Some(v) => Some( PartitionInfo::try_from(v.clone()) .box_err() @@ -388,9 +388,10 @@ async fn handle_create_table_on_shard( let create_table_request = CreateTableRequest { catalog_name: ctx.catalog_manager.default_catalog_name().to_string(), - schema_name: table.schema_name, - schema_id: SchemaId::from_u32(table.schema_id), - table_name: table.name, + schema_name: table_info.schema_name, + schema_id: SchemaId::from_u32(table_info.schema_id), + table_name: table_info.name, + table_id: Some(TableId::new(table_info.id)), table_schema, engine: request.engine, options: request.options, diff --git a/server/src/grpc/mod.rs b/server/src/grpc/mod.rs index 26b4c08afc..f40578a139 100644 --- a/server/src/grpc/mod.rs +++ b/server/src/grpc/mod.rs @@ -1,4 +1,4 @@ -// Copyright 2022 CeresDB Project Authors. Licensed under Apache-2.0. +// Copyright 2022-2023 CeresDB Project Authors. Licensed under Apache-2.0. //! Grpc services @@ -214,6 +214,7 @@ pub struct Builder { opened_wals: Option, schema_config_provider: Option, forward_config: Option, + remote_engine_client_config: remote_engine_client::Config, auto_create_table: bool, } @@ -231,6 +232,7 @@ impl Builder { opened_wals: None, schema_config_provider: None, forward_config: None, + remote_engine_client_config: remote_engine_client::config::Config::default(), auto_create_table: true, } } @@ -287,6 +289,11 @@ impl Builder { self } + pub fn remote_engine_client_config(mut self, config: remote_engine_client::Config) -> Self { + self.remote_engine_client_config = config; + self + } + pub fn timeout(mut self, timeout: Option) -> Self { self.timeout = timeout; self @@ -327,11 +334,13 @@ impl Builder { }; let forward_config = self.forward_config.unwrap_or_default(); + let remote_engine_client_config = self.remote_engine_client_config; let bg_runtime = runtimes.bg_runtime.clone(); let proxy = Proxy::try_new( router, instance, forward_config, + remote_engine_client_config, self.local_endpoint.context(MissingLocalEndpoint)?, self.resp_compress_min_length, self.auto_create_table, diff --git a/server/src/proxy/forward.rs b/server/src/proxy/forward.rs index a54e71e365..eff41d5d7c 100644 --- a/server/src/proxy/forward.rs +++ b/server/src/proxy/forward.rs @@ -1,4 +1,4 @@ -// Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. +// Copyright 2022-2023 CeresDB Project Authors. Licensed under Apache-2.0. //! Forward for grpc services use std::{ @@ -12,7 +12,8 @@ use async_trait::async_trait; use ceresdbproto::storage::{ storage_service_client::StorageServiceClient, RequestContext, RouteRequest, }; -use log::{debug, error, warn}; +use log::{debug, error, info, warn}; +use meta_client::types::TableInfo; use router::{endpoint::Endpoint, RouterRef}; use serde::{Deserialize, Serialize}; use snafu::{Backtrace, ResultExt, Snafu}; @@ -179,6 +180,7 @@ pub struct Forwarder { /// If no forwarding happens, [`Original`] can be used. pub enum ForwardResult { Original, + OriginalPartitionTableInfo(TableInfo), Forwarded(std::result::Result), } @@ -278,15 +280,40 @@ impl Forwarder { let endpoint = match self.router.route(route_req).await { Ok(mut routes) => { - if routes.len() != 1 || routes[0].endpoint.is_none() { + if routes.len() == 1 { + if routes[0].table.is_none() { + info!("xxxxroute table is none",); + return Ok(ForwardResult::Original); + } + + let table = routes[0].table.clone().unwrap(); + + if table.partition_info.is_some() { + info!("xxxxroute partition_info is some",); + return Ok(ForwardResult::OriginalPartitionTableInfo(table)); + } + + if routes[0].endpoint.is_none() { + warn!( + "Fail to forward request for empty route results, routes result:{:?}, req:{:?}", + routes, req + ); + return Ok(ForwardResult::Original); + } + let endpoint = routes.remove(0).endpoint.unwrap(); + if self.is_local_endpoint(&endpoint) { + info!("xxxxroute local endpoint",); + return Ok(ForwardResult::Original); + } + endpoint + } else { warn!( "Fail to forward request for multiple or empty route results, routes result:{:?}, req:{:?}", routes, req ); + // TODO: shall we return an error? return Ok(ForwardResult::Original); } - - Endpoint::from(routes.remove(0).endpoint.unwrap()) } Err(e) => { error!("Fail to route request, req:{:?}, err:{}", req, e); @@ -294,10 +321,6 @@ impl Forwarder { } }; - if self.is_local_endpoint(&endpoint) { - return Ok(ForwardResult::Original); - } - // Update the request. { // TODO: we should use the timeout from the original request. diff --git a/server/src/proxy/grpc/route.rs b/server/src/proxy/grpc/route.rs index 8e74185837..bb0060ebe5 100644 --- a/server/src/proxy/grpc/route.rs +++ b/server/src/proxy/grpc/route.rs @@ -1,6 +1,6 @@ // Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. -use ceresdbproto::storage::{RouteRequest, RouteResponse}; +use ceresdbproto::storage::{Route, RouteRequest, RouteResponse}; use common_util::error::BoxError; use http::StatusCode; use log::error; @@ -29,7 +29,16 @@ impl Proxy { } Ok(v) => { resp.header = Some(error::build_ok_header()); - resp.routes = v; + + resp.routes = v + .into_iter() + .map(|r| { + let mut router = Route::default(); + router.table = r.table_name; + router.endpoint = r.endpoint.map(Into::into); + router + }) + .collect(); } } resp diff --git a/server/src/proxy/grpc/sql_query.rs b/server/src/proxy/grpc/sql_query.rs index f140d8876c..19aaa034a0 100644 --- a/server/src/proxy/grpc/sql_query.rs +++ b/server/src/proxy/grpc/sql_query.rs @@ -1,10 +1,11 @@ -// Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. +// Copyright 2022-2023 CeresDB Project Authors. Licensed under Apache-2.0. //! Query handler use std::{sync::Arc, time::Instant}; use arrow_ext::ipc::{CompressOptions, CompressionMethod, RecordBatchesEncoder}; +use catalog::schema::{CreateOptions, CreateTableRequest}; use ceresdbproto::{ common::ResponseHeader, storage::{ @@ -12,19 +13,26 @@ use ceresdbproto::{ ArrowPayload, SqlQueryRequest, SqlQueryResponse, }, }; -use common_types::{record_batch::RecordBatch, request_id::RequestId}; +use common_types::{record_batch::RecordBatch, request_id::RequestId, table::DEFAULT_SHARD_ID}; use common_util::{error::BoxError, time::InstantExt}; use futures::{stream, stream::BoxStream, FutureExt, StreamExt}; use http::StatusCode; use interpreters::{context::Context as InterpreterContext, factory::Factory, interpreter::Output}; use log::{error, info, warn}; +use meta_client::types::TableInfo; use query_engine::executor::Executor as QueryExecutor; use router::endpoint::Endpoint; -use snafu::{ensure, ResultExt}; +use snafu::{ensure, OptionExt, ResultExt}; use sql::{ frontend::{Context as SqlContext, Frontend}, provider::CatalogMetaProvider, }; +use table_engine::{ + engine::TableState, + partition::{format_sub_partition_table_name, PartitionInfo}, + remote::model::{GetTableInfoRequest, TableIdentifier}, + table::TableId, +}; use tokio::sync::mpsc; use tokio_stream::wrappers::ReceiverStream; use tonic::{transport::Channel, IntoRequest}; @@ -74,12 +82,16 @@ impl Proxy { ctx: Context, req: SqlQueryRequest, ) -> Result { - let req = match self.maybe_forward_sql_query(&req).await { - Some(resp) => return resp, - None => req, + let (req, table) = match self.maybe_forward_sql_query(&req).await { + Some(resp) => match resp { + ForwardResult::Forwarded(resp) => return resp, + ForwardResult::Original => (req, None), + ForwardResult::OriginalPartitionTableInfo(table) => (req, Some(table)), + }, + None => (req, None), }; - let output = self.fetch_sql_query_output(ctx, &req).await?; + let output = self.fetch_sql_query_output(ctx, &req, table).await?; convert_output(&output, self.resp_compress_min_length) } @@ -88,18 +100,22 @@ impl Proxy { ctx: Context, req: SqlQueryRequest, ) -> Result> { - let req = match self - .clone() - .maybe_forward_stream_sql_query(req.clone()) - .await - { - Some(resp) => return resp, - None => req, + let (req, table) = match self.clone().maybe_forward_stream_sql_query(&req).await { + Some(resp) => match resp { + ForwardResult::Forwarded(resp) => return resp, + ForwardResult::Original => (req, None), + ForwardResult::OriginalPartitionTableInfo(table) => (req, Some(table)), + }, + None => (req, None), }; + let (tx, rx) = mpsc::channel(STREAM_QUERY_CHANNEL_LEN); let runtime = ctx.runtime.clone(); let resp_compress_min_length = self.resp_compress_min_length; - let output = self.as_ref().fetch_sql_query_output(ctx, &req).await?; + let output = self + .as_ref() + .fetch_sql_query_output(ctx, &req, table) + .await?; runtime.spawn(async move { match output { Output::AffectedRows(rows) => { @@ -132,7 +148,7 @@ impl Proxy { async fn maybe_forward_sql_query( &self, req: &SqlQueryRequest, - ) -> Option> { + ) -> Option> { if req.tables.len() != 1 { warn!("Unable to forward sql query without exactly one table, req:{req:?}",); @@ -164,11 +180,9 @@ impl Proxy { Box::new(query) as _ }; - match self.forwarder.forward(forward_req, do_query).await { - Ok(forward_res) => match forward_res { - ForwardResult::Forwarded(v) => Some(v), - ForwardResult::Original => None, - }, + let forward_result = self.forwarder.forward(forward_req, do_query).await; + match forward_result { + Ok(forward_res) => Some(forward_res), Err(e) => { error!("Failed to forward sql req but the error is ignored, err:{e}"); None @@ -178,8 +192,8 @@ impl Proxy { async fn maybe_forward_stream_sql_query( self: Arc, - req: SqlQueryRequest, - ) -> Option>> { + req: &SqlQueryRequest, + ) -> Option, Error>> { if req.tables.len() != 1 { warn!("Unable to forward sql query without exactly one table, req:{req:?}",); @@ -226,11 +240,10 @@ impl Proxy { Box::new(query) as _ }; - match self.forwarder.forward(forward_req, do_query).await { - Ok(forward_res) => match forward_res { - ForwardResult::Forwarded(v) => Some(v), - ForwardResult::Original => None, - }, + let forward_result = self.forwarder.forward(forward_req, do_query).await; + + match forward_result { + Ok(forward_res) => Some(forward_res), Err(e) => { error!("Failed to forward stream sql req but the error is ignored, err:{e}"); None @@ -238,7 +251,12 @@ impl Proxy { } } - async fn fetch_sql_query_output(&self, ctx: Context, req: &SqlQueryRequest) -> Result { + async fn fetch_sql_query_output( + &self, + ctx: Context, + req: &SqlQueryRequest, + table_info: Option, + ) -> Result { let request_id = RequestId::next_id(); let begin_instant = Instant::now(); let deadline = ctx.timeout.map(|t| begin_instant + t); @@ -291,6 +309,13 @@ impl Proxy { ), } ); + // Open partition table if needed. + if let Some(v) = table_info { + if v.partition_info.is_some() { + self.open_partition_table_if_not_exist(catalog, schema, v) + .await?; + } + } // Create logical plan // Note: Remember to store sql in error when creating logical plan @@ -373,6 +398,100 @@ impl Proxy { Ok(output) } + + async fn open_partition_table_if_not_exist( + &self, + catalog_name: &str, + schema_name: &str, + table_info: TableInfo, + ) -> Result<()> { + let catalog = self + .instance + .catalog_manager + .catalog_by_name(catalog_name) + .box_err() + .with_context(|| ErrWithCause { + code: StatusCode::INTERNAL_SERVER_ERROR, + msg: format!("Failed to find catalog, catalog_name:{catalog_name}"), + })? + .with_context(|| ErrNoCause { + code: StatusCode::BAD_REQUEST, + msg: format!("Catalog not found, catalog_name:{catalog_name}"), + })?; + + // TODO: support create schema if not exist + let schema = catalog + .schema_by_name(schema_name) + .box_err() + .with_context(|| ErrWithCause { + code: StatusCode::INTERNAL_SERVER_ERROR, + msg: format!("Failed to find schema, schema_name:{schema_name}"), + })? + .context(ErrNoCause { + code: StatusCode::BAD_REQUEST, + msg: format!("Schema not found, schema_name:{schema_name}"), + })?; + let table = schema + .table_by_name(&table_info.name) + .box_err() + .with_context(|| ErrWithCause { + code: StatusCode::INTERNAL_SERVER_ERROR, + msg: format!("Failed to find table, table_name:{}", table_info.name), + })?; + + if table.is_some() { + return Ok(()); + } + + // If table not exists, open it. + if let Some(v) = &table_info.partition_info { + // Get table_schema from first sub partition table. + let first_sub_partition_table_name = + get_sub_partition_name(&table_info.name, v, 0usize); + let table = self + .remote_engine_ref + .get_table_info(GetTableInfoRequest { + table: TableIdentifier { + catalog: catalog_name.to_string(), + schema: schema_name.to_string(), + table: first_sub_partition_table_name, + }, + }) + .await + .box_err() + .with_context(|| ErrWithCause { + code: StatusCode::INTERNAL_SERVER_ERROR, + msg: "Failed to get table", + })?; + + let create_table_request = CreateTableRequest { + catalog_name: catalog_name.to_string(), + schema_name: schema_name.to_string(), + schema_id: schema.id(), + table_name: table_info.name, + table_id: Some(TableId::new(table_info.id)), + table_schema: table.table_schema, + engine: table.engine, + options: Default::default(), + state: TableState::Stable, + shard_id: DEFAULT_SHARD_ID, + partition_info: table_info.partition_info, + }; + let create_opts = CreateOptions { + table_engine: self.instance.table_engine.clone(), + create_if_not_exists: true, + }; + schema + .create_table(create_table_request.clone(), create_opts) + .await + .box_err() + .with_context(|| ErrWithCause { + code: StatusCode::INTERNAL_SERVER_ERROR, + msg: format!("Failed to create table, request:{create_table_request:?}"), + })?; + } + return Ok(()); + } } // TODO(chenxiang): Output can have both `rows` and `affected_rows` @@ -496,3 +615,8 @@ impl QueryResponseWriter { Ok(resp) } } + +fn get_sub_partition_name(table_name: &str, partition_info: &PartitionInfo, id: usize) -> String { + let partition_name = partition_info.get_definitions()[id].name.clone(); + format_sub_partition_table_name(table_name, &partition_name) +} diff --git a/server/src/proxy/mod.rs b/server/src/proxy/mod.rs index d35c638573..bc4cbc7490 100644 --- a/server/src/proxy/mod.rs +++ b/server/src/proxy/mod.rs @@ -12,8 +12,10 @@ use std::{str::FromStr, sync::Arc, time::Duration}; use common_util::{error::BoxError, runtime::Runtime}; use query_engine::executor::Executor as QueryExecutor; +use remote_engine_client::RemoteEngineImpl; use router::{endpoint::Endpoint, Router}; use snafu::ResultExt; +use table_engine::remote::RemoteEngineRef; use crate::{ instance::InstanceRef, @@ -31,6 +33,7 @@ pub struct Proxy { resp_compress_min_length: usize, auto_create_table: bool, schema_config_provider: SchemaConfigProviderRef, + remote_engine_ref: RemoteEngineRef, } impl Proxy { @@ -38,11 +41,16 @@ impl Proxy { router: Arc, instance: InstanceRef, forward_config: forward::Config, + remote_engine_client_config: remote_engine_client::config::Config, local_endpoint: String, resp_compress_min_length: usize, auto_create_table: bool, schema_config_provider: SchemaConfigProviderRef, ) -> Result { + let remote_engine_ref = Arc::new(RemoteEngineImpl::new( + remote_engine_client_config, + router.clone(), + )); let local_endpoint = Endpoint::from_str(&local_endpoint).with_context(|| Internal { msg: format!("invalid local endpoint, input:{local_endpoint}"), })?; @@ -60,6 +68,7 @@ impl Proxy { resp_compress_min_length, auto_create_table, schema_config_provider, + remote_engine_ref, }) } } diff --git a/server/src/server.rs b/server/src/server.rs index e534ddb5cd..40c0b420b9 100644 --- a/server/src/server.rs +++ b/server/src/server.rs @@ -1,4 +1,4 @@ -// Copyright 2022 CeresDB Project Authors. Licensed under Apache-2.0. +// Copyright 2022-2023 CeresDB Project Authors. Licensed under Apache-2.0. //! Server @@ -174,6 +174,7 @@ impl Server { #[must_use] pub struct Builder { server_config: ServerConfig, + remote_engine_client_config: remote_engine_client::config::Config, node_addr: String, config_content: Option, engine_runtimes: Option>, @@ -195,6 +196,7 @@ impl Builder { pub fn new(config: ServerConfig) -> Self { Self { server_config: config, + remote_engine_client_config: remote_engine_client::Config::default(), node_addr: "".to_string(), config_content: None, engine_runtimes: None, @@ -374,6 +376,7 @@ impl Builder { .opened_wals(opened_wals) .schema_config_provider(provider) .forward_config(self.server_config.forward) + .remote_engine_client_config(self.remote_engine_client_config) .timeout(self.server_config.timeout.map(|v| v.0)) .auto_create_table(self.server_config.auto_create_table) .build() diff --git a/sql/src/frontend.rs b/sql/src/frontend.rs index 1000e441dd..da2e7ba136 100644 --- a/sql/src/frontend.rs +++ b/sql/src/frontend.rs @@ -1,4 +1,4 @@ -// Copyright 2022 CeresDB Project Authors. Licensed under Apache-2.0. +// Copyright 2022-2023 CeresDB Project Authors. Licensed under Apache-2.0. //! Frontend @@ -9,6 +9,7 @@ use cluster::config::SchemaConfig; use common_types::request_id::RequestId; use influxql_parser::statement::Statement as InfluxqlStatement; use snafu::{Backtrace, OptionExt, ResultExt, Snafu}; +use sqlparser::ast::{SetExpr, Statement as SqlStatement, TableFactor}; use table_engine::table; use crate::{ @@ -156,3 +157,52 @@ impl Frontend

{ .context(CreatePlan) } } + +pub fn parse_table_name(statements: &StatementVec) -> Option { + match &statements[0] { + Statement::Standard(s) => match *s.clone() { + SqlStatement::Insert { table_name, .. } => Some(table_name.to_string()), + SqlStatement::Explain { statement, .. } => { + if let SqlStatement::Query(q) = *statement { + match *q.body { + SetExpr::Select(select) => { + if select.from.len() != 1 { + None + } else if let TableFactor::Table { name, .. } = &select.from[0].relation + { + Some(name.to_string()) + } else { + None + } + } + _ => None, + } + } else { + None + } + } + SqlStatement::Query(q) => match *q.body { + SetExpr::Select(select) => { + if select.from.len() != 1 { + None + } else if let TableFactor::Table { name, .. } = &select.from[0].relation { + Some(name.to_string()) + } else { + None + } + } + _ => None, + }, + _ => None, + }, + Statement::Create(s) => Some(s.table_name.to_string()), + Statement::Drop(s) => Some(s.table_name.to_string()), + Statement::Describe(s) => Some(s.table_name.to_string()), + Statement::AlterModifySetting(s) => Some(s.table_name.to_string()), + Statement::AlterAddColumn(s) => Some(s.table_name.to_string()), + Statement::ShowCreate(s) => Some(s.table_name.to_string()), + Statement::ShowTables(_s) => None, + Statement::ShowDatabases => None, + Statement::Exists(s) => Some(s.table_name.to_string()), + } +} diff --git a/table_engine/src/partition/mod.rs b/table_engine/src/partition/mod.rs index 68f3bc23a2..4e3d45548b 100644 --- a/table_engine/src/partition/mod.rs +++ b/table_engine/src/partition/mod.rs @@ -1,4 +1,4 @@ -// Copyright 2022 CeresDB Project Authors. Licensed under Apache-2.0. +// Copyright 2022-2023 CeresDB Project Authors. Licensed under Apache-2.0. //! Partitioned table supports @@ -232,10 +232,12 @@ impl TryFrom for PartitionInfo { } } +#[inline] pub fn format_sub_partition_table_name(table_name: &str, partition_name: &str) -> String { format!("{PARTITION_TABLE_PREFIX}{table_name}_{partition_name}") } +#[inline] pub fn is_sub_partition_table(table_name: &str) -> bool { table_name.starts_with(PARTITION_TABLE_PREFIX) } From 80dda5f8bd8fb3c0599b6fda757e291e7d79c2a7 Mon Sep 17 00:00:00 2001 From: "chunshao.rcs" Date: Mon, 3 Apr 2023 19:49:04 +0800 Subject: [PATCH 2/9] refactor code --- analytic_engine/src/manifest/details.rs | 41 +++----------- catalog_impls/src/table_based.rs | 1 + catalog_impls/src/volatile.rs | 1 + integration_tests/src/database.rs | 66 ++--------------------- meta_client/src/types.rs | 2 +- remote_engine_client/src/cached_router.rs | 3 +- router/src/cluster_based.rs | 7 +-- router/src/rule_based.rs | 1 - server/src/proxy/forward.rs | 31 +++++------ server/src/proxy/grpc/route.rs | 12 +++-- server/src/proxy/grpc/sql_query.rs | 2 +- server/src/proxy/mod.rs | 1 + sql/src/frontend.rs | 46 ++++++++++++++-- 13 files changed, 84 insertions(+), 130 deletions(-) diff --git a/analytic_engine/src/manifest/details.rs b/analytic_engine/src/manifest/details.rs index 5c589448f9..2071a98e72 100644 --- a/analytic_engine/src/manifest/details.rs +++ b/analytic_engine/src/manifest/details.rs @@ -689,17 +689,13 @@ where mod tests { use std::{path::PathBuf, sync::Arc, vec}; - use bytes::Bytes; use common_types::{ column_schema, datum::DatumKind, schema, schema::Schema, table::DEFAULT_SHARD_ID, }; use common_util::{runtime, runtime::Runtime, tests::init_log_for_test}; use futures::future::BoxFuture; use object_store::LocalFileSystem; - use table_engine::{ - partition::{HashPartitionInfo, PartitionDefinition, PartitionInfo}, - table::{SchemaId, TableId, TableSeqGenerator}, - }; + use table_engine::table::{SchemaId, TableId, TableSeqGenerator}; use wal::rocks_impl::manager::Builder as WalBuilder; use super::*; @@ -832,15 +828,10 @@ mod tests { table_name, schema: common_types::tests::build_schema(), opts: TableOptions::default(), - partition_info: None, }) } - fn meta_update_add_table_with_partition_info( - &self, - table_id: TableId, - partition_info: Option, - ) -> MetaUpdate { + fn meta_update_add_table_with_partition_info(&self, table_id: TableId) -> MetaUpdate { let table_name = Self::table_name_from_id(table_id); MetaUpdate::AddTable(AddTableMeta { space_id: self.schema_id.as_u32(), @@ -848,7 +839,6 @@ mod tests { table_name, schema: common_types::tests::build_schema(), opts: TableOptions::default(), - partition_info, }) } @@ -898,7 +888,6 @@ mod tests { async fn add_table_with_manifest( &self, table_id: TableId, - partition_info: Option, manifest_data_builder: &mut TableManifestDataBuilder, manifest: &ManifestImpl, ) { @@ -906,8 +895,7 @@ mod tests { shard_id: DEFAULT_SHARD_ID, }; - let add_table = - self.meta_update_add_table_with_partition_info(table_id, partition_info); + let add_table = self.meta_update_add_table_with_partition_info(table_id); let update_req = { MetaUpdateRequest { shard_info, @@ -968,7 +956,7 @@ mod tests { manifest_data_builder: &mut TableManifestDataBuilder, ) { let manifest = self.open_manifest().await; - self.add_table_with_manifest(table_id, None, manifest_data_builder, &manifest) + self.add_table_with_manifest(table_id, manifest_data_builder, &manifest) .await; } @@ -1129,26 +1117,11 @@ mod tests { runtime.block_on(async move { let table_id = ctx.alloc_table_id(); - let default_version = 0; - let partition_info = Some(PartitionInfo::Hash(HashPartitionInfo { - version: default_version, - definitions: vec![PartitionDefinition { - name: "p0".to_string(), - origin_name: Some("region0".to_string()), - }], - expr: Bytes::from("test"), - linear: false, - })); let location = WalLocation::new(DEFAULT_SHARD_ID as u64, table_id.as_u64()); let mut manifest_data_builder = TableManifestDataBuilder::default(); let manifest = ctx.open_manifest().await; - ctx.add_table_with_manifest( - table_id, - partition_info, - &mut manifest_data_builder, - &manifest, - ) - .await; + ctx.add_table_with_manifest(table_id, &mut manifest_data_builder, &manifest) + .await; manifest .maybe_do_snapshot(ctx.schema_id.as_u32(), table_id, location, true) @@ -1189,7 +1162,7 @@ mod tests { }; let mut manifest_data_builder = TableManifestDataBuilder::default(); let manifest = ctx.open_manifest().await; - ctx.add_table_with_manifest(table_id, None, &mut manifest_data_builder, &manifest) + ctx.add_table_with_manifest(table_id, &mut manifest_data_builder, &manifest) .await; for i in 0..500 { diff --git a/catalog_impls/src/table_based.rs b/catalog_impls/src/table_based.rs index 6de5f195b7..5bb0e60aaf 100644 --- a/catalog_impls/src/table_based.rs +++ b/catalog_impls/src/table_based.rs @@ -950,6 +950,7 @@ mod tests { schema_name: schema.name().to_string(), schema_id: schema.id(), table_name: table_name.to_string(), + table_id: None, table_schema: common_types::tests::build_schema(), engine: ANALYTIC_ENGINE_TYPE.to_string(), options: HashMap::new(), diff --git a/catalog_impls/src/volatile.rs b/catalog_impls/src/volatile.rs index 7210865ddf..f0a8564a45 100644 --- a/catalog_impls/src/volatile.rs +++ b/catalog_impls/src/volatile.rs @@ -302,6 +302,7 @@ impl Schema for SchemaImpl { } // Do real create table. + // Partition table is not stored in ShardTableManager. if request.partition_info.is_none() { let _ = self .shard_tables_cache diff --git a/integration_tests/src/database.rs b/integration_tests/src/database.rs index 8752ae4782..3fb4f896ad 100644 --- a/integration_tests/src/database.rs +++ b/integration_tests/src/database.rs @@ -1,4 +1,4 @@ -// Copyright 2022 CeresDB Project Authors. Licensed under Apache-2.0. +// Copyright 2022-2023 CeresDB Project Authors. Licensed under Apache-2.0. use std::{ collections::HashMap, @@ -17,12 +17,8 @@ use ceresdb_client::{ RpcContext, }; use reqwest::{ClientBuilder, Url}; -use sql::{ - ast::{Statement, TableName}, - parser::Parser, -}; +use sql::{frontend::parse_table_name, parser::Parser}; use sqlness::{Database, QueryContext}; -use sqlparser::ast::{SetExpr, Statement as SqlStatement, TableFactor}; const SERVER_GRPC_ENDPOINT_ENV: &str = "CERESDB_SERVER_GRPC_ENDPOINT"; const SERVER_HTTP_ENDPOINT_ENV: &str = "CERESDB_SERVER_HTTP_ENDPOINT"; @@ -275,8 +271,8 @@ impl CeresDB { database: Some("public".to_string()), timeout: None, }; - - let table_name = Self::parse_table_name(&query); + let statements = Parser::parse_sql(&query).unwrap(); + let table_name = parse_table_name(&statements); let query_req = match table_name { Some(table_name) => Request { @@ -302,58 +298,4 @@ impl CeresDB { Err(e) => format!("Failed to execute query, err: {e:?}"), }) } - - fn parse_table_name(query: &str) -> Option { - let statements = Parser::parse_sql(query).unwrap(); - - match &statements[0] { - Statement::Standard(s) => match *s.clone() { - SqlStatement::Insert { table_name, .. } => { - Some(TableName::from(table_name).to_string()) - } - SqlStatement::Explain { statement, .. } => { - if let SqlStatement::Query(q) = *statement { - match *q.body { - SetExpr::Select(select) => { - if select.from.len() != 1 { - None - } else if let TableFactor::Table { name, .. } = - &select.from[0].relation - { - Some(TableName::from(name.clone()).to_string()) - } else { - None - } - } - _ => None, - } - } else { - None - } - } - SqlStatement::Query(q) => match *q.body { - SetExpr::Select(select) => { - if select.from.len() != 1 { - None - } else if let TableFactor::Table { name, .. } = &select.from[0].relation { - Some(TableName::from(name.clone()).to_string()) - } else { - None - } - } - _ => None, - }, - _ => None, - }, - Statement::Create(s) => Some(s.table_name.to_string()), - Statement::Drop(s) => Some(s.table_name.to_string()), - Statement::Describe(s) => Some(s.table_name.to_string()), - Statement::AlterModifySetting(s) => Some(s.table_name.to_string()), - Statement::AlterAddColumn(s) => Some(s.table_name.to_string()), - Statement::ShowCreate(s) => Some(s.table_name.to_string()), - Statement::ShowTables(_s) => None, - Statement::ShowDatabases => None, - Statement::Exists(s) => Some(s.table_name.to_string()), - } - } } diff --git a/meta_client/src/types.rs b/meta_client/src/types.rs index 0d389452a5..46796df831 100644 --- a/meta_client/src/types.rs +++ b/meta_client/src/types.rs @@ -289,7 +289,7 @@ impl TryFrom for TablesOfShard { .tables .into_iter() .map(TryInto::::try_into) - .collect::>>()?, + .collect::>>()?, }) } } diff --git a/remote_engine_client/src/cached_router.rs b/remote_engine_client/src/cached_router.rs index 860b252d32..ef57f0d4bf 100644 --- a/remote_engine_client/src/cached_router.rs +++ b/remote_engine_client/src/cached_router.rs @@ -1,4 +1,4 @@ -// Copyright 2022 CeresDB Project Authors. Licensed under Apache-2.0. +// Copyright 2022-2023 CeresDB Project Authors. Licensed under Apache-2.0. //! Cached router @@ -120,7 +120,6 @@ impl CachedRouter { msg: "no endpoint in route info", })?; - let endpoint = endpoint.into(); let channel = self.channel_pool.get(&endpoint).await?; Ok(channel) diff --git a/router/src/cluster_based.rs b/router/src/cluster_based.rs index 7b2bb47db9..52c346bbbd 100644 --- a/router/src/cluster_based.rs +++ b/router/src/cluster_based.rs @@ -64,7 +64,7 @@ fn make_route(table: TableInfo, endpoint: &str) -> Result { Ok(RouteData { table_name: table.name.clone(), table: Some(table), - endpoint: Some(endpoint.into()), + endpoint: Some(endpoint), }) } @@ -188,6 +188,7 @@ mod tests { name: table.clone(), schema_name: String::from("public"), schema_id: 0, + partition_info: None, }, node_shards: vec![NodeShard { endpoint: String::from("127.0.0.1:8831"), @@ -250,7 +251,7 @@ mod tests { let mut routes = Vec::with_capacity(tables.len()); let miss = router.route_from_cache(tables, &mut routes); assert_eq!(routes.len(), 1); - assert_eq!(routes[0].table, table1.to_string()); + assert_eq!(routes[0].table_name, table1.to_string()); assert_eq!(miss.len(), 0); // sleep 1.5s, table2 will be evicted, and table1 in cache @@ -259,7 +260,7 @@ mod tests { let mut routes = Vec::with_capacity(tables.len()); let miss = router.route_from_cache(tables, &mut routes); assert_eq!(routes.len(), 1); - assert_eq!(routes[0].table, table1.to_string()); + assert_eq!(routes[0].table_name, table1.to_string()); assert_eq!(miss.len(), 1); assert_eq!(miss[0], table2.to_string()); } diff --git a/router/src/rule_based.rs b/router/src/rule_based.rs index b330d9de14..c1b52125ec 100644 --- a/router/src/rule_based.rs +++ b/router/src/rule_based.rs @@ -158,7 +158,6 @@ impl Router for RuleBasedRouter { table: &table_name, })?; - // let pb_endpoint = storage::Endpoint::from(endpoint.clone()); let route = RouteData { table_name, table: None, diff --git a/server/src/proxy/forward.rs b/server/src/proxy/forward.rs index eff41d5d7c..2fc1acf67e 100644 --- a/server/src/proxy/forward.rs +++ b/server/src/proxy/forward.rs @@ -12,7 +12,7 @@ use async_trait::async_trait; use ceresdbproto::storage::{ storage_service_client::StorageServiceClient, RequestContext, RouteRequest, }; -use log::{debug, error, info, warn}; +use log::{debug, error, warn}; use meta_client::types::TableInfo; use router::{endpoint::Endpoint, RouterRef}; use serde::{Deserialize, Serialize}; @@ -281,16 +281,11 @@ impl Forwarder { let endpoint = match self.router.route(route_req).await { Ok(mut routes) => { if routes.len() == 1 { - if routes[0].table.is_none() { - info!("xxxxroute table is none",); - return Ok(ForwardResult::Original); - } - - let table = routes[0].table.clone().unwrap(); - - if table.partition_info.is_some() { - info!("xxxxroute partition_info is some",); - return Ok(ForwardResult::OriginalPartitionTableInfo(table)); + if routes[0].table.is_some() { + let table = routes[0].table.clone().unwrap(); + if table.partition_info.is_some() { + return Ok(ForwardResult::OriginalPartitionTableInfo(table)); + } } if routes[0].endpoint.is_none() { @@ -302,7 +297,6 @@ impl Forwarder { } let endpoint = routes.remove(0).endpoint.unwrap(); if self.is_local_endpoint(&endpoint) { - info!("xxxxroute local endpoint",); return Ok(ForwardResult::Original); } endpoint @@ -370,9 +364,9 @@ impl Forwarder { #[cfg(test)] mod tests { use catalog::consts::DEFAULT_SCHEMA; - use ceresdbproto::storage::{Route, SqlQueryRequest, SqlQueryResponse}; + use ceresdbproto::storage::{SqlQueryRequest, SqlQueryResponse}; use futures::FutureExt; - use router::Router; + use router::{RouteData, Router}; use tonic::IntoRequest; use super::*; @@ -403,13 +397,14 @@ mod tests { #[async_trait] impl Router for MockRouter { - async fn route(&self, req: RouteRequest) -> router::Result> { + async fn route(&self, req: RouteRequest) -> router::Result> { let endpoint = self.routing_tables.get(&req.tables[0]); match endpoint { None => Ok(vec![]), - Some(v) => Ok(vec![Route { - table: req.tables[0].clone(), - endpoint: Some(v.clone().into()), + Some(v) => Ok(vec![RouteData { + table_name: req.tables[0].clone(), + table: None, + endpoint: Some(v.clone()), }]), } } diff --git a/server/src/proxy/grpc/route.rs b/server/src/proxy/grpc/route.rs index bb0060ebe5..c73f42b86f 100644 --- a/server/src/proxy/grpc/route.rs +++ b/server/src/proxy/grpc/route.rs @@ -33,10 +33,14 @@ impl Proxy { resp.routes = v .into_iter() .map(|r| { - let mut router = Route::default(); - router.table = r.table_name; - router.endpoint = r.endpoint.map(Into::into); - router + Route { + table: r.table_name, + endpoint: r.endpoint.map(Into::into), + } + // let mut router = Route::default(); + // router.table = r.table_name; + // router.endpoint = r.endpoint.map(Into::into); + // router }) .collect(); } diff --git a/server/src/proxy/grpc/sql_query.rs b/server/src/proxy/grpc/sql_query.rs index 19aaa034a0..85be3e6a8f 100644 --- a/server/src/proxy/grpc/sql_query.rs +++ b/server/src/proxy/grpc/sql_query.rs @@ -490,7 +490,7 @@ impl Proxy { msg: format!("Failed to create table, request:{create_table_request:?}"), })?; } - return Ok(()); + Ok(()) } } diff --git a/server/src/proxy/mod.rs b/server/src/proxy/mod.rs index bc4cbc7490..eeb2da6411 100644 --- a/server/src/proxy/mod.rs +++ b/server/src/proxy/mod.rs @@ -37,6 +37,7 @@ pub struct Proxy { } impl Proxy { + #[allow(clippy::too_many_arguments)] pub fn try_new( router: Arc, instance: InstanceRef, diff --git a/sql/src/frontend.rs b/sql/src/frontend.rs index da2e7ba136..8bd64ac8fe 100644 --- a/sql/src/frontend.rs +++ b/sql/src/frontend.rs @@ -13,7 +13,7 @@ use sqlparser::ast::{SetExpr, Statement as SqlStatement, TableFactor}; use table_engine::table; use crate::{ - ast::Statement, + ast::{Statement, TableName}, parser::Parser, plan::Plan, planner::Planner, @@ -161,7 +161,9 @@ impl Frontend

{ pub fn parse_table_name(statements: &StatementVec) -> Option { match &statements[0] { Statement::Standard(s) => match *s.clone() { - SqlStatement::Insert { table_name, .. } => Some(table_name.to_string()), + SqlStatement::Insert { table_name, .. } => { + Some(TableName::from(table_name).to_string()) + } SqlStatement::Explain { statement, .. } => { if let SqlStatement::Query(q) = *statement { match *q.body { @@ -170,7 +172,7 @@ pub fn parse_table_name(statements: &StatementVec) -> Option { None } else if let TableFactor::Table { name, .. } = &select.from[0].relation { - Some(name.to_string()) + Some(TableName::from(name.clone()).to_string()) } else { None } @@ -186,7 +188,7 @@ pub fn parse_table_name(statements: &StatementVec) -> Option { if select.from.len() != 1 { None } else if let TableFactor::Table { name, .. } = &select.from[0].relation { - Some(name.to_string()) + Some(TableName::from(name.clone()).to_string()) } else { None } @@ -206,3 +208,39 @@ pub fn parse_table_name(statements: &StatementVec) -> Option { Statement::Exists(s) => Some(s.table_name.to_string()), } } + +#[cfg(test)] +mod tests { + use crate::{frontend::parse_table_name, parser::Parser}; + + #[test] + fn test_parse_table_name() { + let table = "test_parse_table_name"; + let test_cases=vec![ + format!("INSERT INTO {table} (t, name, value) VALUES (1651737067000, 'ceresdb', 100)"), + format!("INSERT INTO `{table}` (t, name, value) VALUES (1651737067000,'ceresdb', 100)"), + format!("select * from {table}"), + format!("select * from `{table}`"), + format!("explain select * from {table}"), + format!("explain select * from `{table}`"), + format!("CREATE TABLE {table} (`name`string TAG,`value` double NOT NULL, `t` timestamp NOT NULL, TIMESTAMP KEY(t))"), + format!("CREATE TABLE `{table}` (`name`string TAG,`value` double NOT NULL, `t` timestamp NOT NULL, TIMESTAMP KEY(t))"), + format!("drop table {table}"), + format!("drop table `{table}`"), + format!("describe table {table}"), + format!("describe table `{table}`"), + format!("alter table {table} modify setting enable_ttl='false'"), + format!("alter table `{table}` modify setting enable_ttl='false'"), + format!("alter table {table} add column c1 int"), + format!("alter table `{table}` add column c1 int"), + format!("show create table {table}"), + format!("show create table `{table}`"), + format!("exists table {table}"), + format!("exists table `{table}`"), + ]; + for sql in test_cases { + let statements = Parser::parse_sql(&sql).unwrap(); + assert_eq!(parse_table_name(&statements), Some(table.to_string())); + } + } +} From da0599485427b7265ef2b50ee55d70901898b870 Mon Sep 17 00:00:00 2001 From: "chunshao.rcs" Date: Tue, 4 Apr 2023 16:10:34 +0800 Subject: [PATCH 3/9] feat: introduce partition table engine --- Cargo.lock | 17 +++ Cargo.toml | 2 + analytic_engine/src/engine.rs | 76 ++++--------- analytic_engine/src/instance/create.rs | 49 ++++----- analytic_engine/src/instance/mod.rs | 5 +- analytic_engine/src/instance/open.rs | 4 +- analytic_engine/src/setup.rs | 22 +--- analytic_engine/src/table/data.rs | 8 +- analytic_engine/src/table/metrics.rs | 19 +--- analytic_engine/src/table/mod.rs | 5 +- analytic_engine/src/tests/util.rs | 1 - partition_table_engine/Cargo.toml | 23 ++++ partition_table_engine/src/error.rs | 12 ++ partition_table_engine/src/lib.rs | 78 +++++++++++++ partition_table_engine/src/metrics.rs | 24 ++++ .../src}/partition.rs | 104 +++++++++--------- server/Cargo.toml | 1 + server/src/grpc/meta_event_service/mod.rs | 11 +- server/src/grpc/mod.rs | 9 -- server/src/instance.rs | 6 +- server/src/proxy/grpc/sql_query.rs | 3 +- server/src/proxy/mod.rs | 10 +- server/src/server.rs | 14 ++- src/setup.rs | 4 +- table_engine/src/engine.rs | 4 +- table_engine/src/lib.rs | 3 +- 26 files changed, 293 insertions(+), 221 deletions(-) create mode 100644 partition_table_engine/Cargo.toml create mode 100644 partition_table_engine/src/error.rs create mode 100644 partition_table_engine/src/lib.rs create mode 100644 partition_table_engine/src/metrics.rs rename {analytic_engine/src/table => partition_table_engine/src}/partition.rs (77%) diff --git a/Cargo.lock b/Cargo.lock index df74835fd3..194fe49b6b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4256,6 +4256,22 @@ dependencies = [ "regex", ] +[[package]] +name = "partition_table_engine" +version = "1.1.0" +dependencies = [ + "async-stream", + "async-trait", + "common_types", + "common_util", + "futures 0.3.25", + "lazy_static", + "prometheus 0.12.0", + "remote_engine_client", + "snafu 0.6.10", + "table_engine", +] + [[package]] name = "paste" version = "0.1.18" @@ -5619,6 +5635,7 @@ dependencies = [ "logger", "meta_client", "opensrv-mysql", + "partition_table_engine", "paste 1.0.8", "profile", "prom-remote-api", diff --git a/Cargo.toml b/Cargo.toml index e4cd6639a4..b21e0fef8a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -38,6 +38,7 @@ members = [ "integration_tests/sdk/rust", "interpreters", "meta_client", + "partition_table_engine", "query_engine", "remote_engine_client", "router", @@ -91,6 +92,7 @@ interpreters = { path = "interpreters" } itertools = "0.10.5" meta_client = { path = "meta_client" } object_store = { path = "components/object_store" } +partition_table_engine = { path = "partition_table_engine" } parquet_ext = { path = "components/parquet_ext" } parquet = { version = "32.0.0" } paste = "1.0" diff --git a/analytic_engine/src/engine.rs b/analytic_engine/src/engine.rs index cc4c0e0e47..1038a05c99 100644 --- a/analytic_engine/src/engine.rs +++ b/analytic_engine/src/engine.rs @@ -1,4 +1,4 @@ -// Copyright 2022 CeresDB Project Authors. Licensed under Apache-2.0. +// Copyright 2022-2023 CeresDB Project Authors. Licensed under Apache-2.0. //! Implements the TableEngine trait @@ -7,21 +7,17 @@ use std::sync::Arc; use async_trait::async_trait; use common_util::error::BoxError; use log::info; -use snafu::{OptionExt, ResultExt}; +use snafu::ResultExt; use table_engine::{ engine::{ Close, CloseTableRequest, CreateTableRequest, DropTableRequest, OpenTableRequest, Result, - TableEngine, Unexpected, UnexpectedNoCause, + TableEngine, }, table::{SchemaId, TableRef}, ANALYTIC_ENGINE_TYPE, }; -use crate::{ - instance::InstanceRef, - space::SpaceId, - table::{partition::PartitionTableImpl, TableImpl}, -}; +use crate::{instance::InstanceRef, space::SpaceId, table::TableImpl}; /// TableEngine implementation pub struct TableEngineImpl { @@ -76,30 +72,14 @@ impl TableEngine for TableEngineImpl { let space_table = self.instance.create_table(space_id, request).await?; - let table_impl: TableRef = match &space_table.table_data().partition_info { - None => Arc::new(TableImpl::new( - self.instance.clone(), - ANALYTIC_ENGINE_TYPE.to_string(), - space_id, - space_table.table_data().id, - space_table.table_data().clone(), - space_table, - )), - Some(_v) => Arc::new( - PartitionTableImpl::new( - self.instance - .remote_engine - .clone() - .context(UnexpectedNoCause { - msg: "remote engine not found", - })?, - ANALYTIC_ENGINE_TYPE.to_string(), - space_table, - ) - .box_err() - .context(Unexpected)?, - ), - }; + let table_impl: TableRef = Arc::new(TableImpl::new( + self.instance.clone(), + ANALYTIC_ENGINE_TYPE.to_string(), + space_id, + space_table.table_data().id, + space_table.table_data().clone(), + space_table, + )); Ok(table_impl) } @@ -128,30 +108,14 @@ impl TableEngine for TableEngineImpl { None => return Ok(None), }; - let table_impl: TableRef = match &space_table.table_data().partition_info { - None => Arc::new(TableImpl::new( - self.instance.clone(), - ANALYTIC_ENGINE_TYPE.to_string(), - space_id, - space_table.table_data().id, - space_table.table_data().clone(), - space_table, - )), - Some(_v) => Arc::new( - PartitionTableImpl::new( - self.instance - .remote_engine - .clone() - .context(UnexpectedNoCause { - msg: "remote engine is empty", - })?, - ANALYTIC_ENGINE_TYPE.to_string(), - space_table, - ) - .box_err() - .context(Unexpected)?, - ), - }; + let table_impl = Arc::new(TableImpl::new( + self.instance.clone(), + ANALYTIC_ENGINE_TYPE.to_string(), + space_id, + space_table.table_data().id, + space_table.table_data().clone(), + space_table, + )); Ok(Some(table_impl)) } diff --git a/analytic_engine/src/instance/create.rs b/analytic_engine/src/instance/create.rs index 1d199d948c..a0c299b872 100644 --- a/analytic_engine/src/instance/create.rs +++ b/analytic_engine/src/instance/create.rs @@ -105,32 +105,29 @@ impl Instance { return Ok(table_data); }; - // Partition table is not stored in manifest. - if table_data.partition_info.is_none() { - // Store table info into meta - let update_req = { - let meta_update = MetaUpdate::AddTable(AddTableMeta { - space_id: space.id, - table_id: table_data.id, - table_name: table_data.name.clone(), - schema: table_data.schema(), - opts: table_data.table_options().as_ref().clone(), - }); - MetaUpdateRequest { - shard_info: table_data.shard_info, - meta_update, - } - }; - self.space_store - .manifest - .store_update(update_req) - .await - .context(WriteManifest { - space_id: space.id, - table: &table_data.name, - table_id: table_data.id, - })?; - } + // Store table info into meta + let update_req = { + let meta_update = MetaUpdate::AddTable(AddTableMeta { + space_id: space.id, + table_id: table_data.id, + table_name: table_data.name.clone(), + schema: table_data.schema(), + opts: table_data.table_options().as_ref().clone(), + }); + MetaUpdateRequest { + shard_info: table_data.shard_info, + meta_update, + } + }; + self.space_store + .manifest + .store_update(update_req) + .await + .context(WriteManifest { + space_id: space.id, + table: &table_data.name, + table_id: table_data.id, + })?; space.insert_table(table_data.clone()); Ok(table_data) diff --git a/analytic_engine/src/instance/mod.rs b/analytic_engine/src/instance/mod.rs index 179d59b073..9e17bc0fb7 100644 --- a/analytic_engine/src/instance/mod.rs +++ b/analytic_engine/src/instance/mod.rs @@ -1,4 +1,4 @@ -// Copyright 2022 CeresDB Project Authors. Licensed under Apache-2.0. +// Copyright 2022-2023 CeresDB Project Authors. Licensed under Apache-2.0. //! A table engine instance //! @@ -27,7 +27,7 @@ use common_util::{define_result, runtime::Runtime}; use log::info; use mem_collector::MemUsageCollector; use snafu::{ResultExt, Snafu}; -use table_engine::{engine::EngineRuntimes, remote::RemoteEngineRef}; +use table_engine::engine::EngineRuntimes; use wal::manager::{WalLocation, WalManagerRef}; use crate::{ @@ -177,7 +177,6 @@ pub struct Instance { /// Options for scanning sst pub(crate) scan_options: ScanOptions, pub(crate) iter_options: Option, - pub(crate) remote_engine: Option, } impl Instance { diff --git a/analytic_engine/src/instance/open.rs b/analytic_engine/src/instance/open.rs index 941c04eeb2..1e226f06c1 100644 --- a/analytic_engine/src/instance/open.rs +++ b/analytic_engine/src/instance/open.rs @@ -10,7 +10,7 @@ use std::{ use common_types::schema::IndexInWriterSchema; use log::{debug, error, info, trace, warn}; use snafu::ResultExt; -use table_engine::{engine::OpenTableRequest, remote::RemoteEngineRef}; +use table_engine::engine::OpenTableRequest; use tokio::sync::oneshot; use wal::{ log_batch::LogEntry, @@ -54,7 +54,6 @@ impl Instance { wal_manager: WalManagerRef, store_picker: ObjectStorePickerRef, sst_factory: SstFactoryRef, - remote_engine_ref: Option, ) -> Result> { let space_store = Arc::new(SpaceStore { spaces: RwLock::new(Spaces::default()), @@ -111,7 +110,6 @@ impl Instance { .map(|v| v.as_byte() as usize), iter_options, scan_options, - remote_engine: remote_engine_ref, }); Ok(instance) diff --git a/analytic_engine/src/setup.rs b/analytic_engine/src/setup.rs index f98a2bb8ab..c68f635bfd 100644 --- a/analytic_engine/src/setup.rs +++ b/analytic_engine/src/setup.rs @@ -1,4 +1,4 @@ -// Copyright 2022 CeresDB Project Authors. Licensed under Apache-2.0. +// Copyright 2022-2023 CeresDB Project Authors. Licensed under Apache-2.0. //! Setup the analytic engine @@ -16,13 +16,8 @@ use object_store::{ prefix::StoreWithPrefix, LocalFileSystem, ObjectStoreRef, }; -use remote_engine_client::RemoteEngineImpl; -use router::RouterRef; use snafu::{Backtrace, ResultExt, Snafu}; -use table_engine::{ - engine::{EngineRuntimes, TableEngineRef}, - remote::RemoteEngineRef, -}; +use table_engine::engine::{EngineRuntimes, TableEngineRef}; use table_kv::{memory::MemoryImpl, obkv::ObkvImpl, TableKv}; use wal::{ manager::{self, WalManagerRef}, @@ -110,7 +105,6 @@ const DISK_CACHE_DIR_NAME: &str = "sst_cache"; #[derive(Clone)] pub struct EngineBuilder<'a> { pub config: &'a Config, - pub router: Option, pub engine_runtimes: Arc, pub opened_wals: OpenedWals, } @@ -132,7 +126,6 @@ impl<'a> EngineBuilder<'a> { self.opened_wals.data_wal, Arc::new(manifest), Arc::new(opened_storages), - self.router, ) .await?; Ok(Arc::new(TableEngineImpl::new(instance))) @@ -360,17 +353,7 @@ async fn open_instance( wal_manager: WalManagerRef, manifest: ManifestRef, store_picker: ObjectStorePickerRef, - router: Option, ) -> Result { - let remote_engine_ref: Option = if let Some(v) = router { - Some(Arc::new(RemoteEngineImpl::new( - config.remote_engine_client.clone(), - v, - ))) - } else { - None - }; - let meta_cache: Option = config .sst_meta_cache_cap .map(|cap| Arc::new(MetaCache::new(cap))); @@ -387,7 +370,6 @@ async fn open_instance( wal_manager, store_picker, Arc::new(FactoryImpl::default()), - remote_engine_ref, ) .await .context(OpenInstance)?; diff --git a/analytic_engine/src/table/data.rs b/analytic_engine/src/table/data.rs index a50dcf787f..29c80f52d6 100644 --- a/analytic_engine/src/table/data.rs +++ b/analytic_engine/src/table/data.rs @@ -27,7 +27,7 @@ use common_util::define_result; use log::{debug, info}; use object_store::Path; use snafu::{Backtrace, OptionExt, ResultExt, Snafu}; -use table_engine::{engine::CreateTableRequest, partition::PartitionInfo, table::TableId}; +use table_engine::{engine::CreateTableRequest, table::TableId}; use crate::{ instance::write_worker::{choose_worker, WorkerLocal, WriteHandle}, @@ -146,9 +146,6 @@ pub struct TableData { /// Shard id pub shard_info: TableShardInfo, - - /// Partition info - pub partition_info: Option, } impl fmt::Debug for TableData { @@ -164,7 +161,6 @@ impl fmt::Debug for TableData { .field("last_file_id", &self.last_file_id) .field("dropped", &self.dropped.load(Ordering::Relaxed)) .field("shard_info", &self.shard_info) - .field("partition_info", &self.partition_info) .finish() } } @@ -219,7 +215,6 @@ impl TableData { dropped: AtomicBool::new(false), metrics, shard_info: TableShardInfo::new(request.shard_id), - partition_info: request.partition_info, }) } @@ -256,7 +251,6 @@ impl TableData { dropped: AtomicBool::new(false), metrics, shard_info: TableShardInfo::new(shard_id), - partition_info: None, }) } diff --git a/analytic_engine/src/table/metrics.rs b/analytic_engine/src/table/metrics.rs index 585708bea2..95c16c7e7a 100644 --- a/analytic_engine/src/table/metrics.rs +++ b/analytic_engine/src/table/metrics.rs @@ -1,4 +1,4 @@ -// Copyright 2022 CeresDB Project Authors. Licensed under Apache-2.0. +// Copyright 2022-2023 CeresDB Project Authors. Licensed under Apache-2.0. //! Metrics of table. @@ -101,23 +101,6 @@ lazy_static! { exponential_buckets(0.01, 2.0, 13).unwrap() ).unwrap(); - // Buckets: 0, 0.01, .., 0.01 * 2^12 - pub static ref PARTITION_TABLE_WRITE_DURATION_HISTOGRAM: HistogramVec = register_histogram_vec!( - "partition_table_write_duration", - "Histogram for write duration of the partition table in seconds", - &["type"], - exponential_buckets(0.01, 2.0, 13).unwrap() - ) - .unwrap(); - - // Buckets: 0, 0.01, .., 0.01 * 2^12 - pub static ref PARTITION_TABLE_PARTITIONED_READ_DURATION_HISTOGRAM: HistogramVec = register_histogram_vec!( - "partition_table_partitioned_read_duration", - "Histogram for partitioned read duration of the partition table in seconds", - &["type"], - exponential_buckets(0.01, 2.0, 13).unwrap() - ) - .unwrap(); // End of histograms. } diff --git a/analytic_engine/src/table/mod.rs b/analytic_engine/src/table/mod.rs index d00637644d..77da11679f 100644 --- a/analytic_engine/src/table/mod.rs +++ b/analytic_engine/src/table/mod.rs @@ -1,4 +1,4 @@ -// Copyright 2022 CeresDB Project Authors. Licensed under Apache-2.0. +// Copyright 2022-2023 CeresDB Project Authors. Licensed under Apache-2.0. //! Table implementation @@ -32,7 +32,6 @@ use crate::{ pub mod data; pub mod metrics; -pub mod partition; pub mod sst_util; pub mod version; pub mod version_edit; @@ -103,7 +102,7 @@ impl Table for TableImpl { } fn partition_info(&self) -> Option { - self.table_data.partition_info.clone() + None } fn engine_type(&self) -> &str { diff --git a/analytic_engine/src/tests/util.rs b/analytic_engine/src/tests/util.rs index dc9ccfe250..be68abac7b 100644 --- a/analytic_engine/src/tests/util.rs +++ b/analytic_engine/src/tests/util.rs @@ -130,7 +130,6 @@ impl TestContext { let engine_builder = EngineBuilder { config: &self.config, - router: None, engine_runtimes: self.runtimes.clone(), opened_wals: opened_wals.clone(), }; diff --git a/partition_table_engine/Cargo.toml b/partition_table_engine/Cargo.toml new file mode 100644 index 0000000000..302857d9e6 --- /dev/null +++ b/partition_table_engine/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "partition_table_engine" + +[package.version] +workspace = true + +[package.authors] +workspace = true + +[package.edition] +workspace = true + +[dependencies] +async-stream = { workspace = true } +async-trait = { workspace = true } +common_types = { workspace = true } +common_util = { workspace = true } +futures = { workspace = true } +lazy_static = { workspace = true } +prometheus = { workspace = true } +remote_engine_client = { workspace = true } +snafu = { workspace = true } +table_engine = { workspace = true } diff --git a/partition_table_engine/src/error.rs b/partition_table_engine/src/error.rs new file mode 100644 index 0000000000..1b18e79e46 --- /dev/null +++ b/partition_table_engine/src/error.rs @@ -0,0 +1,12 @@ +// Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. + +use common_util::{define_result, error::GenericError}; +use snafu::Snafu; +define_result!(Error); + +#[derive(Snafu, Debug)] +#[snafu(visibility(pub))] +pub enum Error { + #[snafu(display("Internal error, message:{}, err:{}", msg, source))] + Internal { msg: String, source: GenericError }, +} diff --git a/partition_table_engine/src/lib.rs b/partition_table_engine/src/lib.rs new file mode 100644 index 0000000000..fe7edd2e1e --- /dev/null +++ b/partition_table_engine/src/lib.rs @@ -0,0 +1,78 @@ +// Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. + +//! Partition table engine implementations + +mod error; +mod metrics; +mod partition; + +use std::sync::Arc; + +use async_trait::async_trait; +use common_util::error::BoxError; +use snafu::{OptionExt, ResultExt}; +use table_engine::{ + engine::{ + CloseTableRequest, CreateTableRequest, DropTableRequest, OpenTableRequest, Result, + TableEngine, Unexpected, UnexpectedNoCause, + }, + remote::RemoteEngineRef, + table::TableRef, + PARTITION_TABLE_ENGINE_TYPE, +}; + +use crate::partition::PartitionTableImpl; + +/// Partition table engine implementation. +pub struct PartitionTableEngine { + remote_engine_ref: RemoteEngineRef, +} + +impl PartitionTableEngine { + pub fn new(remote_engine_ref: RemoteEngineRef) -> Self { + Self { remote_engine_ref } + } +} + +#[async_trait] +impl TableEngine for PartitionTableEngine { + fn engine_type(&self) -> &str { + PARTITION_TABLE_ENGINE_TYPE + } + + async fn close(&self) -> Result<()> { + Ok(()) + } + + async fn create_table(&self, request: CreateTableRequest) -> Result { + Ok(Arc::new( + PartitionTableImpl::new( + request.catalog_name, + request.schema_name, + request.table_name, + request.table_id, + request.table_schema, + request.partition_info.context(UnexpectedNoCause { + msg: "partition info not found", + })?, + request.options, + request.engine, + self.remote_engine_ref.clone(), + ) + .box_err() + .context(Unexpected)?, + )) + } + + async fn drop_table(&self, _request: DropTableRequest) -> Result { + Ok(true) + } + + async fn open_table(&self, _request: OpenTableRequest) -> Result> { + Ok(None) + } + + async fn close_table(&self, _request: CloseTableRequest) -> Result<()> { + Ok(()) + } +} diff --git a/partition_table_engine/src/metrics.rs b/partition_table_engine/src/metrics.rs new file mode 100644 index 0000000000..d19978bdf3 --- /dev/null +++ b/partition_table_engine/src/metrics.rs @@ -0,0 +1,24 @@ +// Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. + +use lazy_static::lazy_static; +use prometheus::{exponential_buckets, register_histogram_vec, HistogramVec}; + +lazy_static! { + // Buckets: 0, 0.01, .., 0.01 * 2^12 + pub static ref PARTITION_TABLE_WRITE_DURATION_HISTOGRAM: HistogramVec = register_histogram_vec!( + "partition_table_write_duration", + "Histogram for write duration of the partition table in seconds", + &["type"], + exponential_buckets(0.01, 2.0, 13).unwrap() + ) + .unwrap(); + + // Buckets: 0, 0.01, .., 0.01 * 2^12 + pub static ref PARTITION_TABLE_PARTITIONED_READ_DURATION_HISTOGRAM: HistogramVec = register_histogram_vec!( + "partition_table_partitioned_read_duration", + "Histogram for partitioned read duration of the partition table in seconds", + &["type"], + exponential_buckets(0.01, 2.0, 13).unwrap() + ) + .unwrap(); +} diff --git a/analytic_engine/src/table/partition.rs b/partition_table_engine/src/partition.rs similarity index 77% rename from analytic_engine/src/table/partition.rs rename to partition_table_engine/src/partition.rs index c68dfff2e0..46a796c406 100644 --- a/analytic_engine/src/table/partition.rs +++ b/partition_table_engine/src/partition.rs @@ -1,4 +1,4 @@ -// Copyright 2022 CeresDB Project Authors. Licensed under Apache-2.0. +// Copyright 2022-2023 CeresDB Project Authors. Licensed under Apache-2.0. //! Distributed Table implementation @@ -11,7 +11,7 @@ use common_types::{ }; use common_util::error::BoxError; use futures::future::try_join_all; -use snafu::{ensure, ResultExt}; +use snafu::ResultExt; use table_engine::{ partition::{ format_sub_partition_table_name, rule::df_adapter::DfPartitionRuleAdapter, PartitionInfo, @@ -28,60 +28,57 @@ use table_engine::{ ReadRequest, Result, Scan, Table, TableId, TableStats, UnexpectedWithMsg, UnsupportedMethod, Write, WriteRequest, }, + PARTITION_TABLE_ENGINE_TYPE, }; -use crate::{ - space::SpaceAndTable, - table::metrics::{ - PARTITION_TABLE_PARTITIONED_READ_DURATION_HISTOGRAM, - PARTITION_TABLE_WRITE_DURATION_HISTOGRAM, - }, +use crate::metrics::{ + PARTITION_TABLE_PARTITIONED_READ_DURATION_HISTOGRAM, PARTITION_TABLE_WRITE_DURATION_HISTOGRAM, }; /// Table trait implementation pub struct PartitionTableImpl { - /// Space table - space_table: SpaceAndTable, - /// Remote engine - remote_engine: RemoteEngineRef, - /// Engine type + catalog_name: String, + schema_name: String, + table_name: String, + table_id: TableId, + table_schema: Schema, + partition_info: PartitionInfo, + options: HashMap, engine_type: String, + remote_engine: RemoteEngineRef, } impl PartitionTableImpl { pub fn new( - remote_engine: RemoteEngineRef, + catalog_name: String, + schema_name: String, + table_name: String, + table_id: TableId, + table_schema: Schema, + partition_info: PartitionInfo, + options: HashMap, engine_type: String, - space_table: SpaceAndTable, + remote_engine: RemoteEngineRef, ) -> Result { - ensure!( - space_table.table_data().partition_info.is_some(), - UnexpectedWithMsg { - msg: "partition table partition info can't be empty" - } - ); Ok(Self { - space_table, - remote_engine, + catalog_name, + schema_name, + table_name, + table_id, + table_schema, + partition_info, + options, engine_type, + remote_engine, }) } fn get_sub_table_ident(&self, id: usize) -> TableIdentifier { - let partition_name = self - .space_table - .table_data() - .partition_info - .as_ref() - .map(|v| v.get_definitions()[id].name.clone()) - .unwrap(); + let partition_name = self.partition_info.get_definitions()[id].name.clone(); TableIdentifier { - catalog: self.space_table.space().context.catalog_name.clone(), - schema: self.space_table.space().context.schema_name.clone(), - table: format_sub_partition_table_name( - &self.space_table.table_data().name, - &partition_name, - ), + catalog: self.catalog_name.clone(), + schema: self.schema_name.clone(), + table: format_sub_partition_table_name(&self.table_name, &partition_name), } } } @@ -89,8 +86,10 @@ impl PartitionTableImpl { impl fmt::Debug for PartitionTableImpl { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("PartitionTableImpl") - .field("space_id", &self.space_table.space().id) - .field("table_id", &self.space_table.table_data().id) + .field("catalog_name", &self.catalog_name) + .field("schema_name", &self.schema_name) + .field("table_name", &self.table_name) + .field("table_id", &self.table_id) .finish() } } @@ -98,23 +97,24 @@ impl fmt::Debug for PartitionTableImpl { #[async_trait] impl Table for PartitionTableImpl { fn name(&self) -> &str { - &self.space_table.table_data().name + &self.table_name } fn id(&self) -> TableId { - self.space_table.table_data().id + self.table_id } fn schema(&self) -> Schema { - self.space_table.table_data().schema() + self.table_schema.clone() } + // TODO: get options from sub partition table with remote engine fn options(&self) -> HashMap { - self.space_table.table_data().table_options().to_raw_map() + self.options.clone() } fn partition_info(&self) -> Option { - self.space_table.table_data().partition_info.clone() + Some(self.partition_info.clone()) } fn engine_type(&self) -> &str { @@ -122,7 +122,7 @@ impl Table for PartitionTableImpl { } fn stats(&self) -> TableStats { - self.space_table.table_data().metrics.table_stats() + TableStats::default() } async fn write(&self, request: WriteRequest) -> Result { @@ -136,11 +136,9 @@ impl Table for PartitionTableImpl { msg: "partition table partition info can't be empty", } .fail()?, - Some(partition_info) => { - DfPartitionRuleAdapter::new(partition_info, &self.space_table.table_data().schema()) - .box_err() - .context(CreatePartitionRule)? - } + Some(partition_info) => DfPartitionRuleAdapter::new(partition_info, &self.table_schema) + .box_err() + .context(CreatePartitionRule)?, }; // Split write request. @@ -221,11 +219,9 @@ impl Table for PartitionTableImpl { msg: "partition table partition info can't be empty", } .fail()?, - Some(partition_info) => { - DfPartitionRuleAdapter::new(partition_info, &self.space_table.table_data().schema()) - .box_err() - .context(CreatePartitionRule)? - } + Some(partition_info) => DfPartitionRuleAdapter::new(partition_info, &self.table_schema) + .box_err() + .context(CreatePartitionRule)?, }; // Evaluate expr and locate partition. diff --git a/server/Cargo.toml b/server/Cargo.toml index 94bfa89965..da70f907f5 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -33,6 +33,7 @@ log = { workspace = true } logger = { workspace = true } meta_client = { workspace = true } opensrv-mysql = "0.1.0" +partition_table_engine = { workspace = true } paste = { workspace = true } profile = { workspace = true } prom-remote-api = { version = "0.2.1", features = ["warp"] } diff --git a/server/src/grpc/meta_event_service/mod.rs b/server/src/grpc/meta_event_service/mod.rs index 6effa5c91f..0816d6500d 100644 --- a/server/src/grpc/meta_event_service/mod.rs +++ b/server/src/grpc/meta_event_service/mod.rs @@ -172,6 +172,7 @@ impl MetaServiceImpl { cluster: self.cluster.clone(), catalog_manager: self.instance.catalog_manager.clone(), table_engine: self.instance.table_engine.clone(), + partition_table_engine: self.instance.partition_table_engine.clone(), wal_region_closer: self.wal_region_closer.clone(), } } @@ -182,6 +183,7 @@ struct HandlerContext { cluster: ClusterRef, catalog_manager: ManagerRef, table_engine: TableEngineRef, + partition_table_engine: TableEngineRef, wal_region_closer: WalRegionCloserRef, } @@ -386,6 +388,12 @@ async fn handle_create_table_on_shard( None => None, }; + let table_engine = if partition_info.is_some() { + ctx.partition_table_engine.clone() + } else { + ctx.table_engine.clone() + }; + let create_table_request = CreateTableRequest { catalog_name: ctx.catalog_manager.default_catalog_name().to_string(), schema_name: table_info.schema_name, @@ -399,8 +407,9 @@ async fn handle_create_table_on_shard( shard_id: shard_info.id, partition_info, }; + let create_opts = CreateOptions { - table_engine: ctx.table_engine, + table_engine, create_if_not_exists: request.create_if_not_exist, }; diff --git a/server/src/grpc/mod.rs b/server/src/grpc/mod.rs index f40578a139..0988617bc2 100644 --- a/server/src/grpc/mod.rs +++ b/server/src/grpc/mod.rs @@ -214,7 +214,6 @@ pub struct Builder { opened_wals: Option, schema_config_provider: Option, forward_config: Option, - remote_engine_client_config: remote_engine_client::Config, auto_create_table: bool, } @@ -232,7 +231,6 @@ impl Builder { opened_wals: None, schema_config_provider: None, forward_config: None, - remote_engine_client_config: remote_engine_client::config::Config::default(), auto_create_table: true, } } @@ -289,11 +287,6 @@ impl Builder { self } - pub fn remote_engine_client_config(mut self, config: remote_engine_client::Config) -> Self { - self.remote_engine_client_config = config; - self - } - pub fn timeout(mut self, timeout: Option) -> Self { self.timeout = timeout; self @@ -334,13 +327,11 @@ impl Builder { }; let forward_config = self.forward_config.unwrap_or_default(); - let remote_engine_client_config = self.remote_engine_client_config; let bg_runtime = runtimes.bg_runtime.clone(); let proxy = Proxy::try_new( router, instance, forward_config, - remote_engine_client_config, self.local_endpoint.context(MissingLocalEndpoint)?, self.resp_compress_min_length, self.auto_create_table, diff --git a/server/src/instance.rs b/server/src/instance.rs index 5044a29e0b..5518a340c9 100644 --- a/server/src/instance.rs +++ b/server/src/instance.rs @@ -1,4 +1,4 @@ -// Copyright 2022 CeresDB Project Authors. Licensed under Apache-2.0. +// Copyright 2022-2023 CeresDB Project Authors. Licensed under Apache-2.0. //! Instance contains shared states of service @@ -7,7 +7,7 @@ use std::sync::Arc; use catalog::manager::ManagerRef; use df_operator::registry::FunctionRegistryRef; use interpreters::table_manipulator::TableManipulatorRef; -use table_engine::engine::TableEngineRef; +use table_engine::{engine::TableEngineRef, remote::RemoteEngineRef}; use crate::limiter::Limiter; @@ -18,10 +18,12 @@ pub struct Instance { pub catalog_manager: ManagerRef, pub query_executor: Q, pub table_engine: TableEngineRef, + pub partition_table_engine: TableEngineRef, // User defined functions registry. pub function_registry: FunctionRegistryRef, pub limiter: Limiter, pub table_manipulator: TableManipulatorRef, + pub remote_engine_ref: RemoteEngineRef, } /// A reference counted instance pointer diff --git a/server/src/proxy/grpc/sql_query.rs b/server/src/proxy/grpc/sql_query.rs index 85be3e6a8f..2fe8b5c894 100644 --- a/server/src/proxy/grpc/sql_query.rs +++ b/server/src/proxy/grpc/sql_query.rs @@ -449,6 +449,7 @@ impl Proxy { let first_sub_partition_table_name = get_sub_partition_name(&table_info.name, v, 0usize); let table = self + .instance .remote_engine_ref .get_table_info(GetTableInfoRequest { table: TableIdentifier { @@ -478,7 +479,7 @@ impl Proxy { partition_info: table_info.partition_info, }; let create_opts = CreateOptions { - table_engine: self.instance.table_engine.clone(), + table_engine: self.instance.partition_table_engine.clone(), create_if_not_exists: true, }; schema diff --git a/server/src/proxy/mod.rs b/server/src/proxy/mod.rs index eeb2da6411..22d7358b4b 100644 --- a/server/src/proxy/mod.rs +++ b/server/src/proxy/mod.rs @@ -12,10 +12,8 @@ use std::{str::FromStr, sync::Arc, time::Duration}; use common_util::{error::BoxError, runtime::Runtime}; use query_engine::executor::Executor as QueryExecutor; -use remote_engine_client::RemoteEngineImpl; use router::{endpoint::Endpoint, Router}; use snafu::ResultExt; -use table_engine::remote::RemoteEngineRef; use crate::{ instance::InstanceRef, @@ -33,7 +31,6 @@ pub struct Proxy { resp_compress_min_length: usize, auto_create_table: bool, schema_config_provider: SchemaConfigProviderRef, - remote_engine_ref: RemoteEngineRef, } impl Proxy { @@ -42,16 +39,11 @@ impl Proxy { router: Arc, instance: InstanceRef, forward_config: forward::Config, - remote_engine_client_config: remote_engine_client::config::Config, local_endpoint: String, resp_compress_min_length: usize, auto_create_table: bool, schema_config_provider: SchemaConfigProviderRef, ) -> Result { - let remote_engine_ref = Arc::new(RemoteEngineImpl::new( - remote_engine_client_config, - router.clone(), - )); let local_endpoint = Endpoint::from_str(&local_endpoint).with_context(|| Internal { msg: format!("invalid local endpoint, input:{local_endpoint}"), })?; @@ -62,6 +54,7 @@ impl Proxy { msg: "fail to init forward", })?, ); + Ok(Self { router, instance, @@ -69,7 +62,6 @@ impl Proxy { resp_compress_min_length, auto_create_table, schema_config_provider, - remote_engine_ref, }) } } diff --git a/server/src/server.rs b/server/src/server.rs index 40c0b420b9..204d4b6867 100644 --- a/server/src/server.rs +++ b/server/src/server.rs @@ -11,7 +11,9 @@ use df_operator::registry::FunctionRegistryRef; use interpreters::table_manipulator::TableManipulatorRef; use log::{info, warn}; use logger::RuntimeLevel; +use partition_table_engine::PartitionTableEngine; use query_engine::executor::Executor as QueryExecutor; +use remote_engine_client::RemoteEngineImpl; use router::{endpoint::Endpoint, RouterRef}; use snafu::{Backtrace, OptionExt, ResultExt, Snafu}; use table_engine::engine::{EngineRuntimes, TableEngineRef}; @@ -302,15 +304,25 @@ impl Builder { let table_manipulator = self.table_manipulator.context(MissingTableManipulator)?; let function_registry = self.function_registry.context(MissingFunctionRegistry)?; let opened_wals = self.opened_wals.context(MissingWals)?; + let router = self.router.context(MissingRouter)?; + + let remote_engine_ref = Arc::new(RemoteEngineImpl::new( + self.remote_engine_client_config.clone(), + router.clone(), + )); + + let partition_table_engine = Arc::new(PartitionTableEngine::new(remote_engine_ref.clone())); let instance = { let instance = Instance { catalog_manager, query_executor, table_engine, + partition_table_engine, function_registry, limiter: self.limiter, table_manipulator, + remote_engine_ref, }; InstanceRef::new(instance) }; @@ -356,7 +368,6 @@ impl Builder { .build() .context(BuildMysqlService)?; - let router = self.router.context(MissingRouter)?; let rpc_services = grpc::Builder::new() .endpoint( @@ -376,7 +387,6 @@ impl Builder { .opened_wals(opened_wals) .schema_config_provider(provider) .forward_config(self.server_config.forward) - .remote_engine_client_config(self.remote_engine_client_config) .timeout(self.server_config.timeout.map(|v| v.0)) .auto_create_table(self.server_config.auto_create_table) .build() diff --git a/src/setup.rs b/src/setup.rs index ebf74d79ad..4328794b69 100644 --- a/src/setup.rs +++ b/src/setup.rs @@ -1,4 +1,4 @@ -// Copyright 2022 CeresDB Project Authors. Licensed under Apache-2.0. +// Copyright 2022-2023 CeresDB Project Authors. Licensed under Apache-2.0. //! Setup server @@ -225,7 +225,6 @@ async fn build_with_meta( .expect("Failed to setup analytic engine"); let engine_builder = EngineBuilder { config: &config.analytic, - router: Some(router.clone()), engine_runtimes: runtimes.clone(), opened_wals: opened_wals.clone(), }; @@ -265,7 +264,6 @@ async fn build_without_meta( .expect("Failed to setup analytic engine"); let engine_builder = EngineBuilder { config: &config.analytic, - router: None, engine_runtimes: runtimes.clone(), opened_wals: opened_wals.clone(), }; diff --git a/table_engine/src/engine.rs b/table_engine/src/engine.rs index b9ce2c8047..5bea5091da 100644 --- a/table_engine/src/engine.rs +++ b/table_engine/src/engine.rs @@ -27,7 +27,7 @@ pub enum Error { #[snafu(display("Table already exists, table:{}.\nBacktrace:\n{}", table, backtrace))] TableExists { table: String, backtrace: Backtrace }, - #[snafu(display("Invalid arguments, err:{}", source))] + #[snafu(display("Invalid arguments, table:{table}, err:{}", source))] InvalidArguments { table: String, source: GenericError }, #[snafu(display("Failed to write meta data, err:{}", source))] @@ -36,7 +36,7 @@ pub enum Error { #[snafu(display("Unexpected error, err:{}", source))] Unexpected { source: GenericError }, - #[snafu(display("Unexpected error, :msg{}", msg))] + #[snafu(display("Unexpected error, msg:{}", msg))] UnexpectedNoCause { msg: String }, #[snafu(display( diff --git a/table_engine/src/lib.rs b/table_engine/src/lib.rs index 9874c6c239..f8ddf0fb42 100644 --- a/table_engine/src/lib.rs +++ b/table_engine/src/lib.rs @@ -1,4 +1,4 @@ -// Copyright 2022 CeresDB Project Authors. Licensed under Apache-2.0. +// Copyright 2022-2023 CeresDB Project Authors. Licensed under Apache-2.0. //! Table engine facade, provides read/write interfaces of table @@ -20,3 +20,4 @@ pub const OPTION_KEY_ENABLE_TTL: &str = "enable_ttl"; pub const MEMORY_ENGINE_TYPE: &str = "Memory"; pub const ANALYTIC_ENGINE_TYPE: &str = "Analytic"; +pub const PARTITION_TABLE_ENGINE_TYPE: &str = "PartitionTable"; From 53f551d1aeebb4686756badfdfa288d806fcd00a Mon Sep 17 00:00:00 2001 From: "chunshao.rcs" Date: Tue, 4 Apr 2023 16:11:11 +0800 Subject: [PATCH 4/9] test: modify integration_tests/cases/env/cluster/05_ddl/partition_table.result --- .../cases/env/cluster/05_ddl/partition_table.result | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration_tests/cases/env/cluster/05_ddl/partition_table.result b/integration_tests/cases/env/cluster/05_ddl/partition_table.result index 5580ee1321..17ae305ebf 100644 --- a/integration_tests/cases/env/cluster/05_ddl/partition_table.result +++ b/integration_tests/cases/env/cluster/05_ddl/partition_table.result @@ -19,7 +19,7 @@ affected_rows: 0 SHOW CREATE TABLE partition_table_t; Table,Create Table, -String("partition_table_t"),String("CREATE TABLE `partition_table_t` (`tsid` uint64 NOT NULL, `t` timestamp NOT NULL, `name` string TAG, `id` int TAG, `value` double NOT NULL, PRIMARY KEY(tsid,t), TIMESTAMP KEY(t)) PARTITION BY KEY(name) PARTITIONS 4 ENGINE=Analytic WITH(arena_block_size='2097152', compaction_strategy='default', compression='ZSTD', enable_ttl='false', num_rows_per_row_group='8192', segment_duration='', storage_format='AUTO', ttl='7d', update_mode='OVERWRITE', write_buffer_size='33554432')"), +String("partition_table_t"),String("CREATE TABLE `partition_table_t` (`tsid` uint64 NOT NULL, `t` timestamp NOT NULL, `name` string TAG, `id` int TAG, `value` double NOT NULL, PRIMARY KEY(tsid,t), TIMESTAMP KEY(t)) PARTITION BY KEY(name) PARTITIONS 4 ENGINE=Analytic WITH(enable_ttl='false')"), SHOW CREATE TABLE __partition_table_t_0; From 0e04a475489ebbe4ba25ae1f5e71702b6260a63a Mon Sep 17 00:00:00 2001 From: "chunshao.rcs" Date: Tue, 4 Apr 2023 16:16:57 +0800 Subject: [PATCH 5/9] refactor code --- analytic_engine/src/manifest/meta_update.rs | 1 + docs/example-cluster-0.toml | 4 +- docs/example-cluster-1.toml | 4 +- partition_table_engine/src/lib.rs | 32 ++++---- partition_table_engine/src/partition.rs | 86 +++++++++------------ server/src/grpc/mod.rs | 2 +- 6 files changed, 62 insertions(+), 67 deletions(-) diff --git a/analytic_engine/src/manifest/meta_update.rs b/analytic_engine/src/manifest/meta_update.rs index 7e641da45d..3068e5c7bf 100644 --- a/analytic_engine/src/manifest/meta_update.rs +++ b/analytic_engine/src/manifest/meta_update.rs @@ -160,6 +160,7 @@ impl From for manifest_pb::AddTableMeta { table_name: v.table_name, schema: Some(schema_pb::TableSchema::from(&v.schema)), options: Some(manifest_pb::TableOptions::from(v.opts)), + // Deprecated. partition_info: None, } } diff --git a/docs/example-cluster-0.toml b/docs/example-cluster-0.toml index d767846d31..da3fbe7631 100644 --- a/docs/example-cluster-0.toml +++ b/docs/example-cluster-0.toml @@ -1,12 +1,14 @@ [node] addr = "ceresdb0" +[logger] +level = "info" + [server] bind_addr = "0.0.0.0" http_port = 5440 grpc_port = 8831 mysql_port = 3307 -log_level = "info" deploy_mode = "Cluster" [server.forward] diff --git a/docs/example-cluster-1.toml b/docs/example-cluster-1.toml index c0b0f0f123..6a04569cbd 100644 --- a/docs/example-cluster-1.toml +++ b/docs/example-cluster-1.toml @@ -1,12 +1,14 @@ [node] addr = "ceresdb1" +[logger] +level = "info" + [server] bind_addr = "0.0.0.0" http_port = 5441 grpc_port = 8832 mysql_port = 13307 -log_level = "info" deploy_mode = "Cluster" [server.forward] diff --git a/partition_table_engine/src/lib.rs b/partition_table_engine/src/lib.rs index fe7edd2e1e..db5b89644d 100644 --- a/partition_table_engine/src/lib.rs +++ b/partition_table_engine/src/lib.rs @@ -21,7 +21,7 @@ use table_engine::{ PARTITION_TABLE_ENGINE_TYPE, }; -use crate::partition::PartitionTableImpl; +use crate::partition::{PartitionTableImpl, TableData}; /// Partition table engine implementation. pub struct PartitionTableEngine { @@ -45,22 +45,22 @@ impl TableEngine for PartitionTableEngine { } async fn create_table(&self, request: CreateTableRequest) -> Result { + let table_data = TableData { + catalog_name: request.catalog_name, + schema_name: request.schema_name, + table_name: request.table_name, + table_id: request.table_id, + table_schema: request.table_schema, + partition_info: request.partition_info.context(UnexpectedNoCause { + msg: "partition info not found", + })?, + options: request.options, + engine_type: request.engine, + }; Ok(Arc::new( - PartitionTableImpl::new( - request.catalog_name, - request.schema_name, - request.table_name, - request.table_id, - request.table_schema, - request.partition_info.context(UnexpectedNoCause { - msg: "partition info not found", - })?, - request.options, - request.engine, - self.remote_engine_ref.clone(), - ) - .box_err() - .context(Unexpected)?, + PartitionTableImpl::new(table_data, self.remote_engine_ref.clone()) + .box_err() + .context(Unexpected)?, )) } diff --git a/partition_table_engine/src/partition.rs b/partition_table_engine/src/partition.rs index 46a796c406..bb06ab9533 100644 --- a/partition_table_engine/src/partition.rs +++ b/partition_table_engine/src/partition.rs @@ -28,57 +28,46 @@ use table_engine::{ ReadRequest, Result, Scan, Table, TableId, TableStats, UnexpectedWithMsg, UnsupportedMethod, Write, WriteRequest, }, - PARTITION_TABLE_ENGINE_TYPE, }; use crate::metrics::{ PARTITION_TABLE_PARTITIONED_READ_DURATION_HISTOGRAM, PARTITION_TABLE_WRITE_DURATION_HISTOGRAM, }; +#[derive(Debug)] +pub struct TableData { + pub catalog_name: String, + pub schema_name: String, + pub table_name: String, + pub table_id: TableId, + pub table_schema: Schema, + pub partition_info: PartitionInfo, + pub options: HashMap, + pub engine_type: String, +} + /// Table trait implementation pub struct PartitionTableImpl { - catalog_name: String, - schema_name: String, - table_name: String, - table_id: TableId, - table_schema: Schema, - partition_info: PartitionInfo, - options: HashMap, - engine_type: String, + table_data: TableData, remote_engine: RemoteEngineRef, } impl PartitionTableImpl { - pub fn new( - catalog_name: String, - schema_name: String, - table_name: String, - table_id: TableId, - table_schema: Schema, - partition_info: PartitionInfo, - options: HashMap, - engine_type: String, - remote_engine: RemoteEngineRef, - ) -> Result { + pub fn new(table_data: TableData, remote_engine: RemoteEngineRef) -> Result { Ok(Self { - catalog_name, - schema_name, - table_name, - table_id, - table_schema, - partition_info, - options, - engine_type, + table_data, remote_engine, }) } fn get_sub_table_ident(&self, id: usize) -> TableIdentifier { - let partition_name = self.partition_info.get_definitions()[id].name.clone(); + let partition_name = self.table_data.partition_info.get_definitions()[id] + .name + .clone(); TableIdentifier { - catalog: self.catalog_name.clone(), - schema: self.schema_name.clone(), - table: format_sub_partition_table_name(&self.table_name, &partition_name), + catalog: self.table_data.catalog_name.clone(), + schema: self.table_data.schema_name.clone(), + table: format_sub_partition_table_name(&self.table_data.table_name, &partition_name), } } } @@ -86,10 +75,7 @@ impl PartitionTableImpl { impl fmt::Debug for PartitionTableImpl { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("PartitionTableImpl") - .field("catalog_name", &self.catalog_name) - .field("schema_name", &self.schema_name) - .field("table_name", &self.table_name) - .field("table_id", &self.table_id) + .field("table_data", &self.table_data) .finish() } } @@ -97,28 +83,28 @@ impl fmt::Debug for PartitionTableImpl { #[async_trait] impl Table for PartitionTableImpl { fn name(&self) -> &str { - &self.table_name + &self.table_data.table_name } fn id(&self) -> TableId { - self.table_id + self.table_data.table_id } fn schema(&self) -> Schema { - self.table_schema.clone() + self.table_data.table_schema.clone() } // TODO: get options from sub partition table with remote engine fn options(&self) -> HashMap { - self.options.clone() + self.table_data.options.clone() } fn partition_info(&self) -> Option { - Some(self.partition_info.clone()) + Some(self.table_data.partition_info.clone()) } fn engine_type(&self) -> &str { - &self.engine_type + &self.table_data.engine_type } fn stats(&self) -> TableStats { @@ -136,9 +122,11 @@ impl Table for PartitionTableImpl { msg: "partition table partition info can't be empty", } .fail()?, - Some(partition_info) => DfPartitionRuleAdapter::new(partition_info, &self.table_schema) - .box_err() - .context(CreatePartitionRule)?, + Some(partition_info) => { + DfPartitionRuleAdapter::new(partition_info, &self.table_data.table_schema) + .box_err() + .context(CreatePartitionRule)? + } }; // Split write request. @@ -219,9 +207,11 @@ impl Table for PartitionTableImpl { msg: "partition table partition info can't be empty", } .fail()?, - Some(partition_info) => DfPartitionRuleAdapter::new(partition_info, &self.table_schema) - .box_err() - .context(CreatePartitionRule)?, + Some(partition_info) => { + DfPartitionRuleAdapter::new(partition_info, &self.table_data.table_schema) + .box_err() + .context(CreatePartitionRule)? + } }; // Evaluate expr and locate partition. diff --git a/server/src/grpc/mod.rs b/server/src/grpc/mod.rs index 0988617bc2..26b4c08afc 100644 --- a/server/src/grpc/mod.rs +++ b/server/src/grpc/mod.rs @@ -1,4 +1,4 @@ -// Copyright 2022-2023 CeresDB Project Authors. Licensed under Apache-2.0. +// Copyright 2022 CeresDB Project Authors. Licensed under Apache-2.0. //! Grpc services From e0dc54e4b7bd08953707cac448c85cb732cac618 Mon Sep 17 00:00:00 2001 From: "chunshao.rcs" Date: Mon, 10 Apr 2023 16:02:16 +0800 Subject: [PATCH 6/9] refactor by CR --- analytic_engine/src/manifest/details.rs | 13 +------------ integration_tests/src/database.rs | 4 ++-- server/src/proxy/grpc/route.rs | 12 +++--------- sql/src/frontend.rs | 2 +- table_engine/src/engine.rs | 2 +- 5 files changed, 8 insertions(+), 25 deletions(-) diff --git a/analytic_engine/src/manifest/details.rs b/analytic_engine/src/manifest/details.rs index 2071a98e72..781d85a305 100644 --- a/analytic_engine/src/manifest/details.rs +++ b/analytic_engine/src/manifest/details.rs @@ -831,17 +831,6 @@ mod tests { }) } - fn meta_update_add_table_with_partition_info(&self, table_id: TableId) -> MetaUpdate { - let table_name = Self::table_name_from_id(table_id); - MetaUpdate::AddTable(AddTableMeta { - space_id: self.schema_id.as_u32(), - table_id, - table_name, - schema: common_types::tests::build_schema(), - opts: TableOptions::default(), - }) - } - fn meta_update_drop_table(&self, table_id: TableId) -> MetaUpdate { let table_name = Self::table_name_from_id(table_id); MetaUpdate::DropTable(DropTableMeta { @@ -895,7 +884,7 @@ mod tests { shard_id: DEFAULT_SHARD_ID, }; - let add_table = self.meta_update_add_table_with_partition_info(table_id); + let add_table = self.meta_update_add_table(table_id); let update_req = { MetaUpdateRequest { shard_info, diff --git a/integration_tests/src/database.rs b/integration_tests/src/database.rs index 3fb4f896ad..5e85b88442 100644 --- a/integration_tests/src/database.rs +++ b/integration_tests/src/database.rs @@ -17,7 +17,7 @@ use ceresdb_client::{ RpcContext, }; use reqwest::{ClientBuilder, Url}; -use sql::{frontend::parse_table_name, parser::Parser}; +use sql::{frontend, parser::Parser}; use sqlness::{Database, QueryContext}; const SERVER_GRPC_ENDPOINT_ENV: &str = "CERESDB_SERVER_GRPC_ENDPOINT"; @@ -272,7 +272,7 @@ impl CeresDB { timeout: None, }; let statements = Parser::parse_sql(&query).unwrap(); - let table_name = parse_table_name(&statements); + let table_name = frontend::parse_table_name(&statements); let query_req = match table_name { Some(table_name) => Request { diff --git a/server/src/proxy/grpc/route.rs b/server/src/proxy/grpc/route.rs index c73f42b86f..6ef7c6167f 100644 --- a/server/src/proxy/grpc/route.rs +++ b/server/src/proxy/grpc/route.rs @@ -32,15 +32,9 @@ impl Proxy { resp.routes = v .into_iter() - .map(|r| { - Route { - table: r.table_name, - endpoint: r.endpoint.map(Into::into), - } - // let mut router = Route::default(); - // router.table = r.table_name; - // router.endpoint = r.endpoint.map(Into::into); - // router + .map(|r| Route { + table: r.table_name, + endpoint: r.endpoint.map(Into::into), }) .collect(); } diff --git a/sql/src/frontend.rs b/sql/src/frontend.rs index 8bd64ac8fe..702de3ebb0 100644 --- a/sql/src/frontend.rs +++ b/sql/src/frontend.rs @@ -216,7 +216,7 @@ mod tests { #[test] fn test_parse_table_name() { let table = "test_parse_table_name"; - let test_cases=vec![ + let test_cases = vec![ format!("INSERT INTO {table} (t, name, value) VALUES (1651737067000, 'ceresdb', 100)"), format!("INSERT INTO `{table}` (t, name, value) VALUES (1651737067000,'ceresdb', 100)"), format!("select * from {table}"), diff --git a/table_engine/src/engine.rs b/table_engine/src/engine.rs index 5bea5091da..6bc2492501 100644 --- a/table_engine/src/engine.rs +++ b/table_engine/src/engine.rs @@ -27,7 +27,7 @@ pub enum Error { #[snafu(display("Table already exists, table:{}.\nBacktrace:\n{}", table, backtrace))] TableExists { table: String, backtrace: Backtrace }, - #[snafu(display("Invalid arguments, table:{table}, err:{}", source))] + #[snafu(display("Invalid arguments, table:{table}, err:{source}"))] InvalidArguments { table: String, source: GenericError }, #[snafu(display("Failed to write meta data, err:{}", source))] From 0f249f2110c1479c904a7c577496711454768e0f Mon Sep 17 00:00:00 2001 From: "chunshao.rcs" Date: Mon, 10 Apr 2023 18:14:13 +0800 Subject: [PATCH 7/9] refactor by CR --- Cargo.lock | 1 + catalog/src/schema.rs | 6 +- meta_client/src/types.rs | 4 +- remote_engine_client/src/cached_router.rs | 3 +- router/Cargo.toml | 1 + router/src/cluster_based.rs | 69 +++++++-- router/src/lib.rs | 28 ++-- router/src/rule_based.rs | 32 ++-- server/src/proxy/forward.rs | 54 +++---- server/src/proxy/grpc/route.rs | 11 +- server/src/proxy/grpc/sql_query.rs | 174 +++++++++++++--------- server/src/proxy/mod.rs | 1 - 12 files changed, 225 insertions(+), 159 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 194fe49b6b..a2cb3be3e7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5280,6 +5280,7 @@ dependencies = [ "moka", "serde", "snafu 0.6.10", + "table_engine", "tokio", "twox-hash", ] diff --git a/catalog/src/schema.rs b/catalog/src/schema.rs index 2dbcad69be..03c27564cf 100644 --- a/catalog/src/schema.rs +++ b/catalog/src/schema.rs @@ -214,11 +214,7 @@ impl CreateTableRequest { self, table_id: Option, ) -> engine::CreateTableRequest { - let table_id = match (self.table_id, table_id) { - (Some(v), _) => v, - (None, Some(v)) => v, - (None, None) => TableId::MIN, - }; + let table_id = self.table_id.unwrap_or(table_id.unwrap_or(TableId::MIN)); engine::CreateTableRequest { catalog_name: self.catalog_name, schema_name: self.schema_name, diff --git a/meta_client/src/types.rs b/meta_client/src/types.rs index 46796df831..193665d93d 100644 --- a/meta_client/src/types.rs +++ b/meta_client/src/types.rs @@ -401,7 +401,7 @@ pub struct NodeShard { #[derive(Debug, Clone)] pub struct RouteEntry { - pub table: TableInfo, + pub table_info: TableInfo, pub node_shards: Vec, } @@ -457,7 +457,7 @@ impl TryFrom for RouteEntry { msg: "table info is missing in route entry", })?; Ok(RouteEntry { - table: TableInfo::try_from(table_info)?, + table_info: TableInfo::try_from(table_info)?, node_shards, }) } diff --git a/remote_engine_client/src/cached_router.rs b/remote_engine_client/src/cached_router.rs index ef57f0d4bf..860b252d32 100644 --- a/remote_engine_client/src/cached_router.rs +++ b/remote_engine_client/src/cached_router.rs @@ -1,4 +1,4 @@ -// Copyright 2022-2023 CeresDB Project Authors. Licensed under Apache-2.0. +// Copyright 2022 CeresDB Project Authors. Licensed under Apache-2.0. //! Cached router @@ -120,6 +120,7 @@ impl CachedRouter { msg: "no endpoint in route info", })?; + let endpoint = endpoint.into(); let channel = self.channel_pool.get(&endpoint).await?; Ok(channel) diff --git a/router/Cargo.toml b/router/Cargo.toml index 4df5a99ca7..2a7aaa5922 100644 --- a/router/Cargo.toml +++ b/router/Cargo.toml @@ -21,5 +21,6 @@ meta_client = { workspace = true } moka = { version = "0.10", features = ["future"] } serde = { workspace = true } snafu = { workspace = true } +table_engine = { workspace = true } tokio = { workspace = true } twox-hash = "1.6" diff --git a/router/src/cluster_based.rs b/router/src/cluster_based.rs index 52c346bbbd..63332feb86 100644 --- a/router/src/cluster_based.rs +++ b/router/src/cluster_based.rs @@ -1,22 +1,23 @@ -// Copyright 2022-2023 CeresDB Project Authors. Licensed under Apache-2.0. +// Copyright 2022 CeresDB Project Authors. Licensed under Apache-2.0. //! A router based on the [`cluster::Cluster`]. use async_trait::async_trait; -use ceresdbproto::storage::RouteRequest; +use ceresdbproto::storage::{Route, RouteRequest}; use cluster::ClusterRef; use common_util::error::BoxError; -use meta_client::types::{RouteTablesRequest, TableInfo}; +use meta_client::types::RouteTablesRequest; use moka::future::Cache; use snafu::ResultExt; use crate::{ - endpoint::Endpoint, OtherWithCause, ParseEndpoint, Result, RouteCacheConfig, RouteData, Router, + endpoint::Endpoint, OtherWithCause, ParseEndpoint, PartitionTableInfo, Result, + RouteCacheConfig, Router, }; pub struct ClusterBasedRouter { cluster: ClusterRef, - cache: Option>, + cache: Option>, } impl ClusterBasedRouter { @@ -38,7 +39,7 @@ impl ClusterBasedRouter { /// route table from local cache, return cache routes and tables which are /// not in cache - fn route_from_cache(&self, tables: Vec, routes: &mut Vec) -> Vec { + fn route_from_cache(&self, tables: Vec, routes: &mut Vec) -> Vec { let mut miss = vec![]; if let Some(cache) = &self.cache { @@ -58,19 +59,18 @@ impl ClusterBasedRouter { } /// Make a route according to the table name and the raw endpoint. -fn make_route(table: TableInfo, endpoint: &str) -> Result { +fn make_route(table_name: &str, endpoint: &str) -> Result { let endpoint: Endpoint = endpoint.parse().context(ParseEndpoint { endpoint })?; - Ok(RouteData { - table_name: table.name.clone(), - table: Some(table), - endpoint: Some(endpoint), + Ok(Route { + table: table_name.to_string(), + endpoint: Some(endpoint.into()), }) } #[async_trait] impl Router for ClusterBasedRouter { - async fn route(&self, req: RouteRequest) -> Result> { + async fn route(&self, req: RouteRequest) -> Result> { let req_ctx = req.context.unwrap(); // Firstly route table from local cache. @@ -99,7 +99,7 @@ impl Router for ClusterBasedRouter { for (table_name, route_entry) in route_resp.entries { for node_shard in route_entry.node_shards { if node_shard.shard_info.is_leader() { - let route = make_route(route_entry.table.clone(), &node_shard.endpoint)?; + let route = make_route(&table_name, &node_shard.endpoint)?; if let Some(cache) = &self.cache { // There may be data race here, and it is acceptable currently. cache.insert(table_name.clone(), route.clone()).await; @@ -110,6 +110,43 @@ impl Router for ClusterBasedRouter { } return Ok(routes); } + + async fn fetch_partition_table_info( + &self, + schema: &str, + table: &str, + ) -> Result> { + let route_tables_req = RouteTablesRequest { + schema_name: schema.to_string(), + table_names: vec![table.to_string()], + }; + let route_resp = self + .cluster + .route_tables(&route_tables_req) + .await + .box_err() + .with_context(|| OtherWithCause { + msg: format!("Failed to route tables by cluster, req:{route_tables_req:?}"), + })?; + + let table_info = route_resp + .entries + .get(table) + .map(|entry| entry.table_info.clone()); + if let Some(v) = table_info { + if v.partition_info.is_some() { + let partition_table_info = PartitionTableInfo { + id: v.id, + name: v.name, + schema_id: v.schema_id, + schema_name: v.schema_name, + partition_info: v.partition_info.unwrap(), + }; + return Ok(Some(partition_table_info)); + } + } + Ok(None) + } } #[cfg(test)] @@ -183,7 +220,7 @@ mod tests { entries.insert( table.clone(), RouteEntry { - table: TableInfo { + table_info: TableInfo { id: 0, name: table.clone(), schema_name: String::from("public"), @@ -251,7 +288,7 @@ mod tests { let mut routes = Vec::with_capacity(tables.len()); let miss = router.route_from_cache(tables, &mut routes); assert_eq!(routes.len(), 1); - assert_eq!(routes[0].table_name, table1.to_string()); + assert_eq!(routes[0].table, table1.to_string()); assert_eq!(miss.len(), 0); // sleep 1.5s, table2 will be evicted, and table1 in cache @@ -260,7 +297,7 @@ mod tests { let mut routes = Vec::with_capacity(tables.len()); let miss = router.route_from_cache(tables, &mut routes); assert_eq!(routes.len(), 1); - assert_eq!(routes[0].table_name, table1.to_string()); + assert_eq!(routes[0].table, table1.to_string()); assert_eq!(miss.len(), 1); assert_eq!(miss[0], table2.to_string()); } diff --git a/router/src/lib.rs b/router/src/lib.rs index 5ff6bca616..3a388d2025 100644 --- a/router/src/lib.rs +++ b/router/src/lib.rs @@ -1,4 +1,4 @@ -// Copyright 2022-2023 CeresDB Project Authors. Licensed under Apache-2.0. +// Copyright 2022 CeresDB Project Authors. Licensed under Apache-2.0. pub mod cluster_based; pub mod endpoint; @@ -7,15 +7,14 @@ pub mod rule_based; use std::{sync::Arc, time::Duration}; use async_trait::async_trait; -use ceresdbproto::storage::RouteRequest; +use ceresdbproto::storage::{Route, RouteRequest}; pub use cluster_based::ClusterBasedRouter; +use common_types::{schema::SchemaId, table::TableId}; use common_util::{config::ReadableDuration, define_result}; -use meta_client::types::TableInfo; pub use rule_based::{RuleBasedRouter, RuleList}; use serde::{Deserialize, Serialize}; use snafu::{Backtrace, Snafu}; - -use crate::endpoint::Endpoint; +use table_engine::partition::PartitionInfo; #[derive(Snafu, Debug)] #[snafu(visibility(pub))] @@ -62,16 +61,23 @@ define_result!(Error); pub type RouterRef = Arc; -#[derive(Debug, Clone)] -pub struct RouteData { - pub table_name: String, - pub table: Option, - pub endpoint: Option, +#[derive(Clone, Debug)] +pub struct PartitionTableInfo { + pub id: TableId, + pub name: String, + pub schema_id: SchemaId, + pub schema_name: String, + pub partition_info: PartitionInfo, } #[async_trait] pub trait Router { - async fn route(&self, req: RouteRequest) -> Result>; + async fn route(&self, req: RouteRequest) -> Result>; + async fn fetch_partition_table_info( + &self, + schema: &str, + table: &str, + ) -> Result>; } #[derive(Clone, Debug, Deserialize, Serialize)] diff --git a/router/src/rule_based.rs b/router/src/rule_based.rs index c1b52125ec..7ad772bdad 100644 --- a/router/src/rule_based.rs +++ b/router/src/rule_based.rs @@ -1,18 +1,20 @@ -// Copyright 2022-2023 CeresDB Project Authors. Licensed under Apache-2.0. +// Copyright 2022 CeresDB Project Authors. Licensed under Apache-2.0. //! A router based on rules. use std::collections::HashMap; use async_trait::async_trait; -use ceresdbproto::storage::RouteRequest; +use ceresdbproto::storage::{self, Route, RouteRequest}; use cluster::config::SchemaConfig; use log::info; use meta_client::types::ShardId; use serde::{Deserialize, Serialize}; use snafu::{ensure, OptionExt}; -use crate::{endpoint::Endpoint, hash, Result, RouteData, RouteNotFound, Router, ShardNotFound}; +use crate::{ + endpoint::Endpoint, hash, PartitionTableInfo, Result, RouteNotFound, Router, ShardNotFound, +}; pub type ShardNodes = HashMap; @@ -138,7 +140,7 @@ impl RuleBasedRouter { #[async_trait] impl Router for RuleBasedRouter { - async fn route(&self, req: RouteRequest) -> Result> { + async fn route(&self, req: RouteRequest) -> Result> { let req_ctx = req.context.unwrap(); let schema = &req_ctx.database; if let Some(shard_nodes) = self.cluster_view.schema_shards.get(schema) { @@ -150,18 +152,18 @@ impl Router for RuleBasedRouter { // TODO(yingwen): Better way to get total shard number let total_shards = shard_nodes.len(); let mut route_results = Vec::with_capacity(req.tables.len()); - for table_name in req.tables { - let shard_id = Self::route_table(&table_name, rule_list_opt, total_shards); + for table in req.tables { + let shard_id = Self::route_table(&table, rule_list_opt, total_shards); let endpoint = shard_nodes.get(&shard_id).with_context(|| ShardNotFound { schema, - table: &table_name, + table: &table, })?; - let route = RouteData { - table_name, - table: None, - endpoint: Some(endpoint.to_owned()), + let pb_endpoint = storage::Endpoint::from(endpoint.clone()); + let route = Route { + table, + endpoint: Some(pb_endpoint), }; route_results.push(route); } @@ -170,4 +172,12 @@ impl Router for RuleBasedRouter { Ok(Vec::new()) } + + async fn fetch_partition_table_info( + &self, + _schema: &str, + _table: &str, + ) -> Result> { + return Ok(None); + } } diff --git a/server/src/proxy/forward.rs b/server/src/proxy/forward.rs index 2fc1acf67e..b29b302470 100644 --- a/server/src/proxy/forward.rs +++ b/server/src/proxy/forward.rs @@ -1,4 +1,4 @@ -// Copyright 2022-2023 CeresDB Project Authors. Licensed under Apache-2.0. +// Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. //! Forward for grpc services use std::{ @@ -13,7 +13,6 @@ use ceresdbproto::storage::{ storage_service_client::StorageServiceClient, RequestContext, RouteRequest, }; use log::{debug, error, warn}; -use meta_client::types::TableInfo; use router::{endpoint::Endpoint, RouterRef}; use serde::{Deserialize, Serialize}; use snafu::{Backtrace, ResultExt, Snafu}; @@ -180,7 +179,6 @@ pub struct Forwarder { /// If no forwarding happens, [`Original`] can be used. pub enum ForwardResult { Original, - OriginalPartitionTableInfo(TableInfo), Forwarded(std::result::Result), } @@ -280,34 +278,15 @@ impl Forwarder { let endpoint = match self.router.route(route_req).await { Ok(mut routes) => { - if routes.len() == 1 { - if routes[0].table.is_some() { - let table = routes[0].table.clone().unwrap(); - if table.partition_info.is_some() { - return Ok(ForwardResult::OriginalPartitionTableInfo(table)); - } - } - - if routes[0].endpoint.is_none() { - warn!( - "Fail to forward request for empty route results, routes result:{:?}, req:{:?}", - routes, req - ); - return Ok(ForwardResult::Original); - } - let endpoint = routes.remove(0).endpoint.unwrap(); - if self.is_local_endpoint(&endpoint) { - return Ok(ForwardResult::Original); - } - endpoint - } else { + if routes.len() != 1 || routes[0].endpoint.is_none() { warn!( "Fail to forward request for multiple or empty route results, routes result:{:?}, req:{:?}", routes, req ); - // TODO: shall we return an error? return Ok(ForwardResult::Original); } + + Endpoint::from(routes.remove(0).endpoint.unwrap()) } Err(e) => { error!("Fail to route request, req:{:?}, err:{}", req, e); @@ -315,6 +294,10 @@ impl Forwarder { } }; + if self.is_local_endpoint(&endpoint) { + return Ok(ForwardResult::Original); + } + // Update the request. { // TODO: we should use the timeout from the original request. @@ -364,9 +347,9 @@ impl Forwarder { #[cfg(test)] mod tests { use catalog::consts::DEFAULT_SCHEMA; - use ceresdbproto::storage::{SqlQueryRequest, SqlQueryResponse}; + use ceresdbproto::storage::{Route, SqlQueryRequest, SqlQueryResponse}; use futures::FutureExt; - use router::{RouteData, Router}; + use router::{PartitionTableInfo, Router}; use tonic::IntoRequest; use super::*; @@ -397,17 +380,24 @@ mod tests { #[async_trait] impl Router for MockRouter { - async fn route(&self, req: RouteRequest) -> router::Result> { + async fn route(&self, req: RouteRequest) -> router::Result> { let endpoint = self.routing_tables.get(&req.tables[0]); match endpoint { None => Ok(vec![]), - Some(v) => Ok(vec![RouteData { - table_name: req.tables[0].clone(), - table: None, - endpoint: Some(v.clone()), + Some(v) => Ok(vec![Route { + table: req.tables[0].clone(), + endpoint: Some(v.clone().into()), }]), } } + + async fn fetch_partition_table_info( + &self, + _schema: &str, + _table: &str, + ) -> router::Result> { + return Ok(None); + } } struct MockClientBuilder; diff --git a/server/src/proxy/grpc/route.rs b/server/src/proxy/grpc/route.rs index 6ef7c6167f..8e74185837 100644 --- a/server/src/proxy/grpc/route.rs +++ b/server/src/proxy/grpc/route.rs @@ -1,6 +1,6 @@ // Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. -use ceresdbproto::storage::{Route, RouteRequest, RouteResponse}; +use ceresdbproto::storage::{RouteRequest, RouteResponse}; use common_util::error::BoxError; use http::StatusCode; use log::error; @@ -29,14 +29,7 @@ impl Proxy { } Ok(v) => { resp.header = Some(error::build_ok_header()); - - resp.routes = v - .into_iter() - .map(|r| Route { - table: r.table_name, - endpoint: r.endpoint.map(Into::into), - }) - .collect(); + resp.routes = v; } } resp diff --git a/server/src/proxy/grpc/sql_query.rs b/server/src/proxy/grpc/sql_query.rs index 2fe8b5c894..833115233a 100644 --- a/server/src/proxy/grpc/sql_query.rs +++ b/server/src/proxy/grpc/sql_query.rs @@ -5,7 +5,7 @@ use std::{sync::Arc, time::Instant}; use arrow_ext::ipc::{CompressOptions, CompressionMethod, RecordBatchesEncoder}; -use catalog::schema::{CreateOptions, CreateTableRequest}; +use catalog::schema::{CreateOptions, CreateTableRequest, DropOptions, DropTableRequest}; use ceresdbproto::{ common::ResponseHeader, storage::{ @@ -19,11 +19,11 @@ use futures::{stream, stream::BoxStream, FutureExt, StreamExt}; use http::StatusCode; use interpreters::{context::Context as InterpreterContext, factory::Factory, interpreter::Output}; use log::{error, info, warn}; -use meta_client::types::TableInfo; use query_engine::executor::Executor as QueryExecutor; use router::endpoint::Endpoint; use snafu::{ensure, OptionExt, ResultExt}; use sql::{ + frontend, frontend::{Context as SqlContext, Frontend}, provider::CatalogMetaProvider, }; @@ -32,6 +32,7 @@ use table_engine::{ partition::{format_sub_partition_table_name, PartitionInfo}, remote::model::{GetTableInfoRequest, TableIdentifier}, table::TableId, + PARTITION_TABLE_ENGINE_TYPE, }; use tokio::sync::mpsc; use tokio_stream::wrappers::ReceiverStream; @@ -82,16 +83,15 @@ impl Proxy { ctx: Context, req: SqlQueryRequest, ) -> Result { - let (req, table) = match self.maybe_forward_sql_query(&req).await { + let req = match self.maybe_forward_sql_query(&req).await { Some(resp) => match resp { ForwardResult::Forwarded(resp) => return resp, - ForwardResult::Original => (req, None), - ForwardResult::OriginalPartitionTableInfo(table) => (req, Some(table)), + ForwardResult::Original => req, }, - None => (req, None), + None => req, }; - let output = self.fetch_sql_query_output(ctx, &req, table).await?; + let output = self.fetch_sql_query_output(ctx, &req).await?; convert_output(&output, self.resp_compress_min_length) } @@ -100,22 +100,18 @@ impl Proxy { ctx: Context, req: SqlQueryRequest, ) -> Result> { - let (req, table) = match self.clone().maybe_forward_stream_sql_query(&req).await { + let req = match self.clone().maybe_forward_stream_sql_query(&req).await { Some(resp) => match resp { ForwardResult::Forwarded(resp) => return resp, - ForwardResult::Original => (req, None), - ForwardResult::OriginalPartitionTableInfo(table) => (req, Some(table)), + ForwardResult::Original => req, }, - None => (req, None), + None => req, }; let (tx, rx) = mpsc::channel(STREAM_QUERY_CHANNEL_LEN); let runtime = ctx.runtime.clone(); let resp_compress_min_length = self.resp_compress_min_length; - let output = self - .as_ref() - .fetch_sql_query_output(ctx, &req, table) - .await?; + let output = self.as_ref().fetch_sql_query_output(ctx, &req).await?; runtime.spawn(async move { match output { Output::AffectedRows(rows) => { @@ -251,12 +247,7 @@ impl Proxy { } } - async fn fetch_sql_query_output( - &self, - ctx: Context, - req: &SqlQueryRequest, - table_info: Option, - ) -> Result { + async fn fetch_sql_query_output(&self, ctx: Context, req: &SqlQueryRequest) -> Result { let request_id = RequestId::next_id(); let begin_instant = Instant::now(); let deadline = ctx.timeout.map(|t| begin_instant + t); @@ -309,12 +300,12 @@ impl Proxy { ), } ); + // Open partition table if needed. - if let Some(v) = table_info { - if v.partition_info.is_some() { - self.open_partition_table_if_not_exist(catalog, schema, v) - .await?; - } + let table_name = frontend::parse_table_name(&stmts); + if let Some(table_name) = table_name { + self.open_partition_table_if_not_exist(catalog, schema, &table_name) + .await?; } // Create logical plan @@ -403,8 +394,18 @@ impl Proxy { &self, catalog_name: &str, schema_name: &str, - table_info: TableInfo, + table_name: &str, ) -> Result<()> { + let partition_table_info = self + .router + .fetch_partition_table_info(schema_name, table_name) + .await?; + if partition_table_info.is_none() { + return Ok(()); + } + + let partition_table_info = partition_table_info.unwrap(); + let catalog = self .instance .catalog_manager @@ -432,65 +433,96 @@ impl Proxy { msg: format!("Schema not found, schema_name:{schema_name}"), })?; let table = schema - .table_by_name(&table_info.name) + .table_by_name(&partition_table_info.name) .box_err() .with_context(|| ErrWithCause { code: StatusCode::INTERNAL_SERVER_ERROR, - msg: format!("Failed to find table, table_name:{}", table_info.name), + msg: format!( + "Failed to find table, table_name:{}", + partition_table_info.name + ), })?; - if table.is_some() { - return Ok(()); - } - - // If table not exists, open it. - if let Some(v) = &table_info.partition_info { - // Get table_schema from first sub partition table. - let first_sub_partition_table_name = - get_sub_partition_name(&table_info.name, v, 0usize); - let table = self - .instance - .remote_engine_ref - .get_table_info(GetTableInfoRequest { - table: TableIdentifier { - catalog: catalog_name.to_string(), - schema: schema_name.to_string(), - table: first_sub_partition_table_name, - }, - }) - .await - .box_err() - .with_context(|| ErrWithCause { - code: StatusCode::INTERNAL_SERVER_ERROR, - msg: "Failed to get table", - })?; + if let Some(table) = table { + if table.id().as_u64() == partition_table_info.id { + return Ok(()); + } - let create_table_request = CreateTableRequest { - catalog_name: catalog_name.to_string(), - schema_name: schema_name.to_string(), - schema_id: schema.id(), - table_name: table_info.name, - table_id: Some(TableId::new(table_info.id)), - table_schema: table.table_schema, - engine: table.engine, - options: Default::default(), - state: TableState::Stable, - shard_id: DEFAULT_SHARD_ID, - partition_info: table_info.partition_info, - }; - let create_opts = CreateOptions { + // Drop partition table if table id not match. + let opts = DropOptions { table_engine: self.instance.partition_table_engine.clone(), - create_if_not_exists: true, }; schema - .create_table(create_table_request.clone(), create_opts) + .drop_table( + DropTableRequest { + catalog_name: catalog_name.to_string(), + schema_name: schema_name.to_string(), + schema_id: schema.id(), + table_name: table_name.to_string(), + engine: PARTITION_TABLE_ENGINE_TYPE.to_string(), + }, + opts, + ) .await .box_err() .with_context(|| ErrWithCause { code: StatusCode::INTERNAL_SERVER_ERROR, - msg: format!("Failed to create table, request:{create_table_request:?}"), + msg: format!("Failed to drop partition table, table_name:{table_name}"), })?; } + + // If table not exists, open it. + // Get table_schema from first sub partition table. + let first_sub_partition_table_name = get_sub_partition_name( + &partition_table_info.name, + &partition_table_info.partition_info, + 0usize, + ); + let table = self + .instance + .remote_engine_ref + .get_table_info(GetTableInfoRequest { + table: TableIdentifier { + catalog: catalog_name.to_string(), + schema: schema_name.to_string(), + table: first_sub_partition_table_name, + }, + }) + .await + .box_err() + .with_context(|| ErrWithCause { + code: StatusCode::INTERNAL_SERVER_ERROR, + msg: "Failed to get table", + })?; + + // Partition table is a virtual table, so we need to create it manually. + // Partition info is stored in ceresmeta, so we need to use create_table_request + // to create it. + let create_table_request = CreateTableRequest { + catalog_name: catalog_name.to_string(), + schema_name: schema_name.to_string(), + schema_id: schema.id(), + table_name: partition_table_info.name, + table_id: Some(TableId::new(partition_table_info.id)), + table_schema: table.table_schema, + engine: table.engine, + options: Default::default(), + state: TableState::Stable, + shard_id: DEFAULT_SHARD_ID, + partition_info: Some(partition_table_info.partition_info), + }; + let create_opts = CreateOptions { + table_engine: self.instance.partition_table_engine.clone(), + create_if_not_exists: true, + }; + schema + .create_table(create_table_request.clone(), create_opts) + .await + .box_err() + .with_context(|| ErrWithCause { + code: StatusCode::INTERNAL_SERVER_ERROR, + msg: format!("Failed to create table, request:{create_table_request:?}"), + })?; Ok(()) } } diff --git a/server/src/proxy/mod.rs b/server/src/proxy/mod.rs index 22d7358b4b..cadff199b9 100644 --- a/server/src/proxy/mod.rs +++ b/server/src/proxy/mod.rs @@ -34,7 +34,6 @@ pub struct Proxy { } impl Proxy { - #[allow(clippy::too_many_arguments)] pub fn try_new( router: Arc, instance: InstanceRef, From 2e9c523c282be0cf4094bdb59bcd1169b09d57c0 Mon Sep 17 00:00:00 2001 From: "chunshao.rcs" Date: Tue, 11 Apr 2023 11:26:55 +0800 Subject: [PATCH 8/9] refactor by CR --- catalog/src/schema.rs | 1 + catalog_impls/src/volatile.rs | 2 +- router/src/cluster_based.rs | 122 ++++++++++++---------- server/src/grpc/meta_event_service/mod.rs | 23 ++-- server/src/proxy/grpc/sql_query.rs | 4 +- 5 files changed, 79 insertions(+), 73 deletions(-) diff --git a/catalog/src/schema.rs b/catalog/src/schema.rs index 621062e42d..5885a35a1c 100644 --- a/catalog/src/schema.rs +++ b/catalog/src/schema.rs @@ -192,6 +192,7 @@ pub struct CreateTableRequest { /// Table name pub table_name: String, /// Table id + // TODO: remove this field pub table_id: Option, /// Table schema pub table_schema: common_types::schema::Schema, diff --git a/catalog_impls/src/volatile.rs b/catalog_impls/src/volatile.rs index 8c95db016b..dcf0ecd173 100644 --- a/catalog_impls/src/volatile.rs +++ b/catalog_impls/src/volatile.rs @@ -313,7 +313,7 @@ impl Schema for SchemaImpl { .with_context(|| schema::CreateTable { request: request.clone(), msg: format!("table with shards is not found in the ShardTableManager, catalog_name:{}, schema_name:{}, table_name:{}", - request.catalog_name,request.schema_name,request.table_name), + request.catalog_name, request.schema_name, request.table_name), })?; } let request = request.into_engine_create_request(None, self.schema_id); diff --git a/router/src/cluster_based.rs b/router/src/cluster_based.rs index 63332feb86..8c29f88b20 100644 --- a/router/src/cluster_based.rs +++ b/router/src/cluster_based.rs @@ -1,4 +1,4 @@ -// Copyright 2022 CeresDB Project Authors. Licensed under Apache-2.0. +// Copyright 2022-2023 CeresDB Project Authors. Licensed under Apache-2.0. //! A router based on the [`cluster::Cluster`]. @@ -6,7 +6,7 @@ use async_trait::async_trait; use ceresdbproto::storage::{Route, RouteRequest}; use cluster::ClusterRef; use common_util::error::BoxError; -use meta_client::types::RouteTablesRequest; +use meta_client::types::{RouteTablesRequest, TableInfo}; use moka::future::Cache; use snafu::ResultExt; @@ -15,9 +15,16 @@ use crate::{ RouteCacheConfig, Router, }; +#[derive(Clone)] +struct RouteData { + table_name: String, + table_info: TableInfo, + endpoint: Option, +} + pub struct ClusterBasedRouter { cluster: ClusterRef, - cache: Option>, + cache: Option>, } impl ClusterBasedRouter { @@ -39,7 +46,7 @@ impl ClusterBasedRouter { /// route table from local cache, return cache routes and tables which are /// not in cache - fn route_from_cache(&self, tables: Vec, routes: &mut Vec) -> Vec { + fn route_from_cache(&self, tables: Vec, routes: &mut Vec) -> Vec { let mut miss = vec![]; if let Some(cache) = &self.cache { @@ -56,33 +63,20 @@ impl ClusterBasedRouter { miss } -} - -/// Make a route according to the table name and the raw endpoint. -fn make_route(table_name: &str, endpoint: &str) -> Result { - let endpoint: Endpoint = endpoint.parse().context(ParseEndpoint { endpoint })?; - - Ok(Route { - table: table_name.to_string(), - endpoint: Some(endpoint.into()), - }) -} - -#[async_trait] -impl Router for ClusterBasedRouter { - async fn route(&self, req: RouteRequest) -> Result> { - let req_ctx = req.context.unwrap(); + async fn route_with_cache( + &self, + tables: Vec, + database: String, + ) -> Result> { // Firstly route table from local cache. - let mut routes = Vec::with_capacity(req.tables.len()); - let miss = self.route_from_cache(req.tables, &mut routes); - + let mut routes = Vec::with_capacity(tables.len()); + let miss = self.route_from_cache(tables, &mut routes); if miss.is_empty() { return Ok(routes); } - let route_tables_req = RouteTablesRequest { - schema_name: req_ctx.database, + schema_name: database, table_names: miss, }; @@ -99,7 +93,7 @@ impl Router for ClusterBasedRouter { for (table_name, route_entry) in route_resp.entries { for node_shard in route_entry.node_shards { if node_shard.shard_info.is_leader() { - let route = make_route(&table_name, &node_shard.endpoint)?; + let route = make_route(route_entry.table_info.clone(), &node_shard.endpoint)?; if let Some(cache) = &self.cache { // There may be data race here, and it is acceptable currently. cache.insert(table_name.clone(), route.clone()).await; @@ -108,7 +102,33 @@ impl Router for ClusterBasedRouter { } } } - return Ok(routes); + Ok(routes) + } +} + +/// Make a route according to the table_info and the raw endpoint. +fn make_route(table_info: TableInfo, endpoint: &str) -> Result { + let endpoint: Endpoint = endpoint.parse().context(ParseEndpoint { endpoint })?; + + Ok(RouteData { + table_name: table_info.name.clone(), + table_info, + endpoint: Some(endpoint), + }) +} + +#[async_trait] +impl Router for ClusterBasedRouter { + async fn route(&self, req: RouteRequest) -> Result> { + let req_ctx = req.context.unwrap(); + let route_data_vec = self.route_with_cache(req.tables, req_ctx.database).await?; + Ok(route_data_vec + .into_iter() + .map(|v| Route { + table: v.table_name, + endpoint: v.endpoint.map(Into::into), + }) + .collect()) } async fn fetch_partition_table_info( @@ -116,35 +136,25 @@ impl Router for ClusterBasedRouter { schema: &str, table: &str, ) -> Result> { - let route_tables_req = RouteTablesRequest { - schema_name: schema.to_string(), - table_names: vec![table.to_string()], - }; - let route_resp = self - .cluster - .route_tables(&route_tables_req) - .await - .box_err() - .with_context(|| OtherWithCause { - msg: format!("Failed to route tables by cluster, req:{route_tables_req:?}"), - })?; + let mut route_data_vec = self + .route_with_cache(vec![table.to_string()], schema.to_string()) + .await?; + if route_data_vec.is_empty() { + return Ok(None); + } - let table_info = route_resp - .entries - .get(table) - .map(|entry| entry.table_info.clone()); - if let Some(v) = table_info { - if v.partition_info.is_some() { - let partition_table_info = PartitionTableInfo { - id: v.id, - name: v.name, - schema_id: v.schema_id, - schema_name: v.schema_name, - partition_info: v.partition_info.unwrap(), - }; - return Ok(Some(partition_table_info)); - } + let route_data = route_data_vec.remove(0); + let table_info = route_data.table_info; + if table_info.partition_info.is_some() { + return Ok(Some(PartitionTableInfo { + id: table_info.id, + name: table_info.name, + schema_id: table_info.schema_id, + schema_name: table_info.schema_name, + partition_info: table_info.partition_info.unwrap(), + })); } + Ok(None) } } @@ -288,7 +298,7 @@ mod tests { let mut routes = Vec::with_capacity(tables.len()); let miss = router.route_from_cache(tables, &mut routes); assert_eq!(routes.len(), 1); - assert_eq!(routes[0].table, table1.to_string()); + assert_eq!(routes[0].table_name, table1.to_string()); assert_eq!(miss.len(), 0); // sleep 1.5s, table2 will be evicted, and table1 in cache @@ -297,7 +307,7 @@ mod tests { let mut routes = Vec::with_capacity(tables.len()); let miss = router.route_from_cache(tables, &mut routes); assert_eq!(routes.len(), 1); - assert_eq!(routes[0].table, table1.to_string()); + assert_eq!(routes[0].table_name, table1.to_string()); assert_eq!(miss.len(), 1); assert_eq!(miss[0], table2.to_string()); } diff --git a/server/src/grpc/meta_event_service/mod.rs b/server/src/grpc/meta_event_service/mod.rs index 9b1432d329..fb15127622 100644 --- a/server/src/grpc/meta_event_service/mod.rs +++ b/server/src/grpc/meta_event_service/mod.rs @@ -333,22 +333,17 @@ async fn handle_create_table_on_shard( ), })?; - let partition_info = match table_info.partition_info { - Some(v) => Some( - PartitionInfo::try_from(v.clone()) - .box_err() - .with_context(|| ErrWithCause { + let (table_engine, partition_info) = match table_info.partition_info { + Some(v) => { + let partition_info = Some(PartitionInfo::try_from(v.clone()).box_err().with_context( + || ErrWithCause { code: StatusCode::BadRequest, msg: format!("fail to parse partition info, partition_info:{v:?}"), - })?, - ), - None => None, - }; - - let table_engine = if partition_info.is_some() { - ctx.partition_table_engine.clone() - } else { - ctx.table_engine.clone() + }, + )?); + (ctx.partition_table_engine.clone(), partition_info) + } + None => (ctx.table_engine.clone(), None), }; // Build create table request and options. diff --git a/server/src/proxy/grpc/sql_query.rs b/server/src/proxy/grpc/sql_query.rs index 4e8a37d47c..ca8aff418c 100644 --- a/server/src/proxy/grpc/sql_query.rs +++ b/server/src/proxy/grpc/sql_query.rs @@ -306,7 +306,7 @@ impl Proxy { // Open partition table if needed. let table_name = frontend::parse_table_name(&stmts); if let Some(table_name) = table_name { - self.open_partition_table_if_not_exist(catalog, schema, &table_name) + self.maybe_open_partition_table_if_not_exist(catalog, schema, &table_name) .await?; } @@ -392,7 +392,7 @@ impl Proxy { Ok(output) } - async fn open_partition_table_if_not_exist( + async fn maybe_open_partition_table_if_not_exist( &self, catalog_name: &str, schema_name: &str, From 72b0550594402146023f9e7d10f5e598305365de Mon Sep 17 00:00:00 2001 From: "chunshao.rcs" Date: Tue, 11 Apr 2023 14:33:23 +0800 Subject: [PATCH 9/9] refactor by CR --- meta_client/src/types.rs | 19 +++++++++---------- router/src/cluster_based.rs | 29 ++++++++++++++--------------- sql/src/frontend.rs | 1 + 3 files changed, 24 insertions(+), 25 deletions(-) diff --git a/meta_client/src/types.rs b/meta_client/src/types.rs index 193665d93d..c5dfc95ed4 100644 --- a/meta_client/src/types.rs +++ b/meta_client/src/types.rs @@ -94,16 +94,15 @@ impl TryFrom for TableInfo { type Error = Error; fn try_from(pb_table_info: meta_service_pb::TableInfo) -> Result { - let partition_info = match pb_table_info.partition_info { - Some(partition_info) => Some( - PartitionInfo::try_from(partition_info) - .box_err() - .context(Convert { - msg: "Failed to parse partition", - })?, - ), - None => None, - }; + let partition_info = pb_table_info + .partition_info + .map(|v| { + PartitionInfo::try_from(v).box_err().context(Convert { + msg: "Failed to parse partition", + }) + }) + .transpose()?; + Ok(TableInfo { id: pb_table_info.id, name: pb_table_info.name, diff --git a/router/src/cluster_based.rs b/router/src/cluster_based.rs index 8c29f88b20..5129fe7a53 100644 --- a/router/src/cluster_based.rs +++ b/router/src/cluster_based.rs @@ -17,7 +17,6 @@ use crate::{ #[derive(Clone)] struct RouteData { - table_name: String, table_info: TableInfo, endpoint: Option, } @@ -46,19 +45,19 @@ impl ClusterBasedRouter { /// route table from local cache, return cache routes and tables which are /// not in cache - fn route_from_cache(&self, tables: Vec, routes: &mut Vec) -> Vec { + fn route_from_cache(&self, tables: &[String], routes: &mut Vec) -> Vec { let mut miss = vec![]; if let Some(cache) = &self.cache { for table in tables { - if let Some(route) = cache.get(&table) { + if let Some(route) = cache.get(table) { routes.push(route.clone()); } else { miss.push(table.clone()); } } } else { - miss = tables; + miss = tables.to_vec(); } miss @@ -66,7 +65,7 @@ impl ClusterBasedRouter { async fn route_with_cache( &self, - tables: Vec, + tables: &Vec, database: String, ) -> Result> { // Firstly route table from local cache. @@ -93,12 +92,13 @@ impl ClusterBasedRouter { for (table_name, route_entry) in route_resp.entries { for node_shard in route_entry.node_shards { if node_shard.shard_info.is_leader() { - let route = make_route(route_entry.table_info.clone(), &node_shard.endpoint)?; + let route = make_route(route_entry.table_info, &node_shard.endpoint)?; if let Some(cache) = &self.cache { // There may be data race here, and it is acceptable currently. cache.insert(table_name.clone(), route.clone()).await; } routes.push(route); + break; } } } @@ -111,7 +111,6 @@ fn make_route(table_info: TableInfo, endpoint: &str) -> Result { let endpoint: Endpoint = endpoint.parse().context(ParseEndpoint { endpoint })?; Ok(RouteData { - table_name: table_info.name.clone(), table_info, endpoint: Some(endpoint), }) @@ -121,11 +120,11 @@ fn make_route(table_info: TableInfo, endpoint: &str) -> Result { impl Router for ClusterBasedRouter { async fn route(&self, req: RouteRequest) -> Result> { let req_ctx = req.context.unwrap(); - let route_data_vec = self.route_with_cache(req.tables, req_ctx.database).await?; + let route_data_vec = self.route_with_cache(&req.tables, req_ctx.database).await?; Ok(route_data_vec .into_iter() .map(|v| Route { - table: v.table_name, + table: v.table_info.name, endpoint: v.endpoint.map(Into::into), }) .collect()) @@ -137,7 +136,7 @@ impl Router for ClusterBasedRouter { table: &str, ) -> Result> { let mut route_data_vec = self - .route_with_cache(vec![table.to_string()], schema.to_string()) + .route_with_cache(&vec![table.to_string()], schema.to_string()) .await?; if route_data_vec.is_empty() { return Ok(None); @@ -288,7 +287,7 @@ mod tests { assert_eq!(result.unwrap().len(), 2); let mut routes = Vec::with_capacity(tables.len()); - let miss = router.route_from_cache(tables, &mut routes); + let miss = router.route_from_cache(&tables, &mut routes); assert_eq!(routes.len(), 2); assert_eq!(miss.len(), 0); sleep(Duration::from_secs(1)); @@ -296,18 +295,18 @@ mod tests { // try to get table1 let tables = vec![table1.to_string()]; let mut routes = Vec::with_capacity(tables.len()); - let miss = router.route_from_cache(tables, &mut routes); + let miss = router.route_from_cache(&tables, &mut routes); assert_eq!(routes.len(), 1); - assert_eq!(routes[0].table_name, table1.to_string()); + assert_eq!(routes[0].table_info.name, table1.to_string()); assert_eq!(miss.len(), 0); // sleep 1.5s, table2 will be evicted, and table1 in cache sleep(Duration::from_millis(1500)); let tables = vec![table1.to_string(), table2.to_string()]; let mut routes = Vec::with_capacity(tables.len()); - let miss = router.route_from_cache(tables, &mut routes); + let miss = router.route_from_cache(&tables, &mut routes); assert_eq!(routes.len(), 1); - assert_eq!(routes[0].table_name, table1.to_string()); + assert_eq!(routes[0].table_info.name, table1.to_string()); assert_eq!(miss.len(), 1); assert_eq!(miss[0], table2.to_string()); } diff --git a/sql/src/frontend.rs b/sql/src/frontend.rs index 2a179e6ff4..a8c840495d 100644 --- a/sql/src/frontend.rs +++ b/sql/src/frontend.rs @@ -214,6 +214,7 @@ pub fn parse_table_name(statements: &StatementVec) -> Option { None } } + // TODO: return unsupported error rather than none. _ => None, } } else {