From 7ca525698add193c58025b75ed3b27aa8974012a Mon Sep 17 00:00:00 2001 From: universalmind303 Date: Tue, 12 Mar 2024 10:45:08 -0500 Subject: [PATCH] refactor(table options): serde (#2770) --- Cargo.lock | 1 + crates/datasources/src/debug/mod.rs | 2 +- crates/datasources/src/debug/options.rs | 65 +- crates/datasources/src/lance/mod.rs | 2 +- crates/datasources/src/native/access.rs | 2 +- crates/datasources/src/object_store/mod.rs | 10 +- crates/protogen/Cargo.toml | 2 +- crates/protogen/proto/metastore/options.proto | 20 +- .../protogen/src/metastore/types/catalog.rs | 5 +- .../protogen/src/metastore/types/options.rs | 1357 ++--------------- .../sqlbuiltins/src/functions/table/debug.rs | 20 +- testdata/sqllogictests_bigquery/basic.slt | 4 +- .../sqllogictests_clickhouse/datatypes.slt | 12 +- .../local/basic.slt | 10 +- 14 files changed, 191 insertions(+), 1321 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 60d241fe4..e68e95dc8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6045,6 +6045,7 @@ dependencies = [ "prost", "prost-build", "prost-types", + "serde", "serde_json", "thiserror", "tonic", diff --git a/crates/datasources/src/debug/mod.rs b/crates/datasources/src/debug/mod.rs index 59727fc56..a4493587c 100644 --- a/crates/datasources/src/debug/mod.rs +++ b/crates/datasources/src/debug/mod.rs @@ -279,7 +279,7 @@ impl crate::Datasource for DebugDatasource { options: &TableOptions, tunnel_opts: Option<&TunnelOptions>, ) -> Result, DatasourceError> { - let options = TableOptionsDebug::try_from(options).unwrap(); + let options: TableOptionsDebug = options.extract()?; Ok(Arc::new(DebugTableProvider { typ: options.table_type, diff --git a/crates/datasources/src/debug/options.rs b/crates/datasources/src/debug/options.rs index e3ada594f..2a57938af 100644 --- a/crates/datasources/src/debug/options.rs +++ b/crates/datasources/src/debug/options.rs @@ -1,4 +1,3 @@ -use std::collections::BTreeMap; use std::str::FromStr; use std::sync::Arc; @@ -8,8 +7,7 @@ use datafusion::arrow::error::Result as ArrowResult; use datafusion::arrow::record_batch::RecordBatch; use parser::errors::ParserError; use parser::options::{OptionValue as SqlOptionValue, ParseOptionValue}; -use protogen::metastore::types::options::{OptionValue, TableOptions}; -use protogen::ProtoConvError; +use protogen::metastore::types::options::TableOptionsImpl; use serde::{Deserialize, Serialize}; use super::errors::DebugError; @@ -22,30 +20,6 @@ pub enum DebugTableType { NeverEnding, } -impl From for OptionValue { - fn from(t: DebugTableType) -> Self { - t.to_string().into() - } -} - -impl TryFrom<&OptionValue> for DebugTableType { - type Error = DebugError; - - fn try_from(value: &OptionValue) -> Result { - match value { - OptionValue::String(s) => s.parse(), - _ => Err(DebugError::UnknownDebugTableType("".to_string())), - } - } -} - -impl TryFrom for DebugTableType { - type Error = DebugError; - - fn try_from(value: OptionValue) -> Result { - (&value).try_into() - } -} impl ParseOptionValue for SqlOptionValue { fn parse_opt(self) -> Result { @@ -63,6 +37,7 @@ impl ParseOptionValue for SqlOptionValue { Ok(opt) } } + impl std::fmt::Display for DebugTableType { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!(f, "{}", self.as_str()) @@ -149,44 +124,14 @@ impl DebugTableType { } -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct TableOptionsDebug { pub table_type: DebugTableType, } - -impl From for TableOptions { - fn from(value: TableOptionsDebug) -> Self { - let mut options = BTreeMap::new(); - options.insert( - "table_type".to_string(), - OptionValue::String(value.table_type.to_string()), - ); - - TableOptions { - name: "debug".to_string(), - options, - } - } +impl TableOptionsImpl for TableOptionsDebug { + const NAME: &'static str = "debug"; } -impl TryFrom<&TableOptions> for TableOptionsDebug { - type Error = ProtoConvError; - fn try_from(value: &TableOptions) -> Result { - if matches!(value.name.as_ref(), "debug") { - let table_type: DebugTableType = value - .options - .get("table_type") - .cloned() - .ok_or_else(|| ProtoConvError::RequiredField("table_type".to_string()))? - .try_into() - .map_err(|e: DebugError| ProtoConvError::ParseError(e.to_string()))?; - - Ok(TableOptionsDebug { table_type }) - } else { - Err(ProtoConvError::UnknownVariant(value.name.to_string())) - } - } -} impl Default for TableOptionsDebug { fn default() -> Self { diff --git a/crates/datasources/src/lance/mod.rs b/crates/datasources/src/lance/mod.rs index ff6eee93a..cbd1ed3e2 100644 --- a/crates/datasources/src/lance/mod.rs +++ b/crates/datasources/src/lance/mod.rs @@ -121,7 +121,7 @@ impl Datasource for LanceDatasource { location, storage_options, .. - } = TableOptionsObjectStore::try_from(options)?; + } = options.extract()?; if let Some(file_type) = file_type { if file_type.as_str() != "lance" { return Err(ObjectStoreSourceError::NotSupportFileType(file_type).into()); diff --git a/crates/datasources/src/native/access.rs b/crates/datasources/src/native/access.rs index a74961c07..1d961066b 100644 --- a/crates/datasources/src/native/access.rs +++ b/crates/datasources/src/native/access.rs @@ -264,7 +264,7 @@ impl NativeTableStorage { fn opts_from_ent(table: &TableEntry) -> Result { let opts = match table.options.name.as_ref() { - "internal" => TableOptionsInternal::try_from(&table.options).unwrap(), + "internal" => (&table.options).extract_unchecked(), // variant is already checked _ => return Err(NativeError::NotNative(table.clone())), }; Ok(opts) diff --git a/crates/datasources/src/object_store/mod.rs b/crates/datasources/src/object_store/mod.rs index d11c6c240..fbf936c9c 100644 --- a/crates/datasources/src/object_store/mod.rs +++ b/crates/datasources/src/object_store/mod.rs @@ -398,7 +398,7 @@ pub fn init_session_registry<'a>( // TODO: Consider consolidating Gcs, S3 and Delta and Iceberg `TableOptions` and // `ObjStoreAccess` since they largely overlap "gcs" => { - let gcs_opts = TableOptionsGcs::try_from(opts).unwrap(); + let gcs_opts: TableOptionsGcs = opts.extract_unchecked(); Arc::new(GcsStoreAccess { bucket: gcs_opts.bucket, service_account_key: gcs_opts.service_account_key, @@ -407,7 +407,8 @@ pub fn init_session_registry<'a>( } "s3" => { // todo! - let s3_opts = TableOptionsS3::try_from(opts).unwrap(); + let s3_opts: TableOptionsS3 = opts.extract_unchecked(); + Arc::new(S3StoreAccess { bucket: s3_opts.bucket.clone(), region: Some(s3_opts.region.clone()), @@ -417,7 +418,7 @@ pub fn init_session_registry<'a>( }) } "azure" => { - let azure_opts = TableOptionsObjectStore::try_from(opts).unwrap(); + let azure_opts: TableOptionsObjectStore = opts.extract_unchecked(); let uri = DatasourceUrl::try_new(azure_opts.location)?; Arc::new(AzureStoreAccess::try_from_uri( &uri, @@ -425,8 +426,7 @@ pub fn init_session_registry<'a>( )?) } "lance" => { - let opts = TableOptionsObjectStore::try_from(opts).unwrap(); - + let opts: TableOptionsObjectStore = opts.extract_unchecked(); let url = DatasourceUrl::try_new(opts.location)?; storage_options_into_store_access(&url, &opts.storage_options) diff --git a/crates/protogen/Cargo.toml b/crates/protogen/Cargo.toml index afc6dac6e..4305f783f 100644 --- a/crates/protogen/Cargo.toml +++ b/crates/protogen/Cargo.toml @@ -16,7 +16,7 @@ prost-types = { workspace = true } uuid = { version = "1.7.0", features = ["v4", "fast-rng", "macro-diagnostics"] } tracing = { workspace = true } serde_json = { workspace = true } - +serde = { workspace = true } [build-dependencies] tonic-build = "0.11" # Only needed to handle custom btree mapping; can be removed when we bump the tonic version which will have this too diff --git a/crates/protogen/proto/metastore/options.proto b/crates/protogen/proto/metastore/options.proto index e2d7e82ab..0f8a2a15b 100644 --- a/crates/protogen/proto/metastore/options.proto +++ b/crates/protogen/proto/metastore/options.proto @@ -119,27 +119,9 @@ message StorageOptions { map inner = 1; } -message OptionValue { - oneof value { - string string_value = 1; - int64 int_value = 2; - bool bool_value = 4; - OptionValueArray array_value = 5; - OptionValueMap map_value = 6; - } -} - -message OptionValueArray { - repeated OptionValue values = 1; -} - -message OptionValueMap { - map values = 1; -} - message TableOptions { string name = 1; - map options = 2; + bytes options = 2; } // Table options diff --git a/crates/protogen/src/metastore/types/catalog.rs b/crates/protogen/src/metastore/types/catalog.rs index 51c5cab81..444cfcc68 100644 --- a/crates/protogen/src/metastore/types/catalog.rs +++ b/crates/protogen/src/metastore/types/catalog.rs @@ -447,9 +447,8 @@ pub struct TableEntry { impl TableEntry { /// Try to get the columns for this table if available. pub fn get_internal_columns(&self) -> Option> { - TableOptionsInternal::try_from(&self.options) - .ok() - .map(|o| o.columns) + let options: TableOptionsInternal = self.options.extract().ok()?; + Some(options.columns) } pub fn get_columns(&self) -> Option> { self.get_internal_columns().map(|val| { diff --git a/crates/protogen/src/metastore/types/options.rs b/crates/protogen/src/metastore/types/options.rs index e6690ed49..d6b740bfd 100644 --- a/crates/protogen/src/metastore/types/options.rs +++ b/crates/protogen/src/metastore/types/options.rs @@ -1,64 +1,24 @@ -use std::collections::{BTreeMap, HashMap}; +use std::collections::BTreeMap; use std::fmt::{self, Display}; use std::hash::Hash; use std::sync::Arc; use datafusion::arrow::datatypes::{DataType, Field, Fields, SchemaRef}; use datafusion::common::DFSchemaRef; +use serde::de::DeserializeOwned; +use serde::{Deserialize, Serialize}; use crate::gen::common::arrow; use crate::gen::metastore::options; use crate::{FromOptionalField, ProtoConvError}; -#[derive(Debug, Clone, PartialEq, Eq, Hash)] +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] pub struct InternalColumnDefinition { pub name: String, pub nullable: bool, pub arrow_type: DataType, } -impl From for OptionValue { - fn from(value: InternalColumnDefinition) -> Self { - OptionValue::Object( - vec![ - ("name".to_string(), OptionValue::String(value.name)), - ("nullable".to_string(), OptionValue::Bool(value.nullable)), - ( - "arrow_type".to_string(), - OptionValue::String(serde_json::to_string(&value.arrow_type).unwrap()), - ), - ] - .into_iter() - .collect(), - ) - } -} - -impl TryFrom for InternalColumnDefinition { - type Error = ProtoConvError; - fn try_from(value: OptionValue) -> Result { - if let OptionValue::Object(mut obj) = value { - let name = obj - .remove("name") - .ok_or_else(|| ProtoConvError::RequiredField("name".to_string()))?; - let nullable = obj - .remove("nullable") - .ok_or_else(|| ProtoConvError::RequiredField("nullable".to_string()))?; - let arrow_type = obj - .remove("arrow_type") - .ok_or_else(|| ProtoConvError::RequiredField("arrow_type".to_string()))?; - let arrow_type: DataType = arrow_type.try_into()?; - Ok(InternalColumnDefinition { - name: name.try_into()?, - nullable: nullable.try_into()?, - arrow_type: arrow_type.try_into()?, - }) - } else { - Err(ProtoConvError::ParseError("Expected object".to_string())) - } - } -} - impl InternalColumnDefinition { /// Create a vec of column definitions. /// @@ -129,29 +89,18 @@ impl TryFrom for options::InternalColumnDefinition { } } -#[cfg(test)] -mod tests { - use datafusion::arrow::datatypes::DataType; - - use super::*; - - #[test] - fn test_roundtrip() { - let def = super::InternalColumnDefinition { - name: "foo".to_string(), - nullable: true, - arrow_type: DataType::Int64, - }; +// #[cfg(test)] +// mod tests { +// use datafusion::arrow::datatypes::DataType; - let proto: OptionValue = def.clone().into(); - let back: super::InternalColumnDefinition = proto.try_into().unwrap(); +// use super::*; - assert_eq!(def, back); - } -} +// #[test] +// fn test_roundtrip() {} +// } // Database options -#[derive(Debug, Clone, PartialEq, Eq, Hash)] +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] pub enum DatabaseOptions { Internal(DatabaseOptionsInternal), Debug(DatabaseOptionsDebug), @@ -283,7 +232,7 @@ impl From for options::DatabaseOptions { } } -#[derive(Debug, Clone, PartialEq, Eq, Hash)] +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] pub struct DatabaseOptionsInternal {} impl TryFrom for DatabaseOptionsInternal { @@ -299,7 +248,7 @@ impl From for options::DatabaseOptionsInternal { } } -#[derive(Debug, Clone, PartialEq, Eq, Hash)] +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] pub struct DatabaseOptionsDebug {} impl TryFrom for DatabaseOptionsDebug { @@ -315,7 +264,7 @@ impl From for options::DatabaseOptionsDebug { } } -#[derive(Debug, Clone, PartialEq, Eq, Hash)] +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] pub struct DatabaseOptionsPostgres { pub connection_string: String, } @@ -337,7 +286,7 @@ impl From for options::DatabaseOptionsPostgres { } } -#[derive(Debug, Clone, PartialEq, Eq, Hash)] +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] pub struct DatabaseOptionsBigQuery { pub service_account_key: String, pub project_id: String, @@ -362,7 +311,7 @@ impl From for options::DatabaseOptionsBigQuery { } } -#[derive(Debug, Clone, PartialEq, Eq, Hash)] +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] pub struct DatabaseOptionsMysql { pub connection_string: String, } @@ -384,7 +333,7 @@ impl From for options::DatabaseOptionsMysql { } } -#[derive(Debug, Clone, PartialEq, Eq, Hash)] +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] pub struct DatabaseOptionsMongoDb { pub connection_string: String, } @@ -406,7 +355,7 @@ impl From for options::DatabaseOptionsMongoDb { } } -#[derive(Debug, Clone, PartialEq, Eq, Hash)] +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] pub struct DatabaseOptionsSqlServer { pub connection_string: String, } @@ -428,7 +377,7 @@ impl From for options::DatabaseOptionsSqlServer { } } -#[derive(Debug, Clone, PartialEq, Eq, Hash)] +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] pub struct DatabaseOptionsClickhouse { pub connection_string: String, } @@ -449,7 +398,7 @@ impl From for options::DatabaseOptionsClickhouse { } } } -#[derive(Debug, Clone, PartialEq, Eq, Hash)] +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] pub struct DatabaseOptionsCassandra { pub host: String, pub username: Option, @@ -477,7 +426,7 @@ impl From for options::DatabaseOptionsCassandra { } } -#[derive(Debug, Clone, PartialEq, Eq, Hash)] +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] pub struct DatabaseOptionsSqlite { pub location: String, } @@ -499,7 +448,7 @@ impl From for options::DatabaseOptionsSqlite { } } -#[derive(Debug, Clone, PartialEq, Eq, Hash)] +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] pub struct DatabaseOptionsSnowflake { pub account_name: String, pub login_name: String, @@ -536,7 +485,7 @@ impl From for options::DatabaseOptionsSnowflake { } } -#[derive(Debug, Clone, PartialEq, Eq, Hash)] +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] pub struct DatabaseOptionsDeltaLake { pub catalog: DeltaLakeCatalog, pub storage_options: StorageOptions, @@ -563,7 +512,8 @@ impl From for options::DatabaseOptionsDeltaLake { } } -#[derive(Debug, Clone, PartialEq, Eq, Hash)] +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] + pub enum DeltaLakeCatalog { Unity(DeltaLakeUnityCatalog), } @@ -589,7 +539,7 @@ impl From for options::database_options_delta_lake::Catalog { } } -#[derive(Debug, Clone, PartialEq, Eq, Hash)] +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] pub struct DeltaLakeUnityCatalog { pub catalog_id: String, pub databricks_access_token: String, @@ -629,8 +579,9 @@ impl From for options::DeltaLakeUnityCatalog { /// - [Azure options](https://docs.rs/object_store/latest/object_store/azure/enum.AzureConfigKey.html#variants) /// - [S3 options](https://docs.rs/object_store/latest/object_store/aws/enum.AmazonS3ConfigKey.html#variants) /// - [Google options](https://docs.rs/object_store/latest/object_store/gcp/enum.GoogleConfigKey.html#variants) -#[derive(Debug, Default, Clone, PartialEq, Eq, Hash)] +#[derive(Debug, Default, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] pub struct StorageOptions { + #[serde(flatten)] pub inner: BTreeMap, } @@ -648,32 +599,6 @@ impl StorageOptions { } } -impl From for OptionValue { - fn from(value: StorageOptions) -> Self { - OptionValue::Object( - value - .inner - .into_iter() - .map(|(k, v)| (k, v.into())) - .collect(), - ) - } -} -impl TryFrom for StorageOptions { - type Error = ProtoConvError; - fn try_from(value: OptionValue) -> Result { - if let OptionValue::Object(inner) = value { - Ok(StorageOptions { - inner: inner - .into_iter() - .map(|(k, v)| Ok::<_, ProtoConvError>((k, v.try_into()?))) - .collect::>()?, - }) - } else { - Err(ProtoConvError::ParseError("Expected object".to_string())) - } - } -} impl TryFrom for StorageOptions { type Error = ProtoConvError; @@ -688,12 +613,17 @@ impl From for options::StorageOptions { } } -#[derive(Debug, Clone, PartialEq, Eq, Hash)] +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] pub struct TableOptions { pub name: String, - pub options: BTreeMap, + pub options: Vec, +} + +pub trait TableOptionsImpl: Serialize + DeserializeOwned { + const NAME: &'static str; } + impl Display for TableOptions { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{}", self.name) @@ -704,133 +634,53 @@ impl TableOptions { pub fn as_str(&self) -> &str { &self.name } -} - -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub enum OptionValue { - String(String), - Int(i64), - Bool(bool), - Array(Vec), - Object(BTreeMap), -} -impl TryFrom for String { - type Error = ProtoConvError; + pub fn new(options: &S) -> Self { + let options = serde_json::to_vec(options).expect("options must serialize"); - fn try_from(value: OptionValue) -> Result { - match value { - OptionValue::String(v) => Ok(v), - _ => Err(ProtoConvError::ParseError("Expected string".to_string())), - } - } -} - -impl TryFrom for DataType { - type Error = ProtoConvError; - - fn try_from(value: OptionValue) -> Result { - match value { - OptionValue::String(v) => serde_json::from_str(&v).map_err(|e| { - ProtoConvError::ParseError(format!("Failed to parse arrow type: {}", e)) - }), - _ => Err(ProtoConvError::ParseError("Expected DataType".to_string())), - } - } -} - -impl TryFrom for i64 { - type Error = ProtoConvError; - - fn try_from(value: OptionValue) -> Result { - match value { - OptionValue::Int(v) => Ok(v), - _ => Err(ProtoConvError::ParseError("Expected int".to_string())), + TableOptions { + name: S::NAME.to_string(), + options, } } -} - -impl TryFrom for bool { - type Error = ProtoConvError; - fn try_from(value: OptionValue) -> Result { - match value { - OptionValue::Bool(v) => Ok(v), - _ => Err(ProtoConvError::ParseError("Expected bool".to_string())), + pub fn extract(&self) -> Result { + if self.name != T::NAME { + return Err(ProtoConvError::ParseError(format!( + "Expected table options of type {}, got {}", + T::NAME, + self.name + ))); } - } -} - -impl TryFrom for Vec { - type Error = ProtoConvError; - fn try_from(value: OptionValue) -> Result { - match value { - OptionValue::Array(v) => Ok(v), - _ => Err(ProtoConvError::ParseError("Expected array".to_string())), - } + Ok(self.extract_unchecked()) } -} -impl TryFrom for BTreeMap { - type Error = ProtoConvError; - - fn try_from(value: OptionValue) -> Result { - match value { - OptionValue::Object(v) => Ok(v), - _ => Err(ProtoConvError::ParseError("Expected object".to_string())), - } + /// Extract the table options to a specific type. + /// This does not check the variant of the table options, so this will panic if the variant is not the expected one. + /// Use `extract` if you want to handle the error case. + /// This should only be used if the variant is checked elsewhere. + pub fn extract_unchecked(&self) -> T { + serde_json::from_slice(&self.options) + .expect("Options should infallibly deserialize. This indicates a programming error.") } } -impl From for OptionValue { - fn from(value: String) -> Self { - OptionValue::String(value) - } -} -impl From for OptionValue { - fn from(value: i64) -> Self { - OptionValue::Int(value) - } -} -impl From for OptionValue { - fn from(value: bool) -> Self { - OptionValue::Bool(value) - } -} - -impl From> for OptionValue +impl From for TableOptions where - T: Into, + T: TableOptionsImpl, { - fn from(value: Vec) -> Self { - OptionValue::Array(value.into_iter().map(|v| v.into()).collect()) + fn from(value: T) -> Self { + TableOptions::new(&value) } } -impl TryFrom for Vec -where - T: TryFrom, -{ - type Error = ProtoConvError; - - fn try_from(value: OptionValue) -> Result { - match value { - OptionValue::Array(v) => v.into_iter().map(|v| v.try_into()).collect(), - _ => Err(ProtoConvError::ParseError("Expected array".to_string())), - } - } -} impl TryFrom for TableOptions { type Error = ProtoConvError; fn try_from(value: options::TableOptions) -> Result { let name = value.name; - let values = value - .options - .into_iter() - .map(|(k, v)| Ok::<_, ProtoConvError>((k, v.try_into()?))) - .collect::>()?; + let values = value.options; Ok(TableOptions { options: values, @@ -843,75 +693,13 @@ impl From for options::TableOptions { fn from(value: TableOptions) -> Self { options::TableOptions { name: value.name, - options: value - .options - .into_iter() - .map(|(k, v)| (k, v.into())) - .collect::>(), + options: value.options, } } } -impl TryFrom for OptionValue { - type Error = ProtoConvError; - - fn try_from(value: options::OptionValue) -> Result { - let value: options::option_value::Value = value.value.required("value")?; - - Ok(match value { - options::option_value::Value::StringValue(s) => OptionValue::String(s), - options::option_value::Value::IntValue(i) => OptionValue::Int(i), - options::option_value::Value::BoolValue(b) => OptionValue::Bool(b), - options::option_value::Value::ArrayValue(arr) => OptionValue::Array( - arr.values - .into_iter() - .map(|v| v.try_into()) - .collect::>()?, - ), - options::option_value::Value::MapValue(map) => OptionValue::Object( - map.values - .into_iter() - .map(|(k, v)| Ok::<_, ProtoConvError>((k, v.try_into()?))) - .collect::>()?, - ), - }) - } -} - -impl From for options::OptionValue { - fn from(value: OptionValue) -> Self { - match value { - OptionValue::String(v) => options::OptionValue { - value: Some(options::option_value::Value::StringValue(v)), - }, - OptionValue::Int(v) => options::OptionValue { - value: Some(options::option_value::Value::IntValue(v)), - }, - OptionValue::Bool(v) => options::OptionValue { - value: Some(options::option_value::Value::BoolValue(v)), - }, - OptionValue::Array(v) => options::OptionValue { - value: Some(options::option_value::Value::ArrayValue( - options::OptionValueArray { - values: v.into_iter().map(|v| v.into()).collect(), - }, - )), - }, - OptionValue::Object(v) => options::OptionValue { - value: Some(options::option_value::Value::MapValue( - options::OptionValueMap { - values: v - .into_iter() - .map(|(k, v)| (k, v.into())) - .collect::>(), - }, - )), - }, - } - } -} -#[derive(Debug, Clone, PartialEq, Eq, Hash)] +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] pub enum TableOptionsOld { Internal(TableOptionsInternal), Postgres(TableOptionsPostgres), @@ -983,28 +771,29 @@ impl TableOptionsOld { } } } + impl From for TableOptions { fn from(value: TableOptionsOld) -> Self { match value { - TableOptionsOld::Internal(opts) => opts.into(), - TableOptionsOld::Postgres(opts) => opts.into(), - TableOptionsOld::BigQuery(opts) => opts.into(), - TableOptionsOld::Mysql(opts) => opts.into(), - TableOptionsOld::Local(opts) => opts.into(), - TableOptionsOld::Gcs(opts) => opts.into(), - TableOptionsOld::S3(opts) => opts.into(), - TableOptionsOld::MongoDb(opts) => opts.into(), - TableOptionsOld::Snowflake(opts) => opts.into(), - TableOptionsOld::Delta(opts) => opts.into(), - TableOptionsOld::Iceberg(opts) => opts.into(), - TableOptionsOld::Azure(opts) => opts.into(), - TableOptionsOld::SqlServer(opts) => opts.into(), - TableOptionsOld::Lance(opts) => opts.into(), - TableOptionsOld::Bson(opts) => opts.into(), - TableOptionsOld::Clickhouse(opts) => opts.into(), - TableOptionsOld::Cassandra(opts) => opts.into(), - TableOptionsOld::Excel(opts) => opts.into(), - TableOptionsOld::Sqlite(opts) => opts.into(), + TableOptionsOld::Internal(opts) => TableOptions::new(&opts), + TableOptionsOld::Postgres(opts) => TableOptions::new(&opts), + TableOptionsOld::BigQuery(opts) => TableOptions::new(&opts), + TableOptionsOld::Mysql(opts) => TableOptions::new(&opts), + TableOptionsOld::Local(opts) => TableOptions::new(&opts), + TableOptionsOld::Gcs(opts) => TableOptions::new(&opts), + TableOptionsOld::S3(opts) => TableOptions::new(&opts), + TableOptionsOld::MongoDb(opts) => TableOptions::new(&opts), + TableOptionsOld::Snowflake(opts) => TableOptions::new(&opts), + TableOptionsOld::Delta(opts) => TableOptions::new(&opts), + TableOptionsOld::Iceberg(opts) => TableOptions::new(&opts), + TableOptionsOld::Azure(opts) => TableOptions::new(&opts), + TableOptionsOld::SqlServer(opts) => TableOptions::new(&opts), + TableOptionsOld::Lance(opts) => TableOptions::new(&opts), + TableOptionsOld::Bson(opts) => TableOptions::new(&opts), + TableOptionsOld::Clickhouse(opts) => TableOptions::new(&opts), + TableOptionsOld::Cassandra(opts) => TableOptions::new(&opts), + TableOptionsOld::Excel(opts) => TableOptions::new(&opts), + TableOptionsOld::Sqlite(opts) => TableOptions::new(&opts), } } } @@ -1015,79 +804,79 @@ impl TryFrom<&TableOptions> for TableOptionsOld { fn try_from(value: &TableOptions) -> Result { match value.name.as_ref() { Self::INTERNAL => { - let internal: TableOptionsInternal = value.try_into()?; + let internal: TableOptionsInternal = value.extract()?; Ok(TableOptionsOld::Internal(internal)) } Self::POSTGRES => { - let postgres: TableOptionsPostgres = value.try_into()?; + let postgres: TableOptionsPostgres = value.extract()?; Ok(TableOptionsOld::Postgres(postgres)) } Self::BIGQUERY => { - let bigquery: TableOptionsBigQuery = value.try_into()?; + let bigquery: TableOptionsBigQuery = value.extract()?; Ok(TableOptionsOld::BigQuery(bigquery)) } Self::MYSQL => { - let mysql: TableOptionsMysql = value.try_into()?; + let mysql: TableOptionsMysql = value.extract()?; Ok(TableOptionsOld::Mysql(mysql)) } Self::LOCAL => { - let local: TableOptionsLocal = value.try_into()?; + let local: TableOptionsLocal = value.extract()?; Ok(TableOptionsOld::Local(local)) } Self::GCS => { - let gcs: TableOptionsGcs = value.try_into()?; + let gcs: TableOptionsGcs = value.extract()?; Ok(TableOptionsOld::Gcs(gcs)) } Self::S3_STORAGE => { - let s3: TableOptionsS3 = value.try_into()?; + let s3: TableOptionsS3 = value.extract()?; Ok(TableOptionsOld::S3(s3)) } Self::MONGODB => { - let mongo: TableOptionsMongoDb = value.try_into()?; + let mongo: TableOptionsMongoDb = value.extract()?; Ok(TableOptionsOld::MongoDb(mongo)) } Self::SNOWFLAKE => { - let snowflake: TableOptionsSnowflake = value.try_into()?; + let snowflake: TableOptionsSnowflake = value.extract()?; Ok(TableOptionsOld::Snowflake(snowflake)) } Self::DELTA => { - let delta: TableOptionsObjectStore = value.try_into()?; + let delta: TableOptionsObjectStore = value.extract()?; Ok(TableOptionsOld::Delta(delta)) } Self::ICEBERG => { - let iceberg: TableOptionsObjectStore = value.try_into()?; + let iceberg: TableOptionsObjectStore = value.extract()?; Ok(TableOptionsOld::Iceberg(iceberg)) } Self::AZURE => { - let azure: TableOptionsObjectStore = value.try_into()?; + let azure: TableOptionsObjectStore = value.extract()?; Ok(TableOptionsOld::Azure(azure)) } Self::SQL_SERVER => { - let sql_server: TableOptionsSqlServer = value.try_into()?; + let sql_server: TableOptionsSqlServer = value.extract()?; Ok(TableOptionsOld::SqlServer(sql_server)) } Self::LANCE => { - let lance: TableOptionsObjectStore = value.try_into()?; + let lance: TableOptionsObjectStore = value.extract()?; Ok(TableOptionsOld::Lance(lance)) } Self::BSON => { - let bson: TableOptionsObjectStore = value.try_into()?; + let bson: TableOptionsObjectStore = value.extract()?; Ok(TableOptionsOld::Bson(bson)) } Self::CLICKHOUSE => { - let clickhouse: TableOptionsClickhouse = value.try_into()?; + let clickhouse: TableOptionsClickhouse = value.extract()?; Ok(TableOptionsOld::Clickhouse(clickhouse)) } Self::CASSANDRA => { - let cassandra: TableOptionsCassandra = value.try_into()?; + let cassandra: TableOptionsCassandra = value.extract()?; Ok(TableOptionsOld::Cassandra(cassandra)) } Self::EXCEL => { - let excel: TableOptionsExcel = value.try_into()?; + let excel: TableOptionsExcel = value.extract()?; Ok(TableOptionsOld::Excel(excel)) } Self::SQLITE => { - let sqlite: TableOptionsSqlite = value.try_into()?; + let sqlite: TableOptionsSqlite = value.extract()?; Ok(TableOptionsOld::Sqlite(sqlite)) } _ => Err(ProtoConvError::UnknownVariant(value.name.to_string())), @@ -1147,53 +936,15 @@ impl TryFrom for options::TableOptionsOld { } } -#[derive(Debug, Clone, PartialEq, Eq, Hash)] +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] pub struct TableOptionsInternal { pub columns: Vec, } -impl From for TableOptions { - fn from(value: TableOptionsInternal) -> Self { - let mut options = BTreeMap::new(); - let columns = value.columns.into_iter().map(|col| col.into()); - options.insert("columns".to_string(), OptionValue::Array(columns.collect())); - - TableOptions { - name: "internal".to_string(), - options, - } - } -} - -impl TryFrom for TableOptionsInternal { - type Error = ProtoConvError; - fn try_from(value: TableOptions) -> Result { - (&value).try_into() - } +impl TableOptionsImpl for TableOptionsInternal { + const NAME: &'static str = "internal"; } -impl TryFrom<&TableOptions> for TableOptionsInternal { - type Error = ProtoConvError; - fn try_from(value: &TableOptions) -> Result { - if matches!(value.name.as_ref(), "internal") { - let columns: Vec = value - .options - .get("columns") - .cloned() - .ok_or_else(|| ProtoConvError::RequiredField("columns".to_string()))? - .try_into()?; - - let columns = columns - .into_iter() - .map(|col| col.try_into()) - .collect::>()?; - - Ok(TableOptionsInternal { columns }) - } else { - Err(ProtoConvError::UnknownVariant(value.name.to_string())) - } - } -} impl From for TableOptionsInternal { fn from(value: DFSchemaRef) -> Self { @@ -1253,62 +1004,17 @@ impl TryFrom for options::TableOptionsInternal { } } -#[derive(Debug, Clone, PartialEq, Eq, Hash)] +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] pub struct TableOptionsPostgres { pub connection_string: String, pub schema: String, pub table: String, } -impl From for TableOptions { - fn from(value: TableOptionsPostgres) -> Self { - let name = "postgres".to_string(); - let mut options = BTreeMap::new(); - options.insert( - "connection_string".to_string(), - value.connection_string.into(), - ); - options.insert("schema".to_string(), value.schema.into()); - options.insert("table".to_string(), value.table.into()); - - TableOptions { name, options } - } +impl TableOptionsImpl for TableOptionsPostgres { + const NAME: &'static str = "postgres"; } -impl TryFrom<&TableOptions> for TableOptionsPostgres { - type Error = ProtoConvError; - - fn try_from(value: &TableOptions) -> Result { - if matches!(value.name.as_ref(), "postgres") { - let connection_string: String = value - .options - .get("connection_string") - .cloned() - .ok_or_else(|| ProtoConvError::RequiredField("connection_string".to_string()))? - .try_into()?; - let schema: String = value - .options - .get("schema") - .cloned() - .ok_or_else(|| ProtoConvError::RequiredField("schema".to_string()))? - .try_into()?; - let table: String = value - .options - .get("table") - .cloned() - .ok_or_else(|| ProtoConvError::RequiredField("table".to_string()))? - .try_into()?; - - Ok(TableOptionsPostgres { - connection_string, - schema, - table, - }) - } else { - Err(ProtoConvError::UnknownVariant(value.name.to_string())) - } - } -} impl TryFrom for TableOptionsPostgres { type Error = ProtoConvError; @@ -1331,7 +1037,7 @@ impl From for options::TableOptionsPostgres { } } -#[derive(Debug, Clone, PartialEq, Eq, Hash)] +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] pub struct TableOptionsBigQuery { pub service_account_key: String, pub project_id: String, @@ -1339,22 +1045,11 @@ pub struct TableOptionsBigQuery { pub table_id: String, } -impl From for TableOptions { - fn from(value: TableOptionsBigQuery) -> Self { - let name = "bigquery".to_string(); - let mut options = BTreeMap::new(); - options.insert( - "service_account_key".to_string(), - value.service_account_key.into(), - ); - options.insert("project_id".to_string(), value.project_id.into()); - options.insert("dataset_id".to_string(), value.dataset_id.into()); - options.insert("table_id".to_string(), value.table_id.into()); - - TableOptions { name, options } - } +impl TableOptionsImpl for TableOptionsBigQuery { + const NAME: &'static str = "bigquery"; } + impl TryFrom for TableOptionsBigQuery { type Error = ProtoConvError; fn try_from(value: options::TableOptionsBigQuery) -> Result { @@ -1378,104 +1073,17 @@ impl From for options::TableOptionsBigQuery { } } -impl TryFrom<&TableOptions> for TableOptionsBigQuery { - type Error = ProtoConvError; - - fn try_from(value: &TableOptions) -> Result { - if matches!(value.name.as_ref(), "bigquery") { - let service_account_key: String = value - .options - .get("service_account_key") - .cloned() - .ok_or_else(|| ProtoConvError::RequiredField("service_account_key".to_string()))? - .try_into()?; - let project_id: String = value - .options - .get("project_id") - .cloned() - .ok_or_else(|| ProtoConvError::RequiredField("project_id".to_string()))? - .try_into()?; - let dataset_id: String = value - .options - .get("dataset_id") - .cloned() - .ok_or_else(|| ProtoConvError::RequiredField("dataset_id".to_string()))? - .try_into()?; - let table_id: String = value - .options - .get("table_id") - .cloned() - .ok_or_else(|| ProtoConvError::RequiredField("table_id".to_string()))? - .try_into()?; - - Ok(TableOptionsBigQuery { - service_account_key, - project_id, - dataset_id, - table_id, - }) - } else { - Err(ProtoConvError::UnknownVariant(value.name.to_string())) - } - } -} - -#[derive(Debug, Clone, PartialEq, Eq, Hash)] +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] pub struct TableOptionsMysql { pub connection_string: String, pub schema: String, pub table: String, } -impl From for TableOptions { - fn from(value: TableOptionsMysql) -> Self { - let name = "mysql".to_string(); - let mut options = BTreeMap::new(); - options.insert( - "connection_string".to_string(), - value.connection_string.into(), - ); - options.insert("schema".to_string(), value.schema.into()); - options.insert("table".to_string(), value.table.into()); - - TableOptions { name, options } - } +impl TableOptionsImpl for TableOptionsMysql { + const NAME: &'static str = "mysql"; } -impl TryFrom<&TableOptions> for TableOptionsMysql { - type Error = ProtoConvError; - - fn try_from(value: &TableOptions) -> Result { - if matches!(value.name.as_ref(), "mysql") { - let connection_string: String = value - .options - .get("connection_string") - .cloned() - .ok_or_else(|| ProtoConvError::RequiredField("connection_string".to_string()))? - .try_into()?; - let schema: String = value - .options - .get("schema") - .cloned() - .ok_or_else(|| ProtoConvError::RequiredField("schema".to_string()))? - .try_into()?; - let table: String = value - .options - .get("table") - .cloned() - .ok_or_else(|| ProtoConvError::RequiredField("table".to_string()))? - .try_into()?; - - Ok(TableOptionsMysql { - connection_string, - schema, - table, - }) - } else { - Err(ProtoConvError::UnknownVariant(value.name.to_string())) - } - } -} impl TryFrom for TableOptionsMysql { type Error = ProtoConvError; @@ -1498,61 +1106,17 @@ impl From for options::TableOptionsMysql { } } -#[derive(Debug, Clone, PartialEq, Eq, Hash)] +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] pub struct TableOptionsLocal { pub location: String, pub file_type: String, pub compression: Option, } -impl From for TableOptions { - fn from(value: TableOptionsLocal) -> Self { - let name = "local".to_string(); - let mut options = BTreeMap::new(); - options.insert("location".to_string(), value.location.into()); - options.insert("file_type".to_string(), value.file_type.into()); - if let Some(compression) = value.compression { - options.insert("compression".to_string(), compression.into()); - } - - TableOptions { name, options } - } +impl TableOptionsImpl for TableOptionsLocal { + const NAME: &'static str = "local"; } -impl TryFrom<&TableOptions> for TableOptionsLocal { - type Error = ProtoConvError; - - fn try_from(value: &TableOptions) -> Result { - if matches!(value.name.as_ref(), "local") { - let location: String = value - .options - .get("location") - .cloned() - .ok_or_else(|| ProtoConvError::RequiredField("location".to_string()))? - .try_into()?; - let file_type: String = value - .options - .get("file_type") - .cloned() - .ok_or_else(|| ProtoConvError::RequiredField("file_type".to_string()))? - .try_into()?; - let compression = value - .options - .get("compression") - .cloned() - .map(|v| v.try_into()) - .transpose()?; - - Ok(TableOptionsLocal { - location, - file_type, - compression, - }) - } else { - Err(ProtoConvError::UnknownVariant(value.name.to_string())) - } - } -} impl TryFrom for TableOptionsLocal { type Error = ProtoConvError; @@ -1575,7 +1139,7 @@ impl From for options::TableOptionsLocal { } } -#[derive(Debug, Clone, PartialEq, Eq, Hash)] +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] pub struct TableOptionsGcs { pub service_account_key: Option, pub bucket: String, @@ -1584,72 +1148,8 @@ pub struct TableOptionsGcs { pub compression: Option, } -impl From for TableOptions { - fn from(value: TableOptionsGcs) -> Self { - let name = "gcs".to_string(); - let mut options = BTreeMap::new(); - if let Some(service_account_key) = value.service_account_key { - options.insert( - "service_account_key".to_string(), - service_account_key.into(), - ); - } - options.insert("bucket".to_string(), value.bucket.into()); - options.insert("location".to_string(), value.location.into()); - options.insert("file_type".to_string(), value.file_type.into()); - if let Some(compression) = value.compression { - options.insert("compression".to_string(), compression.into()); - } - - TableOptions { name, options } - } -} -impl TryFrom<&TableOptions> for TableOptionsGcs { - type Error = ProtoConvError; - fn try_from(value: &TableOptions) -> Result { - let service_account_key = value - .options - .get("service_account_key") - .cloned() - .map(|v| v.try_into()) - .transpose()?; - - let bucket = value - .options - .get("bucket") - .cloned() - .ok_or_else(|| ProtoConvError::RequiredField("bucket".to_string()))? - .try_into()?; - - let location = value - .options - .get("location") - .cloned() - .ok_or_else(|| ProtoConvError::RequiredField("location".to_string()))? - .try_into()?; - - let file_type = value - .options - .get("file_type") - .cloned() - .ok_or_else(|| ProtoConvError::RequiredField("file_type".to_string()))? - .try_into()?; - - let compression = value - .options - .get("compression") - .cloned() - .map(|v| v.try_into()) - .transpose()?; - - Ok(TableOptionsGcs { - service_account_key, - bucket, - location, - file_type, - compression, - }) - } +impl TableOptionsImpl for TableOptionsGcs { + const NAME: &'static str = "gcs"; } impl TryFrom for TableOptionsGcs { @@ -1677,7 +1177,7 @@ impl From for options::TableOptionsGcs { } } -#[derive(Debug, Clone, PartialEq, Eq, Hash)] +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] pub struct TableOptionsS3 { pub access_key_id: Option, pub secret_access_key: Option, @@ -1688,93 +1188,8 @@ pub struct TableOptionsS3 { pub compression: Option, } -impl From for TableOptions { - fn from(value: TableOptionsS3) -> Self { - let name = "s3".to_string(); - let mut options = BTreeMap::new(); - if let Some(access_key_id) = value.access_key_id { - options.insert("access_key_id".to_string(), access_key_id.into()); - } - if let Some(secret_access_key) = value.secret_access_key { - options.insert("secret_access_key".to_string(), secret_access_key.into()); - } - options.insert("region".to_string(), value.region.into()); - options.insert("bucket".to_string(), value.bucket.into()); - options.insert("location".to_string(), value.location.into()); - options.insert("file_type".to_string(), value.file_type.into()); - if let Some(compression) = value.compression { - options.insert("compression".to_string(), compression.into()); - } - - TableOptions { name, options } - } -} -impl TryFrom<&TableOptions> for TableOptionsS3 { - type Error = ProtoConvError; - - fn try_from(value: &TableOptions) -> Result { - if value.name != "s3" { - return Err(ProtoConvError::UnknownVariant(value.name.to_string())); - } - let access_key_id = value - .options - .get("access_key_id") - .cloned() - .map(|v| v.try_into()) - .transpose()?; - - let secret_access_key = value - .options - .get("secret_access_key") - .cloned() - .map(|v| v.try_into()) - .transpose()?; - - let region = value - .options - .get("region") - .cloned() - .ok_or_else(|| ProtoConvError::RequiredField("region".to_string()))? - .try_into()?; - - let bucket = value - .options - .get("bucket") - .cloned() - .ok_or_else(|| ProtoConvError::RequiredField("bucket".to_string()))? - .try_into()?; - - let location = value - .options - .get("location") - .cloned() - .ok_or_else(|| ProtoConvError::RequiredField("location".to_string()))? - .try_into()?; - - let file_type = value - .options - .get("file_type") - .cloned() - .ok_or_else(|| ProtoConvError::RequiredField("file_type".to_string()))? - .try_into()?; - - let compression = value - .options - .get("compression") - .cloned() - .map(|v| v.try_into()) - .transpose()?; - - Ok(TableOptionsS3 { - access_key_id, - secret_access_key, - region, - bucket, - location, - file_type, - compression, - }) - } +impl TableOptionsImpl for TableOptionsS3 { + const NAME: &'static str = "s3"; } impl TryFrom for TableOptionsS3 { @@ -1805,63 +1220,17 @@ impl From for options::TableOptionsS3 { } } } -#[derive(Debug, Clone, PartialEq, Eq, Hash)] +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] pub struct TableOptionsMongoDb { pub connection_string: String, pub database: String, pub collection: String, } -impl From for TableOptions { - fn from(value: TableOptionsMongoDb) -> Self { - let name = "mongodb".to_string(); - let mut options = BTreeMap::new(); - options.insert( - "connection_string".to_string(), - value.connection_string.into(), - ); - options.insert("database".to_string(), value.database.into()); - options.insert("collection".to_string(), value.collection.into()); - - TableOptions { name, options } - } +impl TableOptionsImpl for TableOptionsMongoDb { + const NAME: &'static str = "mongo"; } -impl TryFrom<&TableOptions> for TableOptionsMongoDb { - type Error = ProtoConvError; - - fn try_from(value: &TableOptions) -> Result { - if matches!(value.name.as_ref(), "mongodb") { - let connection_string: String = value - .options - .get("connection_string") - .cloned() - .ok_or_else(|| ProtoConvError::RequiredField("connection_string".to_string()))? - .try_into()?; - let database: String = value - .options - .get("database") - .cloned() - .ok_or_else(|| ProtoConvError::RequiredField("database".to_string()))? - .try_into()?; - let collection: String = value - .options - .get("collection") - .cloned() - .ok_or_else(|| ProtoConvError::RequiredField("collection".to_string()))? - .try_into()?; - - - Ok(TableOptionsMongoDb { - connection_string, - database, - collection, - }) - } else { - Err(ProtoConvError::UnknownVariant(value.name.to_string())) - } - } -} impl TryFrom for TableOptionsMongoDb { type Error = ProtoConvError; @@ -1884,7 +1253,7 @@ impl From for options::TableOptionsMongo { } } -#[derive(Debug, Clone, PartialEq, Eq, Hash)] +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] pub struct TableOptionsExcel { pub location: String, pub storage_options: StorageOptions, @@ -1894,82 +1263,8 @@ pub struct TableOptionsExcel { pub has_header: bool, } -impl From for TableOptions { - fn from(value: TableOptionsExcel) -> Self { - let name = "excel".to_string(); - let mut options = BTreeMap::new(); - options.insert("location".to_string(), value.location.into()); - - options.insert("storage_options".to_string(), value.storage_options.into()); - if let Some(file_type) = value.file_type { - options.insert("file_type".to_string(), file_type.into()); - } - if let Some(compression) = value.compression { - options.insert("compression".to_string(), compression.into()); - } - if let Some(sheet_name) = value.sheet_name { - options.insert("sheet_name".to_string(), sheet_name.into()); - } - options.insert("has_header".to_string(), value.has_header.into()); - - TableOptions { name, options } - } -} - -impl TryFrom<&TableOptions> for TableOptionsExcel { - type Error = ProtoConvError; - - fn try_from(value: &TableOptions) -> Result { - if matches!(value.name.as_ref(), "excel") { - let location: String = value - .options - .get("location") - .cloned() - .ok_or_else(|| ProtoConvError::RequiredField("location".to_string()))? - .try_into()?; - let storage_options: StorageOptions = value - .options - .get("storage_options") - .cloned() - .ok_or_else(|| ProtoConvError::RequiredField("storage_options".to_string()))? - .try_into()?; - let file_type = value - .options - .get("file_type") - .cloned() - .map(|v| v.try_into()) - .transpose()?; - let compression = value - .options - .get("compression") - .cloned() - .map(|v| v.try_into()) - .transpose()?; - let sheet_name = value - .options - .get("sheet_name") - .cloned() - .map(|v| v.try_into()) - .transpose()?; - let has_header: bool = value - .options - .get("has_header") - .cloned() - .ok_or_else(|| ProtoConvError::RequiredField("has_header".to_string()))? - .try_into()?; - - Ok(TableOptionsExcel { - location, - storage_options, - file_type, - compression, - sheet_name, - has_header, - }) - } else { - Err(ProtoConvError::UnknownVariant(value.name.to_string())) - } - } +impl TableOptionsImpl for TableOptionsExcel { + const NAME: &'static str = "excel"; } @@ -2000,62 +1295,17 @@ impl From for options::TableOptionsExcel { } } -#[derive(Debug, Clone, PartialEq, Eq, Hash)] +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] pub struct TableOptionsSqlServer { pub connection_string: String, pub schema: String, pub table: String, } -impl From for TableOptions { - fn from(value: TableOptionsSqlServer) -> Self { - let name = "sql_server".to_string(); - let mut options = BTreeMap::new(); - options.insert( - "connection_string".to_string(), - value.connection_string.into(), - ); - options.insert("schema".to_string(), value.schema.into()); - options.insert("table".to_string(), value.table.into()); - - TableOptions { name, options } - } +impl TableOptionsImpl for TableOptionsSqlServer { + const NAME: &'static str = "sql_server"; } -impl TryFrom<&TableOptions> for TableOptionsSqlServer { - type Error = ProtoConvError; - - fn try_from(value: &TableOptions) -> Result { - if matches!(value.name.as_ref(), "sql_server") { - let connection_string: String = value - .options - .get("connection_string") - .cloned() - .ok_or_else(|| ProtoConvError::RequiredField("connection_string".to_string()))? - .try_into()?; - let schema: String = value - .options - .get("schema") - .cloned() - .ok_or_else(|| ProtoConvError::RequiredField("schema".to_string()))? - .try_into()?; - let table: String = value - .options - .get("table") - .cloned() - .ok_or_else(|| ProtoConvError::RequiredField("table".to_string()))? - .try_into()?; - - Ok(TableOptionsSqlServer { - connection_string, - schema, - table, - }) - } else { - Err(ProtoConvError::UnknownVariant(value.name.to_string())) - } - } -} impl TryFrom for TableOptionsSqlServer { type Error = ProtoConvError; @@ -2078,63 +1328,15 @@ impl From for options::TableOptionsSqlServer { } } -#[derive(Debug, Clone, PartialEq, Eq, Hash)] +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] pub struct TableOptionsClickhouse { pub connection_string: String, pub table: String, pub database: Option, } -impl From for TableOptions { - fn from(value: TableOptionsClickhouse) -> Self { - let name = "clickhouse".to_string(); - let mut options = BTreeMap::new(); - options.insert( - "connection_string".to_string(), - value.connection_string.into(), - ); - options.insert("table".to_string(), value.table.into()); - if let Some(database) = value.database { - options.insert("database".to_string(), database.into()); - } - - TableOptions { name, options } - } -} - -impl TryFrom<&TableOptions> for TableOptionsClickhouse { - type Error = ProtoConvError; - - fn try_from(value: &TableOptions) -> Result { - if matches!(value.name.as_ref(), "clickhouse") { - let connection_string: String = value - .options - .get("connection_string") - .cloned() - .ok_or_else(|| ProtoConvError::RequiredField("connection_string".to_string()))? - .try_into()?; - let table: String = value - .options - .get("table") - .cloned() - .ok_or_else(|| ProtoConvError::RequiredField("table".to_string()))? - .try_into()?; - let database = value - .options - .get("database") - .cloned() - .map(|v| v.try_into()) - .transpose()?; - - Ok(TableOptionsClickhouse { - connection_string, - table, - database, - }) - } else { - Err(ProtoConvError::UnknownVariant(value.name.to_string())) - } - } +impl TableOptionsImpl for TableOptionsClickhouse { + const NAME: &'static str = "clickhouse"; } impl TryFrom for TableOptionsClickhouse { @@ -2158,7 +1360,7 @@ impl From for options::TableOptionsClickhouse { } } -#[derive(Debug, Clone, PartialEq, Eq, Hash)] +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] pub struct TableOptionsCassandra { pub host: String, pub keyspace: String, @@ -2167,71 +1369,8 @@ pub struct TableOptionsCassandra { pub password: Option, } -impl From for TableOptions { - fn from(value: TableOptionsCassandra) -> Self { - let name = "cassandra".to_string(); - let mut options = BTreeMap::new(); - options.insert("host".to_string(), value.host.into()); - options.insert("keyspace".to_string(), value.keyspace.into()); - options.insert("table".to_string(), value.table.into()); - if let Some(username) = value.username { - options.insert("username".to_string(), username.into()); - } - if let Some(password) = value.password { - options.insert("password".to_string(), password.into()); - } - - TableOptions { name, options } - } -} - -impl TryFrom<&TableOptions> for TableOptionsCassandra { - type Error = ProtoConvError; - - fn try_from(value: &TableOptions) -> Result { - if matches!(value.name.as_ref(), "cassandra") { - let host: String = value - .options - .get("host") - .cloned() - .ok_or_else(|| ProtoConvError::RequiredField("host".to_string()))? - .try_into()?; - let keyspace: String = value - .options - .get("keyspace") - .cloned() - .ok_or_else(|| ProtoConvError::RequiredField("keyspace".to_string()))? - .try_into()?; - let table: String = value - .options - .get("table") - .cloned() - .ok_or_else(|| ProtoConvError::RequiredField("table".to_string()))? - .try_into()?; - let username = value - .options - .get("username") - .cloned() - .map(|v| v.try_into()) - .transpose()?; - let password = value - .options - .get("password") - .cloned() - .map(|v| v.try_into()) - .transpose()?; - - Ok(TableOptionsCassandra { - host, - keyspace, - table, - username, - password, - }) - } else { - Err(ProtoConvError::UnknownVariant(value.name.to_string())) - } - } +impl TableOptionsImpl for TableOptionsCassandra { + const NAME: &'static str = "cassandra"; } impl TryFrom for TableOptionsCassandra { @@ -2259,47 +1398,16 @@ impl From for options::TableOptionsCassandra { } } -#[derive(Debug, Clone, PartialEq, Eq, Hash)] +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] pub struct TableOptionsSqlite { pub location: String, pub table: String, } -impl From for TableOptions { - fn from(value: TableOptionsSqlite) -> Self { - let name = "sqlite".to_string(); - let mut options = BTreeMap::new(); - options.insert("location".to_string(), value.location.into()); - options.insert("table".to_string(), value.table.into()); - - TableOptions { name, options } - } +impl TableOptionsImpl for TableOptionsSqlite { + const NAME: &'static str = "sqlite"; } -impl TryFrom<&TableOptions> for TableOptionsSqlite { - type Error = ProtoConvError; - - fn try_from(value: &TableOptions) -> Result { - if matches!(value.name.as_ref(), "sqlite") { - let location: String = value - .options - .get("location") - .cloned() - .ok_or_else(|| ProtoConvError::RequiredField("location".to_string()))? - .try_into()?; - let table: String = value - .options - .get("table") - .cloned() - .ok_or_else(|| ProtoConvError::RequiredField("table".to_string()))? - .try_into()?; - - Ok(TableOptionsSqlite { location, table }) - } else { - Err(ProtoConvError::UnknownVariant(value.name.to_string())) - } - } -} impl TryFrom for TableOptionsSqlite { type Error = ProtoConvError; @@ -2320,7 +1428,7 @@ impl From for options::TableOptionsSqlite { } } -#[derive(Debug, Clone, PartialEq, Eq, Hash)] +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] pub struct TableOptionsSnowflake { pub account_name: String, pub login_name: String, @@ -2332,91 +1440,8 @@ pub struct TableOptionsSnowflake { pub table_name: String, } -impl From for TableOptions { - fn from(value: TableOptionsSnowflake) -> Self { - let name = "snowflake".to_string(); - let mut options = BTreeMap::new(); - options.insert("account_name".to_string(), value.account_name.into()); - options.insert("login_name".to_string(), value.login_name.into()); - options.insert("password".to_string(), value.password.into()); - options.insert("database_name".to_string(), value.database_name.into()); - options.insert("warehouse".to_string(), value.warehouse.into()); - options.insert("role_name".to_string(), value.role_name.into()); - options.insert("schema_name".to_string(), value.schema_name.into()); - options.insert("table_name".to_string(), value.table_name.into()); - - TableOptions { name, options } - } -} - -impl TryFrom<&TableOptions> for TableOptionsSnowflake { - type Error = ProtoConvError; - - fn try_from(value: &TableOptions) -> Result { - if matches!(value.name.as_ref(), "snowflake") { - let account_name: String = value - .options - .get("account_name") - .cloned() - .ok_or_else(|| ProtoConvError::RequiredField("account_name".to_string()))? - .try_into()?; - let login_name: String = value - .options - .get("login_name") - .cloned() - .ok_or_else(|| ProtoConvError::RequiredField("login_name".to_string()))? - .try_into()?; - let password: String = value - .options - .get("password") - .cloned() - .ok_or_else(|| ProtoConvError::RequiredField("password".to_string()))? - .try_into()?; - let database_name: String = value - .options - .get("database_name") - .cloned() - .ok_or_else(|| ProtoConvError::RequiredField("database_name".to_string()))? - .try_into()?; - let warehouse: String = value - .options - .get("warehouse") - .cloned() - .ok_or_else(|| ProtoConvError::RequiredField("warehouse".to_string()))? - .try_into()?; - let role_name: String = value - .options - .get("role_name") - .cloned() - .ok_or_else(|| ProtoConvError::RequiredField("role_name".to_string()))? - .try_into()?; - let schema_name: String = value - .options - .get("schema_name") - .cloned() - .ok_or_else(|| ProtoConvError::RequiredField("schema_name".to_string()))? - .try_into()?; - let table_name: String = value - .options - .get("table_name") - .cloned() - .ok_or_else(|| ProtoConvError::RequiredField("table_name".to_string()))? - .try_into()?; - - Ok(TableOptionsSnowflake { - account_name, - login_name, - password, - database_name, - warehouse, - role_name, - schema_name, - table_name, - }) - } else { - Err(ProtoConvError::UnknownVariant(value.name.to_string())) - } - } +impl TableOptionsImpl for TableOptionsSnowflake { + const NAME: &'static str = "snowflake"; } impl TryFrom for TableOptionsSnowflake { @@ -2450,7 +1475,7 @@ impl From for options::TableOptionsSnowflake { } } -#[derive(Debug, Clone, PartialEq, Eq, Hash)] +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] pub struct TableOptionsObjectStore { pub location: String, pub storage_options: StorageOptions, @@ -2459,76 +1484,11 @@ pub struct TableOptionsObjectStore { pub schema_sample_size: Option, } -impl From for TableOptions { - fn from(value: TableOptionsObjectStore) -> Self { - let mut options = BTreeMap::new(); - options.insert("location".to_string(), value.location.into()); - options.insert("storage_options".to_string(), value.storage_options.into()); - let name = if let Some(file_type) = value.file_type { - options.insert("file_type".to_string(), file_type.clone().into()); - file_type - } else { - "object_store".to_string() - }; - if let Some(compression) = value.compression { - options.insert("compression".to_string(), compression.into()); - } - if let Some(schema_sample_size) = value.schema_sample_size { - options.insert("schema_sample_size".to_string(), schema_sample_size.into()); - } - TableOptions { name, options } - } +impl TableOptionsImpl for TableOptionsObjectStore { + const NAME: &'static str = "object_store"; } -impl TryFrom<&TableOptions> for TableOptionsObjectStore { - type Error = ProtoConvError; - fn try_from(value: &TableOptions) -> Result { - let file_type = value - .options - .get("file_type") - .cloned() - .map(|v| v.try_into()) - .transpose()?; - - if value.name != "object_store" && file_type.is_none() { - return Err(ProtoConvError::UnknownVariant(value.name.to_string())); - } - - let location = value - .options - .get("location") - .cloned() - .ok_or_else(|| ProtoConvError::RequiredField("location".to_string()))? - .try_into()?; - let storage_options = value - .options - .get("storage_options") - .cloned() - .ok_or_else(|| ProtoConvError::RequiredField("storage_options".to_string()))? - .try_into()?; - - let compression = value - .options - .get("compression") - .cloned() - .map(|v| v.try_into()) - .transpose()?; - let schema_sample_size = value - .options - .get("schema_sample_size") - .cloned() - .map(|v| v.try_into()) - .transpose()?; - Ok(TableOptionsObjectStore { - location, - storage_options, - file_type, - compression, - schema_sample_size, - }) - } -} impl TryFrom for TableOptionsObjectStore { type Error = ProtoConvError; fn try_from(value: options::TableOptionsObjectStore) -> Result { @@ -2554,7 +1514,7 @@ impl From for options::TableOptionsObjectStore { } } -#[derive(Debug, Clone, PartialEq, Eq, Hash)] +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] pub enum TunnelOptions { Internal(TunnelOptionsInternal), Debug(TunnelOptionsDebug), @@ -2617,7 +1577,7 @@ impl From for options::TunnelOptions { } } -#[derive(Debug, Clone, PartialEq, Eq, Hash)] +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] pub struct TunnelOptionsInternal {} impl TryFrom for TunnelOptionsInternal { @@ -2633,7 +1593,7 @@ impl From for options::TunnelOptionsInternal { } } -#[derive(Debug, Clone, PartialEq, Eq, Hash)] +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] pub struct TunnelOptionsDebug {} impl TryFrom for TunnelOptionsDebug { @@ -2649,7 +1609,7 @@ impl From for options::TunnelOptionsDebug { } } -#[derive(Debug, Clone, PartialEq, Eq, Hash)] +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] pub struct TunnelOptionsSsh { pub connection_string: String, pub ssh_key: Vec, @@ -2673,7 +1633,8 @@ impl From for options::TunnelOptionsSsh { } } } -#[derive(Debug, Clone, PartialEq, Eq, Hash)] +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] + pub enum CredentialsOptions { Debug(CredentialsOptionsDebug), Gcp(CredentialsOptionsGcp), @@ -2748,7 +1709,7 @@ impl From for options::CredentialsOptions { } } -#[derive(Debug, Clone, PartialEq, Eq, Hash)] +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] pub struct CredentialsOptionsDebug { pub table_type: String, } @@ -2770,7 +1731,7 @@ impl From for options::CredentialsOptionsDebug { } } -#[derive(Debug, Clone, PartialEq, Eq, Hash)] +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] pub struct CredentialsOptionsGcp { pub service_account_key: String, } @@ -2792,7 +1753,7 @@ impl From for options::CredentialsOptionsGcp { } } -#[derive(Debug, Clone, PartialEq, Eq, Hash)] +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] pub struct CredentialsOptionsAws { pub access_key_id: String, pub secret_access_key: String, @@ -2817,7 +1778,7 @@ impl From for options::CredentialsOptionsAws { } } -#[derive(Debug, Clone, PartialEq, Eq, Hash)] +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] pub struct CredentialsOptionsAzure { pub account_name: String, pub access_key: String, @@ -2842,7 +1803,7 @@ impl From for options::CredentialsOptionsAzure { } } -#[derive(Debug, Clone, PartialEq, Eq, Hash)] +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] pub struct CredentialsOptionsOpenAI { pub api_key: String, pub api_base: Option, diff --git a/crates/sqlbuiltins/src/functions/table/debug.rs b/crates/sqlbuiltins/src/functions/table/debug.rs index 62bf71a88..5c2366ae0 100644 --- a/crates/sqlbuiltins/src/functions/table/debug.rs +++ b/crates/sqlbuiltins/src/functions/table/debug.rs @@ -1,4 +1,4 @@ -use std::collections::{BTreeMap, HashMap}; +use std::collections::HashMap; use std::sync::Arc; use async_trait::async_trait; @@ -10,7 +10,6 @@ use datafusion_ext::functions::{FuncParamValue, TableFuncContextProvider}; use datasources::debug::{DebugTableProvider, DebugTableType}; pub use datasources::Datasource; use protogen::metastore::types::catalog::{FunctionType, RuntimePreference}; -use protogen::metastore::types::options::TableOptions; use super::TableFunc; use crate::functions::ConstBuiltinFunction; @@ -62,20 +61,3 @@ impl TableFunc for Dummy { Ok(Arc::new(DebugTableProvider { typ, tunnel: false })) } } - -#[derive(Debug, Clone)] -pub struct TableOptionsDebug2 { - pub table_type: String, -} - -impl From for TableOptions { - fn from(value: TableOptionsDebug2) -> Self { - let mut options = BTreeMap::new(); - options.insert("table_type".to_string(), value.table_type.into()); - - TableOptions { - name: "debug".to_string(), - options, - } - } -} diff --git a/testdata/sqllogictests_bigquery/basic.slt b/testdata/sqllogictests_bigquery/basic.slt index f07f49013..7e5237924 100644 --- a/testdata/sqllogictests_bigquery/basic.slt +++ b/testdata/sqllogictests_bigquery/basic.slt @@ -1,8 +1,8 @@ # Basic tests for bigquery external tables statement ok -CREATE EXTERNAL TABLE basic - FROM bigquery +CREATE EXTERNAL TABLE basic +FROM bigquery OPTIONS ( service_account_key = '${GCP_SERVICE_ACCOUNT_KEY}', project_id = '${GCP_PROJECT_ID}', diff --git a/testdata/sqllogictests_clickhouse/datatypes.slt b/testdata/sqllogictests_clickhouse/datatypes.slt index 90ed612d8..19cc9b20a 100644 --- a/testdata/sqllogictests_clickhouse/datatypes.slt +++ b/testdata/sqllogictests_clickhouse/datatypes.slt @@ -2,12 +2,12 @@ # Create an external table that connects to the datatypes table. statement ok -CREATE EXTERNAL TABLE datatypes - FROM clickhouse - OPTIONS ( - connection_string = '${CLICKHOUSE_CONN_STRING}', - table = 'datatypes' - ); +CREATE EXTERNAL TABLE datatypes +FROM clickhouse +OPTIONS ( + connection_string = '${CLICKHOUSE_CONN_STRING}', + table = 'datatypes' +); # Check if we can fetch contents of the datatype table. # diff --git a/testdata/sqllogictests_object_store/local/basic.slt b/testdata/sqllogictests_object_store/local/basic.slt index b51a0a0f7..d99dba26a 100644 --- a/testdata/sqllogictests_object_store/local/basic.slt +++ b/testdata/sqllogictests_object_store/local/basic.slt @@ -7,11 +7,11 @@ statement ok set search_path = local_object_store; statement ok -CREATE EXTERNAL TABLE basic - FROM local - OPTIONS ( - location = '${PWD}/testdata/sqllogictests_datasources_common/data/bikeshare_stations.csv' - ); +CREATE EXTERNAL TABLE basic +FROM local +OPTIONS ( + location = '${PWD}/testdata/sqllogictests_datasources_common/data/bikeshare_stations.csv' +); # TODO re-enable when CSV related querying is improved, (timestamp issue) halt