Add DataTypeExt and FieldExt to assist converting to/from Fields and …

datafusion/common/src/datatype.rs

@@ -0,0 +1,96 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! [DataTypeExt] extension trait for converting DataTypes to Fields
+
+use crate::arrow::datatypes::{DataType, Field};
+use std::sync::Arc;
+
+/// DataFusion extension methods for Arrow [`DataType`]
+pub trait DataTypeExt {
+    /// convert the type to field with nullable type and "" name
+    fn into_nullable_field(self) -> Field;
+
+    /// convert the type to field ref with nullable type and "" name
+    fn into_nullable_field_ref(self) -> Arc<Field>;
+
+    //
+}
+
+impl DataTypeExt for DataType {
+    fn into_nullable_field(self) -> Field {
+        Field::new("", self, true)
+    }
+
+    fn into_nullable_field_ref(self) -> Arc<Field> {
+        Arc::new(Field::new("", self, true))
+    }
+}
+
+/// DataFusion extension methods for Arrow [`Field`]
+pub trait FieldExt {
+    /// Returns a new Field representing a List of this Field's DataType.
+    fn into_list(self) -> Self;
+
+    /// Return a new Field representing this Field as the item type of a FixedSizeList
+    fn into_fixed_size_list(self, list_size: i32) -> Self;
+
+    /// Note that lists are allowed to have an arbitrarily named field;
+    /// however, a name other than 'item' will cause it to fail an
+    /// == check against a more idiomatically created list in
+    /// arrow-rs which causes issues.
+    fn into_list_item(self) -> Self;
+}
+
+impl FieldExt for Field {
+    fn into_list(self) -> Self {
+        DataType::List(Arc::new(self.into_list_item())).into_nullable_field()
+    }
+
+    fn into_fixed_size_list(self, list_size: i32) -> Self {
+        DataType::FixedSizeList(self.into_list_item().into(), list_size)
+            .into_nullable_field()
+    }
+    fn into_list_item(self) -> Self {
+        if self.name() != "item" {
+            self.with_name("item")
+        } else {
+            self
+        }
+    }
+}
+
+impl FieldExt for Arc<Field> {
+    fn into_list(self) -> Self {
+        DataType::List(self.into_list_item())
+            .into_nullable_field()
+            .into()
+    }
+
+    fn into_fixed_size_list(self, list_size: i32) -> Self {
+        DataType::FixedSizeList(self.into_list_item(), list_size)
+            .into_nullable_field()
+            .into()
+    }
+    fn into_list_item(self) -> Self {
+        if self.name() != "item" {
+            Arc::unwrap_or_clone(self).with_name("item").into()
+        } else {
+            self
+        }
+    }
+}

datafusion/common/src/lib.rs

@@ -39,6 +39,7 @@ pub mod alias;
 pub mod cast;
 pub mod config;
 pub mod cse;
+pub mod datatype;
 pub mod diagnostic;
 pub mod display;
 pub mod encryption;

datafusion/sql/src/planner.rs

@@ -21,8 +21,10 @@ use std::str::FromStr;
 use std::sync::Arc;
 use std::vec;
 
+use crate::utils::make_decimal_type;
 use arrow::datatypes::*;
 use datafusion_common::config::SqlParserOptions;
+use datafusion_common::datatype::{DataTypeExt, FieldExt};
 use datafusion_common::error::add_possible_columns_to_diag;
 use datafusion_common::TableReference;
 use datafusion_common::{
@@ -31,15 +33,13 @@ use datafusion_common::{
 };
 use datafusion_common::{not_impl_err, plan_err, DFSchema, DataFusionError, Result};
 use datafusion_expr::logical_plan::{LogicalPlan, LogicalPlanBuilder};
+pub use datafusion_expr::planner::ContextProvider;
 use datafusion_expr::utils::find_column_exprs;
 use datafusion_expr::{col, Expr};
 use sqlparser::ast::{ArrayElemTypeDef, ExactNumberInfo, TimezoneInfo};
 use sqlparser::ast::{ColumnDef as SQLColumnDef, ColumnOption};
 use sqlparser::ast::{DataType as SQLDataType, Ident, ObjectName, TableAlias};
 
-use crate::utils::make_decimal_type;
-pub use datafusion_expr::planner::ContextProvider;
-
 /// SQL parser options
 #[derive(Debug, Clone, Copy)]
 pub struct ParserOptions {
@@ -596,60 +596,30 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> {
         // First check if any of the registered type_planner can handle this type
         if let Some(type_planner) = self.context_provider.get_type_planner() {
             if let Some(data_type) = type_planner.plan_type(sql_type)? {
-                return Ok(Field::new("", data_type, true).into());
+                return Ok(data_type.into_nullable_field_ref());
             }
         }
 
         // If no type_planner can handle this type, use the default conversion
         match sql_type {
             SQLDataType::Array(ArrayElemTypeDef::AngleBracket(inner_sql_type)) => {
                 // Arrays may be multi-dimensional.
-                let inner_data_type = self.convert_data_type_to_field(inner_sql_type)?;
-                Ok(Field::new(
-                    "",
-                    DataType::List(
-                        inner_data_type
-                            .as_ref()
-                            .clone()
-                            .with_name(Field::LIST_FIELD_DEFAULT_NAME)
-                            .into(),
-                    ),
-                    true,
-                )
-                .into())
+                Ok(self.convert_data_type_to_field(inner_sql_type)?.into_list())
             }
             SQLDataType::Array(ArrayElemTypeDef::SquareBracket(
                 inner_sql_type,
                 maybe_array_size,
             )) => {
-                let inner_data_type = self.convert_data_type_to_field(inner_sql_type)?;
+                let inner_field = self.convert_data_type_to_field(inner_sql_type)?;
                 if let Some(array_size) = maybe_array_size {
-                    Ok(Field::new(
-                        "",
-                        DataType::FixedSizeList(
-                            inner_data_type
-                                .as_ref()
-                                .clone()
-                                .with_name(Field::LIST_FIELD_DEFAULT_NAME)
-                                .into(),
-                            *array_size as i32,
-                        ),
-                        true,
-                    )
-                    .into())
+                    let array_size: i32 = (*array_size).try_into().map_err(|_| {
+                        plan_datafusion_err!(
+                            "Array size must be a positive 32 bit integer, got {array_size}"
+                        )
+                    })?;
+                    Ok(inner_field.into_fixed_size_list(array_size))
                 } else {
-                    Ok(Field::new(
-                        "",
-                        DataType::List(
-                            inner_data_type
-                                .as_ref()
-                                .clone()
-                                .with_name(Field::LIST_FIELD_DEFAULT_NAME)
-                                .into(),
-                        ),
-                        true,
-                    )
-                    .into())
+                    Ok(inner_field.into_list())
                 }
             }
             SQLDataType::Array(ArrayElemTypeDef::None) => {