From 6f019d362294f787dcaf86977121013f85c77f76 Mon Sep 17 00:00:00 2001 From: Liang-Chi Hsieh Date: Mon, 19 Feb 2024 21:21:20 -0800 Subject: [PATCH] feat: Nested map support for columnar shuffle --- .../apache/comet/vector/CometMapVector.java | 77 + .../org/apache/comet/vector/CometVector.java | 3 + core/src/execution/proto/expr.proto | 12 +- core/src/execution/serde.rs | 30 +- core/src/execution/shuffle/map.rs | 2853 +++++++++++++++++ core/src/execution/shuffle/mod.rs | 1 + core/src/execution/shuffle/row.rs | 2497 ++++++++++++++- .../apache/comet/serde/QueryPlanSerde.scala | 29 +- .../apache/comet/exec/CometShuffleSuite.scala | 349 ++ 9 files changed, 5805 insertions(+), 46 deletions(-) create mode 100644 common/src/main/java/org/apache/comet/vector/CometMapVector.java create mode 100644 core/src/execution/shuffle/map.rs diff --git a/common/src/main/java/org/apache/comet/vector/CometMapVector.java b/common/src/main/java/org/apache/comet/vector/CometMapVector.java new file mode 100644 index 000000000..a634cf470 --- /dev/null +++ b/common/src/main/java/org/apache/comet/vector/CometMapVector.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.comet.vector; + +import org.apache.arrow.vector.*; +import org.apache.arrow.vector.complex.MapVector; +import org.apache.arrow.vector.complex.StructVector; +import org.apache.arrow.vector.util.TransferPair; +import org.apache.spark.sql.vectorized.ColumnVector; +import org.apache.spark.sql.vectorized.ColumnarMap; + +/** A Comet column vector for map type. */ +public class CometMapVector extends CometDecodedVector { + final MapVector mapVector; + final ValueVector dataVector; + final CometStructVector dataColumnVector; + + final ColumnVector keys; + final ColumnVector values; + + public CometMapVector(ValueVector vector, boolean useDecimal128) { + super(vector, vector.getField(), useDecimal128); + + this.mapVector = ((MapVector) vector); + this.dataVector = mapVector.getDataVector(); + + if (dataVector instanceof StructVector) { + this.dataColumnVector = new CometStructVector(dataVector, useDecimal128); + + if (dataColumnVector.children.size() != 2) { + throw new RuntimeException( + "MapVector's dataVector should have 2 children, but got: " + + dataColumnVector.children.size()); + } + + this.keys = dataColumnVector.getChild(0); + this.values = dataColumnVector.getChild(1); + } else { + throw new RuntimeException( + "MapVector's dataVector should be StructVector, but got: " + + dataVector.getClass().getSimpleName()); + } + } + + @Override + public ColumnarMap getMap(int i) { + int start = mapVector.getOffsetBuffer().getInt(i * MapVector.OFFSET_WIDTH); + int end = mapVector.getOffsetBuffer().getInt((i + 1) * MapVector.OFFSET_WIDTH); + + return new ColumnarMap(keys, values, start, end - start); + } + + @Override + public CometVector slice(int offset, int length) { + TransferPair tp = this.valueVector.getTransferPair(this.valueVector.getAllocator()); + tp.splitAndTransfer(offset, length); + + return new CometMapVector(tp.getTo(), useDecimal128); + } +} diff --git a/common/src/main/java/org/apache/comet/vector/CometVector.java b/common/src/main/java/org/apache/comet/vector/CometVector.java index 744f24395..205c97815 100644 --- a/common/src/main/java/org/apache/comet/vector/CometVector.java +++ b/common/src/main/java/org/apache/comet/vector/CometVector.java @@ -25,6 +25,7 @@ import org.apache.arrow.vector.FixedWidthVector; import org.apache.arrow.vector.ValueVector; import org.apache.arrow.vector.complex.ListVector; +import org.apache.arrow.vector.complex.MapVector; import org.apache.arrow.vector.complex.StructVector; import org.apache.arrow.vector.dictionary.Dictionary; import org.apache.arrow.vector.dictionary.DictionaryProvider; @@ -206,6 +207,8 @@ protected static CometVector getVector( ValueVector vector, boolean useDecimal128, DictionaryProvider dictionaryProvider) { if (vector instanceof StructVector) { return new CometStructVector(vector, useDecimal128); + } else if (vector instanceof MapVector) { + return new CometMapVector(vector, useDecimal128); } else if (vector instanceof ListVector) { return new CometListVector(vector, useDecimal128); } else { diff --git a/core/src/execution/proto/expr.proto b/core/src/execution/proto/expr.proto index 7311c1245..53035b87f 100644 --- a/core/src/execution/proto/expr.proto +++ b/core/src/execution/proto/expr.proto @@ -421,7 +421,8 @@ message DataType { DATE = 12; NULL = 13; LIST = 14; - STRUCT = 15; + MAP = 15; + STRUCT = 16; } DataTypeId type_id = 1; @@ -429,7 +430,8 @@ message DataType { oneof datatype_struct { DecimalInfo decimal = 2; ListInfo list = 3; - StructInfo struct = 4; + MapInfo map = 4; + StructInfo struct = 5; } } @@ -443,6 +445,12 @@ message DataType { bool contains_null = 2; } + message MapInfo { + DataType key_type = 1; + DataType value_type = 2; + bool value_contains_null = 3; + } + message StructInfo { repeated string field_names = 1; repeated DataType field_datatypes = 2; diff --git a/core/src/execution/serde.rs b/core/src/execution/serde.rs index 2f38b4527..b88e3d651 100644 --- a/core/src/execution/serde.rs +++ b/core/src/execution/serde.rs @@ -29,7 +29,7 @@ use crate::{ }, }; use arrow::datatypes::{DataType as ArrowDataType, TimeUnit}; -use arrow_schema::Field; +use arrow_schema::{Field, Fields}; use prost::Message; use std::{io::Cursor, sync::Arc}; @@ -118,6 +118,34 @@ pub fn to_arrow_datatype(dt_value: &DataType) -> ArrowDataType { } _ => unreachable!(), }, + DataTypeId::Map => match dt_value + .type_info + .as_ref() + .unwrap() + .datatype_struct + .as_ref() + .unwrap() + { + DatatypeStruct::Map(info) => { + let key_field = Field::new( + "key", + to_arrow_datatype(info.key_type.as_ref().unwrap()), + false, + ); + let value_field = Field::new( + "value", + to_arrow_datatype(info.value_type.as_ref().unwrap()), + info.value_contains_null, + ); + let struct_field = Field::new( + "entries", + ArrowDataType::Struct(Fields::from(vec![key_field, value_field])), + false, + ); + ArrowDataType::Map(Arc::new(struct_field), false) + } + _ => unreachable!(), + }, DataTypeId::Struct => match dt_value .type_info .as_ref() diff --git a/core/src/execution/shuffle/map.rs b/core/src/execution/shuffle/map.rs new file mode 100644 index 000000000..1c0f8abb1 --- /dev/null +++ b/core/src/execution/shuffle/map.rs @@ -0,0 +1,2853 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::{ + errors::CometError, + execution::shuffle::{ + list::SparkUnsafeArray, + row::{append_field, downcast_builder_ref, SparkUnsafeObject}, + }, +}; +use arrow_array::builder::{ + ArrayBuilder, BinaryBuilder, BooleanBuilder, Date32Builder, Decimal128Builder, Float32Builder, + Float64Builder, Int16Builder, Int32Builder, Int64Builder, Int8Builder, MapBuilder, + MapFieldNames, StringBuilder, StructBuilder, TimestampMicrosecondBuilder, +}; +use arrow_schema::{DataType, FieldRef, Fields, TimeUnit}; + +pub struct SparkUnsafeMap { + row_addr: i64, + row_size: i32, + pub(crate) keys: SparkUnsafeArray, + pub(crate) values: SparkUnsafeArray, +} + +impl SparkUnsafeMap { + /// Creates a `SparkUnsafeMap` which points to the given address and size in bytes. + pub(crate) fn new(addr: i64, size: i32) -> Self { + // Read the number of bytes of key array from the first 8 bytes. + let slice: &[u8] = unsafe { std::slice::from_raw_parts(addr as *const u8, 8) }; + let key_array_size = i64::from_le_bytes(slice.try_into().unwrap()); + + if key_array_size < 0 { + panic!("Negative key size in bytes of map: {}", key_array_size); + } + + if key_array_size > i32::MAX as i64 { + panic!( + "Number of key size in bytes should <= i32::MAX: {}", + key_array_size + ); + } + + let value_array_size = size - key_array_size as i32 - 8; + if value_array_size < 0 { + panic!("Negative value size in bytes of map: {}", value_array_size); + } + + let keys = SparkUnsafeArray::new(addr + 8, key_array_size as i32); + let values = SparkUnsafeArray::new(addr + 8 + key_array_size, value_array_size); + + if keys.get_num_elements() != values.get_num_elements() { + panic!( + "Number of elements of keys and values should be the same: {} vs {}", + keys.get_num_elements(), + values.get_num_elements() + ); + } + + Self { + row_addr: addr, + row_size: size, + keys, + values, + } + } + + pub(crate) fn get_num_elements(&self) -> usize { + self.keys.get_num_elements() + } +} + +/// A macro defining a function to append elements of a map to a map builder with given key builder +/// and given value builder. +macro_rules! define_append_map_element { + ($func:ident, $key_builder_type:ty, $value_builder_type:ty, $key_accessor:expr, $value_accessor:expr) => { + #[allow(clippy::redundant_closure_call)] + fn $func( + map_builder: &mut MapBuilder<$key_builder_type, $value_builder_type>, + map: &SparkUnsafeMap, + ) { + let keys = &map.keys; + let values = &map.values; + + for idx in 0..keys.get_num_elements() { + // Map key cannot be null + // TODO: Getting key/value builders outside loop when new API is available + // from upstream release + let key_builder = downcast_builder_ref!($key_builder_type, map_builder.keys()); + $key_accessor(key_builder, keys, idx); + + let value_builder = + downcast_builder_ref!($value_builder_type, map_builder.values()); + if values.is_null_at(idx) { + value_builder.append_null(); + } else { + $value_accessor(value_builder, values, idx); + } + } + map_builder.append(true).unwrap(); + } + }; +} + +/// A macro defining a function to append elements of a map to a map builder with given key builder +/// and struct builder as value builder. +macro_rules! define_append_map_struct_value_element { + ($func:ident, $key_builder_type:ty, $key_accessor:expr) => { + #[allow(clippy::redundant_closure_call)] + fn $func( + map_builder: &mut MapBuilder<$key_builder_type, StructBuilder>, + map: &SparkUnsafeMap, + fields: &Fields, + ) { + let keys = &map.keys; + let values = &map.values; + + for idx in 0..keys.get_num_elements() { + // Map key cannot be null + // TODO: Getting key/value builders outside loop when new API is available + // from upstream release + let key_builder = downcast_builder_ref!($key_builder_type, map_builder.keys()); + $key_accessor(key_builder, keys, idx); + + let value_builder = downcast_builder_ref!(StructBuilder, map_builder.values()); + if values.is_null_at(idx) { + value_builder.append_null(); + } else { + let nested_row = values.get_struct(idx, fields.len()); + value_builder.append(true); + for (field_idx, field) in fields.into_iter().enumerate() { + append_field(field.data_type(), value_builder, &nested_row, field_idx); + } + } + } + map_builder.append(true).unwrap(); + } + }; +} + +/// A macro defining a function to append elements of a map to a map builder with struct builder as +/// key builder and given value builder. +macro_rules! define_append_map_struct_key_element { + ($func:ident, $value_builder_type:ty, $value_accessor:expr) => { + #[allow(clippy::redundant_closure_call)] + fn $func( + map_builder: &mut MapBuilder, + map: &SparkUnsafeMap, + fields: &Fields, + ) { + let keys = &map.keys; + let values = &map.values; + + for idx in 0..keys.get_num_elements() { + // Map key cannot be null + // TODO: Getting key/value builders outside loop when new API is available + // from upstream release + let key_builder = downcast_builder_ref!(StructBuilder, map_builder.keys()); + let nested_row = keys.get_struct(idx, fields.len()); + key_builder.append(true); + for (field_idx, field) in fields.into_iter().enumerate() { + append_field(field.data_type(), key_builder, &nested_row, field_idx); + } + + let value_builder = + downcast_builder_ref!($value_builder_type, map_builder.values()); + if values.is_null_at(idx) { + value_builder.append_null(); + } else { + $value_accessor(value_builder, values, idx); + } + } + map_builder.append(true).unwrap(); + } + }; +} + +fn append_map_struct_struct_element( + map_builder: &mut MapBuilder, + map: &SparkUnsafeMap, + key_fields: &Fields, + value_fields: &Fields, +) { + let keys = &map.keys; + let values = &map.values; + + for idx in 0..keys.get_num_elements() { + // Map key cannot be null + // TODO: Getting key/value builders outside loop when new API is available + // from upstream release + let key_builder = downcast_builder_ref!(StructBuilder, map_builder.keys()); + let nested_row = keys.get_struct(idx, key_fields.len()); + key_builder.append(true); + for (field_idx, field) in key_fields.into_iter().enumerate() { + append_field(field.data_type(), key_builder, &nested_row, field_idx); + } + + let value_builder = downcast_builder_ref!(StructBuilder, map_builder.values()); + if values.is_null_at(idx) { + value_builder.append_null(); + } else { + let nested_row = values.get_struct(idx, value_fields.len()); + value_builder.append(true); + for (field_idx, field) in value_fields.into_iter().enumerate() { + append_field(field.data_type(), value_builder, &nested_row, field_idx); + } + } + } + map_builder.append(true).unwrap(); +} + +/// A macro defining a function to append elements of a map to a map builder with given key builder +/// and decimal builder as value builder. +macro_rules! define_append_map_decimal_value_element { + ($func:ident, $key_builder_type:ty, $key_accessor:expr) => { + #[allow(clippy::redundant_closure_call)] + fn $func( + map_builder: &mut MapBuilder<$key_builder_type, Decimal128Builder>, + map: &SparkUnsafeMap, + precision: u8, + ) { + let keys = &map.keys; + let values = &map.values; + + for idx in 0..keys.get_num_elements() { + // Map key cannot be null + // TODO: Getting key/value builders outside loop when new API is available + // from upstream release + let key_builder = downcast_builder_ref!($key_builder_type, map_builder.keys()); + $key_accessor(key_builder, keys, idx); + + let value_builder = downcast_builder_ref!(Decimal128Builder, map_builder.values()); + if values.is_null_at(idx) { + value_builder.append_null(); + } else { + value_builder.append_value(values.get_decimal(idx, precision)); + } + } + map_builder.append(true).unwrap(); + } + }; +} + +/// A macro defining a function to append elements of a map to a map builder with decimal builder as +/// key builder and given value builder. +macro_rules! define_append_map_decimal_key_element { + ($func:ident, $value_builder_type:ty, $value_accessor:expr) => { + #[allow(clippy::redundant_closure_call)] + fn $func( + map_builder: &mut MapBuilder, + map: &SparkUnsafeMap, + precision: u8, + ) { + let keys = &map.keys; + let values = &map.values; + + for idx in 0..keys.get_num_elements() { + // Map key cannot be null + // TODO: Getting key/value builders outside loop when new API is available + // from upstream release + let key_builder = downcast_builder_ref!(Decimal128Builder, map_builder.keys()); + key_builder.append_value(keys.get_decimal(idx, precision)); + + let value_builder = + downcast_builder_ref!($value_builder_type, map_builder.values()); + if values.is_null_at(idx) { + value_builder.append_null(); + } else { + $value_accessor(value_builder, values, idx); + } + } + map_builder.append(true).unwrap(); + } + }; +} + +fn append_map_decimal_decimal_element( + map_builder: &mut MapBuilder, + map: &SparkUnsafeMap, + key_precision: u8, + value_precision: u8, +) { + let keys = &map.keys; + let values = &map.values; + + for idx in 0..keys.get_num_elements() { + // Map key cannot be null + // TODO: Getting key/value builders outside loop when new API is available + // from upstream release + let key_builder = downcast_builder_ref!(Decimal128Builder, map_builder.keys()); + key_builder.append_value(keys.get_decimal(idx, key_precision)); + + let value_builder = downcast_builder_ref!(Decimal128Builder, map_builder.values()); + if values.is_null_at(idx) { + value_builder.append_null(); + } else { + value_builder.append_value(values.get_decimal(idx, value_precision)); + } + } + map_builder.append(true).unwrap(); +} + +fn append_map_decimal_struct_element( + map_builder: &mut MapBuilder, + map: &SparkUnsafeMap, + precision: u8, + fields: &Fields, +) { + let keys = &map.keys; + let values = &map.values; + + for idx in 0..keys.get_num_elements() { + // Map key cannot be null + // TODO: Getting key/value builders outside loop when new API is available + // from upstream release + let key_builder = downcast_builder_ref!(Decimal128Builder, map_builder.keys()); + key_builder.append_value(keys.get_decimal(idx, precision)); + + let value_builder = downcast_builder_ref!(StructBuilder, map_builder.values()); + if values.is_null_at(idx) { + value_builder.append_null(); + } else { + let nested_row = values.get_struct(idx, fields.len()); + value_builder.append(true); + for (field_idx, field) in fields.into_iter().enumerate() { + append_field(field.data_type(), value_builder, &nested_row, field_idx); + } + } + } + map_builder.append(true).unwrap(); +} + +fn append_map_struct_decimal_element( + map_builder: &mut MapBuilder, + map: &SparkUnsafeMap, + precision: u8, + fields: &Fields, +) { + let keys = &map.keys; + let values = &map.values; + + for idx in 0..keys.get_num_elements() { + // Map key cannot be null + // TODO: Getting key/value builders outside loop when new API is available + // from upstream release + let key_builder = downcast_builder_ref!(StructBuilder, map_builder.keys()); + let nested_row = values.get_struct(idx, fields.len()); + key_builder.append(true); + for (field_idx, field) in fields.into_iter().enumerate() { + append_field(field.data_type(), key_builder, &nested_row, field_idx); + } + + let value_builder = downcast_builder_ref!(Decimal128Builder, map_builder.values()); + if values.is_null_at(idx) { + value_builder.append_null(); + } else { + value_builder.append_value(values.get_decimal(idx, precision)); + } + } + map_builder.append(true).unwrap(); +} + +// Boolean key +define_append_map_element!( + append_map_boolean_boolean_element, + BooleanBuilder, + BooleanBuilder, + |builder: &mut BooleanBuilder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_boolean(idx)), + |builder: &mut BooleanBuilder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_boolean(idx)) +); + +define_append_map_element!( + append_map_boolean_int8_element, + BooleanBuilder, + Int8Builder, + |builder: &mut BooleanBuilder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_boolean(idx)), + |builder: &mut Int8Builder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_byte(idx)) +); + +define_append_map_element!( + append_map_boolean_int16_element, + BooleanBuilder, + Int16Builder, + |builder: &mut BooleanBuilder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_boolean(idx)), + |builder: &mut Int16Builder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_short(idx)) +); + +define_append_map_element!( + append_map_boolean_int32_element, + BooleanBuilder, + Int32Builder, + |builder: &mut BooleanBuilder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_boolean(idx)), + |builder: &mut Int32Builder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_int(idx)) +); + +define_append_map_element!( + append_map_boolean_int64_element, + BooleanBuilder, + Int64Builder, + |builder: &mut BooleanBuilder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_boolean(idx)), + |builder: &mut Int64Builder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_long(idx)) +); + +define_append_map_element!( + append_map_boolean_float32_element, + BooleanBuilder, + Float32Builder, + |builder: &mut BooleanBuilder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_boolean(idx)), + |builder: &mut Float32Builder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_float(idx)) +); + +define_append_map_element!( + append_map_boolean_float64_element, + BooleanBuilder, + Float64Builder, + |builder: &mut BooleanBuilder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_boolean(idx)), + |builder: &mut Float64Builder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_double(idx)) +); + +define_append_map_element!( + append_map_boolean_date32_element, + BooleanBuilder, + Date32Builder, + |builder: &mut BooleanBuilder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_boolean(idx)), + |builder: &mut Date32Builder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_date(idx)) +); + +define_append_map_element!( + append_map_boolean_timestamp_element, + BooleanBuilder, + TimestampMicrosecondBuilder, + |builder: &mut BooleanBuilder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_boolean(idx)), + |builder: &mut TimestampMicrosecondBuilder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_timestamp(idx)) +); + +define_append_map_element!( + append_map_boolean_binary_element, + BooleanBuilder, + BinaryBuilder, + |builder: &mut BooleanBuilder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_boolean(idx)), + |builder: &mut BinaryBuilder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_binary(idx)) +); + +define_append_map_element!( + append_map_boolean_string_element, + BooleanBuilder, + StringBuilder, + |builder: &mut BooleanBuilder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_boolean(idx)), + |builder: &mut StringBuilder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_string(idx)) +); + +define_append_map_decimal_value_element!( + append_map_boolean_decimal_element, + BooleanBuilder, + |builder: &mut BooleanBuilder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_boolean(idx)) +); + +define_append_map_struct_value_element!( + append_map_boolean_struct_element, + BooleanBuilder, + |builder: &mut BooleanBuilder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_boolean(idx)) +); + +// Int8 key +define_append_map_element!( + append_map_int8_boolean_element, + Int8Builder, + BooleanBuilder, + |builder: &mut Int8Builder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_byte(idx)), + |builder: &mut BooleanBuilder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_boolean(idx)) +); + +define_append_map_element!( + append_map_int8_int8_element, + Int8Builder, + Int8Builder, + |builder: &mut Int8Builder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_byte(idx)), + |builder: &mut Int8Builder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_byte(idx)) +); + +define_append_map_element!( + append_map_int8_int16_element, + Int8Builder, + Int16Builder, + |builder: &mut Int8Builder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_byte(idx)), + |builder: &mut Int16Builder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_short(idx)) +); + +define_append_map_element!( + append_map_int8_int32_element, + Int8Builder, + Int32Builder, + |builder: &mut Int8Builder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_byte(idx)), + |builder: &mut Int32Builder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_int(idx)) +); + +define_append_map_element!( + append_map_int8_int64_element, + Int8Builder, + Int64Builder, + |builder: &mut Int8Builder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_byte(idx)), + |builder: &mut Int64Builder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_long(idx)) +); + +define_append_map_element!( + append_map_int8_float32_element, + Int8Builder, + Float32Builder, + |builder: &mut Int8Builder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_byte(idx)), + |builder: &mut Float32Builder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_float(idx)) +); + +define_append_map_element!( + append_map_int8_float64_element, + Int8Builder, + Float64Builder, + |builder: &mut Int8Builder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_byte(idx)), + |builder: &mut Float64Builder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_double(idx)) +); + +define_append_map_element!( + append_map_int8_date32_element, + Int8Builder, + Date32Builder, + |builder: &mut Int8Builder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_byte(idx)), + |builder: &mut Date32Builder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_date(idx)) +); + +define_append_map_element!( + append_map_int8_timestamp_element, + Int8Builder, + TimestampMicrosecondBuilder, + |builder: &mut Int8Builder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_byte(idx)), + |builder: &mut TimestampMicrosecondBuilder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_timestamp(idx)) +); + +define_append_map_element!( + append_map_int8_binary_element, + Int8Builder, + BinaryBuilder, + |builder: &mut Int8Builder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_byte(idx)), + |builder: &mut BinaryBuilder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_binary(idx)) +); + +define_append_map_element!( + append_map_int8_string_element, + Int8Builder, + StringBuilder, + |builder: &mut Int8Builder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_byte(idx)), + |builder: &mut StringBuilder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_string(idx)) +); + +define_append_map_decimal_value_element!( + append_map_int8_decimal_element, + Int8Builder, + |builder: &mut Int8Builder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_byte(idx)) +); + +define_append_map_struct_value_element!( + append_map_int8_struct_element, + Int8Builder, + |builder: &mut Int8Builder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_byte(idx)) +); + +// Int16 key +define_append_map_element!( + append_map_int16_boolean_element, + Int16Builder, + BooleanBuilder, + |builder: &mut Int16Builder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_short(idx)), + |builder: &mut BooleanBuilder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_boolean(idx)) +); + +define_append_map_element!( + append_map_int16_int8_element, + Int16Builder, + Int8Builder, + |builder: &mut Int16Builder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_short(idx)), + |builder: &mut Int8Builder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_byte(idx)) +); + +define_append_map_element!( + append_map_int16_int16_element, + Int16Builder, + Int16Builder, + |builder: &mut Int16Builder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_short(idx)), + |builder: &mut Int16Builder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_short(idx)) +); + +define_append_map_element!( + append_map_int16_int32_element, + Int16Builder, + Int32Builder, + |builder: &mut Int16Builder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_short(idx)), + |builder: &mut Int32Builder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_int(idx)) +); + +define_append_map_element!( + append_map_int16_int64_element, + Int16Builder, + Int64Builder, + |builder: &mut Int16Builder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_short(idx)), + |builder: &mut Int64Builder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_long(idx)) +); + +define_append_map_element!( + append_map_int16_float32_element, + Int16Builder, + Float32Builder, + |builder: &mut Int16Builder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_short(idx)), + |builder: &mut Float32Builder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_float(idx)) +); + +define_append_map_element!( + append_map_int16_float64_element, + Int16Builder, + Float64Builder, + |builder: &mut Int16Builder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_short(idx)), + |builder: &mut Float64Builder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_double(idx)) +); + +define_append_map_element!( + append_map_int16_date32_element, + Int16Builder, + Date32Builder, + |builder: &mut Int16Builder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_short(idx)), + |builder: &mut Date32Builder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_date(idx)) +); + +define_append_map_element!( + append_map_int16_timestamp_element, + Int16Builder, + TimestampMicrosecondBuilder, + |builder: &mut Int16Builder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_short(idx)), + |builder: &mut TimestampMicrosecondBuilder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_timestamp(idx)) +); + +define_append_map_element!( + append_map_int16_binary_element, + Int16Builder, + BinaryBuilder, + |builder: &mut Int16Builder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_short(idx)), + |builder: &mut BinaryBuilder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_binary(idx)) +); + +define_append_map_element!( + append_map_int16_string_element, + Int16Builder, + StringBuilder, + |builder: &mut Int16Builder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_short(idx)), + |builder: &mut StringBuilder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_string(idx)) +); + +define_append_map_decimal_value_element!( + append_map_int16_decimal_element, + Int16Builder, + |builder: &mut Int16Builder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_short(idx)) +); + +define_append_map_struct_value_element!( + append_map_int16_struct_element, + Int16Builder, + |builder: &mut Int16Builder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_short(idx)) +); + +// Int32 key +define_append_map_element!( + append_map_int32_boolean_element, + Int32Builder, + BooleanBuilder, + |builder: &mut Int32Builder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_int(idx)), + |builder: &mut BooleanBuilder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_boolean(idx)) +); + +define_append_map_element!( + append_map_int32_int8_element, + Int32Builder, + Int8Builder, + |builder: &mut Int32Builder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_int(idx)), + |builder: &mut Int8Builder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_byte(idx)) +); + +define_append_map_element!( + append_map_int32_int16_element, + Int32Builder, + Int16Builder, + |builder: &mut Int32Builder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_int(idx)), + |builder: &mut Int16Builder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_short(idx)) +); + +define_append_map_element!( + append_map_int32_int32_element, + Int32Builder, + Int32Builder, + |builder: &mut Int32Builder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_int(idx)), + |builder: &mut Int32Builder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_int(idx)) +); + +define_append_map_element!( + append_map_int32_int64_element, + Int32Builder, + Int64Builder, + |builder: &mut Int32Builder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_int(idx)), + |builder: &mut Int64Builder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_long(idx)) +); + +define_append_map_element!( + append_map_int32_float32_element, + Int32Builder, + Float32Builder, + |builder: &mut Int32Builder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_int(idx)), + |builder: &mut Float32Builder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_float(idx)) +); + +define_append_map_element!( + append_map_int32_float64_element, + Int32Builder, + Float64Builder, + |builder: &mut Int32Builder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_int(idx)), + |builder: &mut Float64Builder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_double(idx)) +); + +define_append_map_element!( + append_map_int32_date32_element, + Int32Builder, + Date32Builder, + |builder: &mut Int32Builder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_int(idx)), + |builder: &mut Date32Builder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_date(idx)) +); + +define_append_map_element!( + append_map_int32_timestamp_element, + Int32Builder, + TimestampMicrosecondBuilder, + |builder: &mut Int32Builder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_int(idx)), + |builder: &mut TimestampMicrosecondBuilder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_timestamp(idx)) +); + +define_append_map_element!( + append_map_int32_binary_element, + Int32Builder, + BinaryBuilder, + |builder: &mut Int32Builder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_int(idx)), + |builder: &mut BinaryBuilder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_binary(idx)) +); + +define_append_map_element!( + append_map_int32_string_element, + Int32Builder, + StringBuilder, + |builder: &mut Int32Builder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_int(idx)), + |builder: &mut StringBuilder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_string(idx)) +); + +define_append_map_decimal_value_element!( + append_map_int32_decimal_element, + Int32Builder, + |builder: &mut Int32Builder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_int(idx)) +); + +define_append_map_struct_value_element!( + append_map_int32_struct_element, + Int32Builder, + |builder: &mut Int32Builder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_int(idx)) +); + +// Int64 key +define_append_map_element!( + append_map_int64_boolean_element, + Int64Builder, + BooleanBuilder, + |builder: &mut Int64Builder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_long(idx)), + |builder: &mut BooleanBuilder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_boolean(idx)) +); + +define_append_map_element!( + append_map_int64_int8_element, + Int64Builder, + Int8Builder, + |builder: &mut Int64Builder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_long(idx)), + |builder: &mut Int8Builder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_byte(idx)) +); + +define_append_map_element!( + append_map_int64_int16_element, + Int64Builder, + Int16Builder, + |builder: &mut Int64Builder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_long(idx)), + |builder: &mut Int16Builder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_short(idx)) +); + +define_append_map_element!( + append_map_int64_int32_element, + Int64Builder, + Int32Builder, + |builder: &mut Int64Builder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_long(idx)), + |builder: &mut Int32Builder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_int(idx)) +); + +define_append_map_element!( + append_map_int64_int64_element, + Int64Builder, + Int64Builder, + |builder: &mut Int64Builder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_long(idx)), + |builder: &mut Int64Builder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_long(idx)) +); + +define_append_map_element!( + append_map_int64_float32_element, + Int64Builder, + Float32Builder, + |builder: &mut Int64Builder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_long(idx)), + |builder: &mut Float32Builder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_float(idx)) +); + +define_append_map_element!( + append_map_int64_float64_element, + Int64Builder, + Float64Builder, + |builder: &mut Int64Builder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_long(idx)), + |builder: &mut Float64Builder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_double(idx)) +); + +define_append_map_element!( + append_map_int64_date32_element, + Int64Builder, + Date32Builder, + |builder: &mut Int64Builder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_long(idx)), + |builder: &mut Date32Builder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_date(idx)) +); + +define_append_map_element!( + append_map_int64_timestamp_element, + Int64Builder, + TimestampMicrosecondBuilder, + |builder: &mut Int64Builder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_long(idx)), + |builder: &mut TimestampMicrosecondBuilder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_timestamp(idx)) +); + +define_append_map_element!( + append_map_int64_binary_element, + Int64Builder, + BinaryBuilder, + |builder: &mut Int64Builder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_long(idx)), + |builder: &mut BinaryBuilder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_binary(idx)) +); + +define_append_map_element!( + append_map_int64_string_element, + Int64Builder, + StringBuilder, + |builder: &mut Int64Builder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_long(idx)), + |builder: &mut StringBuilder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_string(idx)) +); + +define_append_map_decimal_value_element!( + append_map_int64_decimal_element, + Int64Builder, + |builder: &mut Int64Builder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_long(idx)) +); + +define_append_map_struct_value_element!( + append_map_int64_struct_element, + Int64Builder, + |builder: &mut Int64Builder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_long(idx)) +); + +// Float32 key +define_append_map_element!( + append_map_float32_boolean_element, + Float32Builder, + BooleanBuilder, + |builder: &mut Float32Builder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_float(idx)), + |builder: &mut BooleanBuilder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_boolean(idx)) +); + +define_append_map_element!( + append_map_float32_int8_element, + Float32Builder, + Int8Builder, + |builder: &mut Float32Builder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_float(idx)), + |builder: &mut Int8Builder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_byte(idx)) +); + +define_append_map_element!( + append_map_float32_int16_element, + Float32Builder, + Int16Builder, + |builder: &mut Float32Builder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_float(idx)), + |builder: &mut Int16Builder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_short(idx)) +); + +define_append_map_element!( + append_map_float32_int32_element, + Float32Builder, + Int32Builder, + |builder: &mut Float32Builder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_float(idx)), + |builder: &mut Int32Builder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_int(idx)) +); + +define_append_map_element!( + append_map_float32_int64_element, + Float32Builder, + Int64Builder, + |builder: &mut Float32Builder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_float(idx)), + |builder: &mut Int64Builder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_long(idx)) +); + +define_append_map_element!( + append_map_float32_float32_element, + Float32Builder, + Float32Builder, + |builder: &mut Float32Builder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_float(idx)), + |builder: &mut Float32Builder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_float(idx)) +); + +define_append_map_element!( + append_map_float32_float64_element, + Float32Builder, + Float64Builder, + |builder: &mut Float32Builder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_float(idx)), + |builder: &mut Float64Builder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_double(idx)) +); + +define_append_map_element!( + append_map_float32_date32_element, + Float32Builder, + Date32Builder, + |builder: &mut Float32Builder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_float(idx)), + |builder: &mut Date32Builder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_date(idx)) +); + +define_append_map_element!( + append_map_float32_timestamp_element, + Float32Builder, + TimestampMicrosecondBuilder, + |builder: &mut Float32Builder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_float(idx)), + |builder: &mut TimestampMicrosecondBuilder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_timestamp(idx)) +); + +define_append_map_element!( + append_map_float32_binary_element, + Float32Builder, + BinaryBuilder, + |builder: &mut Float32Builder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_float(idx)), + |builder: &mut BinaryBuilder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_binary(idx)) +); + +define_append_map_element!( + append_map_float32_string_element, + Float32Builder, + StringBuilder, + |builder: &mut Float32Builder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_float(idx)), + |builder: &mut StringBuilder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_string(idx)) +); + +define_append_map_decimal_value_element!( + append_map_float32_decimal_element, + Float32Builder, + |builder: &mut Float32Builder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_float(idx)) +); + +define_append_map_struct_value_element!( + append_map_float32_struct_element, + Float32Builder, + |builder: &mut Float32Builder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_float(idx)) +); + +// Float64 key +define_append_map_element!( + append_map_float64_boolean_element, + Float64Builder, + BooleanBuilder, + |builder: &mut Float64Builder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_double(idx)), + |builder: &mut BooleanBuilder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_boolean(idx)) +); + +define_append_map_element!( + append_map_float64_int8_element, + Float64Builder, + Int8Builder, + |builder: &mut Float64Builder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_double(idx)), + |builder: &mut Int8Builder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_byte(idx)) +); + +define_append_map_element!( + append_map_float64_int16_element, + Float64Builder, + Int16Builder, + |builder: &mut Float64Builder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_double(idx)), + |builder: &mut Int16Builder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_short(idx)) +); + +define_append_map_element!( + append_map_float64_int32_element, + Float64Builder, + Int32Builder, + |builder: &mut Float64Builder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_double(idx)), + |builder: &mut Int32Builder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_int(idx)) +); + +define_append_map_element!( + append_map_float64_int64_element, + Float64Builder, + Int64Builder, + |builder: &mut Float64Builder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_double(idx)), + |builder: &mut Int64Builder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_long(idx)) +); + +define_append_map_element!( + append_map_float64_float32_element, + Float64Builder, + Float32Builder, + |builder: &mut Float64Builder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_double(idx)), + |builder: &mut Float32Builder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_float(idx)) +); + +define_append_map_element!( + append_map_float64_float64_element, + Float64Builder, + Float64Builder, + |builder: &mut Float64Builder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_double(idx)), + |builder: &mut Float64Builder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_double(idx)) +); + +define_append_map_element!( + append_map_float64_date32_element, + Float64Builder, + Date32Builder, + |builder: &mut Float64Builder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_double(idx)), + |builder: &mut Date32Builder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_date(idx)) +); + +define_append_map_element!( + append_map_float64_timestamp_element, + Float64Builder, + TimestampMicrosecondBuilder, + |builder: &mut Float64Builder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_double(idx)), + |builder: &mut TimestampMicrosecondBuilder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_timestamp(idx)) +); + +define_append_map_element!( + append_map_float64_binary_element, + Float64Builder, + BinaryBuilder, + |builder: &mut Float64Builder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_double(idx)), + |builder: &mut BinaryBuilder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_binary(idx)) +); + +define_append_map_element!( + append_map_float64_string_element, + Float64Builder, + StringBuilder, + |builder: &mut Float64Builder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_double(idx)), + |builder: &mut StringBuilder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_string(idx)) +); + +define_append_map_decimal_value_element!( + append_map_float64_decimal_element, + Float64Builder, + |builder: &mut Float64Builder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_double(idx)) +); + +define_append_map_struct_value_element!( + append_map_float64_struct_element, + Float64Builder, + |builder: &mut Float64Builder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_double(idx)) +); + +// Date32 key +define_append_map_element!( + append_map_date32_boolean_element, + Date32Builder, + BooleanBuilder, + |builder: &mut Date32Builder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_date(idx)), + |builder: &mut BooleanBuilder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_boolean(idx)) +); + +define_append_map_element!( + append_map_date32_int8_element, + Date32Builder, + Int8Builder, + |builder: &mut Date32Builder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_date(idx)), + |builder: &mut Int8Builder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_byte(idx)) +); + +define_append_map_element!( + append_map_date32_int16_element, + Date32Builder, + Int16Builder, + |builder: &mut Date32Builder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_date(idx)), + |builder: &mut Int16Builder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_short(idx)) +); + +define_append_map_element!( + append_map_date32_int32_element, + Date32Builder, + Int32Builder, + |builder: &mut Date32Builder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_date(idx)), + |builder: &mut Int32Builder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_int(idx)) +); + +define_append_map_element!( + append_map_date32_int64_element, + Date32Builder, + Int64Builder, + |builder: &mut Date32Builder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_date(idx)), + |builder: &mut Int64Builder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_long(idx)) +); + +define_append_map_element!( + append_map_date32_float32_element, + Date32Builder, + Float32Builder, + |builder: &mut Date32Builder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_date(idx)), + |builder: &mut Float32Builder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_float(idx)) +); + +define_append_map_element!( + append_map_date32_float64_element, + Date32Builder, + Float64Builder, + |builder: &mut Date32Builder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_date(idx)), + |builder: &mut Float64Builder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_double(idx)) +); + +define_append_map_element!( + append_map_date32_date32_element, + Date32Builder, + Date32Builder, + |builder: &mut Date32Builder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_date(idx)), + |builder: &mut Date32Builder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_date(idx)) +); + +define_append_map_element!( + append_map_date32_timestamp_element, + Date32Builder, + TimestampMicrosecondBuilder, + |builder: &mut Date32Builder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_date(idx)), + |builder: &mut TimestampMicrosecondBuilder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_timestamp(idx)) +); + +define_append_map_element!( + append_map_date32_binary_element, + Date32Builder, + BinaryBuilder, + |builder: &mut Date32Builder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_date(idx)), + |builder: &mut BinaryBuilder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_binary(idx)) +); + +define_append_map_element!( + append_map_date32_string_element, + Date32Builder, + StringBuilder, + |builder: &mut Date32Builder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_date(idx)), + |builder: &mut StringBuilder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_string(idx)) +); + +define_append_map_decimal_value_element!( + append_map_date32_decimal_element, + Date32Builder, + |builder: &mut Date32Builder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_date(idx)) +); + +define_append_map_struct_value_element!( + append_map_date32_struct_element, + Date32Builder, + |builder: &mut Date32Builder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_date(idx)) +); + +// Timestamp key +define_append_map_element!( + append_map_timestamp_boolean_element, + TimestampMicrosecondBuilder, + BooleanBuilder, + |builder: &mut TimestampMicrosecondBuilder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_timestamp(idx)), + |builder: &mut BooleanBuilder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_boolean(idx)) +); + +define_append_map_element!( + append_map_timestamp_int8_element, + TimestampMicrosecondBuilder, + Int8Builder, + |builder: &mut TimestampMicrosecondBuilder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_timestamp(idx)), + |builder: &mut Int8Builder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_byte(idx)) +); + +define_append_map_element!( + append_map_timestamp_int16_element, + TimestampMicrosecondBuilder, + Int16Builder, + |builder: &mut TimestampMicrosecondBuilder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_timestamp(idx)), + |builder: &mut Int16Builder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_short(idx)) +); + +define_append_map_element!( + append_map_timestamp_int32_element, + TimestampMicrosecondBuilder, + Int32Builder, + |builder: &mut TimestampMicrosecondBuilder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_timestamp(idx)), + |builder: &mut Int32Builder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_int(idx)) +); + +define_append_map_element!( + append_map_timestamp_int64_element, + TimestampMicrosecondBuilder, + Int64Builder, + |builder: &mut TimestampMicrosecondBuilder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_timestamp(idx)), + |builder: &mut Int64Builder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_long(idx)) +); + +define_append_map_element!( + append_map_timestamp_float32_element, + TimestampMicrosecondBuilder, + Float32Builder, + |builder: &mut TimestampMicrosecondBuilder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_timestamp(idx)), + |builder: &mut Float32Builder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_float(idx)) +); + +define_append_map_element!( + append_map_timestamp_float64_element, + TimestampMicrosecondBuilder, + Float64Builder, + |builder: &mut TimestampMicrosecondBuilder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_timestamp(idx)), + |builder: &mut Float64Builder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_double(idx)) +); + +define_append_map_element!( + append_map_timestamp_date32_element, + TimestampMicrosecondBuilder, + Date32Builder, + |builder: &mut TimestampMicrosecondBuilder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_timestamp(idx)), + |builder: &mut Date32Builder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_date(idx)) +); + +define_append_map_element!( + append_map_timestamp_timestamp_element, + TimestampMicrosecondBuilder, + TimestampMicrosecondBuilder, + |builder: &mut TimestampMicrosecondBuilder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_timestamp(idx)), + |builder: &mut TimestampMicrosecondBuilder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_timestamp(idx)) +); + +define_append_map_element!( + append_map_timestamp_binary_element, + TimestampMicrosecondBuilder, + BinaryBuilder, + |builder: &mut TimestampMicrosecondBuilder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_timestamp(idx)), + |builder: &mut BinaryBuilder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_binary(idx)) +); + +define_append_map_element!( + append_map_timestamp_string_element, + TimestampMicrosecondBuilder, + StringBuilder, + |builder: &mut TimestampMicrosecondBuilder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_timestamp(idx)), + |builder: &mut StringBuilder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_string(idx)) +); + +define_append_map_decimal_value_element!( + append_map_timestamp_decimal_element, + TimestampMicrosecondBuilder, + |builder: &mut TimestampMicrosecondBuilder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_timestamp(idx)) +); + +define_append_map_struct_value_element!( + append_map_timestamp_struct_element, + TimestampMicrosecondBuilder, + |builder: &mut TimestampMicrosecondBuilder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_timestamp(idx)) +); + +// Binary key +define_append_map_element!( + append_map_binary_boolean_element, + BinaryBuilder, + BooleanBuilder, + |builder: &mut BinaryBuilder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_binary(idx)), + |builder: &mut BooleanBuilder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_boolean(idx)) +); + +define_append_map_element!( + append_map_binary_int8_element, + BinaryBuilder, + Int8Builder, + |builder: &mut BinaryBuilder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_binary(idx)), + |builder: &mut Int8Builder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_byte(idx)) +); + +define_append_map_element!( + append_map_binary_int16_element, + BinaryBuilder, + Int16Builder, + |builder: &mut BinaryBuilder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_binary(idx)), + |builder: &mut Int16Builder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_short(idx)) +); + +define_append_map_element!( + append_map_binary_int32_element, + BinaryBuilder, + Int32Builder, + |builder: &mut BinaryBuilder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_binary(idx)), + |builder: &mut Int32Builder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_int(idx)) +); + +define_append_map_element!( + append_map_binary_int64_element, + BinaryBuilder, + Int64Builder, + |builder: &mut BinaryBuilder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_binary(idx)), + |builder: &mut Int64Builder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_long(idx)) +); + +define_append_map_element!( + append_map_binary_float32_element, + BinaryBuilder, + Float32Builder, + |builder: &mut BinaryBuilder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_binary(idx)), + |builder: &mut Float32Builder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_float(idx)) +); + +define_append_map_element!( + append_map_binary_float64_element, + BinaryBuilder, + Float64Builder, + |builder: &mut BinaryBuilder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_binary(idx)), + |builder: &mut Float64Builder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_double(idx)) +); + +define_append_map_element!( + append_map_binary_date32_element, + BinaryBuilder, + Date32Builder, + |builder: &mut BinaryBuilder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_binary(idx)), + |builder: &mut Date32Builder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_date(idx)) +); + +define_append_map_element!( + append_map_binary_timestamp_element, + BinaryBuilder, + TimestampMicrosecondBuilder, + |builder: &mut BinaryBuilder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_binary(idx)), + |builder: &mut TimestampMicrosecondBuilder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_timestamp(idx)) +); + +define_append_map_element!( + append_map_binary_binary_element, + BinaryBuilder, + BinaryBuilder, + |builder: &mut BinaryBuilder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_binary(idx)), + |builder: &mut BinaryBuilder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_binary(idx)) +); + +define_append_map_element!( + append_map_binary_string_element, + BinaryBuilder, + StringBuilder, + |builder: &mut BinaryBuilder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_binary(idx)), + |builder: &mut StringBuilder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_string(idx)) +); + +define_append_map_decimal_value_element!( + append_map_binary_decimal_element, + BinaryBuilder, + |builder: &mut BinaryBuilder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_binary(idx)) +); + +define_append_map_struct_value_element!( + append_map_binary_struct_element, + BinaryBuilder, + |builder: &mut BinaryBuilder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_binary(idx)) +); + +// String key +define_append_map_element!( + append_map_string_boolean_element, + StringBuilder, + BooleanBuilder, + |builder: &mut StringBuilder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_string(idx)), + |builder: &mut BooleanBuilder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_boolean(idx)) +); + +define_append_map_element!( + append_map_string_int8_element, + StringBuilder, + Int8Builder, + |builder: &mut StringBuilder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_string(idx)), + |builder: &mut Int8Builder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_byte(idx)) +); + +define_append_map_element!( + append_map_string_int16_element, + StringBuilder, + Int16Builder, + |builder: &mut StringBuilder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_string(idx)), + |builder: &mut Int16Builder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_short(idx)) +); + +define_append_map_element!( + append_map_string_int32_element, + StringBuilder, + Int32Builder, + |builder: &mut StringBuilder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_string(idx)), + |builder: &mut Int32Builder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_int(idx)) +); + +define_append_map_element!( + append_map_string_int64_element, + StringBuilder, + Int64Builder, + |builder: &mut StringBuilder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_string(idx)), + |builder: &mut Int64Builder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_long(idx)) +); + +define_append_map_element!( + append_map_string_float32_element, + StringBuilder, + Float32Builder, + |builder: &mut StringBuilder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_string(idx)), + |builder: &mut Float32Builder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_float(idx)) +); + +define_append_map_element!( + append_map_string_float64_element, + StringBuilder, + Float64Builder, + |builder: &mut StringBuilder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_string(idx)), + |builder: &mut Float64Builder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_double(idx)) +); + +define_append_map_element!( + append_map_string_date32_element, + StringBuilder, + Date32Builder, + |builder: &mut StringBuilder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_string(idx)), + |builder: &mut Date32Builder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_date(idx)) +); + +define_append_map_element!( + append_map_string_timestamp_element, + StringBuilder, + TimestampMicrosecondBuilder, + |builder: &mut StringBuilder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_string(idx)), + |builder: &mut TimestampMicrosecondBuilder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_timestamp(idx)) +); + +define_append_map_element!( + append_map_string_binary_element, + StringBuilder, + BinaryBuilder, + |builder: &mut StringBuilder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_string(idx)), + |builder: &mut BinaryBuilder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_binary(idx)) +); + +define_append_map_element!( + append_map_string_string_element, + StringBuilder, + StringBuilder, + |builder: &mut StringBuilder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_string(idx)), + |builder: &mut StringBuilder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_string(idx)) +); + +define_append_map_decimal_value_element!( + append_map_string_decimal_element, + StringBuilder, + |builder: &mut StringBuilder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_string(idx)) +); + +define_append_map_struct_value_element!( + append_map_string_struct_element, + StringBuilder, + |builder: &mut StringBuilder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_string(idx)) +); + +// Decimal key +define_append_map_decimal_key_element!( + append_map_decimal_boolean_element, + BooleanBuilder, + |builder: &mut BooleanBuilder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_boolean(idx)) +); + +define_append_map_decimal_key_element!( + append_map_decimal_int8_element, + Int8Builder, + |builder: &mut Int8Builder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_byte(idx)) +); + +define_append_map_decimal_key_element!( + append_map_decimal_int16_element, + Int16Builder, + |builder: &mut Int16Builder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_short(idx)) +); + +define_append_map_decimal_key_element!( + append_map_decimal_int32_element, + Int32Builder, + |builder: &mut Int32Builder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_int(idx)) +); + +define_append_map_decimal_key_element!( + append_map_decimal_int64_element, + Int64Builder, + |builder: &mut Int64Builder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_long(idx)) +); + +define_append_map_decimal_key_element!( + append_map_decimal_float32_element, + Float32Builder, + |builder: &mut Float32Builder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_float(idx)) +); + +define_append_map_decimal_key_element!( + append_map_decimal_float64_element, + Float64Builder, + |builder: &mut Float64Builder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_double(idx)) +); + +define_append_map_decimal_key_element!( + append_map_decimal_date32_element, + Date32Builder, + |builder: &mut Date32Builder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_date(idx)) +); + +define_append_map_decimal_key_element!( + append_map_decimal_timestamp_element, + TimestampMicrosecondBuilder, + |builder: &mut TimestampMicrosecondBuilder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_timestamp(idx)) +); + +define_append_map_decimal_key_element!( + append_map_decimal_binary_element, + BinaryBuilder, + |builder: &mut BinaryBuilder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_binary(idx)) +); + +define_append_map_decimal_key_element!( + append_map_decimal_string_element, + StringBuilder, + |builder: &mut StringBuilder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_string(idx)) +); + +// Struct key +define_append_map_struct_key_element!( + append_map_struct_boolean_element, + BooleanBuilder, + |builder: &mut BooleanBuilder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_boolean(idx)) +); + +define_append_map_struct_key_element!( + append_map_struct_int8_element, + Int8Builder, + |builder: &mut Int8Builder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_byte(idx)) +); + +define_append_map_struct_key_element!( + append_map_struct_int16_element, + Int16Builder, + |builder: &mut Int16Builder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_short(idx)) +); + +define_append_map_struct_key_element!( + append_map_struct_int32_element, + Int32Builder, + |builder: &mut Int32Builder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_int(idx)) +); + +define_append_map_struct_key_element!( + append_map_struct_int64_element, + Int64Builder, + |builder: &mut Int64Builder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_long(idx)) +); + +define_append_map_struct_key_element!( + append_map_struct_float32_element, + Float32Builder, + |builder: &mut Float32Builder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_float(idx)) +); + +define_append_map_struct_key_element!( + append_map_struct_float64_element, + Float64Builder, + |builder: &mut Float64Builder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_double(idx)) +); + +define_append_map_struct_key_element!( + append_map_struct_date32_element, + Date32Builder, + |builder: &mut Date32Builder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_date(idx)) +); + +define_append_map_struct_key_element!( + append_map_struct_timestamp_element, + TimestampMicrosecondBuilder, + |builder: &mut TimestampMicrosecondBuilder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_timestamp(idx)) +); + +define_append_map_struct_key_element!( + append_map_struct_binary_element, + BinaryBuilder, + |builder: &mut BinaryBuilder, values: &SparkUnsafeArray, idx: usize| builder + .append_value(values.get_binary(idx)) +); + +define_append_map_struct_key_element!( + append_map_struct_string_element, + StringBuilder, + |builder: &mut StringBuilder, keys: &SparkUnsafeArray, idx: usize| builder + .append_value(keys.get_string(idx)) +); + +/// Appending the given map stored in `SparkUnsafeMap` into `MapBuilder`. +/// `field` includes data types of the map element. `map_builder` is the map builder. +/// `map` is the map stored in `SparkUnsafeMap`. +pub fn append_map_elements( + field: &FieldRef, + map_builder: &mut MapBuilder, + map: &SparkUnsafeMap, +) -> Result<(), CometError> { + let (key_dt, value_dt, _) = get_map_key_value_dt(field).unwrap(); + + // macro cannot expand to match arm + match (key_dt, value_dt) { + (DataType::Boolean, DataType::Boolean) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_boolean_boolean_element(map_builder, map); + } + (DataType::Boolean, DataType::Int8) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_boolean_int8_element(map_builder, map); + } + (DataType::Boolean, DataType::Int16) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_boolean_int16_element(map_builder, map); + } + (DataType::Boolean, DataType::Int32) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_boolean_int32_element(map_builder, map); + } + (DataType::Boolean, DataType::Int64) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_boolean_int64_element(map_builder, map); + } + (DataType::Boolean, DataType::Float32) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_boolean_float32_element(map_builder, map); + } + (DataType::Boolean, DataType::Float64) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_boolean_float64_element(map_builder, map); + } + (DataType::Boolean, DataType::Date32) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_boolean_date32_element(map_builder, map); + } + (DataType::Boolean, DataType::Timestamp(TimeUnit::Microsecond, _)) => { + let map_builder = downcast_builder_ref!( + MapBuilder, + map_builder + ); + append_map_boolean_timestamp_element(map_builder, map); + } + (DataType::Boolean, DataType::Binary) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_boolean_binary_element(map_builder, map); + } + (DataType::Boolean, DataType::Utf8) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_boolean_string_element(map_builder, map); + } + (DataType::Boolean, DataType::Decimal128(p, _)) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_boolean_decimal_element(map_builder, map, *p); + } + (DataType::Boolean, DataType::Struct(fields)) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_boolean_struct_element(map_builder, map, fields); + } + (DataType::Int8, DataType::Boolean) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_int8_boolean_element(map_builder, map); + } + (DataType::Int8, DataType::Int8) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_int8_int8_element(map_builder, map); + } + (DataType::Int8, DataType::Int16) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_int8_int16_element(map_builder, map); + } + (DataType::Int8, DataType::Int32) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_int8_int32_element(map_builder, map); + } + (DataType::Int8, DataType::Int64) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_int8_int64_element(map_builder, map); + } + (DataType::Int8, DataType::Float32) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_int8_float32_element(map_builder, map); + } + (DataType::Int8, DataType::Float64) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_int8_float64_element(map_builder, map); + } + (DataType::Int8, DataType::Date32) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_int8_date32_element(map_builder, map); + } + (DataType::Int8, DataType::Timestamp(TimeUnit::Microsecond, _)) => { + let map_builder = downcast_builder_ref!( + MapBuilder, + map_builder + ); + append_map_int8_timestamp_element(map_builder, map); + } + (DataType::Int8, DataType::Binary) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_int8_binary_element(map_builder, map); + } + (DataType::Int8, DataType::Utf8) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_int8_string_element(map_builder, map); + } + (DataType::Int8, DataType::Decimal128(p, _)) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_int8_decimal_element(map_builder, map, *p); + } + (DataType::Int8, DataType::Struct(fields)) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_int8_struct_element(map_builder, map, fields); + } + (DataType::Int16, DataType::Boolean) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_int16_boolean_element(map_builder, map); + } + (DataType::Int16, DataType::Int8) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_int16_int8_element(map_builder, map); + } + (DataType::Int16, DataType::Int16) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_int16_int16_element(map_builder, map); + } + (DataType::Int16, DataType::Int32) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_int16_int32_element(map_builder, map); + } + (DataType::Int16, DataType::Int64) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_int16_int64_element(map_builder, map); + } + (DataType::Int16, DataType::Float32) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_int16_float32_element(map_builder, map); + } + (DataType::Int16, DataType::Float64) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_int16_float64_element(map_builder, map); + } + (DataType::Int16, DataType::Date32) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_int16_date32_element(map_builder, map); + } + (DataType::Int16, DataType::Timestamp(TimeUnit::Microsecond, _)) => { + let map_builder = downcast_builder_ref!( + MapBuilder, + map_builder + ); + append_map_int16_timestamp_element(map_builder, map); + } + (DataType::Int16, DataType::Binary) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_int16_binary_element(map_builder, map); + } + (DataType::Int16, DataType::Utf8) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_int16_string_element(map_builder, map); + } + (DataType::Int16, DataType::Decimal128(p, _)) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_int16_decimal_element(map_builder, map, *p); + } + (DataType::Int16, DataType::Struct(fields)) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_int16_struct_element(map_builder, map, fields); + } + (DataType::Int32, DataType::Boolean) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_int32_boolean_element(map_builder, map); + } + (DataType::Int32, DataType::Int8) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_int32_int8_element(map_builder, map); + } + (DataType::Int32, DataType::Int16) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_int32_int16_element(map_builder, map); + } + (DataType::Int32, DataType::Int32) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_int32_int32_element(map_builder, map); + } + (DataType::Int32, DataType::Int64) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_int32_int64_element(map_builder, map); + } + (DataType::Int32, DataType::Float32) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_int32_float32_element(map_builder, map); + } + (DataType::Int32, DataType::Float64) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_int32_float64_element(map_builder, map); + } + (DataType::Int32, DataType::Date32) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_int32_date32_element(map_builder, map); + } + (DataType::Int32, DataType::Timestamp(TimeUnit::Microsecond, _)) => { + let map_builder = downcast_builder_ref!( + MapBuilder, + map_builder + ); + append_map_int32_timestamp_element(map_builder, map); + } + (DataType::Int32, DataType::Binary) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_int32_binary_element(map_builder, map); + } + (DataType::Int32, DataType::Utf8) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_int32_string_element(map_builder, map); + } + (DataType::Int32, DataType::Decimal128(p, _)) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_int32_decimal_element(map_builder, map, *p); + } + (DataType::Int32, DataType::Struct(fields)) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_int32_struct_element(map_builder, map, fields); + } + (DataType::Int64, DataType::Boolean) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_int64_boolean_element(map_builder, map); + } + (DataType::Int64, DataType::Int8) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_int64_int8_element(map_builder, map); + } + (DataType::Int64, DataType::Int16) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_int64_int16_element(map_builder, map); + } + (DataType::Int64, DataType::Int32) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_int64_int32_element(map_builder, map); + } + (DataType::Int64, DataType::Int64) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_int64_int64_element(map_builder, map); + } + (DataType::Int64, DataType::Float32) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_int64_float32_element(map_builder, map); + } + (DataType::Int64, DataType::Float64) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_int64_float64_element(map_builder, map); + } + (DataType::Int64, DataType::Date32) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_int64_date32_element(map_builder, map); + } + (DataType::Int64, DataType::Timestamp(TimeUnit::Microsecond, _)) => { + let map_builder = downcast_builder_ref!( + MapBuilder, + map_builder + ); + append_map_int64_timestamp_element(map_builder, map); + } + (DataType::Int64, DataType::Binary) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_int64_binary_element(map_builder, map); + } + (DataType::Int64, DataType::Utf8) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_int64_string_element(map_builder, map); + } + (DataType::Int64, DataType::Decimal128(p, _)) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_int64_decimal_element(map_builder, map, *p); + } + (DataType::Int64, DataType::Struct(fields)) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_int64_struct_element(map_builder, map, fields); + } + (DataType::Float32, DataType::Boolean) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_float32_boolean_element(map_builder, map); + } + (DataType::Float32, DataType::Int8) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_float32_int8_element(map_builder, map); + } + (DataType::Float32, DataType::Int16) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_float32_int16_element(map_builder, map); + } + (DataType::Float32, DataType::Int32) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_float32_int32_element(map_builder, map); + } + (DataType::Float32, DataType::Int64) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_float32_int64_element(map_builder, map); + } + (DataType::Float32, DataType::Float32) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_float32_float32_element(map_builder, map); + } + (DataType::Float32, DataType::Float64) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_float32_float64_element(map_builder, map); + } + (DataType::Float32, DataType::Date32) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_float32_date32_element(map_builder, map); + } + (DataType::Float32, DataType::Timestamp(TimeUnit::Microsecond, _)) => { + let map_builder = downcast_builder_ref!( + MapBuilder, + map_builder + ); + append_map_float32_timestamp_element(map_builder, map); + } + (DataType::Float32, DataType::Binary) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_float32_binary_element(map_builder, map); + } + (DataType::Float32, DataType::Utf8) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_float32_string_element(map_builder, map); + } + (DataType::Float32, DataType::Decimal128(p, _)) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_float32_decimal_element(map_builder, map, *p); + } + (DataType::Float32, DataType::Struct(fields)) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_float32_struct_element(map_builder, map, fields); + } + (DataType::Float64, DataType::Boolean) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_float64_boolean_element(map_builder, map); + } + (DataType::Float64, DataType::Int8) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_float64_int8_element(map_builder, map); + } + (DataType::Float64, DataType::Int16) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_float64_int16_element(map_builder, map); + } + (DataType::Float64, DataType::Int32) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_float64_int32_element(map_builder, map); + } + (DataType::Float64, DataType::Int64) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_float64_int64_element(map_builder, map); + } + (DataType::Float64, DataType::Float32) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_float64_float32_element(map_builder, map); + } + (DataType::Float64, DataType::Float64) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_float64_float64_element(map_builder, map); + } + (DataType::Float64, DataType::Date32) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_float64_date32_element(map_builder, map); + } + (DataType::Float64, DataType::Timestamp(TimeUnit::Microsecond, _)) => { + let map_builder = downcast_builder_ref!( + MapBuilder, + map_builder + ); + append_map_float64_timestamp_element(map_builder, map); + } + (DataType::Float64, DataType::Binary) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_float64_binary_element(map_builder, map); + } + (DataType::Float64, DataType::Utf8) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_float64_string_element(map_builder, map); + } + (DataType::Float64, DataType::Decimal128(p, _)) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_float64_decimal_element(map_builder, map, *p); + } + (DataType::Float64, DataType::Struct(fields)) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_float64_struct_element(map_builder, map, fields); + } + (DataType::Date32, DataType::Boolean) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_date32_boolean_element(map_builder, map); + } + (DataType::Date32, DataType::Int8) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_date32_int8_element(map_builder, map); + } + (DataType::Date32, DataType::Int16) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_date32_int16_element(map_builder, map); + } + (DataType::Date32, DataType::Int32) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_date32_int32_element(map_builder, map); + } + (DataType::Date32, DataType::Int64) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_date32_int64_element(map_builder, map); + } + (DataType::Date32, DataType::Float32) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_date32_float32_element(map_builder, map); + } + (DataType::Date32, DataType::Float64) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_date32_float64_element(map_builder, map); + } + (DataType::Date32, DataType::Date32) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_date32_date32_element(map_builder, map); + } + (DataType::Date32, DataType::Timestamp(TimeUnit::Microsecond, _)) => { + let map_builder = downcast_builder_ref!( + MapBuilder, + map_builder + ); + append_map_date32_timestamp_element(map_builder, map); + } + (DataType::Date32, DataType::Binary) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_date32_binary_element(map_builder, map); + } + (DataType::Date32, DataType::Utf8) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_date32_string_element(map_builder, map); + } + (DataType::Date32, DataType::Decimal128(p, _)) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_date32_decimal_element(map_builder, map, *p); + } + (DataType::Date32, DataType::Struct(fields)) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_date32_struct_element(map_builder, map, fields); + } + (DataType::Timestamp(TimeUnit::Microsecond, _), DataType::Boolean) => { + let map_builder = downcast_builder_ref!( + MapBuilder, + map_builder + ); + append_map_timestamp_boolean_element(map_builder, map); + } + (DataType::Timestamp(TimeUnit::Microsecond, _), DataType::Int8) => { + let map_builder = downcast_builder_ref!( + MapBuilder, + map_builder + ); + append_map_timestamp_int8_element(map_builder, map); + } + (DataType::Timestamp(TimeUnit::Microsecond, _), DataType::Int16) => { + let map_builder = downcast_builder_ref!( + MapBuilder, + map_builder + ); + append_map_timestamp_int16_element(map_builder, map); + } + (DataType::Timestamp(TimeUnit::Microsecond, _), DataType::Int32) => { + let map_builder = downcast_builder_ref!( + MapBuilder, + map_builder + ); + append_map_timestamp_int32_element(map_builder, map); + } + (DataType::Timestamp(TimeUnit::Microsecond, _), DataType::Int64) => { + let map_builder = downcast_builder_ref!( + MapBuilder, + map_builder + ); + append_map_timestamp_int64_element(map_builder, map); + } + (DataType::Timestamp(TimeUnit::Microsecond, _), DataType::Float32) => { + let map_builder = downcast_builder_ref!( + MapBuilder, + map_builder + ); + append_map_timestamp_float32_element(map_builder, map); + } + (DataType::Timestamp(TimeUnit::Microsecond, _), DataType::Float64) => { + let map_builder = downcast_builder_ref!( + MapBuilder, + map_builder + ); + append_map_timestamp_float64_element(map_builder, map); + } + (DataType::Timestamp(TimeUnit::Microsecond, _), DataType::Date32) => { + let map_builder = downcast_builder_ref!( + MapBuilder, + map_builder + ); + append_map_timestamp_date32_element(map_builder, map); + } + ( + DataType::Timestamp(TimeUnit::Microsecond, _), + DataType::Timestamp(TimeUnit::Microsecond, _), + ) => { + let map_builder = downcast_builder_ref!( + MapBuilder, + map_builder + ); + append_map_timestamp_timestamp_element(map_builder, map); + } + (DataType::Timestamp(TimeUnit::Microsecond, _), DataType::Binary) => { + let map_builder = downcast_builder_ref!( + MapBuilder, + map_builder + ); + append_map_timestamp_binary_element(map_builder, map); + } + (DataType::Timestamp(TimeUnit::Microsecond, _), DataType::Utf8) => { + let map_builder = downcast_builder_ref!( + MapBuilder, + map_builder + ); + append_map_timestamp_string_element(map_builder, map); + } + (DataType::Timestamp(TimeUnit::Microsecond, _), DataType::Decimal128(p, _)) => { + let map_builder = downcast_builder_ref!(MapBuilder, map_builder); + append_map_timestamp_decimal_element(map_builder, map, *p); + } + (DataType::Timestamp(TimeUnit::Microsecond, _), DataType::Struct(fields)) => { + let map_builder = downcast_builder_ref!(MapBuilder, map_builder); + append_map_timestamp_struct_element(map_builder, map, fields); + } + (DataType::Binary, DataType::Boolean) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_binary_boolean_element(map_builder, map); + } + (DataType::Binary, DataType::Int8) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_binary_int8_element(map_builder, map); + } + (DataType::Binary, DataType::Int16) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_binary_int16_element(map_builder, map); + } + (DataType::Binary, DataType::Int32) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_binary_int32_element(map_builder, map); + } + (DataType::Binary, DataType::Int64) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_binary_int64_element(map_builder, map); + } + (DataType::Binary, DataType::Float32) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_binary_float32_element(map_builder, map); + } + (DataType::Binary, DataType::Float64) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_binary_float64_element(map_builder, map); + } + (DataType::Binary, DataType::Date32) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_binary_date32_element(map_builder, map); + } + (DataType::Binary, DataType::Timestamp(TimeUnit::Microsecond, _)) => { + let map_builder = downcast_builder_ref!( + MapBuilder, + map_builder + ); + append_map_binary_timestamp_element(map_builder, map); + } + (DataType::Binary, DataType::Binary) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_binary_binary_element(map_builder, map); + } + (DataType::Binary, DataType::Utf8) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_binary_string_element(map_builder, map); + } + (DataType::Binary, DataType::Decimal128(p, _)) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_binary_decimal_element(map_builder, map, *p); + } + (DataType::Binary, DataType::Struct(fields)) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_binary_struct_element(map_builder, map, fields); + } + (DataType::Utf8, DataType::Boolean) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_string_boolean_element(map_builder, map); + } + (DataType::Utf8, DataType::Int8) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_string_int8_element(map_builder, map); + } + (DataType::Utf8, DataType::Int16) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_string_int16_element(map_builder, map); + } + (DataType::Utf8, DataType::Int32) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_string_int32_element(map_builder, map); + } + (DataType::Utf8, DataType::Int64) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_string_int64_element(map_builder, map); + } + (DataType::Utf8, DataType::Float32) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_string_float32_element(map_builder, map); + } + (DataType::Utf8, DataType::Float64) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_string_float64_element(map_builder, map); + } + (DataType::Utf8, DataType::Date32) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_string_date32_element(map_builder, map); + } + (DataType::Utf8, DataType::Timestamp(TimeUnit::Microsecond, _)) => { + let map_builder = downcast_builder_ref!( + MapBuilder, + map_builder + ); + append_map_string_timestamp_element(map_builder, map); + } + (DataType::Utf8, DataType::Binary) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_string_binary_element(map_builder, map); + } + (DataType::Utf8, DataType::Utf8) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_string_string_element(map_builder, map); + } + (DataType::Utf8, DataType::Decimal128(p, _)) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_string_decimal_element(map_builder, map, *p); + } + (DataType::Utf8, DataType::Struct(fields)) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_string_struct_element(map_builder, map, fields); + } + (DataType::Decimal128(p, _), DataType::Boolean) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_decimal_boolean_element(map_builder, map, *p); + } + (DataType::Decimal128(p, _), DataType::Int8) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_decimal_int8_element(map_builder, map, *p); + } + (DataType::Decimal128(p, _), DataType::Int16) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_decimal_int16_element(map_builder, map, *p); + } + (DataType::Decimal128(p, _), DataType::Int32) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_decimal_int32_element(map_builder, map, *p); + } + (DataType::Decimal128(p, _), DataType::Int64) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_decimal_int64_element(map_builder, map, *p); + } + (DataType::Decimal128(p, _), DataType::Float32) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_decimal_float32_element(map_builder, map, *p); + } + (DataType::Decimal128(p, _), DataType::Float64) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_decimal_float64_element(map_builder, map, *p); + } + (DataType::Decimal128(p, _), DataType::Date32) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_decimal_date32_element(map_builder, map, *p); + } + (DataType::Decimal128(p, _), DataType::Timestamp(TimeUnit::Microsecond, _)) => { + let map_builder = downcast_builder_ref!( + MapBuilder, + map_builder + ); + append_map_decimal_timestamp_element(map_builder, map, *p); + } + (DataType::Decimal128(p, _), DataType::Binary) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_decimal_binary_element(map_builder, map, *p); + } + (DataType::Decimal128(p, _), DataType::Utf8) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_decimal_string_element(map_builder, map, *p); + } + (DataType::Decimal128(p1, _), DataType::Decimal128(p2, _)) => { + let map_builder = downcast_builder_ref!(MapBuilder, map_builder); + append_map_decimal_decimal_element(map_builder, map, *p1, *p2); + } + (DataType::Decimal128(p, _), DataType::Struct(fields)) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_decimal_struct_element(map_builder, map, *p, fields); + } + (DataType::Struct(fields), DataType::Boolean) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_struct_boolean_element(map_builder, map, fields); + } + (DataType::Struct(fields), DataType::Int8) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_struct_int8_element(map_builder, map, fields); + } + (DataType::Struct(fields), DataType::Int16) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_struct_int16_element(map_builder, map, fields); + } + (DataType::Struct(fields), DataType::Int32) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_struct_int32_element(map_builder, map, fields); + } + (DataType::Struct(fields), DataType::Int64) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_struct_int64_element(map_builder, map, fields); + } + (DataType::Struct(fields), DataType::Float32) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_struct_float32_element(map_builder, map, fields); + } + (DataType::Struct(fields), DataType::Float64) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_struct_float64_element(map_builder, map, fields); + } + (DataType::Struct(fields), DataType::Date32) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_struct_date32_element(map_builder, map, fields); + } + (DataType::Struct(fields), DataType::Timestamp(TimeUnit::Microsecond, _)) => { + let map_builder = downcast_builder_ref!( + MapBuilder, + map_builder + ); + append_map_struct_timestamp_element(map_builder, map, fields); + } + (DataType::Struct(fields), DataType::Binary) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_struct_binary_element(map_builder, map, fields); + } + (DataType::Struct(fields), DataType::Utf8) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_struct_string_element(map_builder, map, fields); + } + (DataType::Struct(fields), DataType::Decimal128(p, _)) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_struct_decimal_element(map_builder, map, *p, fields); + } + (DataType::Struct(key_fields), DataType::Struct(value_fields)) => { + let map_builder = + downcast_builder_ref!(MapBuilder, map_builder); + append_map_struct_struct_element(map_builder, map, key_fields, value_fields); + } + _ => { + return Err(CometError::Internal(format!( + "Unsupported map key/value data type: {:?}/{:?}", + key_dt, value_dt + ))) + } + } + + Ok(()) +} + +#[allow(clippy::field_reassign_with_default)] +pub fn get_map_key_value_dt( + field: &FieldRef, +) -> Result<(&DataType, &DataType, MapFieldNames), CometError> { + let mut map_fieldnames = MapFieldNames::default(); + map_fieldnames.entry = field.name().to_string(); + + let (key_dt, value_dt) = match field.data_type() { + DataType::Struct(fields) => { + if fields.len() != 2 { + return Err(CometError::Internal(format!( + "Map field should have 2 fields, but got {}", + fields.len() + ))); + } + + map_fieldnames.key = fields[0].name().to_string(); + map_fieldnames.value = fields[1].name().to_string(); + + let key_dt = fields[0].data_type(); + let value_dt = fields[1].data_type(); + (key_dt, value_dt) + } + _ => { + return Err(CometError::Internal(format!( + "Map field should be a struct, but got {:?}", + field.data_type() + ))); + } + }; + + Ok((key_dt, value_dt, map_fieldnames)) +} diff --git a/core/src/execution/shuffle/mod.rs b/core/src/execution/shuffle/mod.rs index 0b11597d0..b052df29b 100644 --- a/core/src/execution/shuffle/mod.rs +++ b/core/src/execution/shuffle/mod.rs @@ -16,4 +16,5 @@ // under the License. mod list; +mod map; pub mod row; diff --git a/core/src/execution/shuffle/row.rs b/core/src/execution/shuffle/row.rs index 110b57827..d1f73d2a6 100644 --- a/core/src/execution/shuffle/row.rs +++ b/core/src/execution/shuffle/row.rs @@ -21,7 +21,10 @@ use crate::{ errors::CometError, execution::{ datafusion::shuffle_writer::{compute_checksum, write_ipc_compressed, ChecksumAlgorithm}, - shuffle::list::{append_list_element, SparkUnsafeArray}, + shuffle::{ + list::{append_list_element, SparkUnsafeArray}, + map::{append_map_elements, get_map_key_value_dt, SparkUnsafeMap}, + }, utils::bytes_to_i128, }, }; @@ -30,7 +33,7 @@ use arrow_array::{ builder::{ ArrayBuilder, BinaryBuilder, BinaryDictionaryBuilder, BooleanBuilder, Date32Builder, Decimal128Builder, Float32Builder, Float64Builder, Int16Builder, Int32Builder, - Int64Builder, Int8Builder, ListBuilder, StringBuilder, StringDictionaryBuilder, + Int64Builder, Int8Builder, ListBuilder, MapBuilder, StringBuilder, StringDictionaryBuilder, StructBuilder, TimestampMicrosecondBuilder, }, types::Int32Type, @@ -47,7 +50,7 @@ use std::{ const WORD_SIZE: i64 = 8; const MAX_LONG_DIGITS: u8 = 18; -const LIST_BUILDER_CAPACITY: usize = 100; +const NESTED_TYPE_BUILDER_CAPACITY: usize = 100; /// A common trait for Spark Unsafe classes that can be used to access the underlying data, /// e.g., `UnsafeRow` and `UnsafeArray`. This defines a set of methods that can be used to @@ -170,6 +173,11 @@ pub trait SparkUnsafeObject { let (offset, len) = self.get_offset_and_len(index); SparkUnsafeArray::new(self.get_row_addr() + offset as i64, len) } + + fn get_map(&self, index: usize) -> SparkUnsafeMap { + let (offset, len) = self.get_offset_and_len(index); + SparkUnsafeMap::new(self.get_row_addr() + offset as i64, len) + } } pub struct SparkUnsafeRow { @@ -256,6 +264,18 @@ macro_rules! downcast_builder { }; } +macro_rules! downcast_builder_ref { + ($builder_type:ty, $builder:expr) => { + $builder + .as_any_mut() + .downcast_mut::<$builder_type>() + .unwrap() + }; +} + +// Expose the macro for other modules. +pub(crate) use downcast_builder_ref; + /// Appends field of row to the given struct builder. `dt` is the data type of the field. /// `struct_builder` is the struct builder of the row. `row` is the row that contains the field. /// `idx` is the index of the field in the row. @@ -280,6 +300,28 @@ pub(crate) fn append_field( }}; } + /// The macros for generating code of appending value into Map field builder of Arrow struct + /// builder. + macro_rules! append_map_element { + ($key_builder_type:ty, $value_builder_type:ty, $field:expr) => {{ + let field_builder = struct_builder + .field_builder::>(idx) + .unwrap(); + let is_null = row.is_null_at(idx); + + if is_null { + field_builder.append(false).unwrap(); + } else { + append_map_elements::<$key_builder_type, $value_builder_type>( + $field, + field_builder, + &row.get_map(idx), + ) + .unwrap(); + } + }}; + } + /// A macro for generating code of appending value into list field builder of Arrow struct /// builder. macro_rules! append_list_field_to_builder { @@ -363,6 +405,533 @@ pub(crate) fn append_field( } }); } + DataType::Map(field, _) => { + let (key_dt, value_dt, _) = get_map_key_value_dt(field).unwrap(); + + // macro cannot expand to match arm + match (key_dt, value_dt) { + (DataType::Boolean, DataType::Boolean) => { + append_map_element!(BooleanBuilder, BooleanBuilder, field); + } + (DataType::Boolean, DataType::Int8) => { + append_map_element!(BooleanBuilder, Int8Builder, field); + } + (DataType::Boolean, DataType::Int16) => { + append_map_element!(BooleanBuilder, Int16Builder, field); + } + (DataType::Boolean, DataType::Int32) => { + append_map_element!(BooleanBuilder, Int32Builder, field); + } + (DataType::Boolean, DataType::Int64) => { + append_map_element!(BooleanBuilder, Int64Builder, field); + } + (DataType::Boolean, DataType::Float32) => { + append_map_element!(BooleanBuilder, Float32Builder, field); + } + (DataType::Boolean, DataType::Float64) => { + append_map_element!(BooleanBuilder, Float64Builder, field); + } + (DataType::Boolean, DataType::Date32) => { + append_map_element!(BooleanBuilder, Date32Builder, field); + } + (DataType::Boolean, DataType::Timestamp(TimeUnit::Microsecond, _)) => { + append_map_element!(BooleanBuilder, TimestampMicrosecondBuilder, field); + } + (DataType::Boolean, DataType::Binary) => { + append_map_element!(BooleanBuilder, BinaryBuilder, field); + } + (DataType::Boolean, DataType::Utf8) => { + append_map_element!(BooleanBuilder, StringBuilder, field); + } + (DataType::Boolean, DataType::Decimal128(_, _)) => { + append_map_element!(BooleanBuilder, Decimal128Builder, field); + } + (DataType::Boolean, DataType::Struct(_)) => { + append_map_element!(BooleanBuilder, StructBuilder, field); + } + (DataType::Int8, DataType::Boolean) => { + append_map_element!(Int8Builder, BooleanBuilder, field); + } + (DataType::Int8, DataType::Int8) => { + append_map_element!(Int8Builder, Int8Builder, field); + } + (DataType::Int8, DataType::Int16) => { + append_map_element!(Int8Builder, Int16Builder, field); + } + (DataType::Int8, DataType::Int32) => { + append_map_element!(Int8Builder, Int32Builder, field); + } + (DataType::Int8, DataType::Int64) => { + append_map_element!(Int8Builder, Int64Builder, field); + } + (DataType::Int8, DataType::Float32) => { + append_map_element!(Int8Builder, Float32Builder, field); + } + (DataType::Int8, DataType::Float64) => { + append_map_element!(Int8Builder, Float64Builder, field); + } + (DataType::Int8, DataType::Date32) => { + append_map_element!(Int8Builder, Date32Builder, field); + } + (DataType::Int8, DataType::Timestamp(TimeUnit::Microsecond, _)) => { + append_map_element!(Int8Builder, TimestampMicrosecondBuilder, field); + } + (DataType::Int8, DataType::Binary) => { + append_map_element!(Int8Builder, BinaryBuilder, field); + } + (DataType::Int8, DataType::Utf8) => { + append_map_element!(Int8Builder, StringBuilder, field); + } + (DataType::Int8, DataType::Decimal128(_, _)) => { + append_map_element!(Int8Builder, Decimal128Builder, field); + } + (DataType::Int8, DataType::Struct(_)) => { + append_map_element!(Int8Builder, StructBuilder, field); + } + (DataType::Int16, DataType::Boolean) => { + append_map_element!(Int16Builder, BooleanBuilder, field); + } + (DataType::Int16, DataType::Int8) => { + append_map_element!(Int16Builder, Int8Builder, field); + } + (DataType::Int16, DataType::Int16) => { + append_map_element!(Int16Builder, Int16Builder, field); + } + (DataType::Int16, DataType::Int32) => { + append_map_element!(Int16Builder, Int32Builder, field); + } + (DataType::Int16, DataType::Int64) => { + append_map_element!(Int16Builder, Int64Builder, field); + } + (DataType::Int16, DataType::Float32) => { + append_map_element!(Int16Builder, Float32Builder, field); + } + (DataType::Int16, DataType::Float64) => { + append_map_element!(Int16Builder, Float64Builder, field); + } + (DataType::Int16, DataType::Date32) => { + append_map_element!(Int16Builder, Date32Builder, field); + } + (DataType::Int16, DataType::Timestamp(TimeUnit::Microsecond, _)) => { + append_map_element!(Int16Builder, TimestampMicrosecondBuilder, field); + } + (DataType::Int16, DataType::Binary) => { + append_map_element!(Int16Builder, BinaryBuilder, field); + } + (DataType::Int16, DataType::Utf8) => { + append_map_element!(Int16Builder, StringBuilder, field); + } + (DataType::Int16, DataType::Decimal128(_, _)) => { + append_map_element!(Int16Builder, Decimal128Builder, field); + } + (DataType::Int16, DataType::Struct(_)) => { + append_map_element!(Int16Builder, StructBuilder, field); + } + (DataType::Int32, DataType::Boolean) => { + append_map_element!(Int32Builder, BooleanBuilder, field); + } + (DataType::Int32, DataType::Int8) => { + append_map_element!(Int32Builder, Int8Builder, field); + } + (DataType::Int32, DataType::Int16) => { + append_map_element!(Int32Builder, Int16Builder, field); + } + (DataType::Int32, DataType::Int32) => { + append_map_element!(Int32Builder, Int32Builder, field); + } + (DataType::Int32, DataType::Int64) => { + append_map_element!(Int32Builder, Int64Builder, field); + } + (DataType::Int32, DataType::Float32) => { + append_map_element!(Int32Builder, Float32Builder, field); + } + (DataType::Int32, DataType::Float64) => { + append_map_element!(Int32Builder, Float64Builder, field); + } + (DataType::Int32, DataType::Date32) => { + append_map_element!(Int32Builder, Date32Builder, field); + } + (DataType::Int32, DataType::Timestamp(TimeUnit::Microsecond, _)) => { + append_map_element!(Int32Builder, TimestampMicrosecondBuilder, field); + } + (DataType::Int32, DataType::Binary) => { + append_map_element!(Int32Builder, BinaryBuilder, field); + } + (DataType::Int32, DataType::Utf8) => { + append_map_element!(Int32Builder, StringBuilder, field); + } + (DataType::Int32, DataType::Decimal128(_, _)) => { + append_map_element!(Int32Builder, Decimal128Builder, field); + } + (DataType::Int32, DataType::Struct(_)) => { + append_map_element!(Int32Builder, StructBuilder, field); + } + (DataType::Int64, DataType::Boolean) => { + append_map_element!(Int64Builder, BooleanBuilder, field); + } + (DataType::Int64, DataType::Int8) => { + append_map_element!(Int64Builder, Int8Builder, field); + } + (DataType::Int64, DataType::Int16) => { + append_map_element!(Int64Builder, Int16Builder, field); + } + (DataType::Int64, DataType::Int32) => { + append_map_element!(Int64Builder, Int32Builder, field); + } + (DataType::Int64, DataType::Int64) => { + append_map_element!(Int64Builder, Int64Builder, field); + } + (DataType::Int64, DataType::Float32) => { + append_map_element!(Int64Builder, Float32Builder, field); + } + (DataType::Int64, DataType::Float64) => { + append_map_element!(Int64Builder, Float64Builder, field); + } + (DataType::Int64, DataType::Date32) => { + append_map_element!(Int64Builder, Date32Builder, field); + } + (DataType::Int64, DataType::Timestamp(TimeUnit::Microsecond, _)) => { + append_map_element!(Int64Builder, TimestampMicrosecondBuilder, field); + } + (DataType::Int64, DataType::Binary) => { + append_map_element!(Int64Builder, BinaryBuilder, field); + } + (DataType::Int64, DataType::Utf8) => { + append_map_element!(Int64Builder, StringBuilder, field); + } + (DataType::Int64, DataType::Decimal128(_, _)) => { + append_map_element!(Int64Builder, Decimal128Builder, field); + } + (DataType::Int64, DataType::Struct(_)) => { + append_map_element!(Int64Builder, StructBuilder, field); + } + (DataType::Float32, DataType::Boolean) => { + append_map_element!(Float32Builder, BooleanBuilder, field); + } + (DataType::Float32, DataType::Int8) => { + append_map_element!(Float32Builder, Int8Builder, field); + } + (DataType::Float32, DataType::Int16) => { + append_map_element!(Float32Builder, Int16Builder, field); + } + (DataType::Float32, DataType::Int32) => { + append_map_element!(Float32Builder, Int32Builder, field); + } + (DataType::Float32, DataType::Int64) => { + append_map_element!(Float32Builder, Int64Builder, field); + } + (DataType::Float32, DataType::Float32) => { + append_map_element!(Float32Builder, Float32Builder, field); + } + (DataType::Float32, DataType::Float64) => { + append_map_element!(Float32Builder, Float64Builder, field); + } + (DataType::Float32, DataType::Date32) => { + append_map_element!(Float32Builder, Date32Builder, field); + } + (DataType::Float32, DataType::Timestamp(TimeUnit::Microsecond, _)) => { + append_map_element!(Float32Builder, TimestampMicrosecondBuilder, field); + } + (DataType::Float32, DataType::Binary) => { + append_map_element!(Float32Builder, BinaryBuilder, field); + } + (DataType::Float32, DataType::Utf8) => { + append_map_element!(Float32Builder, StringBuilder, field); + } + (DataType::Float32, DataType::Decimal128(_, _)) => { + append_map_element!(Float32Builder, Decimal128Builder, field); + } + (DataType::Float32, DataType::Struct(_)) => { + append_map_element!(Float32Builder, StructBuilder, field); + } + (DataType::Float64, DataType::Boolean) => { + append_map_element!(Float64Builder, BooleanBuilder, field); + } + (DataType::Float64, DataType::Int8) => { + append_map_element!(Float64Builder, Int8Builder, field); + } + (DataType::Float64, DataType::Int16) => { + append_map_element!(Float64Builder, Int16Builder, field); + } + (DataType::Float64, DataType::Int32) => { + append_map_element!(Float64Builder, Int32Builder, field); + } + (DataType::Float64, DataType::Int64) => { + append_map_element!(Float64Builder, Int64Builder, field); + } + (DataType::Float64, DataType::Float32) => { + append_map_element!(Float64Builder, Float32Builder, field); + } + (DataType::Float64, DataType::Float64) => { + append_map_element!(Float64Builder, Float64Builder, field); + } + (DataType::Float64, DataType::Date32) => { + append_map_element!(Float64Builder, Date32Builder, field); + } + (DataType::Float64, DataType::Timestamp(TimeUnit::Microsecond, _)) => { + append_map_element!(Float64Builder, TimestampMicrosecondBuilder, field); + } + (DataType::Float64, DataType::Binary) => { + append_map_element!(Float64Builder, BinaryBuilder, field); + } + (DataType::Float64, DataType::Utf8) => { + append_map_element!(Float64Builder, StringBuilder, field); + } + (DataType::Float64, DataType::Decimal128(_, _)) => { + append_map_element!(Float64Builder, Decimal128Builder, field); + } + (DataType::Float64, DataType::Struct(_)) => { + append_map_element!(Float64Builder, StructBuilder, field); + } + (DataType::Date32, DataType::Boolean) => { + append_map_element!(Date32Builder, BooleanBuilder, field); + } + (DataType::Date32, DataType::Int8) => { + append_map_element!(Date32Builder, Int8Builder, field); + } + (DataType::Date32, DataType::Int16) => { + append_map_element!(Date32Builder, Int16Builder, field); + } + (DataType::Date32, DataType::Int32) => { + append_map_element!(Date32Builder, Int32Builder, field); + } + (DataType::Date32, DataType::Int64) => { + append_map_element!(Date32Builder, Int64Builder, field); + } + (DataType::Date32, DataType::Float32) => { + append_map_element!(Date32Builder, Float32Builder, field); + } + (DataType::Date32, DataType::Float64) => { + append_map_element!(Date32Builder, Float64Builder, field); + } + (DataType::Date32, DataType::Date32) => { + append_map_element!(Date32Builder, Date32Builder, field); + } + (DataType::Date32, DataType::Timestamp(TimeUnit::Microsecond, _)) => { + append_map_element!(Date32Builder, TimestampMicrosecondBuilder, field); + } + (DataType::Date32, DataType::Binary) => { + append_map_element!(Date32Builder, BinaryBuilder, field); + } + (DataType::Date32, DataType::Utf8) => { + append_map_element!(Date32Builder, StringBuilder, field); + } + (DataType::Date32, DataType::Decimal128(_, _)) => { + append_map_element!(Date32Builder, Decimal128Builder, field); + } + (DataType::Date32, DataType::Struct(_)) => { + append_map_element!(Date32Builder, StructBuilder, field); + } + (DataType::Timestamp(TimeUnit::Microsecond, _), DataType::Boolean) => { + append_map_element!(TimestampMicrosecondBuilder, BooleanBuilder, field); + } + (DataType::Timestamp(TimeUnit::Microsecond, _), DataType::Int8) => { + append_map_element!(TimestampMicrosecondBuilder, Int8Builder, field); + } + (DataType::Timestamp(TimeUnit::Microsecond, _), DataType::Int16) => { + append_map_element!(TimestampMicrosecondBuilder, Int16Builder, field); + } + (DataType::Timestamp(TimeUnit::Microsecond, _), DataType::Int32) => { + append_map_element!(TimestampMicrosecondBuilder, Int32Builder, field); + } + (DataType::Timestamp(TimeUnit::Microsecond, _), DataType::Int64) => { + append_map_element!(TimestampMicrosecondBuilder, Int64Builder, field); + } + (DataType::Timestamp(TimeUnit::Microsecond, _), DataType::Float32) => { + append_map_element!(TimestampMicrosecondBuilder, Float32Builder, field); + } + (DataType::Timestamp(TimeUnit::Microsecond, _), DataType::Float64) => { + append_map_element!(TimestampMicrosecondBuilder, Float64Builder, field); + } + (DataType::Timestamp(TimeUnit::Microsecond, _), DataType::Date32) => { + append_map_element!(TimestampMicrosecondBuilder, Date32Builder, field); + } + ( + DataType::Timestamp(TimeUnit::Microsecond, _), + DataType::Timestamp(TimeUnit::Microsecond, _), + ) => { + append_map_element!( + TimestampMicrosecondBuilder, + TimestampMicrosecondBuilder, + field + ); + } + (DataType::Timestamp(TimeUnit::Microsecond, _), DataType::Binary) => { + append_map_element!(TimestampMicrosecondBuilder, BinaryBuilder, field); + } + (DataType::Timestamp(TimeUnit::Microsecond, _), DataType::Utf8) => { + append_map_element!(TimestampMicrosecondBuilder, StringBuilder, field); + } + (DataType::Timestamp(TimeUnit::Microsecond, _), DataType::Decimal128(_, _)) => { + append_map_element!(TimestampMicrosecondBuilder, Decimal128Builder, field); + } + (DataType::Timestamp(TimeUnit::Microsecond, _), DataType::Struct(_)) => { + append_map_element!(TimestampMicrosecondBuilder, StructBuilder, field); + } + (DataType::Binary, DataType::Boolean) => { + append_map_element!(BinaryBuilder, BooleanBuilder, field); + } + (DataType::Binary, DataType::Int8) => { + append_map_element!(BinaryBuilder, Int8Builder, field); + } + (DataType::Binary, DataType::Int16) => { + append_map_element!(BinaryBuilder, Int16Builder, field); + } + (DataType::Binary, DataType::Int32) => { + append_map_element!(BinaryBuilder, Int32Builder, field); + } + (DataType::Binary, DataType::Int64) => { + append_map_element!(BinaryBuilder, Int64Builder, field); + } + (DataType::Binary, DataType::Float32) => { + append_map_element!(BinaryBuilder, Float32Builder, field); + } + (DataType::Binary, DataType::Float64) => { + append_map_element!(BinaryBuilder, Float64Builder, field); + } + (DataType::Binary, DataType::Date32) => { + append_map_element!(BinaryBuilder, Date32Builder, field); + } + (DataType::Binary, DataType::Timestamp(TimeUnit::Microsecond, _)) => { + append_map_element!(BinaryBuilder, TimestampMicrosecondBuilder, field); + } + (DataType::Binary, DataType::Binary) => { + append_map_element!(BinaryBuilder, BinaryBuilder, field); + } + (DataType::Binary, DataType::Utf8) => { + append_map_element!(BinaryBuilder, StringBuilder, field); + } + (DataType::Binary, DataType::Decimal128(_, _)) => { + append_map_element!(BinaryBuilder, Decimal128Builder, field); + } + (DataType::Binary, DataType::Struct(_)) => { + append_map_element!(BinaryBuilder, StructBuilder, field); + } + (DataType::Utf8, DataType::Boolean) => { + append_map_element!(StringBuilder, BooleanBuilder, field); + } + (DataType::Utf8, DataType::Int8) => { + append_map_element!(StringBuilder, Int8Builder, field); + } + (DataType::Utf8, DataType::Int16) => { + append_map_element!(StringBuilder, Int16Builder, field); + } + (DataType::Utf8, DataType::Int32) => { + append_map_element!(StringBuilder, Int32Builder, field); + } + (DataType::Utf8, DataType::Int64) => { + append_map_element!(StringBuilder, Int64Builder, field); + } + (DataType::Utf8, DataType::Float32) => { + append_map_element!(StringBuilder, Float32Builder, field); + } + (DataType::Utf8, DataType::Float64) => { + append_map_element!(StringBuilder, Float64Builder, field); + } + (DataType::Utf8, DataType::Date32) => { + append_map_element!(StringBuilder, Date32Builder, field); + } + (DataType::Utf8, DataType::Timestamp(TimeUnit::Microsecond, _)) => { + append_map_element!(StringBuilder, TimestampMicrosecondBuilder, field); + } + (DataType::Utf8, DataType::Binary) => { + append_map_element!(StringBuilder, BinaryBuilder, field); + } + (DataType::Utf8, DataType::Utf8) => { + append_map_element!(StringBuilder, StringBuilder, field); + } + (DataType::Utf8, DataType::Decimal128(_, _)) => { + append_map_element!(StringBuilder, Decimal128Builder, field); + } + (DataType::Utf8, DataType::Struct(_)) => { + append_map_element!(StringBuilder, StructBuilder, field); + } + (DataType::Decimal128(_, _), DataType::Boolean) => { + append_map_element!(Decimal128Builder, BooleanBuilder, field); + } + (DataType::Decimal128(_, _), DataType::Int8) => { + append_map_element!(Decimal128Builder, Int8Builder, field); + } + (DataType::Decimal128(_, _), DataType::Int16) => { + append_map_element!(Decimal128Builder, Int16Builder, field); + } + (DataType::Decimal128(_, _), DataType::Int32) => { + append_map_element!(Decimal128Builder, Int32Builder, field); + } + (DataType::Decimal128(_, _), DataType::Int64) => { + append_map_element!(Decimal128Builder, Int64Builder, field); + } + (DataType::Decimal128(_, _), DataType::Float32) => { + append_map_element!(Decimal128Builder, Float32Builder, field); + } + (DataType::Decimal128(_, _), DataType::Float64) => { + append_map_element!(Decimal128Builder, Float64Builder, field); + } + (DataType::Decimal128(_, _), DataType::Date32) => { + append_map_element!(Decimal128Builder, Date32Builder, field); + } + (DataType::Decimal128(_, _), DataType::Timestamp(TimeUnit::Microsecond, _)) => { + append_map_element!(Decimal128Builder, TimestampMicrosecondBuilder, field); + } + (DataType::Decimal128(_, _), DataType::Binary) => { + append_map_element!(Decimal128Builder, BinaryBuilder, field); + } + (DataType::Decimal128(_, _), DataType::Utf8) => { + append_map_element!(Decimal128Builder, StringBuilder, field); + } + (DataType::Decimal128(_, _), DataType::Decimal128(_, _)) => { + append_map_element!(Decimal128Builder, Decimal128Builder, field); + } + (DataType::Decimal128(_, _), DataType::Struct(_)) => { + append_map_element!(Decimal128Builder, StructBuilder, field); + } + (DataType::Struct(_), DataType::Boolean) => { + append_map_element!(StructBuilder, BooleanBuilder, field); + } + (DataType::Struct(_), DataType::Int8) => { + append_map_element!(StructBuilder, Int8Builder, field); + } + (DataType::Struct(_), DataType::Int16) => { + append_map_element!(StructBuilder, Int16Builder, field); + } + (DataType::Struct(_), DataType::Int32) => { + append_map_element!(StructBuilder, Int32Builder, field); + } + (DataType::Struct(_), DataType::Int64) => { + append_map_element!(StructBuilder, Int64Builder, field); + } + (DataType::Struct(_), DataType::Float32) => { + append_map_element!(StructBuilder, Float32Builder, field); + } + (DataType::Struct(_), DataType::Float64) => { + append_map_element!(StructBuilder, Float64Builder, field); + } + (DataType::Struct(_), DataType::Date32) => { + append_map_element!(StructBuilder, Date32Builder, field); + } + (DataType::Struct(_), DataType::Timestamp(TimeUnit::Microsecond, _)) => { + append_map_element!(StructBuilder, TimestampMicrosecondBuilder, field); + } + (DataType::Struct(_), DataType::Binary) => { + append_map_element!(StructBuilder, BinaryBuilder, field); + } + (DataType::Struct(_), DataType::Utf8) => { + append_map_element!(StructBuilder, StringBuilder, field); + } + (DataType::Struct(_), DataType::Decimal128(_, _)) => { + append_map_element!(StructBuilder, Decimal128Builder, field); + } + (DataType::Struct(_), DataType::Struct(_)) => { + append_map_element!(StructBuilder, StructBuilder, field); + } + _ => { + unreachable!( + "Unsupported data type of map key and value: {:?} and {:?}", + key_dt, value_dt + ) + } + } + } DataType::List(field) => match field.data_type() { DataType::Boolean => { append_list_field_to_builder!(BooleanBuilder, field.data_type()); @@ -422,7 +991,7 @@ pub(crate) fn append_columns( column_idx: usize, builder: &mut Box, prefer_dictionary_ratio: f64, -) { +) -> Result<(), CometError> { /// A macro for generating code of appending values into Arrow array builders. macro_rules! append_column_to_builder { ($builder_type:ty, $accessor:expr) => {{ @@ -478,6 +1047,36 @@ pub(crate) fn append_columns( }}; } + /// A macro for generating code of appending values into Arrow `MapBuilder`. + macro_rules! append_column_to_map_builder { + ($key_builder_type:ty, $value_builder_type:ty, $field:expr) => {{ + let builder = builder + .as_any_mut() + .downcast_mut::>() + .unwrap(); + let mut row = SparkUnsafeRow::new(schema); + + for i in row_start..row_end { + let row_addr = unsafe { *row_addresses_ptr.add(i) }; + let row_size = unsafe { *row_sizes_ptr.add(i) }; + row.point_to(row_addr, row_size); + + let is_null = row.is_null_at(column_idx); + + if is_null { + builder.append(false)?; + } else { + append_map_elements::<$key_builder_type, $value_builder_type>( + $field, + builder, + &row.get_map(column_idx), + ) + .unwrap() + } + } + }}; + } + let dt = &schema[column_idx]; match dt { @@ -583,41 +1182,525 @@ pub(crate) fn append_columns( .append_value(row.get_timestamp(idx)) ); } - DataType::List(field) => match field.data_type() { - DataType::Boolean => { - append_column_to_list_builder!(BooleanBuilder, field.data_type()); - } - DataType::Int8 => { - append_column_to_list_builder!(Int8Builder, field.data_type()); - } - DataType::Int16 => { - append_column_to_list_builder!(Int16Builder, field.data_type()); - } - DataType::Int32 => { - append_column_to_list_builder!(Int32Builder, field.data_type()); - } - DataType::Int64 => { - append_column_to_list_builder!(Int64Builder, field.data_type()); - } - DataType::Float32 => { - append_column_to_list_builder!(Float32Builder, field.data_type()); - } - DataType::Float64 => { - append_column_to_list_builder!(Float64Builder, field.data_type()); - } - DataType::Timestamp(TimeUnit::Microsecond, _) => { - append_column_to_list_builder!(TimestampMicrosecondBuilder, field.data_type()); - } - DataType::Date32 => { - append_column_to_list_builder!(Date32Builder, field.data_type()); - } - DataType::Binary => { - append_column_to_list_builder!(BinaryBuilder, field.data_type()); - } - DataType::Utf8 => { - append_column_to_list_builder!(StringBuilder, field.data_type()); - } - DataType::Struct(_) => { + DataType::Map(field, _) => { + let (key_dt, value_dt, _) = get_map_key_value_dt(field)?; + + // macro cannot expand to match arm + match (key_dt, value_dt) { + (DataType::Boolean, DataType::Boolean) => { + append_column_to_map_builder!(BooleanBuilder, BooleanBuilder, field) + } + (DataType::Boolean, DataType::Int8) => { + append_column_to_map_builder!(BooleanBuilder, Int8Builder, field) + } + (DataType::Boolean, DataType::Int16) => { + append_column_to_map_builder!(BooleanBuilder, Int16Builder, field) + } + (DataType::Boolean, DataType::Int32) => { + append_column_to_map_builder!(BooleanBuilder, Int32Builder, field) + } + (DataType::Boolean, DataType::Int64) => { + append_column_to_map_builder!(BooleanBuilder, Int64Builder, field) + } + (DataType::Boolean, DataType::Float32) => { + append_column_to_map_builder!(BooleanBuilder, Float32Builder, field) + } + (DataType::Boolean, DataType::Float64) => { + append_column_to_map_builder!(BooleanBuilder, Float64Builder, field) + } + (DataType::Boolean, DataType::Timestamp(TimeUnit::Microsecond, _)) => { + append_column_to_map_builder!( + BooleanBuilder, + TimestampMicrosecondBuilder, + field + ) + } + (DataType::Boolean, DataType::Date32) => { + append_column_to_map_builder!(BooleanBuilder, Date32Builder, field) + } + (DataType::Boolean, DataType::Binary) => { + append_column_to_map_builder!(BooleanBuilder, BinaryBuilder, field) + } + (DataType::Boolean, DataType::Utf8) => { + append_column_to_map_builder!(BooleanBuilder, StringBuilder, field) + } + (DataType::Boolean, DataType::Decimal128(_, _)) => { + append_column_to_map_builder!(BooleanBuilder, Decimal128Builder, field) + } + (DataType::Int8, DataType::Boolean) => { + append_column_to_map_builder!(Int8Builder, BooleanBuilder, field) + } + (DataType::Int8, DataType::Int8) => { + append_column_to_map_builder!(Int8Builder, Int8Builder, field) + } + (DataType::Int8, DataType::Int16) => { + append_column_to_map_builder!(Int8Builder, Int16Builder, field) + } + (DataType::Int8, DataType::Int32) => { + append_column_to_map_builder!(Int8Builder, Int32Builder, field) + } + (DataType::Int8, DataType::Int64) => { + append_column_to_map_builder!(Int8Builder, Int64Builder, field) + } + (DataType::Int8, DataType::Float32) => { + append_column_to_map_builder!(Int8Builder, Float32Builder, field) + } + (DataType::Int8, DataType::Float64) => { + append_column_to_map_builder!(Int8Builder, Float64Builder, field) + } + (DataType::Int8, DataType::Timestamp(TimeUnit::Microsecond, _)) => { + append_column_to_map_builder!(Int8Builder, TimestampMicrosecondBuilder, field) + } + (DataType::Int8, DataType::Date32) => { + append_column_to_map_builder!(Int8Builder, Date32Builder, field) + } + (DataType::Int8, DataType::Binary) => { + append_column_to_map_builder!(Int8Builder, BinaryBuilder, field) + } + (DataType::Int8, DataType::Utf8) => { + append_column_to_map_builder!(Int8Builder, StringBuilder, field) + } + (DataType::Int8, DataType::Decimal128(_, _)) => { + append_column_to_map_builder!(Int8Builder, Decimal128Builder, field) + } + (DataType::Int16, DataType::Boolean) => { + append_column_to_map_builder!(Int16Builder, BooleanBuilder, field) + } + (DataType::Int16, DataType::Int8) => { + append_column_to_map_builder!(Int16Builder, Int8Builder, field) + } + (DataType::Int16, DataType::Int16) => { + append_column_to_map_builder!(Int16Builder, Int16Builder, field) + } + (DataType::Int16, DataType::Int32) => { + append_column_to_map_builder!(Int16Builder, Int32Builder, field) + } + (DataType::Int16, DataType::Int64) => { + append_column_to_map_builder!(Int16Builder, Int64Builder, field) + } + (DataType::Int16, DataType::Float32) => { + append_column_to_map_builder!(Int16Builder, Float32Builder, field) + } + (DataType::Int16, DataType::Float64) => { + append_column_to_map_builder!(Int16Builder, Float64Builder, field) + } + (DataType::Int16, DataType::Timestamp(TimeUnit::Microsecond, _)) => { + append_column_to_map_builder!(Int16Builder, TimestampMicrosecondBuilder, field) + } + (DataType::Int16, DataType::Date32) => { + append_column_to_map_builder!(Int16Builder, Date32Builder, field) + } + (DataType::Int16, DataType::Binary) => { + append_column_to_map_builder!(Int16Builder, BinaryBuilder, field) + } + (DataType::Int16, DataType::Utf8) => { + append_column_to_map_builder!(Int16Builder, StringBuilder, field) + } + (DataType::Int16, DataType::Decimal128(_, _)) => { + append_column_to_map_builder!(Int16Builder, Decimal128Builder, field) + } + (DataType::Int32, DataType::Boolean) => { + append_column_to_map_builder!(Int32Builder, BooleanBuilder, field) + } + (DataType::Int32, DataType::Int8) => { + append_column_to_map_builder!(Int32Builder, Int8Builder, field) + } + (DataType::Int32, DataType::Int16) => { + append_column_to_map_builder!(Int32Builder, Int16Builder, field) + } + (DataType::Int32, DataType::Int32) => { + append_column_to_map_builder!(Int32Builder, Int32Builder, field) + } + (DataType::Int32, DataType::Int64) => { + append_column_to_map_builder!(Int32Builder, Int64Builder, field) + } + (DataType::Int32, DataType::Float32) => { + append_column_to_map_builder!(Int32Builder, Float32Builder, field) + } + (DataType::Int32, DataType::Float64) => { + append_column_to_map_builder!(Int32Builder, Float64Builder, field) + } + (DataType::Int32, DataType::Timestamp(TimeUnit::Microsecond, _)) => { + append_column_to_map_builder!(Int32Builder, TimestampMicrosecondBuilder, field) + } + (DataType::Int32, DataType::Date32) => { + append_column_to_map_builder!(Int32Builder, Date32Builder, field) + } + (DataType::Int32, DataType::Binary) => { + append_column_to_map_builder!(Int32Builder, BinaryBuilder, field) + } + (DataType::Int32, DataType::Utf8) => { + append_column_to_map_builder!(Int32Builder, StringBuilder, field) + } + (DataType::Int32, DataType::Decimal128(_, _)) => { + append_column_to_map_builder!(Int32Builder, Decimal128Builder, field) + } + (DataType::Int64, DataType::Boolean) => { + append_column_to_map_builder!(Int64Builder, BooleanBuilder, field) + } + (DataType::Int64, DataType::Int8) => { + append_column_to_map_builder!(Int64Builder, Int8Builder, field) + } + (DataType::Int64, DataType::Int16) => { + append_column_to_map_builder!(Int64Builder, Int16Builder, field) + } + (DataType::Int64, DataType::Int32) => { + append_column_to_map_builder!(Int64Builder, Int32Builder, field) + } + (DataType::Int64, DataType::Int64) => { + append_column_to_map_builder!(Int64Builder, Int64Builder, field) + } + (DataType::Int64, DataType::Float32) => { + append_column_to_map_builder!(Int64Builder, Float32Builder, field) + } + (DataType::Int64, DataType::Float64) => { + append_column_to_map_builder!(Int64Builder, Float64Builder, field) + } + (DataType::Int64, DataType::Timestamp(TimeUnit::Microsecond, _)) => { + append_column_to_map_builder!(Int64Builder, TimestampMicrosecondBuilder, field) + } + (DataType::Int64, DataType::Date32) => { + append_column_to_map_builder!(Int64Builder, Date32Builder, field) + } + (DataType::Int64, DataType::Binary) => { + append_column_to_map_builder!(Int64Builder, BinaryBuilder, field) + } + (DataType::Int64, DataType::Utf8) => { + append_column_to_map_builder!(Int64Builder, StringBuilder, field) + } + (DataType::Int64, DataType::Decimal128(_, _)) => { + append_column_to_map_builder!(Int64Builder, Decimal128Builder, field) + } + (DataType::Float32, DataType::Boolean) => { + append_column_to_map_builder!(Float32Builder, BooleanBuilder, field) + } + (DataType::Float32, DataType::Int8) => { + append_column_to_map_builder!(Float32Builder, Int8Builder, field) + } + (DataType::Float32, DataType::Int16) => { + append_column_to_map_builder!(Float32Builder, Int16Builder, field) + } + (DataType::Float32, DataType::Int32) => { + append_column_to_map_builder!(Float32Builder, Int32Builder, field) + } + (DataType::Float32, DataType::Int64) => { + append_column_to_map_builder!(Float32Builder, Int64Builder, field) + } + (DataType::Float32, DataType::Float32) => { + append_column_to_map_builder!(Float32Builder, Float32Builder, field) + } + (DataType::Float32, DataType::Float64) => { + append_column_to_map_builder!(Float32Builder, Float64Builder, field) + } + (DataType::Float32, DataType::Timestamp(TimeUnit::Microsecond, _)) => { + append_column_to_map_builder!( + Float32Builder, + TimestampMicrosecondBuilder, + field + ) + } + (DataType::Float32, DataType::Date32) => { + append_column_to_map_builder!(Float32Builder, Date32Builder, field) + } + (DataType::Float32, DataType::Binary) => { + append_column_to_map_builder!(Float32Builder, BinaryBuilder, field) + } + (DataType::Float32, DataType::Utf8) => { + append_column_to_map_builder!(Float32Builder, StringBuilder, field) + } + (DataType::Float32, DataType::Decimal128(_, _)) => { + append_column_to_map_builder!(Float32Builder, Decimal128Builder, field) + } + (DataType::Float64, DataType::Boolean) => { + append_column_to_map_builder!(Float64Builder, BooleanBuilder, field) + } + (DataType::Float64, DataType::Int8) => { + append_column_to_map_builder!(Float64Builder, Int8Builder, field) + } + (DataType::Float64, DataType::Int16) => { + append_column_to_map_builder!(Float64Builder, Int16Builder, field) + } + (DataType::Float64, DataType::Int32) => { + append_column_to_map_builder!(Float64Builder, Int32Builder, field) + } + (DataType::Float64, DataType::Int64) => { + append_column_to_map_builder!(Float64Builder, Int64Builder, field) + } + (DataType::Float64, DataType::Float32) => { + append_column_to_map_builder!(Float64Builder, Float32Builder, field) + } + (DataType::Float64, DataType::Float64) => { + append_column_to_map_builder!(Float64Builder, Float64Builder, field) + } + (DataType::Float64, DataType::Timestamp(TimeUnit::Microsecond, _)) => { + append_column_to_map_builder!( + Float64Builder, + TimestampMicrosecondBuilder, + field + ) + } + (DataType::Float64, DataType::Date32) => { + append_column_to_map_builder!(Float64Builder, Date32Builder, field) + } + (DataType::Float64, DataType::Binary) => { + append_column_to_map_builder!(Float64Builder, BinaryBuilder, field) + } + (DataType::Float64, DataType::Utf8) => { + append_column_to_map_builder!(Float64Builder, StringBuilder, field) + } + (DataType::Float64, DataType::Decimal128(_, _)) => { + append_column_to_map_builder!(Float64Builder, Decimal128Builder, field) + } + (DataType::Timestamp(TimeUnit::Microsecond, _), DataType::Boolean) => { + append_column_to_map_builder!( + TimestampMicrosecondBuilder, + BooleanBuilder, + field + ) + } + (DataType::Timestamp(TimeUnit::Microsecond, _), DataType::Int8) => { + append_column_to_map_builder!(TimestampMicrosecondBuilder, Int8Builder, field) + } + (DataType::Timestamp(TimeUnit::Microsecond, _), DataType::Int16) => { + append_column_to_map_builder!(TimestampMicrosecondBuilder, Int16Builder, field) + } + (DataType::Timestamp(TimeUnit::Microsecond, _), DataType::Int32) => { + append_column_to_map_builder!(TimestampMicrosecondBuilder, Int32Builder, field) + } + (DataType::Timestamp(TimeUnit::Microsecond, _), DataType::Int64) => { + append_column_to_map_builder!(TimestampMicrosecondBuilder, Int64Builder, field) + } + (DataType::Timestamp(TimeUnit::Microsecond, _), DataType::Float32) => { + append_column_to_map_builder!( + TimestampMicrosecondBuilder, + Float32Builder, + field + ) + } + (DataType::Timestamp(TimeUnit::Microsecond, _), DataType::Float64) => { + append_column_to_map_builder!( + TimestampMicrosecondBuilder, + Float64Builder, + field + ) + } + ( + DataType::Timestamp(TimeUnit::Microsecond, _), + DataType::Timestamp(TimeUnit::Microsecond, _), + ) => { + append_column_to_map_builder!( + TimestampMicrosecondBuilder, + TimestampMicrosecondBuilder, + field + ) + } + (DataType::Timestamp(TimeUnit::Microsecond, _), DataType::Date32) => { + append_column_to_map_builder!(TimestampMicrosecondBuilder, Date32Builder, field) + } + (DataType::Timestamp(TimeUnit::Microsecond, _), DataType::Binary) => { + append_column_to_map_builder!(TimestampMicrosecondBuilder, BinaryBuilder, field) + } + (DataType::Timestamp(TimeUnit::Microsecond, _), DataType::Utf8) => { + append_column_to_map_builder!(TimestampMicrosecondBuilder, StringBuilder, field) + } + (DataType::Timestamp(TimeUnit::Microsecond, _), DataType::Decimal128(_, _)) => { + append_column_to_map_builder!( + TimestampMicrosecondBuilder, + Decimal128Builder, + field + ) + } + (DataType::Date32, DataType::Boolean) => { + append_column_to_map_builder!(Date32Builder, BooleanBuilder, field) + } + (DataType::Date32, DataType::Int8) => { + append_column_to_map_builder!(Date32Builder, Int8Builder, field) + } + (DataType::Date32, DataType::Int16) => { + append_column_to_map_builder!(Date32Builder, Int16Builder, field) + } + (DataType::Date32, DataType::Int32) => { + append_column_to_map_builder!(Date32Builder, Int32Builder, field) + } + (DataType::Date32, DataType::Int64) => { + append_column_to_map_builder!(Date32Builder, Int64Builder, field) + } + (DataType::Date32, DataType::Float32) => { + append_column_to_map_builder!(Date32Builder, Float32Builder, field) + } + (DataType::Date32, DataType::Float64) => { + append_column_to_map_builder!(Date32Builder, Float64Builder, field) + } + (DataType::Date32, DataType::Timestamp(TimeUnit::Microsecond, _)) => { + append_column_to_map_builder!(Date32Builder, TimestampMicrosecondBuilder, field) + } + (DataType::Date32, DataType::Date32) => { + append_column_to_map_builder!(Date32Builder, Date32Builder, field) + } + (DataType::Date32, DataType::Binary) => { + append_column_to_map_builder!(Date32Builder, BinaryBuilder, field) + } + (DataType::Date32, DataType::Utf8) => { + append_column_to_map_builder!(Date32Builder, StringBuilder, field) + } + (DataType::Date32, DataType::Decimal128(_, _)) => { + append_column_to_map_builder!(Date32Builder, Decimal128Builder, field) + } + (DataType::Binary, DataType::Boolean) => { + append_column_to_map_builder!(BinaryBuilder, BooleanBuilder, field) + } + (DataType::Binary, DataType::Int8) => { + append_column_to_map_builder!(BinaryBuilder, Int8Builder, field) + } + (DataType::Binary, DataType::Int16) => { + append_column_to_map_builder!(BinaryBuilder, Int16Builder, field) + } + (DataType::Binary, DataType::Int32) => { + append_column_to_map_builder!(BinaryBuilder, Int32Builder, field) + } + (DataType::Binary, DataType::Int64) => { + append_column_to_map_builder!(BinaryBuilder, Int64Builder, field) + } + (DataType::Binary, DataType::Float32) => { + append_column_to_map_builder!(BinaryBuilder, Float32Builder, field) + } + (DataType::Binary, DataType::Float64) => { + append_column_to_map_builder!(BinaryBuilder, Float64Builder, field) + } + (DataType::Binary, DataType::Timestamp(TimeUnit::Microsecond, _)) => { + append_column_to_map_builder!(BinaryBuilder, TimestampMicrosecondBuilder, field) + } + (DataType::Binary, DataType::Date32) => { + append_column_to_map_builder!(BinaryBuilder, Date32Builder, field) + } + (DataType::Binary, DataType::Binary) => { + append_column_to_map_builder!(BinaryBuilder, BinaryBuilder, field) + } + (DataType::Binary, DataType::Utf8) => { + append_column_to_map_builder!(BinaryBuilder, StringBuilder, field) + } + (DataType::Binary, DataType::Decimal128(_, _)) => { + append_column_to_map_builder!(BinaryBuilder, Decimal128Builder, field) + } + (DataType::Utf8, DataType::Boolean) => { + append_column_to_map_builder!(StringBuilder, BooleanBuilder, field) + } + (DataType::Utf8, DataType::Int8) => { + append_column_to_map_builder!(StringBuilder, Int8Builder, field) + } + (DataType::Utf8, DataType::Int16) => { + append_column_to_map_builder!(StringBuilder, Int16Builder, field) + } + (DataType::Utf8, DataType::Int32) => { + append_column_to_map_builder!(StringBuilder, Int32Builder, field) + } + (DataType::Utf8, DataType::Int64) => { + append_column_to_map_builder!(StringBuilder, Int64Builder, field) + } + (DataType::Utf8, DataType::Float32) => { + append_column_to_map_builder!(StringBuilder, Float32Builder, field) + } + (DataType::Utf8, DataType::Float64) => { + append_column_to_map_builder!(StringBuilder, Float64Builder, field) + } + (DataType::Utf8, DataType::Timestamp(TimeUnit::Microsecond, _)) => { + append_column_to_map_builder!(StringBuilder, TimestampMicrosecondBuilder, field) + } + (DataType::Utf8, DataType::Date32) => { + append_column_to_map_builder!(StringBuilder, Date32Builder, field) + } + (DataType::Utf8, DataType::Binary) => { + append_column_to_map_builder!(StringBuilder, BinaryBuilder, field) + } + (DataType::Utf8, DataType::Utf8) => { + append_column_to_map_builder!(StringBuilder, StringBuilder, field) + } + (DataType::Utf8, DataType::Decimal128(_, _)) => { + append_column_to_map_builder!(StringBuilder, Decimal128Builder, field) + } + (DataType::Decimal128(_, _), DataType::Boolean) => { + append_column_to_map_builder!(Decimal128Builder, BooleanBuilder, field) + } + (DataType::Decimal128(_, _), DataType::Int8) => { + append_column_to_map_builder!(Decimal128Builder, Int8Builder, field) + } + (DataType::Decimal128(_, _), DataType::Int16) => { + append_column_to_map_builder!(Decimal128Builder, Int16Builder, field) + } + (DataType::Decimal128(_, _), DataType::Int32) => { + append_column_to_map_builder!(Decimal128Builder, Int32Builder, field) + } + (DataType::Decimal128(_, _), DataType::Int64) => { + append_column_to_map_builder!(Decimal128Builder, Int64Builder, field) + } + (DataType::Decimal128(_, _), DataType::Float32) => { + append_column_to_map_builder!(Decimal128Builder, Float32Builder, field) + } + (DataType::Decimal128(_, _), DataType::Float64) => { + append_column_to_map_builder!(Decimal128Builder, Float64Builder, field) + } + (DataType::Decimal128(_, _), DataType::Timestamp(TimeUnit::Microsecond, _)) => { + append_column_to_map_builder!( + Decimal128Builder, + TimestampMicrosecondBuilder, + field + ) + } + (DataType::Decimal128(_, _), DataType::Date32) => { + append_column_to_map_builder!(Decimal128Builder, Date32Builder, field) + } + (DataType::Decimal128(_, _), DataType::Binary) => { + append_column_to_map_builder!(Decimal128Builder, BinaryBuilder, field) + } + (DataType::Decimal128(_, _), DataType::Utf8) => { + append_column_to_map_builder!(Decimal128Builder, StringBuilder, field) + } + (DataType::Decimal128(_, _), DataType::Decimal128(_, _)) => { + append_column_to_map_builder!(Decimal128Builder, Decimal128Builder, field) + } + _ => { + return Err(CometError::Internal(format!( + "Unsupported type: {:?}", + field.data_type() + ))) + } + } + } + DataType::List(field) => match field.data_type() { + DataType::Boolean => { + append_column_to_list_builder!(BooleanBuilder, field.data_type()); + } + DataType::Int8 => { + append_column_to_list_builder!(Int8Builder, field.data_type()); + } + DataType::Int16 => { + append_column_to_list_builder!(Int16Builder, field.data_type()); + } + DataType::Int32 => { + append_column_to_list_builder!(Int32Builder, field.data_type()); + } + DataType::Int64 => { + append_column_to_list_builder!(Int64Builder, field.data_type()); + } + DataType::Float32 => { + append_column_to_list_builder!(Float32Builder, field.data_type()); + } + DataType::Float64 => { + append_column_to_list_builder!(Float64Builder, field.data_type()); + } + DataType::Timestamp(TimeUnit::Microsecond, _) => { + append_column_to_list_builder!(TimestampMicrosecondBuilder, field.data_type()); + } + DataType::Date32 => { + append_column_to_list_builder!(Date32Builder, field.data_type()); + } + DataType::Binary => { + append_column_to_list_builder!(BinaryBuilder, field.data_type()); + } + DataType::Utf8 => { + append_column_to_list_builder!(StringBuilder, field.data_type()); + } + DataType::Struct(_) => { append_column_to_list_builder!(StructBuilder, field.data_type()); } DataType::Decimal128(_, _) => { @@ -641,6 +1724,8 @@ pub(crate) fn append_columns( unreachable!("Unsupported data type of column: {:?}", dt) } } + + Ok(()) } fn make_builders( @@ -685,9 +1770,1339 @@ fn make_builders( DataType::Timestamp(TimeUnit::Microsecond, _) => { Box::new(TimestampMicrosecondBuilder::with_capacity(row_num).with_data_type(dt.clone())) } + DataType::Map(field, _) => { + let (key_dt, value_dt, map_fieldnames) = get_map_key_value_dt(field)?; + + let key_builder = make_builders(key_dt, NESTED_TYPE_BUILDER_CAPACITY, 1.0)?; + let value_builder = make_builders(value_dt, NESTED_TYPE_BUILDER_CAPACITY, 1.0)?; + + // TODO: support other types of map after new release of Arrow. In new API, `MapBuilder` + // can take general `Box` as key/value builder. + match (key_dt, value_dt) { + (DataType::Boolean, DataType::Boolean) => { + let key_builder = downcast_builder!(BooleanBuilder, key_builder); + let value_builder = downcast_builder!(BooleanBuilder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Boolean, DataType::Int8) => { + let key_builder = downcast_builder!(BooleanBuilder, key_builder); + let value_builder = downcast_builder!(Int8Builder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Boolean, DataType::Int16) => { + let key_builder = downcast_builder!(BooleanBuilder, key_builder); + let value_builder = downcast_builder!(Int16Builder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Boolean, DataType::Int32) => { + let key_builder = downcast_builder!(BooleanBuilder, key_builder); + let value_builder = downcast_builder!(Int32Builder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Boolean, DataType::Int64) => { + let key_builder = downcast_builder!(BooleanBuilder, key_builder); + let value_builder = downcast_builder!(Int64Builder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Boolean, DataType::Float32) => { + let key_builder = downcast_builder!(BooleanBuilder, key_builder); + let value_builder = downcast_builder!(Float32Builder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Boolean, DataType::Float64) => { + let key_builder = downcast_builder!(BooleanBuilder, key_builder); + let value_builder = downcast_builder!(Float64Builder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Boolean, DataType::Timestamp(TimeUnit::Microsecond, _)) => { + let key_builder = downcast_builder!(BooleanBuilder, key_builder); + let value_builder = + downcast_builder!(TimestampMicrosecondBuilder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Boolean, DataType::Date32) => { + let key_builder = downcast_builder!(BooleanBuilder, key_builder); + let value_builder = downcast_builder!(Date32Builder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Boolean, DataType::Binary) => { + let key_builder = downcast_builder!(BooleanBuilder, key_builder); + let value_builder = downcast_builder!(BinaryBuilder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Boolean, DataType::Utf8) => { + let key_builder = downcast_builder!(BooleanBuilder, key_builder); + let value_builder = downcast_builder!(StringBuilder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Boolean, DataType::Decimal128(_, _)) => { + let key_builder = downcast_builder!(BooleanBuilder, key_builder); + let value_builder = downcast_builder!(Decimal128Builder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Int8, DataType::Boolean) => { + let key_builder = downcast_builder!(Int8Builder, key_builder); + let value_builder = downcast_builder!(BooleanBuilder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Int8, DataType::Int8) => { + let key_builder = downcast_builder!(Int8Builder, key_builder); + let value_builder = downcast_builder!(Int8Builder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Int8, DataType::Int16) => { + let key_builder = downcast_builder!(Int8Builder, key_builder); + let value_builder = downcast_builder!(Int16Builder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Int8, DataType::Int32) => { + let key_builder = downcast_builder!(Int8Builder, key_builder); + let value_builder = downcast_builder!(Int32Builder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Int8, DataType::Int64) => { + let key_builder = downcast_builder!(Int8Builder, key_builder); + let value_builder = downcast_builder!(Int64Builder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Int8, DataType::Float32) => { + let key_builder = downcast_builder!(Int8Builder, key_builder); + let value_builder = downcast_builder!(Float32Builder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Int8, DataType::Float64) => { + let key_builder = downcast_builder!(Int8Builder, key_builder); + let value_builder = downcast_builder!(Float64Builder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Int8, DataType::Timestamp(TimeUnit::Microsecond, _)) => { + let key_builder = downcast_builder!(Int8Builder, key_builder); + let value_builder = + downcast_builder!(TimestampMicrosecondBuilder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Int8, DataType::Date32) => { + let key_builder = downcast_builder!(Int8Builder, key_builder); + let value_builder = downcast_builder!(Date32Builder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Int8, DataType::Binary) => { + let key_builder = downcast_builder!(Int8Builder, key_builder); + let value_builder = downcast_builder!(BinaryBuilder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Int8, DataType::Utf8) => { + let key_builder = downcast_builder!(Int8Builder, key_builder); + let value_builder = downcast_builder!(StringBuilder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Int8, DataType::Decimal128(_, _)) => { + let key_builder = downcast_builder!(Int8Builder, key_builder); + let value_builder = downcast_builder!(Decimal128Builder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Int16, DataType::Boolean) => { + let key_builder = downcast_builder!(Int16Builder, key_builder); + let value_builder = downcast_builder!(BooleanBuilder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Int16, DataType::Int8) => { + let key_builder = downcast_builder!(Int16Builder, key_builder); + let value_builder = downcast_builder!(Int8Builder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Int16, DataType::Int16) => { + let key_builder = downcast_builder!(Int16Builder, key_builder); + let value_builder = downcast_builder!(Int16Builder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Int16, DataType::Int32) => { + let key_builder = downcast_builder!(Int16Builder, key_builder); + let value_builder = downcast_builder!(Int32Builder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Int16, DataType::Int64) => { + let key_builder = downcast_builder!(Int16Builder, key_builder); + let value_builder = downcast_builder!(Int64Builder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Int16, DataType::Float32) => { + let key_builder = downcast_builder!(Int16Builder, key_builder); + let value_builder = downcast_builder!(Float32Builder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Int16, DataType::Float64) => { + let key_builder = downcast_builder!(Int16Builder, key_builder); + let value_builder = downcast_builder!(Float64Builder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Int16, DataType::Timestamp(TimeUnit::Microsecond, _)) => { + let key_builder = downcast_builder!(Int16Builder, key_builder); + let value_builder = + downcast_builder!(TimestampMicrosecondBuilder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Int16, DataType::Date32) => { + let key_builder = downcast_builder!(Int16Builder, key_builder); + let value_builder = downcast_builder!(Date32Builder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Int16, DataType::Binary) => { + let key_builder = downcast_builder!(Int16Builder, key_builder); + let value_builder = downcast_builder!(BinaryBuilder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Int16, DataType::Utf8) => { + let key_builder = downcast_builder!(Int16Builder, key_builder); + let value_builder = downcast_builder!(StringBuilder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Int16, DataType::Decimal128(_, _)) => { + let key_builder = downcast_builder!(Int16Builder, key_builder); + let value_builder = downcast_builder!(Decimal128Builder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Int32, DataType::Boolean) => { + let key_builder = downcast_builder!(Int32Builder, key_builder); + let value_builder = downcast_builder!(BooleanBuilder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Int32, DataType::Int8) => { + let key_builder = downcast_builder!(Int32Builder, key_builder); + let value_builder = downcast_builder!(Int8Builder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Int32, DataType::Int16) => { + let key_builder = downcast_builder!(Int32Builder, key_builder); + let value_builder = downcast_builder!(Int16Builder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Int32, DataType::Int32) => { + let key_builder = downcast_builder!(Int32Builder, key_builder); + let value_builder = downcast_builder!(Int32Builder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Int32, DataType::Int64) => { + let key_builder = downcast_builder!(Int32Builder, key_builder); + let value_builder = downcast_builder!(Int64Builder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Int32, DataType::Float32) => { + let key_builder = downcast_builder!(Int32Builder, key_builder); + let value_builder = downcast_builder!(Float32Builder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Int32, DataType::Float64) => { + let key_builder = downcast_builder!(Int32Builder, key_builder); + let value_builder = downcast_builder!(Float64Builder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Int32, DataType::Timestamp(TimeUnit::Microsecond, _)) => { + let key_builder = downcast_builder!(Int32Builder, key_builder); + let value_builder = + downcast_builder!(TimestampMicrosecondBuilder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Int32, DataType::Date32) => { + let key_builder = downcast_builder!(Int32Builder, key_builder); + let value_builder = downcast_builder!(Date32Builder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Int32, DataType::Binary) => { + let key_builder = downcast_builder!(Int32Builder, key_builder); + let value_builder = downcast_builder!(BinaryBuilder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Int32, DataType::Utf8) => { + let key_builder = downcast_builder!(Int32Builder, key_builder); + let value_builder = downcast_builder!(StringBuilder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Int32, DataType::Decimal128(_, _)) => { + let key_builder = downcast_builder!(Int32Builder, key_builder); + let value_builder = downcast_builder!(Decimal128Builder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Int64, DataType::Boolean) => { + let key_builder = downcast_builder!(Int64Builder, key_builder); + let value_builder = downcast_builder!(BooleanBuilder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Int64, DataType::Int8) => { + let key_builder = downcast_builder!(Int64Builder, key_builder); + let value_builder = downcast_builder!(Int8Builder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Int64, DataType::Int16) => { + let key_builder = downcast_builder!(Int64Builder, key_builder); + let value_builder = downcast_builder!(Int16Builder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Int64, DataType::Int32) => { + let key_builder = downcast_builder!(Int64Builder, key_builder); + let value_builder = downcast_builder!(Int32Builder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Int64, DataType::Int64) => { + let key_builder = downcast_builder!(Int64Builder, key_builder); + let value_builder = downcast_builder!(Int64Builder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Int64, DataType::Float32) => { + let key_builder = downcast_builder!(Int64Builder, key_builder); + let value_builder = downcast_builder!(Float32Builder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Int64, DataType::Float64) => { + let key_builder = downcast_builder!(Int64Builder, key_builder); + let value_builder = downcast_builder!(Float64Builder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Int64, DataType::Timestamp(TimeUnit::Microsecond, _)) => { + let key_builder = downcast_builder!(Int64Builder, key_builder); + let value_builder = + downcast_builder!(TimestampMicrosecondBuilder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Int64, DataType::Date32) => { + let key_builder = downcast_builder!(Int64Builder, key_builder); + let value_builder = downcast_builder!(Date32Builder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Int64, DataType::Binary) => { + let key_builder = downcast_builder!(Int64Builder, key_builder); + let value_builder = downcast_builder!(BinaryBuilder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Int64, DataType::Utf8) => { + let key_builder = downcast_builder!(Int64Builder, key_builder); + let value_builder = downcast_builder!(StringBuilder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Int64, DataType::Decimal128(_, _)) => { + let key_builder = downcast_builder!(Int64Builder, key_builder); + let value_builder = downcast_builder!(Decimal128Builder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Float32, DataType::Boolean) => { + let key_builder = downcast_builder!(Float32Builder, key_builder); + let value_builder = downcast_builder!(BooleanBuilder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Float32, DataType::Int8) => { + let key_builder = downcast_builder!(Float32Builder, key_builder); + let value_builder = downcast_builder!(Int8Builder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Float32, DataType::Int16) => { + let key_builder = downcast_builder!(Float32Builder, key_builder); + let value_builder = downcast_builder!(Int16Builder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Float32, DataType::Int32) => { + let key_builder = downcast_builder!(Float32Builder, key_builder); + let value_builder = downcast_builder!(Int32Builder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Float32, DataType::Int64) => { + let key_builder = downcast_builder!(Float32Builder, key_builder); + let value_builder = downcast_builder!(Int64Builder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Float32, DataType::Float32) => { + let key_builder = downcast_builder!(Float32Builder, key_builder); + let value_builder = downcast_builder!(Float32Builder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Float32, DataType::Float64) => { + let key_builder = downcast_builder!(Float32Builder, key_builder); + let value_builder = downcast_builder!(Float64Builder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Float32, DataType::Timestamp(TimeUnit::Microsecond, _)) => { + let key_builder = downcast_builder!(Float32Builder, key_builder); + let value_builder = + downcast_builder!(TimestampMicrosecondBuilder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Float32, DataType::Date32) => { + let key_builder = downcast_builder!(Float32Builder, key_builder); + let value_builder = downcast_builder!(Date32Builder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Float32, DataType::Binary) => { + let key_builder = downcast_builder!(Float32Builder, key_builder); + let value_builder = downcast_builder!(BinaryBuilder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Float32, DataType::Utf8) => { + let key_builder = downcast_builder!(Float32Builder, key_builder); + let value_builder = downcast_builder!(StringBuilder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Float32, DataType::Decimal128(_, _)) => { + let key_builder = downcast_builder!(Float32Builder, key_builder); + let value_builder = downcast_builder!(Decimal128Builder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Float64, DataType::Boolean) => { + let key_builder = downcast_builder!(Float64Builder, key_builder); + let value_builder = downcast_builder!(BooleanBuilder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Float64, DataType::Int8) => { + let key_builder = downcast_builder!(Float64Builder, key_builder); + let value_builder = downcast_builder!(Int8Builder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Float64, DataType::Int16) => { + let key_builder = downcast_builder!(Float64Builder, key_builder); + let value_builder = downcast_builder!(Int16Builder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Float64, DataType::Int32) => { + let key_builder = downcast_builder!(Float64Builder, key_builder); + let value_builder = downcast_builder!(Int32Builder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Float64, DataType::Int64) => { + let key_builder = downcast_builder!(Float64Builder, key_builder); + let value_builder = downcast_builder!(Int64Builder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Float64, DataType::Float32) => { + let key_builder = downcast_builder!(Float64Builder, key_builder); + let value_builder = downcast_builder!(Float32Builder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Float64, DataType::Float64) => { + let key_builder = downcast_builder!(Float64Builder, key_builder); + let value_builder = downcast_builder!(Float64Builder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Float64, DataType::Timestamp(TimeUnit::Microsecond, _)) => { + let key_builder = downcast_builder!(Float64Builder, key_builder); + let value_builder = + downcast_builder!(TimestampMicrosecondBuilder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Float64, DataType::Date32) => { + let key_builder = downcast_builder!(Float64Builder, key_builder); + let value_builder = downcast_builder!(Date32Builder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Float64, DataType::Binary) => { + let key_builder = downcast_builder!(Float64Builder, key_builder); + let value_builder = downcast_builder!(BinaryBuilder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Float64, DataType::Utf8) => { + let key_builder = downcast_builder!(Float64Builder, key_builder); + let value_builder = downcast_builder!(StringBuilder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Float64, DataType::Decimal128(_, _)) => { + let key_builder = downcast_builder!(Float64Builder, key_builder); + let value_builder = downcast_builder!(Decimal128Builder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Date32, DataType::Boolean) => { + let key_builder = downcast_builder!(Date32Builder, key_builder); + let value_builder = downcast_builder!(BooleanBuilder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Date32, DataType::Int8) => { + let key_builder = downcast_builder!(Date32Builder, key_builder); + let value_builder = downcast_builder!(Int8Builder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Date32, DataType::Int16) => { + let key_builder = downcast_builder!(Date32Builder, key_builder); + let value_builder = downcast_builder!(Int16Builder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Date32, DataType::Int32) => { + let key_builder = downcast_builder!(Date32Builder, key_builder); + let value_builder = downcast_builder!(Int32Builder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Date32, DataType::Int64) => { + let key_builder = downcast_builder!(Date32Builder, key_builder); + let value_builder = downcast_builder!(Int64Builder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Date32, DataType::Float32) => { + let key_builder = downcast_builder!(Date32Builder, key_builder); + let value_builder = downcast_builder!(Float32Builder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Date32, DataType::Float64) => { + let key_builder = downcast_builder!(Date32Builder, key_builder); + let value_builder = downcast_builder!(Float64Builder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Date32, DataType::Timestamp(TimeUnit::Microsecond, _)) => { + let key_builder = downcast_builder!(Date32Builder, key_builder); + let value_builder = + downcast_builder!(TimestampMicrosecondBuilder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Date32, DataType::Date32) => { + let key_builder = downcast_builder!(Date32Builder, key_builder); + let value_builder = downcast_builder!(Date32Builder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Date32, DataType::Binary) => { + let key_builder = downcast_builder!(Date32Builder, key_builder); + let value_builder = downcast_builder!(BinaryBuilder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Date32, DataType::Utf8) => { + let key_builder = downcast_builder!(Date32Builder, key_builder); + let value_builder = downcast_builder!(StringBuilder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Date32, DataType::Decimal128(_, _)) => { + let key_builder = downcast_builder!(Date32Builder, key_builder); + let value_builder = downcast_builder!(Decimal128Builder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Timestamp(TimeUnit::Microsecond, _), DataType::Boolean) => { + let key_builder = downcast_builder!(TimestampMicrosecondBuilder, key_builder); + let value_builder = downcast_builder!(BooleanBuilder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Timestamp(TimeUnit::Microsecond, _), DataType::Int8) => { + let key_builder = downcast_builder!(TimestampMicrosecondBuilder, key_builder); + let value_builder = downcast_builder!(Int8Builder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Timestamp(TimeUnit::Microsecond, _), DataType::Int16) => { + let key_builder = downcast_builder!(TimestampMicrosecondBuilder, key_builder); + let value_builder = downcast_builder!(Int16Builder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Timestamp(TimeUnit::Microsecond, _), DataType::Int32) => { + let key_builder = downcast_builder!(TimestampMicrosecondBuilder, key_builder); + let value_builder = downcast_builder!(Int32Builder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Timestamp(TimeUnit::Microsecond, _), DataType::Int64) => { + let key_builder = downcast_builder!(TimestampMicrosecondBuilder, key_builder); + let value_builder = downcast_builder!(Int64Builder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Timestamp(TimeUnit::Microsecond, _), DataType::Float32) => { + let key_builder = downcast_builder!(TimestampMicrosecondBuilder, key_builder); + let value_builder = downcast_builder!(Float32Builder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Timestamp(TimeUnit::Microsecond, _), DataType::Float64) => { + let key_builder = downcast_builder!(TimestampMicrosecondBuilder, key_builder); + let value_builder = downcast_builder!(Float64Builder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + ( + DataType::Timestamp(TimeUnit::Microsecond, _), + DataType::Timestamp(TimeUnit::Microsecond, _), + ) => { + let key_builder = downcast_builder!(TimestampMicrosecondBuilder, key_builder); + let value_builder = + downcast_builder!(TimestampMicrosecondBuilder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Timestamp(TimeUnit::Microsecond, _), DataType::Date32) => { + let key_builder = downcast_builder!(TimestampMicrosecondBuilder, key_builder); + let value_builder = downcast_builder!(Date32Builder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Timestamp(TimeUnit::Microsecond, _), DataType::Binary) => { + let key_builder = downcast_builder!(TimestampMicrosecondBuilder, key_builder); + let value_builder = downcast_builder!(BinaryBuilder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Timestamp(TimeUnit::Microsecond, _), DataType::Utf8) => { + let key_builder = downcast_builder!(TimestampMicrosecondBuilder, key_builder); + let value_builder = downcast_builder!(StringBuilder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Timestamp(TimeUnit::Microsecond, _), DataType::Decimal128(_, _)) => { + let key_builder = downcast_builder!(TimestampMicrosecondBuilder, key_builder); + let value_builder = downcast_builder!(Decimal128Builder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Binary, DataType::Boolean) => { + let key_builder = downcast_builder!(BinaryBuilder, key_builder); + let value_builder = downcast_builder!(BooleanBuilder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Binary, DataType::Int8) => { + let key_builder = downcast_builder!(BinaryBuilder, key_builder); + let value_builder = downcast_builder!(Int8Builder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Binary, DataType::Int16) => { + let key_builder = downcast_builder!(BinaryBuilder, key_builder); + let value_builder = downcast_builder!(Int16Builder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Binary, DataType::Int32) => { + let key_builder = downcast_builder!(BinaryBuilder, key_builder); + let value_builder = downcast_builder!(Int32Builder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Binary, DataType::Int64) => { + let key_builder = downcast_builder!(BinaryBuilder, key_builder); + let value_builder = downcast_builder!(Int64Builder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Binary, DataType::Float32) => { + let key_builder = downcast_builder!(BinaryBuilder, key_builder); + let value_builder = downcast_builder!(Float32Builder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Binary, DataType::Float64) => { + let key_builder = downcast_builder!(BinaryBuilder, key_builder); + let value_builder = downcast_builder!(Float64Builder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Binary, DataType::Timestamp(TimeUnit::Microsecond, _)) => { + let key_builder = downcast_builder!(BinaryBuilder, key_builder); + let value_builder = + downcast_builder!(TimestampMicrosecondBuilder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Binary, DataType::Date32) => { + let key_builder = downcast_builder!(BinaryBuilder, key_builder); + let value_builder = downcast_builder!(Date32Builder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Binary, DataType::Binary) => { + let key_builder = downcast_builder!(BinaryBuilder, key_builder); + let value_builder = downcast_builder!(BinaryBuilder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Binary, DataType::Utf8) => { + let key_builder = downcast_builder!(BinaryBuilder, key_builder); + let value_builder = downcast_builder!(StringBuilder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Binary, DataType::Decimal128(_, _)) => { + let key_builder = downcast_builder!(BinaryBuilder, key_builder); + let value_builder = downcast_builder!(Decimal128Builder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Utf8, DataType::Boolean) => { + let key_builder = downcast_builder!(StringBuilder, key_builder); + let value_builder = downcast_builder!(BooleanBuilder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Utf8, DataType::Int8) => { + let key_builder = downcast_builder!(StringBuilder, key_builder); + let value_builder = downcast_builder!(Int8Builder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Utf8, DataType::Int16) => { + let key_builder = downcast_builder!(StringBuilder, key_builder); + let value_builder = downcast_builder!(Int16Builder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Utf8, DataType::Int32) => { + let key_builder = downcast_builder!(StringBuilder, key_builder); + let value_builder = downcast_builder!(Int32Builder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Utf8, DataType::Int64) => { + let key_builder = downcast_builder!(StringBuilder, key_builder); + let value_builder = downcast_builder!(Int64Builder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Utf8, DataType::Float32) => { + let key_builder = downcast_builder!(StringBuilder, key_builder); + let value_builder = downcast_builder!(Float32Builder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Utf8, DataType::Float64) => { + let key_builder = downcast_builder!(StringBuilder, key_builder); + let value_builder = downcast_builder!(Float64Builder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Utf8, DataType::Timestamp(TimeUnit::Microsecond, _)) => { + let key_builder = downcast_builder!(StringBuilder, key_builder); + let value_builder = + downcast_builder!(TimestampMicrosecondBuilder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Utf8, DataType::Date32) => { + let key_builder = downcast_builder!(StringBuilder, key_builder); + let value_builder = downcast_builder!(Date32Builder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Utf8, DataType::Binary) => { + let key_builder = downcast_builder!(StringBuilder, key_builder); + let value_builder = downcast_builder!(BinaryBuilder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Utf8, DataType::Utf8) => { + let key_builder = downcast_builder!(StringBuilder, key_builder); + let value_builder = downcast_builder!(StringBuilder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Utf8, DataType::Decimal128(_, _)) => { + let key_builder = downcast_builder!(StringBuilder, key_builder); + let value_builder = downcast_builder!(Decimal128Builder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Decimal128(_, _), DataType::Boolean) => { + let key_builder = downcast_builder!(Decimal128Builder, key_builder); + let value_builder = downcast_builder!(BooleanBuilder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Decimal128(_, _), DataType::Int8) => { + let key_builder = downcast_builder!(Decimal128Builder, key_builder); + let value_builder = downcast_builder!(Int8Builder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Decimal128(_, _), DataType::Int16) => { + let key_builder = downcast_builder!(Decimal128Builder, key_builder); + let value_builder = downcast_builder!(Int16Builder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Decimal128(_, _), DataType::Int32) => { + let key_builder = downcast_builder!(Decimal128Builder, key_builder); + let value_builder = downcast_builder!(Int32Builder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Decimal128(_, _), DataType::Int64) => { + let key_builder = downcast_builder!(Decimal128Builder, key_builder); + let value_builder = downcast_builder!(Int64Builder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Decimal128(_, _), DataType::Float32) => { + let key_builder = downcast_builder!(Decimal128Builder, key_builder); + let value_builder = downcast_builder!(Float32Builder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Decimal128(_, _), DataType::Float64) => { + let key_builder = downcast_builder!(Decimal128Builder, key_builder); + let value_builder = downcast_builder!(Float64Builder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Decimal128(_, _), DataType::Timestamp(TimeUnit::Microsecond, _)) => { + let key_builder = downcast_builder!(Decimal128Builder, key_builder); + let value_builder = + downcast_builder!(TimestampMicrosecondBuilder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Decimal128(_, _), DataType::Date32) => { + let key_builder = downcast_builder!(Decimal128Builder, key_builder); + let value_builder = downcast_builder!(Date32Builder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Decimal128(_, _), DataType::Binary) => { + let key_builder = downcast_builder!(Decimal128Builder, key_builder); + let value_builder = downcast_builder!(BinaryBuilder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Decimal128(_, _), DataType::Utf8) => { + let key_builder = downcast_builder!(Decimal128Builder, key_builder); + let value_builder = downcast_builder!(StringBuilder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + (DataType::Decimal128(_, _), DataType::Decimal128(_, _)) => { + let key_builder = downcast_builder!(Decimal128Builder, key_builder); + let value_builder = downcast_builder!(Decimal128Builder, value_builder); + Box::new(MapBuilder::new( + Some(map_fieldnames), + *key_builder, + *value_builder, + )) + } + + _ => { + return Err(CometError::Internal(format!( + "Unsupported type: {:?}", + field.data_type() + ))) + } + } + } DataType::List(field) => { // Disable dictionary encoding for array element - let value_builder = make_builders(field.data_type(), LIST_BUILDER_CAPACITY, 1.0)?; + let value_builder = + make_builders(field.data_type(), NESTED_TYPE_BUILDER_CAPACITY, 1.0)?; match field.data_type() { DataType::Boolean => { let builder = downcast_builder!(BooleanBuilder, value_builder); @@ -810,7 +3225,7 @@ pub fn process_sorted_row_partition( idx, builder, prefer_dictionary_ratio, - ); + )?; } row_start = row_end; diff --git a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala index e52018b38..f178a2f37 100644 --- a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala +++ b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala @@ -39,7 +39,7 @@ import org.apache.spark.unsafe.types.UTF8String import org.apache.comet.CometSparkSessionExtensions.{isCometOperatorEnabled, isCometScan, isSpark32, isSpark34Plus} import org.apache.comet.serde.ExprOuterClass.{AggExpr, DataType => ProtoDataType, Expr, ScalarFunc} -import org.apache.comet.serde.ExprOuterClass.DataType.{DataTypeInfo, DecimalInfo, ListInfo, StructInfo} +import org.apache.comet.serde.ExprOuterClass.DataType.{DataTypeInfo, DecimalInfo, ListInfo, MapInfo, StructInfo} import org.apache.comet.serde.OperatorOuterClass.{AggregateMode => CometAggregateMode, Operator} import org.apache.comet.shims.ShimQueryPlanSerde @@ -85,7 +85,8 @@ object QueryPlanSerde extends Logging with ShimQueryPlanSerde { case _: DateType => 12 case _: NullType => 13 case _: ArrayType => 14 - case _: StructType => 15 + case _: MapType => 15 + case _: StructType => 16 case dt => emitWarning(s"Cannot serialize Spark data type: $dt") return None @@ -119,6 +120,26 @@ object QueryPlanSerde extends Logging with ShimQueryPlanSerde { info.setList(list) builder.setTypeInfo(info.build()).build() + case m: MapType => + val keyType = serializeDataType(m.keyType) + if (keyType.isEmpty) { + return None + } + + val valueType = serializeDataType(m.valueType) + if (valueType.isEmpty) { + return None + } + + val info = DataTypeInfo.newBuilder() + val map = MapInfo.newBuilder() + map.setKeyType(keyType.get) + map.setValueType(valueType.get) + map.setValueContainsNull(m.valueContainsNull) + + info.setMap(map) + builder.setTypeInfo(info.build()).build() + case s: StructType => val info = DataTypeInfo.newBuilder() val struct = StructInfo.newBuilder() @@ -1764,6 +1785,10 @@ object QueryPlanSerde extends Logging with ShimQueryPlanSerde { case ArrayType(ArrayType(_, _), _) => false // TODO: nested array is not supported case ArrayType(elementType, _) => supportedDataType(elementType) + case MapType(MapType(_, _, _), _, _) => false // TODO: nested map is not supported + case MapType(_, MapType(_, _, _), _) => false + case MapType(keyType, valueType, _) => + supportedDataType(keyType) && supportedDataType(valueType) case _ => false } diff --git a/spark/src/test/scala/org/apache/comet/exec/CometShuffleSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometShuffleSuite.scala index 431a08fe6..a7aad2b0e 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometShuffleSuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometShuffleSuite.scala @@ -87,6 +87,355 @@ abstract class CometShuffleSuiteBase extends CometTestBase with AdaptiveSparkPla } } + test("columnar shuffle on map") { + def genTuples[K](num: Int, keys: Seq[K]): Seq[( + Int, + Map[K, Boolean], + Map[K, Byte], + Map[K, Short], + Map[K, Int], + Map[K, Long], + Map[K, Float], + Map[K, Double], + Map[K, java.sql.Date], + Map[K, java.sql.Timestamp], + Map[K, java.math.BigDecimal], + Map[K, Array[Byte]], + Map[K, String])] = { + (0 until num).map(i => + ( + i + 1, + Map(keys(0) -> (i > 10), keys(1) -> (i > 20)), + Map(keys(0) -> i.toByte, keys(1) -> (i + 1).toByte), + Map(keys(0) -> i.toShort, keys(1) -> (i + 1).toShort), + Map(keys(0) -> i, keys(1) -> (i + 1)), + Map(keys(0) -> i.toLong, keys(1) -> (i + 1).toLong), + Map(keys(0) -> i.toFloat, keys(1) -> (i + 1).toFloat), + Map(keys(0) -> i.toDouble, keys(1) -> (i + 1).toDouble), + Map( + keys(0) -> new java.sql.Date(i.toLong), + keys(1) -> new java.sql.Date((i + 1).toLong)), + Map( + keys(0) -> new java.sql.Timestamp(i.toLong), + keys(1) -> new java.sql.Timestamp((i + 1).toLong)), + Map( + keys(0) -> new java.math.BigDecimal(i.toLong), + keys(1) -> new java.math.BigDecimal((i + 1).toLong)), + Map(keys(0) -> i.toString.getBytes(), keys(1) -> (i + 1).toString.getBytes()), + Map(keys(0) -> i.toString, keys(1) -> (i + 1).toString))) + } + + Seq(10, 201).foreach { numPartitions => + Seq("1.0", "10.0").foreach { ratio => + withSQLConf( + CometConf.COMET_EXEC_ENABLED.key -> "false", + CometConf.COMET_COLUMNAR_SHUFFLE_ENABLED.key -> "true", + CometConf.COMET_EXEC_SHUFFLE_ENABLED.key -> "true", + CometConf.COMET_SHUFFLE_PREFER_DICTIONARY_RATIO.key -> ratio) { + // Boolean key + withParquetTable(genTuples(50, Seq(true, false)), "tbl") { + val df = sql("SELECT * FROM tbl") + .filter($"_1" > 10) + .repartition( + numPartitions, + $"_2", + $"_3", + $"_4", + $"_5", + $"_6", + $"_7", + $"_8", + $"_9", + $"_10", + $"_11", + $"_12", + $"_13") + .sortWithinPartitions($"_1") + + checkSparkAnswer(df) + checkCometExchange(df, 1, false) + } + + // Byte key + withParquetTable(genTuples(50, Seq(0.toByte, 1.toByte)), "tbl") { + val df = sql("SELECT * FROM tbl") + .filter($"_1" > 10) + .repartition( + numPartitions, + $"_2", + $"_3", + $"_4", + $"_5", + $"_6", + $"_7", + $"_8", + $"_9", + $"_10", + $"_11", + $"_12", + $"_13") + .sortWithinPartitions($"_1") + + checkSparkAnswer(df) + checkCometExchange(df, 1, false) + } + + // Short key + withParquetTable(genTuples(50, Seq(0.toShort, 1.toShort)), "tbl") { + val df = sql("SELECT * FROM tbl") + .filter($"_1" > 10) + .repartition( + numPartitions, + $"_2", + $"_3", + $"_4", + $"_5", + $"_6", + $"_7", + $"_8", + $"_9", + $"_10", + $"_11", + $"_12", + $"_13") + .sortWithinPartitions($"_1") + + checkSparkAnswer(df) + checkCometExchange(df, 1, false) + } + + // Int key + withParquetTable(genTuples(50, Seq(0, 1)), "tbl") { + val df = sql("SELECT * FROM tbl") + .filter($"_1" > 10) + .repartition( + numPartitions, + $"_2", + $"_3", + $"_4", + $"_5", + $"_6", + $"_7", + $"_8", + $"_9", + $"_10", + $"_11", + $"_12", + $"_13") + .sortWithinPartitions($"_1") + + checkSparkAnswer(df) + checkCometExchange(df, 1, false) + } + + // Long key + withParquetTable(genTuples(50, Seq(0.toLong, 1.toLong)), "tbl") { + val df = sql("SELECT * FROM tbl") + .filter($"_1" > 10) + .repartition( + numPartitions, + $"_2", + $"_3", + $"_4", + $"_5", + $"_6", + $"_7", + $"_8", + $"_9", + $"_10", + $"_11", + $"_12", + $"_13") + .sortWithinPartitions($"_1") + + checkSparkAnswer(df) + checkCometExchange(df, 1, false) + } + + // Float key + withParquetTable(genTuples(50, Seq(0.toFloat, 1.toFloat)), "tbl") { + val df = sql("SELECT * FROM tbl") + .filter($"_1" > 10) + .repartition( + numPartitions, + $"_2", + $"_3", + $"_4", + $"_5", + $"_6", + $"_7", + $"_8", + $"_9", + $"_10", + $"_11", + $"_12", + $"_13") + .sortWithinPartitions($"_1") + + checkSparkAnswer(df) + checkCometExchange(df, 1, false) + } + + // Double key + withParquetTable(genTuples(50, Seq(0.toDouble, 1.toDouble)), "tbl") { + val df = sql("SELECT * FROM tbl") + .filter($"_1" > 10) + .repartition( + numPartitions, + $"_2", + $"_3", + $"_4", + $"_5", + $"_6", + $"_7", + $"_8", + $"_9", + $"_10", + $"_11", + $"_12", + $"_13") + .sortWithinPartitions($"_1") + + checkSparkAnswer(df) + checkCometExchange(df, 1, false) + } + + // Date key + withParquetTable( + genTuples(50, Seq(new java.sql.Date(0.toLong), new java.sql.Date(1.toLong))), + "tbl") { + val df = sql("SELECT * FROM tbl") + .filter($"_1" > 10) + .repartition( + numPartitions, + $"_2", + $"_3", + $"_4", + $"_5", + $"_6", + $"_7", + $"_8", + $"_9", + $"_10", + $"_11", + $"_12", + $"_13") + .sortWithinPartitions($"_1") + + checkSparkAnswer(df) + checkCometExchange(df, 1, false) + } + + // Timestamp key + withParquetTable( + genTuples( + 50, + Seq(new java.sql.Timestamp(0.toLong), new java.sql.Timestamp(1.toLong))), + "tbl") { + val df = sql("SELECT * FROM tbl") + .filter($"_1" > 10) + .repartition( + numPartitions, + $"_2", + $"_3", + $"_4", + $"_5", + $"_6", + $"_7", + $"_8", + $"_9", + $"_10", + $"_11", + $"_12", + $"_13") + .sortWithinPartitions($"_1") + + checkSparkAnswer(df) + checkCometExchange(df, 1, false) + } + + // Decimal key + withParquetTable( + genTuples( + 50, + Seq(new java.math.BigDecimal(0.toLong), new java.math.BigDecimal(1.toLong))), + "tbl") { + val df = sql("SELECT * FROM tbl") + .filter($"_1" > 10) + .repartition( + numPartitions, + $"_2", + $"_3", + $"_4", + $"_5", + $"_6", + $"_7", + $"_8", + $"_9", + $"_10", + $"_11", + $"_12", + $"_13") + .sortWithinPartitions($"_1") + + checkSparkAnswer(df) + checkCometExchange(df, 1, false) + } + + // String key + withParquetTable(genTuples(50, Seq(0.toString, 1.toString)), "tbl") { + val df = sql("SELECT * FROM tbl") + .filter($"_1" > 10) + .repartition( + numPartitions, + $"_2", + $"_3", + $"_4", + $"_5", + $"_6", + $"_7", + $"_8", + $"_9", + $"_10", + $"_11", + $"_12", + $"_13") + .sortWithinPartitions($"_1") + + checkSparkAnswer(df) + checkCometExchange(df, 1, false) + } + + // Binary key + withParquetTable( + genTuples(50, Seq(0.toString.getBytes(), 1.toString.getBytes())), + "tbl") { + val df = sql("SELECT * FROM tbl") + .filter($"_1" > 10) + .repartition( + numPartitions, + $"_2", + $"_3", + $"_4", + $"_5", + $"_6", + $"_7", + $"_8", + $"_9", + $"_10", + $"_11", + $"_12", + $"_13") + .sortWithinPartitions($"_1") + + checkSparkAnswer(df) + checkCometExchange(df, 1, false) + } + } + } + } + } + test("columnar shuffle on array") { Seq(10, 201).foreach { numPartitions => Seq("1.0", "10.0").foreach { ratio =>